Private Sub btnSend_Click
Dim msg As String = txtInput.Text.Trim
If msg.Length = 0 Then Return
CustomListView1.AddTextItem("User: " & msg, "")
PrevMessages.Add(msg)
SetBusy(True)
Dim SystemPrompt As String = "You are a frienly assistant named Eric. Provide concise and short answers."
Wait For (GenerateResponse(SystemPrompt, PrevMessages)) Complete (Response As String)
SetBusy(False)
PrevMessages.Add(Response)
txtInput.SelectAll
txtInput.RequestFocus
CustomListView1.AddTextItem("Assistant: " & Response, "")
Sleep(0)
CustomListView1.ScrollToItem(CustomListView1.Size - 1)
End Sub
Private Sub GenerateResponse (SystemPrompt As String, Messages As List) As ResumableSub
Dim chat As PyWrapper = CreateChatStructure(SystemPrompt, Messages)
chat.Print
Dim inputs As PyWrapper = Tokenizer.Run2("__call__", Array(Array(chat)), _
CreateMap("return_tensors": "pt")).Run("to", Array(Device))
Dim InputSize As PyWrapper = inputs.Get("input_ids").GetField("shape").Get(-1)
Dim MaxNewTokens As Int = 1000 'this has large effect on the performance
Dim output_ids As PyWrapper = Model.Run2("generate", Null, _
CreateMap("input_ids": inputs.Get("input_ids"), _
"attention_mask": inputs.Get("attention_mask"), _
"max_new_tokens": MaxNewTokens, "pad_token_id": Tokenizer.GetField("eos_token_id"), _
"do_sample": True))
output_ids = output_ids.Get(Array(Py.Utils.Slice(Null, Null), Py.Utils.Slice(InputSize, Null)))
Dim responses As PyWrapper = Tokenizer.Run2("batch_decode", Array(output_ids), CreateMap("skip_special_tokens": True))
Wait For (responses.Get(0).Fetch) Complete (Result As PyWrapper)
Return Result.Value
End Sub
Private Sub CreateChatStructure (SystemPrompt As String, Messages As List) As PyWrapper
Dim MessagesWithRoles As List
MessagesWithRoles.Initialize
MessagesWithRoles.Add(CreateMap("role": "system", "content": SystemPrompt))
For i = 0 To Messages.Size - 1
MessagesWithRoles.Add(CreateMap("role": IIf(i Mod 2 = 0, "user", "assistant"), _
"content": Messages.Get(i)))
Next
Return Tokenizer.Run2("apply_chat_template", Array(MessagesWithRoles), _
CreateMap("add_generation_prompt": True, "tokenize": False))
End Sub
