some changes
This commit is contained in:
1 parent
cccdb7d231
commit
a3700ca15a
7 files changed
+36
-12
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import threading
|
||||
import ollama
|
||||
from stream import Stream
|
||||
|
||||
|
||||
@@ -18,13 +19,15 @@ class Models:
|
||||
qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
|
||||
qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
|
||||
|
||||
async def ollama_chat_completion(app, messages, model, opts={}):
|
||||
from ollama import AsyncClient
|
||||
async def ollama_list(app, messages, model, opts={}):
|
||||
# TODO: test the output, and use this
|
||||
return ollama.list()
|
||||
|
||||
async def ollama_chat_completion(app, messages, model, opts={}):
|
||||
content_stream = Stream(app)
|
||||
|
||||
print("START OLLAMA INFERENCE")
|
||||
stream_response = await AsyncClient().chat(
|
||||
stream_response = await ollama.AsyncClient().chat(
|
||||
model=model,
|
||||
messages=[{'role': msg.role, 'content': msg.content} for msg in messages],
|
||||
stream=True,
|
||||
|
||||
Reference in new issue
Block a user