WIP Stash

backend/src/inference.py

@@ -2,12 +2,12 @@ import threading
 import asyncio
 import json
 import os
-import ollama
+#import ollama
 import traceback
 from stream import Stream
 from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec, ToolCall
 
-from llama_cpp import Llama
+#from llama_cpp import Llama
 from openrouter import OpenRouter
 
 VALID_ROLES = ['user', 'assistant', 'system', 'tool']
@@ -35,12 +35,60 @@ class Models:
     qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
+def format_openai(messages):
+    formatted = []
+    last_assistant_message = None
+    for msg in messages:
+        if msg.role == 'tool':
+            if not last_assistant_message:
+                raise Exception("tool message before any assistant message")
+            tool_calls = last_assistant_message.get('tool_calls')
+            if tool_calls == None:
+                tool_calls = []
+                last_assistant_message['tool_calls'] = tool_calls
+            tool_calls.append({
+                'id': msg.content.id,
+                'type': 'function',
+                'function': {
+                    'name': msg.content.tool_name,
+                    'arguments': json.dumps(msg.content.arguments),
+                },
+            })
+            formatted.append({
+                'role': 'tool',
+                'tool_call_id': msg.content.id,
+                'content': json.dumps(msg.content.output),
+            })
+        else:
+            fmsg = {'role': msg.role, 'content': msg.content}
+            formatted.append(fmsg)
+            if msg.role == 'assistant':
+                last_assistant_message = fmsg
+    return formatted
+
+
 # llama cpp models
 model_cache = {}
 last_model_file = None
 last_model_instance = None
 
+def safe_queue(callback):
+    queue = asyncio.Queue()
+    loop = asyncio.get_running_loop()
+    async def read_loop():
+        while True:
+            args = await queue.get()
+            await callback(*args)
+    loop.create_task(read_loop())
+
+    def send(*args):
+        def queue_put():
+            loop.create_task(queue.put(args))
+        loop.call_soon_threadsafe(queue_put)
+    return send
+
 def get_llama_cpp_model(app, model_file):
+    raise Exception("llama-cpp disabled")
     global last_model_file, last_model_instance
     if last_model_file == model_file:
         return last_model_instance
@@ -95,24 +143,13 @@ def build_messages(system_prompt="", instruction=None, input=None, messages=None
 
 async def chat_completion(app,
     model_id=None,
-    instruction=None,
-    input=None,
     messages=None,
-    system_prompt="",
     tools=[],
     opts={},
     agent_run=None
 ):
     db_model = (await app.get_model(model_id)).value
 
-    # Integrate arguments into message list
-    messages = build_messages(
-        system_prompt=system_prompt,
-        instruction=instruction,
-        input=input,
-        messages=messages,
-    )
-
     model_engine = db_model['engine']
     model_name = db_model['model']
 
@@ -140,10 +177,22 @@ async def chat_completion(app,
 
 # llama.cpp
 async def local_chat_completion(app, messages, model, tools=[], opts={}, agent_run=None):
-    if agent_run:
-        content_stream = agent_run.get_output_stream()
-    else:
-        content_stream = Stream(app)
+    if not agent_run:
+        raise Exception("agent_run required")
+
+    content_stream = None
+    async def message_loop(name, value=None):
+        nonlocal content_stream, agent_run
+        if name == 'output':
+            if content_stream == None:
+                content_stream = Stream(app)
+                agent_run.emit_event('output', content_stream)
+            content_stream.feed(value)
+        elif name == 'close':
+            if content_stream:
+                content_stream.close()
+            agent_run.emit_event('inference-end', {})
+    send_message = safe_queue(message_loop)
 
     def worker():
         llm = get_llama_cpp_model(app, model)
@@ -179,21 +228,16 @@ async def local_chat_completion(app, messages, model, tools=[], opts={}, agent_r
                 
             delta = chunk['choices'][0]['delta']
             if 'content' in delta:
-                chunk_text = delta['content']
-                content_stream.feed(chunk_text)
+                send_message('output', delta['content'])
             # Handle tool_call deltas if the model emits them
             if 'tool_calls' in delta:
                 # Forward tool call chunks as JSON strings for the consumer
-                for tc in delta['tool_calls']:
-                    content_stream.feed(json.dumps({'tool_call': tc}))
-                if agent_run:
-                    for tc in delta['tool_calls']:
-                        agent_run.emit_event("tool_call", {
-                            "id": tc.get("id"),
-                            "function": tc.get("function"),
-                        })
+                send_message("tool-call", {
+                    "id": tc.get("id"),
+                    "function": tc.get("function"),
+                })
 
-        content_stream.close()
+        send_message('close')
 
     thread = threading.Thread(target=worker)
     if agent_run:
@@ -209,6 +253,7 @@ async def ollama_list(app, messages, model, opts={}):
     return ollama.list()
 
 async def ollama_chat_completion(app, messages, model, tools=[], opts={}, agent_run=None):
+    raise Exception("ollama disabled")
     if agent_run:
         content_stream = agent_run.stream
     else:
@@ -259,6 +304,29 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
     if not agent_run:
         raise Exception("agent_run required")
 
+    # TODO: Separate as InferenceRun class
+    content_stream = None
+    reasoning_stream = None
+    async def message_loop(name, value=None):
+        nonlocal content_stream, reasoning_stream, agent_run
+        if name == 'output':
+            if content_stream == None:
+                content_stream = Stream(app)
+                agent_run.emit_event('output', content_stream)
+            content_stream.feed(value)
+        if name == 'reasoning':
+            if reasoning_stream == None:
+                reasoning_stream = Stream(app)
+                agent_run.emit_event('output', reasoning_stream)
+            reasoning_stream.feed(value)
+        if name == 'tool-call':
+            agent_run.emit_event('tool-call', value)
+        elif name == 'close':
+            if content_stream: content_stream.close()
+            if reasoning_stream: reasoning_stream.close()
+            agent_run.emit_event('inference-end', {})
+    emit = safe_queue(message_loop)
+
     async def worker():
         print("START OPENROUTER INFERENCE")
         if not app.openrouter_client:
@@ -266,7 +334,7 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
 
         kwargs = {
             'model': model,
-            'messages': [msg.to_dict() for msg in messages],
+            'messages': format_openai(messages),
             'stream': True,
         }
 
@@ -294,9 +362,9 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
                     delta = event.choices[0].delta
 
                     if delta.content:
-                        agent_run.feed_output(delta.content)
+                        emit('output', delta.content)
                     if delta.reasoning:
-                        agent_run.feed_reasoning(delta.reasoning)
+                        emit('reasoning', delta.reasoning)
                     if delta.tool_calls:
                         print("[openrouter] tool call:" + repr(delta.tool_calls)) 
                         for tc in delta.tool_calls:
@@ -325,9 +393,9 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
                 try:
                     arguments = json.loads(tc.arguments)
                     tc.arguments = arguments
-                    agent_run.emit_event('tool-call', tc)
+                    emit('tool-call', tc)
                 except Exception:
-                    print("[openrouter] Bad tool call arguments")
+                    print("[openrouter] Bad tool call arguments for " + repr(tc))
                     print(traceback.format_exc())
         except asyncio.CancelledError:
             print(f"[openrouter] task cancelled for run {agent_run.id}")
@@ -336,7 +404,7 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
             print(f"[openrouter] exception during inference")
             print(traceback.format_exc())
 
-        agent_run.close()
+        emit('close')
         print("END OPENROUTER INFERENCE")
 
     task = asyncio.create_task(worker())