feat: (WIP) improve openrouter tool parsing and openai tool encoding

backend/src/inference.py

@@ -3,8 +3,9 @@ import asyncio
 import json
 import os
 import ollama
+import traceback
 from stream import Stream
-from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec
+from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec, ToolCall
 
 from llama_cpp import Llama
 from openrouter import OpenRouter
@@ -115,11 +116,6 @@ async def chat_completion(app,
     model_engine = db_model['engine']
     model_name = db_model['model']
 
-    # Warn when tools are passed to engines that do not support native tool calling
-    if tools:
-        if model_engine not in ("llama-cpp",):
-            print(f"WARNING: Tool calls not supported for engine '{model_engine}'. Tools will be ignored.")
-
     if model_engine == "llama-cpp":
         result = await local_chat_completion(app, messages, model=model_name, tools=tools, opts=opts, agent_run=agent_run)
     elif model_engine == "replicate":
@@ -260,10 +256,8 @@ async def ollama_chat_completion(app, messages, model, tools=[], opts={}, agent_
 
 
 async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, agent_run=None):
-    if agent_run:
-        content_stream = agent_run.stream
-    else:
-        content_stream = Stream(app)
+    if not agent_run:
+        raise Exception("agent_run required")
 
     async def worker():
         print("START OPENROUTER INFERENCE")
@@ -284,6 +278,12 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
         try:
             stream_response = await app.openrouter_client.chat.send_async(**kwargs)
 
+            # OpenAI format streams tool calls in parts
+            # We could, but I won't assume any tool call is complete until the message is over
+            # TODO: tool_call_by_index = {}
+            tool_calls = []
+            last_tool_call = None
+
             async for event in stream_response:
                 if agent_run and agent_run.cancelled:
                     print(f"Cancelling openrouter inference for run {agent_run.id}")
@@ -292,22 +292,55 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
                     
                 if event.choices and len(event.choices) > 0:
                     delta = event.choices[0].delta
-                    if delta.content:
-                        content_stream.feed(delta.content)
-        except asyncio.CancelledError:
-            print(f"OpenRouter task cancelled for run {agent_run.id if agent_run else 'unknown'}")
-            raise
 
-        content_stream.close()
+                    if delta.content:
+                        agent_run.feed_output(delta.content)
+                    if delta.reasoning:
+                        agent_run.feed_reasoning(delta.reasoning)
+                    if delta.tool_calls:
+                        print("[openrouter] tool call:" + repr(delta.tool_calls)) 
+                        for tc in delta.tool_calls:
+                            if not last_tool_call:
+                                last_tool_call = ToolCall(None, arguments="")
+                                tool_calls.append(last_tool_call)
+
+                            if tc.id:
+                                if last_tool_call.id is None:
+                                    last_tool_call.id = tc.id
+                                elif last_tool_call.id != tc.id:
+                                    # New id = new tool
+                                    # TODO: WRONG! new id must append value to accumulating id. new index signals new tool
+                                    last_tool_call = ToolCall(None, arguments="", id=tc.id)
+                                    tool_calls.append(last_tool_call)
+
+                            if tc.function:
+                                if tc.function.name:
+                                    last_tool_call.tool_name = tc.function.name
+                                if tc.function.arguments:
+                                    last_tool_call.arguments += tc.function.arguments
+                else:
+                    print("[openrouter] no event choices: " + repr(event))
+
+            for tc in tool_calls:
+                try:
+                    arguments = json.loads(tc.arguments)
+                    tc.arguments = arguments
+                    agent_run.emit_event('tool-call', tc)
+                except Exception:
+                    print("[openrouter] Bad tool call arguments")
+                    print(traceback.format_exc())
+        except asyncio.CancelledError:
+            print(f"[openrouter] task cancelled for run {agent_run.id}")
+            raise
+        except Exception:
+            print(f"[openrouter] exception during inference")
+            print(traceback.format_exc())
+
+        agent_run.close()
         print("END OPENROUTER INFERENCE")
 
     task = asyncio.create_task(worker())
-    if agent_run:
-        agent_run.task = task
-
-    if agent_run:
-        return None
-    return content_stream
+    agent_run.task = task
 
 
 async def replicate_completion(app, messages, model=None, tools=[], opts={}, agent_run=None):