feat(backend): tool support in send agent prompt, regularize inference messages (wip)

backend/src/inference.py

@@ -5,22 +5,20 @@ import os
 import traceback
 import random
 from stream import Stream
-from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec, ToolCall
+from tool import Tool, ToolSpec, ToolCall
 
 #from llama_cpp import Llama
 from openrouter import OpenRouter
 
-VALID_ROLES = ['user', 'assistant', 'system', 'tool']
+VALID_ROLES = ['user', 'assistant', 'system', 'tool', 'reasoning']
 CONTEXT_WINDOW = 10_000
 
-# TODO: Rename to ChatMessage. Add methods for constructing from instruct/openai/etc formatted objects.
-class LlamaMessage:
-    def __init__(self, role, content, reasoning=None):
+class Message:
+    def __init__(self, role, content):
         if role not in VALID_ROLES:
             raise Exception("message role must be one of: " + ",".join(VALID_ROLES))
         self.role = role
         self.content = content
-        self.reasoning = reasoning
 
     def to_dict(self):
         """Serialize to the dict format expected by llama-cpp / ollama."""
@@ -30,44 +28,68 @@ class LlamaMessage:
         return d
 
     def __repr__(self):
-        return f"LlamaMessage(role={self.role!r}, content={self.content!r}, reasoning={self.reasoning!r})"
+        return f"Message(role={self.role!r}, content={self.content!r})"
 
-class Models:
-    llama_8b = "meta/meta-llama-3-8b-instruct"
-    llama_70b = "meta/meta-llama-3-70b-instruct"
-    qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
-    qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
+def format_tools(tools):
+    specs = []
+    for t in tools:
+        if isinstance(t, dict):
+            specs.append(t)
+        elif isinstance(t, ToolSpec):
+            specs.append(t.format_openai())
+        elif isinstance(t, Tool):
+            specs.append(t.get_spec().format_openai())
+    return specs
 
 def format_openai(messages):
+    print('format openai ' + repr(messages))
     formatted = []
-    last_assistant_message = None
+    def last_assistant_message():
+        if formatted and formatted[-1]['role'] == 'assistant':
+            return formatted[-1]
+        else:
+            msg = {'role': 'assistant', 'content': ''}
+            formatted.append(msg)
+            return msg
+
     for msg in messages:
         if msg.role == 'tool':
-            if not last_assistant_message:
-                # Create one implicitly
-                last_assistant_message = {'role': 'assistant', 'content': ''}
-            tool_calls = last_assistant_message.get('tool_calls')
+            last = last_assistant_message()
+            tool_calls = last.get('tool_calls')
             if tool_calls == None:
                 tool_calls = []
-                last_assistant_message['tool_calls'] = tool_calls
+                last['tool_calls'] = tool_calls
+            tc = msg.content
+            if isinstance(tc, dict):
+                tc = ToolCall(
+                    tool_name=tc['tool_name'],
+                    arguments=tc['arguments'],
+                    result=tc['result'],
+                    id=tc['id']
+                )
+                # Skip non-resolved calls
+                if not tc.result: continue
+                # TODO: Fix call id if missing
             tool_calls.append({
-                'id': msg.content.id,
+                'id': tc.id,
                 'type': 'function',
                 'function': {
-                    'name': msg.content.tool_name,
-                    'arguments': json.dumps(msg.content.arguments),
+                    'name': tc.tool_name,
+                    'arguments': json.dumps(tc.arguments),
                 },
             })
             formatted.append({
                 'role': 'tool',
-                'tool_call_id': msg.content.id,
-                'content': json.dumps(msg.content.result),
+                'tool_call_id': tc.id,
+                'content': json.dumps(tc.result),
             })
+        elif msg.role == 'reasoning':
+            last = last_assistant_message()
+            reasoning = last.get('reasoning', '')
+            last['reasoning'] = reasoning + msg.content
         else:
             fmsg = {'role': msg.role, 'content': msg.content}
             formatted.append(fmsg)
-            if msg.role == 'assistant':
-                last_assistant_message = fmsg
     return formatted
 
 
@@ -116,10 +138,10 @@ def get_llama_cpp_model(app, model_file):
 # Generate a conversation based on agent inputs
 def build_messages(system_prompt="", instruction=None, input=None, messages=None):
     def normalize(message):
-        if isinstance(message, LlamaMessage):
+        if isinstance(message, Message):
             return message
         elif isinstance(message, dict):
-            return LlamaMessage(message['role'], message['content'])
+            return Message(message['role'], message['content'])
         else:
             raise Exception("Unsupported message type: " + repr(type(message)))
 
@@ -139,9 +161,9 @@ def build_messages(system_prompt="", instruction=None, input=None, messages=None
 
     # Add default messages to list
     if system_prompt:
-        messages.insert(0, LlamaMessage("system", system_prompt))
+        messages.insert(0, Message("system", system_prompt))
     if prompt:
-        messages.append(LlamaMessage("user", prompt))
+        messages.append(Message("user", prompt))
 
     return messages
 
@@ -205,7 +227,7 @@ async def local_chat_completion(app, messages, model, tools=[], opts={}, agent_r
             kwargs['repeat_penalty'] = repeat
 
         # Format tools for llama-cpp (OpenAI-compatible function format)
-        formatted_tools = format_tools_for_engine(tools, "llama-cpp") if tools else None
+        formatted_tools = format_tools(tools) if tools else None
         if formatted_tools:
             kwargs['tools'] = formatted_tools
             kwargs['tool_choice'] = 'auto'
@@ -301,9 +323,10 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
         }
 
         # Pass tools in OpenAI-compatible format if provided
-        formatted_tools = format_tools_for_engine(tools, "openrouter") if tools else None
+        formatted_tools = format_tools(tools) if tools else None
         if formatted_tools:
             kwargs['tools'] = formatted_tools
+        print(f"Tools: {tools} and {formatted_tools}")
 
         temp = agent_run.input.get("temperature")
         if temp is not None: