feat: OpenRouter engine, with OPENROUTER_API_KEY env var

backend/src/inference.py

@@ -1,11 +1,13 @@
 import threading
 import asyncio
 import json
+import os
 import ollama
 from stream import Stream
 from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec
 
 from llama_cpp import Llama
+from openrouter import OpenRouter
 
 VALID_ROLES = ['user', 'assistant', 'system', 'tool']
 CONTEXT_WINDOW = 10_000
@@ -130,6 +132,8 @@ async def chat_completion(app,
         result = await replicate_completion(app, messages, model=model_name, tools=tools, opts=opts, agent_run=agent_run)
     elif model_engine == "ollama":
         result = await ollama_chat_completion(app, messages, model=model_name, tools=tools, opts=opts, agent_run=agent_run)
+    elif model_engine == "openrouter":
+        result = await openrouter_chat_completion(app, messages, model=model_name, tools=tools, opts=opts, agent_run=agent_run)
     else:
         raise Exception("Unsupported model: " + repr(model))
 
@@ -236,6 +240,46 @@ async def ollama_chat_completion(app, messages, model, tools=[], opts={}, agent_
     return content_stream
 
 
+async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, agent_run=None):
+    if agent_run:
+        content_stream = agent_run.stream
+    else:
+        content_stream = Stream(app)
+
+    async def worker():
+        print("START OPENROUTER INFERENCE")
+        if not app.openrouter_client:
+            raise Exception("OPENROUTER_API_KEY environment variable is not set")
+
+        kwargs = {
+            'model': model,
+            'messages': [msg.to_dict() for msg in messages],
+            'stream': True,
+        }
+
+        # Pass tools in OpenAI-compatible format if provided
+        formatted_tools = format_tools_for_engine(tools, "openrouter") if tools else None
+        if formatted_tools:
+            kwargs['tools'] = formatted_tools
+
+        stream_response = await app.openrouter_client.chat.send_async(**kwargs)
+
+        async for event in stream_response:
+            if event.choices and len(event.choices) > 0:
+                delta = event.choices[0].delta
+                if delta.content:
+                    content_stream.feed(delta.content)
+
+        content_stream.close()
+        print("END OPENROUTER INFERENCE")
+
+    asyncio.create_task(worker())
+
+    if agent_run:
+        return None
+    return content_stream
+
+
 async def replicate_completion(app, messages, model=None, tools=[], opts={}, agent_run=None):
     if model is None:
         model = Models.llama_8b
@@ -263,40 +307,6 @@ async def replicate_completion(app, messages, model=None, tools=[], opts={}, age
         return None
     return result
 
-# DEPRECATED
-async def replicate_llama_completion(app, messages, model=None, opts={}):
-    if model is None:
-        model = Models.llama_8b
-
-    return await replicate_base_completion(app, model, {
-        "prompt": format_instruct(messages),
-        "temperature": opts.get('temperature', 0.75),
-        "max_new_tokens": opts.get('max_tokens', 2048),
-        "system_prompt": "" # System prompt is formatted in the prompt
-    })
-
-# DEPRECATED
-async def replicate_qwen_completion(app, messages, model=None, opts={}):
-    if model is None:
-        model = Models.qwen3_8b
-
-    if model == Models.qwen3_8b:
-        input = {
-            "prompt": format_instruct(messages),
-            "temperature": opts.get('temperature', 0.75),
-            "max_new_tokens": opts.get('max_tokens', 2048),
-        }
-    elif model == Models.qwen3_235b:
-        input = {
-            "prompt": format_instruct(messages),
-            "temperature": opts.get('temperature', 0.75),
-            "max_tokens": opts.get('max_tokens', 2048),
-        }
-    else:
-        raise Exception("Invalid qwen model: " + model)
-
-    return await replicate_base_completion(app, model, input)
-
 async def replicate_base_completion(app, model, input, agent_run=None):
     if agent_run:
         content_stream = agent_run.stream