feat(chat): support chat resources, implement chat-based remote discovery via resource messages

backend/src/agent.py

@@ -85,11 +85,19 @@ class Agent:
         chat_view = await self.app.get_chat_view(message.chat_id)
         thread = chat_view.get_thread(message.thread_id)
 
+        remotes = []
         messages = []
         for msg in thread.messages:
             role = self._role_for_message(msg)
             messages.append({"role": role, "content": msg.content})
 
+        for msg in thread.resources:
+            # Remote declarations: content is the remote name
+            if msg.subkind == 'remote':
+                name = msg.content
+                if (self.user_id, name) in self.app.remotes and name not in remotes:
+                    remotes.append(name)
+
         run = AgentRun.create(
             self.app,
             agent_id=self.id,
@@ -98,6 +106,7 @@ class Agent:
                 "messages": messages,
                 "chat_id": message.chat_id,
                 "thread_id": message.thread_id,
+                "remotes": remotes,
             },
         )
 

backend/src/agents/expert.py

@@ -24,19 +24,24 @@ class ExpertAgent(Agent):
     async def run(self, run):
         opts = {}
 
-        # Inject remotes as tools
-        if 'remotes' in run.input:
-            run.input['tools'] = run.input.get('tools', []) + self.app.getRemoteTools(self.user_id, run.input['remotes'])
-
         # Is this roon using *Agent's* tools, as opposed to run tools
         using_agent_tools = False
-        if 'tools' not in run.input:
+        tools = run.input.get('tools', [])
+
+        if not tools:
             # Agent tools will fully replace run tools if none are present,
             # otherwise they won't be used
             if self.tools:
-                run.input['tools'] = self.tools
+                tools = self.tools
                 using_agent_tools = True
 
+            # Inject remotes as tools
+            if 'remotes' in run.input:
+                tools += self.app.getRemoteTools(self.user_id, run.input['remotes'])
+
+        print("Final tool object: " + repr(tools))
+        if tools: run.input['tools'] = tools
+
         if self.config.get("temperature") is not None:
             opts["temperature"] = self.config["temperature"]
         if self.config.get("repetition_penalty") is not None:
@@ -83,8 +88,8 @@ class ExpertAgent(Agent):
 
             await inference_end.wait()
 
-            content = await output_stream.content_wait();
-            reasoning = await reasoning_stream.content_wait();
+            content = output_stream and await output_stream.content_wait();
+            reasoning = reasoning_stream and await reasoning_stream.content_wait();
 
             if content or reasoning:
                 msg = inference.LlamaMessage('assistant', content or "")
@@ -95,7 +100,7 @@ class ExpertAgent(Agent):
             if using_agent_tools and tool_calls:
                 print("Will execute tool calls")
                 for tc in tool_calls:
-                    tool = next(t for t in self.tools if t.name == tc.tool_name)
+                    tool = next(t for t in tools if t.name == tc.tool_name)
                     if not tool:
                         print(f"Bad agent tool name: {tc.name}")
                         continue

backend/src/app.py

@@ -138,15 +138,15 @@ class App(ChatMixin):
         tools = []
         for spec in remote_specs or []:
             if isinstance(spec, str):
-                name, prefix = spec, ''
+                name, prefix = spec, None
             else:
-                name, prefix = spec.get('name'), spec.get('prefix', '')
+                name, prefix = spec.get('name'), spec.get('prefix', None)
             connection = self.remotes.get((user_id, name))
             if connection is None:
                 print(f"WARNING: remote '{name}' requested by run input is not connected, skipping")
                 continue
             for fn in connection.functions.values():
-                tools.append(RemoteToolSpec(connection, fn, name=prefix + fn.name))
+                tools.append(fn.get_spec(connection, prefix=prefix))
         return tools
 
     def resolve_remote_call(self, request_id, result=None, error=None):

backend/src/app_mixins/chat.py

@@ -152,6 +152,10 @@ class ChatMixin:
         # Notification order depends on whether the content is a stream
 
         async def notify_agents():
+            # Not all messages are visible to agents
+            if message.kind == "resource":
+                return
+
             # Fetch participant agents
             if tmp.is_tmp_id(chat_id):
                 agent_ids = self.tmp.chats[chat_id].participants

backend/src/chat.py

@@ -53,7 +53,7 @@ class Thread:
 
         self.messages = []
         self.notes = []
-        self.memories = []
+        self.resources = []
 
 class ChatView:
     def __init__(self, app, chat_id, participant_type=None, participant_id=None):
@@ -157,8 +157,8 @@ class ChatView:
 
             if message.kind == 'note':
                 thread.notes.insert(0, message)
-            elif message.kind == 'memory':
-                thread.memories.insert(0, message)
+            elif message.kind == 'resource':
+                thread.resources.insert(0, message)
             else:
                 thread.messages.insert(0, message)
     

backend/src/inference.py

@@ -350,6 +350,8 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag
         if formatted_tools:
             kwargs['tools'] = formatted_tools
 
+        print("all kwargs: " + repr(kwargs))
+
         stream_response = await app.openrouter_client.chat.send_async(**kwargs)
 
         # OpenAI format streams tool calls in parts

backend/src/remote.py

@@ -2,6 +2,8 @@ import asyncio
 import itertools
 import json
 
+from tool import ToolSpec, ToolParameter
+
 # Seconds to wait for a remote to answer a call before failing it
 REMOTE_CALL_TIMEOUT = 30
 
@@ -24,6 +26,32 @@ class RemoteFunction:
             "type": "object", "properties": {}, "required": [],
         }
 
+    def get_spec(self, connection, prefix=None):
+
+        async def call_function(**kwargs):
+            try:
+                result = await connection.call(self.name, kwargs)
+            except RemoteError as err:
+                result = {"error": str(err)}
+            # ToolCall.result is string content; remote results are JSON values
+            if not isinstance(result, str):
+                result = json.dumps(result)
+            return result
+
+        return ToolSpec(
+            name=(prefix or '') + self.name,
+            description=self.description,
+            parameters = [
+                ToolParameter(
+                    name=name,
+                    type=p.get("type", {}),
+                    description=p.get("description"),
+                    required=name in self.parameters.get('required', [])
+                ) for (name, p) in self.parameters.get('properties', {}).items()
+            ],
+            func=call_function
+        )
+
 
 def parse_function_specs(specs):
     """Parse the 'tools' list of a remote-connect message into a