diff --git a/backend/src/agent.py b/backend/src/agent.py index 4a34fa1..1462968 100644 --- a/backend/src/agent.py +++ b/backend/src/agent.py @@ -85,11 +85,19 @@ class Agent: chat_view = await self.app.get_chat_view(message.chat_id) thread = chat_view.get_thread(message.thread_id) + remotes = [] messages = [] for msg in thread.messages: role = self._role_for_message(msg) messages.append({"role": role, "content": msg.content}) + for msg in thread.resources: + # Remote declarations: content is the remote name + if msg.subkind == 'remote': + name = msg.content + if (self.user_id, name) in self.app.remotes and name not in remotes: + remotes.append(name) + run = AgentRun.create( self.app, agent_id=self.id, @@ -98,6 +106,7 @@ class Agent: "messages": messages, "chat_id": message.chat_id, "thread_id": message.thread_id, + "remotes": remotes, }, ) diff --git a/backend/src/agents/expert.py b/backend/src/agents/expert.py index d0b262e..9fedeff 100644 --- a/backend/src/agents/expert.py +++ b/backend/src/agents/expert.py @@ -24,19 +24,24 @@ class ExpertAgent(Agent): async def run(self, run): opts = {} - # Inject remotes as tools - if 'remotes' in run.input: - run.input['tools'] = run.input.get('tools', []) + self.app.getRemoteTools(self.user_id, run.input['remotes']) - # Is this roon using *Agent's* tools, as opposed to run tools using_agent_tools = False - if 'tools' not in run.input: + tools = run.input.get('tools', []) + + if not tools: # Agent tools will fully replace run tools if none are present, # otherwise they won't be used if self.tools: - run.input['tools'] = self.tools + tools = self.tools using_agent_tools = True + # Inject remotes as tools + if 'remotes' in run.input: + tools += self.app.getRemoteTools(self.user_id, run.input['remotes']) + + print("Final tool object: " + repr(tools)) + if tools: run.input['tools'] = tools + if self.config.get("temperature") is not None: opts["temperature"] = self.config["temperature"] if self.config.get("repetition_penalty") is not None: @@ -83,8 +88,8 @@ class ExpertAgent(Agent): await inference_end.wait() - content = await output_stream.content_wait(); - reasoning = await reasoning_stream.content_wait(); + content = output_stream and await output_stream.content_wait(); + reasoning = reasoning_stream and await reasoning_stream.content_wait(); if content or reasoning: msg = inference.LlamaMessage('assistant', content or "") @@ -95,7 +100,7 @@ class ExpertAgent(Agent): if using_agent_tools and tool_calls: print("Will execute tool calls") for tc in tool_calls: - tool = next(t for t in self.tools if t.name == tc.tool_name) + tool = next(t for t in tools if t.name == tc.tool_name) if not tool: print(f"Bad agent tool name: {tc.name}") continue diff --git a/backend/src/app.py b/backend/src/app.py index 00b0f88..fb99cd3 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -138,15 +138,15 @@ class App(ChatMixin): tools = [] for spec in remote_specs or []: if isinstance(spec, str): - name, prefix = spec, '' + name, prefix = spec, None else: - name, prefix = spec.get('name'), spec.get('prefix', '') + name, prefix = spec.get('name'), spec.get('prefix', None) connection = self.remotes.get((user_id, name)) if connection is None: print(f"WARNING: remote '{name}' requested by run input is not connected, skipping") continue for fn in connection.functions.values(): - tools.append(RemoteToolSpec(connection, fn, name=prefix + fn.name)) + tools.append(fn.get_spec(connection, prefix=prefix)) return tools def resolve_remote_call(self, request_id, result=None, error=None): diff --git a/backend/src/app_mixins/chat.py b/backend/src/app_mixins/chat.py index fc1c107..3369cbb 100644 --- a/backend/src/app_mixins/chat.py +++ b/backend/src/app_mixins/chat.py @@ -152,6 +152,10 @@ class ChatMixin: # Notification order depends on whether the content is a stream async def notify_agents(): + # Not all messages are visible to agents + if message.kind == "resource": + return + # Fetch participant agents if tmp.is_tmp_id(chat_id): agent_ids = self.tmp.chats[chat_id].participants diff --git a/backend/src/chat.py b/backend/src/chat.py index 1a63cf8..1e23211 100644 --- a/backend/src/chat.py +++ b/backend/src/chat.py @@ -53,7 +53,7 @@ class Thread: self.messages = [] self.notes = [] - self.memories = [] + self.resources = [] class ChatView: def __init__(self, app, chat_id, participant_type=None, participant_id=None): @@ -157,8 +157,8 @@ class ChatView: if message.kind == 'note': thread.notes.insert(0, message) - elif message.kind == 'memory': - thread.memories.insert(0, message) + elif message.kind == 'resource': + thread.resources.insert(0, message) else: thread.messages.insert(0, message) diff --git a/backend/src/inference.py b/backend/src/inference.py index 99c7b13..927eec2 100644 --- a/backend/src/inference.py +++ b/backend/src/inference.py @@ -350,6 +350,8 @@ async def openrouter_chat_completion(app, messages, model, tools=[], opts={}, ag if formatted_tools: kwargs['tools'] = formatted_tools + print("all kwargs: " + repr(kwargs)) + stream_response = await app.openrouter_client.chat.send_async(**kwargs) # OpenAI format streams tool calls in parts diff --git a/backend/src/remote.py b/backend/src/remote.py index 158ee86..5a906ef 100644 --- a/backend/src/remote.py +++ b/backend/src/remote.py @@ -2,6 +2,8 @@ import asyncio import itertools import json +from tool import ToolSpec, ToolParameter + # Seconds to wait for a remote to answer a call before failing it REMOTE_CALL_TIMEOUT = 30 @@ -24,6 +26,32 @@ class RemoteFunction: "type": "object", "properties": {}, "required": [], } + def get_spec(self, connection, prefix=None): + + async def call_function(**kwargs): + try: + result = await connection.call(self.name, kwargs) + except RemoteError as err: + result = {"error": str(err)} + # ToolCall.result is string content; remote results are JSON values + if not isinstance(result, str): + result = json.dumps(result) + return result + + return ToolSpec( + name=(prefix or '') + self.name, + description=self.description, + parameters = [ + ToolParameter( + name=name, + type=p.get("type", {}), + description=p.get("description"), + required=name in self.parameters.get('required', []) + ) for (name, p) in self.parameters.get('properties', {}).items() + ], + func=call_function + ) + def parse_function_specs(specs): """Parse the 'tools' list of a remote-connect message into a diff --git a/frontend/src/components/Message.vue b/frontend/src/components/Message.vue index 7975207..032b62c 100644 --- a/frontend/src/components/Message.vue +++ b/frontend/src/components/Message.vue @@ -32,6 +32,8 @@ function chatEntryClass(msg) { return ['note']; } else if (msg.type == 'message') { return ['message', msg.sender == 'user' ? 'sent' : 'received']; + } else if (msg.type == 'resource') { + return ['resource', 'resource-' + (msg.subkind || 'generic')]; } else if (msg.type == 'log') { return ['message', 'log']; } else { @@ -39,6 +41,11 @@ function chatEntryClass(msg) { } } +function resourceLabel(msg) { + if (msg.subkind == 'remote') return 'Remote'; + return msg.subkind || 'Resource'; +} + const format = ref(null); const content = ref(""); @@ -79,6 +86,11 @@ watch(() => [props.message.format, props.message.content], ([newFormat, newConte
{{content}}
+ +