Refactor agent execution to use AgentRun with OpenAI-compatible API
Major refactoring to support isolated run objects with streaming, events, and OpenAI-compatible endpoints. ## Backend Changes - **Agent class**: Added `run()` method, `received_message()` wraps chat into AgentRun - **AgentRun class**: New class for run state with event system, serialization, and user tracking - **inference.py**: Complete refactor with `build_messages()`, `chat_completion()`, tool support for llama-cpp/ollama/replicate - **tools.py**: New centralized tool management (ToolSpec, ToolParameter, format_tools_for_engine) - **tools.py**: Removed legacy Toolset/Action/ToolDescription system - **server.py**: Added OpenAI-compatible `/api/openai/v1/chat/completions` endpoint with streaming support - **temporary.py**: Added `self.runs` dict to State class - **expert.py**: Refactored to use new Agent.run() signature - **scripted.py**: Updated to raise NotImplementedError ## Frontend Changes - **backend.js**: Added `onRunEvents()` for run event subscriptions - **Sidebar.vue**: Added runs navigation mode - **router/index.js**: Renamed User route to RunsView - **RunsView.vue**: New view for listing/monitoring agent runs ## Breaking Changes - Agent `prompt()` signature changed: `(instruction, input=None, **kwargs)` instead of `(content)` - Agent `received_message()` must use AgentRun pattern - Tools.py removed, functionality moved to tool.py - Deprecated tool prompt/action system removed
This commit is contained in:
Shivering Ocelot committed
1 parent
eb9a1ad9e8
commit
4e15b44d39
14 files changed
+1151
-522
No files matched your search
+293
-6
@@ -1,12 +1,15 @@
|
||||
from quart import Quart, Response, request, send_from_directory, websocket
|
||||
from quart_auth import (
|
||||
QuartAuth, current_user,
|
||||
AuthUser, login_user, logout_user
|
||||
AuthUser, login_user, logout_user, login_required
|
||||
)
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
import nest_asyncio
|
||||
from stream import Stream
|
||||
from tool import ToolSpec, ToolParameter
|
||||
from agent import AgentRun
|
||||
|
||||
# Vue frontend
|
||||
# relative to the module file???? wtf
|
||||
@@ -66,6 +69,7 @@ async def start_server(app):
|
||||
return "bad login credentials", 401
|
||||
|
||||
token = auth_manager.dump_token(user_id)
|
||||
login_user(AuthUser(auth_id=user_id))
|
||||
return {"token": token}
|
||||
|
||||
@server.route('/user/signup', methods=['POST'])
|
||||
@@ -134,6 +138,11 @@ async def start_server(app):
|
||||
else:
|
||||
app.add_user_connection(user_id, send_queue)
|
||||
asyncio.create_task(sender_task())
|
||||
|
||||
# Listen for events and broadcast them
|
||||
async def on_event(name, data):
|
||||
await send({"action": event, "name": name, "data": data})
|
||||
app.on_event(on_event)
|
||||
|
||||
while True:
|
||||
req = json.loads(await websocket.receive())
|
||||
@@ -207,9 +216,31 @@ async def start_server(app):
|
||||
await send({'message': "agent kind retrieved", 'info': app.get_agent_kind_info(body['kind']), 'nonce': nonce})
|
||||
|
||||
elif action == "get-agents":
|
||||
#await send({'agents': [], 'nonce': nonce})
|
||||
await send({'message': "agents retrieved", 'agents': await app.get_agents(user_id), 'nonce': nonce})
|
||||
|
||||
elif action == "get-agent-runs":
|
||||
await send({
|
||||
'runs': await app.get_agent_runs(agent_id=body.get('agentId'), user_id=user_id),
|
||||
'nonce': nonce
|
||||
})
|
||||
|
||||
elif action == "get-agent-run":
|
||||
await send({
|
||||
'run': await app.get_agent_run(body['runId']),
|
||||
'nonce': nonce
|
||||
})
|
||||
stream_id = body['streamId']
|
||||
print("Read Stream" + str(stream_id))
|
||||
stream = app.streams.get(stream_id)
|
||||
if stream is None:
|
||||
await send({'error': 'bad stream id: ' + str(stream_id), 'nonce': nonce})
|
||||
else:
|
||||
async def chunk_fn(chunk):
|
||||
await send({'data': chunk, 'eof': False, 'nonce': nonce})
|
||||
async def eof_fn():
|
||||
await send({'eof': True, 'fullData': stream.content, 'nonce': nonce})
|
||||
stream.listen(chunk_fn, eof_fn)
|
||||
|
||||
elif action == "create-chat":
|
||||
chat_id = await app.create_chat(
|
||||
user_id=user_id,
|
||||
@@ -284,10 +315,12 @@ async def start_server(app):
|
||||
response = await response.content_wait()
|
||||
await send({'message': "prompt sent", 'response': response, 'nonce': nonce})
|
||||
|
||||
elif action == "call-agent-action":
|
||||
result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments'])
|
||||
await send({'message': "agent action executed", 'result': result, 'nonce': nonce})
|
||||
|
||||
elif action == "get-agent-runs":
|
||||
await send({
|
||||
'runs': await app.get_agent_runs(agent_id=body.get('agentId'), user_id=user_id),
|
||||
'nonce': nonce
|
||||
})
|
||||
|
||||
elif action == "read-stream":
|
||||
stream_id = body['streamId']
|
||||
print("Read Stream" + str(stream_id))
|
||||
@@ -307,4 +340,258 @@ async def start_server(app):
|
||||
print("User logged out:", user_id)
|
||||
app.remove_user_connection(user_id, send_queue)
|
||||
|
||||
# https://github.com/hrgarber/openai-compatible-api/blob/main/app/main.py
|
||||
@server.post('/api/openai/v1/chat/completions')
|
||||
@login_required
|
||||
async def openai_chat_completion():
|
||||
data = await request.get_json()
|
||||
|
||||
print("Completion: " + json.dumps(data))
|
||||
|
||||
# Extract required fields
|
||||
agent_id = data.get("model")
|
||||
openai_messages = data.get("messages", [])
|
||||
openai_tools = data.get("tools", [])
|
||||
stream_mode = data.get("stream", False)
|
||||
|
||||
if not agent_id or not openai_messages:
|
||||
return {"error": "model and messages are required"}, 400
|
||||
|
||||
user_id = current_user.auth_id
|
||||
if not user_id: pass
|
||||
|
||||
# --- Parse messages ---
|
||||
message_errors = []
|
||||
messages = []
|
||||
for msg in openai_messages:
|
||||
role = msg['role']
|
||||
if role not in ['user', 'assistant', 'system']:
|
||||
message_errors.append("Bad role: " + role)
|
||||
continue
|
||||
content = msg.get('content', None)
|
||||
|
||||
if type(content) == list:
|
||||
parts = content
|
||||
content = ""
|
||||
for part in parts:
|
||||
if part['type'] == 'text':
|
||||
content += part['text']
|
||||
|
||||
if type(content) != str:
|
||||
message_errors.append("Bad content")
|
||||
continue
|
||||
|
||||
messages.append({'role': role, 'content': content})
|
||||
|
||||
if message_errors:
|
||||
print("[openai] ERROR: bad messages input\n" + "\n".join([f"\t- {e}" for e in message_errors]))
|
||||
|
||||
# --- Parse tools (OpenAI format -> ToolSpec) ---
|
||||
tools = []
|
||||
for tool in openai_tools:
|
||||
if tool.get('type') != 'function':
|
||||
continue
|
||||
func = tool['function']
|
||||
params_schema = func.get('parameters', {})
|
||||
required_params = params_schema.get('required', [])
|
||||
parameters = []
|
||||
for param_name, param_spec in params_schema.get('properties', {}).items():
|
||||
parameters.append(ToolParameter(
|
||||
name=param_name,
|
||||
type=param_spec.get('type', 'string'),
|
||||
description=param_spec.get('description', ''),
|
||||
required=param_name in required_params,
|
||||
enum=param_spec.get('enum'),
|
||||
))
|
||||
tools.append(ToolSpec(
|
||||
name=func['name'],
|
||||
description=func.get('description', ''),
|
||||
parameters=parameters,
|
||||
))
|
||||
|
||||
# --- Get agent control and create AgentRun directly ---
|
||||
control = await app.get_agent_control(agent_id)
|
||||
|
||||
run = AgentRun.create(
|
||||
app,
|
||||
agent_id=agent_id,
|
||||
user_id=user_id,
|
||||
input={
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
},
|
||||
)
|
||||
print("run input: " + repr(run.input))
|
||||
|
||||
# --- Streaming ---
|
||||
if stream_mode:
|
||||
queue = asyncio.Queue()
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
# Listen to the stream for both content and tool_call chunks
|
||||
async def on_chunk(chunk):
|
||||
# Tool calls are fed as JSON strings: '{"tool_call": {...}}'
|
||||
try:
|
||||
parsed = json.loads(chunk)
|
||||
if isinstance(parsed, dict) and 'tool_call' in parsed:
|
||||
tc = parsed['tool_call']
|
||||
await queue.put(('tool_call', tc))
|
||||
return
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
# Regular content
|
||||
await queue.put(('content', chunk))
|
||||
|
||||
async def on_eof():
|
||||
await queue.put(('eof', None))
|
||||
|
||||
run.stream.listen(on_chunk, on_eof)
|
||||
|
||||
# Start the agent run
|
||||
await control.run(run)
|
||||
|
||||
async def generate():
|
||||
try:
|
||||
while True:
|
||||
msg_type, data = await queue.get()
|
||||
|
||||
if msg_type == 'content':
|
||||
chunk_body = {
|
||||
"id": "chatcmpl-" + str(time.time()),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": agent_id,
|
||||
"choices": [{"delta": {"content": data}, "index": 0, "finish_reason": None}]
|
||||
}
|
||||
yield f"data: {json.dumps(chunk_body)}\n\n"
|
||||
|
||||
elif msg_type == 'tool_call':
|
||||
# Forward the tool call delta (already in OpenAI format from llama-cpp)
|
||||
tc_entry = {"index": data.get('index', 0)}
|
||||
if data.get('id'):
|
||||
tc_entry['id'] = data['id']
|
||||
if 'function' in data:
|
||||
func_part = {}
|
||||
fn = data['function']
|
||||
if fn.get('name'):
|
||||
func_part['name'] = fn['name']
|
||||
if fn.get('arguments'):
|
||||
func_part['arguments'] = fn['arguments']
|
||||
if func_part:
|
||||
tc_entry['function'] = func_part
|
||||
|
||||
chunk_body = {
|
||||
"id": "chatcmpl-" + str(time.time()),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": agent_id,
|
||||
"choices": [{
|
||||
"delta": {"tool_calls": [tc_entry]},
|
||||
"index": 0,
|
||||
"finish_reason": None
|
||||
}]
|
||||
}
|
||||
yield f"data: {json.dumps(chunk_body)}\n\n"
|
||||
|
||||
elif msg_type == 'eof':
|
||||
# Determine finish_reason — was there a tool call?
|
||||
finish_reason = "tool_calls" if any(
|
||||
e['name'] == 'tool_call' for e in run.events
|
||||
) else "stop"
|
||||
|
||||
chunk_body = {
|
||||
"id": "chatcmpl-" + str(time.time()),
|
||||
"object": "chat.completion.chunk",
|
||||
"created": int(time.time()),
|
||||
"model": agent_id,
|
||||
"choices": [{"delta": {}, "index": 0, "finish_reason": finish_reason}]
|
||||
}
|
||||
yield f"data: {json.dumps(chunk_body)}\n\n"
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print("[openai] ERROR: " + str(e))
|
||||
yield f"data: {json.dumps({'error': str(e)})}\n\n"
|
||||
|
||||
return Response(generate(), content_type="text/event-stream")
|
||||
|
||||
# --- Non-streaming ---
|
||||
else:
|
||||
await control.run(run)
|
||||
await run.stream.content_wait()
|
||||
|
||||
# Collect any tool calls that were emitted
|
||||
tool_events = [e['data'] for e in run.events if e['name'] == 'tool_call']
|
||||
|
||||
if tool_events:
|
||||
# Aggregate tool calls: combine partial argument chunks by id
|
||||
tc_map = {}
|
||||
for evt in tool_events:
|
||||
tid = evt.get('id', '')
|
||||
if tid not in tc_map:
|
||||
tc_map[tid] = {
|
||||
"id": tid,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": evt.get('function', {}).get('name', ''),
|
||||
"arguments": evt.get('function', {}).get('arguments', ''),
|
||||
}
|
||||
}
|
||||
else:
|
||||
tc_map[tid]['function']['arguments'] += \
|
||||
evt.get('function', {}).get('arguments', '')
|
||||
|
||||
return {
|
||||
"id": "chatcmpl-" + str(time.time()),
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": agent_id,
|
||||
"choices": [{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": list(tc_map.values()),
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}]
|
||||
}
|
||||
else:
|
||||
# No tool calls: return text content
|
||||
content = run.stream.content
|
||||
return {
|
||||
"id": "chatcmpl-" + str(time.time()),
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": agent_id,
|
||||
"choices": [{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": content,
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}]
|
||||
}
|
||||
|
||||
@server.get("/api/openai/v1/models")
|
||||
@login_required
|
||||
async def openai_models():
|
||||
"""
|
||||
Return a list of available models.
|
||||
Customize this list based on your available models.
|
||||
"""
|
||||
|
||||
user_id = current_user.auth_id
|
||||
if not user_id: pass
|
||||
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [{
|
||||
"id": agent.id,
|
||||
"object": "model",
|
||||
"created": int(time.time()),
|
||||
"owned_by": user_id,
|
||||
} for agent in await app.get_agents(user_id)]
|
||||
}
|
||||
|
||||
|
||||
server.run(debug=True)
|
||||
Reference in new issue
Block a user