From fe596761dc76077973e9651e8c240265ec338be3 Mon Sep 17 00:00:00 2001 From: Shivering Ocelot Date: Fri, 27 Mar 2026 20:06:30 -0500 Subject: [PATCH] Reimplement replicate inference. Allow model deletion. Implement repetition penalty. Fixes in stream reading. --- backend/src/agents/expert.py | 18 +++++++--- backend/src/app.py | 19 +++++++---- backend/src/inference.py | 26 ++++++++++++-- backend/src/server.py | 43 ++++++++++++++++++++---- frontend/src/backend.js | 9 +++++ frontend/src/components/main/Sidebar.vue | 6 ++++ frontend/src/router/index.js | 5 +++ frontend/src/views/Chat.vue | 18 ++++++++++ frontend/src/views/Models.vue | 5 +-- frontend/src/views/User.vue | 40 ++++++++++++++++++++++ 10 files changed, 169 insertions(+), 20 deletions(-) create mode 100644 frontend/src/views/User.vue diff --git a/backend/src/agents/expert.py b/backend/src/agents/expert.py index 3f75bfb..22e59ad 100644 --- a/backend/src/agents/expert.py +++ b/backend/src/agents/expert.py @@ -21,6 +21,7 @@ class ExpertAgent(Agent): AgentField('system_prompt', 'string', default="You are an expert AI assistant."), AgentField('model', 'model_id', default=None), AgentField('temperature', 'number', default=None), + AgentField('repetition_penalty', 'number', default=None), ] def __init__(self, app, data): @@ -55,18 +56,27 @@ class ExpertAgent(Agent): if type(model) is int: db_model = await self.app.get_model(model) model_engine = db_model.value["engine"] + + opts = {} + if 'temperature' in self.config: + opts['temperature'] = self.config['temperature'] + if 'repetition_penalty' in self.config: + opts['repetition_penalty'] = self.config['repetition_penalty'] if model_engine == "llama-cpp": response = await inference.local_chat_completion( self.app, messages, model=db_model.value['model'], - opts={ - 'temperature':self.config.get('temperature') - } + opts=opts, ) + + return response elif model_engine == "replicate": - response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model']) + if 'parameters' in db_model: + opts['parameters'] = db_model['parameters'] + + response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts) else: raise Exception("Unsuported model: " + repr(db_model.value)) diff --git a/backend/src/app.py b/backend/src/app.py index 7c6bfb4..1a4cf1b 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -253,6 +253,8 @@ class App(ChatMixin): async def get_model(self, model_id): return await db.Model.get(id=model_id) + # Serves for creating and updating models. + # Deletes the model by the id if all other parameters are empty. async def save_model(self, model_id, kind, name, value): if model_id is None: model = await db.Model.create( @@ -261,12 +263,17 @@ class App(ChatMixin): value=value, ) else: - model = await db.Model.update_or_create( - id=model_id, - name=name, - kind=kind, - value=value, - ) + if value is None and name is None and kind is None: + await db.Model.filter(id=model_id).delete() + print(f"Deleted model: {model_id}") + return None + else: + model = await db.Model.update_or_create( + id=model_id, + name=name, + kind=kind, + value=value, + ) print(f"Saved model: {model.id} - {name}") return model diff --git a/backend/src/inference.py b/backend/src/inference.py index c71b5e6..c2d280b 100644 --- a/backend/src/inference.py +++ b/backend/src/inference.py @@ -5,6 +5,7 @@ from stream import Stream from llama_cpp import Llama VALID_ROLES = ['user', 'assistant', 'system'] +CONTEXT_WINDOW = 10_000 class LlamaMessage: def __init__(self, role, content): @@ -35,7 +36,7 @@ async def local_chat_completion(app, messages, model, opts={}): model_path=f"{app.data_directory}/models/{model}", n_gpu_layers=-1, # Uncomment to use GPU acceleration # seed=1337, # Uncomment to set a specific seed - n_ctx=2048, # Uncomment to increase the context window + n_ctx=CONTEXT_WINDOW, ) model_cache[model] = llm @@ -43,10 +44,15 @@ async def local_chat_completion(app, messages, model, opts={}): if temp is None: temp = 0.25 + repeat = opts.get("repetition_penalty") + if repeat is None: + repeat = 1.0 + # With stream=True, returns an iterator that infers tokens on demand (openai compatible) llm_output = llm.create_chat_completion( messages = [{'role': msg.role, 'content': msg.content} for msg in messages], temperature = temp, + repeat_penalty = repeat, stream = True, ) @@ -90,6 +96,22 @@ async def ollama_chat_completion(app, messages, model, opts={}): #return content_stream +async def replicate_completion(app, messages, model=None, opts={}): + if model is None: + model = Models.llama_8b + + parameter_names = opts.get('parameters', {}) + + arguments = {} + arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages) + arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75) + arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048) + + arguments[parameter_names.get('system_prompt', 'system_prompt')] = "" + + return await replicate_base_completion(app, model, arguments) + +# DEPRECATED async def replicate_llama_completion(app, messages, model=None, opts={}): if model is None: model = Models.llama_8b @@ -101,7 +123,7 @@ async def replicate_llama_completion(app, messages, model=None, opts={}): "system_prompt": "" # System prompt is formatted in the prompt }) - +# DEPRECATED async def replicate_qwen_completion(app, messages, model=None, opts={}): if model is None: model = Models.qwen3_8b diff --git a/backend/src/server.py b/backend/src/server.py index 5538f5f..4a75356 100644 --- a/backend/src/server.py +++ b/backend/src/server.py @@ -6,6 +6,7 @@ from quart_auth import ( import json import asyncio import nest_asyncio +from stream import Stream # Vue frontend # relative to the module file???? wtf @@ -18,6 +19,11 @@ def start_server(app): server.secret_key = 'horde-secret' auth_manager = QuartAuth(server, mode="brearer") + stream_listeners = {} + + def cleanup(self): + stream_listeners = [] + @server.after_request def after_request(response): response.headers.add('Access-Control-Allow-Origin', '*') @@ -48,10 +54,14 @@ def start_server(app): async def login(): data = await request.get_json() - if "username" not in data or "password" not in data: + # token has precedence if it's present + if "token" in data: + user_id = auth_manager.load_token(data['token']) + elif "username" in data and "password" in data: + user_id = await app.verify_user_credentials(data['username'], data['password']) + else: return "invalid login arguments", 400 - user_id = await app.verify_user_credentials(data['username'], data['password']) if user_id == None: return "bad login credentials", 401 @@ -177,11 +187,14 @@ def start_server(app): elif action == "save-model": model = await app.save_model( model_id=body.get("modelId"), - name=body["name"], - kind=body["kind"], - value=body["value"], + name=body.get("name"), + kind=body.get("kind"), + value=body.get("value"), ) - await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce}) + if model: + await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce}) + elif modelId in body: + await send({'message': "model deleted", 'modelId': body["modelId"], 'nonce': nonce}) elif action == "get-agent-config-fields": await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce}) @@ -262,11 +275,29 @@ def start_server(app): body.get('input'), body.get('options', {}) ) + if isinstance(response, Stream): + if body.get('stream'): + response = {"streamId": response.id} + else: + response = await response.content_wait() await send({'message': "prompt sent", 'response': response, 'nonce': nonce}) elif action == "call-agent-action": result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments']) await send({'message': "agent action executed", 'result': result, 'nonce': nonce}) + + elif action == "read-stream": + stream_id = body['streamId'] + print("Read Stream" + str(stream_id)) + stream = app.streams.get(stream_id) + if stream is None: + await send({'error': 'bad stream id: ' + str(stream_id), 'nonce': nonce}) + else: + async def chunk_fn(chunk): + await send({'data': chunk, 'eof': False, 'nonce': nonce}) + async def eof_fn(): + await send({'eof': True, 'fullData': stream.content, 'nonce': nonce}) + stream.listen(chunk_fn, eof_fn) else: await send({'error': 'unknown action: ' + action, 'nonce': nonce}) diff --git a/frontend/src/backend.js b/frontend/src/backend.js index 70a2f98..8805651 100644 --- a/frontend/src/backend.js +++ b/frontend/src/backend.js @@ -24,6 +24,7 @@ let cache = { let loginToken = null; let loginTokenDeferred = createDeferred(); let authDeferred = createDeferred(); +let credentialsDeferred = createDeferred(); // Handlers const pendingActions = {}; // Maps nonce to { resolve, reject } @@ -98,6 +99,7 @@ export async function signup(username, email, password) { // Returns a login token for the given credentials export async function login(username, password) { try { + credentialsDeferred.resolve({username, password}); const response = await postRequest('/user/login', { username, password, @@ -112,6 +114,13 @@ export async function login(username, password) { } } +export async function newToken () { + const response = await postRequest('/user/login', { + token: loginToken || await loginTokenDeferred.promise, + }); + return response.token; +} + // --- Communication --- diff --git a/frontend/src/components/main/Sidebar.vue b/frontend/src/components/main/Sidebar.vue index 66b5bfc..b3f0866 100644 --- a/frontend/src/components/main/Sidebar.vue +++ b/frontend/src/components/main/Sidebar.vue @@ -90,6 +90,9 @@
+
+ +
+ +