Reimplement replicate inference. Allow model deletion. Implement repetition penalty. Fixes in stream reading.

backend/src/agents/expert.py

@@ -21,6 +21,7 @@ class ExpertAgent(Agent):
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
         AgentField('model', 'model_id', default=None),
         AgentField('temperature', 'number', default=None),
+        AgentField('repetition_penalty', 'number', default=None),
     ]
 
     def __init__(self, app, data):
@@ -55,18 +56,27 @@ class ExpertAgent(Agent):
         if type(model) is int:
             db_model = await self.app.get_model(model)
             model_engine = db_model.value["engine"]
+
+            opts = {}
+            if 'temperature' in self.config:
+                opts['temperature'] = self.config['temperature']
+            if 'repetition_penalty' in self.config:
+                opts['repetition_penalty'] = self.config['repetition_penalty']
             
             if model_engine == "llama-cpp":
                 response = await inference.local_chat_completion(
                     self.app,
                     messages,
                     model=db_model.value['model'],
-                    opts={
-                        'temperature':self.config.get('temperature')
-                    }
+                    opts=opts,
                 )
+
+                return response
             elif model_engine == "replicate":
-                response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model'])
+                if 'parameters' in db_model:
+                    opts['parameters'] = db_model['parameters']
+
+                response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts)
             else:
                 raise Exception("Unsuported model: " + repr(db_model.value))
 

backend/src/app.py

@@ -253,6 +253,8 @@ class App(ChatMixin):
     async def get_model(self, model_id):
         return await db.Model.get(id=model_id)
 
+    # Serves for creating and updating models.
+    # Deletes the model by the id if all other parameters are empty.
     async def save_model(self, model_id, kind, name, value):
         if model_id is None:
             model = await db.Model.create(
@@ -261,12 +263,17 @@ class App(ChatMixin):
                 value=value,
             )
         else:
-            model = await db.Model.update_or_create(
-                id=model_id, 
-                name=name, 
-                kind=kind, 
-                value=value,
-            )
+            if value is None and name is None and kind is None:
+                await db.Model.filter(id=model_id).delete()
+                print(f"Deleted model: {model_id}")
+                return None
+            else:
+                model = await db.Model.update_or_create(
+                    id=model_id, 
+                    name=name, 
+                    kind=kind, 
+                    value=value,
+                )
         print(f"Saved model: {model.id} - {name}")
         return model
 

backend/src/inference.py

@@ -5,6 +5,7 @@ from stream import Stream
 from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
+CONTEXT_WINDOW = 10_000
 
 class LlamaMessage:
     def __init__(self, role, content):
@@ -35,7 +36,7 @@ async def local_chat_completion(app, messages, model, opts={}):
                 model_path=f"{app.data_directory}/models/{model}",
                 n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=CONTEXT_WINDOW,
             )
             model_cache[model] = llm
 
@@ -43,10 +44,15 @@ async def local_chat_completion(app, messages, model, opts={}):
         if temp is None:
             temp = 0.25
 
+        repeat = opts.get("repetition_penalty")
+        if repeat is None:
+            repeat = 1.0
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
             temperature = temp,
+            repeat_penalty = repeat,
             stream = True,
         )
 
@@ -90,6 +96,22 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     #return content_stream
 
 
+async def replicate_completion(app, messages, model=None, opts={}):
+    if model is None:
+        model = Models.llama_8b
+
+    parameter_names = opts.get('parameters', {})
+
+    arguments = {}
+    arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages)
+    arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75)
+    arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048)
+
+    arguments[parameter_names.get('system_prompt', 'system_prompt')] = ""
+
+    return await replicate_base_completion(app, model, arguments)
+
+# DEPRECATED
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.llama_8b
@@ -101,7 +123,7 @@ async def replicate_llama_completion(app, messages, model=None, opts={}):
         "system_prompt": "" # System prompt is formatted in the prompt
     })
 
-
+# DEPRECATED
 async def replicate_qwen_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.qwen3_8b

backend/src/server.py

@@ -6,6 +6,7 @@ from quart_auth import (
 import json
 import asyncio
 import nest_asyncio
+from stream import Stream
 
 # Vue frontend
 # relative to the module file???? wtf
@@ -18,6 +19,11 @@ def start_server(app):
     server.secret_key = 'horde-secret'
     auth_manager = QuartAuth(server, mode="brearer")
 
+    stream_listeners = {}
+
+    def cleanup(self):
+        stream_listeners = []
+
     @server.after_request
     def after_request(response):
         response.headers.add('Access-Control-Allow-Origin', '*')
@@ -48,10 +54,14 @@ def start_server(app):
     async def login():
         data = await request.get_json()
 
-        if "username" not in data or "password" not in data:
+        # token has precedence if it's present
+        if "token" in data:
+            user_id = auth_manager.load_token(data['token'])
+        elif "username" in data and "password" in data:
+            user_id = await app.verify_user_credentials(data['username'], data['password'])
+        else:
             return "invalid login arguments", 400
 
-        user_id = await app.verify_user_credentials(data['username'], data['password'])
         if user_id == None:
             return "bad login credentials", 401
 
@@ -177,11 +187,14 @@ def start_server(app):
                 elif action == "save-model":
                     model = await app.save_model(
                         model_id=body.get("modelId"),
-                        name=body["name"],
-                        kind=body["kind"],
-                        value=body["value"],
+                        name=body.get("name"),
+                        kind=body.get("kind"),
+                        value=body.get("value"),
                     )
-                    await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    if model:
+                        await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    elif modelId in body:
+                        await send({'message': "model deleted", 'modelId': body["modelId"], 'nonce': nonce})
 
                 elif action == "get-agent-config-fields":
                     await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce})
@@ -262,11 +275,29 @@ def start_server(app):
                         body.get('input'),
                         body.get('options', {})
                     )
+                    if isinstance(response, Stream):
+                        if body.get('stream'):
+                            response = {"streamId": response.id}
+                        else:
+                            response = await response.content_wait()
                     await send({'message': "prompt sent", 'response': response, 'nonce': nonce})
 
                 elif action == "call-agent-action":
                     result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments'])
                     await send({'message': "agent action executed", 'result': result, 'nonce': nonce})
+                
+                elif action == "read-stream":
+                    stream_id = body['streamId']
+                    print("Read Stream" + str(stream_id))
+                    stream = app.streams.get(stream_id)
+                    if stream is None:
+                        await send({'error': 'bad stream id: ' + str(stream_id), 'nonce': nonce})
+                    else:
+                        async def chunk_fn(chunk):
+                            await send({'data': chunk, 'eof': False, 'nonce': nonce})
+                        async def eof_fn():
+                            await send({'eof': True, 'fullData': stream.content, 'nonce': nonce})
+                        stream.listen(chunk_fn, eof_fn)
 
                 else:
                     await send({'error': 'unknown action: ' + action, 'nonce': nonce})