Reimplement replicate inference. Allow model deletion. Implement repetition penalty. Fixes in stream reading.

backend/src/agents/expert.py

@@ -21,6 +21,7 @@ class ExpertAgent(Agent):
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
         AgentField('model', 'model_id', default=None),
         AgentField('temperature', 'number', default=None),
+        AgentField('repetition_penalty', 'number', default=None),
     ]
 
     def __init__(self, app, data):
@@ -56,17 +57,26 @@ class ExpertAgent(Agent):
             db_model = await self.app.get_model(model)
             model_engine = db_model.value["engine"]
 
+            opts = {}
+            if 'temperature' in self.config:
+                opts['temperature'] = self.config['temperature']
+            if 'repetition_penalty' in self.config:
+                opts['repetition_penalty'] = self.config['repetition_penalty']
+            
             if model_engine == "llama-cpp":
                 response = await inference.local_chat_completion(
                     self.app,
                     messages,
                     model=db_model.value['model'],
-                    opts={
+                    opts=opts,
-                        'temperature':self.config.get('temperature')
-                    }
                 )
+
+                return response
             elif model_engine == "replicate":
-                response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model'])
+                if 'parameters' in db_model:
+                    opts['parameters'] = db_model['parameters']
+
+                response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts)
             else:
                 raise Exception("Unsuported model: " + repr(db_model.value))
 

backend/src/app.py

@@ -253,6 +253,8 @@ class App(ChatMixin):
     async def get_model(self, model_id):
         return await db.Model.get(id=model_id)
 
+    # Serves for creating and updating models.
+    # Deletes the model by the id if all other parameters are empty.
     async def save_model(self, model_id, kind, name, value):
         if model_id is None:
             model = await db.Model.create(
@@ -261,12 +263,17 @@ class App(ChatMixin):
                 value=value,
             )
         else:
-            model = await db.Model.update_or_create(
+            if value is None and name is None and kind is None:
-                id=model_id, 
+                await db.Model.filter(id=model_id).delete()
-                name=name, 
+                print(f"Deleted model: {model_id}")
-                kind=kind, 
+                return None
-                value=value,
+            else:
-            )
+                model = await db.Model.update_or_create(
+                    id=model_id, 
+                    name=name, 
+                    kind=kind, 
+                    value=value,
+                )
         print(f"Saved model: {model.id} - {name}")
         return model
 

backend/src/inference.py

@@ -5,6 +5,7 @@ from stream import Stream
 from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
+CONTEXT_WINDOW = 10_000
 
 class LlamaMessage:
     def __init__(self, role, content):
@@ -35,7 +36,7 @@ async def local_chat_completion(app, messages, model, opts={}):
                 model_path=f"{app.data_directory}/models/{model}",
                 n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=CONTEXT_WINDOW,
             )
             model_cache[model] = llm
 
@@ -43,10 +44,15 @@ async def local_chat_completion(app, messages, model, opts={}):
         if temp is None:
             temp = 0.25
 
+        repeat = opts.get("repetition_penalty")
+        if repeat is None:
+            repeat = 1.0
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
             temperature = temp,
+            repeat_penalty = repeat,
             stream = True,
         )
 
@@ -90,6 +96,22 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     #return content_stream
 
 
+async def replicate_completion(app, messages, model=None, opts={}):
+    if model is None:
+        model = Models.llama_8b
+
+    parameter_names = opts.get('parameters', {})
+
+    arguments = {}
+    arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages)
+    arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75)
+    arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048)
+
+    arguments[parameter_names.get('system_prompt', 'system_prompt')] = ""
+
+    return await replicate_base_completion(app, model, arguments)
+
+# DEPRECATED
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.llama_8b
@@ -101,7 +123,7 @@ async def replicate_llama_completion(app, messages, model=None, opts={}):
         "system_prompt": "" # System prompt is formatted in the prompt
     })
 
-
+# DEPRECATED
 async def replicate_qwen_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.qwen3_8b

backend/src/server.py

@@ -6,6 +6,7 @@ from quart_auth import (
 import json
 import asyncio
 import nest_asyncio
+from stream import Stream
 
 # Vue frontend
 # relative to the module file???? wtf
@@ -18,6 +19,11 @@ def start_server(app):
     server.secret_key = 'horde-secret'
     auth_manager = QuartAuth(server, mode="brearer")
 
+    stream_listeners = {}
+
+    def cleanup(self):
+        stream_listeners = []
+
     @server.after_request
     def after_request(response):
         response.headers.add('Access-Control-Allow-Origin', '*')
@@ -48,10 +54,14 @@ def start_server(app):
     async def login():
         data = await request.get_json()
 
-        if "username" not in data or "password" not in data:
+        # token has precedence if it's present
+        if "token" in data:
+            user_id = auth_manager.load_token(data['token'])
+        elif "username" in data and "password" in data:
+            user_id = await app.verify_user_credentials(data['username'], data['password'])
+        else:
             return "invalid login arguments", 400
 
-        user_id = await app.verify_user_credentials(data['username'], data['password'])
         if user_id == None:
             return "bad login credentials", 401
 
@@ -177,11 +187,14 @@ def start_server(app):
                 elif action == "save-model":
                     model = await app.save_model(
                         model_id=body.get("modelId"),
-                        name=body["name"],
+                        name=body.get("name"),
-                        kind=body["kind"],
+                        kind=body.get("kind"),
-                        value=body["value"],
+                        value=body.get("value"),
                     )
-                    await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    if model:
+                        await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    elif modelId in body:
+                        await send({'message': "model deleted", 'modelId': body["modelId"], 'nonce': nonce})
 
                 elif action == "get-agent-config-fields":
                     await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce})
@@ -262,12 +275,30 @@ def start_server(app):
                         body.get('input'),
                         body.get('options', {})
                     )
+                    if isinstance(response, Stream):
+                        if body.get('stream'):
+                            response = {"streamId": response.id}
+                        else:
+                            response = await response.content_wait()
                     await send({'message': "prompt sent", 'response': response, 'nonce': nonce})
 
                 elif action == "call-agent-action":
                     result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments'])
                     await send({'message': "agent action executed", 'result': result, 'nonce': nonce})
                 
+                elif action == "read-stream":
+                    stream_id = body['streamId']
+                    print("Read Stream" + str(stream_id))
+                    stream = app.streams.get(stream_id)
+                    if stream is None:
+                        await send({'error': 'bad stream id: ' + str(stream_id), 'nonce': nonce})
+                    else:
+                        async def chunk_fn(chunk):
+                            await send({'data': chunk, 'eof': False, 'nonce': nonce})
+                        async def eof_fn():
+                            await send({'eof': True, 'fullData': stream.content, 'nonce': nonce})
+                        stream.listen(chunk_fn, eof_fn)
+
                 else:
                     await send({'error': 'unknown action: ' + action, 'nonce': nonce})
         finally:

frontend/src/backend.js

@@ -24,6 +24,7 @@ let cache = {
 let loginToken = null;
 let loginTokenDeferred = createDeferred();
 let authDeferred = createDeferred();
+let credentialsDeferred = createDeferred();
 
 // Handlers
 const pendingActions = {}; // Maps nonce to { resolve, reject }
@@ -98,6 +99,7 @@ export async function signup(username, email, password) {
 // Returns a login token for the given credentials
 export async function login(username, password) {
   try {
+    credentialsDeferred.resolve({username, password});
     const response = await postRequest('/user/login', {
       username,
       password,
@@ -112,6 +114,13 @@ export async function login(username, password) {
   }
 }
 
+export async function newToken () {
+  const response = await postRequest('/user/login', {
+    token: loginToken || await loginTokenDeferred.promise,
+  });
+  return response.token;
+}
+
 
 // --- Communication ---
 

frontend/src/components/main/Sidebar.vue

@@ -90,6 +90,9 @@
         <div @click="mode='agents'" class="tab" :class="{active: mode=='agents'}">
           <i class="fa-solid fa-user-group"></i>
         </div>
+        <div @click="mode='settings'" class="tab" :class="{active: mode=='settings'}">
+          <i class="fa-solid fa-gears"></i>
+        </div>
       </div>
       <div class="tab-group">
         <button class="header-button" @click="sidebarExpanded=false;">
@@ -105,7 +108,10 @@
     <nav v-if="mode=='agents'">
       <RouterLink v-for="it in agents" :to="'/agent/' + it.id" class="nav-item">{{it.name}}</RouterLink>
       <RouterLink to="/agent/create" class="nav-item">Create Agent</RouterLink>
+    </nav>
+    <nav v-if="mode=='settings'">
       <RouterLink to="/models" class="nav-item">Manage Models</RouterLink>
+      <RouterLink to="/user" class="nav-item">User</RouterLink>
     </nav>
   </aside>
 </template>

frontend/src/router/index.js

@@ -43,6 +43,11 @@ const router = createRouter({
       path: `/models`,
       component: () => import('../views/Models.vue')
     },
+    {
+      name: 'user',
+      path: `/user`,
+      component: () => import('../views/User.vue')
+    },
     objectView('chat', () => import('../views/Chat.vue')),
     objectView('agent', () => import('../views/Agent.vue')),
   ],

frontend/src/views/Chat.vue

@@ -326,6 +326,18 @@
   }
 
 
+  // === Special Chat Functions === //
+
+  async function sendPicture () {
+    // todo
+  }
+
+  async function uploadFile () {
+    // todo
+  }
+
+
+
   // === UI Helper Functions === //
 
   async function chatEntryContextMenu (message) {
@@ -453,6 +465,12 @@
         <button v-if="editingMessageId != null" @click="editMessage(null)">
           Cancel Edit
         </button>
+        <button @click="sendPicture()">
+          Send Picture
+        </button>
+        <button @click="uploadFile()">
+          Upload File
+        </button>
         <button id="send-button" @click="sendMessage()">
           Send
           <span style="font-size: 66%;">(Ctrl+Enter)</span>

frontend/src/views/Models.vue

@@ -48,8 +48,9 @@
     alert(JSON.stringify(model));
   }
 
-  function deleteModel (model) {
+  async function deleteModel (model) {
-    alert("TODO: delete Model. ID: " + model.id);
+    let res = await backend.sendAction("save-model", {modelId: model.id, value: null});
+    ui.alert(res.message);
   }
 </script>
 

frontend/src/views/User.vue

@@ -0,0 +1,40 @@
+<script setup>
+  //import TheWelcome from '../components/TheWelcome.vue'
+  import TitleBar from '../components/TitleBar.vue';
+  import {
+    ref, computed, reactive, watch,
+    onMounted, onUnmounted,
+  } from "vue";
+  import * as backend from '../backend';
+  import * as ui from '../ui';
+
+  let token = ref(null);
+
+  async function generateToken() {
+    token.value = await backend.newToken();
+  }
+</script>
+
+<template>
+  <main>
+    <TitleBar>
+      <h1>User Details</h1>
+    </TitleBar>
+
+    <div class="home-content">
+      <p>This is the Hub for Horde Agents. You can configure agents, interact and chat with them.</p>
+      <button @click="generateToken()">Generate Access Token</button>
+      <pre v-if="token != null" @input.stop.prevent="false">{{ token }}</pre>
+    </div>
+
+    <!--TheWelcome /-->
+  </main>
+</template>
+
+<style scoped>
+  .home-content {
+    max-width: 900px;
+    margin: 2em auto auto;
+    padding: 0 5em 0 2em;
+  }
+</style>