Llama-cpp inference, database stored models

.gitignore

@@ -4,4 +4,5 @@ __pycache__
 horde.db
 
 /test-data
-/local-data
+/local-data
+/models

backend/requirements.txt

@@ -6,4 +6,5 @@ nest_asyncio
 replicate
 qdrant-client[fastembed]
 ollama
-tortoise-orm
+tortoise-orm
+llama-cpp-python

backend/src/agents/expert.py

@@ -19,7 +19,7 @@ class ExpertAgent(Agent):
     description = CLASS_DESCRIPTION
     schema_fields = [
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
-        AgentField('model', 'string', default="qwen-big"),
+        AgentField('model', 'model_id', default=None),
     ]
 
     def __init__(self, app, data):
@@ -51,7 +51,15 @@ class ExpertAgent(Agent):
 
         model = self.config['model']
 
-        if model.startswith("ollama:"):
+        if type(model) is int:
+            db_model = await self.app.get_model(model)
+            
+            if db_model.value["engine"] == "llama-cpp":
+                response = await inference.local_chat_completion(self.app, messages, model=db_model.value['model'])
+            else:
+                raise Exception("Unknown model: " + repr(model))
+
+        elif model.startswith("ollama:"):
             response = await inference.ollama_chat_completion(self.app, messages, model=model[7:])
         elif model == "qwen-big":
             response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b)

backend/src/app.py

@@ -244,10 +244,32 @@ class App(ChatMixin):
             agent_id=agent_id, 
             kind=kind, 
             key=key,
-            defaults={"value": value}
+            value=value,
         )
         print(f"Saved agent resource: {agent_id} - {kind} - {key}")
 
+    async def get_models(self, kind=None):
+        return await db.Model.all()
+    async def get_model(self, model_id):
+        return await db.Model.get(id=model_id)
+
+    async def save_model(self, model_id, kind, name, value):
+        if model_id is None:
+            model = await db.Model.create(
+                name=name, 
+                kind=kind, 
+                value=value,
+            )
+        else:
+            model = await db.Model.update_or_create(
+                id=model_id, 
+                name=name, 
+                kind=kind, 
+                value=value,
+            )
+        print(f"Saved model: {model.id} - {name}")
+        return model
+
 # --- Main Entry Point ---
 
 async def main():

backend/src/database.py

@@ -128,6 +128,11 @@ class MessageDatum(models.Model):
     class Meta:
         table = "message_data"
 
+class Model(models.Model):
+    id = fields.IntField(pk=True)
+    name = fields.CharField(max_length=255)
+    kind = fields.CharField(max_length=50)
+    value = fields.JSONField(null=True)
 
 """
 class User(Base):

backend/src/inference.py

@@ -2,6 +2,7 @@ import threading
 import ollama
 from stream import Stream
 
+from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
 
@@ -19,6 +20,27 @@ class Models:
     qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
+# llama cpp models
+# stores a single model
+class ModelCache:
+    last_used = None
+    last_model = None
+
+# llama.cpp
+async def local_chat_completion(app, messages, model, opts={}):
+    llm = Llama(    
+        model_path=f"{app.data_directory}/models/{model}",
+        # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+        # seed=1337, # Uncomment to set a specific seed
+        # n_ctx=2048, # Uncomment to increase the context window
+    )
+    response = llm.create_chat_completion(
+        messages = [{'role': msg.role, 'content': msg.content} for msg in messages]
+    )
+
+    # TODO: KeyError: 'text'
+    return response["choices"][0]["text"]
+
 async def ollama_list(app, messages, model, opts={}):
     # TODO: test the output, and use this
     return ollama.list()
@@ -42,6 +64,8 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     print("END OLLAMA INFERENCE")
     return await content_stream.content_wait()
 
+    #return content_stream
+
 
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None:

backend/src/server.py

@@ -31,6 +31,7 @@ def start_server(app):
 
     # These paths must match the routes in the frontend SPA
     # file: frontend/src/routes/index.js
+    @server.route('/models')
     @server.route('/chat')
     @server.route('/agent')
     @server.route('/chat/<path:subpath>')
@@ -167,6 +168,21 @@ def start_server(app):
                     )
                     await send({'message': "agent updated", 'agentId': agent_id, 'nonce': nonce})
 
+                elif action == "get-models":
+                    await send({
+                        'models': [{"id": model.id, "name": model.name} for model in await app.get_models()],
+                        'nonce': nonce
+                    })
+
+                elif action == "save-model":
+                    model = await app.save_model(
+                        model_id=body.get("modelId"),
+                        name=body["name"],
+                        kind=body["kind"],
+                        value=body["value"],
+                    )
+                    await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+
                 elif action == "get-agent-config-fields":
                     await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce})
 

frontend/src/components/main/Sidebar.vue

@@ -105,6 +105,7 @@
     <nav v-if="mode=='agents'">
       <RouterLink v-for="it in agents" :to="'/agent/' + it.id" class="nav-item">{{it.name}}</RouterLink>
       <RouterLink to="/agent/create" class="nav-item">Create Agent</RouterLink>
+      <RouterLink to="/models" class="nav-item">Manage Models</RouterLink>
     </nav>
   </aside>
 </template>

frontend/src/router/index.js

@@ -38,6 +38,11 @@ const router = createRouter({
       path: `/agent/create`,
       component: () => import('../views/AgentCreate.vue')
     },
+    {
+      name: 'models',
+      path: `/models`,
+      component: () => import('../views/Models.vue')
+    },
     objectView('chat', () => import('../views/Chat.vue')),
     objectView('agent', () => import('../views/Agent.vue')),
   ],

frontend/src/views/Agent.vue

@@ -21,6 +21,7 @@
   const configFields = ref([]);
   // Only used for scripted agents.
   const actionDefinitions = ref([]);
+  const modelList = ref([]);
 
 
   // === State Management === //
@@ -54,6 +55,7 @@
         name: field.name,
         type: field.type,
         value: field.value,
+        kind: field.type,
         default: null,
         enabled: field.value != null,
       }));
@@ -72,6 +74,11 @@
     });
   }
 
+  async function loadModels () {
+    let res = await backend.sendAction("get-models");
+    modelList.value = res.models;
+  }
+
   // This won't fire when immediately, the mounted callback will take care
   watch(() => props.id, newId => {
     loadAgentData(newId);
@@ -79,6 +86,7 @@
 
   onMounted(async () => {
     loadAgentData(props.id);
+    loadModels();
   });
 
   // === Prompt === //
@@ -336,7 +344,11 @@
         <span class="label">{{ configField.name }}</span>
         <input type="checkbox" v-model="configField.enabled" />
         <br>
-        <textarea v-model="configField.value" class="config-input" :disabled="!configField.enabled"/>
+        <select v-if="configField.kind == 'model_id'" v-model="configField.value">
+          <option v-if="!modelList.length" disabled>empty</option>
+          <option :value="model.id" v-for="model in modelList">{{ model.name }}</option>
+        </select>
+        <textarea v-else v-model="configField.value" class="config-input" :disabled="!configField.enabled"/>
       </div>
       
       <button type="submit" @click="saveChanges()">Save</button>

frontend/src/views/Models.vue

@@ -0,0 +1,91 @@
+<script setup>
+  import {
+    ref, computed, reactive,
+    onMounted, onUnmounted,
+  } from "vue";
+
+  import TitleBar from '../components/TitleBar.vue';
+
+  import * as backend from '../backend';
+  import * as ui from '../ui';
+
+  const models = ref([]);
+  const createExpanded = ref(false);
+
+  const editModel = ref({kind: 'text', name: null, inputValue: '{}'});
+
+  onMounted(async () => {
+    let res = await backend.sendAction("get-models");
+    models.value = res.models;
+  });
+
+  async function createModel () {
+    let current = Object.assign({}, editModel.value);
+
+    if (!current.name) {
+      ui.alert("Missing name");
+      return;
+    }
+
+    try {
+      current.value = JSON.parse(current.inputValue);
+      delete current.inputValue;
+    } catch (exception) {
+      ui.alert("Bad value: " + exception.toString());
+      return;
+    }
+    
+    let res = await backend.sendAction("save-model", current);
+    if (res.modelId) {
+      current.id = res.modelId;
+      models.value.push(current)
+    } else {
+      ui.alert("bad response: " + JSON.stringify(res));
+    }
+  }
+
+  function viewModel (model) {
+    alert(JSON.stringify(model));
+  }
+
+  function deleteModel (model) {
+    alert("TODO: delete Model. ID: " + model.id);
+  }
+</script>
+
+<template>
+  <main>
+    <TitleBar>
+      <h1>Models</h1>
+    </TitleBar>
+
+    <div class="entry" v-for="model in models">
+      {{ model.name }}
+      <button @click="viewModel(model)">View</button>
+      <button @click="deleteModel(model)">Delete</button>
+    </div>
+
+    <div class="form-container">
+      <template v-if="createExpanded">
+        <p>Kinds is only text for now. Value must be like:</p>
+        <pre>{}</pre>
+        <input type="text" v-model="editModel.kind" placeholder="kind">
+        <input type="text" v-model="editModel.name" placeholder="name">
+        <textarea v-model="editModel.inputValue" placeholder="{}"></textarea>
+        <div>
+          <button @click="createModel()">Create</button>
+          <button @click="createExpanded=false">Cancel</button>
+        </div>
+      </template>
+      <button @click="createExpanded=true" v-else>Create Model</button>
+    </div>
+  </main>
+</template>
+
+<style scoped>
+  .form-container {
+    display: flex;
+    flex-direction: column;
+  }
+  .entry {}
+</style>