Llama-cpp inference, database stored models

backend/requirements.txt

@@ -6,4 +6,5 @@ nest_asyncio
 replicate
 qdrant-client[fastembed]
 ollama
-tortoise-orm
+tortoise-orm
+llama-cpp-python

backend/src/agents/expert.py

@@ -19,7 +19,7 @@ class ExpertAgent(Agent):
     description = CLASS_DESCRIPTION
     schema_fields = [
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
-        AgentField('model', 'string', default="qwen-big"),
+        AgentField('model', 'model_id', default=None),
     ]
 
     def __init__(self, app, data):
@@ -51,7 +51,15 @@ class ExpertAgent(Agent):
 
         model = self.config['model']
 
-        if model.startswith("ollama:"):
+        if type(model) is int:
+            db_model = await self.app.get_model(model)
+            
+            if db_model.value["engine"] == "llama-cpp":
+                response = await inference.local_chat_completion(self.app, messages, model=db_model.value['model'])
+            else:
+                raise Exception("Unknown model: " + repr(model))
+
+        elif model.startswith("ollama:"):
             response = await inference.ollama_chat_completion(self.app, messages, model=model[7:])
         elif model == "qwen-big":
             response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b)

backend/src/app.py

@@ -244,10 +244,32 @@ class App(ChatMixin):
             agent_id=agent_id, 
             kind=kind, 
             key=key,
-            defaults={"value": value}
+            value=value,
         )
         print(f"Saved agent resource: {agent_id} - {kind} - {key}")
 
+    async def get_models(self, kind=None):
+        return await db.Model.all()
+    async def get_model(self, model_id):
+        return await db.Model.get(id=model_id)
+
+    async def save_model(self, model_id, kind, name, value):
+        if model_id is None:
+            model = await db.Model.create(
+                name=name, 
+                kind=kind, 
+                value=value,
+            )
+        else:
+            model = await db.Model.update_or_create(
+                id=model_id, 
+                name=name, 
+                kind=kind, 
+                value=value,
+            )
+        print(f"Saved model: {model.id} - {name}")
+        return model
+
 # --- Main Entry Point ---
 
 async def main():

backend/src/database.py

@@ -128,6 +128,11 @@ class MessageDatum(models.Model):
     class Meta:
         table = "message_data"
 
+class Model(models.Model):
+    id = fields.IntField(pk=True)
+    name = fields.CharField(max_length=255)
+    kind = fields.CharField(max_length=50)
+    value = fields.JSONField(null=True)
 
 """
 class User(Base):

backend/src/inference.py

@@ -2,6 +2,7 @@ import threading
 import ollama
 from stream import Stream
 
+from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
 
@@ -19,6 +20,27 @@ class Models:
     qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
+# llama cpp models
+# stores a single model
+class ModelCache:
+    last_used = None
+    last_model = None
+
+# llama.cpp
+async def local_chat_completion(app, messages, model, opts={}):
+    llm = Llama(    
+        model_path=f"{app.data_directory}/models/{model}",
+        # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+        # seed=1337, # Uncomment to set a specific seed
+        # n_ctx=2048, # Uncomment to increase the context window
+    )
+    response = llm.create_chat_completion(
+        messages = [{'role': msg.role, 'content': msg.content} for msg in messages]
+    )
+
+    # TODO: KeyError: 'text'
+    return response["choices"][0]["text"]
+
 async def ollama_list(app, messages, model, opts={}):
     # TODO: test the output, and use this
     return ollama.list()
@@ -42,6 +64,8 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     print("END OLLAMA INFERENCE")
     return await content_stream.content_wait()
 
+    #return content_stream
+
 
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None:

backend/src/server.py

@@ -31,6 +31,7 @@ def start_server(app):
 
     # These paths must match the routes in the frontend SPA
     # file: frontend/src/routes/index.js
+    @server.route('/models')
     @server.route('/chat')
     @server.route('/agent')
     @server.route('/chat/<path:subpath>')
@@ -167,6 +168,21 @@ def start_server(app):
                     )
                     await send({'message': "agent updated", 'agentId': agent_id, 'nonce': nonce})
 
+                elif action == "get-models":
+                    await send({
+                        'models': [{"id": model.id, "name": model.name} for model in await app.get_models()],
+                        'nonce': nonce
+                    })
+
+                elif action == "save-model":
+                    model = await app.save_model(
+                        model_id=body.get("modelId"),
+                        name=body["name"],
+                        kind=body["kind"],
+                        value=body["value"],
+                    )
+                    await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+
                 elif action == "get-agent-config-fields":
                     await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce})