Llama-cpp inference, database stored models

backend/src/inference.py

@@ -2,6 +2,7 @@ import threading
 import ollama
 from stream import Stream
 
+from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
 
@@ -19,6 +20,27 @@ class Models:
     qwen3_8b = "lucataco/qwen3-vl-8b-instruct:39e893666996acf464cff75688ad49ac95ef54e9f1c688fbc677330acc478e11"
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
+# llama cpp models
+# stores a single model
+class ModelCache:
+    last_used = None
+    last_model = None
+
+# llama.cpp
+async def local_chat_completion(app, messages, model, opts={}):
+    llm = Llama(    
+        model_path=f"{app.data_directory}/models/{model}",
+        # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+        # seed=1337, # Uncomment to set a specific seed
+        # n_ctx=2048, # Uncomment to increase the context window
+    )
+    response = llm.create_chat_completion(
+        messages = [{'role': msg.role, 'content': msg.content} for msg in messages]
+    )
+
+    # TODO: KeyError: 'text'
+    return response["choices"][0]["text"]
+
 async def ollama_list(app, messages, model, opts={}):
     # TODO: test the output, and use this
     return ollama.list()
@@ -42,6 +64,8 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     print("END OLLAMA INFERENCE")
     return await content_stream.content_wait()
 
+    #return content_stream
+
 
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None: