From 39fa4f1123d46e95ea4bed0f9ca78bcec67c7e6c Mon Sep 17 00:00:00 2001
From: Shivering Ocelot <shiveringocelot@pm.me>
Date: Mon, 19 Jan 2026 02:44:17 -0500
Subject: [PATCH] llama model cache, a few bugfixes

---
 backend/src/chat.py          |  3 ++-
 backend/src/inference.py     | 21 +++++++++++----------
 frontend/src/views/Agent.vue |  2 +-
 3 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/backend/src/chat.py b/backend/src/chat.py
index e226bff..c83a333 100644
--- a/backend/src/chat.py
+++ b/backend/src/chat.py
@@ -91,7 +91,8 @@ class ChatView:
             filters["id__lte"] = self._earliest_message_id
 
         # Tortoise query: filter, order, and limit
-        records_rev = await db.Message.filter(**filters).order_by("-time").limit(load_count)
+        # ordering by id is more robust than by time
+        records_rev = await db.Message.filter(**filters).order_by("-id").limit(load_count)
 
         # Update the earliest id for the next load operation
         if records_rev:
diff --git a/backend/src/inference.py b/backend/src/inference.py
index b6ecef0..4280de0 100644
--- a/backend/src/inference.py
+++ b/backend/src/inference.py
@@ -21,22 +21,23 @@ class Models:
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
 # llama cpp models
-# stores a single model
-class ModelCache:
-    last_used = None
-    last_model = None
+model_cache = {}
 
 # llama.cpp 
 async def local_chat_completion(app, messages, model, opts={}):
     content_stream = Stream(app)
 
     def worker():
-        llm = Llama(    
-            model_path=f"{app.data_directory}/models/{model}",
-            # n_gpu_layers=-1, # Uncomment to use GPU acceleration
-            # seed=1337, # Uncomment to set a specific seed
-            # n_ctx=2048, # Uncomment to increase the context window
-        )
+        if model in model_cache:
+            llm = model_cache[model]
+        else:
+            llm = Llama(    
+                model_path=f"{app.data_directory}/models/{model}",
+                # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+                # seed=1337, # Uncomment to set a specific seed
+                # n_ctx=2048, # Uncomment to increase the context window
+            )
+            model_cache[model] = llm
 
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
diff --git a/frontend/src/views/Agent.vue b/frontend/src/views/Agent.vue
index e88ad18..40da151 100644
--- a/frontend/src/views/Agent.vue
+++ b/frontend/src/views/Agent.vue
@@ -344,7 +344,7 @@
         <span class="label">{{ configField.name }}</span>
         <input type="checkbox" v-model="configField.enabled" />
         <br>
-        <select v-if="configField.kind == 'model_id'" v-model="configField.value">
+        <select v-if="configField.kind == 'model_id'" v-model="configField.value" :disabled="!configField.enabled">
           <option v-if="!modelList.length" disabled>empty</option>
           <option :value="model.id" v-for="model in modelList">{{ model.name }}</option>
         </select>