llama model cache, a few bugfixes

backend/src/inference.py

@@ -21,22 +21,23 @@ class Models:
     qwen3_235b = "qwen/qwen3-235b-a22b-instruct-2507"
 
 # llama cpp models
-# stores a single model
-class ModelCache:
-    last_used = None
-    last_model = None
+model_cache = {}
 
 # llama.cpp 
 async def local_chat_completion(app, messages, model, opts={}):
     content_stream = Stream(app)
 
     def worker():
-        llm = Llama(    
-            model_path=f"{app.data_directory}/models/{model}",
-            # n_gpu_layers=-1, # Uncomment to use GPU acceleration
-            # seed=1337, # Uncomment to set a specific seed
-            # n_ctx=2048, # Uncomment to increase the context window
-        )
+        if model in model_cache:
+            llm = model_cache[model]
+        else:
+            llm = Llama(    
+                model_path=f"{app.data_directory}/models/{model}",
+                # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+                # seed=1337, # Uncomment to set a specific seed
+                # n_ctx=2048, # Uncomment to increase the context window
+            )
+            model_cache[model] = llm
 
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(