temperature config for expert

backend/src/inference.py

@@ -33,15 +33,20 @@ async def local_chat_completion(app, messages, model, opts={}):
         else:
             llm = Llama(    
                 model_path=f"{app.data_directory}/models/{model}",
-                # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+                n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                # n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=2048, # Uncomment to increase the context window
             )
             model_cache[model] = llm
 
+        temp = opts.get("temperature")
+        if temp is None:
+            temp = 0.25
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
+            temperature = temp,
             stream = True,
         )