temperature config for expert

backend/src/agents/expert.py

@@ -20,6 +20,7 @@ class ExpertAgent(Agent):
     schema_fields = [
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
         AgentField('model', 'model_id', default=None),
+        AgentField('temperature', 'number', default=None),
     ]
 
     def __init__(self, app, data):
@@ -56,7 +57,14 @@ class ExpertAgent(Agent):
             model_engine = db_model.value["engine"]
             
             if model_engine == "llama-cpp":
-                response = await inference.local_chat_completion(self.app, messages, model=db_model.value['model'])
+                response = await inference.local_chat_completion(
+                    self.app,
+                    messages,
+                    model=db_model.value['model'],
+                    opts={
+                        'temperature':self.config.get('temperature')
+                    }
+                )
             elif model_engine == "replicate":
                 response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model'])
             else:

backend/src/inference.py

@@ -33,15 +33,20 @@ async def local_chat_completion(app, messages, model, opts={}):
         else:
             llm = Llama(    
                 model_path=f"{app.data_directory}/models/{model}",
-                # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+                n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                # n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=2048, # Uncomment to increase the context window
             )
             model_cache[model] = llm
 
+        temp = opts.get("temperature")
+        if temp is None:
+            temp = 0.25
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
+            temperature = temp,
             stream = True,
         )