temperature config for expert

backend/src/agents/expert.py

@@ -20,6 +20,7 @@ class ExpertAgent(Agent):
     schema_fields = [
         AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
         AgentField('model', 'model_id', default=None),
+        AgentField('temperature', 'number', default=None),
     ]
 
     def __init__(self, app, data):
@@ -56,7 +57,14 @@ class ExpertAgent(Agent):
             model_engine = db_model.value["engine"]
             
             if model_engine == "llama-cpp":
-                response = await inference.local_chat_completion(self.app, messages, model=db_model.value['model'])
+                response = await inference.local_chat_completion(
+                    self.app,
+                    messages,
+                    model=db_model.value['model'],
+                    opts={
+                        'temperature':self.config.get('temperature')
+                    }
+                )
             elif model_engine == "replicate":
                 response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model'])
             else:

backend/src/inference.py

@@ -33,15 +33,20 @@ async def local_chat_completion(app, messages, model, opts={}):
         else:
             llm = Llama(    
                 model_path=f"{app.data_directory}/models/{model}",
-                # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+                n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                # n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=2048, # Uncomment to increase the context window
             )
             model_cache[model] = llm
 
+        temp = opts.get("temperature")
+        if temp is None:
+            temp = 0.25
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
+            temperature = temp,
             stream = True,
         )
 

frontend/src/views/Agent.vue

@@ -195,7 +195,23 @@
     const dataObject = {};
     for (let field of configFields.value) {
       if (field.enabled) {
-        dataObject[field.name] = field.value;
+        let value = field.value;
+        if (field.kind == 'number') {
+          try {
+            value = parseFloat(value);
+          } catch (e) {
+            ui.alert("bad number at field " + field.name);
+            return;
+          }
+        } else if (field.kind != 'string') {
+          try {
+            value = JSON.parse(value);
+          } catch (e) {
+            ui.alert("bad JSON value at field " + field.name);
+            return;
+          }
+        }
+        dataObject[field.name] = value;
       }
     }
 
@@ -348,6 +364,7 @@
           <option v-if="!modelList.length" disabled>empty</option>
           <option :value="model.id" v-for="model in modelList">{{ model.name }}</option>
         </select>
+        <input v-else-if="configField.kind=='number'" type="number" v-model="configField.value" class="config-input" :disabled="!configField.enabled"/>
         <textarea v-else v-model="configField.value" class="config-input" :disabled="!configField.enabled"/>
       </div>