diff --git a/backend/src/agents/expert.py b/backend/src/agents/expert.py
index 48fa08d..3f75bfb 100644
--- a/backend/src/agents/expert.py
+++ b/backend/src/agents/expert.py
@@ -20,6 +20,7 @@ class ExpertAgent(Agent):
schema_fields = [
AgentField('system_prompt', 'string', default="You are an expert AI assistant."),
AgentField('model', 'model_id', default=None),
+ AgentField('temperature', 'number', default=None),
]
def __init__(self, app, data):
@@ -56,7 +57,14 @@ class ExpertAgent(Agent):
model_engine = db_model.value["engine"]
if model_engine == "llama-cpp":
- response = await inference.local_chat_completion(self.app, messages, model=db_model.value['model'])
+ response = await inference.local_chat_completion(
+ self.app,
+ messages,
+ model=db_model.value['model'],
+ opts={
+ 'temperature':self.config.get('temperature')
+ }
+ )
elif model_engine == "replicate":
response = await inference.replicate_llama_completion(self.app, messages, model=db_model.value['model'])
else:
diff --git a/backend/src/inference.py b/backend/src/inference.py
index 4280de0..c71b5e6 100644
--- a/backend/src/inference.py
+++ b/backend/src/inference.py
@@ -33,15 +33,20 @@ async def local_chat_completion(app, messages, model, opts={}):
else:
llm = Llama(
model_path=f"{app.data_directory}/models/{model}",
- # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+ n_gpu_layers=-1, # Uncomment to use GPU acceleration
# seed=1337, # Uncomment to set a specific seed
- # n_ctx=2048, # Uncomment to increase the context window
+ n_ctx=2048, # Uncomment to increase the context window
)
model_cache[model] = llm
+ temp = opts.get("temperature")
+ if temp is None:
+ temp = 0.25
+
# With stream=True, returns an iterator that infers tokens on demand (openai compatible)
llm_output = llm.create_chat_completion(
messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
+ temperature = temp,
stream = True,
)
diff --git a/frontend/src/views/Agent.vue b/frontend/src/views/Agent.vue
index 40da151..e95053f 100644
--- a/frontend/src/views/Agent.vue
+++ b/frontend/src/views/Agent.vue
@@ -195,7 +195,23 @@
const dataObject = {};
for (let field of configFields.value) {
if (field.enabled) {
- dataObject[field.name] = field.value;
+ let value = field.value;
+ if (field.kind == 'number') {
+ try {
+ value = parseFloat(value);
+ } catch (e) {
+ ui.alert("bad number at field " + field.name);
+ return;
+ }
+ } else if (field.kind != 'string') {
+ try {
+ value = JSON.parse(value);
+ } catch (e) {
+ ui.alert("bad JSON value at field " + field.name);
+ return;
+ }
+ }
+ dataObject[field.name] = value;
}
}
@@ -348,6 +364,7 @@
+