Reimplement replicate inference. Allow model deletion. Implement repetition penalty. Fixes in stream reading.

backend/src/inference.py

@@ -5,6 +5,7 @@ from stream import Stream
 from llama_cpp import Llama
 
 VALID_ROLES = ['user', 'assistant', 'system']
+CONTEXT_WINDOW = 10_000
 
 class LlamaMessage:
     def __init__(self, role, content):
@@ -35,7 +36,7 @@ async def local_chat_completion(app, messages, model, opts={}):
                 model_path=f"{app.data_directory}/models/{model}",
                 n_gpu_layers=-1, # Uncomment to use GPU acceleration
                 # seed=1337, # Uncomment to set a specific seed
-                n_ctx=2048, # Uncomment to increase the context window
+                n_ctx=CONTEXT_WINDOW,
             )
             model_cache[model] = llm
 
@@ -43,10 +44,15 @@ async def local_chat_completion(app, messages, model, opts={}):
         if temp is None:
             temp = 0.25
 
+        repeat = opts.get("repetition_penalty")
+        if repeat is None:
+            repeat = 1.0
+
         # With stream=True, returns an iterator that infers tokens on demand (openai compatible)
         llm_output = llm.create_chat_completion(
             messages = [{'role': msg.role, 'content': msg.content} for msg in messages],
             temperature = temp,
+            repeat_penalty = repeat,
             stream = True,
         )
 
@@ -90,6 +96,22 @@ async def ollama_chat_completion(app, messages, model, opts={}):
     #return content_stream
 
 
+async def replicate_completion(app, messages, model=None, opts={}):
+    if model is None:
+        model = Models.llama_8b
+
+    parameter_names = opts.get('parameters', {})
+
+    arguments = {}
+    arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages)
+    arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75)
+    arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048)
+
+    arguments[parameter_names.get('system_prompt', 'system_prompt')] = ""
+
+    return await replicate_base_completion(app, model, arguments)
+
+# DEPRECATED
 async def replicate_llama_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.llama_8b
@@ -101,7 +123,7 @@ async def replicate_llama_completion(app, messages, model=None, opts={}):
         "system_prompt": "" # System prompt is formatted in the prompt
     })
 
-
+# DEPRECATED
 async def replicate_qwen_completion(app, messages, model=None, opts={}):
     if model is None:
         model = Models.qwen3_8b