Reimplement replicate inference. Allow model deletion. Implement repetition penalty. Fixes in stream reading.

backend/src/server.py

@@ -6,6 +6,7 @@ from quart_auth import (
 import json
 import asyncio
 import nest_asyncio
+from stream import Stream
 
 # Vue frontend
 # relative to the module file???? wtf
@@ -18,6 +19,11 @@ def start_server(app):
     server.secret_key = 'horde-secret'
     auth_manager = QuartAuth(server, mode="brearer")
 
+    stream_listeners = {}
+
+    def cleanup(self):
+        stream_listeners = []
+
     @server.after_request
     def after_request(response):
         response.headers.add('Access-Control-Allow-Origin', '*')
@@ -48,10 +54,14 @@ def start_server(app):
     async def login():
         data = await request.get_json()
 
-        if "username" not in data or "password" not in data:
+        # token has precedence if it's present
+        if "token" in data:
+            user_id = auth_manager.load_token(data['token'])
+        elif "username" in data and "password" in data:
+            user_id = await app.verify_user_credentials(data['username'], data['password'])
+        else:
             return "invalid login arguments", 400
 
-        user_id = await app.verify_user_credentials(data['username'], data['password'])
         if user_id == None:
             return "bad login credentials", 401
 
@@ -177,11 +187,14 @@ def start_server(app):
                 elif action == "save-model":
                     model = await app.save_model(
                         model_id=body.get("modelId"),
-                        name=body["name"],
-                        kind=body["kind"],
-                        value=body["value"],
+                        name=body.get("name"),
+                        kind=body.get("kind"),
+                        value=body.get("value"),
                     )
-                    await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    if model:
+                        await send({'message': "model saved", 'modelId': model.id, 'nonce': nonce})
+                    elif modelId in body:
+                        await send({'message': "model deleted", 'modelId': body["modelId"], 'nonce': nonce})
 
                 elif action == "get-agent-config-fields":
                     await send({'fields': await app.get_agent_config_fields(body['agentId']), 'nonce': nonce})
@@ -262,11 +275,29 @@ def start_server(app):
                         body.get('input'),
                         body.get('options', {})
                     )
+                    if isinstance(response, Stream):
+                        if body.get('stream'):
+                            response = {"streamId": response.id}
+                        else:
+                            response = await response.content_wait()
                     await send({'message': "prompt sent", 'response': response, 'nonce': nonce})
 
                 elif action == "call-agent-action":
                     result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments'])
                     await send({'message': "agent action executed", 'result': result, 'nonce': nonce})
+                
+                elif action == "read-stream":
+                    stream_id = body['streamId']
+                    print("Read Stream" + str(stream_id))
+                    stream = app.streams.get(stream_id)
+                    if stream is None:
+                        await send({'error': 'bad stream id: ' + str(stream_id), 'nonce': nonce})
+                    else:
+                        async def chunk_fn(chunk):
+                            await send({'data': chunk, 'eof': False, 'nonce': nonce})
+                        async def eof_fn():
+                            await send({'eof': True, 'fullData': stream.content, 'nonce': nonce})
+                        stream.listen(chunk_fn, eof_fn)
 
                 else:
                     await send({'error': 'unknown action: ' + action, 'nonce': nonce})