chat: proper fork behavior in backend, wip frontend

backend/src/inference.py

@@ -4,6 +4,7 @@ import json
 import os
 #import ollama
 import traceback
+import random
 from stream import Stream
 from tool import format_tools_for_engine, build_tool_prompt_prefix, ToolSpec, ToolCall
 
@@ -152,6 +153,8 @@ async def chat_completion(app,
     opts={},
     agent_run=None
 ):
+    # TESTING OVERRIDE
+    return await mock_completion(app, agent_run)
     db_model = (await app.get_model(model_id)).value
 
     model_engine = db_model['engine']
@@ -462,6 +465,32 @@ async def replicate_base_completion(app, model, input, agent_run=None):
         return None
     return content_stream
 
+async def mock_completion(app, agent_run=None):
+    VERBS = ['fix','scan','sync','deploy','parse','render','cache','trace',
+        'merge','purge','index','probe','flush','bind','warm','drain']
+    OBJECTS = ['nodes','packets','tokens','frames','shards','clusters',
+       'buffers','threads','sockets','records','layers','streams',
+       'handles','slots','cursors','segments']
+    DELAY = 0.3
+
+    content_stream = Stream(app)
+    if agent_run:
+        agent_run.emit_event('output', content_stream)
+
+    async def content_task ():
+        await asyncio.sleep(DELAY)
+        content_stream.feed(random.choice(VERBS))
+        await asyncio.sleep(DELAY)
+        content_stream.feed(f" {random.randint(1, 99)} ")
+        await asyncio.sleep(DELAY)
+        content_stream.feed(random.choice(VERBS))
+        await asyncio.sleep(DELAY)
+        content_stream.close()
+        agent_run.emit_event('inference-end', {})
+
+    asyncio.create_task(content_task())
+    if not agent_run: return content_stream
+
 
 def run_replicate_inference_loop(replicate_client, model, input, content_stream, agent_run=None):
     print("START INFERENCE")