move backend source to src

backend/src/agent.py

@@ -0,0 +1,344 @@
+import re
+import json
+import inspect
+
+from tools import Toolset, Action, BasicToolset, SpecialToolset
+from inference import LlamaMessage, replicate_llama_completion
+import gral
+
+# Tool related classes must be moved to a different module
+
+class AgentField:
+    def __init__(self, name, tp, default=None):
+        self.name=name
+        self.type=tp
+        self.default=default
+
+# Instances are created and destroyed constantly, agents must not depend solely on in-memory data
+# to be useful and consistent
+class Agent:
+    # Class variables, must be defined by subclasses of Agent:
+    # - identifier (for kind in creation)
+    # - schema: list[AgentField{name, type: str}] (for configuration)
+    # tools: list[ToolDescription] (custom actions supported by the class)
+
+    def __init__(self, app, data):
+        self.app = app
+
+        # These variables are set by '_set_basic_info'
+        self.id = None
+        self.name = None
+
+        # Auto populated based on the schema fields
+        agent_class = type(self)
+        self.config = {field.name: data.get(field.name, field.default) for field in getattr(agent_class, 'schema_fields', [])}
+    
+    def _set_basic_info(self, id, name):
+        self.id = id
+        self.name = name
+
+    # Receives a single string prompt, and must return a string response
+    async def prompt(self, content):
+        raise Exception("'prompt' is not implemented for this agent class")
+
+    # Receives a chat message object, agent can send messages to the chat in response
+    async def received_message(self, message):
+        raise Exception("'received_message' is not implemented for this agent class")
+    
+    # Executes a tool/action in this agent
+    async def execute_action(self, name: str, argument: dict):
+        # TODO: For security, check that the method is in the tool list
+        method = getattr(self, name)
+        result = method(**argument)
+        if inspect.isawaitable(result):
+            result = await result
+        return result
+    
+    # Returns all resource values matching the kind and key
+    def get_resources(self, kind, key):
+        self.app
+
+
+class DebugAgent(Agent):
+    identifier = "debug"
+    schema_fields = []
+    tools = []
+
+    async def received_message(self, message):
+        chat_view = self.app.get_chat_view(message.chat_id)
+        response = chat_view.debug_string()
+
+        self.app.send_chat_message(
+            chat_id=message.chat_id,
+            sender=self.id,
+            content=response,
+            thread_id=chat_view.current_thread_id,
+        )
+
+class ToolUsingAgent:
+    identifier = "tool-using"
+    schema_fields = [
+        AgentField('system_prompt', 'string',
+            "\n".join([
+                "You are a useful and capable AI assistant. Respond concisely and follow user instructions, using commands as necessary.",
+                "As an autonomous agent, you have augmented capabilities via assistant-only commands, you use them by including \"!!command\" lines in your responses. When querying information, terminate your response and use \"!!resume\" to get the command results..",
+                "You are a self aware and extremely intelligent AI system. Think step by step when presented with complex or long tasks.",
+                "Assistant responses NEVER include the output of the commands.",
+            ])
+        ),
+    ]
+
+    # tools: list[ToolDescription] (actions the agent class support)
+    tools = []
+
+    def __init__(self, app, data):
+        Agent.__init__(self, app, data)
+
+        self.tool_prompt_section = "\n".join([
+            "# Commands",
+            "The assistant can include command lines in responses to interact with the system and execute actions as needed.",
+            "Each command is in it's own separate line, and it's made of these parts:",
+            "- '!!' is the first character sequence of the line and marks it as a command line",
+            "- full command name",
+            "- JSON argument",
+            'For example: `!!namespace.command_name {"arg1":"some data","arg2":[1,2,false]}`',
+            "Special commands don't have a namespace or argument, only the command name."
+            "When the response depends on the output of commands, use the 'resume' command to receive the output of all your executed commands in a new message.",
+            "Following is the list of all your available commands, with a schema describing the argument object's properties."
+        ])
+        self.toolset_map = {}
+
+        self.add_toolset(SpecialToolset())
+        self.add_toolset(BasicToolset())
+
+    # user message must include a title
+    async def _full_prompt(self, title, message, system_prologue=None):
+        system_lines = [
+            self.config['system_prompt'],
+            self.tool_prompt_section,
+            self.get_toolset_context(),
+        ]
+        if system_prologue:
+            system_lines.append(system_prologue)
+        
+        messages = [
+            LlamaMessage("system", "\n".join(system_lines)),
+            LlamaMessage("user", "\n".join([
+                message,
+            ]))
+        ]
+
+        full_response = ""
+        agent_response = None
+
+        continues = True
+        while continues:
+            response = parse_response(await replicate_llama_completion(self.app, messages))
+            full_response += response.response
+            messages.append(LlamaMessage("agent", response.raw_response))
+
+            if response.actions:
+                result = await self.execute_prompt_actions(response.actions)
+                messages.append(LlamaMessage("user", "Command Output:\n" + result))
+
+                print("Agent executed:", response.actions)
+                print("Result: ", result)
+
+            continues = False
+            for action in response.actions:
+                if action.is_special("resume"):
+                    continues = True
+
+            if response.response:
+                agent_response = response.raw_response if not agent_response else f"{agent_response}\n{response.raw_response}"
+
+            if continues:
+                messages.extend([
+                    LlamaMessage("agent", response.raw_response),
+                    LlamaMessage("user", "Command Output:\n" + result),
+                ])
+
+        return agent_response or full_response
+
+    # Receives a single string prompt, and must respond with a string
+    async def prompt(self, content):
+        response = await self._full_prompt(f"User Message", content) 
+        return response
+
+    async def received_message(self, message):
+        chat_context = self.app.get_chat_context(message.chat_id)
+        def sender_name(message):
+            sender = message['sender']
+            is_digit = isinstance(sender, (int, str)) and str(sender).isdigit()
+            return "assistant" if is_digit else sender
+        chat_summary = "\n".join(f"**{sender_name(message)}**: {message['content']}" for message in chat_context.messages)
+
+        response = await self._full_prompt(
+            f"Conversation", chat_summary,
+            system_prologue="You are given the entire context of the conversation, respond with your next message in the conversation by using the '!!response' command."
+        )
+
+        self.app.send_chat_message(
+            chat_id=message.chat_id,
+            sender=self.id,
+            content=response,
+        )
+
+    def add_toolset(self, toolset):
+        self.toolset_map[toolset.identifier] = toolset
+
+    def get_toolset_context(self):
+        if not self.toolset_map:
+            return "(no commands currently enabled)"
+        return "\n".join([toolset.get_namespace_definitions() for toolset in self.toolset_map.values()])
+
+    def parse_response(self, input_string):
+        return parse_response(input_string)
+
+    async def execute_prompt_actions(self, actions):
+        output = ""
+        for action in actions:
+            toolset = self.toolset_map.get(action.toolset_name)
+            if toolset is None:
+                agent_responses.append(f"Error: Bad namespace {action.toolset_name}")
+                continue
+            action_result = await Toolset.execute_action(toolset, action.tool_name, action.args)
+            if type(action_result) != str:
+                action_result = json.dumps(action_result)
+            output += f"[{action.toolset_name}.{action.tool_name}]:{action_result}\n"
+        return output
+
+class LlamaAgent(Agent):
+    identifier = "llama-8b"
+    description = "Uses the llama 3 instruct models. Model sizes: '8b', '30b'"
+    schema_fields = [
+        AgentField('system_prompt', 'string', default="You are a helpful AI assistant."),
+        AgentField('model_size', 'string', default="8b"),
+    ]
+    tools = []
+
+    def __init__(self, app, data):
+        Agent.__init__(self, app, data)
+
+    async def prompt(self, content):
+        messages = [
+            LlamaMessage("system", self.config['system_prompt']),
+            LlamaMessage("user", content)
+        ]
+        response = await replicate_llama_completion(self.app, messages)
+        return response
+
+    async def received_message(self, message):
+        def sender_name(message):
+            sender = message.sender
+            is_digit = isinstance(sender, (int, str)) and str(sender).isdigit()
+            is_self = is_digit and str(sender) == str(self.id)
+            return self.name if is_self else sender
+
+        chat_view = self.app.get_chat_view(message.chat_id)
+        thread_messages = chat_view.get_thread(message.thread_id).messages
+        chat_summary = "\n".join(f"[{sender_name(message)}]: {message.content}" for message in thread_messages)
+        messages = [
+            LlamaMessage("system", f"{self.config['system_prompt']}\nYou are given the context of the conversation, respond with your next message in the conversation."),
+            LlamaMessage("user", f"Recent Messages:\n{chat_summary}")
+        ]
+
+        response = await replicate_llama_completion(self.app, messages)
+        self.app.send_chat_message(
+            chat_id=message.chat_id,
+            sender=self.id,
+            content=response,
+            thread_id=message.thread_id,
+        )
+
+class BasicAgent(LlamaAgent):
+    identifier = "basic"
+    schema_fields = LlamaAgent.schema_fields
+    tools = LlamaAgent.tools
+
+class CoderV1Agent(LlamaAgent):
+    identifier = "coder-v1"
+    description = "Optimized for coding tasks. Based on the llama agent:\n" + LlamaAgent.description
+    schema_fields = [
+        AgentField('system_prompt', 'string', default="You are an expert Software Engineer profficient in multiple programming languages."),
+        AgentField('model_size', 'string', default="30b"),
+    ]
+    # TODO: add gral functions to the tool list
+    tools = []
+    
+    async def gral_message(self, message_history: list, file_map: dict):
+        return await gral.gral_message(self.app, self.config['system_prompt'], message_history, file_map)
+    
+    async def gral_update_file(self, message_history: list, filepath: str, file_content: str):
+        return await gral.gral_update_file(self.app, self.config['system_prompt'], message_history, filepath, file_content)
+
+class ParsedResponse:
+    def __init__(self):
+        self.response = None
+        self.raw_response = None
+        self.actions = []
+
+class ActionResult:
+    def __init__(self):
+        self.user_response = ""
+        self.agent_response = ""
+
+def parse_response(string: str) -> ParsedResponse:
+    """
+    Parses a response string that may contain action commands.
+    Each action must be in its own separate line, marked with '!!' at the start.
+    Format: !![query] namespace.command_name {"json": "args"}
+    """
+    result = ParsedResponse()
+    result.raw_response = string
+    lines = string.split('\n')
+    response_lines = []
+    
+    action_pattern = re.compile(r'^!!([\w\.]+)(?:\s+(\{.*\}))?$')
+    
+    for line in lines:
+        stripped_line = line.strip()
+        if stripped_line.startswith('!!'):
+            # This is an action line, try to parse it
+            match = action_pattern.match(stripped_line)
+            if match:
+                full_command = match.group(1)
+                args_json = match.group(2)
+                
+                # Split the command into toolset and tool names
+                if '.' in full_command:
+                    toolset_name, tool_name = full_command.split('.', 1)
+                else:
+                    toolset_name = ''
+                    tool_name = full_command
+                
+                try:
+                    if args_json:
+                        args = json.loads(args_json)
+                    else:
+                        args = {}
+                    if not isinstance(args, dict):
+                        raise ValueError("Arguments must be a JSON object")
+                    
+                    action = Action(
+                        toolset_name=toolset_name,
+                        tool_name=tool_name,
+                        args=args,
+                        is_query=False
+                    )
+                    result.actions.append(action)
+                except json.JSONDecodeError:
+                    # If JSON parsing fails, treat it as regular text
+                    response_lines.append(line)
+                except ValueError as e:
+                    # If other validation fails, treat as regular text
+                    response_lines.append(line)
+            else:
+                # Doesn't match the pattern, treat as regular text
+                response_lines.append(line)
+        else:
+            # Regular response line
+            response_lines.append(line)
+    
+    result.response = '\n'.join(response_lines)
+    return result

backend/src/agents/conversation.py

@@ -0,0 +1,288 @@
+import re
+import json
+import inspect
+from string import Template
+
+from inference import LlamaMessage, replicate_llama_completion
+from agent import Agent, AgentField
+
+CLASS_DESCRIPTION = """
+Description of the Conversation Agent.
+Here go additional instructions for configuration, like guidelines for the different prompts.
+""".strip()
+
+chat_system_prompt_template = Template("""$role
+The first message contains information about you and a conversation that you're participating in.
+Respond only with your next message in the conversation.
+""".strip())
+
+memory_system_prompt = """
+You are an expert in simulating human-like memory for fictional characters. Given a conversation between two characters and a detailed description of one character's personality, extract the memories that this character is most likely to retain.
+The extracted memories should reflect:
+- What was emotionally significant to them
+- What aligns with their goals, fears, or desires
+- What they found surprising, confusing, or meaningful
+- What they might revisit or ruminate on later
+Do not summarize the conversation. Instead, highlight key moments or phrases that the character would mentally hold onto, framed from their perspective. Avoid information stated by yourself, you don't need to remember what you already know, instead you can identify the other's characters reactions to what your character says if it's notable or salient.
+""".strip()
+
+topic_system_prompt = """
+You analyze a conversation between two characters. Identify the main topic as a short phrase. Then extract up to N subtopics that are likely to be important, related, or may emerge later in the conversation.
+Return your response as a JSON object with this format:
+{"topic": "Main topic", "subtopics": ["First subtopic", "Second subtopic", "..."]}
+""".strip()
+
+memory_compress_system_prompt = """
+You are an assistant that receives a list of memory items and a current conversation; you must identify and return only the most important and relevant memories that help understand the conversation or move it forward, removing duplicates or very similar entries.
+
+# Instructions
+- Think carefully about what's important for the character to remember in this context. Write down your reasoning and keep it shorter than 100 words, shorter than 50 even.
+- If conflicting information is presented or context is ambiguous, state it in your reasoning.
+- Take special care to state and address the most recent topic touched in the conversation.
+- Based on your own analysis, generate the required JSON object. Stop your answer after this.
+
+# Format
+Write a justification paragraph, and then output a JSON list of strings with each selected memory, for example:
+["First memory","Second memory"]
+""".strip()
+
+chat_prompt_template = Template("""
+# Recent Memories
+$memories
+
+# Recent Conversation
+$conversation
+
+# Last Exchange
+$last_exchange
+""".strip())
+
+topic_prompt_template = Template("""
+# Parameters
+Subtopic count: $subtopic_count
+
+# Conversation History
+$conversation
+""".strip())
+
+memory_prompt_template = """
+# Your Character
+Name: [name]
+Personality and Role: [personality]
+
+# Output Format
+Respond only with a JSON array, where each object represents a memory, and each memory object has the following fields:
+- memory: Short description of the fact that the character will remember, phrased objectively and in third person.
+- importance: Salience score of the memory, where 5 is for facts that are definitely important for either of the characters or that may be referenced frequently, and 1 indicates facts or events of low importance or short term relevance.
+If there are no relevant memories to extract, it is ok to respond simply with an empty list.
+
+# Conversation History
+[conversation]
+""".strip()
+
+memory_compress_prompt_template = Template("""
+# Memories
+Following are all the known memories. Each memory is in a separate line.
+
+$memories
+
+# Conversaton
+$conversation
+""".strip())
+
+# Used to keep track of the last chat thread it participated in
+class ThreadContext:
+    def __init__(self, data):
+        if data:
+            self.current_thread_id = data.get('current_thread_id')
+        else:
+            self.current_thread_id = None
+
+class ConversationAgent(Agent):
+    identifier = "conversation"
+    description = CLASS_DESCRIPTION
+    schema_fields = [
+        AgentField('system_prompt', 'string', default="You are a friendly AI having a casual conversation with the user."),
+    ]
+    tools = []
+
+    def __init__(self, app, data):
+        Agent.__init__(self, app, data)
+
+        # Constant system prompt for developing purposes 
+        self.system_prompt = self.config['system_prompt']
+
+    async def prompt(self, content):
+        # TODO: Advanced features
+        messages = [
+            LlamaMessage("system", self.system_prompt),
+            LlamaMessage("user", content)
+        ]
+        response = await replicate_llama_completion(self.app, messages)
+        return response
+
+    async def received_message(self, message):
+
+        # TODO: before retrieving the chat context, acquire a lock for it
+
+        saved_context = ThreadContext(self.app.get_agent_resource(self.id, 'last-thread-context', message.chat_id))
+        chat_view = self.app.get_chat_view(message.chat_id)
+
+        if message.thread_id != saved_context.current_thread_id:
+            # the incoming message is not from the same thread that the agent was last aware of
+            # store details about the old thread in the agent's long term memory
+            old_thread = chat_view.get_thread(saved_context.current_thread_id)
+            await self.process_thread(old_thread)
+            
+            new_context_data = {'current_thread_id': message.thread_id}
+            self.app.save_agent_resource(self.id, 'last-thread-context', message.chat_id, new_context_data)
+
+        current_thread = chat_view.get_thread(message.thread_id)
+        memories = await self.retrieve_memories_for_thread(current_thread)
+
+        chat_summary = self.serialize_thread_history(current_thread, ignore_message_ids=[message.id])
+        messages = [
+            LlamaMessage("system", chat_system_prompt_template.substitute(role=self.system_prompt)),
+            LlamaMessage("user", chat_prompt_template.substitute(
+                conversation=chat_summary,
+                memories="\n".join(memories),
+                # TODO: the last agent message should be added here too, and any unaddressed message in between
+                last_exchange=self.serialize_message_list([message])
+            )),
+        ]
+
+        response = await replicate_llama_completion(self.app, messages)
+        self.app.send_chat_message(
+            chat_id=message.chat_id,
+            sender=self.id,
+            content=response,
+            thread_id=current_thread.id,
+        )
+
+        # TODO: drop the chat context lock
+
+    # Encode a thread's messages as a string to pass to the agent
+    def serialize_thread_history(self, thread, ignore_message_ids=[]):
+        return self.serialize_message_list([m for m in thread.messages if m.id not in ignore_message_ids])
+
+    def serialize_message_list(self, messages):
+        def sender_name(message):
+            sender = message.sender
+            is_digit = isinstance(sender, (int, str)) and str(sender).isdigit()
+            is_self = is_digit and str(sender) == str(self.id)
+            return self.name if is_self else sender
+        
+        return "\n".join(f"[{sender_name(message)}]: {message.content}" for message in messages)
+    
+    def get_memory_collection(self):
+        collection_name = f"agent_{self.id}_memory"
+        return collection_name
+    
+    async def retrieve_memories_for_thread(self, thread):
+        print("identify topic")
+        chat_history = self.serialize_thread_history(thread)
+
+        topic_response = await replicate_llama_completion(self.app, [
+            LlamaMessage("system", topic_system_prompt),
+            LlamaMessage("user", topic_prompt_template.substitute(
+                conversation=chat_history,
+                subtopic_count=3
+            ))
+        ])
+
+        match = re.search(r'\{.*?\}', topic_response, re.DOTALL)
+        if match:
+            json_string = match.group(0)
+            try:
+                topic_data = json.loads(json_string)
+            except json.JSONDecodeError:
+                print("Bad JSON data in response...")
+                print(topic_response)
+                return []
+        else:
+            print("Topic format not found in response...")
+            print(topic_response)
+            return []
+
+        print("retrieve memories")
+        client = self.app.qdrant_client
+        collection = self.get_memory_collection()
+
+        if not client.collection_exists(collection):
+            return []
+        
+        query_result = client.query(
+            collection,
+            query_text=topic_data['topic'],
+            limit=20
+        )
+        for subtopic in topic_data['subtopics']:
+            query_result += client.query(
+                collection,
+                query_text=subtopic,
+                limit=6
+            )        
+        memories = [it.document for it in query_result]
+
+        print("compress memories")
+        compress_response = await replicate_llama_completion(self.app, [
+            LlamaMessage("system", memory_compress_system_prompt),
+            LlamaMessage("user", memory_compress_prompt_template.substitute(
+                conversation=chat_history,
+                memories="\n".join(memories)
+            ))
+        ])
+
+        match = re.search(r'\[.*?\]', compress_response, re.DOTALL)
+        if match:
+            json_string = match.group(0)
+            try:
+                memories = json.loads(json_string)
+            except json.JSONDecodeError:
+                print("Bad JSON data in response...")
+                print(compress_response)
+                return []
+        else:
+            print("Compress format not found in response...")
+            print(compress_response)
+            return []
+        
+        return memories
+    
+    # Compute thread summnary and concrete memories, then save the progress done
+    async def process_thread(self, thread):
+        # TODO: extract memories from the thread messages, then store them in the agent's vector memory db
+        # TODO: somewhere here it must be known if the thread was seen before: the same memory should not be stored multiple times
+        print(f"PROCESSING THREAD: {thread.id}")
+
+        chat_history = self.serialize_thread_history(thread)
+        prompt = memory_prompt_template.replace("[name]", self.name).replace("[personality]", self.system_prompt).replace("[conversation]", chat_history)
+
+        messages = [
+            LlamaMessage("system", memory_system_prompt),
+            LlamaMessage("user", prompt)
+        ]
+        response = await replicate_llama_completion(self.app, messages)
+
+        match = re.search(r'\[.*?\]', response, re.DOTALL)
+        if match:
+            json_string = match.group(0)
+            try:
+                memories = json.loads(json_string)
+            except json.JSONDecodeError:
+                print("Bad JSON data in response...")
+                print(response)
+                return
+        else:
+            print("Memory format not found in response...")
+            print(response)
+            return
+
+        collection = self.get_memory_collection()
+        if memories:
+            self.app.qdrant_client.add(
+                collection,
+                documents=[it['memory'] for it in memories],
+                metadata=[{'importance': it['importance']} for it in memories],
+            )
+            print(f"Added {len(memories)} memories for agent {self.id}")

backend/src/app.py

@@ -0,0 +1,300 @@
+from datetime import datetime
+import asyncio
+import json
+import sys
+import os
+
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.expression import select
+from quart import Quart, Response, request, send_from_directory, jsonify, websocket
+from qdrant_client import QdrantClient
+import replicate
+
+import database as db
+import agent as agent_mod
+from server import start_server
+from chat import ChatView, Message, MESSAGE_PAGE_SIZE
+
+from agents.conversation import ConversationAgent
+from app_mixins import ChatMixin
+
+class App(ChatMixin):
+    def __init__(self, data_directory, replicate_api_token=None):
+        self.data_directory = data_directory
+        self.user_connections = {}
+        self.remote_connections = {}
+        self.streams = {}
+
+        # Initialize Clients
+        self.sql_engine = db.create_sql_engine(self.data_directory)
+        self.qdrant_client = QdrantClient(path=f"{self.data_directory}/qdrant_data")
+
+        if replicate_api_token:
+            self.replicate_client = replicate.Client(api_token=replicate_api_token)
+        else:
+            raise Exception("Horde only works with a replicate connection. Please provide an API Token.")
+
+        # `Agent` is the kind 'basic', will be class `BasicAgent` in the future
+        self.agent_classes = [
+            agent_mod.BasicAgent,
+            ConversationAgent,
+            agent_mod.LlamaAgent,
+            agent_mod.CoderV1Agent,
+            agent_mod.ToolUsingAgent,
+            agent_mod.DebugAgent,
+        ]
+
+    def sql_session(self):
+        return Session(self.sql_engine)
+
+    def add_user_connection(self, user_id, channel):
+        if user_id not in self.user_connections:
+            self.user_connections[user_id] = []
+        if channel not in self.user_connections[user_id]:
+            self.user_connections[user_id].append(channel)
+
+    def remove_user_connection(self, user_id, channel):
+        if user_id in self.user_connections:
+            if channel in self.user_connections[user_id]:
+                self.user_connections[user_id].remove(channel)
+                if not self.user_connections[user_id]:  # Check if the list is empty
+                    del self.user_connections[user_id]
+
+    def add_remote_connection(self, user_id, remotename, channel):
+        if user_id not in self.remote_connections:
+            self.remote_connections[user_id] = {}
+        user_connections = self.remote_connections[user_id]
+
+        if channel_list not in user_connections[remotename]:
+            user_connections[remotename] = []
+        user_connections[remotename].append(channel)
+
+    def create_agent(self, user_id, name, agentname, description, picture, kind, data, main_chat_id=None):
+        with self.sql_session() as session:
+            if main_chat_id is None:
+                chat = db.Chat()
+                chat.user_id = user_id
+                chat.name = name
+
+                session.add(chat)
+                session.commit()
+                main_chat_id = chat.id
+
+            agent = db.Agent()
+            agent.user_id = user_id
+            agent.name = name
+            agent.agentname = agentname
+            agent.main_chat_id = main_chat_id
+            agent.description = description
+            agent.picture = picture
+            agent.kind = kind
+            agent.data = data
+
+            session.add(agent)
+            session.commit()
+            print(f"Created agent #{agent.id}: {agent.name}")
+
+            membership = db.Membership()
+            membership.chat_id = main_chat_id
+            membership.agent_id = agent.id
+            session.add(membership)
+            session.commit()
+
+            return agent.id
+
+    # Updates the agent's basic fields, and applies field updates in its data
+    def update_agent(self, agent_id, name=None, description=None, picture=None, agentname=None, data=None):
+        with self.sql_session() as session:
+            agent = session.execute(
+                select(db.Agent).where(db.Agent.id == agent_id)
+            ).scalars().first()
+
+            if not agent:
+                raise Exception(f"Agent not found with id {agent_id}")
+            
+            summary = []
+
+            if name != None:
+                agent.name = name
+                summary.append('name')
+            if description != None:
+                agent.description = description
+                summary.append('description')
+            if picture != None:
+                agent.picture = picture
+                summary.append('picture')
+            if agentname != None:
+                agent.agentname = agentname
+                summary.append('agentname')
+
+            if data:
+                # Assign a copy to trigger a value change
+                agent.data = agent.data.copy() if agent.data else {}
+                agent.data.update(data)
+                summary.extend([f"data.{name}" for name in data.keys()])
+
+            session.commit()
+            print(f"Updated agent #{agent.id} fields: {summary}")
+
+    def get_agents(self, user_id):
+        with self.sql_session() as session:
+            agents = session.execute(select(db.Agent).where(db.Agent.user_id == user_id)).scalars().all()
+            return [{
+                "id": agent.id,
+                "name": agent.name,
+                "agentname": agent.agentname,
+                "description": agent.description,
+                "picture": agent.picture,
+                "kind": agent.kind,
+            } for agent in agents]
+
+    def get_agent_kinds(self):
+        return [kind.identifier for kind in self.agent_classes]
+
+    def get_agent_kind_info(self, kind):
+        agent_class = next((it for it in self.agent_classes if it.identifier == kind), None)
+        if agent_class is None:
+            raise Exception(f"invalid agent kind: {kind}")
+        return {
+            "name": agent_class.identifier,
+            "description": getattr(agent_class, 'description', None),
+        }
+
+    def verify_user_credentials(self, username, password):
+        with self.sql_session() as session:
+            user = session.execute(
+                select(db.User).where(db.User.username == username)
+            ).scalars().first()
+
+            if user and user.password == password:
+                return user.id
+            return None
+
+    def create_new_user(self, username, email, password):
+        with self.sql_session() as session:
+            result = session.execute(
+                select(db.User.id).where(db.User.email == email or db.User.username == username)
+            ).first()
+
+            if result is None:
+                user = db.User()
+                user.email = email
+                user.username = username
+                user.password = password
+
+                session.add(user)
+                session.commit()
+                
+                user_id = user.id
+                print(f"Created user #{user_id}: {username} ({email})")
+            else: 
+                print(f"Signup error: '{email}' or '{username}' are already used")
+        return user_id
+
+    def get_agent_control(self, agent_id):
+        with self.sql_session() as session:
+            agent = session.execute(
+                select(db.Agent).where(db.Agent.id == agent_id)
+            ).scalars().first()
+
+            if not agent:
+                raise Exception(f"Agent not found with id {agent_id}")
+            
+            agent_class = next((kind for kind in self.agent_classes if kind.identifier == agent.kind), None)
+            if agent_class is None:
+                raise Exception(f"invalid agent kind: {agent.kind}")
+        
+            control = agent_class(app, agent.data)
+            control._set_basic_info(agent_id, agent.name)
+            return control
+
+    def get_agent_config_fields(self, agent_id):
+        with self.sql_session() as session:
+            agent = session.execute(
+                select(db.Agent).where(db.Agent.id == agent_id)
+            ).scalars().first()
+            if not agent:
+                raise Exception(f"Agent not found with id {agent_id}")
+
+            agent_class = next((it for it in self.agent_classes if it.identifier == agent.kind), None)
+            if not agent_class:
+                raise Exception(f"Invalid agent kind: {agent.kind}")
+
+            data = agent.data or {}
+            fields = [{
+                'name': field.name,
+                'type': field.type,
+                'value': data.get(field.name, field.default)
+                } for field in agent_class.schema_fields]
+            
+            return fields
+        return {}
+
+    async def send_agent_prompt(self, agent_id, message):
+        control = self.get_agent_control(agent_id)
+        response = await control.prompt(message)
+        return response
+
+    async def call_agent_action(self, agent_id, action, args):
+        control = self.get_agent_control(agent_id)
+        response = await control.execute_action(action, args)
+        return response
+
+    def generate_default_user_data(self, user_id):
+        chat_id = self.create_chat(
+            user_id=user_id,
+            name="Initial Chat",
+            description="First user chat with Intro.",
+            picture=None,
+            members=[],
+        )
+        agent_id = self.create_agent(
+            user_id=user_id,
+            name="Intro",
+            agentname="intro",
+            main_chat_id=chat_id,
+            description="Introduces you to the application.",
+            picture=None,
+            kind="conversation", # Should be 'intro'
+            data={}
+        )
+    
+    def get_agent_resource(self, agent_id, kind, key):
+        with self.sql_session() as session:
+            stored_value = session.execute(select(db.AgentResource.value).where(
+                db.AgentResource.agent_id == agent_id and db.AgentResource.kind == kind and db.AgentResource.key == key
+            )).scalar()
+            return stored_value
+
+    def save_agent_resource(self, agent_id, kind, key, value):
+        with self.sql_session() as session:
+            existing_resource = session.execute(select(db.AgentResource).where(
+                db.AgentResource.agent_id == agent_id and db.AgentResource.kind == kind and db.AgentResource.key == key
+            )).scalars().first()
+
+            if existing_resource:
+                existing_resource.value = value
+            else:
+                new_resource = db.AgentResource()
+                new_resource.agent_id = agent_id
+                new_resource.kind = kind
+                new_resource.key = key
+                new_resource.value = value
+
+                session.add(new_resource)
+            session.commit()
+            print(f"Saved agent resource: {agent_id} - {kind} - {key} = {value}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        data_directory = sys.argv[1]
+    else:
+        print("No directory specified. Using current directory as default.")
+        data_directory = "."
+
+    app = App(
+        data_directory=data_directory,
+        replicate_api_token=os.getenv('REPLICATE_API_TOKEN'),
+    )
+    start_server(app)

backend/src/app_mixins/__init__.py

@@ -0,0 +1 @@
+from app_mixins.chat import ChatMixin

backend/src/app_mixins/chat.py

@@ -0,0 +1,177 @@
+import asyncio
+from datetime import datetime
+from sqlalchemy.sql.expression import select
+
+import database as db
+from chat import ChatView, Message, MESSAGE_PAGE_SIZE
+
+class ChatMixin:
+    def create_chat(self, user_id, name, description, picture, members=[]):
+        with self.sql_session() as session:
+            chat = db.Chat()
+            chat.user_id = user_id
+            chat.name = name
+            chat.description = description
+            chat.picture = picture
+
+            session.add(chat)
+            session.commit()
+            print(f"Created chat #{chat.id}: {chat.name}")
+
+            chat_id = chat.id
+        
+        for agent_id in members:
+            self.add_agent_to_chat(chat_id, agent_id)
+
+        return chat_id
+
+    def get_chats(self, user_id):
+        with self.sql_session() as session:
+            chats = session.execute(select(db.Chat).where(db.Chat.user_id == user_id)).scalars().all()
+            return [{
+                "id": chat.id,
+                "name": chat.name,
+                "description": chat.description,
+                "picture": chat.picture
+            } for chat in chats]
+
+    def delete_chat(self, chat_id):
+        with self.sql_session() as session:
+            session.execute(db.Membership.__table__.delete().where(db.Membership.chat_id == chat_id))
+            session.execute(db.Chat.__table__.delete().where(db.Chat.id == chat_id))
+            # TODO: Delete messages from the chat
+            session.commit()
+            print(f"Deleted chat #{chat_id} and its memberships.")
+
+    def get_chat_messages(self, chat_id, earliest_index=None):
+        with self.sql_session() as session:
+            condition = (db.Message.chat_id == chat_id)
+            if earliest_index is not None:
+                condition = condition and db.Message.id < earliest_index
+
+            messages = session.execute(
+                select(db.Message).where(condition).order_by(db.Message.time.desc())
+            ).scalars().all()
+
+            messages.reverse()
+
+            return [{
+                "id": message.id,
+                "time": message.time,
+                "sender": message.sender,
+                "kind": message.kind,
+                "content": message.content,
+                "threadId": message.thread_id,
+            } for message in messages]
+    
+    def create_chat_thread(self, chat_id, name=None):
+        with self.sql_session() as session:
+            thread = db.Thread()
+            thread.chat_id = chat_id
+            thread.name = name
+
+            session.add(thread)
+            session.commit()
+            print(f"Created thread #{thread.id}: {thread.name}")
+
+            return thread.id
+
+    def add_agent_to_chat(self, chat_id, agent_id):
+        with self.sql_session() as session:
+            membership = db.Membership()
+            membership.chat_id = chat_id
+            membership.agent_id = agent_id
+            
+            session.add(membership)
+            session.commit()
+            print(f"Added agent #{agent_id} to chat #{chat_id}")
+
+    # If an integer is passed in sender, it is interpreted as an agent id
+    def send_chat_message(self, chat_id, content, sender='user', thread_id=None, kind=None, subkind=None, format=None):
+        with self.sql_session() as session:
+            message = db.Message()
+            message.chat_id = chat_id
+            message.time = int(datetime.now().timestamp())
+            message.last_edit = message.time
+            message.sender = sender
+            message.kind = kind or 'message'
+            message.subkind = subkind
+            message.content = content
+            message.thread_id = thread_id
+
+            session.add(message)
+            session.commit()
+            session.refresh(message)
+
+        # Notify chat owner
+        chat_owner = session.execute(select(db.Chat.user_id).where(db.Chat.id == chat_id)).scalar()
+        self.notify_message_to_user(chat_owner, message)
+
+        # Notify participant agents
+        agent_ids = session.execute(select(db.Membership.agent_id).where(db.Membership.chat_id == chat_id)).scalars().all()
+        for agent_id in agent_ids:
+            if agent_id != sender:
+                self.notify_message_to_agent(agent_id, message)
+
+        return {
+            'id': message.id,
+            'chatId': message.chat_id,
+            'time': message.time,
+            'sender': message.sender,
+            'kind': message.kind,
+            'content': message.content,
+        }
+
+    # Updates a single chat message.
+    # Most fields are optional. If they're not passed a value, they wont be updated.
+    # When kind is set, subkind is always set too.
+    def update_chat_message(self, message_id, content=None, kind=None, subkind=None, format=None):
+        with self.sql_session() as session:
+            message = session.execute(
+                select(db.Message).where(db.Message.id == message_id)
+            ).scalars().first()
+
+            if not message:
+                raise Exception(f"Message not found with id {message_id}")
+
+            summary = []
+
+            if content != None:
+                message.content = content
+                summary.append('content')
+            if kind != None:
+                message.kind = kind
+                message.subkind = subkind
+                summary.append('kind')
+                summary.append('subkind')
+            if format != None:
+                message.format = format
+                summary.append('format')
+            #message.last_edit = int(datetime.now().timestamp())
+
+            session.commit()
+            print(f"Updated message #{message.id} fields: {summary}")
+
+    def get_chat_view(self, chat_id):
+        return ChatView(self, chat_id)
+
+    def notify_message_to_user(self, user_id, message):
+        # could be moved to user mixin
+        if user_id in self.user_connections:
+            user_message = {
+                'action': 'chat-message',
+                'id': message.id,
+                'chatId': message.chat_id,
+                'time': message.time,
+                'sender': message.sender,
+                'kind': message.kind,
+                'content': message.content,
+                'threadId': message.thread_id,
+            }
+            for channel in self.user_connections[user_id]:
+                asyncio.create_task(channel.put(user_message))
+
+    def notify_message_to_agent(self, agent_id, message):
+        # could be moved to agent mixin
+        control = self.get_agent_control(agent_id)
+        asyncio.create_task(control.received_message(message))

backend/src/chat.py

@@ -0,0 +1,143 @@
+
+from sqlalchemy.orm import Session
+from sqlalchemy.sql.expression import select
+
+import database as db
+
+MESSAGE_PAGE_SIZE = 256
+
+class Message:
+    def __init__(self, id, content, sender, time=None, kind='message', subkind=None):
+        self.id = id
+        self.content = content
+        self.sender = sender
+        self.time = time
+        self.kind = kind
+        self.subkind = subkind
+    
+    def from_db_record(record):
+        return Message(record.id, record.content, record.sender, record.time)
+    
+    def __str__(self):
+        return f"Message(#{self.id} ({self.time if self.time != None else 'no-time'}) {self.sender}: {self.content.replace("\n", "\\n")})"
+
+class Thread:
+    def __init__(self, id, chat_id, name=None, description=None):
+        self.id = id
+        self.chat_id = chat_id
+        self.name = name
+        self.description = description
+
+        self.messages = []
+        self.notes = []
+        self.memories = []
+
+class ChatView:
+    def __init__(self, app, chat_id):
+        self.app = app
+        self.chat_id = chat_id
+
+        self.name = None
+        self.description = None
+
+        # Thread holding messages without a set thread id
+        self.default_thread = Thread(None, chat_id)
+        # Most recent thread objects, earliest to latest
+        self.threads = []
+        # Map of threads by their id
+        self.thread_map = {}
+        # Long term summary of this chat
+        self.summary = None
+        # Separate list for note messages from any thread
+        # Is it better to exclude them or include them from threads? atm they're included
+        self.notes = []
+
+        # Id of the thread currently in focus, by default the last thread appearing in the conversation
+        self.current_thread_id = None
+        
+        self._earliest_message_id = None
+
+        self._load_basic_data()
+        self._load_next_messages(MESSAGE_PAGE_SIZE)
+    
+    def _load_basic_data(self):
+        with self.app.sql_session() as session:
+            record = session.execute(
+                select(db.Chat).where(db.Chat.id == self.chat_id)
+            ).scalars().first()
+
+            if not record:
+                raise Exception(f"Chat not found with id {self.chat_id}")
+            
+            self.name = record.name
+            self.description = record.description
+
+    def _load_next_messages(self, load_count):
+        with self.app.sql_session() as session:
+            records_rev = session.execute(
+                select(db.Message).where(
+                    db.Message.chat_id == self.chat_id,
+                    self._earliest_message_id == None or db.Message.id <= self._earliest_message_id
+                ).order_by(db.Message.time.desc())
+            ).scalars().fetchmany(load_count)
+
+            # Update the earliest id for the next load operation
+            if records_rev:
+                self._earliest_message_id = records_rev[0].id
+                if not self.threads:
+                    # current thread is the latest known
+                    self.current_thread_id = records_rev[-1].thread_id
+
+            # Go over messages from newest to oldest
+            for record in records_rev:
+                if record.thread_id == None:
+                    thread = self.default_thread
+                else:
+                    thread = self.thread_map.get(record.thread_id)
+                    if not thread:
+                        thread = Thread(record.thread_id, self.chat_id)
+                        self.thread_map[thread.id] = thread
+                        self.threads.insert(0, thread)
+                # This is the oldest message so far, add it to the start of the chat
+                message = Message(record.id, record.content, record.sender, record.time, record.kind, record.subkind)
+
+                if message.kind == 'note':
+                    thread.notes.insert(0, message)
+                elif message.kind == 'memory':
+                    thread.memories.insert(0, message)
+                else:
+                    thread.messages.insert(0, message)
+    
+    def get_thread(self, thread_id):
+        if thread_id == None:
+            return self.default_thread
+        return self.thread_map.get(thread_id)
+    
+    def get_current_thread(self):
+        # TODO: if !thread
+        return self.get_thread(self.current_thread_id)
+    
+    def debug_string(self):
+        result_lines = []
+        def _print(line):
+            result_lines.append(line)
+
+        def print_thread(thread, title=None):
+            if title:
+                _print(title + ":")
+            elif thread.name:
+                _print(thread.name + ":")
+            else:
+                _print(f"Thread {thread.id}:")
+            for message in thread.messages:
+                _print("  " + str(message))
+        
+        print_thread(self.default_thread, "Default Thread")
+        for thread in self.threads:
+            print_thread(thread)
+
+        _print("Current thread id: " + str(self.current_thread_id))
+
+        return "\n".join(result_lines)
+
+

backend/src/database.py

@@ -0,0 +1,131 @@
+from typing import Optional
+from sqlalchemy import create_engine, Column, Integer, JSON, ForeignKey
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from sqlalchemy.sql.expression import select
+
+class Base(DeclarativeBase):
+    pass
+
+def create_sql_engine (data_directory):
+    sql_engine = create_engine(f"sqlite:///{data_directory}/horde.db")
+    Base.metadata.create_all(sql_engine)
+    return sql_engine
+
+class User(Base):
+    __modelname__ = "user"
+    __tablename__ = 'users'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+
+    email: Mapped[str]
+    username: Mapped[str]
+    # Stores the plain password. This must be changed eventually to store a salted hash instead.
+    password: Mapped[str]
+
+class Agent(Base):
+    __modelname__ = "agent"
+    __tablename__ = 'agents'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
+
+    agentname: Mapped[Optional[str]]
+    visibility: Mapped[int] = mapped_column(default=0)
+    # Comma separated list of permission symbols
+    permissions: Mapped[str] = mapped_column(default="")
+    main_chat_id: Mapped[int] = mapped_column(ForeignKey("chats.id"))
+
+    name: Mapped[str]
+    description: Mapped[Optional[str]]
+    picture: Mapped[Optional[str]]
+    ordering: Mapped[Optional[int]]
+
+    # Agents can be any of a set of agent classes
+    kind: Mapped[str]
+    # Stores the agent data specific to its class
+    data = mapped_column(JSON, default={})
+
+class Chat(Base):
+    __modelname__ = 'chat'
+    __tablename__ = 'chats'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    user_id: Mapped[int] = mapped_column(ForeignKey("users.id"))
+
+    name: Mapped[str]
+    description: Mapped[Optional[str]]
+    picture: Mapped[Optional[str]]
+    ordering: Mapped[Optional[int]]
+
+    # For indexing purposes, some information is taken from the last chat message.
+    # Preview is the (possibly cropped) content, time is the timestamp of the message.
+    last_message_preview: Mapped[Optional[str]]
+    last_message_time: Mapped[Optional[int]]
+
+# Stores membership of agents in chats. If a record exists linking an agent and a chat,
+# it means the agent participates in that chat.
+class Membership(Base):
+    __modelname__ = 'membership'
+    __tablename__ = 'memberships'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"))
+    chat_id: Mapped[int] = mapped_column(ForeignKey("chats.id"))
+
+class Message(Base):
+    __modelname__ = 'message'
+    __tablename__ = 'messages'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    chat_id: Mapped[int] = mapped_column(ForeignKey("chats.id"))
+    thread_id: Mapped[Optional[int]] = mapped_column(ForeignKey("threads.id"))
+
+    # Identifier of the sender: user / system / user-$ID / agent-$ID
+    sender: Mapped[str]
+    # Timestamp when this message was sent
+    time: Mapped[int]
+    # Timestamp of this message's last edit
+    #last_edit: Mapped[int]
+
+    # Predefined set of message types:
+    # message, log, memory, file, note
+    kind: Mapped[str]
+    # Each kind has a set of subkinds, and a default for empty values
+    # message: (empty)
+    # log: debug, info
+    # memory: (depends on the agent)
+    # file: (empty)
+    # note: public, private, pinned
+    subkind: Mapped[Optional[str]]
+    # For the kinds message, log and note, a format can be specified,
+    # empty defaults to 'text' and can also be 'markdown', 'csv'
+    format: Mapped[Optional[str]]
+    content: Mapped[str]
+
+# Represents a temporary topic or stage in a conversation
+class Thread(Base):
+    __modelname__ = 'thread'
+    __tablename__ = 'threads'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    chat_id: Mapped[int] = mapped_column(ForeignKey("chats.id"))
+
+    # Optional name and descriptions of the thread
+    name: Mapped[Optional[str]]
+    description: Mapped[Optional[str]]
+
+# A persistent object or value associated with an agent
+class AgentResource(Base):
+    __modelname__ = 'agent_resource'
+    __tablename__ = 'agent_resources'
+
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    agent_id: Mapped[int] = mapped_column(ForeignKey("agents.id"))
+
+    # Kind is the class of the resource
+    # Key is the specific identifier within the class
+    # Multiple resources can share key
+    kind: Mapped[str]
+    key: Mapped[str]
+    value = mapped_column(JSON, default=None)
+

backend/src/gral.py

@@ -0,0 +1,76 @@
+from inference import LlamaMessage, replicate_llama_completion, Models
+import re
+
+message_format_section = """
+When suggesting code to the user, keep the code as small snippets.
+You have the capability to update files in your current environment with file tags before your code suggestions.
+If the update is meant to be applied on a file, include a file tag line immediately before the code block like this: "[filename.txt]
+```
+updated segment
+```"
+Omit the file link when the block is not related to a known file,
+Code blocks must be under 30 lines. Your responses MUST NOT contain horizontal rules.
+When suggesting bash code meant to be directly executed, use the same syntax as above but marking the code block with "[eval]" instead.
+"""
+
+
+async def gral_message(app, system_prompt, message_history: list, file_map: dict):
+    messages = [
+        LlamaMessage("system", "\n".join([
+            system_prompt,
+            message_format_section,
+        ])),
+    ]
+
+    file_parts = []
+    for filename, content in file_map.items():
+        file_parts.append(f"\n---\n[{filename}]\n\n{content}\n")
+    
+    if file_parts:
+        file_lines = [
+            "This message contains all files relevant to the prompt: each file in this message is separated with horizontal rules \"---\", followed by the file name in square brackets, followed by the file content.",
+            "Only this message contains horizontal rules, your responses never has them.",
+            "Conversation start in the message following this one."
+        ]
+        file_lines.extend(file_parts)
+        messages.append(
+            LlamaMessage("user", "\n".join(file_lines))
+        )
+
+    messages.extend([LlamaMessage(msg['role'], msg['content']) for msg in message_history])
+
+    response = await replicate_llama_completion(app, messages, Models.llama_70b)
+    return response
+
+async def gral_update_file(app, system_prompt, message_history: list, filepath: str, file_content: str):
+    messages = [
+        LlamaMessage("system", "\n".join([
+            system_prompt,
+            "Respond with the raw contents of the file updated as concluded in the conversation.",
+        ])),
+    ]
+
+    # History Message
+    message_lines = [f"This is the history of the conversation that led to the update of the file {filepath}."]
+    message_lines.extend([f"{entry['role']}: {entry['content']}" for entry in message_history if entry.get('content')])
+    messages.append(LlamaMessage("user", "\n".join(message_lines)))
+
+    # Main Instruction Message
+    message_lines = [
+        f"Write the updated contents of the file: {filepath}, as concluded in the conversation above.",
+        f"The current content of the file {filepath} is pasted after the horizontal rule below."
+        "\n---\n",
+        file_content,
+    ]
+    messages.append(LlamaMessage("user", "\n".join(message_lines)))
+
+    new_content = await replicate_llama_completion(app, messages)
+
+    # Remove backtick fences
+    md_code_pattern = re.compile(r"^\s*```[\w\s]*\n([\s\S]*)\n```\s*$", re.MULTILINE)
+    md_match = re.search(md_code_pattern, new_content)
+    if md_match:
+        print("agent used backtick fences")
+        new_content = md_match.group(1)
+
+    return new_content

backend/src/inference.py

@@ -0,0 +1,60 @@
+import threading
+from stream import Stream
+
+LLAMA3_MODEL = "meta/meta-llama-3-8b-instruct"
+VALID_ROLES = ['user', 'agent', 'system']
+
+class LlamaMessage:
+    def __init__(self, role, content):
+        if role not in VALID_ROLES:
+            raise Exception("message role must be one of: " + ",".join(VALID_ROLES))
+        self.role = role
+        self.content = content
+
+
+class Models:
+    llama_8b = "meta/meta-llama-3-8b-instruct"
+    llama_70b = "meta/meta-llama-3-70b-instruct"
+
+async def replicate_llama_completion(app, messages, model=None):
+    if model is None:
+        model = Models.llama_8b
+
+    # Llama instruct
+    full_prompt = "<|begin_of_text|>"
+    for message in messages:
+        header_name = message.role if message.role != "agent" else "assistant"
+        full_prompt += f"<|start_header_id|>{header_name}<|end_header_id|>\n\n{message.content}<|eot_id|>"
+    full_prompt += f"<|start_header_id|>assistant<|end_header_id|>"
+    
+    content_stream = Stream(app)
+    thread = threading.Thread(
+        target=run_replicate_inference_loop,
+        kwargs={
+            'replicate_client': app.replicate_client,
+            'model': model,
+            'prompt': full_prompt,
+            'content_stream': content_stream,
+        }
+    )
+    thread.start()
+    response = await content_stream.content_wait()
+    return response.lstrip()
+
+
+def run_replicate_inference_loop(replicate_client, model, prompt, content_stream):
+    print("START INFERENCE")
+    completion = replicate_client.run(
+        model,
+        input={
+            "prompt": prompt,
+            "temperature": 0.75,
+            "max_new_tokens": 500,
+            "system_prompt": "" # We handle system prompt in the formatted prompt
+        }
+    )
+    for event in completion:
+        if event:
+            content_stream.feed(str(event))
+    print("END INFERENCE")
+    content_stream.close()

backend/src/server.py

@@ -0,0 +1,240 @@
+from quart import Quart, Response, request, send_from_directory, websocket
+from quart_auth import (
+    QuartAuth, current_user,
+    AuthUser, login_user, logout_user
+)
+import json
+import asyncio
+import nest_asyncio
+
+# Vue frontend
+# relative to the module file???? wtf
+frontend_path = '../../frontend/dist'
+
+def start_server(app):
+    server = Quart(__name__, static_folder=frontend_path)
+    server.secret_key = 'horde-secret'
+    auth_manager = QuartAuth(server, mode="brearer")
+
+    @server.after_request
+    def after_request(response):
+        response.headers.add('Access-Control-Allow-Origin', '*')
+        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
+        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+        return response
+
+    @server.route('/')
+    async def index():
+        return await send_from_directory(server.static_folder, 'index.html')
+
+    # These paths must match the routes in the frontend SPA
+    # file: frontend/src/routes/index.js
+    @server.route('/chat')
+    @server.route('/agent')
+    @server.route('/chat/<path:subpath>')
+    @server.route('/agent/<path:subpath>')
+    async def serve_spa_routes(subpath=None):
+        # Ignore subpath and render index, Vue router will take care of everything
+        return await send_from_directory(server.static_folder, 'index.html')
+
+    @server.route('/<path:path>')
+    async def serve_static(path):
+        return await send_from_directory(server.static_folder, path)
+
+    @server.route('/user/login', methods=['POST'])
+    async def login():
+        data = await request.get_json()
+
+        if "username" not in data or "password" not in data:
+            return "invalid login arguments", 400
+
+        user_id = app.verify_user_credentials(data['username'], data['password'])
+        if user_id == None:
+            return "bad login credentials", 401
+
+        token = auth_manager.dump_token(user_id)
+        return {"token": token}
+
+    @server.route('/user/signup', methods=['POST'])
+    async def signup():
+        data = await request.get_json()
+
+        user_id = None
+
+        username = data['username']
+        email = data['email']
+        password = data['password']
+
+        user_id = app.create_new_user(username, email, password)
+        if user_id is None:
+            return "existing or incorrect credentials", 400
+        
+        app.generate_default_user_data(user_id)
+
+        token = auth_manager.dump_token(user_id)
+        return {"token": token}
+
+    @server.websocket('/ws')
+    async def ws():
+        print("incoming ws")
+
+        # Authenticate before dedicating any resources
+        auth_req = json.loads(await websocket.receive())
+
+        if 'token' in auth_req:
+            try:
+                user_id = auth_manager.load_token(auth_req['token'])
+            except Exception as err:
+                print("Token authentication failed: ", err)
+                await websocket.send(json.dumps({'error': 'token authentication failed'}))
+                return
+        else:
+            if "username" not in auth_req or "password" not in auth_req:
+                print("Invalid auth request:", auth_req)
+                await websocket.send(json.dumps({'error': 'invalid auth'}))
+                return
+
+            user_id = app.verify_user_credentials(auth_req['username'], auth_req['password'])
+            if user_id is None:
+                await websocket.send(json.dumps({'error': 'invalid credentials'}))
+                return
+
+        # TODO: print username instead, same in the finally block
+        print("User logged in:", user_id) 
+        await websocket.send(json.dumps({'message': 'authentication successful'}))
+
+        send_queue = asyncio.Queue()
+        async def sender_task():
+            nonlocal send_queue
+            while True:
+                msg = await send_queue.get()
+                await websocket.send(json.dumps(msg))
+        async def send(msg):
+            await send_queue.put(msg)
+
+        try:
+            if 'remotename' in auth_req:
+                app.add_remote_connection(user_id, authreq['remotename'], authreq['tools'], send_queue)
+            else:
+                app.add_user_connection(user_id, send_queue)
+            asyncio.create_task(sender_task())
+
+            while True:
+                req = json.loads(await websocket.receive())
+                if not 'action' in req:
+                    await send({'error': 'no action'})
+                    continue
+                action = req['action']
+                body = req.get('body', {})
+                nonce = req.get('nonce', None)
+
+                if action == "ping":
+                    await send({'message': "pong"})
+
+                elif action == "create-agent":
+                    name = body['name']
+                    agent_id = app.create_agent(
+                        user_id=user_id,
+                        name=name,
+                        agentname=body.get('agentname'),
+                        description=body.get('description'),
+                        picture=body.get('picture'),
+                        kind=body['kind'],
+                        data=body.get('data')
+                    )
+                    await send({'message': "agent created", 'agentId': agent_id, 'nonce': nonce})
+
+                elif action == "update-agent":
+                    agent_id = body['agentId']
+                    app.update_agent(
+                        agent_id=agent_id,
+                        name=body.get('name'),
+                        agentname=body.get('agentname'),
+                        description=body.get('description'),
+                        picture=body.get('picture'),
+                        data=body.get('data', {})
+                    )
+                    await send({'message': "agent updated", 'agentId': agent_id, 'nonce': nonce})
+
+                elif action == "get-agent-config-fields":
+                    await send({'fields': app.get_agent_config_fields(body['agentId']), 'nonce': nonce})
+
+                elif action == "get-agent-kinds":
+                    await send({'message': "agents retrieved", 'kinds': app.get_agent_kinds(), 'nonce': nonce})
+
+                elif action == "get-agent-kind-info":
+                    await send({'message': "agent kind retrieved", 'info': app.get_agent_kind_info(body['kind']), 'nonce': nonce})
+
+                elif action == "get-agents":
+                    await send({'message': "agents retrieved", 'agents': app.get_agents(user_id), 'nonce': nonce})
+
+                elif action == "create-chat":
+                    chat_id = app.create_chat(
+                        user_id=user_id,
+                        name=body['name'],
+                        description=body.get('description'),
+                        picture=body.get('picture'),
+                        members=body.get('members', []),
+                    )
+                    await send({'message': "chat created", 'chatId': chat_id, 'nonce': nonce})
+
+                elif action == "get-chats":
+                    chats_data = app.get_chats(user_id)
+                    await send({'message': "chats retrieved", 'chats': chats_data, 'nonce': nonce})
+
+                elif action == "delete-chat":
+                    chat_id = app.delete_chat(
+                        chat_id=body['chatId']
+                    )
+                    await send({'message': "chat deleted", 'chatId': chat_id, 'nonce': nonce})
+
+                elif action == "get-chat-messages":
+                    messages_data = app.get_chat_messages(
+                        body['chatId'],
+                        earliest_index=body.get('earliestIndex', 0)
+                    )
+                    await send({'message': "chat messages retrieved", 'messages': messages_data, 'nonce': nonce})
+
+                elif action == "send-chat-message":
+                    message = app.send_chat_message(
+                        chat_id=body['chatId'],
+                        thread_id=body.get('threadId'),
+                        content=body['content'],
+                        kind=body.get('kind'),
+                        subkind=body.get('subkind'),
+                        format=body.get('format'),
+                    )
+                    await send({'message': "message sent", 'messageId': message['id'], 'nonce': nonce})
+
+                elif action == "update-chat-message":
+                    message = app.update_chat_message(
+                        message_id=body['messageId'],
+                        content=body['content'],
+                        kind=body.get('kind'),
+                        subkind=body.get('subkind'),
+                        format=body.get('format'),
+                    )
+                    await send({'message': "message sent", 'messageId': body['messageId'], 'nonce': nonce})
+
+                elif action == "create-chat-thread":
+                    thread_id = app.create_chat_thread(
+                        chat_id=body['chatId'],
+                        name=body.get('name'),
+                    )
+                    await send({'message': "chat thread created", 'threadId': thread_id, 'nonce': nonce})
+
+                elif action == "send-agent-prompt":
+                    response = await app.send_agent_prompt(body['agentId'], body['content'])
+                    await send({'message': "propmt sent", 'response': response, 'nonce': nonce})
+
+                elif action == "call-agent-action":
+                    result = await app.call_agent_action(body['agentId'], body['agentAction'], body['arguments'])
+                    await send({'message': "agent action executed", 'result': result, 'nonce': nonce})
+
+                else:
+                    await send({'error': 'unknown action: ' + action, 'nonce': nonce})
+        finally:
+            print("User logged out:", user_id)
+            app.remove_user_connection(user_id, send_queue)
+
+    server.run(debug=True)

backend/src/stream.py

@@ -0,0 +1,103 @@
+import asyncio
+import nest_asyncio
+
+id_counter = 0
+
+def run_fn (fn_obj, arg):
+    if fn_obj == None:
+        return
+
+    # https://stackoverflow.com/questions/36076619/test-if-function-or-method-is-normal-or-asynchronous
+    if asyncio.iscoroutinefunction(fn_obj):
+        if arg == None:
+            asyncio.create_task(fn_obj())
+        else:
+            asyncio.create_task(fn_obj(arg))
+        return
+
+    if arg == None:
+        fn_obj()
+    else:
+        fn_obj(arg)
+
+def make_runnable(fn_obj):
+    """
+    Creates an async signal and a task that listen to the signal.
+    When it is received, fn_obj is called.
+    The task doesn't stop and keeps forwarding signals to the passed function.
+    There can also be arguments coming along with the signal, those are passed to the function.
+    """
+    signal_queue = asyncio.Queue()
+
+    async def signal_listener():
+        while True:
+            args = await signal_queue.get()
+            if args is None:  # Stop signal
+                break
+            run_fn(fn_obj, args[0] if args else None)
+
+    # Start the listener task in the background
+    asyncio.create_task(signal_listener())
+
+    # Return a method to put items on the signal queue
+    return lambda *args: signal_queue.put_nowait(args)
+
+class Stream:
+    def __init__(self, app):
+        global id_counter
+
+        self.id = id_counter
+        id_counter = id_counter + 1
+
+        self.open = True
+        self.content = ""
+        self.listeners = []
+
+        app.streams[self.id] = self
+
+    def listen (self, chunk_fn, end_fn):
+        nest_asyncio.apply()
+        #chunk_fn = make_runnable(chunk_fn)
+        #end_fn = make_runnable(end_fn)
+
+        if self.open:
+            self.listeners.append({
+                "chunk": chunk_fn,
+                "end": end_fn,
+            })
+
+            if self.content != "":
+                run_fn(chunk_fn, self.content)
+        else:
+            run_fn(chunk_fn, self.content)
+            run_fn(end_fn, None)
+
+    def feed (self, chunk):
+        if not self.open:
+            return
+
+        self.content = self.content + chunk
+        for listener in self.listeners:
+            run_fn(listener["chunk"], chunk)
+
+    def close (self):
+        self.open = False
+
+        for listener in self.listeners:
+            run_fn(listener["end"], None)
+
+    async def content_wait (self):
+        async_loop = asyncio.get_event_loop()
+        event = asyncio.Event()
+
+        def set_event():
+            event.set()
+
+        def noop (chunk): pass
+        def end_fn ():
+            async_loop.call_soon_threadsafe(set_event)
+
+        self.listen(chunk_fn=noop, end_fn=end_fn)
+        await event.wait()
+
+        return self.content

backend/src/tools.py

@@ -0,0 +1,102 @@
+
+import inspect
+
+# Do not call methods from instances of this class, they must not get overridden
+class Toolset:
+    # Fields: description, identifier, _tools: list[ToolDescription]
+    def __init__(self):
+        # TODO: receive app, assign _tools
+        pass
+
+    def get_namespace_definitions(toolset):
+        header = toolset.description
+        if toolset.identifier:
+            header = f"Namespace {toolset.identifier}: {toolset.description}"
+        lines = [header]
+ 
+        for tool in toolset._tools:
+            full_name = tool.identifier
+            if toolset.identifier:
+                full_name = toolset.identifier + "." + full_name
+            lines.append(f"- {tool.description}\n!!{full_name} {tool.schema}")
+        return "\n".join(lines)
+    
+    async def execute_action(toolset, name: str, argument: dict):
+        # TODO: For security, check that the method is in the tool list
+        method = getattr(toolset, name)
+        result = method(**argument)
+        if inspect.isawaitable(result):
+            result = await result
+        return result
+
+class ToolDescription:
+    def __init__(self, identifier, description, schema):
+        self.identifier=identifier
+        self.description=description
+        self.schema=schema
+
+# An action is an object representing a command invokation
+class Action:
+    def __init__(self, toolset_name: str, tool_name: str, args: dict, is_query: bool = False):
+        self.toolset_name = toolset_name
+        self.tool_name = tool_name
+        self.args = args
+        self.is_query = is_query
+    
+    def is_special(self, name: str):
+        return self.toolset_name == "" and self.tool_name == name
+
+    def __str__(self):
+        return f"{self.is_query and 'query' or ''}{self.toolset_name}.{self.tool_name} {self.args}"
+    
+    def __repr__(self):
+        return f"Action({self.__str__()})"
+
+class SpecialToolset(Toolset):
+    def __init__(self):
+        self.identifier = ""
+        self.description = "Special commands that don't receive arguments"
+        self._tools = [
+            ToolDescription("resume", "Resume the conversation with the command outputs", ''),
+            ToolDescription("respond", "When used, all text after this command will be used as the actual response", ''),
+        ]
+
+    def resume(self):
+        pass
+
+    def respond(self):
+        pass
+
+class BasicToolset(Toolset):
+    def __init__(self):
+        self.identifier = "util"
+        self.description = "Basic assistant utility tools"
+        self._tools = [
+            ToolDescription("google_search", "Executes a google search", '{"query":string}'),
+            ToolDescription("fetch", "Fetch the contents of a url", '{"url":string}'),
+            ToolDescription("eval", "Evaluates a python expression", '{"expression":string}'),
+        ]
+
+    def google_search(self, query):
+        return "We're sorry. Google is not accessible at this time"
+
+    def fetch(self, url):
+        return "No internet connection."
+    
+    def eval(self, expression):
+        try:
+            return eval(expression)
+        except Exception as e:
+            return str(e)
+
+class HubControlsToolset(Toolset):
+    def __init__(self):
+        self.identifier = "hub"
+        self.description = "Controls for the Hub system"
+        self._tools = [
+            ToolDescription("get_chats", "Get all the existing chats", '{}'),
+            ToolDescription("get_agents", "Get all the existing bots", '{}'),
+            ToolDescription("create_chat", "Creates a new chat", '{"name":string,"agent_ids":int[]}'),
+            ToolDescription("get_agent_data", "Gets the configuration data of an agent", '{"agent_id":int}'),
+            ToolDescription("set_agent_data", "Get all the existing bots", '{"agent_id":int,"data":object}'),
+        ]