Temporary agents and chats

backend/src/agents/expert.py

@@ -51,48 +51,33 @@ class ExpertAgent(Agent):
         if prompt:
             messages.append(LlamaMessage("user", prompt))
 
-        model = self.config['model']
+        model_id = self.config['model']
+        db_model = await self.app.get_model(model_id)
+        model_engine = db_model.value["engine"]
 
-        if type(model) is int:
-            db_model = await self.app.get_model(model)
-            model_engine = db_model.value["engine"]
+        opts = {}
+        if 'temperature' in self.config:
+            opts['temperature'] = self.config['temperature']
+        if 'repetition_penalty' in self.config:
+            opts['repetition_penalty'] = self.config['repetition_penalty']
+        
+        if model_engine == "llama-cpp":
+            response = await inference.local_chat_completion(
+                self.app,
+                messages,
+                model=db_model.value['model'],
+                opts=opts,
+            )
+        elif model_engine == "replicate":
+            if 'parameters' in db_model:
+                opts['parameters'] = db_model['parameters']
 
-            opts = {}
-            if 'temperature' in self.config:
-                opts['temperature'] = self.config['temperature']
-            if 'repetition_penalty' in self.config:
-                opts['repetition_penalty'] = self.config['repetition_penalty']
-            
-            if model_engine == "llama-cpp":
-                response = await inference.local_chat_completion(
-                    self.app,
-                    messages,
-                    model=db_model.value['model'],
-                    opts=opts,
-                )
+            response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts)
 
-                return response
-            elif model_engine == "replicate":
-                if 'parameters' in db_model:
-                    opts['parameters'] = db_model['parameters']
-
-                response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts)
-            else:
-                raise Exception("Unsuported model: " + repr(db_model.value))
-
-        elif model.startswith("ollama:"):
-            response = await inference.ollama_chat_completion(self.app, messages, model=model[7:])
-        elif model == "qwen-big":
-            response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b)
-        elif model == "qwen-small":
-            response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_8b)
-        elif model == "llama-big":
-            response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b)
-        elif model == "llama-small":
-            response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_8b)
+        elif model_engine == "ollama":
+            response = await inference.ollama_chat_completion(self.app, messages, model=db_model.value['model'], opts=opts)
         else:
-            # An error here is questionable, model is user configurable
-            raise Exception("Unkonwn agent model: " + model)
+            raise Exception("Unsuported model: " + repr(db_model.value))
 
         return response
 
@@ -108,6 +93,8 @@ class ExpertAgent(Agent):
                 return 'assistant'
             elif sender == 'user':
                 return 'user'
+            elif sender.startswith("tmp-"):
+                return 'assistant'
             else:
                 raise Exception("Unsupported sender: " + sender)
         

backend/src/app.py

@@ -13,6 +13,7 @@ import replicate
 import database as db
 import agent as agent_mod
 import agents as agents_mod
+import temporary as tmp
 from server import start_server
 from chat import ChatView, Message, MESSAGE_PAGE_SIZE
 
@@ -25,6 +26,8 @@ class App(ChatMixin):
         self.remote_connections = {}
         self.streams = {}
 
+        self.tmp = tmp.State()
+
         # Clients
         self.qdrant_client = QdrantClient(path=f"{self.data_directory}/qdrant_data")
 
@@ -67,9 +70,41 @@ class App(ChatMixin):
 
     # --- Agent Methods (Asynchronous) ---
 
-    async def create_agent(self, user_id, name, agentname, description, picture, kind, data, main_chat_id=None):
+    async def get_agent(self, agent_id):
+        if tmp.is_tmp_id(agent_id):
+            agent = self.tmp.agents.get(agent_id)
+        else:
+            agent = await db.Agent.get_or_none(id=agent_id)
+
+        if not agent:
+            raise Exception(f"Agent not found with id {agent_id}")
+        return agent
+
+    async def create_agent(self, user_id, name, agentname, description, picture, kind, data, main_chat_id=None, temporary=False):
         print("create_agent; user_id, name:", user_id, name)
 
+        if temporary:
+            agent = tmp.Agent.create(
+                self,
+                user_id=user_id,
+                name=name,
+                agentname=agentname,
+                main_chat_id=main_chat_id,
+                description=description,
+                picture=picture,
+                kind=kind,
+                data=data or {}
+            )
+
+            if main_chat_id is not None:
+                if tmp.is_tmp_id(main_chat_id):
+                    self.tmp.chats[main_chat_id].participants.append(agent.id)
+                else:
+                    print(f"ERROR: cannot use {main_chat_id} as main chat for temporary agent {agentname or agent.id}")
+                    agent.main_chat_id = None
+
+            return agent.id
+
         if main_chat_id is None:
             chat = await db.Chat.create(
                 user_id=user_id,
@@ -97,9 +132,7 @@ class App(ChatMixin):
         return agent.id
 
     async def update_agent(self, agent_id, name=None, description=None, picture=None, agentname=None, data=None):
-        agent = await db.Agent.get_or_none(id=agent_id)
-        if not agent:
-            raise Exception(f"Agent not found with id {agent_id}")
+        agent = await self.get_agent(agent_id)
         
         summary = []
         if name is not None:
@@ -120,11 +153,15 @@ class App(ChatMixin):
             agent.data.update(data)
             summary.extend([f"data.{k}" for k in data.keys()])
 
-        await agent.save()
+        if db.is_instance(agent):
+            await agent.save()
         print(f"Updated agent #{agent.id} fields: {summary}")
 
     async def get_agents(self, user_id):
         agents = await db.Agent.filter(user_id=user_id).all()
+        if user_id in self.tmp.user_agents:
+            agents += [self.tmp.agents[agent_id] for agent_id in self.tmp.user_agents[user_id]]
+
         return [{
             "id": agent.id,
             "name": agent.name,
@@ -176,9 +213,7 @@ class App(ChatMixin):
     # --- Agent Control & Actions ---
 
     async def get_agent_control(self, agent_id):
-        agent = await db.Agent.get_or_none(id=agent_id)
-        if not agent:
-            raise Exception(f"Agent not found with id {agent_id}")
+        agent = await self.get_agent(agent_id)
         
         agent_class = next((kind for kind in self.agent_classes if kind.identifier == agent.kind), None)
         if agent_class is None:
@@ -189,9 +224,7 @@ class App(ChatMixin):
         return control
 
     async def get_agent_config_fields(self, agent_id):
-        agent = await db.Agent.get_or_none(id=agent_id)
-        if not agent:
-            raise Exception(f"Agent not found with id {agent_id}")
+        agent = await self.get_agent(agent_id)
 
         agent_class = next((it for it in self.agent_classes if it.identifier == agent.kind), None)
         if not agent_class:

backend/src/app_mixins/chat.py

@@ -2,18 +2,27 @@ import asyncio
 from datetime import datetime
 
 import database as db 
+import temporary as tmp
 from chat import ChatView, Message, MESSAGE_PAGE_SIZE
 from stream import Stream
 
 class ChatMixin:
-    async def create_chat(self, user_id, name, description, picture, members=[]):
-        # Tortoise uses .create() for one-line instantiation and saving
-        chat = await db.Chat.create(
-            user_id=user_id,
-            name=name,
-            description=description,
-            picture=picture
-        )
+    async def create_chat(self, user_id, name, description, picture, members=[], temporary=False):
+        if temporary:
+            chat = tmp.Chat.create(
+                self,
+                user_id=user_id,
+                name=name,
+                description=description,
+                picture=picture
+            )
+        else:
+            chat = await db.Chat.create(
+                user_id=user_id,
+                name=name,
+                description=description,
+                picture=picture
+            )
         print(f"Created chat #{chat.id}: {chat.name}")
 
         for agent_id in members:
@@ -24,6 +33,11 @@ class ChatMixin:
     async def get_chats(self, user_id):
         # Direct filtering on the model class
         chats = await db.Chat.filter(user_id=user_id).all()
+
+        tmp_chats = self.tmp.get_user_chats(user_id)
+        if tmp_chats:
+            chats += [self.tmp.chats[chat_id] for chat_id in tmp_chats]
+
         return [{
             "id": chat.id,
             "name": chat.name,
@@ -32,13 +46,23 @@ class ChatMixin:
         } for chat in chats]
 
     async def delete_chat(self, chat_id):
-        # Filtering and deleting is a one-liner
-        await db.Membership.filter(chat_id=chat_id).delete()
-        await db.Chat.filter(id=chat_id).delete()
-        # TODO: Delete messages from the chat
-        print(f"Deleted chat #{chat_id} and its memberships.")
+        if tmp.is_tmp_id(chat_id):
+            chat = self.tmp.chats[chat_id]
+            # TODO: Delete
+
+        else:
+            # Filtering and deleting is a one-liner
+            await db.Membership.filter(chat_id=chat_id).delete()
+            await db.Chat.filter(id=chat_id).delete()
+            # TODO: Delete messages from the chat
+            print(f"Deleted chat #{chat_id} and its memberships.")
 
     async def get_chat_messages(self, chat_id, earliest_index=None):
+        if tmp.is_tmp_id(chat_id):
+            chat = self.tmp.chats[chat_id]
+            
+            return [message.to_json() for message in chat.messages]
+
         query = db.Message.filter(chat_id=chat_id)
         if earliest_index is not None:
             query = query.filter(id__gt=earliest_index)
@@ -58,18 +82,25 @@ class ChatMixin:
         } for message in messages]
     
     async def create_chat_thread(self, chat_id, name=None):
-        thread = await db.Thread.create(
-            chat_id=chat_id,
-            name=name
-        )
+        if tmp.is_tmp_id(chat_id):
+            thread = tmp.Thread.create(self, chat_id, name)
+        else:
+            thread = await db.Thread.create(
+                chat_id=chat_id,
+                name=name
+            )
         print(f"Created thread #{thread.id}: {thread.name}")
         return thread.id
 
     async def add_agent_to_chat(self, chat_id, agent_id):
-        await db.Membership.create(
-            chat_id=chat_id,
-            agent_id=agent_id
-        )
+        if tmp.is_tmp_id(chat_id):
+            chat = self.tmp.chats[chat_id]
+            chat.participants.append(agent_id)
+        else:
+            await db.Membership.create(
+                chat_id=chat_id,
+                agent_id=agent_id
+            )
         print(f"Added agent #{agent_id} to chat #{chat_id}")
 
     async def send_chat_message(self, chat_id, content, sender='user', thread_id=None, kind=None, subkind=None, format=None):
@@ -82,27 +113,49 @@ class ChatMixin:
 
         # Set to true for streams
         updating = False
-        message = await db.Message.create(
-            chat_id=chat_id,
-            time=now,
-            last_edit=now,
-            sender=sender,
-            kind=kind or 'message',
-            subkind=subkind,
-            format=format,
-            content=initial_content,
-            thread_id=thread_id
-        )
+        if tmp.is_tmp_id(chat_id):
+            message = Message(
+                tmp.generate_tmp_id(self),
+                #chat_id=chat_id,
+                time=now,
+                #last_edit=now,
+                sender=sender,
+                kind=kind or 'message',
+                subkind=subkind,
+                #format=format,
+                content=initial_content,
+                thread_id=thread_id
+            )
+            message.chat_id = chat_id
+            chat = self.tmp.chats[chat_id]
+            chat.messages.append(message)
+            chat_owner_id = chat.user_id
+        else:
+            message = await db.Message.create(
+                chat_id=chat_id,
+                time=now,
+                last_edit=now,
+                sender=sender,
+                kind=kind or 'message',
+                subkind=subkind,
+                format=format,
+                content=initial_content,
+                thread_id=thread_id
+            )
 
-        # Fetch chat owner using values_list for efficiency
-        chat = await db.Chat.get(id=chat_id).prefetch_related("user")
-        chat_owner_id = chat.user_id
+            # Fetch chat owner using values_list for efficiency
+            chat = await db.Chat.get(id=chat_id).prefetch_related("user")
+            chat_owner_id = chat.user_id
 
         # Notification order depends on whether the content is a stream
 
         async def notify_agents():
             # Fetch participant agents
-            agent_ids = await db.Membership.filter(chat_id=chat_id).values_list("agent_id", flat=True)
+            if tmp.is_tmp_id(chat_id):
+                agent_ids = self.tmp.chats[chat_id].participants
+            else:
+                agent_ids = await db.Membership.filter(chat_id=chat_id).values_list("agent_id", flat=True)
+
             for agent_id in agent_ids:
                 if str(agent_id) != str(sender):
                     await self.notify_message_to_agent(agent_id, message)
@@ -120,7 +173,8 @@ class ChatMixin:
                 notify_users()
             async def end_fn ():
                 message.content = content.content
-                await message.save()
+                if db.is_instance(message):
+                    await message.save()
                 await notify_agents()
                 notify_users()
 
@@ -142,7 +196,10 @@ class ChatMixin:
         }
 
     async def update_chat_message(self, message_id, content=None, kind=None, subkind=None, format=None):
-        message = await db.Message.get_or_none(id=message_id)
+        if tmp.is_tmp_id(message_id):
+            raise Exception("Cannot update temporary messages")
+        else:
+            message = await db.Message.get_or_none(id=message_id)
 
         if not message:
             raise Exception(f"Message not found with id {message_id}")
@@ -161,7 +218,8 @@ class ChatMixin:
             summary.append('format')
         
         if summary:
-            await message.save() # Saves the specific instance
+            if db.is_instance(message):
+                await message.save() # Saves the specific instance
             print(f"Updated message #{message.id} fields: {summary}")
 
     async def get_chat_view(self, chat_id):

backend/src/chat.py

@@ -3,17 +3,19 @@ from sqlalchemy.orm import Session
 from sqlalchemy.sql.expression import select
 
 import database as db
+import temporary as tmp
 
 MESSAGE_PAGE_SIZE = 256
 
 class Message:
-    def __init__(self, id, content, sender, time=None, kind='message', subkind=None):
+    def __init__(self, id, content, sender, time=None, kind='message', subkind=None, thread_id=None):
         self.id = id
         self.content = content
         self.sender = sender
         self.time = time
         self.kind = kind
         self.subkind = subkind
+        self.thread_id = thread_id
     
     def from_db_record(record):
         return Message(record.id, record.content, record.sender, record.time)
@@ -23,6 +25,16 @@ class Message:
     
     def is_from_agent(self, agent_id):
         return self.sender == str(agent_id)
+
+    def to_json(self):
+        return {
+            "id": self.id,
+            "time": self.time,
+            "sender": self.sender,
+            "kind": self.kind,
+            "content": self.content,
+            "threadId": self.thread_id,
+        }
     
     def __str__(self):
         time = self.time if self.time != None else 'no-time'
@@ -67,14 +79,18 @@ class ChatView:
         self.current_thread_id = None
         
         self._earliest_message_id = None
+        self._earliest_message_time = None
     
     async def init_load(self):
         await self._load_basic_data()
         await self._load_next_messages(MESSAGE_PAGE_SIZE)
 
     async def _load_basic_data(self):
-        # Using get_or_none for cleaner existence checks
-        record = await db.Chat.get_or_none(id=self.chat_id)
+        if tmp.is_tmp_id(self.chat_id):
+            record = self.app.tmp.chats.get(self.chat_id)
+        else:
+            # Using get_or_none for cleaner existence checks
+            record = await db.Chat.get_or_none(id=self.chat_id)
 
         if not record:
             raise Exception(f"Chat not found with id {self.chat_id}")
@@ -83,21 +99,32 @@ class ChatView:
         self.description = record.description
 
     async def _load_next_messages(self, load_count):
-        # Start building the filter
-        filters = {"chat_id": self.chat_id}
-        
-        # Add the 'earliest message' condition if it exists
-        if self._earliest_message_id is not None:
-            filters["id__lte"] = self._earliest_message_id
+        if tmp.is_tmp_id(self.chat_id):
+            chat = self.app.tmp.chats[self.chat_id]
+            record_batch = [
+                msg for msg in chat.messages
+                if not self._earliest_message_time or msg.id < self._earliest_message_time
+            ][:load_count]
+            record_batch.reverse()
+            records_rev = record_batch
+        else:
+            # Start building the filter
+            filters = {"chat_id": self.chat_id}
+            
+            # Add the 'earliest message' condition if it exists
+            if self._earliest_message_id is not None:
+                filters["id__lte"] = self._earliest_message_id
 
-        # Tortoise query: filter, order, and limit
-        # ordering by id is more robust than by time
-        records_rev = await db.Message.filter(**filters).order_by("-id").limit(load_count)
+            # Tortoise query: filter, order, and limit
+            # ordering by id is more robust than by time
+            records_rev = await db.Message.filter(**filters).order_by("-id").limit(load_count)
 
         # Update the earliest id for the next load operation
         if records_rev:
-            # Note: Since we ordered by -time (descending), index 0 is the newest in this batch
-            self._earliest_message_id = records_rev[-1].id  # The "earliest" is the one with the smallest ID/Time in this set
+            # Note: Since we ordered by -time (descending), index 0 is the newest in this batch (highest id/time)
+            # The "earliest" or oldest is the one with the smallest ID/Time in this set
+            self._earliest_message_id = records_rev[-1].id
+            self._earliest_message_time = records_rev[-1].time
             
             if not self.threads:
                 # current thread is the latest known from the newest record in batch

backend/src/database.py

@@ -29,6 +29,10 @@ async def init_database (data_directory):
     )
     await Tortoise.generate_schemas()
 
+# Checks whether an object is an instance of any database model
+def is_instance (value):
+    isinstance(value, models.Model)
+
 class User(models.Model):
     id = fields.IntField(pk=True)
     email = fields.CharField(max_length=255)

backend/src/inference.py

@@ -1,4 +1,5 @@
 import threading
+import asyncio
 import ollama
 from stream import Stream
 
@@ -23,22 +24,33 @@ class Models:
 
 # llama cpp models
 model_cache = {}
+last_model_file = None
+last_model_instance = None
+
+def get_llama_cpp_model(app, model_file):
+    global last_model_file, last_model_instance
+    if last_model_file == model_file:
+        return last_model_instance
+    
+    if last_model_instance is not None:
+        #last_model_instance = None
+        del last_model_instance
+
+    last_model_file = model_file
+    last_model_instance = Llama(    
+        model_path=f"{app.data_directory}/models/{model_file}",
+        n_gpu_layers=-1, # Uncomment to use GPU acceleration
+        # seed=1337, # Uncomment to set a specific seed
+        n_ctx=CONTEXT_WINDOW,
+    )
+    return last_model_instance
 
 # llama.cpp 
 async def local_chat_completion(app, messages, model, opts={}):
     content_stream = Stream(app)
 
     def worker():
-        if model in model_cache:
-            llm = model_cache[model]
-        else:
-            llm = Llama(    
-                model_path=f"{app.data_directory}/models/{model}",
-                n_gpu_layers=-1, # Uncomment to use GPU acceleration
-                # seed=1337, # Uncomment to set a specific seed
-                n_ctx=CONTEXT_WINDOW,
-            )
-            model_cache[model] = llm
+        llm = get_llama_cpp_model(app, model)
 
         temp = opts.get("temperature")
         if temp is None:
@@ -77,23 +89,23 @@ async def ollama_list(app, messages, model, opts={}):
 async def ollama_chat_completion(app, messages, model, opts={}):
     content_stream = Stream(app)
 
-    print("START OLLAMA INFERENCE")
-    stream_response = await ollama.AsyncClient().chat(
-        model=model,
-        messages=[{'role': msg.role, 'content': msg.content} for msg in messages],
-        stream=True,
-    )
+    async def worker():
+        print("START OLLAMA INFERENCE")
+        stream_response = await ollama.AsyncClient().chat(
+            model=model,
+            messages=[{'role': msg.role, 'content': msg.content} for msg in messages],
+            stream=True,
+        )
 
-    async for part in stream_response:
-        content_stream.feed(part.message.content)
-        print(part['message']['content'], end='', flush=True)
+        async for part in stream_response:
+            content_stream.feed(part.message.content)
 
-    content_stream.close()
+        content_stream.close()
+        print("END OLLAMA INFERENCE")
+    
+    asyncio.create_task(worker())
 
-    print("END OLLAMA INFERENCE")
-    return await content_stream.content_wait()
-
-    #return content_stream
+    return content_stream
 
 
 async def replicate_completion(app, messages, model=None, opts={}):
@@ -104,7 +116,7 @@ async def replicate_completion(app, messages, model=None, opts={}):
 
     arguments = {}
     arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages)
-    arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75)
+    arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature') or 0.75
     arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048)
 
     arguments[parameter_names.get('system_prompt', 'system_prompt')] = ""

backend/src/server.py

@@ -162,7 +162,8 @@ def start_server(app):
                         description=body.get('description'),
                         picture=body.get('picture'),
                         kind=body['kind'],
-                        data=body.get('data')
+                        data=body.get('data'),
+                        temporary=body.get('temporary'),
                     )
                     await send({'message': "agent created", 'agentId': agent_id, 'nonce': nonce})
 
@@ -216,6 +217,7 @@ def start_server(app):
                         description=body.get('description'),
                         picture=body.get('picture'),
                         members=body.get('members', []),
+                        temporary=body.get('temporary'),
                     )
                     await send({'message': "chat created", 'chatId': chat_id, 'nonce': nonce})
 

backend/src/temporary.py

@@ -0,0 +1,87 @@
+
+def generate_tmp_id(app):
+    app.tmp.id_counter += 1
+    return "tmp-" + str(app.tmp.id_counter)
+
+def is_tmp_id(id):
+    return type(id) == str and id.startswith("tmp-")
+
+class State:
+    def __init__(self):
+        self.id_counter = 0
+        self.agents = {}
+        self.chats = {}
+        self.threads = {}
+
+        # list of temporary agent ids that belong to a real user
+        self.user_agents = {}
+        # list of temporary chat ids that belong to a real user
+        self.user_chats = {}
+        # list of temporary chat ids where a real agent is participating
+        self.participating = {}
+    
+    def get_user_agents(self, user_id):
+        if user_id not in self.user_agents:
+            self.user_agents[user_id] = []
+        return self.user_agents[user_id]
+    
+    def get_user_chats(self, user_id):
+        if user_id not in self.user_chats:
+            self.user_chats[user_id] = []
+        return self.user_chats[user_id]
+
+class Agent:
+    def create(app, user_id, name, agentname, main_chat_id, description="", picture=None, kind='default', data={}):
+        self = Agent()
+
+        self.id = None
+        self.user_id = user_id
+        self.name = name
+        self.agentname = agentname
+        self.main_chat_id = main_chat_id
+        self.description = description
+        self.picture = picture
+        self.kind = kind
+        self.data = data
+
+        self.id = generate_tmp_id(app)
+        app.tmp.agents[self.id] = self
+        app.tmp.get_user_agents(user_id).append(self.id)
+
+        return self
+
+class Chat:
+    def create(app, user_id, name, description="", picture=None, ordering=None):
+        self = Chat()
+        self._app = app
+
+        self.id = None
+        self.user_id = user_id
+        self.name = name
+        self.description = description
+        self.picture = picture
+        self.ordering = ordering
+
+        self.messages = []
+        self.threads = []
+        self.participants = []
+
+        self.id = generate_tmp_id(app)
+        app.tmp.chats[self.id] = self
+        app.tmp.get_user_chats(user_id).append(self.id)
+
+        return self
+
+class Thread:
+    def create(app, chat_id, name):
+        self = Thread()
+
+        self.chat_id = chat_id
+        self.name = name
+        self.id = generate_tmp_id(app)
+
+        app.tmp.threads[self.id] = self
+        app.tmp.chats[chat_id].threads.append(self.id)
+
+        return self
+