diff --git a/.gitignore b/.gitignore index 0b9799f..41eb1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ horde.db /test-data /local-data -/models \ No newline at end of file +/models + +package-lock.json +node_modules/ \ No newline at end of file diff --git a/backend/src/agents/expert.py b/backend/src/agents/expert.py index 22e59ad..0942179 100644 --- a/backend/src/agents/expert.py +++ b/backend/src/agents/expert.py @@ -51,48 +51,33 @@ class ExpertAgent(Agent): if prompt: messages.append(LlamaMessage("user", prompt)) - model = self.config['model'] + model_id = self.config['model'] + db_model = await self.app.get_model(model_id) + model_engine = db_model.value["engine"] - if type(model) is int: - db_model = await self.app.get_model(model) - model_engine = db_model.value["engine"] + opts = {} + if 'temperature' in self.config: + opts['temperature'] = self.config['temperature'] + if 'repetition_penalty' in self.config: + opts['repetition_penalty'] = self.config['repetition_penalty'] + + if model_engine == "llama-cpp": + response = await inference.local_chat_completion( + self.app, + messages, + model=db_model.value['model'], + opts=opts, + ) + elif model_engine == "replicate": + if 'parameters' in db_model: + opts['parameters'] = db_model['parameters'] - opts = {} - if 'temperature' in self.config: - opts['temperature'] = self.config['temperature'] - if 'repetition_penalty' in self.config: - opts['repetition_penalty'] = self.config['repetition_penalty'] - - if model_engine == "llama-cpp": - response = await inference.local_chat_completion( - self.app, - messages, - model=db_model.value['model'], - opts=opts, - ) + response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts) - return response - elif model_engine == "replicate": - if 'parameters' in db_model: - opts['parameters'] = db_model['parameters'] - - response = await inference.replicate_completion(self.app, messages, model=db_model.value['model'], opts=opts) - else: - raise Exception("Unsuported model: " + repr(db_model.value)) - - elif model.startswith("ollama:"): - response = await inference.ollama_chat_completion(self.app, messages, model=model[7:]) - elif model == "qwen-big": - response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_235b) - elif model == "qwen-small": - response = await inference.replicate_qwen_completion(self.app, messages, model=inference.Models.qwen3_8b) - elif model == "llama-big": - response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_70b) - elif model == "llama-small": - response = await inference.replicate_llama_completion(self.app, messages, model=inference.Models.llama_8b) + elif model_engine == "ollama": + response = await inference.ollama_chat_completion(self.app, messages, model=db_model.value['model'], opts=opts) else: - # An error here is questionable, model is user configurable - raise Exception("Unkonwn agent model: " + model) + raise Exception("Unsuported model: " + repr(db_model.value)) return response @@ -108,6 +93,8 @@ class ExpertAgent(Agent): return 'assistant' elif sender == 'user': return 'user' + elif sender.startswith("tmp-"): + return 'assistant' else: raise Exception("Unsupported sender: " + sender) diff --git a/backend/src/app.py b/backend/src/app.py index 1a4cf1b..2fe21f2 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -13,6 +13,7 @@ import replicate import database as db import agent as agent_mod import agents as agents_mod +import temporary as tmp from server import start_server from chat import ChatView, Message, MESSAGE_PAGE_SIZE @@ -25,6 +26,8 @@ class App(ChatMixin): self.remote_connections = {} self.streams = {} + self.tmp = tmp.State() + # Clients self.qdrant_client = QdrantClient(path=f"{self.data_directory}/qdrant_data") @@ -67,9 +70,41 @@ class App(ChatMixin): # --- Agent Methods (Asynchronous) --- - async def create_agent(self, user_id, name, agentname, description, picture, kind, data, main_chat_id=None): + async def get_agent(self, agent_id): + if tmp.is_tmp_id(agent_id): + agent = self.tmp.agents.get(agent_id) + else: + agent = await db.Agent.get_or_none(id=agent_id) + + if not agent: + raise Exception(f"Agent not found with id {agent_id}") + return agent + + async def create_agent(self, user_id, name, agentname, description, picture, kind, data, main_chat_id=None, temporary=False): print("create_agent; user_id, name:", user_id, name) + if temporary: + agent = tmp.Agent.create( + self, + user_id=user_id, + name=name, + agentname=agentname, + main_chat_id=main_chat_id, + description=description, + picture=picture, + kind=kind, + data=data or {} + ) + + if main_chat_id is not None: + if tmp.is_tmp_id(main_chat_id): + self.tmp.chats[main_chat_id].participants.append(agent.id) + else: + print(f"ERROR: cannot use {main_chat_id} as main chat for temporary agent {agentname or agent.id}") + agent.main_chat_id = None + + return agent.id + if main_chat_id is None: chat = await db.Chat.create( user_id=user_id, @@ -97,9 +132,7 @@ class App(ChatMixin): return agent.id async def update_agent(self, agent_id, name=None, description=None, picture=None, agentname=None, data=None): - agent = await db.Agent.get_or_none(id=agent_id) - if not agent: - raise Exception(f"Agent not found with id {agent_id}") + agent = await self.get_agent(agent_id) summary = [] if name is not None: @@ -120,11 +153,15 @@ class App(ChatMixin): agent.data.update(data) summary.extend([f"data.{k}" for k in data.keys()]) - await agent.save() + if db.is_instance(agent): + await agent.save() print(f"Updated agent #{agent.id} fields: {summary}") async def get_agents(self, user_id): agents = await db.Agent.filter(user_id=user_id).all() + if user_id in self.tmp.user_agents: + agents += [self.tmp.agents[agent_id] for agent_id in self.tmp.user_agents[user_id]] + return [{ "id": agent.id, "name": agent.name, @@ -176,9 +213,7 @@ class App(ChatMixin): # --- Agent Control & Actions --- async def get_agent_control(self, agent_id): - agent = await db.Agent.get_or_none(id=agent_id) - if not agent: - raise Exception(f"Agent not found with id {agent_id}") + agent = await self.get_agent(agent_id) agent_class = next((kind for kind in self.agent_classes if kind.identifier == agent.kind), None) if agent_class is None: @@ -189,9 +224,7 @@ class App(ChatMixin): return control async def get_agent_config_fields(self, agent_id): - agent = await db.Agent.get_or_none(id=agent_id) - if not agent: - raise Exception(f"Agent not found with id {agent_id}") + agent = await self.get_agent(agent_id) agent_class = next((it for it in self.agent_classes if it.identifier == agent.kind), None) if not agent_class: diff --git a/backend/src/app_mixins/chat.py b/backend/src/app_mixins/chat.py index 2444fa5..bdef4b4 100644 --- a/backend/src/app_mixins/chat.py +++ b/backend/src/app_mixins/chat.py @@ -2,18 +2,27 @@ import asyncio from datetime import datetime import database as db +import temporary as tmp from chat import ChatView, Message, MESSAGE_PAGE_SIZE from stream import Stream class ChatMixin: - async def create_chat(self, user_id, name, description, picture, members=[]): - # Tortoise uses .create() for one-line instantiation and saving - chat = await db.Chat.create( - user_id=user_id, - name=name, - description=description, - picture=picture - ) + async def create_chat(self, user_id, name, description, picture, members=[], temporary=False): + if temporary: + chat = tmp.Chat.create( + self, + user_id=user_id, + name=name, + description=description, + picture=picture + ) + else: + chat = await db.Chat.create( + user_id=user_id, + name=name, + description=description, + picture=picture + ) print(f"Created chat #{chat.id}: {chat.name}") for agent_id in members: @@ -24,6 +33,11 @@ class ChatMixin: async def get_chats(self, user_id): # Direct filtering on the model class chats = await db.Chat.filter(user_id=user_id).all() + + tmp_chats = self.tmp.get_user_chats(user_id) + if tmp_chats: + chats += [self.tmp.chats[chat_id] for chat_id in tmp_chats] + return [{ "id": chat.id, "name": chat.name, @@ -32,13 +46,23 @@ class ChatMixin: } for chat in chats] async def delete_chat(self, chat_id): - # Filtering and deleting is a one-liner - await db.Membership.filter(chat_id=chat_id).delete() - await db.Chat.filter(id=chat_id).delete() - # TODO: Delete messages from the chat - print(f"Deleted chat #{chat_id} and its memberships.") + if tmp.is_tmp_id(chat_id): + chat = self.tmp.chats[chat_id] + # TODO: Delete + + else: + # Filtering and deleting is a one-liner + await db.Membership.filter(chat_id=chat_id).delete() + await db.Chat.filter(id=chat_id).delete() + # TODO: Delete messages from the chat + print(f"Deleted chat #{chat_id} and its memberships.") async def get_chat_messages(self, chat_id, earliest_index=None): + if tmp.is_tmp_id(chat_id): + chat = self.tmp.chats[chat_id] + + return [message.to_json() for message in chat.messages] + query = db.Message.filter(chat_id=chat_id) if earliest_index is not None: query = query.filter(id__gt=earliest_index) @@ -58,18 +82,25 @@ class ChatMixin: } for message in messages] async def create_chat_thread(self, chat_id, name=None): - thread = await db.Thread.create( - chat_id=chat_id, - name=name - ) + if tmp.is_tmp_id(chat_id): + thread = tmp.Thread.create(self, chat_id, name) + else: + thread = await db.Thread.create( + chat_id=chat_id, + name=name + ) print(f"Created thread #{thread.id}: {thread.name}") return thread.id async def add_agent_to_chat(self, chat_id, agent_id): - await db.Membership.create( - chat_id=chat_id, - agent_id=agent_id - ) + if tmp.is_tmp_id(chat_id): + chat = self.tmp.chats[chat_id] + chat.participants.append(agent_id) + else: + await db.Membership.create( + chat_id=chat_id, + agent_id=agent_id + ) print(f"Added agent #{agent_id} to chat #{chat_id}") async def send_chat_message(self, chat_id, content, sender='user', thread_id=None, kind=None, subkind=None, format=None): @@ -82,27 +113,49 @@ class ChatMixin: # Set to true for streams updating = False - message = await db.Message.create( - chat_id=chat_id, - time=now, - last_edit=now, - sender=sender, - kind=kind or 'message', - subkind=subkind, - format=format, - content=initial_content, - thread_id=thread_id - ) + if tmp.is_tmp_id(chat_id): + message = Message( + tmp.generate_tmp_id(self), + #chat_id=chat_id, + time=now, + #last_edit=now, + sender=sender, + kind=kind or 'message', + subkind=subkind, + #format=format, + content=initial_content, + thread_id=thread_id + ) + message.chat_id = chat_id + chat = self.tmp.chats[chat_id] + chat.messages.append(message) + chat_owner_id = chat.user_id + else: + message = await db.Message.create( + chat_id=chat_id, + time=now, + last_edit=now, + sender=sender, + kind=kind or 'message', + subkind=subkind, + format=format, + content=initial_content, + thread_id=thread_id + ) - # Fetch chat owner using values_list for efficiency - chat = await db.Chat.get(id=chat_id).prefetch_related("user") - chat_owner_id = chat.user_id + # Fetch chat owner using values_list for efficiency + chat = await db.Chat.get(id=chat_id).prefetch_related("user") + chat_owner_id = chat.user_id # Notification order depends on whether the content is a stream async def notify_agents(): # Fetch participant agents - agent_ids = await db.Membership.filter(chat_id=chat_id).values_list("agent_id", flat=True) + if tmp.is_tmp_id(chat_id): + agent_ids = self.tmp.chats[chat_id].participants + else: + agent_ids = await db.Membership.filter(chat_id=chat_id).values_list("agent_id", flat=True) + for agent_id in agent_ids: if str(agent_id) != str(sender): await self.notify_message_to_agent(agent_id, message) @@ -120,7 +173,8 @@ class ChatMixin: notify_users() async def end_fn (): message.content = content.content - await message.save() + if db.is_instance(message): + await message.save() await notify_agents() notify_users() @@ -142,7 +196,10 @@ class ChatMixin: } async def update_chat_message(self, message_id, content=None, kind=None, subkind=None, format=None): - message = await db.Message.get_or_none(id=message_id) + if tmp.is_tmp_id(message_id): + raise Exception("Cannot update temporary messages") + else: + message = await db.Message.get_or_none(id=message_id) if not message: raise Exception(f"Message not found with id {message_id}") @@ -161,7 +218,8 @@ class ChatMixin: summary.append('format') if summary: - await message.save() # Saves the specific instance + if db.is_instance(message): + await message.save() # Saves the specific instance print(f"Updated message #{message.id} fields: {summary}") async def get_chat_view(self, chat_id): diff --git a/backend/src/chat.py b/backend/src/chat.py index c83a333..e319d3c 100644 --- a/backend/src/chat.py +++ b/backend/src/chat.py @@ -3,17 +3,19 @@ from sqlalchemy.orm import Session from sqlalchemy.sql.expression import select import database as db +import temporary as tmp MESSAGE_PAGE_SIZE = 256 class Message: - def __init__(self, id, content, sender, time=None, kind='message', subkind=None): + def __init__(self, id, content, sender, time=None, kind='message', subkind=None, thread_id=None): self.id = id self.content = content self.sender = sender self.time = time self.kind = kind self.subkind = subkind + self.thread_id = thread_id def from_db_record(record): return Message(record.id, record.content, record.sender, record.time) @@ -23,6 +25,16 @@ class Message: def is_from_agent(self, agent_id): return self.sender == str(agent_id) + + def to_json(self): + return { + "id": self.id, + "time": self.time, + "sender": self.sender, + "kind": self.kind, + "content": self.content, + "threadId": self.thread_id, + } def __str__(self): time = self.time if self.time != None else 'no-time' @@ -67,14 +79,18 @@ class ChatView: self.current_thread_id = None self._earliest_message_id = None + self._earliest_message_time = None async def init_load(self): await self._load_basic_data() await self._load_next_messages(MESSAGE_PAGE_SIZE) async def _load_basic_data(self): - # Using get_or_none for cleaner existence checks - record = await db.Chat.get_or_none(id=self.chat_id) + if tmp.is_tmp_id(self.chat_id): + record = self.app.tmp.chats.get(self.chat_id) + else: + # Using get_or_none for cleaner existence checks + record = await db.Chat.get_or_none(id=self.chat_id) if not record: raise Exception(f"Chat not found with id {self.chat_id}") @@ -83,21 +99,32 @@ class ChatView: self.description = record.description async def _load_next_messages(self, load_count): - # Start building the filter - filters = {"chat_id": self.chat_id} - - # Add the 'earliest message' condition if it exists - if self._earliest_message_id is not None: - filters["id__lte"] = self._earliest_message_id + if tmp.is_tmp_id(self.chat_id): + chat = self.app.tmp.chats[self.chat_id] + record_batch = [ + msg for msg in chat.messages + if not self._earliest_message_time or msg.id < self._earliest_message_time + ][:load_count] + record_batch.reverse() + records_rev = record_batch + else: + # Start building the filter + filters = {"chat_id": self.chat_id} + + # Add the 'earliest message' condition if it exists + if self._earliest_message_id is not None: + filters["id__lte"] = self._earliest_message_id - # Tortoise query: filter, order, and limit - # ordering by id is more robust than by time - records_rev = await db.Message.filter(**filters).order_by("-id").limit(load_count) + # Tortoise query: filter, order, and limit + # ordering by id is more robust than by time + records_rev = await db.Message.filter(**filters).order_by("-id").limit(load_count) # Update the earliest id for the next load operation if records_rev: - # Note: Since we ordered by -time (descending), index 0 is the newest in this batch - self._earliest_message_id = records_rev[-1].id # The "earliest" is the one with the smallest ID/Time in this set + # Note: Since we ordered by -time (descending), index 0 is the newest in this batch (highest id/time) + # The "earliest" or oldest is the one with the smallest ID/Time in this set + self._earliest_message_id = records_rev[-1].id + self._earliest_message_time = records_rev[-1].time if not self.threads: # current thread is the latest known from the newest record in batch diff --git a/backend/src/database.py b/backend/src/database.py index 6ba7471..4cd626b 100644 --- a/backend/src/database.py +++ b/backend/src/database.py @@ -29,6 +29,10 @@ async def init_database (data_directory): ) await Tortoise.generate_schemas() +# Checks whether an object is an instance of any database model +def is_instance (value): + isinstance(value, models.Model) + class User(models.Model): id = fields.IntField(pk=True) email = fields.CharField(max_length=255) diff --git a/backend/src/inference.py b/backend/src/inference.py index c2d280b..a1815f9 100644 --- a/backend/src/inference.py +++ b/backend/src/inference.py @@ -1,4 +1,5 @@ import threading +import asyncio import ollama from stream import Stream @@ -23,22 +24,33 @@ class Models: # llama cpp models model_cache = {} +last_model_file = None +last_model_instance = None + +def get_llama_cpp_model(app, model_file): + global last_model_file, last_model_instance + if last_model_file == model_file: + return last_model_instance + + if last_model_instance is not None: + #last_model_instance = None + del last_model_instance + + last_model_file = model_file + last_model_instance = Llama( + model_path=f"{app.data_directory}/models/{model_file}", + n_gpu_layers=-1, # Uncomment to use GPU acceleration + # seed=1337, # Uncomment to set a specific seed + n_ctx=CONTEXT_WINDOW, + ) + return last_model_instance # llama.cpp async def local_chat_completion(app, messages, model, opts={}): content_stream = Stream(app) def worker(): - if model in model_cache: - llm = model_cache[model] - else: - llm = Llama( - model_path=f"{app.data_directory}/models/{model}", - n_gpu_layers=-1, # Uncomment to use GPU acceleration - # seed=1337, # Uncomment to set a specific seed - n_ctx=CONTEXT_WINDOW, - ) - model_cache[model] = llm + llm = get_llama_cpp_model(app, model) temp = opts.get("temperature") if temp is None: @@ -77,23 +89,23 @@ async def ollama_list(app, messages, model, opts={}): async def ollama_chat_completion(app, messages, model, opts={}): content_stream = Stream(app) - print("START OLLAMA INFERENCE") - stream_response = await ollama.AsyncClient().chat( - model=model, - messages=[{'role': msg.role, 'content': msg.content} for msg in messages], - stream=True, - ) + async def worker(): + print("START OLLAMA INFERENCE") + stream_response = await ollama.AsyncClient().chat( + model=model, + messages=[{'role': msg.role, 'content': msg.content} for msg in messages], + stream=True, + ) - async for part in stream_response: - content_stream.feed(part.message.content) - print(part['message']['content'], end='', flush=True) + async for part in stream_response: + content_stream.feed(part.message.content) - content_stream.close() + content_stream.close() + print("END OLLAMA INFERENCE") + + asyncio.create_task(worker()) - print("END OLLAMA INFERENCE") - return await content_stream.content_wait() - - #return content_stream + return content_stream async def replicate_completion(app, messages, model=None, opts={}): @@ -104,7 +116,7 @@ async def replicate_completion(app, messages, model=None, opts={}): arguments = {} arguments[parameter_names.get('prompt', 'prompt')] = format_instruct(messages) - arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature', 0.75) + arguments[parameter_names.get('temperature', 'temperature')] = opts.get('temperature') or 0.75 arguments[parameter_names.get('max_tokens', 'max_tokens')] = opts.get('max_tokens', 2048) arguments[parameter_names.get('system_prompt', 'system_prompt')] = "" diff --git a/backend/src/server.py b/backend/src/server.py index 4a75356..723e425 100644 --- a/backend/src/server.py +++ b/backend/src/server.py @@ -162,7 +162,8 @@ def start_server(app): description=body.get('description'), picture=body.get('picture'), kind=body['kind'], - data=body.get('data') + data=body.get('data'), + temporary=body.get('temporary'), ) await send({'message': "agent created", 'agentId': agent_id, 'nonce': nonce}) @@ -216,6 +217,7 @@ def start_server(app): description=body.get('description'), picture=body.get('picture'), members=body.get('members', []), + temporary=body.get('temporary'), ) await send({'message': "chat created", 'chatId': chat_id, 'nonce': nonce}) diff --git a/backend/src/temporary.py b/backend/src/temporary.py new file mode 100644 index 0000000..4fc1fbf --- /dev/null +++ b/backend/src/temporary.py @@ -0,0 +1,87 @@ + +def generate_tmp_id(app): + app.tmp.id_counter += 1 + return "tmp-" + str(app.tmp.id_counter) + +def is_tmp_id(id): + return type(id) == str and id.startswith("tmp-") + +class State: + def __init__(self): + self.id_counter = 0 + self.agents = {} + self.chats = {} + self.threads = {} + + # list of temporary agent ids that belong to a real user + self.user_agents = {} + # list of temporary chat ids that belong to a real user + self.user_chats = {} + # list of temporary chat ids where a real agent is participating + self.participating = {} + + def get_user_agents(self, user_id): + if user_id not in self.user_agents: + self.user_agents[user_id] = [] + return self.user_agents[user_id] + + def get_user_chats(self, user_id): + if user_id not in self.user_chats: + self.user_chats[user_id] = [] + return self.user_chats[user_id] + +class Agent: + def create(app, user_id, name, agentname, main_chat_id, description="", picture=None, kind='default', data={}): + self = Agent() + + self.id = None + self.user_id = user_id + self.name = name + self.agentname = agentname + self.main_chat_id = main_chat_id + self.description = description + self.picture = picture + self.kind = kind + self.data = data + + self.id = generate_tmp_id(app) + app.tmp.agents[self.id] = self + app.tmp.get_user_agents(user_id).append(self.id) + + return self + +class Chat: + def create(app, user_id, name, description="", picture=None, ordering=None): + self = Chat() + self._app = app + + self.id = None + self.user_id = user_id + self.name = name + self.description = description + self.picture = picture + self.ordering = ordering + + self.messages = [] + self.threads = [] + self.participants = [] + + self.id = generate_tmp_id(app) + app.tmp.chats[self.id] = self + app.tmp.get_user_chats(user_id).append(self.id) + + return self + +class Thread: + def create(app, chat_id, name): + self = Thread() + + self.chat_id = chat_id + self.name = name + self.id = generate_tmp_id(app) + + app.tmp.threads[self.id] = self + app.tmp.chats[chat_id].threads.append(self.id) + + return self + \ No newline at end of file diff --git a/frontend/src/views/AgentCreate.vue b/frontend/src/views/AgentCreate.vue index a46620f..5dd0ddb 100644 --- a/frontend/src/views/AgentCreate.vue +++ b/frontend/src/views/AgentCreate.vue @@ -32,6 +32,10 @@ payload.identifier = agent.value.identifier; } + if (agent.value.temporary) { + payload.temporary = true; + } + let response = await backend.sendAction("create-agent", payload); if (response.error) { @@ -54,39 +58,54 @@