diff --git a/backend/src/app.py b/backend/src/app.py index a0a03d5..d35359f 100644 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -18,6 +18,7 @@ import agent as agent_mod import agents as agents_mod import temporary as tmp from server import start_server +from drives import init_drives from chat import ChatView, Message, MESSAGE_PAGE_SIZE from remote import RemoteConnection, RemoteToolSpec, parse_function_specs @@ -225,6 +226,8 @@ async def main(): # 1. Initialize Database First await db.init_database(data_directory) print("Database initialized with Tortoise ORM.") + # Drives as well + await init_drives(data_directory) # 2. Initialize App horde_app = App( diff --git a/backend/src/database.py b/backend/src/database.py index 3418151..62f8cf0 100644 --- a/backend/src/database.py +++ b/backend/src/database.py @@ -52,7 +52,7 @@ def is_instance (value): class User(models.Model): id = fields.IntField(pk=True) email = fields.CharField(max_length=255) - username = fields.CharField(max_length=100) + username = fields.CharField(max_length=100, unique=True) password = fields.CharField(max_length=255) class Meta: @@ -165,4 +165,24 @@ class Model(models.Model): #user = fields.ForeignKeyField("models.User", null=True) name = fields.CharField(max_length=255) kind = fields.CharField(max_length=50) - value = fields.JSONField(null=True) \ No newline at end of file + value = fields.JSONField(null=True) + +class Drive(models.Model): + id = fields.IntField(pk=True) + user = fields.ForeignKeyField("models.User", related_name="drives") + name = fields.CharField(max_length=255) + created_at = fields.DatetimeField(auto_now_add=True) + +class DriveFile(models.Model): + id = fields.CharField(pk=True, max_length=64) + drive = fields.ForeignKeyField("models.Drive", related_name="files") + + filename = fields.CharField(max_length=500) # Readable app-side name + mime_type = fields.CharField(max_length=255) # Content type + size_bytes = fields.BigIntField() # Physical size + + accessed_at = fields.DatetimeField(null=True) # Last access timestamp + created_at = fields.DatetimeField(auto_now_add=True) + + class Meta: + table = "drive_files" \ No newline at end of file diff --git a/backend/src/drives.py b/backend/src/drives.py new file mode 100644 index 0000000..5fefde5 --- /dev/null +++ b/backend/src/drives.py @@ -0,0 +1,103 @@ +import os +from pathlib import Path +import secrets +import datetime +import database as db + +# TODO: Ugly ass vibe coded shit. Someone please rewrite. + +# TODO: This belonds in app +DRIVES_ROOT = None + +async def init_drives(data_directory): + """Initialize DRIVES_ROOT global from app startup.""" + global DRIVES_ROOT + DRIVES_ROOT = os.path.join(data_directory, "drives") + os.makedirs(DRIVES_ROOT, exist_ok=True) + +def _generate_file_id(): + """Generate a random hex string for filesystem-level filenames.""" + return secrets.token_hex(8) + +async def upload_file(user_id, drive_name, filename, stream, mime_type=None): + """Write an incoming stream to disk and register it in the DB. Returns file ID.""" + # 1. Locate or create Drive + drive = await db.Drive.get_or_none(user_id=user_id, name=drive_name) + if not drive: + drive = await db.Drive.create(user_id=user_id, name=drive_name) + + # Check collision + if await db.Drive.get_or_none(drive_id=drive.id, filename=filename): + raise Exception("Duplicated file") + + # 2. Prepare disk path + file_id = _generate_file_id() + drive_dir = os.path.join(DRIVES_ROOT, str(drive.id)) + os.makedirs(drive_dir, exist_ok=True) + real_path = os.path.join(drive_dir, file_id) + + # 3. Write stream and calculate size + size = 0 + with open(real_path, "wb") as f: + while chunk := stream.read(8192): + f.write(chunk) + size += len(chunk) + + # 4. Register in database + # TODO: catch id conflict and retry + record = await db.DriveFile.create( + id=file_id, + drive_id=drive.id, + filename=filename, + mime_type=mime_type or "application/octet-stream", + size_bytes=size, + ) + return record.id + +async def get_metadata(file_id): + """Return JSON metadata for frontend display.""" + file = await db.DriveFile.get_or_none(id=file_id) + if not file: + raise Exception("File not found") + return { + "id": file.id, + "name": file.filename, + "size": file.size_bytes, + "mime": file.mime_type, + "created": file.created_at.isoformat(), + "accessed": file.accessed_at.isoformat() if file.accessed_at else None + } + +async def delete_file(file_id): + file = await db.DriveFile.get_or_none(id=file_id) + if not file: + raise Exception("File not found") + + await file.delete() + + abs_path = os.path.join(DRIVES_ROOT, str(file.drive_id), file.id) + if os.path.exists(abs_path): + os.remove(abs_path) + + return abs_path + +async def resolve_file(user_id, drive_name, filename): + """Resolve file to absolute disk path after permission check. Returns (path, mime).""" + # Eager load drive and verify ownership + drive = await db.Drive.get_or_none(user_id=user_id, name=drive_name) + if drive.user_id != user_id: + raise PermissionError("Access denied") + + file = await db.DriveFile.get_or_none(filename=filename, drive_id=drive.id) + if not file: + raise FileNotFoundError("File not found") + + # can do this asynchronously? + file.accessed_at = datetime.datetime.now(datetime.UTC) + await file.save() + + abs_path = os.path.join(DRIVES_ROOT, str(drive.id), file.id) + if not os.path.exists(abs_path): + raise FileNotFoundError("File missing on disk") + + return abs_path, file.id \ No newline at end of file diff --git a/backend/src/migrations/0002_auto_20260921_1840.py b/backend/src/migrations/0002_auto_20260921_1840.py new file mode 100644 index 0000000..b139037 --- /dev/null +++ b/backend/src/migrations/0002_auto_20260921_1840.py @@ -0,0 +1,42 @@ +from tortoise import migrations +from tortoise.migrations import operations as ops +from tortoise.fields.base import OnDelete +from tortoise import fields + +class Migration(migrations.Migration): + dependencies = [('models', '0001_initial')] + + initial = False + + operations = [ + ops.CreateModel( + name='Drive', + fields=[ + ('id', fields.IntField(generated=True, primary_key=True, unique=True, db_index=True)), + ('user', fields.ForeignKeyField('models.User', source_field='user_id', db_constraint=True, to_field='id', related_name='drives', on_delete=OnDelete.CASCADE)), + ('name', fields.CharField(max_length=255)), + ('created_at', fields.DatetimeField(auto_now=False, auto_now_add=True)), + ], + options={'table': 'drive', 'app': 'models', 'pk_attr': 'id'}, + bases=['Model'], + ), + ops.CreateModel( + name='DriveFile', + fields=[ + ('id', fields.CharField(primary_key=True, unique=True, db_index=True, max_length=64)), + ('drive', fields.ForeignKeyField('models.Drive', source_field='drive_id', db_constraint=True, to_field='id', related_name='files', on_delete=OnDelete.CASCADE)), + ('filename', fields.CharField(max_length=500)), + ('mime_type', fields.CharField(max_length=255)), + ('size_bytes', fields.BigIntField()), + ('accessed_at', fields.DatetimeField(null=True, auto_now=False, auto_now_add=False)), + ('created_at', fields.DatetimeField(auto_now=False, auto_now_add=True)), + ], + options={'table': 'drive_files', 'app': 'models', 'pk_attr': 'id'}, + bases=['Model'], + ), + ops.AlterField( + model_name='User', + name='username', + field=fields.CharField(unique=True, max_length=100), + ), + ] diff --git a/backend/src/server/__init__.py b/backend/src/server/__init__.py index 6288b40..ecea414 100644 --- a/backend/src/server/__init__.py +++ b/backend/src/server/__init__.py @@ -1 +1 @@ -from server.start import start_server \ No newline at end of file +from server.web import start_server \ No newline at end of file diff --git a/backend/src/server/start.py b/backend/src/server/start.py deleted file mode 100644 index 1bd813e..0000000 --- a/backend/src/server/start.py +++ /dev/null @@ -1,90 +0,0 @@ -from quart import Quart, request, send_from_directory -from quart_auth import QuartAuth, AuthUser, login_user -from server.websocket import register as register_websocket -from server.openai import register as register_openai - -# Vue frontend -# relative to the module file???? Quart being unserious -frontend_path = '../../../frontend/dist' - -async def start_server(app, port, shutdown_event): - server = Quart(__name__, static_folder=frontend_path) - server.secret_key = 'horde-secret' - auth_manager = QuartAuth(server, mode="bearer") - - stream_listeners = {} - - def cleanup(self): - stream_listeners = [] - - @server.after_request - def after_request(response): - response.headers.add('Access-Control-Allow-Origin', '*') - response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') - response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') - return response - - @server.route('/') - async def index(): - return await send_from_directory(server.static_folder, 'index.html') - - # These paths must match the routes in the frontend SPA - # file: frontend/src/routes/index.js - @server.route('/user') - @server.route('/models') - @server.route('/remotes') - @server.route('/runs') - @server.route('/chat') - @server.route('/agent') - @server.route('/chat/') - @server.route('/agent/') - async def serve_spa_routes(subpath=None): - # Ignore subpath and render index, Vue router will take care of everything - return await send_from_directory(server.static_folder, 'index.html') - - @server.route('/') - async def serve_static(path): - return await send_from_directory(server.static_folder, path) - - @server.route('/user/login', methods=['POST']) - async def login(): - data = await request.get_json() - - # token has precedence if it's present - if "token" in data: - user_id = auth_manager.load_token(data['token']) - elif "username" in data and "password" in data: - user_id = await app.verify_user_credentials(data['username'], data['password']) - else: - return "invalid login arguments", 400 - - if user_id == None: - return "bad login credentials", 401 - - token = auth_manager.dump_token(user_id) - login_user(AuthUser(auth_id=user_id)) - return {"token": token} - - @server.route('/user/signup', methods=['POST']) - async def signup(): - data = await request.get_json() - - user_id = None - - username = data['username'] - email = data['email'] - password = data['password'] - - user_id = await app.create_new_user(username, email, password) - if user_id is None: - return "existing or incorrect credentials", 400 - - await app.generate_default_user_data(user_id) - - token = auth_manager.dump_token(user_id) - return {"token": token} - - register_websocket(server, app, auth_manager) - register_openai(server, app) - - await server.run_task(debug=True, shutdown_trigger=shutdown_event.wait, port=port) \ No newline at end of file diff --git a/backend/src/server/web.py b/backend/src/server/web.py new file mode 100644 index 0000000..96a7177 --- /dev/null +++ b/backend/src/server/web.py @@ -0,0 +1,179 @@ +from quart import Quart, request, send_from_directory, send_file, current_app +from quart_auth import QuartAuth, AuthUser, login_required, login_user, current_user +from server.websocket import register as register_websocket +from server.openai import register as register_openai + +import database as db +import drives + +from functools import wraps + +# Vue frontend +# relative to the module file???? Quart being unserious +frontend_path = '../../../frontend/dist' + +async def start_server(app, port, shutdown_event): + server = Quart(__name__, static_folder=frontend_path) + server.secret_key = 'horde-secret' + auth_manager = QuartAuth(server, mode="bearer") + + stream_listeners = {} + + def query_auth(func): + """A decorator to login with token. + + Wrap the login_required decorator to inject a current user based + on the request query parameters. + """ + + @wraps(func) + async def wrapper(*args, **kwargs): + user_id = auth_manager.load_token(request.args.get('token', '')) + if user_id: + login_user(AuthUser(auth_id=user_id)) + return await current_app.ensure_async(func)(*args, **kwargs) + + return wrapper + + @server.after_request + def after_request(response): + response.headers.add('Access-Control-Allow-Origin', '*') + response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') + response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') + return response + + @server.route('/') + async def index(): + return await send_from_directory(server.static_folder, 'index.html') + + # These paths must match the routes in the frontend SPA + # file: frontend/src/routes/index.js + @server.route('/user') + @server.route('/models') + @server.route('/remotes') + @server.route('/drives') + @server.route('/runs') + @server.route('/chat') + @server.route('/agent') + @server.route('/chat/') + @server.route('/agent/') + async def serve_spa_routes(subpath=None): + # Ignore subpath and render index, Vue router will take care of everything + return await send_from_directory(server.static_folder, 'index.html') + + @server.route('/') + async def serve_static(path): + return await send_from_directory(server.static_folder, path) + + @server.route('/user/login', methods=['POST']) + async def login(): + data = await request.get_json() + + # token has precedence if it's present + if "token" in data: + user_id = auth_manager.load_token(data['token']) + elif "username" in data and "password" in data: + user_id = await app.verify_user_credentials(data['username'], data['password']) + else: + return "invalid login arguments", 400 + + if user_id == None: + return "bad login credentials", 401 + + token = auth_manager.dump_token(user_id) + login_user(AuthUser(auth_id=user_id)) + return {"token": token} + + @server.route('/user/signup', methods=['POST']) + async def signup(): + data = await request.get_json() + + user_id = None + + username = data['username'] + email = data['email'] + password = data['password'] + + user_id = await app.create_new_user(username, email, password) + if user_id is None: + return "existing or incorrect credentials", 400 + + await app.generate_default_user_data(user_id) + + token = auth_manager.dump_token(user_id) + return {"token": token} + + # === Drives === # + + # this, too, is kinda vibe coded + # I need drives, ok? + + @server.route('/files', methods=['GET']) + @login_required + async def list_drives(): + drive_list = await db.Drive.filter(user_id=current_user.auth_id) + return [{"id": d.id, "name": d.name} for d in drive_list] + + @server.route('/files/', methods=['GET']) + @login_required + async def list_files(drive_name): + files = await db.DriveFile.filter(drive__user_id=current_user.auth_id, drive__name=drive_name) + return [{ + "id": f.id, + "name": f.filename, + "size": f.size_bytes, + "mime": f.mime_type, + "created": f.created_at.isoformat(), + } for f in files] + + @server.route('/files//', methods=['POST']) + @login_required + async def upload_file(drive_name, filename=None): + # whole file in one request, no partial transmission + upload = (await request.files).get('file') + if upload is None: + return "missing file part", 400 + file_id = await drives.upload_file( + current_user.auth_id, + drive_name, + filename or upload.filename, + upload.stream, + upload.content_type, + ) + return {"fileId": file_id} + + @server.route('/files//', methods=['GET']) + @query_auth + @login_required + async def serve_file(drive_name, filename): + try: + user_id = current_user.auth_id + file_path, file_id = await drives.resolve_file(user_id, drive_name, filename) + metadata = await drives.get_metadata(file_id) + if 'metadata' in request.args: + return metadata + else: + return await send_file(file_path, mimetype=metadata["mime"]) + except FileNotFoundError: + return "not found", 404 + except PermissionError: + return "forbidden", 403 + + @server.route('/files//', methods=['DELETE']) + @query_auth + @login_required + async def delete_file(drive_name, filename): + try: + user_id = current_user.auth_id + file_path, file_id = await drives.resolve_file(user_id, drive_name, filename) + metadata = await drives.delete_file(file_id) + return {"fileId": file_id} + except FileNotFoundError: + return "not found", 404 + except PermissionError: + return "forbidden", 403 + + register_websocket(server, app, auth_manager) + register_openai(server, app) + + await server.run_task(debug=True, shutdown_trigger=shutdown_event.wait, port=port) \ No newline at end of file diff --git a/docs/drives-plan.md b/docs/drives-plan.md new file mode 100644 index 0000000..fb24b68 --- /dev/null +++ b/docs/drives-plan.md @@ -0,0 +1,125 @@ +# Phase 3: Server Integration (REST) + +## Objective +Attach drive operations to `backend/src/server/web.py` using standard web routes only. This supersedes the WebSocket-based protocol from `drives-report.md`: no WS actions (`list-drives`, `list-files`, `upload-file`), no chunked transfer. Upstream transfer is a single `multipart/form-data` POST per file. + +Assumes `drive_service` from the earlier phases provides `init_drives`, `upload_file`, `get_drive_file_metadata` and `resolve_drive_file_for_serving`. + +## Routes + +| Route | Method | Description | +|-------|--------|-------------| +| `/drives` | GET | List drives of the authenticated user. | +| `/drives//files` | GET | List file metadata for one drive. | +| `/drives//files` | POST | Upload one file (multipart, field `file`). Creates the drive if missing. | +| `/files/` | GET | Raw download. | +| `/files//meta` | GET | Metadata for existence checks. | + +### Auth +Reuse the existing `QuartAuth(server, mode="bearer")` setup: +- JSON endpoints use `@login_required` and read the user id from `current_user.auth_id`. +- `GET /files/` additionally accepts `?token=`, because `` links cannot set an `Authorization` header. + +## Implementation + +### 3.1 Imports + +Top of `backend/src/server/web.py`: + +```python +import database as db +import drive_service +from quart import Quart, request, send_from_directory, send_file +from quart_auth import QuartAuth, AuthUser, login_user, login_required, current_user +``` + +### 3.2 Route handlers + +Registered inside `start_server`, alongside the existing routes: + +```python +@server.route('/drives', methods=['GET']) +@login_required +async def list_drives(): + drives = await db.Drive.filter(user_id=current_user.auth_id) + return [{"id": d.id, "name": d.name} for d in drives] + +@server.route('/drives//files', methods=['GET']) +@login_required +async def list_files(drive_name): + files = await db.DriveFile.filter(drive__user_id=current_user.auth_id, drive__name=drive_name) + return [{ + "id": f.id, + "name": f.filename, + "size": f.size_bytes, + "mime": f.mime_type, + "created": f.created_at.isoformat(), + } for f in files] + +@server.route('/drives//files', methods=['POST']) +@login_required +async def upload_file(drive_name): +# whole file in one request, no partial transmission + upload = (await request.files).get('file') + if upload is None: + return "missing file part", 400 + file_id = await drive_service.upload_file( + current_user.auth_id, drive_name, upload.filename, upload.stream, upload.content_type, + ) + return {"fileId": file_id} + +@server.route('/files/', methods=['GET']) +async def serve_file(file_id): +# query token instead of @login_required: anchor downloads can't set headers + user_id = auth_manager.load_token(request.args.get('token', '')) + if user_id is None: + return "unauthorized", 401 + try: + file_path, mime_type = await drive_service.resolve_drive_file_for_serving(file_id, user_id) + return await send_file(file_path, mimetype=mime_type) + except FileNotFoundError: + return "not found", 404 + except PermissionError: + return "forbidden", 403 + +@server.route('/files//meta', methods=['GET']) +@login_required +async def get_file_meta(file_id): + try: +# resolve performs the ownership check before anything is returned + await drive_service.resolve_drive_file_for_serving(file_id, current_user.auth_id) + return await drive_service.get_drive_file_metadata(file_id) + except FileNotFoundError: + return "not found", 404 + except PermissionError: + return "forbidden", 403 +``` + +No changes to `server/websocket.py`. + +## Frontend protocol + +```javascript +// listing +const drives = await fetch('/drives', {headers: {Authorization: `Bearer ${token}`}}).then(r => r.json()); +const files = await fetch(`/drives/${encodeURIComponent(name)}/files`, {headers: {Authorization: `Bearer ${token}`}}).then(r => r.json()); + +// upload: single POST, no chunking +const form = new FormData(); +form.append('file', blob, 'report.pdf'); +await fetch(`/drives/${encodeURIComponent(driveName)}/files`, { + method: 'POST', + headers: {Authorization: `Bearer ${token}`}, + body: form, +}); + +// download: plain anchor +// Download +``` + +## Notes + +- Routing is unaffected by the static catch-all: werkzeug prefers the exact/typed routes above over `/`. +- `/drives` is now an API route; the SPA must not mount a view at that exact path. +- `drive_name` is a single URL segment (no `/`). The service layer should reject names containing path separators before they reach `os.path.join`. +- The query-token on the download endpoint trades header auth for `` compatibility (token visible in history/logs). Switch to fetch + blob or cookie auth if that's unacceptable. \ No newline at end of file diff --git a/frontend/src/backend.js b/frontend/src/backend.js index bc1283c..95b6a04 100644 --- a/frontend/src/backend.js +++ b/frontend/src/backend.js @@ -137,6 +137,23 @@ export async function newToken () { // --- Communication --- +export async function webFetch (path, options = {}) { + const tk = loginToken || await loginTokenDeferred.promise; + const response = await fetch(`${backendUrl}${path}`, { + ...options, + headers: { + Authorization: `Bearer ${tk}`, + ...options.headers, + }, + }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Response Error: ${response.status}\n${errorText}`); + } + // TODO: Check content-type before parsing + return response.json(); +} + export function sendAction(action, body = {}, callback = null) { if (socket?.readyState !== WebSocket.OPEN) { return Promise.reject(new Error("WebSocket is not open. Cannot send action.")); @@ -358,6 +375,39 @@ export function invalidateCache({ chats=false, agents=false } = {}) { } +// --- Drive Storage (REST) --- +// Drive endpoints are plain HTTP routes instead of WebSocket actions, +// so they authenticate with the bearer token in the Authorization header. + +export function getDrives () { + return webFetch('/files'); +} + +export function getDriveFiles (driveName) { + return webFetch(`/files/${driveName}`); +} + +export function uploadDriveFile (driveName, file) { + const body = new FormData(); + body.append('file', file, file.name); + return webFetch(`/files/${driveName}`, { + method: 'POST', + body, + }); +} + +// Anchor downloads can't set headers, so the token travels as a query parameter +export function driveFileUrl (driveName, filename) { + return `${backendUrl}/files/${driveName}/${filename}?token=${loginToken}`; +} + +export function deleteDriveFile (driveName, filename) { + const body = new FormData(); + return webFetch(`/files/${driveName}/${filename}`, { + method: 'DELETE', + }); +} + // --- Connection --- diff --git a/frontend/src/components/main/Sidebar.vue b/frontend/src/components/main/Sidebar.vue index 322bfed..3898f47 100644 --- a/frontend/src/components/main/Sidebar.vue +++ b/frontend/src/components/main/Sidebar.vue @@ -117,6 +117,7 @@ User Manage Models Remotes + Drives Agent Runs diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 20e641c..af5dde5 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -61,6 +61,11 @@ const router = createRouter({ path: `/remotes`, component: () => import('../views/Remotes.vue') }, + { + name: 'drives', + path: `/drives`, + component: () => import('../views/Drives.vue') + }, objectView('chat', () => import('../views/Chat.vue')), objectView('agent', () => import('../views/Agent.vue')), staticView('user', () => import('../views/User.vue')), diff --git a/frontend/src/views/Drives.vue b/frontend/src/views/Drives.vue new file mode 100644 index 0000000..cf0c47e --- /dev/null +++ b/frontend/src/views/Drives.vue @@ -0,0 +1,250 @@ + + + + + \ No newline at end of file