diff --git a/AGENTS.md b/AGENTS.md index 1ba197d1..3d696952 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,6 +54,10 @@ hollo/ │ │ ├── v1/ # API v1 endpoints │ │ └── v2/ # API v2 endpoints (search, notifications) │ │ +│ ├── cleanup/ # Cleanup (of old data) +│ │ ├── processors.ts # Cleanup item processors +│ │ └── worker.ts # Background job worker +│ │ │ ├── components/ # Hono JSX components (server-rendered) │ │ │ ├── entities/ # Entity serialization (DB → API response) @@ -111,6 +115,7 @@ Key architectural components responses - *Pages* (*src/pages/*): Web UI pages (profile, setup, dashboard, etc.) - *Import system* (*src/import/*): Background job processing for data imports + - *Cleanup system* (*src/cleanup/*): Background job processing for cleanup actions ### Key files diff --git a/bin/server.ts b/bin/server.ts index 45b7577e..7f194641 100644 --- a/bin/server.ts +++ b/bin/server.ts @@ -65,14 +65,21 @@ if (NODE_TYPE === "web" || NODE_TYPE === "all") { } // Start workers if running as worker or all node -let stopWorker: (() => void) | undefined; +let stopWorkers: (() => void) | undefined; if (NODE_TYPE === "worker" || NODE_TYPE === "all") { - const [{ federation }, { startImportWorker, stopImportWorker }] = - await Promise.all([ - import("../src/federation"), - import("../src/import/worker"), - ]); - stopWorker = stopImportWorker; + const [ + { federation }, + { startImportWorker, stopImportWorker }, + { startCleanupWorker, stopCleanupWorker }, + ] = await Promise.all([ + import("../src/federation"), + import("../src/import/worker"), + import("../src/cleanup/worker"), + ]); + stopWorkers = () => { + stopImportWorker(); + stopCleanupWorker(); + }; // Start the Fedify message queue const controller = new AbortController(); @@ -83,17 +90,18 @@ if (NODE_TYPE === "worker" || NODE_TYPE === "all") { process.exit(1); }); - // Start the import worker for background job processing + // Start the workers for background job processing startImportWorker(); + startCleanupWorker(); - console.log("Worker started (Fedify queue + Import worker)"); + console.log("Worker started (Fedify queue + Import worker + Cleanup worker)"); } // Graceful shutdown handling const shutdown = () => { if (NODE_TYPE === "worker" || NODE_TYPE === "all") { console.log("Stopping workers..."); - stopWorker?.(); + stopWorkers?.(); } process.exit(0); }; diff --git a/docs/src/content/docs/install/env.mdx b/docs/src/content/docs/install/env.mdx index c4b802cf..bd4ce4af 100644 --- a/docs/src/content/docs/install/env.mdx +++ b/docs/src/content/docs/install/env.mdx @@ -46,9 +46,9 @@ e.g., `UTC`, `America/New_York`, `Asia/Tokyo`. Controls which components run in this process. Valid values are: - - `all` (default): Run web server, Fedify message queue, and import worker + - `all` (default): Run web server, Fedify message queue, import worker and cleanup worker - `web`: Run only the web server (HTTP API) - - `worker`: Run only workers (Fedify message queue + import worker) + - `worker`: Run only workers (Fedify message queue + import worker + cleanup worker) This allows separating the web server from background workers for better scalability. When running high-traffic instances with many followers, diff --git a/docs/src/content/docs/install/workers.mdx b/docs/src/content/docs/install/workers.mdx index fbb12471..40bd4671 100644 --- a/docs/src/content/docs/install/workers.mdx +++ b/docs/src/content/docs/install/workers.mdx @@ -35,15 +35,16 @@ Hollo has three main components: 1. *Web server*: Handles HTTP requests (API, web UI) 2. *Fedify message queue*: Processes ActivityPub inbox/outbox messages 3. *Import worker*: Handles background data import jobs + 4. *Cleanup worker*: Handles background cleanup jobs By default (`NODE_TYPE=all`), all three run in a single process. You can separate them using the `NODE_TYPE` environment variable: -| `NODE_TYPE` | Web server | Fedify queue | Import worker | -| ----------- | ---------- | ------------ | ------------- | -| `all` (default) | ✓ | ✓ | ✓ | -| `web` | ✓ | ✗ | ✗ | -| `worker` | ✗ | ✓ | ✓ | +| `NODE_TYPE` | Web server | Fedify queue | Import worker | Cleanup worker | +| ----------- | ---------- | ------------ | ------------- | -------------- | +| `all` (default) | ✓ | ✓ | ✓ | ✓ | +| `web` | ✓ | ✗ | ✗ | ✗ | +| `worker` | ✗ | ✓ | ✓ | ✓ | All nodes share the same PostgreSQL database, which acts as the message queue backend using `LISTEN`/`NOTIFY` for real-time message delivery. @@ -159,7 +160,8 @@ NODE_TYPE=worker pnpm prod pnpm worker ``` -This starts the Fedify message queue and import worker without the web server. +This starts the Fedify message queue, import worker and cleanup worker without +the web server.