[on hold] adrian/auto-ai-image-tagging-2026-04-15

ai-context/core-features-reference.md

@@ -48,11 +48,41 @@ This document provides the correct import paths and type definitions for commonl
 - **Interface Type:** See `packages/api-core/src/features/settings/UpdateSettings/abstractions.ts`
 - **Usage:** Create or update settings records
 
+### Ai
+
+- **Import:** `import { Ai } from "@webiny/api-core/features/ai/index.js"`
+- **Interface Type:** See `packages/api-core/src/features/ai/abstractions.ts`
+- **Usage:** Generate text and stream text using registered AI providers. Model format: `"provider/modelId"` (e.g. `"anthropic/claude-3-5-sonnet-20241022"`, `"openai/gpt-4o"`). Providers: `anthropic` (env: `WEBINY_API_ANTHROPIC_API_KEY`), `openai` (env: `WEBINY_API_OPENAI_API_KEY`). Must register `AiFeature` from `@webiny/api-core/features/ai/index.js`.
+
 ### AiGateway
 
-- **Import:** `import { AiGateway } from "@webiny/api-core/exports/api"`
-- **Interface Type:** See `packages/api-core/src/features/aiGateway/abstractions.ts`
-- **Usage:** Obtain a configured `LanguageModel` (from the `ai` SDK) for the active provider. Provider/token/model are read from env vars (`WEBINY_API_AI_GATEWAY_PROVIDER`, `WEBINY_API_AI_GATEWAY_TOKEN`, `WEBINY_API_AI_GATEWAY_MODEL`). Call `aiGateway.getLanguageModel(modelId?)` and pass the result to `generateText`, `streamText`, etc. from the `ai` package.
+- **Import:** `import { AiGateway } from "@webiny/api-core/features/ai/index.js"`
+- **Interface Type:** See `packages/api-core/src/features/ai/abstractions.ts`
+- **Usage:** Routes `"provider/modelId"` strings to registered providers. Used internally by `Ai`.
+
+### TaskDefinition
+
+- **Import:** `import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js"`
+- **Interface Type:** See `packages/api-core/src/features/task/TaskDefinition/abstractions.ts`
+- **Usage:** Define background tasks. Use `TaskDefinition.createImplementation({ implementation, dependencies })`. Register with `context.container.register(MyTask)`. The `run` method receives `{ input, controller }` where controller provides `response.done/error/aborted/continue` and `runtime.isAborted/isCloseToTimeout`.
+
+### TaskService
+
+- **Import:** `import { TaskService } from "@webiny/api-core/features/task/TaskService/index.js"`
+- **Interface Type:** See `packages/api-core/src/features/task/TaskService/abstractions.ts`
+- **Usage:** Trigger and abort background tasks. Call `taskService.trigger({ definition: "taskId", input: {...} })`. Inject as DI dependency via `TaskService`.
+
+### WebsocketService
+
+- **Import:** `import { WebsocketService } from "@webiny/api-websockets/features/WebsocketService/index.js"`
+- **Interface Type:** See `packages/api-websockets/src/features/WebsocketService/abstractions.ts`
+- **Usage:** Send real-time messages to connected clients. Use `send({ id: userId }, { action, data })` for a specific user or `sendToConnections(connections, { action, data })` for multiple. List connections with `listConnections({ where: { identityId } })`. Make optional with `[WebsocketService, { optional: true }]`.
+
+### FileAfterCreateEventHandler (File Manager)
+
+- **Import:** `import { FileAfterCreateEventHandler } from "@webiny/api-file-manager/features/file/CreateFile/events.js"`
+- **Interface Type:** See `packages/api-file-manager/src/features/file/CreateFile/events.ts`
+- **Usage:** Hook into file creation. Implement `.handle(event)` where `event.payload.file` is the created file. Register via `FileAfterCreateEventHandler.createImplementation({ implementation, dependencies })`.
 
 ---
 

packages/api-file-manager/package.json

@@ -22,13 +22,15 @@
     "@webiny/api": "0.0.0",
     "@webiny/api-core": "0.0.0",
     "@webiny/api-headless-cms": "0.0.0",
+    "@webiny/api-websockets": "0.0.0",
     "@webiny/build-tools": "0.0.0",
     "@webiny/di": "^0.2.3",
     "@webiny/error": "0.0.0",
     "@webiny/feature": "0.0.0",
     "@webiny/handler": "0.0.0",
     "@webiny/handler-graphql": "0.0.0",
     "@webiny/plugins": "0.0.0",
+    "@webiny/tasks": "0.0.0",
     "@webiny/wcp": "0.0.0",
     "cache-control-parser": "^2.2.0",
     "zod": "^4.3.6"

packages/api-file-manager/src/features/ai/AiImageTaggingFeature.ts

@@ -0,0 +1,11 @@
+import { createFeature } from "@webiny/feature/api";
+import { AiTagAfterCreateHandler } from "./AiTagAfterCreateHandler.js";
+import { AiImageTaggingTask } from "~/tasks/AiImageTaggingTask.js";
+
+export const AiImageTaggingFeature = createFeature({
+    name: "FileManagerAi/AiImageTagging",
+    register(container) {
+        container.register(AiTagAfterCreateHandler);
+        container.register(AiImageTaggingTask);
+    }
+});

packages/api-file-manager/src/features/ai/AiTagAfterCreateHandler.ts

@@ -0,0 +1,31 @@
+import { FileAfterCreateEventHandler } from "~/features/file/CreateFile/events.js";
+// import { TaskService } from "@webiny/api-core/features/task/TaskService/index.js";
+// import type { IAiImageTaggingTaskInput } from "~/tasks/AiImageTaggingTask.js";
+// import { AI_IMAGE_TAGGING_TASK_ID } from "~/tasks/AiImageTaggingTask.js";
+
+class AiTagAfterCreateHandlerImpl implements FileAfterCreateEventHandler.Interface {
+    // constructor(private taskService: TaskService.Interface) {}
+
+    async handle(event: FileAfterCreateEventHandler.Event): Promise<void> {
+        const { file } = event.payload;
+
+        if (!file.type.startsWith("image/")) {
+            return;
+        }
+
+        // TODO: enable back once ready.
+        // await this.taskService.trigger<IAiImageTaggingTaskInput>({
+        //     definition: AI_IMAGE_TAGGING_TASK_ID,
+        //     input: {
+        //         fileId: file.id
+        //     }
+        // });
+    }
+}
+
+export const AiTagAfterCreateHandler = FileAfterCreateEventHandler.createImplementation({
+    implementation: AiTagAfterCreateHandlerImpl,
+    dependencies: [
+        /*TaskService*/
+    ]
+});

packages/api-file-manager/src/index.ts

@@ -4,6 +4,7 @@ import { setupAssetDelivery } from "./delivery/setupAssetDelivery.js";
 import { createGraphQLSchemaPlugin } from "./graphql/index.js";
 import { FileManagerFeature } from "~/features/FileManagerFeature.js";
 import { FmPermissionsFeature } from "~/features/permissions/feature.js";
+import { AiImageTaggingFeature } from "~/features/ai/AiImageTaggingFeature.js";
 import { GetModelUseCase } from "@webiny/api-headless-cms/features/contentModel/GetModel/index.js";
 import { FileModel as FileModelAbstraction } from "~/domain/file/abstractions.js";
 import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js";
@@ -29,8 +30,8 @@ export const createFileManagerContext = () => {
         });
 
         FmPermissionsFeature.register(context.container);
-
         FileManagerFeature.register(context.container);
+        AiImageTaggingFeature.register(context.container);
     });
 
     plugin.name = "file-manager.createContext";

packages/api-file-manager/src/tasks/AiImageTaggingTask.ts

@@ -0,0 +1,133 @@
+import "~/types.js";
+import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
+import { Ai } from "@webiny/api-core/features/ai/index.js";
+import { GetFileUseCase } from "~/features/file/GetFile/index.js";
+import { UpdateFileUseCase } from "~/features/file/UpdateFile/index.js";
+import { GetSettingsUseCase } from "~/features/settings/GetSettings/abstractions.js";
+import { WebsocketService } from "@webiny/api-websockets/features/WebsocketService/index.js";
+
+export const AI_IMAGE_TAGGING_TASK_ID = "fmAiImageTagging";
+
+const AI_MODEL = "anthropic/claude-sonnet-4-6";
+
+const AI_PROMPT =
+    'Generate up to 5 descriptive tags for this image. Return only a JSON array of lowercase strings, nothing else. Example: ["nature","landscape","mountain"]';
+
+export interface IAiImageTaggingTaskInput {
+    fileId: string;
+}
+
+class AiImageTaggingTaskImpl implements TaskDefinition.Interface<IAiImageTaggingTaskInput> {
+    id = AI_IMAGE_TAGGING_TASK_ID;
+    title = "File Manager - AI Image Tagging";
+    description = "Automatically generates tags for uploaded images using AI.";
+    maxIterations = 1;
+    isPrivate = true;
+    databaseLogs = false;
+
+    constructor(
+        private getFile: GetFileUseCase.Interface,
+        private getSettings: GetSettingsUseCase.Interface,
+        private updateFile: UpdateFileUseCase.Interface,
+        private ai: Ai.Interface,
+        private websocketService?: WebsocketService.Interface
+    ) {}
+
+    async run({
+        input,
+        controller
+    }: TaskDefinition.RunParams<IAiImageTaggingTaskInput>): Promise<
+        TaskDefinition.Result<IAiImageTaggingTaskInput>
+    > {
+        if (controller.runtime.isAborted()) {
+            return controller.response.aborted();
+        }
+
+        const fileResult = await this.getFile.execute(input.fileId);
+        if (fileResult.isFail()) {
+            return controller.response.error({
+                message: `File not found: ${input.fileId}`
+            });
+        }
+
+        const file = fileResult.value;
+
+        if (!file.type.startsWith("image/")) {
+            return controller.response.done("File is not an image; skipping AI tagging.");
+        }
+
+        const settingsResult = await this.getSettings.execute();
+        const srcPrefix = settingsResult.isOk() ? (settingsResult.value.srcPrefix ?? "") : "";
+        const imageUrl = `${srcPrefix}${file.key}`;
+
+        let tags: string[] = [];
+        try {
+            const aiResult = await this.ai.generateText({
+                model: AI_MODEL,
+                messages: [
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "image",
+                                image: new URL(imageUrl)
+                            },
+                            {
+                                type: "text",
+                                text: AI_PROMPT
+                            }
+                        ]
+                    }
+                ]
+            });
+
+            const parsed = JSON.parse(aiResult.text);
+            if (Array.isArray(parsed)) {
+                tags = parsed.filter((t): t is string => typeof t === "string");
+            }
+        } catch (error) {
+            return controller.response.error({
+                message: `AI tagging failed: ${error instanceof Error ? error.message : String(error)}`
+            });
+        }
+
+        const mergedTags = [...new Set([...file.tags, ...tags])];
+
+        const updateResult = await this.updateFile.execute({
+            id: file.id,
+            tags: mergedTags
+        });
+
+        if (updateResult.isFail()) {
+            return controller.response.error({
+                message: `Failed to update file tags: ${updateResult.error.message}`
+            });
+        }
+
+        if (this.websocketService) {
+            const connectionsResult = await this.websocketService.listConnections();
+            if (connectionsResult.isOk() && connectionsResult.value.length > 0) {
+                await this.websocketService.sendToConnections(connectionsResult.value, {
+                    action: "fm.file.tags",
+                    data: {
+                        id: file.id,
+                        tags: mergedTags
+                    }
+                });
+            }
+        }
+
+        return controller.response.done("AI image tagging completed successfully.");
+    }
+}
+
+export const AiImageTaggingTask = TaskDefinition.createImplementation({
+    implementation: AiImageTaggingTaskImpl,
+    dependencies: [
+        GetFileUseCase,
+        GetSettingsUseCase,
+        UpdateFileUseCase,
+        Ai,
+        [WebsocketService, { optional: true }]
+    ]
+});

packages/api-file-manager/src/types.ts

@@ -1,3 +1,4 @@
+import "@webiny/tasks/features/TaskController/augmentation.js";
 import type { SecurityPermission } from "@webiny/api-core/types/security.js";
 
 export interface FilePermission extends SecurityPermission {

packages/api-file-manager/tsconfig.build.json

@@ -5,11 +5,13 @@
     { "path": "../api/tsconfig.build.json" },
     { "path": "../api-core/tsconfig.build.json" },
     { "path": "../api-headless-cms/tsconfig.build.json" },
+    { "path": "../api-websockets/tsconfig.build.json" },
     { "path": "../error/tsconfig.build.json" },
     { "path": "../feature/tsconfig.build.json" },
     { "path": "../handler/tsconfig.build.json" },
     { "path": "../handler-graphql/tsconfig.build.json" },
     { "path": "../plugins/tsconfig.build.json" },
+    { "path": "../tasks/tsconfig.build.json" },
     { "path": "../wcp/tsconfig.build.json" },
     { "path": "../handler-aws/tsconfig.build.json" },
     { "path": "../utils/tsconfig.build.json" }
@@ -27,6 +29,8 @@
       "@webiny/api-core": ["../api-core/src"],
       "@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"],
       "@webiny/api-headless-cms": ["../api-headless-cms/src"],
+      "@webiny/api-websockets/*": ["../api-websockets/src/*"],
+      "@webiny/api-websockets": ["../api-websockets/src"],
       "@webiny/error/*": ["../error/src/*"],
       "@webiny/error": ["../error/src"],
       "@webiny/feature/api": ["../feature/src/api/index.js"],
@@ -39,6 +43,8 @@
       "@webiny/handler-graphql": ["../handler-graphql/src"],
       "@webiny/plugins/*": ["../plugins/src/*"],
       "@webiny/plugins": ["../plugins/src"],
+      "@webiny/tasks/*": ["../tasks/src/*"],
+      "@webiny/tasks": ["../tasks/src"],
       "@webiny/wcp/*": ["../wcp/src/*"],
       "@webiny/wcp": ["../wcp/src"],
       "@webiny/handler-aws/*": ["../handler-aws/src/*"],

packages/api-file-manager/tsconfig.json

@@ -5,11 +5,13 @@
     { "path": "../api" },
     { "path": "../api-core" },
     { "path": "../api-headless-cms" },
+    { "path": "../api-websockets" },
     { "path": "../error" },
     { "path": "../feature" },
     { "path": "../handler" },
     { "path": "../handler-graphql" },
     { "path": "../plugins" },
+    { "path": "../tasks" },
     { "path": "../wcp" },
     { "path": "../handler-aws" },
     { "path": "../utils" }
@@ -27,6 +29,8 @@
       "@webiny/api-core": ["../api-core/src"],
       "@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"],
       "@webiny/api-headless-cms": ["../api-headless-cms/src"],
+      "@webiny/api-websockets/*": ["../api-websockets/src/*"],
+      "@webiny/api-websockets": ["../api-websockets/src"],
       "@webiny/error/*": ["../error/src/*"],
       "@webiny/error": ["../error/src"],
       "@webiny/feature/api": ["../feature/src/api/index.js"],
@@ -39,6 +43,8 @@
       "@webiny/handler-graphql": ["../handler-graphql/src"],
       "@webiny/plugins/*": ["../plugins/src/*"],
       "@webiny/plugins": ["../plugins/src"],
+      "@webiny/tasks/*": ["../tasks/src/*"],
+      "@webiny/tasks": ["../tasks/src"],
       "@webiny/wcp/*": ["../wcp/src/*"],
       "@webiny/wcp": ["../wcp/src"],
       "@webiny/handler-aws/*": ["../handler-aws/src/*"],

yarn.lock

@@ -12959,6 +12959,7 @@ __metadata:
     "@webiny/api": "npm:0.0.0"
     "@webiny/api-core": "npm:0.0.0"
     "@webiny/api-headless-cms": "npm:0.0.0"
+    "@webiny/api-websockets": "npm:0.0.0"
     "@webiny/build-tools": "npm:0.0.0"
     "@webiny/di": "npm:^0.2.3"
     "@webiny/error": "npm:0.0.0"
@@ -12968,6 +12969,7 @@ __metadata:
     "@webiny/handler-graphql": "npm:0.0.0"
     "@webiny/plugins": "npm:0.0.0"
     "@webiny/project-utils": "npm:0.0.0"
+    "@webiny/tasks": "npm:0.0.0"
     "@webiny/utils": "npm:0.0.0"
     "@webiny/wcp": "npm:0.0.0"
     cache-control-parser: "npm:^2.2.0"