diff --git a/ai-context/core-features-reference.md b/ai-context/core-features-reference.md index a80c18d9155..7817f7de303 100644 --- a/ai-context/core-features-reference.md +++ b/ai-context/core-features-reference.md @@ -48,11 +48,41 @@ This document provides the correct import paths and type definitions for commonl - **Interface Type:** See `packages/api-core/src/features/settings/UpdateSettings/abstractions.ts` - **Usage:** Create or update settings records +### Ai + +- **Import:** `import { Ai } from "@webiny/api-core/features/ai/index.js"` +- **Interface Type:** See `packages/api-core/src/features/ai/abstractions.ts` +- **Usage:** Generate text and stream text using registered AI providers. Model format: `"provider/modelId"` (e.g. `"anthropic/claude-3-5-sonnet-20241022"`, `"openai/gpt-4o"`). Providers: `anthropic` (env: `WEBINY_API_ANTHROPIC_API_KEY`), `openai` (env: `WEBINY_API_OPENAI_API_KEY`). Must register `AiFeature` from `@webiny/api-core/features/ai/index.js`. + ### AiGateway -- **Import:** `import { AiGateway } from "@webiny/api-core/exports/api"` -- **Interface Type:** See `packages/api-core/src/features/aiGateway/abstractions.ts` -- **Usage:** Obtain a configured `LanguageModel` (from the `ai` SDK) for the active provider. Provider/token/model are read from env vars (`WEBINY_API_AI_GATEWAY_PROVIDER`, `WEBINY_API_AI_GATEWAY_TOKEN`, `WEBINY_API_AI_GATEWAY_MODEL`). Call `aiGateway.getLanguageModel(modelId?)` and pass the result to `generateText`, `streamText`, etc. from the `ai` package. +- **Import:** `import { AiGateway } from "@webiny/api-core/features/ai/index.js"` +- **Interface Type:** See `packages/api-core/src/features/ai/abstractions.ts` +- **Usage:** Routes `"provider/modelId"` strings to registered providers. Used internally by `Ai`. + +### TaskDefinition + +- **Import:** `import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js"` +- **Interface Type:** See `packages/api-core/src/features/task/TaskDefinition/abstractions.ts` +- **Usage:** Define background tasks. Use `TaskDefinition.createImplementation({ implementation, dependencies })`. Register with `context.container.register(MyTask)`. The `run` method receives `{ input, controller }` where controller provides `response.done/error/aborted/continue` and `runtime.isAborted/isCloseToTimeout`. + +### TaskService + +- **Import:** `import { TaskService } from "@webiny/api-core/features/task/TaskService/index.js"` +- **Interface Type:** See `packages/api-core/src/features/task/TaskService/abstractions.ts` +- **Usage:** Trigger and abort background tasks. Call `taskService.trigger({ definition: "taskId", input: {...} })`. Inject as DI dependency via `TaskService`. + +### WebsocketService + +- **Import:** `import { WebsocketService } from "@webiny/api-websockets/features/WebsocketService/index.js"` +- **Interface Type:** See `packages/api-websockets/src/features/WebsocketService/abstractions.ts` +- **Usage:** Send real-time messages to connected clients. Use `send({ id: userId }, { action, data })` for a specific user or `sendToConnections(connections, { action, data })` for multiple. List connections with `listConnections({ where: { identityId } })`. Make optional with `[WebsocketService, { optional: true }]`. + +### FileAfterCreateEventHandler (File Manager) + +- **Import:** `import { FileAfterCreateEventHandler } from "@webiny/api-file-manager/features/file/CreateFile/events.js"` +- **Interface Type:** See `packages/api-file-manager/src/features/file/CreateFile/events.ts` +- **Usage:** Hook into file creation. Implement `.handle(event)` where `event.payload.file` is the created file. Register via `FileAfterCreateEventHandler.createImplementation({ implementation, dependencies })`. --- diff --git a/packages/api-file-manager/package.json b/packages/api-file-manager/package.json index 2b45f180735..aaea3f9d75d 100644 --- a/packages/api-file-manager/package.json +++ b/packages/api-file-manager/package.json @@ -22,6 +22,7 @@ "@webiny/api": "0.0.0", "@webiny/api-core": "0.0.0", "@webiny/api-headless-cms": "0.0.0", + "@webiny/api-websockets": "0.0.0", "@webiny/build-tools": "0.0.0", "@webiny/di": "^0.2.3", "@webiny/error": "0.0.0", @@ -29,6 +30,7 @@ "@webiny/handler": "0.0.0", "@webiny/handler-graphql": "0.0.0", "@webiny/plugins": "0.0.0", + "@webiny/tasks": "0.0.0", "@webiny/wcp": "0.0.0", "cache-control-parser": "^2.2.0", "zod": "^4.3.6" diff --git a/packages/api-file-manager/src/features/ai/AiImageTaggingFeature.ts b/packages/api-file-manager/src/features/ai/AiImageTaggingFeature.ts new file mode 100644 index 00000000000..32fd87f3eaf --- /dev/null +++ b/packages/api-file-manager/src/features/ai/AiImageTaggingFeature.ts @@ -0,0 +1,11 @@ +import { createFeature } from "@webiny/feature/api"; +import { AiTagAfterCreateHandler } from "./AiTagAfterCreateHandler.js"; +import { AiImageTaggingTask } from "~/tasks/AiImageTaggingTask.js"; + +export const AiImageTaggingFeature = createFeature({ + name: "FileManagerAi/AiImageTagging", + register(container) { + container.register(AiTagAfterCreateHandler); + container.register(AiImageTaggingTask); + } +}); diff --git a/packages/api-file-manager/src/features/ai/AiTagAfterCreateHandler.ts b/packages/api-file-manager/src/features/ai/AiTagAfterCreateHandler.ts new file mode 100644 index 00000000000..27b09b24776 --- /dev/null +++ b/packages/api-file-manager/src/features/ai/AiTagAfterCreateHandler.ts @@ -0,0 +1,31 @@ +import { FileAfterCreateEventHandler } from "~/features/file/CreateFile/events.js"; +// import { TaskService } from "@webiny/api-core/features/task/TaskService/index.js"; +// import type { IAiImageTaggingTaskInput } from "~/tasks/AiImageTaggingTask.js"; +// import { AI_IMAGE_TAGGING_TASK_ID } from "~/tasks/AiImageTaggingTask.js"; + +class AiTagAfterCreateHandlerImpl implements FileAfterCreateEventHandler.Interface { + // constructor(private taskService: TaskService.Interface) {} + + async handle(event: FileAfterCreateEventHandler.Event): Promise { + const { file } = event.payload; + + if (!file.type.startsWith("image/")) { + return; + } + + // TODO: enable back once ready. + // await this.taskService.trigger({ + // definition: AI_IMAGE_TAGGING_TASK_ID, + // input: { + // fileId: file.id + // } + // }); + } +} + +export const AiTagAfterCreateHandler = FileAfterCreateEventHandler.createImplementation({ + implementation: AiTagAfterCreateHandlerImpl, + dependencies: [ + /*TaskService*/ + ] +}); diff --git a/packages/api-file-manager/src/index.ts b/packages/api-file-manager/src/index.ts index abac5f9a8c6..3d616336694 100644 --- a/packages/api-file-manager/src/index.ts +++ b/packages/api-file-manager/src/index.ts @@ -4,6 +4,7 @@ import { setupAssetDelivery } from "./delivery/setupAssetDelivery.js"; import { createGraphQLSchemaPlugin } from "./graphql/index.js"; import { FileManagerFeature } from "~/features/FileManagerFeature.js"; import { FmPermissionsFeature } from "~/features/permissions/feature.js"; +import { AiImageTaggingFeature } from "~/features/ai/AiImageTaggingFeature.js"; import { GetModelUseCase } from "@webiny/api-headless-cms/features/contentModel/GetModel/index.js"; import { FileModel as FileModelAbstraction } from "~/domain/file/abstractions.js"; import { TenantContext } from "@webiny/api-core/features/tenancy/TenantContext/index.js"; @@ -29,8 +30,8 @@ export const createFileManagerContext = () => { }); FmPermissionsFeature.register(context.container); - FileManagerFeature.register(context.container); + AiImageTaggingFeature.register(context.container); }); plugin.name = "file-manager.createContext"; diff --git a/packages/api-file-manager/src/tasks/AiImageTaggingTask.ts b/packages/api-file-manager/src/tasks/AiImageTaggingTask.ts new file mode 100644 index 00000000000..ec734502752 --- /dev/null +++ b/packages/api-file-manager/src/tasks/AiImageTaggingTask.ts @@ -0,0 +1,133 @@ +import "~/types.js"; +import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js"; +import { Ai } from "@webiny/api-core/features/ai/index.js"; +import { GetFileUseCase } from "~/features/file/GetFile/index.js"; +import { UpdateFileUseCase } from "~/features/file/UpdateFile/index.js"; +import { GetSettingsUseCase } from "~/features/settings/GetSettings/abstractions.js"; +import { WebsocketService } from "@webiny/api-websockets/features/WebsocketService/index.js"; + +export const AI_IMAGE_TAGGING_TASK_ID = "fmAiImageTagging"; + +const AI_MODEL = "anthropic/claude-sonnet-4-6"; + +const AI_PROMPT = + 'Generate up to 5 descriptive tags for this image. Return only a JSON array of lowercase strings, nothing else. Example: ["nature","landscape","mountain"]'; + +export interface IAiImageTaggingTaskInput { + fileId: string; +} + +class AiImageTaggingTaskImpl implements TaskDefinition.Interface { + id = AI_IMAGE_TAGGING_TASK_ID; + title = "File Manager - AI Image Tagging"; + description = "Automatically generates tags for uploaded images using AI."; + maxIterations = 1; + isPrivate = true; + databaseLogs = false; + + constructor( + private getFile: GetFileUseCase.Interface, + private getSettings: GetSettingsUseCase.Interface, + private updateFile: UpdateFileUseCase.Interface, + private ai: Ai.Interface, + private websocketService?: WebsocketService.Interface + ) {} + + async run({ + input, + controller + }: TaskDefinition.RunParams): Promise< + TaskDefinition.Result + > { + if (controller.runtime.isAborted()) { + return controller.response.aborted(); + } + + const fileResult = await this.getFile.execute(input.fileId); + if (fileResult.isFail()) { + return controller.response.error({ + message: `File not found: ${input.fileId}` + }); + } + + const file = fileResult.value; + + if (!file.type.startsWith("image/")) { + return controller.response.done("File is not an image; skipping AI tagging."); + } + + const settingsResult = await this.getSettings.execute(); + const srcPrefix = settingsResult.isOk() ? (settingsResult.value.srcPrefix ?? "") : ""; + const imageUrl = `${srcPrefix}${file.key}`; + + let tags: string[] = []; + try { + const aiResult = await this.ai.generateText({ + model: AI_MODEL, + messages: [ + { + role: "user", + content: [ + { + type: "image", + image: new URL(imageUrl) + }, + { + type: "text", + text: AI_PROMPT + } + ] + } + ] + }); + + const parsed = JSON.parse(aiResult.text); + if (Array.isArray(parsed)) { + tags = parsed.filter((t): t is string => typeof t === "string"); + } + } catch (error) { + return controller.response.error({ + message: `AI tagging failed: ${error instanceof Error ? error.message : String(error)}` + }); + } + + const mergedTags = [...new Set([...file.tags, ...tags])]; + + const updateResult = await this.updateFile.execute({ + id: file.id, + tags: mergedTags + }); + + if (updateResult.isFail()) { + return controller.response.error({ + message: `Failed to update file tags: ${updateResult.error.message}` + }); + } + + if (this.websocketService) { + const connectionsResult = await this.websocketService.listConnections(); + if (connectionsResult.isOk() && connectionsResult.value.length > 0) { + await this.websocketService.sendToConnections(connectionsResult.value, { + action: "fm.file.tags", + data: { + id: file.id, + tags: mergedTags + } + }); + } + } + + return controller.response.done("AI image tagging completed successfully."); + } +} + +export const AiImageTaggingTask = TaskDefinition.createImplementation({ + implementation: AiImageTaggingTaskImpl, + dependencies: [ + GetFileUseCase, + GetSettingsUseCase, + UpdateFileUseCase, + Ai, + [WebsocketService, { optional: true }] + ] +}); diff --git a/packages/api-file-manager/src/types.ts b/packages/api-file-manager/src/types.ts index 0ad40639964..f53518f71e7 100644 --- a/packages/api-file-manager/src/types.ts +++ b/packages/api-file-manager/src/types.ts @@ -1,3 +1,4 @@ +import "@webiny/tasks/features/TaskController/augmentation.js"; import type { SecurityPermission } from "@webiny/api-core/types/security.js"; export interface FilePermission extends SecurityPermission { diff --git a/packages/api-file-manager/tsconfig.build.json b/packages/api-file-manager/tsconfig.build.json index a081a4f0057..1c1233dec74 100644 --- a/packages/api-file-manager/tsconfig.build.json +++ b/packages/api-file-manager/tsconfig.build.json @@ -5,11 +5,13 @@ { "path": "../api/tsconfig.build.json" }, { "path": "../api-core/tsconfig.build.json" }, { "path": "../api-headless-cms/tsconfig.build.json" }, + { "path": "../api-websockets/tsconfig.build.json" }, { "path": "../error/tsconfig.build.json" }, { "path": "../feature/tsconfig.build.json" }, { "path": "../handler/tsconfig.build.json" }, { "path": "../handler-graphql/tsconfig.build.json" }, { "path": "../plugins/tsconfig.build.json" }, + { "path": "../tasks/tsconfig.build.json" }, { "path": "../wcp/tsconfig.build.json" }, { "path": "../handler-aws/tsconfig.build.json" }, { "path": "../utils/tsconfig.build.json" } @@ -27,6 +29,8 @@ "@webiny/api-core": ["../api-core/src"], "@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"], "@webiny/api-headless-cms": ["../api-headless-cms/src"], + "@webiny/api-websockets/*": ["../api-websockets/src/*"], + "@webiny/api-websockets": ["../api-websockets/src"], "@webiny/error/*": ["../error/src/*"], "@webiny/error": ["../error/src"], "@webiny/feature/api": ["../feature/src/api/index.js"], @@ -39,6 +43,8 @@ "@webiny/handler-graphql": ["../handler-graphql/src"], "@webiny/plugins/*": ["../plugins/src/*"], "@webiny/plugins": ["../plugins/src"], + "@webiny/tasks/*": ["../tasks/src/*"], + "@webiny/tasks": ["../tasks/src"], "@webiny/wcp/*": ["../wcp/src/*"], "@webiny/wcp": ["../wcp/src"], "@webiny/handler-aws/*": ["../handler-aws/src/*"], diff --git a/packages/api-file-manager/tsconfig.json b/packages/api-file-manager/tsconfig.json index 9ccf3820066..2f15fdef9c3 100644 --- a/packages/api-file-manager/tsconfig.json +++ b/packages/api-file-manager/tsconfig.json @@ -5,11 +5,13 @@ { "path": "../api" }, { "path": "../api-core" }, { "path": "../api-headless-cms" }, + { "path": "../api-websockets" }, { "path": "../error" }, { "path": "../feature" }, { "path": "../handler" }, { "path": "../handler-graphql" }, { "path": "../plugins" }, + { "path": "../tasks" }, { "path": "../wcp" }, { "path": "../handler-aws" }, { "path": "../utils" } @@ -27,6 +29,8 @@ "@webiny/api-core": ["../api-core/src"], "@webiny/api-headless-cms/*": ["../api-headless-cms/src/*"], "@webiny/api-headless-cms": ["../api-headless-cms/src"], + "@webiny/api-websockets/*": ["../api-websockets/src/*"], + "@webiny/api-websockets": ["../api-websockets/src"], "@webiny/error/*": ["../error/src/*"], "@webiny/error": ["../error/src"], "@webiny/feature/api": ["../feature/src/api/index.js"], @@ -39,6 +43,8 @@ "@webiny/handler-graphql": ["../handler-graphql/src"], "@webiny/plugins/*": ["../plugins/src/*"], "@webiny/plugins": ["../plugins/src"], + "@webiny/tasks/*": ["../tasks/src/*"], + "@webiny/tasks": ["../tasks/src"], "@webiny/wcp/*": ["../wcp/src/*"], "@webiny/wcp": ["../wcp/src"], "@webiny/handler-aws/*": ["../handler-aws/src/*"], diff --git a/yarn.lock b/yarn.lock index 73c9710dfb5..c99806bb8c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12959,6 +12959,7 @@ __metadata: "@webiny/api": "npm:0.0.0" "@webiny/api-core": "npm:0.0.0" "@webiny/api-headless-cms": "npm:0.0.0" + "@webiny/api-websockets": "npm:0.0.0" "@webiny/build-tools": "npm:0.0.0" "@webiny/di": "npm:^0.2.3" "@webiny/error": "npm:0.0.0" @@ -12968,6 +12969,7 @@ __metadata: "@webiny/handler-graphql": "npm:0.0.0" "@webiny/plugins": "npm:0.0.0" "@webiny/project-utils": "npm:0.0.0" + "@webiny/tasks": "npm:0.0.0" "@webiny/utils": "npm:0.0.0" "@webiny/wcp": "npm:0.0.0" cache-control-parser: "npm:^2.2.0"