feat: support more ai image actions (#7027)

packages/backend/server/src/data/migrations/utils/prompts.ts

@@ -68,7 +68,7 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-upscaler',
-    action: 'image',
+    action: 'Clearer',
     model: 'clarity-upscaler',
     messages: [
       {
@@ -79,13 +79,13 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-remove-bg',
-    action: 'image',
+    action: 'Remove background',
     model: 'imageutils/rembg',
     messages: [],
   },
   {
     name: 'debug:action:fal-sdturbo-clay',
-    action: 'image',
+    action: 'AI image filter clay style',
     model: 'fast-sdxl/image-to-image',
     messages: [
       {
@@ -101,7 +101,7 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-sdturbo-pixel',
-    action: 'image',
+    action: 'AI image filter pixel style',
     model: 'fast-sdxl/image-to-image',
     messages: [
       {
@@ -115,7 +115,7 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-sdturbo-sketch',
-    action: 'image',
+    action: 'AI image filter sketch style',
     model: 'fast-sdxl/image-to-image',
     messages: [
       {
@@ -131,7 +131,7 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-sdturbo-fantasy',
-    action: 'image',
+    action: 'AI image filter anime style',
     model: 'fast-sdxl/image-to-image',
     messages: [
       {
@@ -147,13 +147,13 @@ export const prompts: Prompt[] = [
   },
   {
     name: 'debug:action:fal-face-to-sticker',
-    action: 'image',
+    action: 'Convert to sticker',
     model: 'face-to-sticker',
     messages: [],
   },
   {
     name: 'debug:action:fal-summary-caption',
-    action: 'image',
+    action: 'Generate a caption',
     model: 'llava-next',
     messages: [
       {

packages/frontend/core/src/components/affine/setting-modal/general-setting/experimental-features/index.tsx

@@ -127,6 +127,7 @@ const blocksuiteFeatureFlags: Partial<Record<keyof BlockSuiteFlags, string>> = {
   enable_expand_database_block: 'Enable Expand Database Block',
   enable_database_statistics: 'Enable Database Block Statistics',
   enable_block_query: 'Enable Todo Block Query',
+  enable_new_image_actions: 'Enable New Image Actions',
 };
 
 const BlocksuiteFeatureFlagSettings = () => {

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts

@@ -7,7 +7,13 @@ export const promptKeys = [
   'debug:action:dalle3',
   'debug:action:fal-sd15',
   'debug:action:fal-upscaler',
-  'debug:action:fal-rembg',
+  'debug:action:fal-remove-bg',
+  'debug:action:fal-sdturbo-clay',
+  'debug:action:fal-sdturbo-pixel',
+  'debug:action:fal-sdturbo-sketch',
+  'debug:action:fal-sdturbo-fantasy',
+  'debug:action:fal-face-to-sticker',
+  'debug:action:fal-summary-caption',
   'chat:gpt4',
   'Summary',
   'Summary the webpage',

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/setup-provider.tsx

@@ -17,6 +17,23 @@ import {
 } from './request';
 import { setupTracker } from './tracker';
 
+const filterStyleToPromptName = new Map(
+  Object.entries({
+    'Clay style': 'debug:action:fal-sdturbo-clay',
+    'Pixel style': 'debug:action:fal-sdturbo-pixel',
+    'Sketch style': 'debug:action:fal-sdturbo-sketch',
+    'Anime style': 'debug:action:fal-sdturbo-fantasy',
+  })
+);
+
+const processTypeToPromptName = new Map(
+  Object.entries({
+    Clearer: 'debug:action:fal-upscaler',
+    'Remove background': 'debug:action:fal-remove-bg',
+    'Convert to sticker': 'debug:action:fal-face-to-sticker',
+  })
+);
+
 function setupAIProvider() {
   // a single workspace should have only a single chat session
   // user-id:workspace-id:doc-id -> chat session id
@@ -290,6 +307,36 @@ Could you make a new website based on these notes and send back just the html fi
     });
   });
 
+  AIProvider.provide('filterImage', options => {
+    // test to image
+    const promptName = filterStyleToPromptName.get(
+      options.style as string
+    ) as PromptKey;
+    return toImage({
+      ...options,
+      promptName,
+    });
+  });
+
+  AIProvider.provide('processImage', options => {
+    // test to image
+    const promptName = processTypeToPromptName.get(
+      options.type as string
+    ) as PromptKey;
+    return toImage({
+      ...options,
+      promptName,
+    });
+  });
+
+  AIProvider.provide('generateCaption', options => {
+    return textToText({
+      ...options,
+      content: options.input,
+      promptName: 'debug:action:fal-summary-caption',
+    });
+  });
+
   AIProvider.provide('continueWriting', options => {
     return textToText({
       ...options,

packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/tracker.ts

@@ -34,6 +34,7 @@ type AIActionEventProperties = {
     | 'login required'
     | 'insert'
     | 'replace'
+    | 'use as caption'
     | 'discard'
     | 'retry'
     | 'add note'
@@ -195,6 +196,8 @@ function inferControl(
     return 'insert';
   } else if (event.event === 'result:replace') {
     return 'replace';
+  } else if (event.event === 'result:use-as-caption') {
+    return 'use as caption';
   } else if (event.event === 'result:discard') {
     return 'discard';
   } else if (event.event === 'result:retry') {