From 919e40f28eaf24263e9441c9a58db12388a412ef Mon Sep 17 00:00:00 2001 From: Chen <99816898+donteatfriedrice@users.noreply.github.com> Date: Fri, 24 May 2024 18:35:32 +0800 Subject: [PATCH] feat: support more ai image actions (#7027) --- .../src/data/migrations/utils/prompts.ts | 16 +++---- .../experimental-features/index.tsx | 1 + .../block-suite-editor/ai/prompt.ts | 8 +++- .../block-suite-editor/ai/setup-provider.tsx | 47 +++++++++++++++++++ .../block-suite-editor/ai/tracker.ts | 3 ++ 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/packages/backend/server/src/data/migrations/utils/prompts.ts b/packages/backend/server/src/data/migrations/utils/prompts.ts index 0f7558e2a8..4578d242f7 100644 --- a/packages/backend/server/src/data/migrations/utils/prompts.ts +++ b/packages/backend/server/src/data/migrations/utils/prompts.ts @@ -68,7 +68,7 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-upscaler', - action: 'image', + action: 'Clearer', model: 'clarity-upscaler', messages: [ { @@ -79,13 +79,13 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-remove-bg', - action: 'image', + action: 'Remove background', model: 'imageutils/rembg', messages: [], }, { name: 'debug:action:fal-sdturbo-clay', - action: 'image', + action: 'AI image filter clay style', model: 'fast-sdxl/image-to-image', messages: [ { @@ -101,7 +101,7 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-sdturbo-pixel', - action: 'image', + action: 'AI image filter pixel style', model: 'fast-sdxl/image-to-image', messages: [ { @@ -115,7 +115,7 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-sdturbo-sketch', - action: 'image', + action: 'AI image filter sketch style', model: 'fast-sdxl/image-to-image', messages: [ { @@ -131,7 +131,7 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-sdturbo-fantasy', - action: 'image', + action: 'AI image filter anime style', model: 'fast-sdxl/image-to-image', messages: [ { @@ -147,13 +147,13 @@ export const prompts: Prompt[] = [ }, { name: 'debug:action:fal-face-to-sticker', - action: 'image', + action: 'Convert to sticker', model: 'face-to-sticker', messages: [], }, { name: 'debug:action:fal-summary-caption', - action: 'image', + action: 'Generate a caption', model: 'llava-next', messages: [ { diff --git a/packages/frontend/core/src/components/affine/setting-modal/general-setting/experimental-features/index.tsx b/packages/frontend/core/src/components/affine/setting-modal/general-setting/experimental-features/index.tsx index b77c9b231a..b3b65b89a8 100644 --- a/packages/frontend/core/src/components/affine/setting-modal/general-setting/experimental-features/index.tsx +++ b/packages/frontend/core/src/components/affine/setting-modal/general-setting/experimental-features/index.tsx @@ -127,6 +127,7 @@ const blocksuiteFeatureFlags: Partial> = { enable_expand_database_block: 'Enable Expand Database Block', enable_database_statistics: 'Enable Database Block Statistics', enable_block_query: 'Enable Todo Block Query', + enable_new_image_actions: 'Enable New Image Actions', }; const BlocksuiteFeatureFlagSettings = () => { diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts index e85df73e37..493767f4e6 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/prompt.ts @@ -7,7 +7,13 @@ export const promptKeys = [ 'debug:action:dalle3', 'debug:action:fal-sd15', 'debug:action:fal-upscaler', - 'debug:action:fal-rembg', + 'debug:action:fal-remove-bg', + 'debug:action:fal-sdturbo-clay', + 'debug:action:fal-sdturbo-pixel', + 'debug:action:fal-sdturbo-sketch', + 'debug:action:fal-sdturbo-fantasy', + 'debug:action:fal-face-to-sticker', + 'debug:action:fal-summary-caption', 'chat:gpt4', 'Summary', 'Summary the webpage', diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/setup-provider.tsx b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/setup-provider.tsx index 070a8aa790..2c4d3c9e47 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/setup-provider.tsx +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/setup-provider.tsx @@ -17,6 +17,23 @@ import { } from './request'; import { setupTracker } from './tracker'; +const filterStyleToPromptName = new Map( + Object.entries({ + 'Clay style': 'debug:action:fal-sdturbo-clay', + 'Pixel style': 'debug:action:fal-sdturbo-pixel', + 'Sketch style': 'debug:action:fal-sdturbo-sketch', + 'Anime style': 'debug:action:fal-sdturbo-fantasy', + }) +); + +const processTypeToPromptName = new Map( + Object.entries({ + Clearer: 'debug:action:fal-upscaler', + 'Remove background': 'debug:action:fal-remove-bg', + 'Convert to sticker': 'debug:action:fal-face-to-sticker', + }) +); + function setupAIProvider() { // a single workspace should have only a single chat session // user-id:workspace-id:doc-id -> chat session id @@ -290,6 +307,36 @@ Could you make a new website based on these notes and send back just the html fi }); }); + AIProvider.provide('filterImage', options => { + // test to image + const promptName = filterStyleToPromptName.get( + options.style as string + ) as PromptKey; + return toImage({ + ...options, + promptName, + }); + }); + + AIProvider.provide('processImage', options => { + // test to image + const promptName = processTypeToPromptName.get( + options.type as string + ) as PromptKey; + return toImage({ + ...options, + promptName, + }); + }); + + AIProvider.provide('generateCaption', options => { + return textToText({ + ...options, + content: options.input, + promptName: 'debug:action:fal-summary-caption', + }); + }); + AIProvider.provide('continueWriting', options => { return textToText({ ...options, diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/tracker.ts b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/tracker.ts index ae39368bed..eea6355b1e 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/tracker.ts +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/ai/tracker.ts @@ -34,6 +34,7 @@ type AIActionEventProperties = { | 'login required' | 'insert' | 'replace' + | 'use as caption' | 'discard' | 'retry' | 'add note' @@ -195,6 +196,8 @@ function inferControl( return 'insert'; } else if (event.event === 'result:replace') { return 'replace'; + } else if (event.event === 'result:use-as-caption') { + return 'use as caption'; } else if (event.event === 'result:discard') { return 'discard'; } else if (event.event === 'result:retry') {