feat(core): continue with AI (#7253)

Closes: [AFF-1251](https://linear.app/affine-design/issue/AFF-1251/continue-with-ai-的-action-的入口区分-open-with-ai) [BS-515](https://linear.app/affine-design/issue/BS-515/添加白板选区的-continue-with-ai-支持) [AFF-1256](https://linear.app/affine-design/issue/AFF-1256/continue-with-ai-当只有不支持的-block-时,需要特殊处理) [AF-919](https://linear.app/affine-design/issue/AF-919/内容为空时,不需要显示-start-with-this-doc)

* add the selected element to the candidate card list
* select the candidate card to hide the candidate card list and add quote into input
* close quote to show the candidate card list
* show only the three newest candidates
* **`Start with this doc`**: Currently only determines if a document has content after the first load

https://github.com/toeverything/AFFiNE/assets/27926/d19c8ab6-37eb-495f-9c38-e579b2f57000

https://github.com/toeverything/AFFiNE/assets/27926/3ba654c3-6af4-4662-a641-17cfe2ed5ff7
This commit is contained in:
fundon 2024-06-20 04:05:10 +00:00
parent e8fdce514f
commit 671fa1149d
No known key found for this signature in database
GPG Key ID: 398BFA91AC539CF7
8 changed files with 347 additions and 212 deletions

View File

@ -36,6 +36,7 @@ import {
AISearchIcon,
AIStarIconWithAnimation,
ChatWithAIIcon,
CommentIcon,
ExplainIcon,
ImproveWritingIcon,
LanguageIcon,
@ -395,6 +396,15 @@ const GenerateWithAIGroup: AIItemGroupConfig = {
const OthersAIGroup: AIItemGroupConfig = {
name: 'Others',
items: [
{
name: 'Continue with AI',
icon: CommentIcon,
handler: host => {
const panel = getAIPanel(host);
AIProvider.slots.requestContinueWithAIInChat.emit({ host });
panel.hide();
},
},
{
name: 'Open AI Chat',
icon: ChatWithAIIcon,

View File

@ -1061,3 +1061,17 @@ export const MoreIcon = html`<svg
</clipPath>
</defs>
</svg> `;
export const CommentIcon = html`<svg
width="20"
height="20"
viewBox="0 0 20 20"
fill="currentColor"
xmlns="http://www.w3.org/2000/svg"
>
<path
fill-rule="evenodd"
clip-rule="evenodd"
d="M10.4167 3.125C6.84987 3.125 3.95837 6.01649 3.95837 9.58333C3.95837 10.5626 4.1759 11.4893 4.56469 12.3193C4.62513 12.4484 4.64581 12.5938 4.62202 12.7365L4.09372 15.9063L7.26351 15.378C7.40628 15.3542 7.55167 15.3749 7.68071 15.4354C8.51071 15.8241 9.43744 16.0417 10.4167 16.0417C13.9835 16.0417 16.875 13.1502 16.875 9.58333C16.875 6.01649 13.9835 3.125 10.4167 3.125ZM2.70837 9.58333C2.70837 5.32614 6.15951 1.875 10.4167 1.875C14.6739 1.875 18.125 5.32614 18.125 9.58333C18.125 13.8405 14.6739 17.2917 10.4167 17.2917C9.31104 17.2917 8.25828 17.0585 7.30619 16.6382L3.5512 17.264C3.07179 17.3439 2.65615 16.9283 2.73606 16.4488L3.36189 12.6939C2.94154 11.7418 2.70837 10.689 2.70837 9.58333Z"
/>
</svg>`;

View File

@ -1,12 +1,10 @@
import type { BaseSelection, EditorHost } from '@blocksuite/block-std';
import type { EditorHost } from '@blocksuite/block-std';
import { WithDisposable } from '@blocksuite/block-std';
import {
type CopilotSelectionController,
type ImageBlockModel,
type NoteBlockModel,
NoteDisplayMode,
} from '@blocksuite/blocks';
import { debounce } from '@blocksuite/global/utils';
import type { BlockModel } from '@blocksuite/store';
import { css, html, LitElement, nothing, type PropertyValues } from 'lit';
import { customElement, property, state } from 'lit/decorators.js';
@ -18,8 +16,8 @@ import {
DocIcon,
SmallImageIcon,
} from '../_common/icons';
import { AIProvider } from '../provider';
import {
getEdgelessRootFromEditor,
getSelectedImagesAsBlobs,
getSelectedTextContent,
getTextContentFromBlockModels,
@ -54,15 +52,68 @@ const cardsStyles = css`
}
`;
const ChatCardsConfig = [
{
name: 'current-selection',
render: (text?: string, _?: File, __?: string) => {
if (!text) return nothing;
enum CardType {
Text,
Image,
Block,
Doc,
}
const lines = text.split('\n');
type CardBase = {
id: number;
};
return html`<div class="card-wrapper">
type CardText = CardBase & {
type: CardType.Text;
text: string;
markdown: string;
};
type CardImage = CardBase & {
type: CardType.Image;
image: File;
caption?: string;
};
type CardBlock = CardBase & {
type: CardType.Block | CardType.Doc;
text?: string;
markdown?: string;
images?: File[];
};
type Card = CardText | CardImage | CardBlock;
const MAX_CARDS = 3;
@customElement('chat-cards')
export class ChatCards extends WithDisposable(LitElement) {
static override styles = css`
:host {
display: flex;
flex-direction: column;
gap: 12px;
}
${cardsStyles}
`;
@property({ attribute: false })
accessor host!: EditorHost;
@property({ attribute: false })
accessor updateContext!: (context: Partial<ChatContextValue>) => void;
@state()
accessor cards: Card[] = [];
private _selectedCardId: number = 0;
static renderText({ text }: CardText) {
const lines = text.split('\n');
return html`
<div class="card-wrapper">
<div class="card-title">
${CurrentSelectionIcon}
<div>Start with current selection</div>
@ -71,8 +122,8 @@ const ChatCardsConfig = [
${repeat(
lines.slice(0, 2),
line => line,
line => {
return html`<div
line => html`
<div
style=${styleMap({
overflow: 'hidden',
textOverflow: 'ellipsis',
@ -80,34 +131,17 @@ const ChatCardsConfig = [
})}
>
${line}
</div>`;
}
</div>
`
)}
</div>
</div> `;
},
handler: (
updateContext: (context: Partial<ChatContextValue>) => void,
text: string,
markdown: string,
images?: File[]
) => {
const value: Partial<ChatContextValue> = {
quote: text,
markdown: markdown,
};
if (images) {
value.images = images;
}
updateContext(value);
},
},
{
name: 'image',
render: (_?: string, image?: File, caption?: string) => {
if (!image) return nothing;
</div>
`;
}
return html`<div
static renderImage({ caption, image }: CardImage) {
return html`
<div
class="card-wrapper"
style=${styleMap({
display: 'flex',
@ -134,142 +168,154 @@ const ChatCardsConfig = [
})}
src="${URL.createObjectURL(image)}"
/>
</div>`;
},
handler: (
updateContext: (context: Partial<ChatContextValue>) => void,
_: string,
__: string,
images?: File[]
) => {
const value: Partial<ChatContextValue> = {};
if (images) {
value.images = images;
}
updateContext(value);
},
},
{
name: 'doc',
render: () => {
return html`
<div class="card-wrapper">
<div class="card-title">
${DocIcon}
<div>Start with this doc</div>
</div>
<div class="second-text">you've chosen within the doc</div>
</div>
`;
}
static renderDoc(_: CardBlock) {
return html`
<div class="card-wrapper">
<div class="card-title">
${DocIcon}
<div>Start with this doc</div>
</div>
`;
},
handler: (
updateContext: (context: Partial<ChatContextValue>) => void,
text: string,
markdown: string,
images?: File[]
) => {
const value: Partial<ChatContextValue> = {
quote: text,
markdown: markdown,
};
if (images) {
value.images = images;
<div class="second-text">you've chosen within the doc</div>
</div>
`;
}
private _renderCard(card: Card) {
if (card.type === CardType.Text) {
return ChatCards.renderText(card);
}
if (card.type === CardType.Image) {
return ChatCards.renderImage(card);
}
if (card.type === CardType.Doc) {
return ChatCards.renderDoc(card);
}
return nothing;
}
private _updateCards(card: Card) {
this.cards.unshift(card);
if (this.cards.length > MAX_CARDS) {
this.cards.pop();
}
this.requestUpdate();
}
private async _handleDocSelection(card: CardBlock) {
const { text, markdown, images } = await this._extractAll();
card.text = text;
card.markdown = markdown;
card.images = images;
}
private async _handleClick(card: Card) {
AIProvider.slots.toggleChatCards.emit({ visible: false });
this._selectedCardId = card.id;
switch (card.type) {
case CardType.Text: {
this.updateContext({
quote: card.text,
markdown: card.markdown,
});
break;
}
case CardType.Image: {
this.updateContext({
images: [card.image],
});
break;
}
case CardType.Doc: {
await this._handleDocSelection(card);
this.updateContext({
quote: card.text,
markdown: card.markdown,
images: card.images,
});
break;
}
updateContext(value);
},
},
];
@customElement('chat-cards')
export class ChatCards extends WithDisposable(LitElement) {
static override styles = css`
${cardsStyles}
.cards-container {
display: flex;
flex-direction: column;
gap: 12px;
}
`;
@property({ attribute: false })
accessor host!: EditorHost;
@property({ attribute: false })
accessor chatContextValue!: ChatContextValue;
@property({ attribute: false })
accessor updateContext!: (context: Partial<ChatContextValue>) => void;
@property({ attribute: false })
accessor selectionValue: BaseSelection[] = [];
@state()
accessor text: string = '';
@state()
accessor markdown: string = '';
@state()
accessor images: File[] = [];
@state()
accessor caption: string = '';
private _onEdgelessCopilotAreaUpdated() {
if (!this.host.closest('edgeless-editor')) return;
const edgeless = getEdgelessRootFromEditor(this.host);
const copilotSelectionTool = edgeless.tools.controllers
.copilot as CopilotSelectionController;
this._disposables.add(
copilotSelectionTool.draggingAreaUpdated.on(
debounce(() => {
selectedToCanvas(this.host)
.then(canvas => {
canvas?.toBlob(blob => {
if (!blob) return;
const file = new File([blob], 'selected.png');
this.images = [file];
});
})
.catch(console.error);
}, 300)
)
);
}
private async _updateState() {
if (
this.selectionValue.some(
selection => selection.is('text') || selection.is('image')
)
)
private async _extract() {
const text = await getSelectedTextContent(this.host, 'plain-text');
const images = await getSelectedImagesAsBlobs(this.host);
const hasText = text.length > 0;
const hasImages = images.length > 0;
if (hasText && !hasImages) {
const markdown = await getSelectedTextContent(this.host, 'markdown');
this._updateCards({
id: Date.now(),
type: CardType.Text,
text,
markdown,
});
return;
}
if (!hasText && hasImages && images.length === 1) {
const [_, data] = this.host.command
.chain()
.tryAll(chain => [chain.getImageSelections()])
.getSelectedBlocks({
types: ['image'],
})
.run();
let caption = '';
if (data.currentImageSelections?.[0]) {
caption =
(
this.host.doc.getBlock(data.currentImageSelections[0].blockId)
?.model as ImageBlockModel
).caption ?? '';
}
this._updateCards({
id: Date.now(),
type: CardType.Image,
image: images[0],
caption,
});
return;
this.text = await getSelectedTextContent(this.host, 'plain-text');
this.markdown = await getSelectedTextContent(this.host, 'markdown');
this.images = await getSelectedImagesAsBlobs(this.host);
const [_, data] = this.host.command
.chain()
.tryAll(chain => [
chain.getTextSelection(),
chain.getBlockSelections(),
chain.getImageSelections(),
])
.getSelectedBlocks({
types: ['image'],
})
.run();
if (data.currentBlockSelections?.[0]) {
this.caption =
(
this.host.doc.getBlock(data.currentBlockSelections[0].blockId)
?.model as ImageBlockModel
).caption ?? '';
}
}
private async _handleDocSelection() {
private async _extractOnEdgeless() {
if (!this.host.closest('edgeless-editor')) return;
const canvas = await selectedToCanvas(this.host);
if (!canvas) return;
const blob: Blob | null = await new Promise(resolve =>
canvas.toBlob(resolve)
);
if (!blob) return;
this._updateCards({
id: Date.now(),
type: CardType.Image,
image: new File([blob], 'selected.png'),
});
}
private async _extractAll() {
const notes = this.host.doc
.getBlocksByFlavour('affine:note')
.filter(
@ -305,50 +351,68 @@ export class ChatCards extends WithDisposable(LitElement) {
}) ?? []
);
const images = blobs.filter((blob): blob is File => !!blob);
this.text = text;
this.markdown = markdown;
this.images = images;
return {
text,
markdown,
images,
};
}
protected override async updated(_changedProperties: PropertyValues) {
if (_changedProperties.has('selectionValue')) {
await this._updateState();
}
protected override async updated(changedProperties: PropertyValues) {
if (changedProperties.has('host')) {
const { text, images } = await this._extractAll();
const hasText = text.length > 0;
const hasImages = images.length > 0;
if (_changedProperties.has('host')) {
this._onEdgelessCopilotAreaUpdated();
// Currently only supports checking on first load
if (
(hasText || hasImages) &&
!this.cards.some(card => card.type === CardType.Doc)
) {
this._updateCards({
id: Date.now(),
type: CardType.Doc,
});
}
}
}
override async connectedCallback() {
super.connectedCallback();
this._disposables.add(
AIProvider.slots.requestContinueWithAIInChat.on(async ({ mode }) => {
if (mode === 'edgeless') {
await this._extractOnEdgeless();
} else {
await this._extract();
}
})
);
this._disposables.add(
AIProvider.slots.toggleChatCards.on(({ visible, ok }) => {
if (visible && ok && this._selectedCardId > 0) {
this.cards = this.cards.filter(
card => card.id !== this._selectedCardId
);
this._selectedCardId = 0;
}
})
);
}
protected override render() {
return html`<div class="cards-container">
${repeat(
ChatCardsConfig,
card => card.name,
card => {
if (
card.render(this.text, this.images[0], this.caption) !== nothing
) {
return html`<div
@click=${async () => {
if (card.name === 'doc') {
await this._handleDocSelection();
}
card.handler(
this.updateContext,
this.text,
this.markdown,
this.images
);
}}
>
${card.render(this.text, this.images[0], this.caption)}
</div> `;
}
return nothing;
}
)}
</div>`;
return repeat(
this.cards,
card => card.id,
card => html`
<div @click=${() => this._handleClick(card)}>
${this._renderCard(card)}
</div>
`
);
}
}

View File

@ -268,6 +268,8 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
<div
class="close-wrapper"
@click=${() => {
AIProvider.slots.toggleChatCards.emit({ visible: true });
if (this.curIndex >= 0 && this.curIndex < images.length) {
const newImages = [...images];
newImages.splice(this.curIndex, 1);
@ -315,6 +317,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
<div
class="chat-quote-close"
@click=${() => {
AIProvider.slots.toggleChatCards.emit({ visible: true });
this.updateContext({ quote: '', markdown: '' });
}}
>

View File

@ -27,6 +27,7 @@ import {
} from '@blocksuite/blocks';
import { css, html, nothing, type PropertyValues } from 'lit';
import { customElement, property, query, state } from 'lit/decorators.js';
import { cache } from 'lit/directives/cache.js';
import { repeat } from 'lit/directives/repeat.js';
import { styleMap } from 'lit/directives/style-map.js';
@ -144,7 +145,8 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
}
`;
private _selectionValue: BaseSelection[] = [];
@state()
accessor _selectionValue: BaseSelection[] = [];
@state()
accessor showDownIndicator = false;
@ -167,14 +169,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
@query('.chat-panel-messages')
accessor messagesContainer!: HTMLDivElement;
protected override updated(_changedProperties: PropertyValues) {
if (_changedProperties.has('host')) {
@state()
accessor showChatCards = true;
protected override updated(changedProperties: PropertyValues) {
if (changedProperties.has('host')) {
const { disposables } = this;
disposables.add(
this.host.selection.slots.changed.on(() => {
this._selectionValue = this.host.selection.value;
this.requestUpdate();
})
);
const { docModeService } = this.host.spec.getService('affine:page');
@ -226,12 +230,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
: 'What can I help you with?'}
</div>
</div>
<chat-cards
.chatContextValue=${this.chatContextValue}
.updateContext=${this.updateContext}
.host=${this.host}
.selectionValue=${this._selectionValue}
></chat-cards> `
${cache(
this.showChatCards
? html`
<chat-cards
.updateContext=${this.updateContext}
.host=${this.host}
></chat-cards>
`
: nothing
)}`
: repeat(filteredItems, (item, index) => {
const isLast = index === filteredItems.length - 1;
return html`<div class="message">
@ -265,6 +273,12 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
}
})
);
this.disposables.add(
AIProvider.slots.toggleChatCards.on(({ visible }) => {
this.showChatCards = visible;
})
);
}
renderError() {

View File

@ -190,6 +190,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
AIProvider.slots.actions.on(({ action, event }) => {
const { status } = this.chatContextValue;
if (
action !== 'chat' &&
event === 'finished' &&
@ -197,6 +198,13 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
) {
this._resetItems();
}
if (action === 'chat' && event === 'finished') {
AIProvider.slots.toggleChatCards.emit({
visible: true,
ok: status === 'success',
});
}
});
AIProvider.slots.userInfo.on(userInfo => {

View File

@ -19,6 +19,7 @@ import {
AIPresentationIconWithAnimation,
AISearchIcon,
ChatWithAIIcon,
CommentIcon,
ExplainIcon,
ImproveWritingIcon,
LanguageIcon,
@ -98,6 +99,19 @@ export const imageProcessingSubItem = imageProcessingTypes.map(type => {
const othersGroup: AIItemGroupConfig = {
name: 'others',
items: [
{
name: 'Continue with AI',
icon: CommentIcon,
showWhen: () => true,
handler: host => {
const panel = getAIPanel(host);
AIProvider.slots.requestContinueWithAIInChat.emit({
host,
mode: 'edgeless',
});
panel.hide();
},
},
{
name: 'Open AI Chat',
icon: ChatWithAIIcon,

View File

@ -81,6 +81,10 @@ export class AIProvider {
// use case: when user selects "continue in chat" in an ask ai result panel
// do we need to pass the context to the chat panel?
requestContinueInChat: new Slot<{ host: EditorHost; show: boolean }>(),
requestContinueWithAIInChat: new Slot<{
host: EditorHost;
mode?: 'page' | 'edgeless';
}>(),
requestLogin: new Slot<{ host: EditorHost }>(),
requestUpgradePlan: new Slot<{ host: EditorHost }>(),
// when an action is requested to run in edgeless mode (show a toast in affine)
@ -94,6 +98,10 @@ export class AIProvider {
// downstream can emit this slot to notify ai presets that user info has been updated
userInfo: new Slot<AIUserInfo | null>(),
// add more if needed
toggleChatCards: new Slot<{
visible: boolean;
ok?: boolean;
}>(),
};
// track the history of triggered actions (in memory only)