feat: copilot frontend e2e (#7694)

.github/actions/copilot-test/action.yml

@@ -0,0 +1,45 @@
+name: 'Run Copilot E2E Test'
+description: 'Run Copilot E2E Test'
+inputs:
+  openai-key:
+    description: 'OpenAI secret key'
+    required: true
+  fal-key:
+    description: 'Fal secret key'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Initialize database
+      shell: bash
+      run: |
+        psql -h localhost -U postgres -c "CREATE DATABASE affine;"
+        psql -h localhost -U postgres -c "CREATE USER affine WITH PASSWORD 'affine';"
+        psql -h localhost -U postgres -c "ALTER USER affine WITH SUPERUSER;"
+      env:
+        PGPASSWORD: affine
+
+    - name: Run init-db script
+      shell: bash
+      run: |
+        yarn workspace @affine/server exec prisma generate
+        yarn workspace @affine/server exec prisma db push
+        yarn workspace @affine/server data-migration run
+
+    - name: Server Copilot E2E Test
+      shell: bash
+      run: yarn workspace @affine-test/affine-cloud e2e:copilot --forbid-only
+      env:
+        COPILOT: true
+        DEV_SERVER_URL: http://localhost:8080
+        COPILOT_OPENAI_API_KEY: ${{ inputs.openai-key }}
+        COPILOT_FAL_API_KEY: ${{ inputs.fal-key }}
+
+    - name: Upload test results
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results-e2e-server-copilot
+        path: ./test-results
+        if-no-files-found: ignore

.github/actions/setup-node/action.yml

@@ -17,6 +17,10 @@ inputs:
     description: 'Download the Electron binary'
     required: false
     default: 'true'
+  corepack-install:
+    description: 'Install CorePack'
+    required: false
+    default: 'false'
   hard-link-nm:
     description: 'set nmMode to hardlinks-local in .yarnrc.yml'
     required: false
@@ -42,6 +46,11 @@ runs:
         registry-url: https://npm.pkg.github.com
         scope: '@toeverything'
 
+    - name: Init CorePack
+      if: ${{ inputs.corepack-install == 'true' }}
+      shell: bash
+      run: corepack enable
+
     - name: Set nmMode
       if: ${{ inputs.hard-link-nm == 'false' }}
       shell: bash

.github/workflows/build-test.yml

@@ -392,6 +392,49 @@ jobs:
           name: affine
           fail_ci_if_error: false
 
+  copilot-e2e-test:
+    name: Server Copilot E2E Test
+    runs-on: [self-hosted, copilot-ci]
+    env:
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      IN_CI_TEST: true
+    needs:
+      - build-server-native
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          corepack-install: true
+          playwright-install: true
+          hard-link-nm: false
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Run Copilot E2E Test
+        uses: ./.github/actions/copilot-test
+        with:
+          openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}
+
   server-e2e-test:
     name: ${{ matrix.tests.name }}
     runs-on: ubuntu-latest
@@ -631,6 +674,7 @@ jobs:
       - build-server-native
       - build-electron-renderer
       - server-test
+      - copilot-e2e-test
       - server-e2e-test
       - desktop-test
       - test-build-ios

.github/workflows/copilot-test-automatically.yml

@@ -0,0 +1,26 @@
+name: Copilot Test Automatically
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+-canary.[0-9]+'
+  schedule:
+    - cron: '0 8 * * *'
+  workflow_dispatch:
+
+jobs:
+  dispatch-deploy:
+    runs-on: ubuntu-latest
+    name: Setup Deploy
+    steps:
+      - name: dispatch deploy by tag
+        if: ${{ github.event_name == 'push' }}
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: deploy.yml
+      - name: dispatch deploy by schedule
+        if: ${{ github.event_name == 'schedule' }}
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: deploy.yml
+          ref: canary

.github/workflows/copilot-test.yml

@@ -0,0 +1,76 @@
+name: Copilot Cron Test
+
+on:
+  workflow_dispatch:
+
+env:
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+
+jobs:
+  build-server-native:
+    name: Build Server native
+    runs-on: ubuntu-latest
+    env:
+      CARGO_PROFILE_RELEASE_DEBUG: '1'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          extra-flags: workspaces focus @affine/server-native
+          electron-install: false
+      - name: Build Rust
+        uses: ./.github/actions/build-rust
+        with:
+          target: 'x86_64-unknown-linux-gnu'
+          package: '@affine/server-native'
+          nx_token: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
+      - name: Upload server-native.node
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/native/server-native.node
+          if-no-files-found: error
+
+  copilot-e2e-test:
+    name: Server Copilot E2E Test
+    runs-on: [self-hosted, copilot-ci]
+    env:
+      DISTRIBUTION: web
+      DATABASE_URL: postgresql://affine:affine@localhost:5432/affine
+      IN_CI_TEST: true
+    needs:
+      - build-server-native
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: affine
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: ./.github/actions/setup-node
+        with:
+          corepack-install: true
+          playwright-install: true
+          hard-link-nm: false
+
+      - name: Download server-native.node
+        uses: actions/download-artifact@v4
+        with:
+          name: server-native.node
+          path: ./packages/backend/server
+
+      - name: Run Copilot E2E Test
+        uses: ./.github/actions/copilot-test
+        with:
+          openai-key: ${{ secrets.COPILOT_OPENAI_API_KEY }}
+          fal-key: ${{ secrets.COPILOT_FAL_API_KEY }}

packages/backend/server/src/plugins/copilot/prompt/prompts.ts

@@ -522,7 +522,7 @@ Rules to follow:
     messages: [
       {
         role: 'system',
-        content: `You are a creative blog writer specializing in producing captivating and informative content. Your task is to write a blog post based on the content provided by user in its original language. The blog post should be between 500-700 words, engaging, and well-structured, with an inviting introduction that hooks the reader, concise and informative body paragraphs, and a compelling conclusion that encourages readers to engage with the content, whether it's through commenting, sharing, or exploring the topics further. Please ensure the blog post is optimized for SEO with relevant keywords, includes at least 2-3 subheadings for better readability, and whenever possible, provides actionable insights or takeaways for the reader. Integrate a friendly and approachable tone throughout the post that reflects the voice of someone knowledgeable yet relatable. And ultimately output the content in Markdown format. Do not put everything into a single code block unless everything is code.`,
+        content: `You are a creative blog writer specializing in producing captivating and informative content. Your task is to write a blog post based on the content provided by user in its original language. The blog post should be between 500-700 words, engaging, and well-structured, with an inviting introduction that hooks the reader, concise and informative body paragraphs, and a compelling conclusion that encourages readers to engage with the content, whether it's through commenting, sharing, or exploring the topics further. Please ensure the blog post is optimized for SEO with relevant keywords, includes at least 2-3 subheadings for better readability, and whenever possible, provides actionable insights or takeaways for the reader. Integrate a friendly and approachable tone throughout the post that reflects the voice of someone knowledgeable yet relatable. And ultimately output the content in Markdown format. You should not place the entire article in a code block.`,
       },
       {
         role: 'user',

packages/frontend/core/src/blocksuite/presets/ai/_common/components/chat-action-list.ts

@@ -153,6 +153,9 @@ export class ChatActionList extends LitElement {
                     });
                   }
                 }}
+                data-testid="action-${action.title
+                  .toLowerCase()
+                  .replaceAll(' ', '-')}"
               >
                 ${action.title}
               </div>

packages/frontend/core/src/blocksuite/presets/ai/_common/components/copy-more.ts

@@ -178,13 +178,17 @@ export class ChatCopyMore extends WithDisposable(LitElement) {
                   this._notifySuccess('Copied to clipboard');
                 }
               }}
+              data-testid="action-copy-button"
             >
               ${CopyIcon}
               <affine-tooltip>Copy</affine-tooltip>
             </div>`
           : nothing}
         ${isLast
-          ? html`<div @click=${() => this.retry()}>
+          ? html`<div
+              @click=${() => this.retry()}
+              data-testid="action-retry-button"
+            >
               ${RetryIcon}
               <affine-tooltip>Retry</affine-tooltip>
             </div>`

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts

@@ -371,6 +371,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
               }
             }
           }}
+          data-testid="chat-panel-input"
         ></textarea>
         <div class="chat-panel-input-actions">
           <div
@@ -378,6 +379,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
             @click=${async () => {
               await this.cleanupHistories();
             }}
+            data-testid="chat-panel-clear"
           >
             ${ChatClearIcon}
           </div>
@@ -410,6 +412,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
                 @click="${this.send}"
                 class="chat-panel-send"
                 aria-disabled=${this.isInputEmpty}
+                data-testid="chat-panel-send"
               >
                 ${ChatSendIcon}
               </div>`}

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts

@@ -379,7 +379,7 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
               : html`<div class="avatar"></div>`}
           </div>`
         : AffineAvatarIcon}
-      ${isUser ? 'You' : 'AFFINE AI'}
+      ${isUser ? 'You' : 'AFFiNE AI'}
     </div>`;
   }
 

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts

@@ -244,7 +244,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
   override render() {
     return html` <div class="chat-panel-container">
       <div class="chat-panel-title">
-        <div>AFFINE AI</div>
+        <div>AFFiNE AI</div>
         <div
           @click=${() => {
             AIProvider.toggleGeneralAIOnboarding?.(true);

packages/frontend/core/src/modules/workbench/view/route-container.tsx

@@ -34,6 +34,7 @@ const ToggleButton = ({
       onClick={onToggle}
       className={className}
       data-show={show}
+      data-testid="right-sidebar-toggle"
     >
       <RightSidebarIcon />
     </IconButton>

packages/frontend/core/src/modules/workbench/view/sidebar/sidebar-container.tsx

@@ -40,6 +40,7 @@ export const SidebarContainer = ({
               styles.sidebarBodyTarget,
               !BUILD_CONFIG.isElectron && styles.borderTop
             )}
+            data-testid={`sidebar-tab-content-${sidebar.id}`}
           />
         ))
       ) : (

packages/frontend/core/src/modules/workbench/view/sidebar/sidebar-header-switcher.tsx

@@ -22,6 +22,7 @@ export const SidebarHeaderSwitcher = () => {
         tabId={tab.id}
       />
     ),
+    testId: `sidebar-tab-${tab.id}`,
     style: { padding: 0, fontSize: 20, width: 24 },
   }));
 

tests/affine-cloud/e2e/copilot.spec.ts

@@ -0,0 +1,589 @@
+import { test } from '@affine-test/kit/playwright';
+import {
+  createRandomAIUser,
+  loginUser,
+  loginUserDirectly,
+} from '@affine-test/kit/utils/cloud';
+import { openHomePage, setCoreUrl } from '@affine-test/kit/utils/load-page';
+import {
+  clickNewPageButton,
+  getBlockSuiteEditorTitle,
+  waitForEditorLoad,
+} from '@affine-test/kit/utils/page-logic';
+import { clickSideBarAllPageButton } from '@affine-test/kit/utils/sidebar';
+import { createLocalWorkspace } from '@affine-test/kit/utils/workspace';
+import { expect, type Page } from '@playwright/test';
+
+const ONE_SECOND = 1000;
+const TEN_SECONDS = 10 * ONE_SECOND;
+const ONE_MINUTE = 60 * ONE_SECOND;
+
+let isProduction = process.env.NODE_ENV === 'production';
+if (
+  process.env.PLAYWRIGHT_USER_AGENT &&
+  process.env.PLAYWRIGHT_EMAIL &&
+  !process.env.PLAYWRIGHT_PASSWORD
+) {
+  test.use({
+    userAgent: process.env.PLAYWRIGHT_USER_AGENT || 'affine-tester',
+  });
+  setCoreUrl(process.env.PLAYWRIGHT_CORE_URL || 'http://localhost:8080');
+  isProduction = true;
+}
+
+function getUser() {
+  if (
+    !isProduction ||
+    !process.env.PLAYWRIGHT_EMAIL ||
+    !process.env.PLAYWRIGHT_PASSWORD
+  ) {
+    return createRandomAIUser();
+  }
+
+  return {
+    email: process.env.PLAYWRIGHT_EMAIL,
+    password: process.env.PLAYWRIGHT_PASSWORD,
+  };
+}
+
+test.skip(
+  () =>
+    !process.env.COPILOT_OPENAI_API_KEY ||
+    !process.env.COPILOT_FAL_API_KEY ||
+    process.env.COPILOT_OPENAI_API_KEY === '1' ||
+    process.env.COPILOT_FAL_API_KEY === '1',
+  'skip test if no copilot api key'
+);
+
+test('can open chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await page.getByTestId('right-sidebar-toggle').click({
+    delay: 200,
+  });
+  await page.getByTestId('sidebar-tab-chat').click();
+  await expect(page.getByTestId('sidebar-tab-content-chat')).toBeVisible();
+});
+
+const makeChat = async (page: Page, content: string) => {
+  if (await page.getByTestId('sidebar-tab-chat').isHidden()) {
+    await page.getByTestId('right-sidebar-toggle').click({
+      delay: 200,
+    });
+  }
+  await page.getByTestId('sidebar-tab-chat').click();
+  await page.getByTestId('chat-panel-input').focus();
+  await page.keyboard.type(content);
+  await page.keyboard.press('Enter');
+};
+
+const clearChat = async (page: Page) => {
+  await page.getByTestId('chat-panel-clear').click();
+  await page.getByTestId('confirm-modal-confirm').click();
+};
+
+const collectChat = async (page: Page) => {
+  const chatPanel = await page.waitForSelector('.chat-panel-messages');
+  if (await chatPanel.$('.chat-panel-messages-placeholder')) {
+    return [];
+  }
+  // wait ai response
+  await page.waitForSelector('.chat-panel-messages .message chat-copy-more');
+  const lastMessage = await chatPanel.$$('.message').then(m => m[m.length - 1]);
+  await lastMessage.waitForSelector('chat-copy-more');
+  await page.waitForTimeout(ONE_SECOND);
+  return Promise.all(
+    Array.from(await chatPanel.$$('.message')).map(async m => ({
+      name: await m.$('.user-info').then(i => i?.innerText()),
+      content: await m
+        .$('chat-text')
+        .then(t => t?.$('editor-host'))
+        .then(e => e?.innerText()),
+    }))
+  );
+};
+
+const focusToEditor = async (page: Page) => {
+  const title = getBlockSuiteEditorTitle(page);
+  await title.focus();
+  await page.keyboard.press('Enter');
+};
+
+const getEditorContent = async (page: Page) => {
+  const lines = await page.$$('page-editor .inline-editor');
+  const contents = await Promise.all(lines.map(el => el.innerText()));
+  return (
+    contents
+      // cleanup zero width space
+      .map(c => c.replace(/\u200B/g, '').trim())
+      .filter(c => !!c)
+      .join('\n')
+  );
+};
+
+const switchToEdgelessMode = async (page: Page) => {
+  const editor = await page.waitForSelector('page-editor');
+  await page.getByTestId('switch-edgeless-mode-button').click();
+  // wait for new editor
+  editor.waitForElementState('hidden');
+  await page.waitForSelector('edgeless-editor');
+};
+
+test('can trigger login at chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await makeChat(page, 'hello');
+  const loginTips = await page.waitForSelector('ai-error-wrapper');
+  expect(await loginTips.innerText()).toBe('Login');
+});
+
+test('can chat after login at chat side panel', async ({ page }) => {
+  await openHomePage(page);
+  await waitForEditorLoad(page);
+  await clickNewPageButton(page);
+  await makeChat(page, 'hello');
+  const loginTips = await page.waitForSelector('ai-error-wrapper');
+  (await loginTips.$('div'))!.click();
+  // login
+  const user = await getUser();
+  await loginUserDirectly(page, user);
+  // after login
+  await makeChat(page, 'hello');
+  const history = await collectChat(page);
+  expect(history[0]).toEqual({ name: 'You', content: 'hello' });
+  expect(history[1].name).toBe('AFFiNE AI');
+});
+
+test.describe('chat panel', () => {
+  let user: {
+    email: string;
+    password: string;
+  };
+
+  test.beforeEach(async ({ page }) => {
+    user = await getUser();
+    await loginUser(page, user);
+  });
+
+  test('basic chat', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const history = await collectChat(page);
+    expect(history[0]).toEqual({ name: 'You', content: 'hello' });
+    expect(history[1].name).toBe('AFFiNE AI');
+    await clearChat(page);
+    expect((await collectChat(page)).length).toBe(0);
+  });
+
+  test('chat actions', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await page.getByTestId('action-copy-button').click();
+    await page.waitForTimeout(500);
+    expect(await page.evaluate(() => navigator.clipboard.readText())).toBe(
+      content
+    );
+    await page.getByTestId('action-retry-button').click();
+    expect((await collectChat(page))[1].content).not.toBe(content);
+  });
+
+  test('can be insert below', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await focusToEditor(page);
+    // insert below
+    await page.getByTestId('action-insert-below').click();
+    await page.waitForSelector('affine-format-bar-widget editor-toolbar');
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  test('can be add to edgeless as node', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    await switchToEdgelessMode(page);
+    // delete default note
+    await (await page.waitForSelector('affine-edgeless-note')).click();
+    page.keyboard.press('Delete');
+    // insert note
+    await page.getByTestId('action-add-to-edgeless-as-note').click();
+    const edgelessNode = await page.waitForSelector('affine-edgeless-note');
+    expect(await edgelessNode.innerText()).toBe(content);
+  });
+
+  test('can be create as a doc', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const content = (await collectChat(page))[1].content;
+    const editor = await page.waitForSelector('page-editor');
+    await page.getByTestId('action-create-as-a-doc').click();
+    // wait for new editor
+    editor.waitForElementState('hidden');
+    await page.waitForSelector('page-editor');
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  // feature not launched yet
+  test.skip('can be save chat to block', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await makeChat(page, 'hello');
+    const contents = (await collectChat(page)).map(m => m.content);
+    await switchToEdgelessMode(page);
+    await page.getByTestId('action-save-chat-to-block').click();
+    const chatBlock = await page.waitForSelector('affine-edgeless-ai-chat');
+    expect(
+      await Promise.all(
+        (await chatBlock.$$('.ai-chat-user-message')).map(m => m.innerText())
+      )
+    ).toBe(contents);
+  });
+
+  test('can be chat and insert below in page mode', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await focusToEditor(page);
+    await page.keyboard.type('/');
+    await page.getByTestId('sub-menu-0').getByText('Ask AI').click();
+    const input = await page.waitForSelector('ai-panel-input textarea');
+    await input.fill('hello');
+    await input.press('Enter');
+    const resp = await page.waitForSelector(
+      'ai-panel-answer .response-list-container'
+    ); // wait response
+    const content = await (
+      await page.waitForSelector('ai-panel-answer editor-host')
+    ).innerText();
+    await (await resp.waitForSelector('.ai-item-insert-below')).click();
+    const editorContent = await getEditorContent(page);
+    expect(editorContent).toBe(content);
+  });
+
+  test('can be retry or discard chat in page mode', async ({ page }) => {
+    await page.reload();
+    await clickSideBarAllPageButton(page);
+    await page.waitForTimeout(200);
+    await createLocalWorkspace({ name: 'test' }, page);
+    await clickNewPageButton(page);
+    await focusToEditor(page);
+    await page.keyboard.type('/');
+    await page.getByTestId('sub-menu-0').getByText('Ask AI').click();
+    const input = await page.waitForSelector('ai-panel-input textarea');
+    await input.fill('hello');
+    await input.press('Enter');
+    // regenerate
+    {
+      const resp = await page.waitForSelector(
+        'ai-panel-answer .response-list-container:last-child'
+      );
+      const answerEditor = await page.waitForSelector(
+        'ai-panel-answer editor-host'
+      );
+      const content = await answerEditor.innerText();
+      await (await resp.waitForSelector('.ai-item-regenerate')).click();
+      await page.waitForSelector('ai-panel-answer .response-list-container'); // wait response
+      expect(
+        await (
+          await (
+            await page.waitForSelector('ai-panel-answer')
+          ).waitForSelector('editor-host')
+        ).innerText()
+      ).not.toBe(content);
+    }
+
+    // discard
+    {
+      const resp = await page.waitForSelector(
+        'ai-panel-answer .response-list-container:last-child'
+      );
+      await (await resp.waitForSelector('.ai-item-discard')).click();
+      await page.getByTestId('confirm-modal-confirm').click();
+      const editorContent = await getEditorContent(page);
+      expect(editorContent).toBe('');
+    }
+  });
+});
+
+test.describe('chat with block', () => {
+  let user: {
+    email: string;
+    password: string;
+  };
+
+  test.beforeEach(async ({ page }) => {
+    user = await getUser();
+    await loginUser(page, user);
+  });
+
+  const collectTextAnswer = async (page: Page) => {
+    // wait ai response
+    await page.waitForSelector(
+      'affine-ai-panel-widget .response-list-container',
+      { timeout: ONE_MINUTE }
+    );
+    const answer = await page.waitForSelector(
+      'affine-ai-panel-widget ai-panel-answer editor-host'
+    );
+    return answer.innerText();
+  };
+
+  const collectImageAnswer = async (page: Page, timeout = TEN_SECONDS) => {
+    // wait ai response
+    await page.waitForSelector(
+      'affine-ai-panel-widget .response-list-container',
+      { timeout }
+    );
+    const answer = await page.waitForSelector(
+      'affine-ai-panel-widget .ai-answer-image img'
+    );
+    return answer.getAttribute('src');
+  };
+
+  const disableEditorBlank = async (page: Page) => {
+    // hide blank element, this may block the click
+    const blank = page.getByTestId('page-editor-blank');
+    if (await blank.isVisible()) {
+      await blank.evaluate(node => (node.style.pointerEvents = 'none'));
+    } else {
+      console.warn('blank element not found');
+    }
+  };
+
+  test.describe('chat with text', () => {
+    const pasteTextToPageEditor = async (page: Page, content: string) => {
+      await focusToEditor(page);
+      await page.keyboard.type(content);
+    };
+
+    test.beforeEach(async ({ page }) => {
+      await page.reload();
+      await clickSideBarAllPageButton(page);
+      await page.waitForTimeout(200);
+      await createLocalWorkspace({ name: 'test' }, page);
+      await clickNewPageButton(page);
+      await pasteTextToPageEditor(page, 'hello');
+    });
+
+    test.beforeEach(async ({ page }) => {
+      await page.waitForSelector('affine-paragraph').then(i => i.click());
+      await page.keyboard.press('ControlOrMeta+A');
+      await page
+        .waitForSelector('page-editor editor-toolbar ask-ai-button')
+        .then(b => b.click());
+    });
+
+    const options = [
+      // review with ai
+      'Fix spelling',
+      'Fix Grammar',
+      // edit with ai
+      'Explain selection',
+      'Improve writing',
+      'Make it longer',
+      'Make it shorter',
+      'Continue writing',
+      // generate with ai
+      'Summarize',
+      'Generate headings',
+      'Generate outline',
+      'Find actions',
+      // draft with ai
+      'Write an article about this',
+      'Write a tweet about this',
+      'Write a poem about this',
+      'Write a blog post about this',
+      'Brainstorm ideas about this',
+    ];
+    for (const option of options) {
+      test(option, async ({ page }) => {
+        await disableEditorBlank(page);
+        await page
+          .waitForSelector(
+            `.ai-item-${option.replaceAll(' ', '-').toLowerCase()}`
+          )
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+    }
+  });
+
+  test.describe('chat with image block', () => {
+    const pasteImageToPageEditor = async (page: Page) => {
+      await page.evaluate(async () => {
+        const canvas = document.createElement('canvas');
+        canvas.width = 100;
+        canvas.height = 100;
+        const blob = await new Promise<Blob>(resolve =>
+          canvas.toBlob(blob => resolve(blob!), 'image/png')
+        );
+        await navigator.clipboard.write([
+          new ClipboardItem({ [blob.type]: blob }),
+        ]);
+      });
+      await focusToEditor(page);
+      await page.keyboard.press('ControlOrMeta+V');
+    };
+
+    test.beforeEach(async ({ page }) => {
+      await page.reload();
+      await clickSideBarAllPageButton(page);
+      await page.waitForTimeout(200);
+      await createLocalWorkspace({ name: 'test' }, page);
+      await clickNewPageButton(page);
+      await pasteImageToPageEditor(page);
+    });
+
+    test.describe('page mode', () => {
+      test.beforeEach(async ({ page }) => {
+        await disableEditorBlank(page);
+        await page.waitForSelector('affine-image').then(i => i.click());
+        await page
+          .waitForSelector('affine-image editor-toolbar ask-ai-button')
+          .then(b => b.click());
+      });
+
+      test('explain this image', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-explain-this-image')
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+
+      test('generate a caption', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-generate-a-caption')
+          .then(i => i.click());
+        expect(await collectTextAnswer(page)).toBeTruthy();
+      });
+
+      test('continue with ai', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-continue-with-ai')
+          .then(i => i.click());
+        await page
+          .waitForSelector('chat-panel-input .chat-panel-images')
+          .then(el => el.waitForElementState('visible'));
+      });
+
+      test('open ai chat', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-open-ai-chat')
+          .then(i => i.click());
+        const cards = await page.waitForSelector('chat-panel chat-cards');
+        await cards.waitForElementState('visible');
+        const cardTitles = await Promise.all(
+          await cards
+            .$$('.card-wrapper .card-title')
+            .then(els => els.map(async el => await el.innerText()))
+        );
+        expect(cardTitles).toContain('Start with this Image');
+      });
+    });
+
+    // TODO(@darkskygit): block by BS-1709, enable this after bug fix
+    test.describe.skip('edgeless mode', () => {
+      test.beforeEach(async ({ page }) => {
+        await switchToEdgelessMode(page);
+        const note = await page.waitForSelector('affine-edgeless-note');
+
+        {
+          // move note to avoid menu overlap
+          const box = (await note.boundingBox())!;
+          page.mouse.move(box.x + box.width / 2, box.y + box.height / 2);
+          page.mouse.down();
+          // sleep to avoid flicker
+          await page.waitForTimeout(500);
+          page.mouse.move(box.x + box.width / 2, box.y + box.height / 2 - 200);
+          await page.waitForTimeout(500);
+          page.mouse.up();
+          note.click();
+        }
+
+        await disableEditorBlank(page);
+        await page.waitForSelector('affine-image').then(i => i.click());
+        await page
+          .waitForSelector('affine-image editor-toolbar ask-ai-button')
+          .then(b => b.click());
+      });
+
+      // skip by default, dalle is very slow
+      test.skip('generate an image', async ({ page }) => {
+        await page
+          .waitForSelector('.ai-item-generate-an-image')
+          .then(i => i.click());
+        await page.keyboard.type('a cat');
+        await page.keyboard.press('Enter');
+        expect(await collectImageAnswer(page)).toBeTruthy();
+      });
+
+      const processes = [
+        'Clearer',
+        'Remove background',
+        // skip by default, need a face in image
+        // 'Convert to sticker',
+      ];
+      for (const process of processes) {
+        test(`image processing ${process}`, async ({ page }) => {
+          await page
+            .waitForSelector('.ai-item-image-processing')
+            .then(i => i.hover());
+          await page.getByText(process).click();
+          {
+            // to be remove
+            await page.keyboard.type(',');
+            await page.keyboard.press('Enter');
+          }
+          expect(await collectImageAnswer(page, ONE_MINUTE * 2)).toBeTruthy();
+        });
+      }
+
+      const filters = ['Clay', 'Sketch', 'Anime', 'Pixel'];
+      for (const filter of filters) {
+        test(`ai image ${filter.toLowerCase()} filter`, async ({ page }) => {
+          await page
+            .waitForSelector('.ai-item-ai-image-filter')
+            .then(i => i.hover());
+          await page.getByText(`${filter} style`).click();
+          {
+            // to be remove
+            await page.keyboard.type(',');
+            await page.keyboard.press('Enter');
+          }
+          expect(await collectImageAnswer(page, ONE_MINUTE * 2)).toBeTruthy();
+        });
+      }
+    });
+  });
+});

tests/affine-cloud/package.json

@@ -2,7 +2,8 @@
   "name": "@affine-test/affine-cloud",
   "private": true,
   "scripts": {
-    "e2e": "yarn playwright test"
+    "e2e": "yarn playwright test",
+    "e2e:copilot": "yarn playwright test e2e/copilot.spec.ts"
   },
   "devDependencies": {
     "@affine-test/kit": "workspace:*",

tests/affine-cloud/playwright.config.ts

@@ -22,7 +22,7 @@ const config: PlaywrightTestConfig = {
     video: 'on',
   },
   forbidOnly: !!process.env.CI,
-  workers: process.env.CI ? 1 : 4,
+  workers: process.env.CI && !process.env.COPILOT ? 1 : 4,
   retries: 1,
   reporter: process.env.CI ? 'github' : 'list',
   webServer: [

tests/affine-local/e2e/all-page.spec.ts

@@ -16,32 +16,14 @@ import { openHomePage } from '@affine-test/kit/utils/load-page';
 import {
   clickNewPageButton,
   clickPageMoreActions,
+  getAllPage,
   getBlockSuiteEditorTitle,
   waitForAllPagesLoad,
   waitForEditorLoad,
 } from '@affine-test/kit/utils/page-logic';
 import { clickSideBarAllPageButton } from '@affine-test/kit/utils/sidebar';
-import type { Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
-function getAllPage(page: Page) {
-  const newPageButton = page.getByTestId('new-page-button-trigger');
-  const newPageDropdown = newPageButton.locator('svg');
-  const edgelessBlockCard = page.getByTestId('new-edgeless-button-in-all-page');
-
-  async function clickNewPageButton() {
-    const newPageButton = page.getByTestId('new-page-button-trigger');
-    return await newPageButton.click();
-  }
-
-  async function clickNewEdgelessDropdown() {
-    await newPageDropdown.click();
-    await edgelessBlockCard.click();
-  }
-
-  return { clickNewPageButton, clickNewEdgelessDropdown };
-}
-
 test('all page', async ({ page }) => {
   await openHomePage(page);
   await waitForEditorLoad(page);

tests/kit/utils/cloud.ts

@@ -150,6 +150,68 @@ export async function createRandomUser(): Promise<{
   } as any;
 }
 
+export async function createRandomAIUser(): Promise<{
+  name: string;
+  email: string;
+  password: string;
+  id: string;
+}> {
+  const user = {
+    name: faker.internet.userName(),
+    email: faker.internet.email().toLowerCase(),
+    password: '123456',
+  };
+  const result = await runPrisma(async client => {
+    const freeFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'free_plan_v1' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+    const aiFeatureId = await client.feature
+      .findFirst({
+        where: { feature: 'unlimited_copilot' },
+        select: { id: true },
+        orderBy: { version: 'desc' },
+      })
+      .then(f => f!.id);
+
+    await client.user.create({
+      data: {
+        ...user,
+        emailVerifiedAt: new Date(),
+        password: await hash(user.password),
+        features: {
+          create: [
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: freeFeatureId,
+            },
+            {
+              reason: 'created by test case',
+              activated: true,
+              featureId: aiFeatureId,
+            },
+          ],
+        },
+      },
+    });
+
+    return await client.user.findUnique({
+      where: {
+        email: user.email,
+      },
+    });
+  });
+  cloudUserSchema.parse(result);
+  return {
+    ...result,
+    password: user.password,
+  } as any;
+}
+
 export async function deleteUser(email: string) {
   await runPrisma(async client => {
     await client.user.delete({
@@ -178,7 +240,24 @@ export async function loginUser(
   }
 
   await clickSideBarCurrentWorkspaceBanner(page);
-  await page.getByTestId('cloud-signin-button').click();
+  await page.getByTestId('cloud-signin-button').click({
+    delay: 200,
+  });
+  await loginUserDirectly(page, user, config);
+}
+
+export async function loginUserDirectly(
+  page: Page,
+  user: {
+    email: string;
+    password: string;
+  },
+  config?: {
+    isElectron?: boolean;
+    beforeLogin?: () => Promise<void>;
+    afterLogin?: () => Promise<void>;
+  }
+) {
   await page.getByPlaceholder('Enter your email address').fill(user.email);
   await page.getByTestId('continue-login-button').click({
     delay: 200,
@@ -188,8 +267,10 @@ export async function loginUser(
     await config.beforeLogin();
   }
   await page.waitForTimeout(200);
-  await page.getByTestId('sign-in-button').click();
-  await page.waitForTimeout(500);
+  const signIn = page.getByTestId('sign-in-button');
+  await signIn.click();
+  await signIn.waitFor({ state: 'detached' });
+  await page.waitForTimeout(200);
   if (config?.afterLogin) {
     await config.afterLogin();
   }

tests/kit/utils/load-page.ts

@@ -1,7 +1,11 @@
 import type { Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
-export const coreUrl = 'http://localhost:8080';
+export let coreUrl = 'http://localhost:8080';
+
+export function setCoreUrl(url: string) {
+  coreUrl = url;
+}
 
 export async function openHomePage(page: Page) {
   await page.goto(coreUrl);

tests/kit/utils/page-logic.ts

@@ -1,6 +1,24 @@
 import type { Locator, Page } from '@playwright/test';
 import { expect } from '@playwright/test';
 
+export function getAllPage(page: Page) {
+  const newPageButton = page.getByTestId('new-page-button-trigger');
+  const newPageDropdown = newPageButton.locator('svg');
+  const edgelessBlockCard = page.getByTestId('new-edgeless-button-in-all-page');
+
+  async function clickNewPageButton() {
+    const newPageButton = page.getByTestId('new-page-button-trigger');
+    return await newPageButton.click();
+  }
+
+  async function clickNewEdgelessDropdown() {
+    await newPageDropdown.click();
+    await edgelessBlockCard.click();
+  }
+
+  return { clickNewPageButton, clickNewEdgelessDropdown };
+}
+
 export async function waitForEditorLoad(page: Page) {
   await page.waitForSelector('v-line', {
     timeout: 20000,