E2E tests (#6717)

Continuation of #6644 

Now chromium browser is used in workspaces tests instead of firefox and
screenshots after each test are properly saved in one folder when run
from IDE and from terminal using `yarn test:e2e` command

.github/workflows/playwright.yml.bak

@@ -0,0 +1,27 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: lts/*
+    - name: Install dependencies
+      run: npm install -g yarn && yarn
+    - name: Install Playwright Browsers
+      run: yarn playwright install --with-deps
+    - name: Run Playwright tests
+      run: yarn test:e2e companies
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -26,5 +26,5 @@ dist
 storybook-static
 *.tsbuildinfo
 .eslintcache
-.cache
 .nyc_output
+test-results/

package.json

@@ -222,6 +222,7 @@
     "@nx/storybook": "18.3.3",
     "@nx/vite": "18.3.3",
     "@nx/web": "18.3.3",
+    "@playwright/test": "^1.46.0",
     "@sentry/types": "^7.109.0",
     "@storybook/addon-actions": "^7.6.3",
     "@storybook/addon-coverage": "^1.0.0",

packages/twenty-e2e-testing/.env.example

@@ -1,2 +1,22 @@
 # Note that provide always without trailing forward slash to have expected behaviour
-FRONTEND_BASE_URL="http://localhost:3001"
+FRONTEND_BASE_URL=http://localhost:3001
+CI_DEFAULT_BASE_URL=https://demo.twenty.com
+DEFAULT_LOGIN=tim@apple.dev
+NEW_WORKSPACE_LOGIN=test@apple.dev
+DEMO_DEFAULT_LOGIN=noah@demo.dev
+DEFAULT_PASSWORD=Applecar2025
+WEBSITE_URL=https://twenty.com
+
+# === DO NOT USE, WORK IN PROGRESS ===
+# This URL must have trailing forward slash as all REST API endpoints have object after it
+# Documentation for REST API: https://twenty.com/developers/rest-api/core#/
+# REST_API_BASE_URL=http://localhost:3000/rest/
+
+# Documentation for GraphQL API: https://twenty.com/developers/graphql/core
+# GRAPHQL_BASE_URL=http://localhost:3000/graphql
+
+# Without this key, all API tests will fail, to generate this key
+# Log in to Twenty workspace, go to Settings > Developers, generate new key and paste it here
+# In order to use it, header Authorization: Bearer token must be used
+# This key works for both REST and GraphQL API
+# API_DEV_KEY=fill_with_proper_key

packages/twenty-e2e-testing/.gitignore

@@ -1,5 +1,10 @@
-node_modules/
-/test-results/
-/playwright-report/
-/blob-report/
-/playwright/.cache/
+test-results/
+playwright-report/
+blob-report/
+playwright/.cache/
+run_results/
+playwright-report/.last-run.json
+results/
+run_results/.playwright-artifacts-0/
+run_results/.playwright-artifacts-1/
+.auth/

packages/twenty-e2e-testing/README.md

@@ -1,8 +1,8 @@
 # Twenty e2e Testing
 
-## Install
+## Prerequisite
 
-Don't forget to install the browsers before launching the tests  :
+Installing the browsers:
 
 ```
 yarn playwright install 
@@ -35,3 +35,10 @@ yarn run test:e2e <filename>
 ```
 yarn run test:e2e:debug
 ```
+
+## Q&A
+
+#### Why there's `path.resolve()` everywhere?
+That's thanks to differences in root directory when running tests using commands and using IDE. When running tests with commands, 
+the root directory is `twenty/packages/twenty-e2e-testing`, for IDE it depends on how someone sets the configuration. This way, it
+ensures that no matter which IDE or OS Shell is used, the result will be the same.

packages/twenty-e2e-testing/config/customreporter.ts

@@ -0,0 +1,33 @@
+import {
+  Reporter,
+  FullConfig,
+  Suite,
+  TestCase,
+  TestResult,
+  FullResult,
+} from '@playwright/test/reporter';
+
+class CustomReporter implements Reporter {
+  constructor(options: { customOption?: string } = {}) {
+    console.log(
+      `my-awesome-reporter setup with customOption set to ${options.customOption}`,
+    );
+  }
+
+  onBegin(config: FullConfig, suite: Suite) {
+    console.log(`Starting the run with ${suite.allTests().length} tests`);
+  }
+
+  onTestBegin(test: TestCase) {
+    console.log(`Starting test ${test.title}`);
+  }
+
+  onTestEnd(test: TestCase, result: TestResult) {
+    console.log(`Finished test ${test.title}: ${result.status}`);
+  }
+
+  onEnd(result: FullResult) {
+    console.log(`Finished the run: ${result.status}`);
+  }
+}
+export default CustomReporter;

packages/twenty-e2e-testing/drivers/shell_driver.ts

@@ -0,0 +1,13 @@
+import { exec } from 'child_process';
+
+export async function sh(cmd) {
+  return new Promise((resolve, reject) => {
+    exec(cmd, (err, stdout, stderr) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve({ stdout, stderr });
+      }
+    });
+  });
+}

packages/twenty-e2e-testing/e2e/companies.spec.ts

@@ -1,14 +0,0 @@
-import { expect, test } from '@playwright/test';
-
-test.describe('visible table', () => {
-  test('table should be visible on navigation to /objects/companies', async ({
-    page,
-  }) => {
-    // Navigate to the page
-    await page.goto('/objects/companies');
-
-    // Check if the table is visible
-    const table = page.locator('table');
-    await expect(table).toBeVisible();
-  });
-});

packages/twenty-e2e-testing/lib/fixtures/screenshot.ts

@@ -0,0 +1,32 @@
+import { test as base } from '@playwright/test';
+import path from 'path';
+
+const date = new Date();
+
+export const test = base.extend<{ screenshotHook: void }>({
+  screenshotHook: [
+    async ({ page, browserName }, use, workerInfo) => {
+      // here everything is the same as beforeEach()
+      // goto is to go to website as login setup saves the cookies of logged-in user, not the state of browser
+      await page.goto('/');
+      await use(); // here is the code of test
+      // here everything is the same as afterEach()
+      // automatic fixture of making screenshot after each test
+      await page.screenshot({
+        path: path.resolve(
+          __dirname,
+          '..',
+          '..',
+          'results',
+          'screenshots',
+          `${workerInfo.project.name}`,
+          browserName,
+          `${date.toISOString()}.png`,
+        ),
+      });
+    },
+    { auto: true }, // automatic fixture runs with every test
+  ],
+});
+
+export { expect } from '@playwright/test';

packages/twenty-e2e-testing/playwright.config.ts

@@ -1,43 +1,85 @@
 import { defineConfig, devices } from '@playwright/test';
-
 import { config } from 'dotenv';
+import path from 'path';
+
 config();
 
+/* === Run your local dev server before starting the tests === */
+
 /**
  * See https://playwright.dev/docs/test-configuration.
- * See https://playwright.dev/docs/trace-viewer to Collect trace when retrying the failed test
  */
 export default defineConfig({
-  testDir: 'e2e',
-  /* Run tests in files in parallel */
-  fullyParallel: true,
-  reporter: 'html',
-  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  testDir: '.',
+  outputDir: 'run_results/', // directory for screenshots and videos
+  snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}', // just in case, do not delete it
+  fullyParallel: true, // false only for specific tests, overwritten in specific projects or global setups of projects
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined, // undefined = amount of projects * amount of tests
+  timeout: 30 * 1000, // timeout can be changed
   use: {
-    /* Base URL to use in actions like `await page.goto('/')`. */
-    baseURL: process.env.FRONTEND_BASE_URL ?? 'http://localhost:3001',
+    baseURL: process.env.CI
+      ? process.env.CI_DEFAULT_BASE_URL
+      : (process.env.FRONTEND_BASE_URL ?? 'http://localhost:3001'),
+    trace: 'retain-on-failure', // trace takes EVERYTHING from page source, records every single step, should be used only when normal debugging won't work
+    screenshot: 'on', // either 'on' here or in different method in modules, if 'on' all screenshots are overwritten each time the test is run
+    headless: true, // instead of changing it to false, run 'yarn test:e2e:debug' or 'yarn test:e2e:ui'
+    testIdAttribute: 'data-testid', // taken from Twenty source
+    viewport: { width: 1920, height: 1080 }, // most laptops use this resolution
+    launchOptions: {
+      slowMo: 500, // time in milliseconds between each step, better to use it than explicitly define timeout in tests
+    },
   },
-
-  /* Configure projects for major browsers */
+  expect: {
+    timeout: 5000,
+  },
+  reporter: [['html', { open: 'never' }]],
   projects: [
     {
-      name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
+      name: 'Login setup',
+      testMatch: /login\.setup\.ts/, // finds all tests matching this regex, in this case only 1 test should be found
+    },
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+        storageState: path.resolve(__dirname, '.auth', 'user.json'), // takes saved cookies from directory
+      },
+      dependencies: ['Login setup'], // forces to run login setup before running tests from this project - CASE SENSITIVE
     },
-
     {
       name: 'firefox',
-      use: { ...devices['Desktop Firefox'] },
+      use: {
+        ...devices['Desktop Firefox'],
+        storageState: path.resolve(__dirname, '.auth', 'user.json'),
+      },
+      dependencies: ['Login setup'],
     },
 
-    {
-      name: 'webkit',
-      use: { ...devices['Desktop Safari'] },
-    },
+    //{
+    //  name: 'webkit',
+    //  use: { ...devices['Desktop Safari'] },
+    //},
 
-    {
-      name: 'Google Chrome',
-      use: { ...devices['Desktop Chrome'], channel: 'chrome' },
-    },
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    //{
+    //  name: 'Microsoft Edge',
+    //  use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    //},
+    //{
+    //  name: 'Google Chrome',
+    //  use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    //},
   ],
 });

packages/twenty-e2e-testing/tests/companies.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from '../lib/fixtures/screenshot';
+import { config } from 'dotenv';
+import path = require('path');
+config({ path: path.resolve(__dirname, '..', '.env') });
+
+test.describe('Basic check', () => {
+  test('Checking if table in Companies is visible', async ({ page }) => {
+    await expect(page.getByTestId('tooltip').nth(0)).toHaveText('Companies');
+    await expect(page.getByTestId('tooltip').nth(0)).toBeVisible();
+    expect(page.url()).toContain('/companies');
+    await expect(page.locator('table')).toBeVisible();
+    await expect(page.locator('tbody > tr')).toHaveCount(13); // shouldn't be hardcoded in case of tests on demo
+  });
+
+  test('', async ({ page }) => {
+    await page.getByRole('link', { name: 'Opportunities' }).click();
+    await expect(page.locator('table')).toBeVisible();
+  });
+});

packages/twenty-e2e-testing/tests/login.setup.ts

@@ -0,0 +1,18 @@
+import { test as setup, expect } from '@playwright/test';
+import path from 'path';
+
+setup('Login test', async ({ page }) => {
+  await page.goto('/');
+  await page.getByRole('button', { name: 'Continue With Email' }).click();
+  await page.getByPlaceholder('Email').fill(process.env.DEFAULT_LOGIN);
+  await page.getByRole('button', { name: 'Continue', exact: true }).click();
+  await page.getByPlaceholder('Password').fill(process.env.DEFAULT_PASSWORD);
+  await page.getByRole('button', { name: 'Sign in' }).click();
+  await expect(page.getByText('Welcome to Twenty')).not.toBeVisible();
+
+  // End of authentication steps.
+
+  await page.context().storageState({
+    path: path.resolve(__dirname, '..', '.auth', 'user.json'),
+  });
+});

packages/twenty-e2e-testing/tests/workspaces.spec.ts

@@ -0,0 +1,66 @@
+import { test, expect } from '@playwright/test';
+import { sh } from '../drivers/shell_driver';
+
+test.describe('', () => {
+  test('Testing logging', async ({ page }) => {
+    await page.goto('/');
+    await page.getByRole('button', { name: 'Continue With Email' }).click();
+    await page.getByPlaceholder('Email').fill('tim@apple.dev');
+    await page.getByRole('button', { name: 'Continue', exact: true }).click();
+    await page.getByPlaceholder('Password').fill('Applecar2025');
+    await page.getByRole('button', { name: 'Sign in' }).click();
+    await expect(page.getByText('Welcome to Twenty')).not.toBeVisible();
+    expect(page.url()).not.toContain('/welcome');
+    await page.getByRole('link', { name: 'Opportunities' }).click();
+    await expect(page.locator('tbody > tr')).toHaveCount(4);
+  });
+
+  test('Creating new workspace', async ({ page, browserName }) => {
+    // this test must use only 1 browser, otherwise it will lead to success and fail (1 workspace is created instead of x workspaces)
+    if (browserName == 'chromium') {
+      await page.goto('/');
+      await page.getByRole('button', { name: 'Continue With Email' }).click();
+      await page.getByPlaceholder('Email').fill('test@apple.dev'); // email must be changed each time test is run
+      await page.getByPlaceholder('Email').press('Enter'); // otherwise if tests fails after this step, new workspace is created
+      await page.getByPlaceholder('Password').fill('Applecar2025');
+      await page.getByPlaceholder('Password').press('Enter');
+      await page.getByPlaceholder('Apple').fill('Test');
+      await page.getByRole('button', { name: 'Continue' }).click();
+      await page.getByPlaceholder('Tim').click();
+      await page.getByPlaceholder('Tim').fill('Test2');
+      await page.getByPlaceholder('Cook').click();
+      await page.getByPlaceholder('Cook').fill('Test2');
+      await page.getByRole('button', { name: 'Continue' }).click();
+      await page.getByText('Continue without sync').click();
+      await page.getByRole('button', { name: 'Finish' }).click();
+      await expect(page.locator('table')).toBeVisible({ timeout: 1000 });
+    }
+  });
+
+  test('Syncing all workspaces', async () => {
+    await sh('npx nx run twenty-server:command workspace:sync-metadata -f');
+    await sh('npx nx run twenty-server:command workspace:sync-metadata -f');
+  });
+
+  test('Resetting database', async ({ page, browserName }) => {
+    if (browserName === 'chromium') {
+      await sh('yarn nx database:reset twenty-server'); // if this command fails for any reason, database must be restarted manually using the same command because database is in unstable state
+      await page.goto('/');
+      await page.getByRole('button', { name: 'Continue With Email' }).click();
+      await page.getByPlaceholder('Email').fill('tim@apple.dev');
+      await page.getByRole('button', { name: 'Continue' }).click();
+      await page.getByPlaceholder('Password').fill('Applecar2025');
+      await page.getByRole('button', { name: 'Sign in' }).click();
+      await page.getByRole('link', { name: 'Companies' }).click();
+      expect(page.url()).toContain('/companies');
+      await expect(page.locator('table')).toBeVisible();
+    }
+  });
+
+  test('Seeding database', async ({ page, browserName }) => {
+    if (browserName === 'chromium') {
+      await sh('npx nx workspace:seed:demo');
+      await page.goto('/');
+    }
+  });
+});

yarn.lock

@@ -47214,6 +47214,7 @@ __metadata:
     "@nx/vite": "npm:18.3.3"
     "@nx/web": "npm:18.3.3"
     "@octokit/graphql": "npm:^7.0.2"
+    "@playwright/test": "npm:^1.46.0"
     "@ptc-org/nestjs-query-core": "npm:^4.2.0"
     "@ptc-org/nestjs-query-typeorm": "npm:4.2.1-alpha.2"
     "@react-email/components": "npm:0.0.12"