playwright/tests/playwright-test/test-modifiers.spec.ts

/**
 * Copyright Microsoft Corporation. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { test, expect, expectTestHelper } from './playwright-test-fixtures';

test('test modifiers should work', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'helper.ts': `
      import { test as base, expect } from '@playwright/test';
      export const test = base.extend({
        foo: true,
      });
    `,
    'a.test.ts': `
      import { test } from './helper';

      test('passed1', async ({foo}) => {
      });
      test('passed2', async ({foo}) => {
        test.skip(false);
      });
      test('passed3', async () => {
        test.fixme(undefined);
      });
      test('passed4', async () => {
        test.fixme(undefined, 'reason')
      });
      test('passed5', async ({foo}) => {
        test.skip(false);
      });

      test('skipped1', async ({foo}) => {
        test.skip();
      });
      test('skipped2', async ({foo}) => {
        test.skip('reason');
      });
      test('skipped3', async ({foo}) => {
        test.skip(foo);
      });
      test('skipped4', async ({foo}) => {
        test.skip(foo, 'reason');
      });
      test('skipped5', async () => {
        test.fixme();
      });
      test('skipped6', async () => {
        test.fixme(true, 'reason');
      });

      test('failed1', async ({foo}) => {
        test.fail();
        expect(true).toBe(false);
      });
      test('failed2', async ({foo}) => {
        test.fail('reason');
        expect(true).toBe(false);
      });
      test('failed3', async ({foo}) => {
        test.fail(foo);
        expect(true).toBe(false);
      });
      test('failed4', async ({foo}) => {
        test.fail(foo, 'reason');
        expect(true).toBe(false);
      });

      test.describe('suite1', () => {
        test.skip();
        test('suite1', () => {});
      });

      test.describe('suite2', () => {
        test.skip(true);
        test('suite2', () => {});
      });

      test.describe('suite3', () => {
        test.skip(({ foo }) => foo, 'reason');
        test('suite3', () => {});
      });

      test.describe('suite3', () => {
        test.skip(({ foo }) => !foo, 'reason');
        test('suite4', () => {});
      });
    `,
  });

  const expectTest = (title: string, expectedStatus: string, status: string, annotations: any) => {
    const spec = result.report.suites[0].specs.find(s => s.title === title) ||
        result.report.suites[0].suites!.find(s => s.specs[0].title === title)!.specs[0];
    const test = spec.tests[0];
    expect(test.expectedStatus).toBe(expectedStatus);
    expect(test.results[0].status).toBe(status);
    expect(test.annotations).toEqual(annotations);
  };
  expectTest('passed1', 'passed', 'passed', []);
  expectTest('passed2', 'passed', 'passed', []);
  expectTest('passed3', 'passed', 'passed', []);
  expectTest('passed4', 'passed', 'passed', []);
  expectTest('passed5', 'passed', 'passed', []);
  expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip' }]);
  expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip' }]);
  expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip' }]);
  expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
  expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme' }]);
  expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason' }]);
  expectTest('failed1', 'failed', 'failed', [{ type: 'fail' }]);
  expectTest('failed2', 'failed', 'failed', [{ type: 'fail' }]);
  expectTest('failed3', 'failed', 'failed', [{ type: 'fail' }]);
  expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason' }]);
  expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip' }]);
  expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip' }]);
  expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]);
  expectTest('suite4', 'passed', 'passed', []);
  expect(result.passed).toBe(10);
  expect(result.skipped).toBe(9);
});

test.describe('test modifier annotations', () => {
  test('should work', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';

        test.describe('suite1', () => {
          test('no marker', () => {});
          test.skip('skip wrap', () => {});
          test('skip inner', () => { test.skip(); });
          test.fixme('fixme wrap', () => {});
          test('fixme inner', () => { test.fixme(); });
        });

        test('example', () => {});
      `,
    });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(2);
    expect(result.skipped).toBe(4);
    expectTest('no marker', 'passed', 'expected', []);
    expectTest('skip wrap', 'skipped', 'skipped', ['skip']);
    expectTest('skip inner', 'skipped', 'skipped', ['skip']);
    expectTest('fixme wrap', 'skipped', 'skipped', ['fixme']);
    expectTest('fixme inner', 'skipped', 'skipped', ['fixme']);
    expectTest('example', 'passed', 'expected', []);
  });

  test('should work alongside top-level modifier', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';

        test.fixme();

        test.describe('suite1', () => {
          test('no marker', () => {});
          test.skip('skip wrap', () => {});
          test('skip inner', () => { test.skip(); });
          test.fixme('fixme wrap', () => {});
          test('fixme inner', () => { test.fixme(); });
        });

        test('example', () => {});
      `,
    });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(0);
    expect(result.skipped).toBe(6);
    expectTest('no marker', 'skipped', 'skipped', ['fixme']);
    expectTest('skip wrap', 'skipped', 'skipped', ['skip', 'fixme']);
    expectTest('skip inner', 'skipped', 'skipped', ['fixme']);
    expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'fixme']);
    expectTest('fixme inner', 'skipped', 'skipped', ['fixme']);
    expectTest('example', 'skipped', 'skipped', ['fixme']);
  });

  test('should work alongside top-level modifier wrapper-style', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';

        test.describe.skip('suite1', () => {
          test('no marker', () => {});
          test.skip('skip wrap', () => {});
          test('skip inner', () => { test.skip(); });
          test.fixme('fixme wrap', () => {});
          test('fixme inner', () => { test.fixme(); });
        });

        test('example', () => {});
      `,
    });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(1);
    expect(result.skipped).toBe(5);
    expectTest('no marker', 'skipped', 'skipped', ['skip']);
    expectTest('skip wrap', 'skipped', 'skipped', ['skip', 'skip']);
    expectTest('skip inner', 'skipped', 'skipped', ['skip']);
    expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'skip']);
    expectTest('fixme inner', 'skipped', 'skipped', ['skip']);
    expectTest('example', 'passed', 'expected', []);
  });

  test('should work with nesting', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';

        test.fixme();

        test.describe.skip('suite1', () => {
          test.describe.skip('sub', () => {
            test.describe('a', () => {
              test.describe('b', () => {
                test.fixme();

                test.fixme('fixme wrap', () => {});
                test('fixme inner', () => { test.fixme(); });
              })
            })
          })
        });
      `,
    });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(0);
    expect(result.skipped).toBe(2);
    expectTest('fixme wrap', 'skipped', 'skipped', ['fixme', 'fixme', 'skip', 'skip', 'fixme']);
    expectTest('fixme inner', 'skipped', 'skipped', ['fixme', 'skip', 'skip', 'fixme']);
  });

  test('should work with only', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';

        test.describe.only("suite", () => {
          test.skip('focused skip by suite', () => {});
          test.fixme('focused fixme by suite', () => {});
        });

        test.describe.skip('not focused', () => {
          test('no marker', () => {});
        });
      `,
    });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(0);
    expect(result.skipped).toBe(2);
    expectTest('focused skip by suite', 'skipped', 'skipped', ['skip']);
    expectTest('focused fixme by suite', 'skipped', 'skipped', ['fixme']);
  });

  test('should not multiple on retry', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';
        test('retry', () => {
          test.info().annotations.push({ type: 'example' });
          expect(1).toBe(2);
        });
      `,
    }, { retries: 3 });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(1);
    expect(result.passed).toBe(0);
    expectTest('retry', 'passed', 'unexpected', ['example']);
  });

  test('should not multiply on repeat-each', async ({ runInlineTest }) => {
    const result = await runInlineTest({
      'a.test.ts': `
        import { test, expect } from '@playwright/test';
        test('retry', () => {
          test.info().annotations.push({ type: 'example' });
        });
      `,
    }, { 'repeat-each': 3 });
    const expectTest = expectTestHelper(result);

    expect(result.exitCode).toBe(0);
    expect(result.passed).toBe(3);
    expectTest('retry', 'passed', 'expected', ['example']);
  });
});

test('test modifiers should check types', async ({ runTSC }) => {
  const result = await runTSC({
    'helper.ts': `
      import { test as base, expect } from '@playwright/test';
      export const test = base.extend<{ foo: boolean }>({
        foo: async ({}, use, testInfo) => {
          testInfo.skip();
          testInfo.fixme(false);
          testInfo.slow(true, 'reason');
          testInfo.fail(false, 'reason');
          // @ts-expect-error
          testInfo.skip('reason');
          // @ts-expect-error
          testInfo.fixme('foo', 'reason');
          // @ts-expect-error
          testInfo.slow(() => true);
          use(true);
        },
      });
    `,
    'a.test.ts': `
      import { test } from './helper';

      test('passed1', async ({foo}) => {
        test.skip();
      });
      test('passed2', async ({foo}) => {
        test.skip(foo);
      });
      test('passed2', async ({foo}) => {
        test.skip(foo, 'reason');
      });
      test('passed3', async ({foo}) => {
        test.skip(({foo}) => foo);
      });
      test('passed3', async ({foo}) => {
        test.skip(({foo}) => foo, 'reason');
      });
      test('passed3', async ({foo}) => {
        // @ts-expect-error
        test.skip('foo', 'bar');
      });
      test('passed3', async ({foo}) => {
        // @ts-expect-error
        test.skip(({ bar }) => bar, 'reason');
      });
      test('passed3', async ({foo}) => {
        // @ts-expect-error
        test.skip(42);
      });
      test.skip('skipped', async ({}) => {
      });
      test.fixme('fixme', async ({}) => {
      });
      // @ts-expect-error
      test.skip('skipped', 'skipped');
      // @ts-expect-error
      test.fixme('fixme', 'fixme');
      // @ts-expect-error
      test.skip(true, async () => {});
      // @ts-expect-error
      test.fixme(true, async () => {});
    `,
  });
  expect(result.exitCode).toBe(0);
});

test('should skip inside fixture', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test as base, expect } from '@playwright/test';
      const test = base.extend({
        foo: async ({}, run, testInfo) => {
          testInfo.skip(true, 'reason');
          await run();
        },
      });

      test('skipped', async ({ foo }) => {
      });
    `,
  });
  expect(result.exitCode).toBe(0);
  expect(result.skipped).toBe(1);
  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
});

test('modifier with a function should throw in the test', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      test('skipped', async ({}) => {
        test.skip(() => true);
      });
    `,
  });
  expect(result.exitCode).toBe(1);
  expect(result.output).toContain('test.skip() with a function can only be called inside describe block');
});

test('test.skip with worker fixtures only should skip before hooks and tests', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test as base, expect } from '@playwright/test';
      const test = base.extend({
        foo: [ 'foo', { scope: 'worker' }],
      });
      const logs = [];
      test.beforeEach(() => {
        console.log('\\n%%beforeEach');
      });
      test('passed', () => {
        console.log('\\n%%passed');
      });
      test.describe('suite1', () => {
        test.skip(({ foo }) => {
          console.log('\\n%%skip');
          return foo === 'foo';
        }, 'reason');
        test.beforeAll(() => {
          console.log('\\n%%beforeAll');
        });
        test('skipped1', () => {
          console.log('\\n%%skipped1');
        });
        test.describe('suite2', () => {
          test('skipped2', () => {
            console.log('\\n%%skipped2');
          });
        });
      });
    `,
  });
  expect(result.exitCode).toBe(0);
  expect(result.passed).toBe(1);
  expect(result.skipped).toBe(2);
  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);
  expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
  expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
  expect(result.outputLines).toEqual([
    'beforeEach',
    'passed',
    'skip',
  ]);
});

test('test.skip without a callback in describe block should skip hooks', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      const logs = [];
      test.beforeAll(() => {
        console.log('%%beforeAll');
      });
      test.beforeEach(() => {
        console.log('%%beforeEach');
      });
      test.skip(true, 'reason');
      test('skipped1', () => {
        console.log('%%skipped1');
      });
      test.describe('suite1', () => {
        test('skipped2', () => {
          console.log('%%skipped2');
        });
      });
    `,
  });
  expect(result.exitCode).toBe(0);
  expect(result.skipped).toBe(2);
  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
  expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
  expect(result.output).not.toContain('%%');
});

test('test.skip should not define a skipped test inside another test', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      const logs = [];
      test('passes', () => {
        test.skip('foo', () => {
          console.log('%%dontseethis');
          throw new Error('foo');
        });
      });
    `,
  });
  expect(result.exitCode).toBe(1);
  expect(result.failed).toBe(1);
  expect(result.output).toContain('It looks like you are calling test.skip() inside the test and pass a callback');
});

test('modifier timeout should be reported', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      test.skip(async () => new Promise(() => {}));
      test('fails', () => {
      });
    `,
  }, { timeout: 2000 });
  expect(result.exitCode).toBe(1);
  expect(result.failed).toBe(1);
  expect(result.output).toContain('"skip" modifier timeout of 2000ms exceeded.');
  expect(result.output).toContain('3 |       test.skip(async () => new Promise(() => {}));');
});

test('should not run hooks if modifier throws', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      test.skip(() => {
        console.log('%%modifier');
        throw new Error('Oh my');
      });
      test.beforeAll(() => {
        console.log('%%beforeEach');
      });
      test.beforeEach(() => {
        console.log('%%beforeEach');
      });
      test.afterEach(() => {
        console.log('%%afterEach');
      });
      test.afterAll(() => {
        console.log('%%beforeEach');
      });
      test('skipped1', () => {
        console.log('%%skipped1');
      });
    `,
  });
  expect(result.exitCode).toBe(1);
  expect(result.failed).toBe(1);
  expect(result.outputLines).toEqual([
    'modifier',
  ]);
});

test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'reporter.ts': `
      class Reporter {
        onTestBegin(test) {
          console.log('\\n%%begin-' + test.title);
        }
        onTestEnd(test, result) {
          console.log('\\n%%end-' + test.title);
          console.log('\\n%%expectedStatus-' + test.expectedStatus);
          console.log('\\n%%timeout-' + test.timeout);
          console.log('\\n%%retries-' + test.retries);
        }
      }
      export default Reporter;
    `,
    'playwright.config.ts': `
      module.exports = { reporter: [['./reporter.ts']] };
    `,
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      test.describe.configure({ timeout: 1234, retries: 3 });
      test('test1', async ({}) => {
      });
      test.skip('test2', async ({}) => {
      });
      test('test3', async ({}) => {
      });
    `,
  }, { reporter: '', workers: 1 });

  expect(result.exitCode).toBe(0);
  expect(result.outputLines).toEqual([
    'begin-test1',
    'end-test1',
    'expectedStatus-passed',
    'timeout-1234',
    'retries-3',
    'begin-test2',
    'end-test2',
    'expectedStatus-skipped',
    'timeout-1234',
    'retries-3',
    'begin-test3',
    'end-test3',
    'expectedStatus-passed',
    'timeout-1234',
    'retries-3',
  ]);
});

test('should skip tests if beforeEach has skip', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test, expect } from '@playwright/test';
      test.beforeEach(() => {
        test.skip();
      });
      test('no marker', () => {
        console.log('skip-me');
      });
    `,
  });
  const expectTest = expectTestHelper(result);
  expect(result.exitCode).toBe(0);
  expect(result.passed).toBe(0);
  expect(result.skipped).toBe(1);
  expectTest('no marker', 'skipped', 'skipped', ['skip']);
  expect(result.output).not.toContain('skip-me');
});

test('static modifiers should be added in serial mode', async ({ runInlineTest }) => {
  const result = await runInlineTest({
    'a.test.ts': `
      import { test } from '@playwright/test';

      test.describe.configure({ mode: 'serial' });
      test('failed', async ({}) => {
        test.slow();
        throw new Error('blocking error');
      });
      test.fixme('fixmed', async ({}) => {
      });
      test.skip('skipped', async ({}) => {
      });
      test('ignored', async ({}) => {
      });
    `,
  });
  expect(result.exitCode).toBe(1);
  expect(result.passed).toBe(0);
  expect(result.skipped).toBe(3);
  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);
  expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);
  expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);
  expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);
});