chore(ui): maintain test order when updating (#21478)

packages/playwright-test/src/isomorphic/teleReceiver.ts

@@ -26,6 +26,7 @@ export type JsonStackFrame = { file: string, line: number, column: number };
 export type JsonConfig = {
   rootDir: string;
   configFile: string | undefined;
+  listOnly: boolean;
 };
 
 export type JsonPattern = {
@@ -148,6 +149,7 @@ export class TeleReporterReceiver {
   }
 
   private _onBegin(config: JsonConfig, projects: JsonProject[]) {
+    const removeMissing = config.listOnly;
     for (const project of projects) {
       let projectSuite = this._rootSuite.suites.find(suite => suite.project()!.id === project.id);
       if (!projectSuite) {
@@ -157,7 +159,7 @@ export class TeleReporterReceiver {
       }
       const p = this._parseProject(project);
       projectSuite.project = () => p;
-      this._mergeSuitesInto(project.suites, projectSuite);
+      this._mergeSuitesInto(project.suites, projectSuite, removeMissing);
     }
     this._reporter.onBegin?.(this._parseConfig(config), this._rootSuite);
   }
@@ -260,7 +262,7 @@ export class TeleReporterReceiver {
     };
   }
 
-  private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite) {
+  private _mergeSuitesInto(jsonSuites: JsonSuite[], parent: TeleSuite, removeMissing: boolean) {
     for (const jsonSuite of jsonSuites) {
       let targetSuite = parent.suites.find(s => s.title === jsonSuite.title);
       if (!targetSuite) {
@@ -271,12 +273,16 @@ export class TeleReporterReceiver {
       targetSuite.location = jsonSuite.location;
       targetSuite._fileId = jsonSuite.fileId;
       targetSuite._parallelMode = jsonSuite.parallelMode;
-      this._mergeSuitesInto(jsonSuite.suites, targetSuite);
-      this._mergeTestsInto(jsonSuite.tests, targetSuite);
+      this._mergeSuitesInto(jsonSuite.suites, targetSuite, removeMissing);
+      this._mergeTestsInto(jsonSuite.tests, targetSuite, removeMissing);
+    }
+    if (removeMissing) {
+      const suiteMap = new Map(parent.suites.map(p => [p.title, p]));
+      parent.suites = jsonSuites.map(s => suiteMap.get(s.title)).filter(Boolean) as TeleSuite[];
     }
   }
 
-  private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) {
+  private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite, removeMissing: boolean) {
     for (const jsonTest of jsonTests) {
       let targetTest = parent.tests.find(s => s.title === jsonTest.title);
       if (!targetTest) {
@@ -287,6 +293,10 @@ export class TeleReporterReceiver {
       }
       this._updateTest(jsonTest, targetTest);
     }
+    if (removeMissing) {
+      const testMap = new Map(parent.tests.map(p => [p.title, p]));
+      parent.tests = jsonTests.map(s => testMap.get(s.title)).filter(Boolean) as TeleTestCase[];
+    }
   }
 
   private _updateTest(payload: JsonTestCase, test: TeleTestCase): TeleTestCase {

packages/playwright-test/src/reporters/teleEmitter.ts

@@ -18,12 +18,11 @@ import type { FullConfig, FullResult, Reporter, TestError, TestResult, TestStep
 import type { Suite, TestCase } from '../common/test';
 import type { JsonConfig, JsonProject, JsonSuite, JsonTestCase, JsonTestResultEnd, JsonTestResultStart, JsonTestStepEnd, JsonTestStepStart } from '../isomorphic/teleReceiver';
 import type { SuitePrivate } from '../../types/reporterPrivate';
-import type { FullProjectInternal } from '../common/types';
+import type { FullConfigInternal, FullProjectInternal } from '../common/types';
 import { createGuid } from 'playwright-core/lib/utils';
 import { serializeRegexPatterns } from '../isomorphic/teleReceiver';
 
 export class TeleReporterEmitter implements Reporter {
-  private config!: FullConfig;
   private _messageSink: (message: any) => void;
 
   constructor(messageSink: (message: any) => void) {
@@ -31,7 +30,6 @@ export class TeleReporterEmitter implements Reporter {
   }
 
   onBegin(config: FullConfig, suite: Suite) {
-    this.config = config;
     const projects: any[] = [];
     for (const projectSuite of suite.suites) {
       const report = this._serializeProject(projectSuite);
@@ -116,6 +114,7 @@ export class TeleReporterEmitter implements Reporter {
     return {
       rootDir: config.rootDir,
       configFile: config.configFile,
+      listOnly: (config as FullConfigInternal)._internal.listOnly,
     };
   }
 

packages/playwright-test/src/runner/runner.ts

@@ -57,7 +57,7 @@ export class Runner {
     webServerPluginsForConfig(config).forEach(p => config._internal.plugins.push({ factory: p }));
 
     const reporter = await createReporter(config, listOnly ? 'list' : 'run');
-    const taskRunner = listOnly ? createTaskRunnerForList(config, reporter)
+    const taskRunner = listOnly ? createTaskRunnerForList(config, reporter, 'in-process')
       : createTaskRunner(config, reporter);
 
     const context: TaskRunnerState = {

packages/playwright-test/src/runner/tasks.ts

@@ -91,9 +91,9 @@ function addRunTasks(taskRunner: TaskRunner<TaskRunnerState>, config: FullConfig
   return taskRunner;
 }
 
-export function createTaskRunnerForList(config: FullConfigInternal, reporter: Multiplexer): TaskRunner<TaskRunnerState> {
+export function createTaskRunnerForList(config: FullConfigInternal, reporter: Multiplexer, mode: 'in-process' | 'out-of-process'): TaskRunner<TaskRunnerState> {
   const taskRunner = new TaskRunner<TaskRunnerState>(reporter, config.globalTimeout);
-  taskRunner.addTask('load tests', createLoadTask('in-process', false));
+  taskRunner.addTask('load tests', createLoadTask(mode, false));
   taskRunner.addTask('report begin', async ({ reporter, rootSuite }) => {
     reporter.onBegin?.(config, rootSuite!);
     return () => reporter.onEnd();

packages/playwright-test/src/runner/uiMode.ts

@@ -35,9 +35,10 @@ class UIMode {
   private _page!: Page;
   private _testRun: { run: Promise<FullResult['status']>, stop: ManualPromise<void> } | undefined;
   globalCleanup: (() => Promise<FullResult['status']>) | undefined;
-  private _watcher: FSWatcher | undefined;
+  private _testWatcher: FSWatcher | undefined;
   private _watchTestFile: string | undefined;
   private _originalStderr: (buffer: string | Uint8Array) => void;
+  private _globalWatcher: FSWatcher;
 
   constructor(config: FullConfigInternal) {
     this._config = config;
@@ -46,6 +47,7 @@ class UIMode {
       p.retries = 0;
     config._internal.configCLIOverrides.use = config._internal.configCLIOverrides.use || {};
     config._internal.configCLIOverrides.use.trace = 'on';
+
     this._originalStderr = process.stderr.write.bind(process.stderr);
     process.stdout.write = (chunk: string | Buffer) => {
       this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stdout', chunk) });
@@ -55,6 +57,25 @@ class UIMode {
       this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stderr', chunk) });
       return true;
     };
+
+    this._globalWatcher = this._installGlobalWatcher();
+  }
+
+  private _installGlobalWatcher(): FSWatcher {
+    const projectDirs = new Set<string>();
+    for (const p of this._config.projects)
+      projectDirs.add(p.testDir);
+    let coalescingTimer: NodeJS.Timeout | undefined;
+    const watcher = chokidar.watch([...projectDirs], { ignoreInitial: true, persistent: true }).on('all', async event => {
+      if (event !== 'add' && event !== 'change')
+        return;
+      if (coalescingTimer)
+        clearTimeout(coalescingTimer);
+      coalescingTimer = setTimeout(() => {
+        this._dispatchEvent({ method: 'listChanged' });
+      }, 200);
+    });
+    return watcher;
   }
 
   async runGlobalSetup(): Promise<FullResult['status']> {
@@ -79,30 +100,45 @@ class UIMode {
     this._page = await showTraceViewer([], 'chromium', { app: 'watch.html' });
     const exitPromise = new ManualPromise();
     this._page.on('close', () => exitPromise.resolve());
+    let queue = Promise.resolve();
     this._page.exposeBinding('sendMessage', false, async (source, data) => {
       const { method, params }: { method: string, params: any } = data;
-      if (method === 'list')
-        await this._listTests();
-      if (method === 'run')
-        await this._runTests(params.testIds);
-      if (method === 'stop')
-        this._stopTests();
-      if (method === 'watch')
+      if (method === 'exit') {
+        exitPromise.resolve();
+        return;
+      }
+      if (method === 'watch') {
         this._watchFile(params.fileName);
-      if (method === 'open' && params.location)
+        return;
+      }
+      if (method === 'open' && params.location) {
         open.openApp('code', { arguments: ['--goto', params.location] }).catch(() => {});
+        return;
+      }
       if (method === 'resizeTerminal') {
         process.stdout.columns = params.cols;
         process.stdout.rows = params.rows;
         process.stderr.columns = params.cols;
         process.stderr.columns = params.rows;
+        return;
       }
-      if (method === 'exit')
-        exitPromise.resolve();
+      if (method === 'stop') {
+        this._stopTests();
+        return;
+      }
+      queue = queue.then(() => this._queueListOrRun(method, params));
+      await queue;
     });
     await exitPromise;
   }
 
+  private async _queueListOrRun(method: string, params: any) {
+    if (method === 'list')
+      await this._listTests();
+    if (method === 'run')
+      await this._runTests(params.testIds);
+  }
+
   private _dispatchEvent(message: any) {
     // eslint-disable-next-line no-console
     this._page.mainFrame().evaluateExpression(dispatchFuncSource, true, message).catch(e => this._originalStderr(String(e)));
@@ -111,8 +147,11 @@ class UIMode {
   private async _listTests() {
     const listReporter = new TeleReporterEmitter(e => this._dispatchEvent(e));
     const reporter = new Multiplexer([listReporter]);
-    const taskRunner = createTaskRunnerForList(this._config, reporter);
+    this._config._internal.listOnly = true;
+    this._config._internal.testIdMatcher = undefined;
+    const taskRunner = createTaskRunnerForList(this._config, reporter, 'out-of-process');
     const context: TaskRunnerState = { config: this._config, reporter, phases: [] };
+    clearCompilationCache();
     reporter.onConfigure(this._config);
     await taskRunner.run(context, 0);
   }
@@ -121,6 +160,7 @@ class UIMode {
     await this._stopTests();
 
     const testIdSet = testIds ? new Set<string>(testIds) : null;
+    this._config._internal.listOnly = false;
     this._config._internal.testIdMatcher = id => !testIdSet || testIdSet.has(id);
 
     const runReporter = new TeleReporterEmitter(e => this._dispatchEvent(e));
@@ -143,14 +183,14 @@ class UIMode {
   private async _watchFile(fileName: string) {
     if (this._watchTestFile === fileName)
       return;
-    if (this._watcher)
-      await this._watcher.close();
+    if (this._testWatcher)
+      await this._testWatcher.close();
     this._watchTestFile = fileName;
     if (!fileName)
       return;
 
     const files = [fileName, ...dependenciesForTestFile(fileName)];
-    this._watcher = chokidar.watch(files, { ignoreInitial: true }).on('all', async (event, file) => {
+    this._testWatcher = chokidar.watch(files, { ignoreInitial: true }).on('all', async (event, file) => {
       if (event !== 'add' && event !== 'change')
         return;
       this._dispatchEvent({ method: 'fileChanged', params: { fileName: file } });

packages/trace-viewer/src/ui/watchMode.tsx

@@ -90,7 +90,7 @@ export const WatchModeView: React.FC<{}> = ({
           <div className='section-title' style={{ cursor: 'pointer' }} onClick={() => setSettingsVisible(false)}>Tests</div>
           <ToolbarButton icon='play' title='Run' onClick={runVisibleTests} disabled={isRunningTest}></ToolbarButton>
           <ToolbarButton icon='debug-stop' title='Stop' onClick={() => sendMessageNoReply('stop')} disabled={!isRunningTest}></ToolbarButton>
-          <ToolbarButton icon='refresh' title='Reload' onClick={resetCollectingRootSuite} disabled={isRunningTest}></ToolbarButton>
+          <ToolbarButton icon='refresh' title='Reload' onClick={() => refreshRootSuite(true)} disabled={isRunningTest}></ToolbarButton>
           <ToolbarButton icon='eye-watch' title='Watch' toggled={isWatchingFiles} onClick={() => setIsWatchingFiles(!isWatchingFiles)}></ToolbarButton>
           <div className='spacer'></div>
           <ToolbarButton icon='gear' title='Toggle color mode' toggled={settingsVisible} onClick={() => { setSettingsVisible(!settingsVisible); }}></ToolbarButton>
@@ -128,7 +128,7 @@ export const TestList: React.FC<{
 
   React.useEffect(() => {
     inputRef.current?.focus();
-    resetCollectingRootSuite();
+    refreshRootSuite(true);
   }, []);
 
   const { filteredItems, treeItemMap, visibleTestIds } = React.useMemo(() => {
@@ -165,8 +165,8 @@ export const TestList: React.FC<{
   }, [selectedTreeItemId, treeItemMap]);
 
   React.useEffect(() => {
-    sendMessageNoReply('watch', { fileName: isWatchingFiles ? fileName(selectedTestItem) : undefined });
-  }, [selectedTestItem, isWatchingFiles]);
+    sendMessageNoReply('watch', { fileName: isWatchingFiles ? fileName(selectedTreeItem) : undefined });
+  }, [selectedTreeItem, isWatchingFiles]);
 
   onTestItemSelected(selectedTestItem);
 
@@ -327,7 +327,12 @@ declare global {
 
 let receiver: TeleReporterReceiver | undefined;
 
-const resetCollectingRootSuite = () => {
+const refreshRootSuite = (eraseResults: boolean) => {
+  if (!eraseResults) {
+    sendMessageNoReply('list');
+    return;
+  }
+
   let rootSuite: Suite;
   const progress: Progress = {
     total: 0,
@@ -367,18 +372,27 @@ const resetCollectingRootSuite = () => {
 };
 
 (window as any).dispatch = (message: any) => {
+  if (message.method === 'listChanged') {
+    refreshRootSuite(false);
+    return;
+  }
+
   if (message.method === 'fileChanged') {
     runWatchedTests();
-  } else if (message.method === 'stdio') {
+    return;
+  }
+
+  if (message.method === 'stdio') {
     if (message.params.buffer) {
       const data = atob(message.params.buffer);
       xtermDataSource.write(data);
     } else {
       xtermDataSource.write(message.params.text);
     }
-  } else {
-    receiver?.dispatch(message);
+    return;
   }
+
+  receiver?.dispatch(message);
 };
 
 const sendMessage = async (method: string, params: any) => {
@@ -442,6 +456,7 @@ type TreeItemBase = {
 type FileItem = TreeItemBase & {
   kind: 'file',
   file: string;
+  children?: TestCaseItem[];
 };
 
 type TestCaseItem = TreeItemBase & {
@@ -501,6 +516,7 @@ function createTree(rootSuite: Suite | undefined, projects: Map<string, boolean>
           test,
         });
       }
+      (fileItem.children as TestCaseItem[]).sort((a, b) => a.location.line - b.location.line);
     }
   }
   return [...fileItems.values()];
@@ -512,7 +528,7 @@ function filterTree(fileItems: FileItem[], filterText: string): FileItem[] {
   const result: FileItem[] = [];
   for (const fileItem of fileItems) {
     if (trimmedFilterText) {
-      const filteredCases: TreeItem[] = [];
+      const filteredCases: TestCaseItem[] = [];
       for (const testCaseItem of fileItem.children!) {
         const fullTitle = (fileItem.title + ' ' + testCaseItem.title).toLowerCase();
         if (filterTokens.every(token => fullTitle.includes(token)))