chore: follow up to the api review for test server (#30097)

packages/playwright/src/isomorphic/testServerConnection.ts

@@ -110,6 +110,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
       this._onLoadTraceRequestedEmitter.fire(params);
   }
 
+  async setSerializer(params: { serializer: string; }): Promise<void> {
+    await this._sendMessage('setSerializer', params);
+  }
+
   async ping(params: Parameters<TestServerInterface['ping']>[0]): ReturnType<TestServerInterface['ping']> {
     await this._sendMessage('ping');
   }
@@ -126,6 +130,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
     this._sendMessageNoReply('watch', params);
   }
 
+  async watchTestDir(params: Parameters<TestServerInterface['watchTestDir']>[0]): ReturnType<TestServerInterface['watchTestDir']> {
+    await this._sendMessage('watchTestDir', params);
+  }
+
   async open(params: Parameters<TestServerInterface['open']>[0]): ReturnType<TestServerInterface['open']> {
     await this._sendMessage('open', params);
   }

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -16,14 +16,21 @@
 
 import type * as reporterTypes from '../../types/testReporter';
 import type { Event } from './events';
+import type { JsonEvent } from './teleReceiver';
+
+export type ReportEntry = JsonEvent;
 
 export interface TestServerInterface {
+  setSerializer(params: { serializer: string }): Promise<void>;
+
   ping(params: {}): Promise<void>;
 
   watch(params: {
     fileNames: string[];
   }): Promise<void>;
 
+  watchTestDir(params: {}): Promise<void>;
+
   open(params: { location: reporterTypes.Location }): Promise<void>;
 
   resizeTerminal(params: { cols: number, rows: number }): Promise<void>;
@@ -32,34 +39,35 @@ export interface TestServerInterface {
 
   installBrowsers(params: {}): Promise<void>;
 
-  runGlobalSetup(params: {}): Promise<reporterTypes.FullResult['status']>;
+  runGlobalSetup(params: {}): Promise<{
+    report: ReportEntry[],
+    status: reporterTypes.FullResult['status']
+  }>;
 
-  runGlobalTeardown(params: {}): Promise<reporterTypes.FullResult['status']>;
+  runGlobalTeardown(params: {}): Promise<{
+    report: ReportEntry[],
+    status: reporterTypes.FullResult['status']
+  }>;
 
   listFiles(params: {
     projects?: string[];
   }): Promise<{
-    projects: {
+    report: ReportEntry[];
-      name: string;
+    status: reporterTypes.FullResult['status']
-      testDir: string;
-      use: { testIdAttribute?: string };
-      files: string[];
-    }[];
-    cliEntryPoint?: string;
-    error?: reporterTypes.TestError;
   }>;
 
   /**
    * Returns list of teleReporter events.
    */
   listTests(params: {
-    serializer?: string;
     projects?: string[];
     locations?: string[];
-  }): Promise<{ report: any[] }>;
+  }): Promise<{
+    report: ReportEntry[],
+    status: reporterTypes.FullResult['status']
+  }>;
 
   runTests(params: {
-    serializer?: string;
     locations?: string[];
     grep?: string;
     grepInvert?: string;
@@ -72,7 +80,9 @@ export interface TestServerInterface {
     projects?: string[];
     reuseContext?: boolean;
     connectWsEndpoint?: string;
-  }): Promise<{ status: reporterTypes.FullResult['status'] }>;
+  }): Promise<{
+    status: reporterTypes.FullResult['status'];
+  }>;
 
   findRelatedTestFiles(params: {
     files: string[];
@@ -84,7 +94,6 @@ export interface TestServerInterface {
 }
 
 export interface TestServerInterfaceEvents {
-  onClose: Event<void>;
   onReport: Event<any>;
   onStdio: Event<{ type: 'stdout' | 'stderr', text?: string, buffer?: string }>;
   onListChanged: Event<void>;
@@ -93,8 +102,7 @@ export interface TestServerInterfaceEvents {
 }
 
 export interface TestServerInterfaceEventEmitters {
-  dispatchEvent(event: 'close', params: {}): void;
+  dispatchEvent(event: 'report', params: ReportEntry): void;
-  dispatchEvent(event: 'report', params: any): void;
   dispatchEvent(event: 'stdio', params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }): void;
   dispatchEvent(event: 'listChanged', params: {}): void;
   dispatchEvent(event: 'testFilesChanged', params: { testFiles: string[] }): void;

packages/playwright/src/runner/loadUtils.ts

@@ -32,7 +32,7 @@ import { dependenciesForTestFile } from '../transform/compilationCache';
 import { sourceMapSupport } from '../utilsBundle';
 import type { RawSourceMap } from 'source-map';
 
-export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher: Matcher | undefined) {
+export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) {
   const config = testRun.config;
   const fsCache = new Map();
   const sourceMapCache = new Map();
@@ -60,8 +60,7 @@ export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTest
       return hasMatchingSources;
     });
     const filteredFiles = matchedFiles.filter(Boolean) as string[];
-    if (filteredFiles.length)
+    filesToRunByProject.set(project, filteredFiles);
-      filesToRunByProject.set(project, filteredFiles);
   }
 
   // (Re-)add all files for dependent projects, disregard filters.
@@ -316,7 +315,7 @@ export function loadGlobalHook(config: FullConfigInternal, file: string): Promis
   return requireOrImportDefaultFunction(path.resolve(config.config.rootDir, file), false);
 }
 
-export function loadReporter(config: FullConfigInternal, file: string): Promise<new (arg?: any) => Reporter> {
+export function loadReporter(config: FullConfigInternal | null, file: string): Promise<new (arg?: any) => Reporter> {
   return requireOrImportDefaultFunction(config ? path.resolve(config.config.rootDir, file) : file, true);
 }
 

packages/playwright/src/runner/reporters.ts

@@ -78,17 +78,16 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
   return reporters;
 }
 
-export async function createReporterForTestServer(config: FullConfigInternal, mode: 'list' | 'test', file: string, messageSink: (message: any) => void): Promise<ReporterV2> {
+export async function createReporterForTestServer(file: string, messageSink: (message: any) => void): Promise<ReporterV2> {
-  const reporterConstructor = await loadReporter(config, file);
+  const reporterConstructor = await loadReporter(null, file);
-  const runOptions = reporterOptions(config, mode, true, messageSink);
+  return wrapReporterAsV2(new reporterConstructor({
-  const instance = new reporterConstructor(runOptions);
+    _send: messageSink,
-  return wrapReporterAsV2(instance);
+  }));
 }
 
-function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean, send?: (message: any) => void) {
+function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean) {
   return {
     configDir: config.configDir,
-    _send: send,
     _mode: mode,
     _isTestServer: isTestServer,
   };

packages/playwright/src/runner/runner.ts

@@ -40,7 +40,6 @@ type ProjectConfigWithFiles = {
 
 type ConfigListFilesReport = {
   projects: ProjectConfigWithFiles[];
-  cliEntryPoint?: string;
   error?: TestError;
 };
 
@@ -57,11 +56,9 @@ export class Runner {
   }
 
   async listTestFiles(projectNames?: string[]): Promise<ConfigListFilesReport> {
-    const frameworkPackage = (this._config.config as any)['@playwright/test']?.['packageJSON'];
     const projects = filterProjects(this._config.projects, projectNames);
     const report: ConfigListFilesReport = {
       projects: [],
-      cliEntryPoint: frameworkPackage ? path.join(path.dirname(frameworkPackage), 'cli.js') : undefined,
     };
     for (const project of projects) {
       report.projects.push({

packages/playwright/src/runner/tasks.ts

@@ -28,7 +28,7 @@ import { TaskRunner } from './taskRunner';
 import type { FullConfigInternal, FullProjectInternal } from '../common/config';
 import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils';
 import type { Matcher } from '../util';
-import type { Suite } from '../common/test';
+import { Suite } from '../common/test';
 import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';
 import { FailureTracker } from './failureTracker';
 
@@ -116,6 +116,13 @@ export function createTaskRunnerForList(config: FullConfigInternal, reporter: Re
   return taskRunner;
 }
 
+export function createTaskRunnerForListFiles(config: FullConfigInternal, reporter: ReporterV2): TaskRunner<TestRun> {
+  const taskRunner = new TaskRunner<TestRun>(reporter, config.config.globalTimeout);
+  taskRunner.addTask('load tests', createListFilesTask());
+  taskRunner.addTask('report begin', createReportBeginTask());
+  return taskRunner;
+}
+
 function createReportBeginTask(): Task<TestRun> {
   return {
     setup: async ({ reporter, rootSuite }) => {
@@ -195,6 +202,29 @@ function createRemoveOutputDirsTask(): Task<TestRun> {
   };
 }
 
+function createListFilesTask(): Task<TestRun> {
+  return {
+    setup: async (testRun, errors) => {
+      testRun.rootSuite = await createRootSuite(testRun, errors, false);
+      testRun.failureTracker.onRootSuite(testRun.rootSuite);
+      await collectProjectsAndTestFiles(testRun, false);
+      for (const [project, files] of testRun.projectFiles) {
+        const projectSuite = new Suite(project.project.name, 'project');
+        projectSuite._fullProject = project;
+        testRun.rootSuite._addSuite(projectSuite);
+        const suites = files.map(file => {
+          const title = path.relative(testRun.config.config.rootDir, file);
+          const suite =  new Suite(title, 'file');
+          suite.location = { file, line: 0, column: 0 };
+          projectSuite._addSuite(suite);
+          return suite;
+        });
+        testRun.projectSuites.set(project, suites);
+      }
+    },
+  };
+}
+
 function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
   return {
     setup: async (testRun, errors, softErrors) => {

packages/playwright/src/runner/testServer.ts

@@ -24,21 +24,20 @@ import { collectAffectedTestFiles, dependenciesForTestFile } from '../transform/
 import type { FullConfigInternal } from '../common/config';
 import { InternalReporter } from '../reporters/internalReporter';
 import { createReporterForTestServer, createReporters } from './reporters';
-import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup } from './tasks';
+import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles } from './tasks';
 import { open } from 'playwright-core/lib/utilsBundle';
 import ListReporter from '../reporters/list';
 import { Multiplexer } from '../reporters/multiplexer';
 import { SigIntWatcher } from './sigIntWatcher';
 import { Watcher } from '../fsWatcher';
-import type { TestServerInterface, TestServerInterfaceEventEmitters } from '../isomorphic/testServerInterface';
+import type { ReportEntry, TestServerInterface, TestServerInterfaceEventEmitters } from '../isomorphic/testServerInterface';
 import { Runner } from './runner';
-import { serializeError } from '../util';
-import { prepareErrorStack } from '../reporters/base';
 import type { ConfigCLIOverrides } from '../common/ipc';
 import { loadConfig, resolveConfigFile, restartWithExperimentalTsEsm } from '../common/configLoader';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
 import type { TraceViewerRedirectOptions, TraceViewerServerOptions } from 'playwright-core/lib/server/trace/viewer/traceViewer';
 import type { TestRunnerPluginRegistration } from '../plugins';
+import { serializeError } from '../util';
 
 class TestServer {
   private _configFile: string | undefined;
@@ -89,9 +88,11 @@ class TestServerDispatcher implements TestServerInterface {
   private _testRun: { run: Promise<reporterTypes.FullResult['status']>, stop: ManualPromise<void> } | undefined;
   readonly transport: Transport;
   private _queue = Promise.resolve();
-  private _globalCleanup: (() => Promise<reporterTypes.FullResult['status']>) | undefined;
+  private _globalSetup: { cleanup: () => Promise<any>, report: ReportEntry[] } | undefined;
   readonly _dispatchEvent: TestServerInterfaceEventEmitters['dispatchEvent'];
   private _plugins: TestRunnerPluginRegistration[] | undefined;
+  private _serializer = require.resolve('./uiModeReporter');
+  private _watchTestDir = false;
 
   constructor(configFile: string | undefined) {
     this._configFile = configFile;
@@ -108,6 +109,21 @@ class TestServerDispatcher implements TestServerInterface {
     this._dispatchEvent = (method, params) => this.transport.sendEvent?.(method, params);
   }
 
+  async setSerializer(params: { serializer: string; }): Promise<void> {
+    this._serializer = params.serializer;
+  }
+
+  private async _wireReporter(messageSink: (message: any) => void) {
+    return await createReporterForTestServer(this._serializer, messageSink);
+  }
+
+  private async _collectingReporter() {
+    const report: ReportEntry[] = [];
+    const wireReporter = await createReporterForTestServer(this._serializer, e => report.push(e));
+    const reporter = new InternalReporter(wireReporter);
+    return { reporter, report };
+  }
+
   async ready() {}
 
   async ping() {}
@@ -134,14 +150,19 @@ class TestServerDispatcher implements TestServerInterface {
     await installBrowsers();
   }
 
-  async runGlobalSetup(): Promise<reporterTypes.FullResult['status']> {
+  async runGlobalSetup(params: Parameters<TestServerInterface['runGlobalSetup']>[0]): ReturnType<TestServerInterface['runGlobalSetup']> {
     await this.runGlobalTeardown();
 
-    const config = await this._loadConfig(this._configFile);
+    const { reporter, report } = await this._collectingReporter();
-    webServerPluginsForConfig(config).forEach(p => config.plugins.push({ factory: p }));
+    const { config, error } = await this._loadConfig(this._configFile);
+    if (!config) {
+      reporter.onError(error!);
+      return { status: 'failed', report };
+    }
 
-    const reporter = new InternalReporter(new ListReporter());
+    webServerPluginsForConfig(config).forEach(p => config.plugins.push({ factory: p }));
-    const taskRunner = createTaskRunnerForWatchSetup(config, reporter);
+    const listReporter = new InternalReporter(new ListReporter());
+    const taskRunner = createTaskRunnerForWatchSetup(config, new Multiplexer([reporter, listReporter]));
     reporter.onConfigure(config.config);
     const testRun = new TestRun(config, reporter);
     const { status, cleanup: globalCleanup } = await taskRunner.runDeferCleanup(testRun, 0);
@@ -149,28 +170,35 @@ class TestServerDispatcher implements TestServerInterface {
     await reporter.onExit();
     if (status !== 'passed') {
       await globalCleanup();
-      return status;
+      return { report, status };
     }
-    this._globalCleanup = globalCleanup;
+    this._globalSetup = { cleanup: globalCleanup, report };
-    return status;
+    return { report, status };
   }
 
   async runGlobalTeardown() {
-    const result = (await this._globalCleanup?.()) || 'passed';
+    const globalSetup = this._globalSetup;
-    this._globalCleanup = undefined;
+    const status = await globalSetup?.cleanup();
-    return result;
+    this._globalSetup = undefined;
+    return { status, report: globalSetup?.report || [] };
   }
 
   async listFiles(params: Parameters<TestServerInterface['listFiles']>[0]): ReturnType<TestServerInterface['listFiles']> {
-    try {
+    const { reporter, report } = await this._collectingReporter();
-      const config = await this._loadConfig(this._configFile);
+    const { config, error } = await this._loadConfig(this._configFile);
-      const runner = new Runner(config);
+    if (!config) {
-      return runner.listTestFiles(params.projects);
+      reporter.onError(error!);
-    } catch (e) {
+      return { status: 'failed', report };
-      const error: reporterTypes.TestError = serializeError(e);
-      error.location = prepareErrorStack(e.stack).location;
-      return { projects: [], error };
     }
+
+    config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
+    const taskRunner = createTaskRunnerForListFiles(config, reporter);
+    reporter.onConfigure(config.config);
+    const testRun = new TestRun(config, reporter);
+    const status = await taskRunner.run(testRun, 0);
+    await reporter.onEnd({ status });
+    await reporter.onExit();
+    return { report, status };
   }
 
   async listTests(params: Parameters<TestServerInterface['listTests']>[0]): ReturnType<TestServerInterface['listTests']> {
@@ -187,15 +215,17 @@ class TestServerDispatcher implements TestServerInterface {
       repeatEach: 1,
       retries: 0,
     };
-    const config = await this._loadConfig(this._configFile, overrides);
+    const { reporter, report } = await this._collectingReporter();
+    const { config, error } = await this._loadConfig(this._configFile, overrides);
+    if (!config) {
+      reporter.onError(error!);
+      return { report: [], status: 'failed' };
+    }
+
     config.cliArgs = params.locations || [];
     config.cliProjectFilter = params.projects?.length ? params.projects : undefined;
     config.cliListOnly = true;
 
-    const wireReporter = await createReporterForTestServer(config, 'list', params.serializer || require.resolve('./uiModeReporter'), e => report.push(e));
-    const report: any[] = [];
-    const reporter = new InternalReporter(wireReporter);
-
     const taskRunner = createTaskRunnerForList(config, reporter, 'out-of-process', { failOnLoadErrors: false });
     const testRun = new TestRun(config, reporter);
     reporter.onConfigure(config.config);
@@ -216,17 +246,18 @@ class TestServerDispatcher implements TestServerInterface {
       projectOutputs.add(result.outDir);
     }
 
-    this._globalWatcher.update([...projectDirs], [...projectOutputs], false);
+    if (this._watchTestDir)
-    return { report };
+      this._globalWatcher.update([...projectDirs], [...projectOutputs], false);
+    return { report, status };
   }
 
   async runTests(params: Parameters<TestServerInterface['runTests']>[0]): ReturnType<TestServerInterface['runTests']> {
-    let result: Awaited<ReturnType<TestServerInterface['runTests']>>;
+    let result: Awaited<ReturnType<TestServerInterface['runTests']>> = { status: 'passed' };
     this._queue = this._queue.then(async () => {
-      result = await this._innerRunTests(params).catch(printInternalError) || { status: 'failed' };
+      result = await this._innerRunTests(params).catch(e => { printInternalError(e); return { status: 'failed' }; });
     });
     await this._queue;
-    return result!;
+    return result;
   }
 
   private async _innerRunTests(params: Parameters<TestServerInterface['runTests']>[0]): ReturnType<TestServerInterface['runTests']> {
@@ -250,8 +281,14 @@ class TestServerDispatcher implements TestServerInterface {
     else
       process.env.PW_LIVE_TRACE_STACKS = undefined;
 
+    const { config, error } = await this._loadConfig(this._configFile, overrides);
+    if (!config) {
+      const wireReporter = await this._wireReporter(e => this._dispatchEvent('report', e));
+      wireReporter.onError(error!);
+      return { status: 'failed' };
+    }
+
     const testIdSet = params.testIds ? new Set<string>(params.testIds) : null;
-    const config = await this._loadConfig(this._configFile, overrides);
     config.cliListOnly = false;
     config.cliPassWithNoTests = true;
     config.cliArgs = params.locations || [];
@@ -261,7 +298,8 @@ class TestServerDispatcher implements TestServerInterface {
     config.testIdMatcher = testIdSet ? id => testIdSet.has(id) : undefined;
 
     const reporters = await createReporters(config, 'test', true);
-    reporters.push(await createReporterForTestServer(config, 'test', params.serializer || require.resolve('./uiModeReporter'), e => this._dispatchEvent('report', e)));
+    const wireReporter = await this._wireReporter(e => this._dispatchEvent('report', e));
+    reporters.push(wireReporter);
     const reporter = new InternalReporter(new Multiplexer(reporters));
     const taskRunner = createTaskRunnerForTestServer(config, reporter);
     const testRun = new TestRun(config, reporter);
@@ -274,8 +312,11 @@ class TestServerDispatcher implements TestServerInterface {
       return status;
     });
     this._testRun = { run, stop };
-    const status = await run;
+    return { status: await run };
-    return { status };
+  }
+
+  async watchTestDir() {
+    this._watchTestDir = true;
   }
 
   async watch(params: { fileNames: string[]; }) {
@@ -288,8 +329,10 @@ class TestServerDispatcher implements TestServerInterface {
   }
 
   async findRelatedTestFiles(params: Parameters<TestServerInterface['findRelatedTestFiles']>[0]): ReturnType<TestServerInterface['findRelatedTestFiles']> {
-    const config = await this._loadConfig(this._configFile);
+    const { config, error } = await this._loadConfig(this._configFile);
-    const runner = new Runner(config);
+    if (error)
+      return { testFiles: [], errors: [error] };
+    const runner = new Runner(config!);
     return runner.findRelatedTestFiles('out-of-process', params.files);
   }
 
@@ -302,17 +345,20 @@ class TestServerDispatcher implements TestServerInterface {
     gracefullyProcessExitDoNotHang(0);
   }
 
-  private async _loadConfig(configFile: string | undefined, overrides?: ConfigCLIOverrides): Promise<FullConfigInternal> {
+  private async _loadConfig(configFile: string | undefined, overrides?: ConfigCLIOverrides): Promise<{ config: FullConfigInternal | null, error?: reporterTypes.TestError }> {
     const configFileOrDirectory = configFile ? path.resolve(process.cwd(), configFile) : process.cwd();
     const resolvedConfigFile = resolveConfigFile(configFileOrDirectory);
-    const config = await loadConfig({ resolvedConfigFile, configDir: resolvedConfigFile === configFileOrDirectory ? path.dirname(resolvedConfigFile) : configFileOrDirectory  }, overrides);
+    try {
-
+      const config = await loadConfig({ resolvedConfigFile, configDir: resolvedConfigFile === configFileOrDirectory ? path.dirname(resolvedConfigFile) : configFileOrDirectory  }, overrides);
-    // Preserve plugin instances between setup and build.
+      // Preserve plugin instances between setup and build.
-    if (!this._plugins)
+      if (!this._plugins)
-      this._plugins = config.plugins || [];
+        this._plugins = config.plugins || [];
-    else
+      else
-      config.plugins.splice(0, config.plugins.length, ...this._plugins);
+        config.plugins.splice(0, config.plugins.length, ...this._plugins);
-    return config;
+      return { config };
+    } catch (e) {
+      return { config: null, error: serializeError(e) };
+    }
   }
 }
 

packages/trace-viewer/src/ui/uiModeView.tsx

@@ -172,7 +172,8 @@ export const UIModeView: React.FC<{}> = ({
     setIsLoading(true);
     setWatchedTreeIds({ value: new Set() });
     (async () => {
-      const status = await testServerConnection.runGlobalSetup({});
+      await testServerConnection.watchTestDir({});
+      const { status } = await testServerConnection.runGlobalSetup({});
       if (status !== 'passed')
         return;
       const result = await testServerConnection.listTests({ projects: queryParams.projects, locations: queryParams.args });