diff --git a/packages/playwright-test/src/isomorphic/teleReceiver.ts b/packages/playwright-test/src/isomorphic/teleReceiver.ts index 32f22dc910..fc67985fb4 100644 --- a/packages/playwright-test/src/isomorphic/teleReceiver.ts +++ b/packages/playwright-test/src/isomorphic/teleReceiver.ts @@ -232,6 +232,7 @@ export class TeleReporterReceiver { result.status = payload.status; result.statusEx = payload.status; result.errors = payload.errors; + result.error = result.errors?.[0]; result.attachments = this._parseAttachments(payload.attachments); this._reporter.onTestEnd?.(test, result); } @@ -242,7 +243,10 @@ export class TeleReporterReceiver { const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined; const step: TestStep = { - titlePath: () => [], + titlePath: () => { + const parentPath = parentStep?.titlePath() || []; + return [...parentPath, payload.title]; + }, title: payload.title, category: payload.category, location: this._absoluteLocation(payload.location), diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts index 14ff77c071..cb8e5b8eb2 100644 --- a/packages/playwright-test/src/reporters/blob.ts +++ b/packages/playwright-test/src/reporters/blob.ts @@ -92,7 +92,7 @@ export class BlobReporter extends TeleReporterEmitter { override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] { return super._serializeAttachments(attachments).map(attachment => { - if (!attachment.path || !fs.statSync(attachment.path).isFile()) + if (!attachment.path || !fs.statSync(attachment.path, { throwIfNoEntry: false })?.isFile()) return attachment; // Add run guid to avoid clashes between shards. const sha1 = calculateSha1(attachment.path + this._salt); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 32e0143f70..5d887f02e1 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -88,7 +88,7 @@ export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-tes const configFile = (baseDir: string, files: Files): string | undefined => { for (const [name, content] of Object.entries(files)) { if (name.includes('playwright.config')) { - if (content.includes('reporter:')) + if (content.includes('reporter:') || content.includes('reportSlowTests:')) return path.resolve(baseDir, name); } } @@ -349,12 +349,7 @@ export const test = base const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir; const testProcess = childProcess({ command, - env: cleanEnv({ - PW_TEST_DEBUG_REPORTERS: '1', - PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', - PWTEST_TTY_WIDTH: '80', - ...env - }), + env: cleanEnv(env), cwd, }); const { exitCode } = await testProcess.exited; diff --git a/tests/playwright-test/reporter-base.spec.ts b/tests/playwright-test/reporter-base.spec.ts index 22d3eb48ba..494180ec53 100644 --- a/tests/playwright-test/reporter-base.spec.ts +++ b/tests/playwright-test/reporter-base.spec.ts @@ -17,346 +17,352 @@ import { test, expect } from './playwright-test-fixtures'; import * as path from 'path'; -test('handle long test names', async ({ runInlineTest }) => { - const title = 'title'.repeat(30); - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('${title}', async ({}) => { - expect(1).toBe(0); - }); - `, - }); - expect(result.output).toContain('expect(1).toBe'); - expect(result.exitCode).toBe(1); -}); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); -test('print the error name', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}) => { - const error = new Error('my-message'); - error.name = 'FooBarError'; - throw error; + test('handle long test names', async ({ runInlineTest }) => { + const title = 'title'.repeat(30); + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('${title}', async ({}) => { + expect(1).toBe(0); + }); + `, + }); + expect(result.output).toContain('expect(1).toBe'); + expect(result.exitCode).toBe(1); }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('FooBarError: my-message'); -}); -test('print should print the error name without a message', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}) => { - const error = new Error(); - error.name = 'FooBarError'; - throw error; + test('print the error name', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}) => { + const error = new Error('my-message'); + error.name = 'FooBarError'; + throw error; + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('FooBarError: my-message'); }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('FooBarError'); -}); -test('should print an error in a codeframe', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}) => { - const error = new Error('my-message'); - error.name = 'FooBarError'; - throw error; - }); - ` - }, {}, { - FORCE_COLOR: '0', - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('FooBarError: my-message'); - expect(result.output).not.toContain('at a.spec.ts:5'); - expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`); - expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`); - expect(result.output).toContain(`> 4 | const error = new Error('my-message');`); -}); - -test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'node_modules/utils/utils.js': ` - function assert(value) { - if (!value) - throw new Error('Assertion error'); - } - module.exports = { assert }; - `, - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - const { assert } = require('utils/utils.js'); - test('fail', async ({}) => { - assert(false); - }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - const output = result.output; - expect(output).toContain('Error: Assertion error'); - expect(output).toContain('a.spec.ts:4:11 › fail'); - expect(output).toContain(` 4 | test('fail', async ({}) => {`); - expect(output).toContain(`> 5 | assert(false);`); - expect(output).toContain(` | ^`); - expect(output).toContain(`utils.js:4`); - expect(output).toContain(`a.spec.ts:5:9`); -}); - -test('should print codeframe from a helper', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'helper.ts': ` - export function ohMy() { - throw new Error('oh my'); - } - `, - 'a.spec.ts': ` - import { ohMy } from './helper'; - import { test, expect } from '@playwright/test'; - test('foobar', async ({}) => { - ohMy(); - }); - ` - }, {}, { - FORCE_COLOR: '0', - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('Error: oh my'); - expect(result.output).toContain(` 2 | export function ohMy() {`); - expect(result.output).toContain(` > 3 | throw new Error('oh my');`); - expect(result.output).toContain(` | ^`); -}); - -test('should print slow tests', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - projects: [ - { name: 'foo' }, - { name: 'bar' }, - { name: 'baz' }, - { name: 'qux' }, - ], - reportSlowTests: { max: 0, threshold: 2400 }, - }; - `, - 'dir/a.test.js': ` - import { test, expect } from '@playwright/test'; - test('slow test', async ({}) => { - await new Promise(f => setTimeout(f, 2500)); - }); - `, - 'dir/b.test.js': ` - import { test, expect } from '@playwright/test'; - test('fast test', async ({}) => { - await new Promise(f => setTimeout(f, 1)); - }); - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(8); - expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`); - expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`); - expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`); - expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`); - expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`); - expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`); - expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`); - expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`); - expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`); -}); - -test('should not print slow parallel tests', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - reportSlowTests: { max: 0, threshold: 500 }, - }; - `, - 'dir/a.test.js': ` - import { test, expect } from '@playwright/test'; - test.describe.parallel('suite', () => { - test('inner slow test', async ({}) => { - await new Promise(f => setTimeout(f, 1000)); - }); - test('inner fast test', async ({}) => { - await new Promise(f => setTimeout(f, 100)); + test('print should print the error name without a message', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}) => { + const error = new Error(); + error.name = 'FooBarError'; + throw error; }); + ` }); - `, + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('FooBarError'); + }); + + test('should print an error in a codeframe', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}) => { + const error = new Error('my-message'); + error.name = 'FooBarError'; + throw error; + }); + ` + }, {}, { + FORCE_COLOR: '0', + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('FooBarError: my-message'); + expect(result.output).not.toContain('at a.spec.ts:5'); + expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`); + expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`); + expect(result.output).toContain(`> 4 | const error = new Error('my-message');`); + }); + + test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'node_modules/utils/utils.js': ` + function assert(value) { + if (!value) + throw new Error('Assertion error'); + } + module.exports = { assert }; + `, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + const { assert } = require('utils/utils.js'); + test('fail', async ({}) => { + assert(false); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + const output = result.output; + expect(output).toContain('Error: Assertion error'); + expect(output).toContain('a.spec.ts:4:15 › fail'); + expect(output).toContain(` 4 | test('fail', async ({}) => {`); + expect(output).toContain(`> 5 | assert(false);`); + expect(output).toContain(` | ^`); + expect(output).toContain(`utils.js:4`); + expect(output).toContain(`a.spec.ts:5:13`); + }); + + test('should print codeframe from a helper', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'helper.ts': ` + export function ohMy() { + throw new Error('oh my'); + } + `, + 'a.spec.ts': ` + import { ohMy } from './helper'; + import { test, expect } from '@playwright/test'; + test('foobar', async ({}) => { + ohMy(); + }); + ` + }, {}, { + FORCE_COLOR: '0', + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('Error: oh my'); + expect(result.output).toContain(` 2 | export function ohMy() {`); + expect(result.output).toContain(` > 3 | throw new Error('oh my');`); + expect(result.output).toContain(` | ^`); + }); + + test('should print slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'foo' }, + { name: 'bar' }, + { name: 'baz' }, + { name: 'qux' }, + ], + reportSlowTests: { max: 0, threshold: 2400 }, + }; + `, + 'dir/a.test.js': ` + import { test, expect } from '@playwright/test'; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 2500)); + }); + `, + 'dir/b.test.js': ` + import { test, expect } from '@playwright/test'; + test('fast test', async ({}) => { + await new Promise(f => setTimeout(f, 1)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(8); + expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`); + expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`); + expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`); + expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`); + expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`); + expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`); + expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`); + expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`); + expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`); + }); + + test('should not print slow parallel tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reportSlowTests: { max: 0, threshold: 500 }, + }; + `, + 'dir/a.test.js': ` + import { test, expect } from '@playwright/test'; + test.describe.parallel('suite', () => { + test('inner slow test', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); + }); + test('inner fast test', async ({}) => { + await new Promise(f => setTimeout(f, 100)); + }); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.output).not.toContain('Slow test file'); + }); + + test('should not print slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + projects: [ + { name: 'baz' }, + { name: 'qux' }, + ], + reportSlowTests: null, + }; + `, + 'dir/a.test.js': ` + import { test, expect } from '@playwright/test'; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 1000)); + }); + test('fast test', async ({}) => { + await new Promise(f => setTimeout(f, 100)); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(4); + expect(result.output).not.toContain('Slow test'); + }); + + test('should print flaky failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + ` + }, { retries: '1', reporter: 'list' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(result.output).toContain('expect(testInfo.retry).toBe(1)'); + }); + + test('should print flaky timeouts', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}, testInfo) => { + if (!testInfo.retry) + await new Promise(f => setTimeout(f, 2000)); + }); + ` + }, { retries: '1', reporter: 'list', timeout: '1000' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(result.output).toContain('Test timeout of 1000ms exceeded.'); + }); + + test('should print stack-less errors', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}) => { + const e = new Error('Hello'); + delete e.stack; + throw e; + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('Hello'); + }); + + test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async function myTest({}) { + const e = new Error('Hello'); + // Force stack to contain "Hello". + // Otherwise it is computed lazy and will get 'foo bar' instead. + e.stack; + e.message = 'foo bar'; + throw e; + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + const output = result.output; + expect(output).toContain('foo bar'); + expect(output).toContain('function myTest'); + }); + + test('should print "no tests found" error', async ({ runInlineTest }) => { + const result = await runInlineTest({ }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('No tests found'); + }); + + test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + testInfo.attachments.push({ + name: 'foo.txt', + body: undefined, + contentType: 'text/plain' + }); + expect(1).toBe(2); + }); + `, + }); + expect(result.output).not.toContain('Error in reporter'); + expect(result.failed).toBe(1); + expect(result.exitCode).toBe(1); + }); + + test('should report fatal errors at the end', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test as base, expect } from '@playwright/test'; + const test = base.extend({ + fixture: [async ({ }, use) => { + await use(); + throw new Error('oh my!'); + }, { scope: 'worker' }], + }); + test('good', async ({ fixture }) => { + }); + `, + 'b.spec.ts': ` + import { test as base, expect } from '@playwright/test'; + const test = base.extend({ + fixture: [async ({ }, use) => { + await use(); + throw new Error('oh my!'); + }, { scope: 'worker' }], + }); + test('good', async ({ fixture }) => { + }); + `, + }, { reporter: 'list' }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(2); + expect(result.output).toContain('2 errors were not a part of any test, see above for details'); + }); + + test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('should work', () => {}); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/); + }); }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.output).not.toContain('Slow test file'); -}); - -test('should not print slow tests', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - projects: [ - { name: 'baz' }, - { name: 'qux' }, - ], - reportSlowTests: null, - }; - `, - 'dir/a.test.js': ` - import { test, expect } from '@playwright/test'; - test('slow test', async ({}) => { - await new Promise(f => setTimeout(f, 1000)); - }); - test('fast test', async ({}) => { - await new Promise(f => setTimeout(f, 100)); - }); - `, - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(4); - expect(result.output).not.toContain('Slow test'); -}); - -test('should print flaky failures', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}, testInfo) => { - expect(testInfo.retry).toBe(1); - }); - ` - }, { retries: '1', reporter: 'list' }); - expect(result.exitCode).toBe(0); - expect(result.flaky).toBe(1); - expect(result.output).toContain('expect(testInfo.retry).toBe(1)'); -}); - -test('should print flaky timeouts', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}, testInfo) => { - if (!testInfo.retry) - await new Promise(f => setTimeout(f, 2000)); - }); - ` - }, { retries: '1', reporter: 'list', timeout: '1000' }); - expect(result.exitCode).toBe(0); - expect(result.flaky).toBe(1); - expect(result.output).toContain('Test timeout of 1000ms exceeded.'); -}); - -test('should print stack-less errors', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}) => { - const e = new Error('Hello'); - delete e.stack; - throw e; - }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).toContain('Hello'); -}); - -test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async function myTest({}) { - const e = new Error('Hello'); - // Force stack to contain "Hello". - // Otherwise it is computed lazy and will get 'foo bar' instead. - e.stack; - e.message = 'foo bar'; - throw e; - }); - ` - }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - const output = result.output; - expect(output).toContain('foo bar'); - expect(output).toContain('function myTest'); -}); - -test('should print "no tests found" error', async ({ runInlineTest }) => { - const result = await runInlineTest({ }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain('No tests found'); -}); - -test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - testInfo.attachments.push({ - name: 'foo.txt', - body: undefined, - contentType: 'text/plain' - }); - expect(1).toBe(2); - }); - `, - }); - expect(result.output).not.toContain('Error in reporter'); - expect(result.failed).toBe(1); - expect(result.exitCode).toBe(1); -}); - -test('should report fatal errors at the end', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test as base, expect } from '@playwright/test'; - const test = base.extend({ - fixture: [async ({ }, use) => { - await use(); - throw new Error('oh my!'); - }, { scope: 'worker' }], - }); - test('good', async ({ fixture }) => { - }); - `, - 'b.spec.ts': ` - import { test as base, expect } from '@playwright/test'; - const test = base.extend({ - fixture: [async ({ }, use) => { - await use(); - throw new Error('oh my!'); - }, { scope: 'worker' }], - }); - test('good', async ({ fixture }) => { - }); - `, - }, { reporter: 'list' }); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(2); - expect(result.output).toContain('2 errors were not a part of any test, see above for details'); -}); - -test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('should work', () => {}); - ` - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/); -}); \ No newline at end of file +} \ No newline at end of file diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index bd7a91e623..53ad62c7d7 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -361,7 +361,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']); - const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'list'] }); + const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); expect(exitCode).toBe(0); const text = stripAnsi(output); diff --git a/tests/playwright-test/reporter-dot.spec.ts b/tests/playwright-test/reporter-dot.spec.ts index 5fd5d20c45..5afda3f7bd 100644 --- a/tests/playwright-test/reporter-dot.spec.ts +++ b/tests/playwright-test/reporter-dot.spec.ts @@ -17,93 +17,99 @@ import colors from 'colors/safe'; import { test, expect } from './playwright-test-fixtures'; -test('render expected', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: 'dot' }); - expect(result.rawOutput).toContain(colors.green('·')); - expect(result.exitCode).toBe(0); -}); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); -test('render unexpected', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { reporter: 'dot' }); - expect(result.rawOutput).toContain(colors.red('F')); - expect(result.exitCode).toBe(1); -}); + test('render expected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: 'dot' }); + expect(result.rawOutput).toContain(colors.green('·')); + expect(result.exitCode).toBe(0); + }); -test('render unexpected after retry', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { retries: 3, reporter: 'dot' }); - const text = result.output; - expect(text).toContain('×××F'); - expect(result.rawOutput).toContain(colors.red('F')); - expect(result.exitCode).toBe(1); -}); + test('render unexpected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { reporter: 'dot' }); + expect(result.rawOutput).toContain(colors.red('F')); + expect(result.exitCode).toBe(1); + }); -test('render flaky', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - expect(testInfo.retry).toBe(3); - }); - `, - }, { retries: 3, reporter: 'dot' }); - const text = result.output; - expect(text).toContain('×××±'); - expect(result.rawOutput).toContain(colors.yellow('±')); - expect(text).toContain('1 flaky'); - expect(text).toContain('Retry #1'); - expect(result.exitCode).toBe(0); -}); + test('render unexpected after retry', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { retries: 3, reporter: 'dot' }); + const text = result.output; + expect(text).toContain('×××F'); + expect(result.rawOutput).toContain(colors.red('F')); + expect(result.exitCode).toBe(1); + }); -test('should work from config', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { reporter: 'dot' }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: 'dot' }); - expect(result.rawOutput).toContain(colors.green('·')); - expect(result.exitCode).toBe(0); -}); + test('render flaky', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + expect(testInfo.retry).toBe(3); + }); + `, + }, { retries: 3, reporter: 'dot' }); + const text = result.output; + expect(text).toContain('×××±'); + expect(result.rawOutput).toContain(colors.yellow('±')); + expect(text).toContain('1 flaky'); + expect(text).toContain('Retry #1'); + expect(result.exitCode).toBe(0); + }); -test('render 243 tests in rows by 80', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - for (let i = 0; i < 243; i++) { - test('test' + i, () => {}); - } - `, - }, { reporter: 'dot' }); - expect(result.exitCode).toBe(0); - expect(result.rawOutput).toContain( - colors.green('·').repeat(80) + '\n' + - colors.green('·').repeat(80) + '\n' + - colors.green('·').repeat(80) + '\n' + - colors.green('·').repeat(3)); -}); + test('should work from config', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: 'dot' }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: 'dot' }); + expect(result.rawOutput).toContain(colors.green('·')); + expect(result.exitCode).toBe(0); + }); + + test('render 243 tests in rows by 80', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + for (let i = 0; i < 243; i++) { + test('test' + i, () => {}); + } + `, + }, { reporter: 'dot' }); + expect(result.exitCode).toBe(0); + expect(result.rawOutput).toContain( + colors.green('·').repeat(80) + '\n' + + colors.green('·').repeat(80) + '\n' + + colors.green('·').repeat(80) + '\n' + + colors.green('·').repeat(3)); + }); + }); +} \ No newline at end of file diff --git a/tests/playwright-test/reporter-github.spec.ts b/tests/playwright-test/reporter-github.spec.ts index 1197ab4d1f..100feb157f 100644 --- a/tests/playwright-test/reporter-github.spec.ts +++ b/tests/playwright-test/reporter-github.spec.ts @@ -23,73 +23,79 @@ function relativeFilePath(file: string): string { return path.relative(process.cwd(), file); } -test('print GitHub annotations for success', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('example1', async ({}) => { - expect(1 + 1).toBe(2); - }); - ` - }, { reporter: 'github' }); - const text = result.output; - expect(text).not.toContain('::error'); - expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed'); - expect(result.exitCode).toBe(0); -}); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); -test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test, expect } = require('@playwright/test'); - test('example', async ({}) => { - expect(1 + 1).toBe(3); - }); - ` - }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() }); - const text = result.output; - const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); - expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`); - expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`); - expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`); - expect(result.exitCode).toBe(1); -}); + test('print GitHub annotations for success', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('example1', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'github' }); + const text = result.output; + expect(text).not.toContain('::error'); + expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed'); + expect(result.exitCode).toBe(0); + }); -test('print GitHub annotations for slow tests', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - reportSlowTests: { max: 0, threshold: 100 } - }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('slow test', async ({}) => { - await new Promise(f => setTimeout(f, 200)); - }); - ` - }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' }); - const text = result.output; - expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took'); - expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed'); - expect(result.exitCode).toBe(0); -}); + test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test, expect } = require('@playwright/test'); + test('example', async ({}) => { + expect(1 + 1).toBe(3); + }); + ` + }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() }); + const text = result.output; + const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); + expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #1`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #2`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #3`); + expect(result.exitCode).toBe(1); + }); -test('print GitHub annotations for global error', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test as base, expect } from '@playwright/test'; - const test = base.extend({ - w: [async ({}, use) => { - await use(); - throw new Error('Oh my!'); - }, { scope: 'worker' }], - }); - test('passes but...', ({w}) => { - }); - `, - }, { reporter: 'github' }); - const text = result.output; - expect(text).toContain('::error ::Error: Oh my!%0A%0A'); - expect(result.exitCode).toBe(1); -}); + test('print GitHub annotations for slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reportSlowTests: { max: 0, threshold: 100 } + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 200)); + }); + ` + }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' }); + const text = result.output; + expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took'); + expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed'); + expect(result.exitCode).toBe(0); + }); + + test('print GitHub annotations for global error', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test as base, expect } from '@playwright/test'; + const test = base.extend({ + w: [async ({}, use) => { + await use(); + throw new Error('Oh my!'); + }, { scope: 'worker' }], + }); + test('passes but...', ({w}) => { + }); + `, + }, { reporter: 'github' }); + const text = result.output; + expect(text).toContain('::error ::Error: Oh my!%0A%0A'); + expect(result.exitCode).toBe(1); + }); + }); +} \ No newline at end of file diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index c4bbbecd1b..ec8f2950a9 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -19,424 +19,432 @@ import path from 'path'; import { test, expect } from './playwright-test-fixtures'; import fs from 'fs'; -test('should render expected', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - 'b.test.js': ` - import { test, expect } from '@playwright/test'; - test('two', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['$']['tests']).toBe('2'); - expect(xml['testsuites']['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'].length).toBe(2); - expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); - expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); - expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); - expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js'); - expect(result.exitCode).toBe(0); -}); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); -test('should render unexpected', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['$']['tests']).toBe('1'); - expect(xml['testsuites']['$']['failures']).toBe('1'); - const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0]; - expect(failure['$']['message']).toContain('a.test.js'); - expect(failure['$']['message']).toContain('one'); - expect(failure['$']['type']).toBe('FAILURE'); - expect(failure['_']).toContain('expect(1).toBe(0)'); - expect(result.exitCode).toBe(1); -}); - -test('should render unexpected after retry', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { retries: 3, reporter: 'junit' }); - expect(result.output).toContain(`tests="1"`); - expect(result.output).toContain(`failures="1"`); - expect(result.output).toContain(` { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - expect(testInfo.retry).toBe(3); - }); - `, - }, { retries: 3, reporter: 'junit' }); - expect(result.output).not.toContain('Retry #1'); - expect(result.exitCode).toBe(0); -}); - -test('should render stdout', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import colors from 'colors/safe'; - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - console.log(colors.yellow('Hello world')); - console.log('Hello again'); - console.error('My error'); - console.error('\\0'); // null control character - test.expect("abc").toBe('abcd'); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['system-out'].length).toBe(1); - expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again'); - expect(testcase['system-out'][0]).not.toContain('u00'); - expect(testcase['system-err'][0]).toContain('My error'); - expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character - expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`); - expect(result.exitCode).toBe(1); -}); - -test('should render stdout without ansi escapes', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - reporter: [ ['junit', { stripANSIControlSequences: true }] ], - }; - `, - 'a.test.ts': ` - import colors from 'colors/safe'; - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - console.log(colors.yellow('Hello world')); - }); - `, - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['system-out'].length).toBe(1); - expect(testcase['system-out'][0].trim()).toBe('Hello world'); - expect(result.exitCode).toBe(0); -}); - -test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - reporter: [ ['junit'] ], - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - process.stdout.write('Hello world &"\\'<>]]>'); - }); - `, - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['system-out'].length).toBe(1); - expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]>'); - expect(result.output).toContain(`\n]]>]]>\n`); - expect(result.exitCode).toBe(0); -}); - -test('should render skipped', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async () => { - console.log('Hello world'); - }); - test('two', async () => { - test.skip(); - console.log('Hello world'); - }); - `, - }, { retries: 3, reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); - expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); - expect(result.exitCode).toBe(0); -}); - -test('should report skipped due to sharding', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async () => { - }); - test('two', async () => { - test.skip(); - }); - `, - 'b.test.js': ` - import { test, expect } from '@playwright/test'; - test('three', async () => { - }); - test('four', async () => { - test.skip(); - }); - test('five', async () => { - }); - `, - }, { shard: '1/3', reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['testsuite'].length).toBe(1); - expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); - expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); - expect(result.exitCode).toBe(0); -}); - -test('should not render projects if they dont exist', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['$']['tests']).toBe('1'); - expect(xml['testsuites']['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'].length).toBe(1); - - expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); - expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); - expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); - expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); - expect(result.exitCode).toBe(0); -}); - -test('should render projects', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - expect(xml['testsuites']['$']['tests']).toBe('2'); - expect(xml['testsuites']['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'].length).toBe(2); - - expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); - expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1'); - expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); - expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); - expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); - expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); - - expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js'); - expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2'); - expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1'); - expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0'); - expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0'); - expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one'); - expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js'); - expect(result.exitCode).toBe(0); -}); - -test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test.use({ screenshot: 'on' }); - test('one', async ({ page }, testInfo) => { - await page.setContent('hello'); - const file = testInfo.outputPath('file.txt'); - require('fs').writeFileSync(file, 'my file', 'utf8'); - testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' }); - testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' }); - console.log('log here'); - }); - `, - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['system-out'].length).toBe(1); - expect(testcase['system-out'][0].trim()).toBe([ - `log here`, - `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`, - `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`, - ].join('\n')); - expect(result.exitCode).toBe(0); -}); - -function parseXML(xml: string): any { - let result: any; - xml2js.parseString(xml, (err, r) => result = r); - return result; -} - -test('should render annotations to custom testcase properties', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); - }); - ` - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).toBeTruthy(); - expect(testcase['properties'][0]['property'].length).toBe(1); - expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description'); - expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description'); - expect(result.exitCode).toBe(0); -}); - -test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - test.slow(); - }); - ` - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).toBeTruthy(); - expect(testcase['properties'][0]['property'].length).toBe(1); - expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow'); - expect(testcase['properties'][0]['property'][0]['$']['value']).toBe(''); - expect(result.exitCode).toBe(0); -}); - -test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - const xrayOptions = { - embedAnnotationsAsProperties: true - } - module.exports = { - reporter: [ ['junit', xrayOptions] ], - }; - `, - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - testInfo.annotations.push({ type: 'test_id', description: '1234' }); - testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); - testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); - testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); - }); - ` - }, { reporter: '' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).toBeTruthy(); - expect(testcase['properties'][0]['property'].length).toBe(4); - expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id'); - expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234'); - expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key'); - expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2'); - expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary'); - expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary'); - expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements'); - expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6'); - expect(result.exitCode).toBe(0); -}); - -test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - const file = testInfo.outputPath('evidence1.txt'); - require('fs').writeFileSync(file, 'hello', 'utf8'); - testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); - testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); - // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); - // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); - }); - ` - }, { reporter: 'junit' }); - const xml = parseXML(result.output); - const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; - expect(testcase['properties']).not.toBeTruthy(); - expect(result.exitCode).toBe(0); -}); - - -test.describe('report location', () => { - test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'nested/project/playwright.config.ts': ` - module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] }; - `, - 'nested/project/a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}) => { - expect(1).toBe(1); - }); - `, - }, { reporter: '', config: './nested/project/playwright.config.ts' }); - expect(result.exitCode).toBe(0); - expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy(); - }); - - test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'foo/package.json': `{ "name": "foo" }`, - // unused config along "search path" - 'foo/bar/playwright.config.js': ` - module.exports = { projects: [ {} ] }; - `, - 'foo/bar/baz/tests/a.spec.js': ` - import { test, expect } from '@playwright/test'; - const fs = require('fs'); - test('pass', ({}, testInfo) => { - }); - ` - }, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { - cwd: 'foo/bar/baz/tests', + test('should render expected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('two', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['$']['tests']).toBe('2'); + expect(xml['testsuites']['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'].length).toBe(2); + expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); + expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js'); + expect(result.exitCode).toBe(0); + }); + + test('should render unexpected', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['$']['tests']).toBe('1'); + expect(xml['testsuites']['$']['failures']).toBe('1'); + const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0]; + expect(failure['$']['message']).toContain('a.test.js'); + expect(failure['$']['message']).toContain('one'); + expect(failure['$']['type']).toBe('FAILURE'); + expect(failure['_']).toContain('expect(1).toBe(0)'); + expect(result.exitCode).toBe(1); + }); + + test('should render unexpected after retry', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { retries: 3, reporter: 'junit' }); + expect(result.output).toContain(`tests="1"`); + expect(result.output).toContain(`failures="1"`); + expect(result.output).toContain(` { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + expect(testInfo.retry).toBe(3); + }); + `, + }, { retries: 3, reporter: 'junit' }); + expect(result.output).not.toContain('Retry #1'); + expect(result.exitCode).toBe(0); + }); + + test('should render stdout', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import colors from 'colors/safe'; + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + console.log(colors.yellow('Hello world')); + console.log('Hello again'); + console.error('My error'); + console.error('\\0'); // null control character + test.expect("abc").toBe('abcd'); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['system-out'].length).toBe(1); + expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again'); + expect(testcase['system-out'][0]).not.toContain('u00'); + expect(testcase['system-err'][0]).toContain('My error'); + expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character + expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`); + expect(result.exitCode).toBe(1); + }); + + test('should render stdout without ansi escapes', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [ ['junit', { stripANSIControlSequences: true }] ], + }; + `, + 'a.test.ts': ` + import colors from 'colors/safe'; + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + console.log(colors.yellow('Hello world')); + }); + `, + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['system-out'].length).toBe(1); + expect(testcase['system-out'][0].trim()).toBe('Hello world'); + expect(result.exitCode).toBe(0); + }); + + test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reporter: [ ['junit'] ], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + process.stdout.write('Hello world &"\\'<>]]>'); + }); + `, + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['system-out'].length).toBe(1); + expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]>'); + expect(result.output).toContain(`\n]]>]]>\n`); + expect(result.exitCode).toBe(0); + }); + + test('should render skipped', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async () => { + console.log('Hello world'); + }); + test('two', async () => { + test.skip(); + console.log('Hello world'); + }); + `, + }, { retries: 3, reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); + expect(result.exitCode).toBe(0); + }); + + test('should report skipped due to sharding', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async () => { + }); + test('two', async () => { + test.skip(); + }); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('three', async () => { + }); + test('four', async () => { + test.skip(); + }); + test('five', async () => { + }); + `, + }, { shard: '1/3', reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['testsuite'].length).toBe(1); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); + expect(result.exitCode).toBe(0); + }); + + test('should not render projects if they dont exist', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['$']['tests']).toBe('1'); + expect(xml['testsuites']['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'].length).toBe(1); + + expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); + expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); + expect(result.exitCode).toBe(0); + }); + + test('should render projects', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + expect(xml['testsuites']['$']['tests']).toBe('2'); + expect(xml['testsuites']['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'].length).toBe(2); + + expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); + expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1'); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); + expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); + + expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js'); + expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2'); + expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0'); + expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one'); + expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js'); + expect(result.exitCode).toBe(0); + }); + + test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => { + test.skip(useIntermediateMergeReport, 'Blob report hashes attachment paths'); + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test.use({ screenshot: 'on' }); + test('one', async ({ page }, testInfo) => { + await page.setContent('hello'); + const file = testInfo.outputPath('file.txt'); + require('fs').writeFileSync(file, 'my file', 'utf8'); + testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' }); + testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' }); + console.log('log here'); + }); + `, + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['system-out'].length).toBe(1); + expect(testcase['system-out'][0].trim()).toBe([ + `log here`, + `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`, + `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`, + ].join('\n')); + expect(result.exitCode).toBe(0); + }); + + function parseXML(xml: string): any { + let result: any; + xml2js.parseString(xml, (err, r) => result = r); + return result; + } + + test('should render annotations to custom testcase properties', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(1); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description'); + expect(result.exitCode).toBe(0); + }); + + test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': `module.exports = { reporter: 'junit' };`, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + test.slow(); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(1); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe(''); + expect(result.exitCode).toBe(0); + }); + + test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + const xrayOptions = { + embedAnnotationsAsProperties: true + } + module.exports = { + reporter: [ ['junit', xrayOptions] ], + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_id', description: '1234' }); + testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); + testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); + testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(4); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234'); + expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key'); + expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2'); + expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary'); + expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary'); + expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements'); + expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6'); + expect(result.exitCode).toBe(0); + }); + + test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + const file = testInfo.outputPath('evidence1.txt'); + require('fs').writeFileSync(file, 'hello', 'utf8'); + testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); + testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); + // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); + // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); + }); + ` + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).not.toBeTruthy(); + expect(result.exitCode).toBe(0); + }); + + + test.describe('report location', () => { + test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { + test.skip(useIntermediateMergeReport); + const result = await runInlineTest({ + 'nested/project/playwright.config.ts': ` + module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] }; + `, + 'nested/project/a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: '', config: './nested/project/playwright.config.ts' }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy(); + }); + + test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + import { test, expect } from '@playwright/test'; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { + cwd: 'foo/bar/baz/tests', + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); + }); }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); }); -}); +} \ No newline at end of file diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts index 23ccaf878b..5e591b191d 100644 --- a/tests/playwright-test/reporter-line.spec.ts +++ b/tests/playwright-test/reporter-line.spec.ts @@ -16,149 +16,155 @@ import { test, expect } from './playwright-test-fixtures'; -test('render unexpected after retry', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test, expect } = require('@playwright/test'); - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { retries: 3, reporter: 'line' }); - const text = result.output; - expect(text).toContain('[1/1] a.test.js:3:7 › one'); - expect(text).toContain('[2/1] (retries) a.test.js:3:7 › one (retry #1)'); - expect(text).toContain('[3/1] (retries) a.test.js:3:7 › one (retry #2)'); - expect(text).toContain('[4/1] (retries) a.test.js:3:7 › one (retry #3)'); - expect(text).toContain('1 failed'); - expect(text).toContain('1) a.test'); - expect(text).not.toContain('2) a.test'); - expect(text).toContain('Retry #1 ────'); - expect(text).toContain('Retry #2 ────'); - expect(text).toContain('Retry #3 ────'); - expect(result.exitCode).toBe(1); -}); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); -test('render flaky', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - import { test, expect } from '@playwright/test'; - test('one', async ({}, testInfo) => { - expect(testInfo.retry).toBe(3); - }); - `, - }, { retries: 3, reporter: 'line' }); - const text = result.output; - expect(text).toContain('1 flaky'); - expect(result.exitCode).toBe(0); -}); - -test('should print flaky failures', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}, testInfo) => { - expect(testInfo.retry).toBe(1); - }); - ` - }, { retries: '1', reporter: 'line' }); - expect(result.exitCode).toBe(0); - expect(result.flaky).toBe(1); - expect(result.output).toContain('expect(testInfo.retry).toBe(1)'); -}); - -test('should work on CI', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.js': ` - const { test, expect } = require('@playwright/test'); - test('one', async ({}) => { - expect(1).toBe(0); - }); - `, - }, { reporter: 'line' }, { CI: '1' }); - const text = result.output; - expect(text).toContain('[1/1] a.test.js:3:7 › one'); - expect(text).toContain('1 failed'); - expect(text).toContain('1) a.test'); - expect(result.exitCode).toBe(1); -}); - -test('should print output', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.ts': ` - import { test, expect } from '@playwright/test'; - test('foobar', async ({}, testInfo) => { - process.stdout.write('one'); - process.stdout.write('two'); - console.log('full-line'); - }); - ` - }, { reporter: 'line' }); - expect(result.exitCode).toBe(0); - expect(result.output).toContain([ - 'a.spec.ts:3:11 › foobar', - 'one', - '', - 'two', - '', - 'full-line', - ].join('\n')); -}); - -test('should render failed test steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => { - expect(1).toBe(2); + test('render unexpected after retry', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test, expect } = require('@playwright/test'); + test('one', async ({}) => { + expect(1).toBe(0); }); - }); - }); - `, - }, { reporter: 'line' }); - const text = result.output; - expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); - expect(result.exitCode).toBe(1); -}); + `, + }, { retries: 3, reporter: 'line' }); + const text = result.output; + expect(text).toContain('[1/1] a.test.js:3:11 › one'); + expect(text).toContain('[2/1] (retries) a.test.js:3:11 › one (retry #1)'); + expect(text).toContain('[3/1] (retries) a.test.js:3:11 › one (retry #2)'); + expect(text).toContain('[4/1] (retries) a.test.js:3:11 › one (retry #3)'); + expect(text).toContain('1 failed'); + expect(text).toContain('1) a.test'); + expect(text).not.toContain('2) a.test'); + expect(text).toContain('Retry #1 ────'); + expect(text).toContain('Retry #2 ────'); + expect(text).toContain('Retry #3 ────'); + expect(result.exitCode).toBe(1); + }); -test('should not render more than one failed test steps in header', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => { + test('render flaky', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + expect(testInfo.retry).toBe(3); + }); + `, + }, { retries: 3, reporter: 'line' }); + const text = result.output; + expect(text).toContain('1 flaky'); + expect(result.exitCode).toBe(0); + }); + + test('should print flaky failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + ` + }, { retries: '1', reporter: 'line' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(result.output).toContain('expect(testInfo.retry).toBe(1)'); + }); + + test('should work on CI', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test, expect } = require('@playwright/test'); + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { reporter: 'line' }, { CI: '1' }); + const text = result.output; + expect(text).toContain('[1/1] a.test.js:3:11 › one'); + expect(text).toContain('1 failed'); + expect(text).toContain('1) a.test'); + expect(result.exitCode).toBe(1); + }); + + test('should print output', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('foobar', async ({}, testInfo) => { + process.stdout.write('one'); + process.stdout.write('two'); + console.log('full-line'); + }); + ` + }, { reporter: 'line' }); + expect(result.exitCode).toBe(0); + expect(result.output).toContain([ + 'a.spec.ts:3:15 › foobar', + 'one', + '', + 'two', + '', + 'full-line', + ].join('\n')); + }); + + test('should render failed test steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──'); + expect(result.exitCode).toBe(1); + }); + + test('should not render more than one failed test steps in header', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect.soft(1).toBe(2); + }); + await test.step('inner 1.2', async () => { + expect.soft(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 ──'); + expect(result.exitCode).toBe(1); + }); + + test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect.soft(1).toBe(2); + }); + }); expect.soft(1).toBe(2); }); - await test.step('inner 1.2', async () => { - expect.soft(1).toBe(2); - }); - }); - }); - `, - }, { reporter: 'line' }); - const text = result.output; - expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──'); - expect(result.exitCode).toBe(1); -}); - -test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => { - expect.soft(1).toBe(2); - }); - }); - expect.soft(1).toBe(2); - }); - `, - }, { reporter: 'line' }); - const text = result.output; - expect(text).toContain('1) a.test.ts:3:11 › passes ──'); - expect(result.exitCode).toBe(1); -}); + `, + }, { reporter: 'line' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:15 › passes ──'); + expect(result.exitCode).toBe(1); + }); + }); +} \ No newline at end of file diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts index 068765f3f6..b16b078065 100644 --- a/tests/playwright-test/reporter-list.spec.ts +++ b/tests/playwright-test/reporter-list.spec.ts @@ -20,212 +20,217 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ '; const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ '; -test('render each test with project name', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { projects: [ - { name: 'foo' }, - { name: 'bar' }, - ] }; - `, - 'a.test.ts': ` - const { test, expect } = require('@playwright/test'); - test('fails', async ({}) => { - expect(1).toBe(0); - }); - test('passes', async ({}) => { - expect(0).toBe(0); - }); - test.skip('skipped', async () => { - }); - `, - }, { reporter: 'list', workers: '1' }); - const text = result.output; +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); - expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:7 › fails`); - expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:7 › passes`); - expect(text).toContain(`- 3 [foo] › a.test.ts:9:12 › skipped`); - expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:7 › fails`); - expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:7 › passes`); - expect(text).toContain(`- 6 [bar] › a.test.ts:9:12 › skipped`); - expect(result.exitCode).toBe(1); -}); - -test('render steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => {}); - await test.step('inner 1.2', async () => {}); - }); - await test.step('outer 2.0', async () => { - await test.step('inner 2.1', async () => {}); - await test.step('inner 2.2', async () => {}); - }); - }); - `, - }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }); - const text = result.output; - const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); - lines.pop(); // Remove last item that contains [v] and time in ms. - expect(lines).toEqual([ - '0 : 1 a.test.ts:3:11 › passes', - '1 : 1.1 passes › outer 1.0', - '2 : 1.2 passes › outer 1.0 › inner 1.1', - '2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', - '3 : 1.3 passes › outer 1.0 › inner 1.2', - '3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', - '1 : 1.1 passes › outer 1.0 (Xms)', - '4 : 1.4 passes › outer 2.0', - '5 : 1.5 passes › outer 2.0 › inner 2.1', - '5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', - '6 : 1.6 passes › outer 2.0 › inner 2.2', - '6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', - '4 : 1.4 passes › outer 2.0 (Xms)', - ]); -}); - -test('render steps inline', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => {}); - await test.step('inner 1.2', async () => {}); - }); - await test.step('outer 2.0', async () => { - await test.step('inner 2.1', async () => {}); - await test.step('inner 2.2', async () => {}); - }); - }); - `, - }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' }); - const text = result.output; - const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); - lines.pop(); // Remove last item that contains [v] and time in ms. - expect(lines).toEqual([ - '0 : 1 a.test.ts:3:11 › passes', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:3:11 › passes', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:3:11 › passes', - ]); -}); - -test('very long console line should not mess terminal', async ({ runInlineTest }) => { - const TTY_WIDTH = 80; - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - console.log('a'.repeat(80) + 'b'.repeat(20)); - }); - `, - }, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' }); - - const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH); - if (process.platform === 'win32') - expect(renderedText).toContain(' ok 1 a.test.ts:3:11 › passes'); - else - expect(renderedText).toContain(' ✓ 1 a.test.ts:3:11 › passes'); - expect(renderedText).not.toContain(' 1 a.test.ts:3:11 › passes'); - expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20)); -}); - -test('render retries', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('flaky', async ({}, testInfo) => { - expect(testInfo.retry).toBe(1); - }); - `, - }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' }); - const text = result.output; - const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms')); - - expect(lines).toEqual([ - `0 : 1 a.test.ts:3:11 › flaky`, - `0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:11 › flaky (XXms)`, - `1 : 2 a.test.ts:3:11 › flaky (retry #1)`, - `1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:11 › flaky (retry #1) (XXms)`, - ]); -}); - -test('should truncate long test names', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { projects: [ - { name: 'foo' }, - ] }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('failure in very long name', async ({}) => { - expect(1).toBe(0); - }); - test('passes', async ({}) => { - }); - test('passes 2 long name', async () => { - }); - test.skip('skipped very long name', async () => { - }); - `, - }, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' }); - expect(result.exitCode).toBe(1); - - const lines = result.output.split('\n').slice(3, 11); - expect(lines.every(line => line.length <= 50)).toBe(true); - - expect(lines[0]).toBe(` 1 …a.test.ts:3:11 › failure in very long name`); - - expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`); - expect(lines[1]).toContain(`:3:11 › failure in very long name (`); - expect(lines[1].length).toBe(50); - - expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:11 › passes`); - - expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes (`); - - expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:11 › passes 2 long name`); - - expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`); - expect(lines[5]).toContain(`test.ts:8:11 › passes 2 long name (`); - expect(lines[5].length).toBe(50); - - expect(lines[6]).toBe(` 4 …› a.test.ts:10:12 › skipped very long name`); - - expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`); -}); - -test('render failed test steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('passes', async ({}) => { - await test.step('outer 1.0', async () => { - await test.step('inner 1.1', async () => { - expect(1).toBe(2); + test('render each test with project name', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'foo' }, + { name: 'bar' }, + ] }; + `, + 'a.test.ts': ` + const { test, expect } = require('@playwright/test'); + test('fails', async ({}) => { + expect(1).toBe(0); }); + test('passes', async ({}) => { + expect(0).toBe(0); + }); + test.skip('skipped', async () => { + }); + `, + }, { reporter: 'list', workers: '1' }); + const text = result.output; + + expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:11 › fails`); + expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes`); + expect(text).toContain(`- 3 [foo] › a.test.ts:9:16 › skipped`); + expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:11 › fails`); + expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:11 › passes`); + expect(text).toContain(`- 6 [bar] › a.test.ts:9:16 › skipped`); + expect(result.exitCode).toBe(1); + }); + + test('render steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => {}); + await test.step('inner 1.2', async () => {}); + }); + await test.step('outer 2.0', async () => { + await test.step('inner 2.1', async () => {}); + await test.step('inner 2.2', async () => {}); + }); + }); + `, + }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }); + const text = result.output; + const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); + lines.pop(); // Remove last item that contains [v] and time in ms. + expect(lines).toEqual([ + '0 : 1 a.test.ts:3:15 › passes', + '1 : 1.1 passes › outer 1.0', + '2 : 1.2 passes › outer 1.0 › inner 1.1', + '2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', + '3 : 1.3 passes › outer 1.0 › inner 1.2', + '3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', + '1 : 1.1 passes › outer 1.0 (Xms)', + '4 : 1.4 passes › outer 2.0', + '5 : 1.5 passes › outer 2.0 › inner 2.1', + '5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', + '6 : 1.6 passes › outer 2.0 › inner 2.2', + '6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', + '4 : 1.4 passes › outer 2.0 (Xms)', + ]); + }); + + test('render steps inline', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => {}); + await test.step('inner 1.2', async () => {}); }); - }); - `, - }, { reporter: 'list' }); - const text = result.output; - expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); - expect(result.exitCode).toBe(1); -}); + await test.step('outer 2.0', async () => { + await test.step('inner 2.1', async () => {}); + await test.step('inner 2.2', async () => {}); + }); + });`, + }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' }); + const text = result.output; + const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms')); + lines.pop(); // Remove last item that contains [v] and time in ms. + expect(lines).toEqual([ + '0 : 1 a.test.ts:3:11 › passes', + '0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1', + '0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2', + '0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '0 : 1 a.test.ts:3:11 › passes', + '0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1', + '0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2', + '0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '0 : 1 a.test.ts:3:11 › passes', + ]); + }); + + test('very long console line should not mess terminal', async ({ runInlineTest }) => { + const TTY_WIDTH = 80; + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + console.log('a'.repeat(80) + 'b'.repeat(20)); + }); + `, + }, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' }); + + const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH); + if (process.platform === 'win32') + expect(renderedText).toContain(' ok 1 a.test.ts:3:15 › passes'); + else + expect(renderedText).toContain(' ✓ 1 a.test.ts:3:15 › passes'); + expect(renderedText).not.toContain(' 1 a.test.ts:3:15 › passes'); + expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20)); + }); + + test('render retries', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + `, + }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' }); + const text = result.output; + const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms')); + + expect(lines).toEqual([ + `0 : 1 a.test.ts:3:15 › flaky`, + `0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`, + `1 : 2 a.test.ts:3:15 › flaky (retry #1)`, + `1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`, + ]); + }); + + test('should truncate long test names', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { projects: [ + { name: 'foo' }, + ] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('failure in very long name', async ({}) => { + expect(1).toBe(0); + }); + test('passes', async ({}) => { + }); + test('passes 2 long name', async () => { + }); + test.skip('skipped very long name', async () => { + }); + `, + }, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' }); + expect(result.exitCode).toBe(1); + + const lines = result.output.split('\n').slice(3, 11); + expect(lines.every(line => line.length <= 50)).toBe(true); + + expect(lines[0]).toBe(` 1 …a.test.ts:3:15 › failure in very long name`); + + expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`); + expect(lines[1]).toContain(`:3:15 › failure in very long name (`); + expect(lines[1].length).toBe(50); + + expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:15 › passes`); + + expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:15 › passes (`); + + expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:15 › passes 2 long name`); + + expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`); + expect(lines[5]).toContain(`test.ts:8:15 › passes 2 long name (`); + expect(lines[5].length).toBe(50); + + expect(lines[6]).toBe(` 4 …› a.test.ts:10:16 › skipped very long name`); + + expect(lines[7]).toBe(` - 4 …› a.test.ts:10:16 › skipped very long name`); + }); + + test('render failed test steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => { + expect(1).toBe(2); + }); + }); + }); + `, + }, { reporter: 'list' }); + const text = result.output; + expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──'); + expect(result.exitCode).toBe(1); + }); + }); +} function simpleAnsiRenderer(text, ttyWidth) { let lineNumber = 0; diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index f15cd9224d..ff6f8ee3dd 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -74,584 +74,589 @@ class Reporter { module.exports = Reporter; `; -test('should work with custom reporter', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': ` - class Reporter { - constructor(options) { - this.options = options; - } - onBegin(config, suite) { - console.log('\\n%%reporter-begin-' + this.options.begin + '%%'); - console.log('\\n%%version-' + config.version); - } - onTestBegin(test) { - const projectName = test.titlePath()[1]; - console.log('\\n%%reporter-testbegin-' + test.title + '-' + projectName + '%%'); - const suite = test.parent; - if (!suite.tests.includes(test)) - console.log('\\n%%error-inconsistent-parent'); - if (test.parent.project().name !== projectName) - console.log('\\n%%error-inconsistent-project-name'); - } - onStdOut() { - console.log('\\n%%reporter-stdout%%'); - } - onStdErr() { - console.log('\\n%%reporter-stderr%%'); - } - onTestEnd(test, result) { - console.log('\\n%%reporter-testend-' + test.title + '-' + test.titlePath()[1] + '%%'); - if (!result.startTime) - console.log('\\n%%error-no-start-time'); - } - onTimeout() { - console.log('\\n%%reporter-timeout%%'); - } - onError() { - console.log('\\n%%reporter-error%%'); - } - async onEnd() { - await new Promise(f => setTimeout(f, 500)); - console.log('\\n%%reporter-end-' + this.options.end + '%%'); - } - } - export default Reporter; - `, - 'playwright.config.ts': ` - module.exports = { - reporter: [ - [ './reporter.ts', { begin: 'begin', end: 'end' } ] - ], - projects: [ - { name: 'foo', repeatEach: 2 }, - { name: 'bar' }, - ], - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('not run', async ({}) => { - console.log('log'); - console.error('error'); - }); - test.only('is run', async ({}) => { - console.log('log'); - console.error('error'); - }); - ` - }, { reporter: '', workers: 1 }); +for (const useIntermediateMergeReport of [false, true] as const) { + test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => { + test.use({ useIntermediateMergeReport }); - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - 'reporter-begin-begin%%', - 'version-' + require('../../packages/playwright-test/package.json').version, - 'reporter-testbegin-is run-foo%%', - 'reporter-stdout%%', - 'reporter-stderr%%', - 'reporter-testend-is run-foo%%', - 'reporter-testbegin-is run-foo%%', - 'reporter-stdout%%', - 'reporter-stderr%%', - 'reporter-testend-is run-foo%%', - 'reporter-testbegin-is run-bar%%', - 'reporter-stdout%%', - 'reporter-stderr%%', - 'reporter-testend-is run-bar%%', - 'reporter-end-end%%', - ]); -}); + test('should work with custom reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + constructor(options) { + this.options = options; + } + onBegin(config, suite) { + console.log('\\n%%reporter-begin-' + this.options.begin + '%%'); + console.log('\\n%%version-' + config.version); + } + onTestBegin(test) { + const projectName = test.titlePath()[1]; + console.log('\\n%%reporter-testbegin-' + test.title + '-' + projectName + '%%'); + const suite = test.parent; + if (!suite.tests.includes(test)) + console.log('\\n%%error-inconsistent-parent'); + if (test.parent.project().name !== projectName) + console.log('\\n%%error-inconsistent-project-name'); + } + onStdOut() { + console.log('\\n%%reporter-stdout%%'); + } + onStdErr() { + console.log('\\n%%reporter-stderr%%'); + } + onTestEnd(test, result) { + console.log('\\n%%reporter-testend-' + test.title + '-' + test.titlePath()[1] + '%%'); + if (!result.startTime) + console.log('\\n%%error-no-start-time'); + } + onTimeout() { + console.log('\\n%%reporter-timeout%%'); + } + onError() { + console.log('\\n%%reporter-error%%'); + } + async onEnd() { + await new Promise(f => setTimeout(f, 500)); + console.log('\\n%%reporter-end-' + this.options.end + '%%'); + } + } + export default Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: [ + [ './reporter.ts', { begin: 'begin', end: 'end' } ] + ], + projects: [ + { name: 'foo', repeatEach: 2 }, + { name: 'bar' }, + ], + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('not run', async ({}) => { + console.log('log'); + console.error('error'); + }); + test.only('is run', async ({}) => { + console.log('log'); + console.error('error'); + }); + ` + }, { reporter: '', workers: 1 }); -test('should work without a file extension', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({}) => { - }); - ` - }, { reporter: '', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'reporter-begin-begin%%', + 'version-' + require('../../packages/playwright-test/package.json').version, + 'reporter-testbegin-is run-foo%%', + 'reporter-stdout%%', + 'reporter-stderr%%', + 'reporter-testend-is run-foo%%', + 'reporter-testbegin-is run-foo%%', + 'reporter-stdout%%', + 'reporter-stderr%%', + 'reporter-testend-is run-foo%%', + 'reporter-testbegin-is run-bar%%', + 'reporter-stdout%%', + 'reporter-stderr%%', + 'reporter-testend-is run-bar%%', + 'reporter-end-end%%', + ]); + }); - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - 'begin', - 'end', - 'exit', - ]); -}); + test('should work without a file extension', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => { + }); + ` + }, { reporter: '', workers: 1 }); -test('should report onEnd after global teardown', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'globalSetup.ts': ` - module.exports = () => { - return () => console.log('\\n%%global teardown'); - }; - `, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - globalSetup: './globalSetup', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({}) => { - }); - ` - }, { reporter: '', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'begin', + 'end', + 'exit', + ]); + }); - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - 'begin', - 'global teardown', - 'end', - 'exit', - ]); -}); + test('should report onEnd after global teardown', async ({ runInlineTest }) => { + test.skip(useIntermediateMergeReport); + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'globalSetup.ts': ` + module.exports = () => { + return () => console.log('\\n%%global teardown'); + }; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + globalSetup: './globalSetup', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => { + }); + ` + }, { reporter: '', workers: 1 }); -test('should load reporter from node_modules', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'node_modules/my-reporter/index.js': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: 'my-reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({}) => { - }); - ` - }, { reporter: '', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'begin', + 'global teardown', + 'end', + 'exit', + ]); + }); - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - 'begin', - 'end', - 'exit', - ]); -}); + test('should load reporter from node_modules', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'node_modules/my-reporter/index.js': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: 'my-reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => { + }); + ` + }, { reporter: '', workers: 1 }); -test('should report expect steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('fail', async ({}) => { - expect(true).toBeTruthy(); - expect(false).toBeTruthy(); - }); - test('pass', async ({}) => { - expect(false).not.toBeTruthy(); - }); - test('async', async ({ page }) => { - await expect(page).not.toHaveTitle('False'); - }); - ` - }, { reporter: '', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'begin', + 'end', + 'exit', + ]); + }); - expect(result.exitCode).toBe(1); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, - `end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - ]); -}); + test('should report expect steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepsReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({}) => { + expect(true).toBeTruthy(); + expect(false).toBeTruthy(); + }); + test('pass', async ({}) => { + expect(false).not.toBeTruthy(); + }); + test('async', async ({ page }) => { + await expect(page).not.toHaveTitle('False'); + }); + ` + }, { reporter: '', workers: 1 }); -test('should report api steps', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({ page, request }) => { - await Promise.all([ - page.waitForNavigation(), - page.goto('data:text/html,'), - ]); - await page.click('button'); - await page.getByRole('button').click(); - await page.request.get('http://localhost2').catch(() => {}); - await request.get('http://localhost2').catch(() => {}); - }); + expect(result.exitCode).toBe(1); + expect(result.outputLines).toEqual([ + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `begin {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\"}`, + `end {\"title\":\"expect.toBeTruthy\",\"category\":\"expect\",\"error\":{\"message\":\"\\u001b[2mexpect(\\u001b[22m\\u001b[31mreceived\\u001b[39m\\u001b[2m).\\u001b[22mtoBeTruthy\\u001b[2m()\\u001b[22m\\n\\nReceived: \\u001b[31mfalse\\u001b[39m\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, + `end {\"title\":\"expect.not.toBeTruthy\",\"category\":\"expect\"}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, + `begin {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, + `end {\"title\":\"expect.not.toHaveTitle\",\"category\":\"expect\"}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, + ]); + }); - test.describe('suite', () => { - let myPage; - test.beforeAll(async ({ browser }) => { - myPage = await browser.newPage(); - await myPage.setContent(''); - }); + test('should report api steps', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepsReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page, request }) => { + await Promise.all([ + page.waitForNavigation(), + page.goto('data:text/html,'), + ]); + await page.click('button'); + await page.getByRole('button').click(); + await page.request.get('http://localhost2').catch(() => {}); + await request.get('http://localhost2').catch(() => {}); + }); - test('pass1', async () => { - await myPage.click('button'); - }); - test('pass2', async () => { - await myPage.click('button'); - }); + test.describe('suite', () => { + let myPage; + test.beforeAll(async ({ browser }) => { + myPage = await browser.newPage(); + await myPage.setContent(''); + }); - test.afterAll(async () => { - await myPage.close(); - }); - }); - ` - }, { reporter: '', workers: 1 }); + test('pass1', async () => { + await myPage.click('button'); + }); + test('pass2', async () => { + await myPage.click('button'); + }); - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, - `begin {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\",\"steps\":[{\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, - `end {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, - `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, - `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, - `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, - `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, - `begin {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, - `end {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"beforeAll hook\",\"category\":\"hook\"}`, - `begin {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"afterAll hook\",\"category\":\"hook\"}`, - `begin {\"title\":\"page.close\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.close\",\"category\":\"pw:api\"}`, - `end {\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}]}`, - ]); -}); + test.afterAll(async () => { + await myPage.close(); + }); + }); + ` + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, + `begin {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequest.newContext\",\"category\":\"pw:api\"}]}]}`, + `begin {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.waitForNavigation\",\"category\":\"pw:api\",\"steps\":[{\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"page.goto(data:text/html,)\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, + `end {\"title\":\"locator.getByRole('button').click\",\"category\":\"pw:api\"}`, + `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, + `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, + `begin {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api"}`, + `end {"title":"apiRequestContext.get(http://localhost2)","category":"pw:api","error":{"message":"","stack":"","location":"","snippet":""}}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: request\",\"category\":\"fixture\"}`, + `begin {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, + `end {\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: request\",\"category\":\"fixture\",\"steps\":[{\"title\":\"apiRequestContext.dispose\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"beforeAll hook\",\"category\":\"hook\"}`, + `begin {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browser.newPage\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `end {\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"beforeAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"browser.newPage\",\"category\":\"pw:api\"},{\"title\":\"page.setContent\",\"category\":\"pw:api\"}]}]}`, + `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.click(button)\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"afterAll hook\",\"category\":\"hook\"}`, + `begin {\"title\":\"page.close\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.close\",\"category\":\"pw:api\"}`, + `end {\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"afterAll hook\",\"category\":\"hook\",\"steps\":[{\"title\":\"page.close\",\"category\":\"pw:api\"}]}]}`, + ]); + }); -test('should report api step failure', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('fail', async ({ page }) => { - await page.setContent(''); - await page.click('input', { timeout: 1 }); - }); - ` - }, { reporter: '', workers: 1 }); + test('should report api step failure', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepsReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + await page.setContent(''); + await page.click('input', { timeout: 1 }); + }); + ` + }, { reporter: '', workers: 1 }); - expect(result.exitCode).toBe(1); - expect(result.outputLines).toEqual([ - `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, - `begin {\"title\":\"page.click(input)\",\"category\":\"pw:api\"}`, - `end {\"title\":\"page.click(input)\",\"category\":\"pw:api\",\"error\":{\"message\":\"page.click: Timeout 1ms exceeded.\\n=========================== logs ===========================\\nwaiting for locator('input')\\n============================================================\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, - `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"},{\"title\":\"fixture: browser\",\"category\":\"fixture\"}]}`, - ]); -}); + expect(result.exitCode).toBe(1); + expect(result.outputLines).toEqual([ + `begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, + `begin {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.setContent\",\"category\":\"pw:api\"}`, + `begin {\"title\":\"page.click(input)\",\"category\":\"pw:api\"}`, + `end {\"title\":\"page.click(input)\",\"category\":\"pw:api\",\"error\":{\"message\":\"page.click: Timeout 1ms exceeded.\\n=========================== logs ===========================\\nwaiting for locator('input')\\n============================================================\",\"stack\":\"\",\"location\":\"\",\"snippet\":\"\"}}`, + `begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"},{\"title\":\"fixture: browser\",\"category\":\"fixture\"}]}`, + ]); + }); -test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test as base, expect } from '@playwright/test'; - const test = base.extend({ - page: async ({ page }, use) => { - await use(page); - // Timeout in fixture teardown that will resolve on browser.close. - await page.waitForNavigation(); - }, - }); - test('pass', async ({ page }) => { - // Timeout in the test. - await page.click('foo'); - }); - ` - }, { workers: 1, timeout: 1000, reporter: 'dot', retries: 1 }); + test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test as base, expect } from '@playwright/test'; + const test = base.extend({ + page: async ({ page }, use) => { + await use(page); + // Timeout in fixture teardown that will resolve on browser.close. + await page.waitForNavigation(); + }, + }); + test('pass', async ({ page }) => { + // Timeout in the test. + await page.click('foo'); + }); + ` + }, { workers: 1, timeout: 1000, reporter: 'dot', retries: 1 }); - expect(result.exitCode).toBe(1); - expect(result.failed).toBe(1); - expect(result.output).not.toContain('Internal error'); -}); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).not.toContain('Internal error'); + }); -test('should show nice stacks for locators', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepsReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('pass', async ({ page }) => { - await page.setContent(''); - const locator = page.locator('button'); - await locator.evaluate(e => e.innerText); - }); - ` - }, { reporter: '', workers: 1 }); + test('should show nice stacks for locators', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepsReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + const locator = page.locator('button'); + await locator.evaluate(e => e.innerText); + }); + ` + }, { reporter: '', workers: 1 }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(0); - expect(result.output).not.toContain('Internal error'); - expect(result.outputLines).toEqual([ - `begin {"title":"Before Hooks","category":"hook"}`, - `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {"title":"browserContext.newPage","category":"pw:api"}`, - `end {"title":"browserContext.newPage","category":"pw:api"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, - `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, - `begin {"title":"page.setContent","category":"pw:api"}`, - `end {"title":"page.setContent","category":"pw:api"}`, - `begin {"title":"locator.evaluate(button)","category":"pw:api"}`, - `end {"title":"locator.evaluate(button)","category":"pw:api"}`, - `begin {"title":"After Hooks","category":"hook"}`, - `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, - `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, - `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, - ]); -}); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(0); + expect(result.output).not.toContain('Internal error'); + expect(result.outputLines).toEqual([ + `begin {"title":"Before Hooks","category":"hook"}`, + `begin {\"title\":\"fixture: browser\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browserType.launch\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `begin {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"browser.newContext\",\"category\":\"pw:api\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {"title":"browserContext.newPage","category":"pw:api"}`, + `end {"title":"browserContext.newPage","category":"pw:api"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}`, + `end {\"title\":\"Before Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: browser\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserType.launch\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: context\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browser.newContext\",\"category\":\"pw:api\"}]},{\"title\":\"fixture: page\",\"category\":\"fixture\",\"steps\":[{\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}]}]}`, + `begin {"title":"page.setContent","category":"pw:api"}`, + `end {"title":"page.setContent","category":"pw:api"}`, + `begin {"title":"locator.evaluate(button)","category":"pw:api"}`, + `end {"title":"locator.evaluate(button)","category":"pw:api"}`, + `begin {"title":"After Hooks","category":"hook"}`, + `begin {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: page\",\"category\":\"fixture\"}`, + `begin {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"fixture: context\",\"category\":\"fixture\"}`, + `end {\"title\":\"After Hooks\",\"category\":\"hook\",\"steps\":[{\"title\":\"fixture: page\",\"category\":\"fixture\"},{\"title\":\"fixture: context\",\"category\":\"fixture\"}]}`, + ]); + }); -test('should report forbid-only error to reporter', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test.only('pass', () => {}); - ` - }, { 'reporter': '', 'forbid-only': true }); + test('should report forbid-only error to reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test.only('pass', () => {}); + ` + }, { 'reporter': '', 'forbid-only': true }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain(`%%got error: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: \"a.test.ts pass\"`); -}); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`%%got error: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: \"a.test.ts pass\"`); + }); -test('should report no-tests error to reporter', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - ` - }, { 'reporter': '' }); + test('should report no-tests error to reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + ` + }, { 'reporter': '' }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain(`%%got error: No tests found`); -}); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`%%got error: No tests found`); + }); -test('should report require error to reporter', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.spec.js': ` - throw new Error('Oh my!'); - `, - }, { 'reporter': '' }); + test('should report require error to reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.spec.js': ` + throw new Error('Oh my!'); + `, + }, { 'reporter': '' }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain(`%%got error: Oh my!`); -}); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`%%got error: Oh my!`); + }); -test('should report global setup error to reporter', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': smallReporterJS, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - globalSetup: './globalSetup', - }; - `, - 'globalSetup.ts': ` - module.exports = () => { - throw new Error('Oh my!'); - }; - `, - 'a.spec.js': ` - const { test, expect } = require('@playwright/test'); - test('test', () => {}); - `, - }, { 'reporter': '' }); + test('should report global setup error to reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': smallReporterJS, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + globalSetup: './globalSetup', + }; + `, + 'globalSetup.ts': ` + module.exports = () => { + throw new Error('Oh my!'); + }; + `, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('test', () => {}); + `, + }, { 'reporter': '' }); - expect(result.exitCode).toBe(1); - expect(result.output).toContain(`%%got error: Oh my!`); -}); + expect(result.exitCode).toBe(1); + expect(result.output).toContain(`%%got error: Oh my!`); + }); -test('should report correct tests/suites when using grep', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.spec.js': ` - import { test, expect } from '@playwright/test'; + test('should report correct tests/suites when using grep', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.js': ` + import { test, expect } from '@playwright/test'; - test.describe('@foo', () => { - test('test1', async ({ }) => { - console.log('%%test1'); - }); - test('test2', async ({ }) => { - console.log('%%test2'); - }); - }); + test.describe('@foo', () => { + test('test1', async ({ }) => { + console.log('%%test1'); + }); + test('test2', async ({ }) => { + console.log('%%test2'); + }); + }); - test('test3', async ({ }) => { - console.log('%%test3'); - }); - `, - }, { 'grep': '@foo' }); + test('test3', async ({ }) => { + console.log('%%test3'); + }); + `, + }, { 'grep': '@foo' }); - expect(result.exitCode).toBe(0); - expect(result.output).toContain('%%test1'); - expect(result.output).toContain('%%test2'); - expect(result.output).not.toContain('%%test3'); - const fileSuite = result.report.suites[0]; - expect(fileSuite.suites!.length).toBe(1); - expect(fileSuite.suites![0].specs.length).toBe(2); - expect(fileSuite.specs.length).toBe(0); -}); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('%%test1'); + expect(result.output).toContain('%%test2'); + expect(result.output).not.toContain('%%test3'); + const fileSuite = result.report.suites[0]; + expect(fileSuite.suites!.length).toBe(1); + expect(fileSuite.suites![0].specs.length).toBe(2); + expect(fileSuite.specs.length).toBe(0); + }); -test('should use sourceMap-based file suite names', async ({ runInlineTest }) => { - test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11028' }); - const result = await runInlineTest({ - 'reporter.js': ` - class Reporter { - onBegin(config, suite) { - console.log(suite.suites[0].suites[0].location.file); - } - } - module.exports = Reporter; - `, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.spec.js': + test('should use sourceMap-based file suite names', async ({ runInlineTest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/11028' }); + const result = await runInlineTest({ + 'reporter.js': ` + class Reporter { + onBegin(config, suite) { + console.log(suite.suites[0].suites[0].location.file); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.spec.js': `var __create = Object.create; var __defProp = Object.defineProperty; var __getOwnPropDesc = Object.getOwnPropertyDescriptor; @@ -674,117 +679,121 @@ var import_test = __toModule(require("@playwright/test")); (0, import_test.test)("pass", async () => { }); //# sourceMappingURL=data:application/json;base64,ewogICJ2ZXJzaW9uIjogMywKICAic291cmNlcyI6IFsiLi4vc3JjL2Euc3BlYy50cyJdLAogICJzb3VyY2VzQ29udGVudCI6IFsiaW1wb3J0IHsgdGVzdCB9IGZyb20gXCJAcGxheXdyaWdodC90ZXN0XCI7XG5cbnRlc3QoJ3Bhc3MnLCBhc3luYyAoKSA9PiB7fSk7Il0sCiAgIm1hcHBpbmdzIjogIjs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsa0JBQXFCO0FBRXJCLHNCQUFLLFFBQVEsWUFBWTtBQUFBOyIsCiAgIm5hbWVzIjogW10KfQo=`, - }, { 'reporter': '' }); + }, { 'reporter': '' }); - expect(result.exitCode).toBe(0); - expect(result.output).toContain('a.spec.ts'); -}); + expect(result.exitCode).toBe(0); + expect(result.output).toContain('a.spec.ts'); + }); -test('parallelIndex is presented in onTestEnd', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': ` - class Reporter { - onTestEnd(test, result) { - console.log('parallelIndex: ' + result.parallelIndex) - } - } - module.exports = Reporter;`, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.spec.js': ` - const { test, expect } = require('@playwright/test'); - test('test', () => {}); - `, - }, { 'reporter': '', 'workers': 1 }); + test('parallelIndex is presented in onTestEnd', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + onTestEnd(test, result) { + console.log('parallelIndex: ' + result.parallelIndex) + } + } + module.exports = Reporter;`, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('test', () => {}); + `, + }, { 'reporter': '', 'workers': 1 }); - expect(result.output).toContain('parallelIndex: 0'); -}); + expect(result.output).toContain('parallelIndex: 0'); + }); -test('test and step error should have code snippet', async ({ runInlineTest }) => { - const testErrorFile = test.info().outputPath('testError.txt'); - const stepErrorFile = test.info().outputPath('stepError.txt'); - const result = await runInlineTest({ - 'reporter.ts': ` - import fs from 'fs'; - class Reporter { - onStepEnd(test, result, step) { - console.log('\\n%%onStepEnd: ' + step.error?.snippet?.length); - fs.writeFileSync('${stepErrorFile.replace(/\\/g, '\\\\')}', step.error?.snippet); - } - onTestEnd(test, result) { - console.log('\\n%%onTestEnd: ' + result.error?.snippet?.length); - fs.writeFileSync('${testErrorFile.replace(/\\/g, '\\\\')}', result.error?.snippet); - } - onError(error) { - console.log('\\n%%onError: ' + error.snippet?.length); - } - } - module.exports = Reporter;`, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.spec.js': ` - const { test, expect } = require('@playwright/test'); - test('test', async () => { - await test.step('step', async () => { - expect(1).toBe(2); - }); - }); - `, - }, { 'reporter': '', 'workers': 1 }); + test('test and step error should have code snippet', async ({ runInlineTest }) => { + const testErrorFile = test.info().outputPath('testError.txt'); + const stepErrorFile = test.info().outputPath('stepError.txt'); + const result = await runInlineTest({ + 'reporter.ts': ` + import fs from 'fs'; + class Reporter { + onStepEnd(test, result, step) { + console.log('\\n%%onStepEnd: ' + step.error?.snippet?.length); + if (step.error?.snippet) + fs.writeFileSync('${stepErrorFile.replace(/\\/g, '\\\\')}', step.error?.snippet); + } + onTestEnd(test, result) { + console.log('\\n%%onTestEnd: ' + result.error?.snippet?.length); + if (result.error) + fs.writeFileSync('${testErrorFile.replace(/\\/g, '\\\\')}', result.error?.snippet); + } + onError(error) { + console.log('\\n%%onError: ' + error.snippet?.length); + } + } + module.exports = Reporter;`, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + test('test', async () => { + await test.step('step', async () => { + expect(1).toBe(2); + }); + }); + `, + }, { 'reporter': '', 'workers': 1 }); - expect(result.output).toContain('onTestEnd: 522'); - expect(result.output).toContain('onStepEnd: 522'); - expect(stripAnsi(fs.readFileSync(testErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => { - 4 | await test.step('step', async () => { -> 5 | expect(1).toBe(2); - | ^ - 6 | }); - 7 | }); - 8 | `); - expect(stripAnsi(fs.readFileSync(stepErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => { - 4 | await test.step('step', async () => { -> 5 | expect(1).toBe(2); - | ^ - 6 | }); - 7 | }); - 8 | `); -}); + expect(result.output).toContain('onTestEnd: 550'); + expect(result.output).toContain('onStepEnd: 550'); + expect(stripAnsi(fs.readFileSync(testErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => { + 4 | await test.step('step', async () => { +> 5 | expect(1).toBe(2); + | ^ + 6 | }); + 7 | }); + 8 | `); + expect(stripAnsi(fs.readFileSync(stepErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => { + 4 | await test.step('step', async () => { +> 5 | expect(1).toBe(2); + | ^ + 6 | }); + 7 | }); + 8 | `); + }); -test('onError should have code snippet', async ({ runInlineTest }) => { - const errorFile = test.info().outputPath('error.txt'); - const result = await runInlineTest({ - 'reporter.ts': ` - import fs from 'fs'; - class Reporter { - onError(error) { - console.log('\\n%%onError: ' + error.snippet?.length); - fs.writeFileSync('${errorFile.replace(/\\/g, '\\\\')}', error.snippet); - } - } - module.exports = Reporter;`, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter', - }; - `, - 'a.spec.js': ` - const { test, expect } = require('@playwright/test'); - throw new Error('test'); - `, - }, { 'reporter': '', 'workers': 1 }); + test('onError should have code snippet', async ({ runInlineTest }) => { + const errorFile = test.info().outputPath('error.txt'); + const result = await runInlineTest({ + 'reporter.ts': ` + import fs from 'fs'; + class Reporter { + onError(error) { + console.log('\\n%%onError: ' + error.snippet?.length); + fs.writeFileSync('${errorFile.replace(/\\/g, '\\\\')}', error.snippet); + } + } + module.exports = Reporter;`, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter', + }; + `, + 'a.spec.js': ` + const { test, expect } = require('@playwright/test'); + throw new Error('test'); + `, + }, { 'reporter': '', 'workers': 1 }); - expect(result.output).toContain('onError: 396'); - expect(stripAnsi(fs.readFileSync(errorFile, 'utf8'))).toBe(` at a.spec.js:3 + expect(result.output).toContain('onError: 412'); + expect(stripAnsi(fs.readFileSync(errorFile, 'utf8'))).toBe(` at a.spec.js:3 1 | - 2 | const { test, expect } = require('@playwright/test'); -> 3 | throw new Error('test'); - | ^ - 4 | `); -}); + 2 | const { test, expect } = require('@playwright/test'); +> 3 | throw new Error('test'); + | ^ + 4 | `); + }); + }); +} \ No newline at end of file