mirrors
/
playwright
mirror of https://mirror.osredm.com/root/playwright.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: run more reporter tests through blob report, some fixes (#23765)

This commit is contained in:
Yury Semikhatsky 2023-06-16 21:30:55 -07:00 committed by GitHub
parent 09b1e3ffa9
commit f6d86c20f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1953 additions and 1908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/playwright-test/src/isomorphic/teleReceiver.ts
View File

@ -232,6 +232,7 @@ export class TeleReporterReceiver {
result.status = payload.status; result.status = payload.status;
result.statusEx = payload.status; result.statusEx = payload.status;
result.errors = payload.errors; result.errors = payload.errors;
result.error = result.errors?.[0];
result.attachments = this._parseAttachments(payload.attachments); result.attachments = this._parseAttachments(payload.attachments);
this._reporter.onTestEnd?.(test, result); this._reporter.onTestEnd?.(test, result);
} }
@ -242,7 +243,10 @@ export class TeleReporterReceiver {
const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined; const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined;
const step: TestStep = { const step: TestStep = {
titlePath: () => [], titlePath: () => {
const parentPath = parentStep?.titlePath() || [];
return [...parentPath, payload.title];
},
title: payload.title, title: payload.title,
category: payload.category, category: payload.category,
location: this._absoluteLocation(payload.location), location: this._absoluteLocation(payload.location),

2
packages/playwright-test/src/reporters/blob.ts
View File

@ -92,7 +92,7 @@ export class BlobReporter extends TeleReporterEmitter {
override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] { override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] {
return super._serializeAttachments(attachments).map(attachment => { return super._serializeAttachments(attachments).map(attachment => {
if (!attachment.path || !fs.statSync(attachment.path).isFile()) if (!attachment.path || !fs.statSync(attachment.path, { throwIfNoEntry: false })?.isFile())
return attachment; return attachment;
// Add run guid to avoid clashes between shards. // Add run guid to avoid clashes between shards.
const sha1 = calculateSha1(attachment.path + this._salt); const sha1 = calculateSha1(attachment.path + this._salt);

9
tests/playwright-test/playwright-test-fixtures.ts
View File

@ -88,7 +88,7 @@ export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-tes
const configFile = (baseDir: string, files: Files): string | undefined => { const configFile = (baseDir: string, files: Files): string | undefined => {
for (const [name, content] of Object.entries(files)) { for (const [name, content] of Object.entries(files)) {
if (name.includes('playwright.config')) { if (name.includes('playwright.config')) {
if (content.includes('reporter:')) if (content.includes('reporter:') || content.includes('reportSlowTests:'))
return path.resolve(baseDir, name); return path.resolve(baseDir, name);
} }
} }
@ -349,12 +349,7 @@ export const test = base
const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir; const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir;
const testProcess = childProcess({ const testProcess = childProcess({
command, command,
env: cleanEnv({ env: cleanEnv(env),
PW_TEST_DEBUG_REPORTERS: '1',
PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1',
PWTEST_TTY_WIDTH: '80',
...env
}),
cwd, cwd,
}); });
const { exitCode } = await testProcess.exited; const { exitCode } = await testProcess.exited;

676
tests/playwright-test/reporter-base.spec.ts
View File

@ -17,346 +17,352 @@
import { test, expect } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
import * as path from 'path'; import * as path from 'path';
test('handle long test names', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const title = 'title'.repeat(30); test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
const result = await runInlineTest({ test.use({ useIntermediateMergeReport });
'a.test.js': `
import { test, expect } from '@playwright/test';
test('${title}', async ({}) => {
expect(1).toBe(0);
});
`,
});
expect(result.output).toContain('expect(1).toBe');
expect(result.exitCode).toBe(1);
});
test('print the error name', async ({ runInlineTest }) => { test('handle long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({ const title = 'title'.repeat(30);
'a.spec.ts': ` const result = await runInlineTest({
import { test, expect } from '@playwright/test'; 'a.test.js': `
test('foobar', async ({}) => { import { test, expect } from '@playwright/test';
const error = new Error('my-message'); test('${title}', async ({}) => {
error.name = 'FooBarError'; expect(1).toBe(0);
throw error; });
`,
});
expect(result.output).toContain('expect(1).toBe');
expect(result.exitCode).toBe(1);
}); });
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
});
test('print should print the error name without a message', async ({ runInlineTest }) => { test('print the error name', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.spec.ts': ` 'a.spec.ts': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('foobar', async ({}) => { test('foobar', async ({}) => {
const error = new Error(); const error = new Error('my-message');
error.name = 'FooBarError'; error.name = 'FooBarError';
throw error; throw error;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
}); });
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError');
});
test('should print an error in a codeframe', async ({ runInlineTest }) => { test('print should print the error name without a message', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.spec.ts': ` 'a.spec.ts': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('foobar', async ({}) => { test('foobar', async ({}) => {
const error = new Error('my-message'); const error = new Error();
error.name = 'FooBarError'; error.name = 'FooBarError';
throw error; throw error;
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
expect(result.output).not.toContain('at a.spec.ts:5');
expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`);
expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`);
expect(result.output).toContain(`> 4 | const error = new Error('my-message');`);
});
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'node_modules/utils/utils.js': `
function assert(value) {
if (!value)
throw new Error('Assertion error');
}
module.exports = { assert };
`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
const { assert } = require('utils/utils.js');
test('fail', async ({}) => {
assert(false);
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('Error: Assertion error');
expect(output).toContain('a.spec.ts:4:11 › fail');
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
expect(output).toContain(`> 5 | assert(false);`);
expect(output).toContain(` | ^`);
expect(output).toContain(`utils.js:4`);
expect(output).toContain(`a.spec.ts:5:9`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function ohMy() {
throw new Error('oh my');
}
`,
'a.spec.ts': `
import { ohMy } from './helper';
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
ohMy();
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: oh my');
expect(result.output).toContain(` 2 | export function ohMy() {`);
expect(result.output).toContain(` > 3 | throw new Error('oh my');`);
expect(result.output).toContain(` | ^`);
});
test('should print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'foo' },
{ name: 'bar' },
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: { max: 0, threshold: 2400 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 2500));
});
`,
'dir/b.test.js': `
import { test, expect } from '@playwright/test';
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 1));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(8);
expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`);
expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`);
});
test('should not print slow parallel tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 500 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test.describe.parallel('suite', () => {
test('inner slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('inner fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
}); });
`
}); });
`, expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError');
});
test('should print an error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const error = new Error('my-message');
error.name = 'FooBarError';
throw error;
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('FooBarError: my-message');
expect(result.output).not.toContain('at a.spec.ts:5');
expect(result.output).toContain(` 2 | import { test, expect } from '@playwright/test';`);
expect(result.output).toContain(` 3 | test('foobar', async ({}) => {`);
expect(result.output).toContain(`> 4 | const error = new Error('my-message');`);
});
test('should filter out node_modules error in a codeframe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'node_modules/utils/utils.js': `
function assert(value) {
if (!value)
throw new Error('Assertion error');
}
module.exports = { assert };
`,
'a.spec.ts': `
import { test, expect } from '@playwright/test';
const { assert } = require('utils/utils.js');
test('fail', async ({}) => {
assert(false);
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('Error: Assertion error');
expect(output).toContain('a.spec.ts:4:15 › fail');
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
expect(output).toContain(`> 5 | assert(false);`);
expect(output).toContain(` | ^`);
expect(output).toContain(`utils.js:4`);
expect(output).toContain(`a.spec.ts:5:13`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
const result = await runInlineTest({
'helper.ts': `
export function ohMy() {
throw new Error('oh my');
}
`,
'a.spec.ts': `
import { ohMy } from './helper';
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
ohMy();
});
`
}, {}, {
FORCE_COLOR: '0',
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Error: oh my');
expect(result.output).toContain(` 2 | export function ohMy() {`);
expect(result.output).toContain(` > 3 | throw new Error('oh my');`);
expect(result.output).toContain(` | ^`);
});
test('should print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'foo' },
{ name: 'bar' },
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: { max: 0, threshold: 2400 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 2500));
});
`,
'dir/b.test.js': `
import { test, expect } from '@playwright/test';
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 1));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(8);
expect(result.output).toContain(`Slow test file: [foo] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [bar] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [baz] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Slow test file: [qux] › dir${path.sep}a.test.js (`);
expect(result.output).toContain(`Consider splitting slow test files to speed up parallel execution`);
expect(result.output).not.toContain(`Slow test file: [foo] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [bar] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [baz] › dir${path.sep}b.test.js (`);
expect(result.output).not.toContain(`Slow test file: [qux] › dir${path.sep}b.test.js (`);
});
test('should not print slow parallel tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
reportSlowTests: { max: 0, threshold: 500 },
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test.describe.parallel('suite', () => {
test('inner slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('inner fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(2);
expect(result.output).not.toContain('Slow test file');
});
test('should not print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: null,
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.output).not.toContain('Slow test');
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should print flaky timeouts', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
if (!testInfo.retry)
await new Promise(f => setTimeout(f, 2000));
});
`
}, { retries: '1', reporter: 'list', timeout: '1000' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
test('should print stack-less errors', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const e = new Error('Hello');
delete e.stack;
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Hello');
});
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async function myTest({}) {
const e = new Error('Hello');
// Force stack to contain "Hello".
// Otherwise it is computed lazy and will get 'foo bar' instead.
e.stack;
e.message = 'foo bar';
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('foo bar');
expect(output).toContain('function myTest');
});
test('should print "no tests found" error', async ({ runInlineTest }) => {
const result = await runInlineTest({ });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('No tests found');
});
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.attachments.push({
name: 'foo.txt',
body: undefined,
contentType: 'text/plain'
});
expect(1).toBe(2);
});
`,
});
expect(result.output).not.toContain('Error in reporter');
expect(result.failed).toBe(1);
expect(result.exitCode).toBe(1);
});
test('should report fatal errors at the end', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
'b.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
}, { reporter: 'list' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.output).toContain('2 errors were not a part of any test, see above for details');
});
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('should work', () => {});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
});
}); });
expect(result.exitCode).toBe(0); }
expect(result.passed).toBe(2);
expect(result.output).not.toContain('Slow test file');
});
test('should not print slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = {
projects: [
{ name: 'baz' },
{ name: 'qux' },
],
reportSlowTests: null,
};
`,
'dir/a.test.js': `
import { test, expect } from '@playwright/test';
test('slow test', async ({}) => {
await new Promise(f => setTimeout(f, 1000));
});
test('fast test', async ({}) => {
await new Promise(f => setTimeout(f, 100));
});
`,
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
expect(result.output).not.toContain('Slow test');
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'list' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should print flaky timeouts', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
if (!testInfo.retry)
await new Promise(f => setTimeout(f, 2000));
});
`
}, { retries: '1', reporter: 'list', timeout: '1000' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('Test timeout of 1000ms exceeded.');
});
test('should print stack-less errors', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}) => {
const e = new Error('Hello');
delete e.stack;
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('Hello');
});
test('should print errors with inconsistent message/stack', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async function myTest({}) {
const e = new Error('Hello');
// Force stack to contain "Hello".
// Otherwise it is computed lazy and will get 'foo bar' instead.
e.stack;
e.message = 'foo bar';
throw e;
});
`
});
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('foo bar');
expect(output).toContain('function myTest');
});
test('should print "no tests found" error', async ({ runInlineTest }) => {
const result = await runInlineTest({ });
expect(result.exitCode).toBe(1);
expect(result.output).toContain('No tests found');
});
test('should not crash on undefined body with manual attachments', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => {
testInfo.attachments.push({
name: 'foo.txt',
body: undefined,
contentType: 'text/plain'
});
expect(1).toBe(2);
});
`,
});
expect(result.output).not.toContain('Error in reporter');
expect(result.failed).toBe(1);
expect(result.exitCode).toBe(1);
});
test('should report fatal errors at the end', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
'b.spec.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
fixture: [async ({ }, use) => {
await use();
throw new Error('oh my!');
}, { scope: 'worker' }],
});
test('good', async ({ fixture }) => {
});
`,
}, { reporter: 'list' });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.output).toContain('2 errors were not a part of any test, see above for details');
});
test('should contain at most 1 decimal for humanized timing', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('should work', () => {});
`
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
});

2
tests/playwright-test/reporter-blob.spec.ts
View File

@ -361,7 +361,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'list'] }); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
const text = stripAnsi(output); const text = stripAnsi(output);

176
tests/playwright-test/reporter-dot.spec.ts
View File

@ -17,93 +17,99 @@
import colors from 'colors/safe'; import colors from 'colors/safe';
import { test, expect } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
test('render expected', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const result = await runInlineTest({ test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
'a.test.js': ` test.use({ useIntermediateMergeReport });
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(0);
});
test('render unexpected', async ({ runInlineTest }) => { test('render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('one', async ({}) => { test('one', async ({}) => {
expect(1).toBe(0); expect(1).toBe(1);
}); });
`, `,
}, { reporter: 'dot' }); }, { reporter: 'dot' });
expect(result.rawOutput).toContain(colors.red('F')); expect(result.rawOutput).toContain(colors.green('·'));
expect(result.exitCode).toBe(1); expect(result.exitCode).toBe(0);
}); });
test('render unexpected after retry', async ({ runInlineTest }) => { test('render unexpected', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('one', async ({}) => { test('one', async ({}) => {
expect(1).toBe(0); expect(1).toBe(0);
}); });
`, `,
}, { retries: 3, reporter: 'dot' }); }, { reporter: 'dot' });
const text = result.output; expect(result.rawOutput).toContain(colors.red('F'));
expect(text).toContain('×××F'); expect(result.exitCode).toBe(1);
expect(result.rawOutput).toContain(colors.red('F')); });
expect(result.exitCode).toBe(1);
});
test('render flaky', async ({ runInlineTest }) => { test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('one', async ({}, testInfo) => { test('one', async ({}) => {
expect(testInfo.retry).toBe(3); expect(1).toBe(0);
}); });
`, `,
}, { retries: 3, reporter: 'dot' }); }, { retries: 3, reporter: 'dot' });
const text = result.output; const text = result.output;
expect(text).toContain('×××±'); expect(text).toContain('×××F');
expect(result.rawOutput).toContain(colors.yellow('±')); expect(result.rawOutput).toContain(colors.red('F'));
expect(text).toContain('1 flaky'); expect(result.exitCode).toBe(1);
expect(text).toContain('Retry #1'); });
expect(result.exitCode).toBe(0);
});
test('should work from config', async ({ runInlineTest }) => { test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'playwright.config.ts': ` 'a.test.js': `
module.exports = { reporter: 'dot' }; import { test, expect } from '@playwright/test';
`, test('one', async ({}, testInfo) => {
'a.test.js': ` expect(testInfo.retry).toBe(3);
import { test, expect } from '@playwright/test'; });
test('one', async ({}) => { `,
expect(1).toBe(1); }, { retries: 3, reporter: 'dot' });
}); const text = result.output;
`, expect(text).toContain('×××±');
}, { reporter: 'dot' }); expect(result.rawOutput).toContain(colors.yellow('±'));
expect(result.rawOutput).toContain(colors.green('·')); expect(text).toContain('1 flaky');
expect(result.exitCode).toBe(0); expect(text).toContain('Retry #1');
}); expect(result.exitCode).toBe(0);
});
test('render 243 tests in rows by 80', async ({ runInlineTest }) => { test('should work from config', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'playwright.config.ts': `
import { test, expect } from '@playwright/test'; module.exports = { reporter: 'dot' };
for (let i = 0; i < 243; i++) { `,
test('test' + i, () => {}); 'a.test.js': `
} import { test, expect } from '@playwright/test';
`, test('one', async ({}) => {
}, { reporter: 'dot' }); expect(1).toBe(1);
expect(result.exitCode).toBe(0); });
expect(result.rawOutput).toContain( `,
colors.green('·').repeat(80) + '\n' + }, { reporter: 'dot' });
colors.green('·').repeat(80) + '\n' + expect(result.rawOutput).toContain(colors.green('·'));
colors.green('·').repeat(80) + '\n' + expect(result.exitCode).toBe(0);
colors.green('·').repeat(3)); });
});
test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
for (let i = 0; i < 243; i++) {
test('test' + i, () => {});
}
`,
}, { reporter: 'dot' });
expect(result.exitCode).toBe(0);
expect(result.rawOutput).toContain(
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(3));
});
});
}

140
tests/playwright-test/reporter-github.spec.ts
View File

@ -23,73 +23,79 @@ function relativeFilePath(file: string): string {
return path.relative(process.cwd(), file); return path.relative(process.cwd(), file);
} }
test('print GitHub annotations for success', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const result = await runInlineTest({ test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
'a.test.js': ` test.use({ useIntermediateMergeReport });
import { test, expect } from '@playwright/test';
test('example1', async ({}) => {
expect(1 + 1).toBe(2);
});
`
}, { reporter: 'github' });
const text = result.output;
expect(text).not.toContain('::error');
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => { test('print GitHub annotations for success', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
const { test, expect } = require('@playwright/test'); import { test, expect } from '@playwright/test';
test('example', async ({}) => { test('example1', async ({}) => {
expect(1 + 1).toBe(3); expect(1 + 1).toBe(2);
}); });
` `
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() }); }, { reporter: 'github' });
const text = result.output; const text = result.output;
const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); expect(text).not.toContain('::error');
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`); expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`); expect(result.exitCode).toBe(0);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`); });
expect(result.exitCode).toBe(1);
});
test('print GitHub annotations for slow tests', async ({ runInlineTest }) => { test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({ const result = await runInlineTest({
'playwright.config.ts': ` 'a.test.js': `
module.exports = { const { test, expect } = require('@playwright/test');
reportSlowTests: { max: 0, threshold: 100 } test('example', async ({}) => {
}; expect(1 + 1).toBe(3);
`, });
'a.test.js': ` `
import { test, expect } from '@playwright/test'; }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
test('slow test', async ({}) => { const text = result.output;
await new Promise(f => setTimeout(f, 200)); const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
}); expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
` expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' }); expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
const text = result.output; expect(result.exitCode).toBe(1);
expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took'); });
expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for global error', async ({ runInlineTest }) => { test('print GitHub annotations for slow tests', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.ts': ` 'playwright.config.ts': `
import { test as base, expect } from '@playwright/test'; module.exports = {
const test = base.extend({ reportSlowTests: { max: 0, threshold: 100 }
w: [async ({}, use) => { };
await use(); `,
throw new Error('Oh my!'); 'a.test.js': `
}, { scope: 'worker' }], import { test, expect } from '@playwright/test';
}); test('slow test', async ({}) => {
test('passes but...', ({w}) => { await new Promise(f => setTimeout(f, 200));
}); });
`, `
}, { reporter: 'github' }); }, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: '' });
const text = result.output; const text = result.output;
expect(text).toContain('::error ::Error: Oh my!%0A%0A'); expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took');
expect(result.exitCode).toBe(1); expect(text).toContain('::notice title=🎭 Playwright Run Summary:: 1 passed');
}); expect(result.exitCode).toBe(0);
});
test('print GitHub annotations for global error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test as base, expect } from '@playwright/test';
const test = base.extend({
w: [async ({}, use) => {
await use();
throw new Error('Oh my!');
}, { scope: 'worker' }],
});
test('passes but...', ({w}) => {
});
`,
}, { reporter: 'github' });
const text = result.output;
expect(text).toContain('::error ::Error: Oh my!%0A%0A');
expect(result.exitCode).toBe(1);
});
});
}

844
tests/playwright-test/reporter-junit.spec.ts
View File

@ -19,424 +19,432 @@ import path from 'path';
import { test, expect } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
import fs from 'fs'; import fs from 'fs';
test('should render expected', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const result = await runInlineTest({ test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
'a.test.js': ` test.use({ useIntermediateMergeReport });
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
'b.test.js': `
import { test, expect } from '@playwright/test';
test('two', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: 'junit' });
const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('2');
expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
expect(result.exitCode).toBe(0);
});
test('should render unexpected', async ({ runInlineTest }) => { test('should render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('one', async ({}) => { test('one', async ({}) => {
expect(1).toBe(0); expect(1).toBe(1);
}); });
`, `,
}, { reporter: 'junit' }); 'b.test.js': `
const xml = parseXML(result.output); import { test, expect } from '@playwright/test';
expect(xml['testsuites']['$']['tests']).toBe('1'); test('two', async ({}) => {
expect(xml['testsuites']['$']['failures']).toBe('1'); expect(1).toBe(1);
const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0]; });
expect(failure['$']['message']).toContain('a.test.js'); `,
expect(failure['$']['message']).toContain('one'); }, { reporter: 'junit' });
expect(failure['$']['type']).toBe('FAILURE'); const xml = parseXML(result.output);
expect(failure['_']).toContain('expect(1).toBe(0)'); expect(xml['testsuites']['$']['tests']).toBe('2');
expect(result.exitCode).toBe(1); expect(xml['testsuites']['$']['failures']).toBe('0');
}); expect(xml['testsuites']['testsuite'].length).toBe(2);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
test('should render unexpected after retry', async ({ runInlineTest }) => { expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
const result = await runInlineTest({ expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
'a.test.js': ` expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
import { test, expect } from '@playwright/test'; expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('b.test.js');
test('one', async ({}) => { expect(result.exitCode).toBe(0);
expect(1).toBe(0); });
});
`, test('should render unexpected', async ({ runInlineTest }) => {
}, { retries: 3, reporter: 'junit' }); const result = await runInlineTest({
expect(result.output).toContain(`tests="1"`); 'a.test.js': `
expect(result.output).toContain(`failures="1"`); import { test, expect } from '@playwright/test';
expect(result.output).toContain(`<failure`); test('one', async ({}) => {
expect(result.output).toContain('Retry #1'); expect(1).toBe(0);
expect(result.output).toContain('Retry #2'); });
expect(result.output).toContain('Retry #3'); `,
expect(result.exitCode).toBe(1); }, { reporter: 'junit' });
}); const xml = parseXML(result.output);
expect(xml['testsuites']['$']['tests']).toBe('1');
test('should render flaky', async ({ runInlineTest }) => { expect(xml['testsuites']['$']['failures']).toBe('1');
const result = await runInlineTest({ const failure = xml['testsuites']['testsuite'][0]['testcase'][0]['failure'][0];
'a.test.js': ` expect(failure['$']['message']).toContain('a.test.js');
import { test, expect } from '@playwright/test'; expect(failure['$']['message']).toContain('one');
test('one', async ({}, testInfo) => { expect(failure['$']['type']).toBe('FAILURE');
expect(testInfo.retry).toBe(3); expect(failure['_']).toContain('expect(1).toBe(0)');
}); expect(result.exitCode).toBe(1);
`, });
}, { retries: 3, reporter: 'junit' });
expect(result.output).not.toContain('Retry #1'); test('should render unexpected after retry', async ({ runInlineTest }) => {
expect(result.exitCode).toBe(0); const result = await runInlineTest({
}); 'a.test.js': `
import { test, expect } from '@playwright/test';
test('should render stdout', async ({ runInlineTest }) => { test('one', async ({}) => {
const result = await runInlineTest({ expect(1).toBe(0);
'a.test.ts': ` });
import colors from 'colors/safe'; `,
import { test, expect } from '@playwright/test'; }, { retries: 3, reporter: 'junit' });
test('one', async ({}) => { expect(result.output).toContain(`tests="1"`);
console.log(colors.yellow('Hello world')); expect(result.output).toContain(`failures="1"`);
console.log('Hello again'); expect(result.output).toContain(`<failure`);
console.error('My error'); expect(result.output).toContain('Retry #1');
console.error('\\0'); // null control character expect(result.output).toContain('Retry #2');
test.expect("abc").toBe('abcd'); expect(result.output).toContain('Retry #3');
}); expect(result.exitCode).toBe(1);
`, });
}, { reporter: 'junit' });
const xml = parseXML(result.output); test('should render flaky', async ({ runInlineTest }) => {
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; const result = await runInlineTest({
expect(testcase['system-out'].length).toBe(1); 'a.test.js': `
expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again'); import { test, expect } from '@playwright/test';
expect(testcase['system-out'][0]).not.toContain('u00'); test('one', async ({}, testInfo) => {
expect(testcase['system-err'][0]).toContain('My error'); expect(testInfo.retry).toBe(3);
expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character });
expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`); `,
expect(result.exitCode).toBe(1); }, { retries: 3, reporter: 'junit' });
}); expect(result.output).not.toContain('Retry #1');
expect(result.exitCode).toBe(0);
test('should render stdout without ansi escapes', async ({ runInlineTest }) => { });
const result = await runInlineTest({
'playwright.config.ts': ` test('should render stdout', async ({ runInlineTest }) => {
module.exports = { const result = await runInlineTest({
reporter: [ ['junit', { stripANSIControlSequences: true }] ], 'a.test.ts': `
}; import colors from 'colors/safe';
`, import { test, expect } from '@playwright/test';
'a.test.ts': ` test('one', async ({}) => {
import colors from 'colors/safe'; console.log(colors.yellow('Hello world'));
import { test, expect } from '@playwright/test'; console.log('Hello again');
test('one', async ({}) => { console.error('My error');
console.log(colors.yellow('Hello world')); console.error('\\0'); // null control character
}); test.expect("abc").toBe('abcd');
`, });
}, { reporter: '' }); `,
const xml = parseXML(result.output); }, { reporter: 'junit' });
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; const xml = parseXML(result.output);
expect(testcase['system-out'].length).toBe(1); const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'][0].trim()).toBe('Hello world'); expect(testcase['system-out'].length).toBe(1);
expect(result.exitCode).toBe(0); expect(testcase['system-out'][0]).toContain('[33mHello world[39m\nHello again');
}); expect(testcase['system-out'][0]).not.toContain('u00');
expect(testcase['system-err'][0]).toContain('My error');
test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => { expect(testcase['system-err'][0]).not.toContain('\u0000'); // null control character
const result = await runInlineTest({ expect(testcase['failure'][0]['_']).toContain(`> 9 | test.expect("abc").toBe('abcd');`);
'playwright.config.ts': ` expect(result.exitCode).toBe(1);
module.exports = { });
reporter: [ ['junit'] ],
}; test('should render stdout without ansi escapes', async ({ runInlineTest }) => {
`, const result = await runInlineTest({
'a.test.ts': ` 'playwright.config.ts': `
import { test, expect } from '@playwright/test'; module.exports = {
test('one', async ({}) => { reporter: [ ['junit', { stripANSIControlSequences: true }] ],
process.stdout.write('Hello world &"\\'<>]]>'); };
}); `,
`, 'a.test.ts': `
}, { reporter: '' }); import colors from 'colors/safe';
const xml = parseXML(result.output); import { test, expect } from '@playwright/test';
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; test('one', async ({}) => {
expect(testcase['system-out'].length).toBe(1); console.log(colors.yellow('Hello world'));
expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]&gt;'); });
expect(result.output).toContain(`<system-out>\n<![CDATA[Hello world &"\'<>]]&gt;]]>\n</system-out>`); `,
expect(result.exitCode).toBe(0); }, { reporter: '' });
}); const xml = parseXML(result.output);
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
test('should render skipped', async ({ runInlineTest }) => { expect(testcase['system-out'].length).toBe(1);
const result = await runInlineTest({ expect(testcase['system-out'][0].trim()).toBe('Hello world');
'a.test.js': ` expect(result.exitCode).toBe(0);
import { test, expect } from '@playwright/test'; });
test('one', async () => {
console.log('Hello world'); test('should render, by default, character data as CDATA sections', async ({ runInlineTest }) => {
}); const result = await runInlineTest({
test('two', async () => { 'playwright.config.ts': `
test.skip(); module.exports = {
console.log('Hello world'); reporter: [ ['junit'] ],
}); };
`, `,
}, { retries: 3, reporter: 'junit' }); 'a.test.ts': `
const xml = parseXML(result.output); import { test, expect } from '@playwright/test';
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); test('one', async ({}) => {
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); process.stdout.write('Hello world &"\\'<>]]>');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); });
expect(result.exitCode).toBe(0); `,
}); }, { reporter: '' });
const xml = parseXML(result.output);
test('should report skipped due to sharding', async ({ runInlineTest }) => { const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
const result = await runInlineTest({ expect(testcase['system-out'].length).toBe(1);
'a.test.js': ` expect(testcase['system-out'][0].trim()).toBe('Hello world &"\'<>]]&gt;');
import { test, expect } from '@playwright/test'; expect(result.output).toContain(`<system-out>\n<![CDATA[Hello world &"\'<>]]&gt;]]>\n</system-out>`);
test('one', async () => { expect(result.exitCode).toBe(0);
}); });
test('two', async () => {
test.skip(); test('should render skipped', async ({ runInlineTest }) => {
}); const result = await runInlineTest({
`, 'a.test.js': `
'b.test.js': ` import { test, expect } from '@playwright/test';
import { test, expect } from '@playwright/test'; test('one', async () => {
test('three', async () => { console.log('Hello world');
}); });
test('four', async () => { test('two', async () => {
test.skip(); test.skip();
}); console.log('Hello world');
test('five', async () => { });
}); `,
`, }, { retries: 3, reporter: 'junit' });
}, { shard: '1/3', reporter: 'junit' }); const xml = parseXML(result.output);
const xml = parseXML(result.output); expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'].length).toBe(1); expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2'); expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); expect(result.exitCode).toBe(0);
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1'); });
expect(result.exitCode).toBe(0);
}); test('should report skipped due to sharding', async ({ runInlineTest }) => {
const result = await runInlineTest({
test('should not render projects if they dont exist', async ({ runInlineTest }) => { 'a.test.js': `
const result = await runInlineTest({ import { test, expect } from '@playwright/test';
'playwright.config.ts': ` test('one', async () => {
module.exports = { }; });
`, test('two', async () => {
'a.test.js': ` test.skip();
import { test, expect } from '@playwright/test'; });
test('one', async ({}) => { `,
expect(1).toBe(1); 'b.test.js': `
}); import { test, expect } from '@playwright/test';
`, test('three', async () => {
}, { reporter: 'junit' }); });
const xml = parseXML(result.output); test('four', async () => {
expect(xml['testsuites']['$']['tests']).toBe('1'); test.skip();
expect(xml['testsuites']['$']['failures']).toBe('0'); });
expect(xml['testsuites']['testsuite'].length).toBe(1); test('five', async () => {
});
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js'); `,
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); }, { shard: '1/3', reporter: 'junit' });
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); const xml = parseXML(result.output);
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('2');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(result.exitCode).toBe(0); expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('1');
}); expect(result.exitCode).toBe(0);
});
test('should render projects', async ({ runInlineTest }) => {
const result = await runInlineTest({ test('should not render projects if they dont exist', async ({ runInlineTest }) => {
'playwright.config.ts': ` const result = await runInlineTest({
module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] }; 'playwright.config.ts': `
`, module.exports = { };
'a.test.js': ` `,
import { test, expect } from '@playwright/test'; 'a.test.js': `
test('one', async ({}) => { import { test, expect } from '@playwright/test';
expect(1).toBe(1); test('one', async ({}) => {
}); expect(1).toBe(1);
`, });
}, { reporter: 'junit' }); `,
const xml = parseXML(result.output); }, { reporter: 'junit' });
expect(xml['testsuites']['$']['tests']).toBe('2'); const xml = parseXML(result.output);
expect(xml['testsuites']['$']['failures']).toBe('0'); expect(xml['testsuites']['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'].length).toBe(2); expect(xml['testsuites']['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'].length).toBe(1);
expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1'); expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0'); expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one'); expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js'); expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
expect(result.exitCode).toBe(0);
expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js'); });
expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2');
expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1'); test('should render projects', async ({ runInlineTest }) => {
expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0'); const result = await runInlineTest({
expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0'); 'playwright.config.ts': `
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one'); module.exports = { projects: [ { name: 'project1' }, { name: 'project2' } ] };
expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js'); `,
expect(result.exitCode).toBe(0); 'a.test.js': `
}); import { test, expect } from '@playwright/test';
test('one', async ({}) => {
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => { expect(1).toBe(1);
const result = await runInlineTest({ });
'a.test.js': ` `,
import { test, expect } from '@playwright/test'; }, { reporter: 'junit' });
test.use({ screenshot: 'on' }); const xml = parseXML(result.output);
test('one', async ({ page }, testInfo) => { expect(xml['testsuites']['$']['tests']).toBe('2');
await page.setContent('hello'); expect(xml['testsuites']['$']['failures']).toBe('0');
const file = testInfo.outputPath('file.txt'); expect(xml['testsuites']['testsuite'].length).toBe(2);
require('fs').writeFileSync(file, 'my file', 'utf8');
testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' }); expect(xml['testsuites']['testsuite'][0]['$']['name']).toBe('a.test.js');
testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' }); expect(xml['testsuites']['testsuite'][0]['$']['hostname']).toBe('project1');
console.log('log here'); expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1');
}); expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0');
`, expect(xml['testsuites']['testsuite'][0]['$']['skipped']).toBe('0');
}, { reporter: 'junit' }); expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['name']).toBe('one');
const xml = parseXML(result.output); expect(xml['testsuites']['testsuite'][0]['testcase'][0]['$']['classname']).toBe('a.test.js');
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['system-out'].length).toBe(1); expect(xml['testsuites']['testsuite'][1]['$']['name']).toBe('a.test.js');
expect(testcase['system-out'][0].trim()).toBe([ expect(xml['testsuites']['testsuite'][1]['$']['hostname']).toBe('project2');
`log here`, expect(xml['testsuites']['testsuite'][1]['$']['tests']).toBe('1');
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`, expect(xml['testsuites']['testsuite'][1]['$']['failures']).toBe('0');
`\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`, expect(xml['testsuites']['testsuite'][1]['$']['skipped']).toBe('0');
].join('\n')); expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['name']).toBe('one');
expect(result.exitCode).toBe(0); expect(xml['testsuites']['testsuite'][1]['testcase'][0]['$']['classname']).toBe('a.test.js');
}); expect(result.exitCode).toBe(0);
});
function parseXML(xml: string): any {
let result: any; test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
xml2js.parseString(xml, (err, r) => result = r); test.skip(useIntermediateMergeReport, 'Blob report hashes attachment paths');
return result; const result = await runInlineTest({
} 'a.test.js': `
import { test, expect } from '@playwright/test';
test('should render annotations to custom testcase properties', async ({ runInlineTest }) => { test.use({ screenshot: 'on' });
const result = await runInlineTest({ test('one', async ({ page }, testInfo) => {
'playwright.config.ts': `module.exports = { reporter: 'junit' };`, await page.setContent('hello');
'a.test.js': ` const file = testInfo.outputPath('file.txt');
import { test, expect } from '@playwright/test'; require('fs').writeFileSync(file, 'my file', 'utf8');
test('one', async ({}, testInfo) => { testInfo.attachments.push({ name: 'my-file', path: file, contentType: 'text/plain' });
testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); testInfo.attachments.push({ name: 'my-file-missing', path: file + '-missing', contentType: 'text/plain' });
}); console.log('log here');
` });
}, { reporter: '' }); `,
const xml = parseXML(result.output); }, { reporter: 'junit' });
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; const xml = parseXML(result.output);
expect(testcase['properties']).toBeTruthy(); const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
expect(testcase['properties'][0]['property'].length).toBe(1); expect(testcase['system-out'].length).toBe(1);
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description'); expect(testcase['system-out'][0].trim()).toBe([
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description'); `log here`,
expect(result.exitCode).toBe(0); `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}file.txt]]`,
}); `\n[[ATTACHMENT|test-results${path.sep}a-one${path.sep}test-finished-1.png]]`,
].join('\n'));
test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0);
const result = await runInlineTest({ });
'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
'a.test.js': ` function parseXML(xml: string): any {
import { test, expect } from '@playwright/test'; let result: any;
test('one', async ({}, testInfo) => { xml2js.parseString(xml, (err, r) => result = r);
test.slow(); return result;
}); }
`
}, { reporter: '' }); test('should render annotations to custom testcase properties', async ({ runInlineTest }) => {
const xml = parseXML(result.output); const result = await runInlineTest({
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; 'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
expect(testcase['properties']).toBeTruthy(); 'a.test.js': `
expect(testcase['properties'][0]['property'].length).toBe(1); import { test, expect } from '@playwright/test';
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow'); test('one', async ({}, testInfo) => {
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe(''); testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
expect(result.exitCode).toBe(0); });
}); `
}, { reporter: '' });
test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => { const xml = parseXML(result.output);
const result = await runInlineTest({ const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
'playwright.config.ts': ` expect(testcase['properties']).toBeTruthy();
const xrayOptions = { expect(testcase['properties'][0]['property'].length).toBe(1);
embedAnnotationsAsProperties: true expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description');
} expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('sample description');
module.exports = { expect(result.exitCode).toBe(0);
reporter: [ ['junit', xrayOptions] ], });
};
`, test('should render built-in annotations to testcase properties', async ({ runInlineTest }) => {
'a.test.js': ` const result = await runInlineTest({
import { test, expect } from '@playwright/test'; 'playwright.config.ts': `module.exports = { reporter: 'junit' };`,
test('one', async ({}, testInfo) => { 'a.test.js': `
testInfo.annotations.push({ type: 'test_id', description: '1234' }); import { test, expect } from '@playwright/test';
testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); test('one', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); test.slow();
testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); });
}); `
` }, { reporter: '' });
}, { reporter: '' }); const xml = parseXML(result.output);
const xml = parseXML(result.output); const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; expect(testcase['properties']).toBeTruthy();
expect(testcase['properties']).toBeTruthy(); expect(testcase['properties'][0]['property'].length).toBe(1);
expect(testcase['properties'][0]['property'].length).toBe(4); expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('slow');
expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id'); expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('');
expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234'); expect(result.exitCode).toBe(0);
expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key'); });
expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2');
expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary'); test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary'); const result = await runInlineTest({
expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements'); 'playwright.config.ts': `
expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6'); const xrayOptions = {
expect(result.exitCode).toBe(0); embedAnnotationsAsProperties: true
}); }
module.exports = {
test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => { reporter: [ ['junit', xrayOptions] ],
const result = await runInlineTest({ };
'a.test.js': ` `,
import { test, expect } from '@playwright/test'; 'a.test.js': `
test('one', async ({}, testInfo) => { import { test, expect } from '@playwright/test';
const file = testInfo.outputPath('evidence1.txt'); test('one', async ({}, testInfo) => {
require('fs').writeFileSync(file, 'hello', 'utf8'); testInfo.annotations.push({ type: 'test_id', description: '1234' });
testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
// await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
// await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); });
}); `
` }, { reporter: '' });
}, { reporter: 'junit' }); const xml = parseXML(result.output);
const xml = parseXML(result.output); const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; expect(testcase['properties']).toBeTruthy();
expect(testcase['properties']).not.toBeTruthy(); expect(testcase['properties'][0]['property'].length).toBe(4);
expect(result.exitCode).toBe(0); expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id');
}); expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234');
expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key');
expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2');
test.describe('report location', () => { expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary');
test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary');
const result = await runInlineTest({ expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements');
'nested/project/playwright.config.ts': ` expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6');
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] }; expect(result.exitCode).toBe(0);
`, });
'nested/project/a.test.js': `
import { test, expect } from '@playwright/test'; test('should not embed attachments to a custom testcase property, if not explicitly requested', async ({ runInlineTest }) => {
test('one', async ({}) => { const result = await runInlineTest({
expect(1).toBe(1); 'a.test.js': `
}); import { test, expect } from '@playwright/test';
`, test('one', async ({}, testInfo) => {
}, { reporter: '', config: './nested/project/playwright.config.ts' }); const file = testInfo.outputPath('evidence1.txt');
expect(result.exitCode).toBe(0); require('fs').writeFileSync(file, 'hello', 'utf8');
expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy(); testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' });
}); testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' });
// await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
const result = await runInlineTest({ });
'foo/package.json': `{ "name": "foo" }`, `
// unused config along "search path" }, { reporter: 'junit' });
'foo/bar/playwright.config.js': ` const xml = parseXML(result.output);
module.exports = { projects: [ {} ] }; const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
`, expect(testcase['properties']).not.toBeTruthy();
'foo/bar/baz/tests/a.spec.js': ` expect(result.exitCode).toBe(0);
import { test, expect } from '@playwright/test'; });
const fs = require('fs');
test('pass', ({}, testInfo) => {
}); test.describe('report location', () => {
` test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
}, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { test.skip(useIntermediateMergeReport);
cwd: 'foo/bar/baz/tests', const result = await runInlineTest({
'nested/project/playwright.config.ts': `
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
`,
'nested/project/a.test.js': `
import { test, expect } from '@playwright/test';
test('one', async ({}) => {
expect(1).toBe(1);
});
`,
}, { reporter: '', config: './nested/project/playwright.config.ts' });
expect(result.exitCode).toBe(0);
expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy();
});
test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'foo/package.json': `{ "name": "foo" }`,
// unused config along "search path"
'foo/bar/playwright.config.js': `
module.exports = { projects: [ {} ] };
`,
'foo/bar/baz/tests/a.spec.js': `
import { test, expect } from '@playwright/test';
const fs = require('fs');
test('pass', ({}, testInfo) => {
});
`
}, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, {
cwd: 'foo/bar/baz/tests',
});
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
}); });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
}); });
}); }

288
tests/playwright-test/reporter-line.spec.ts
View File

@ -16,149 +16,155 @@
import { test, expect } from './playwright-test-fixtures'; import { test, expect } from './playwright-test-fixtures';
test('render unexpected after retry', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const result = await runInlineTest({ test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
'a.test.js': ` test.use({ useIntermediateMergeReport });
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('[2/1] (retries) a.test.js:3:7 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:3:7 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:3:7 › one (retry #3)');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ────');
expect(text).toContain('Retry #2 ────');
expect(text).toContain('Retry #3 ────');
expect(result.exitCode).toBe(1);
});
test('render flaky', async ({ runInlineTest }) => { test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.js': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; const { test, expect } = require('@playwright/test');
test('one', async ({}, testInfo) => { test('one', async ({}) => {
expect(testInfo.retry).toBe(3); expect(1).toBe(0);
});
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('1 flaky');
expect(result.exitCode).toBe(0);
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should work on CI', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'line' }, { CI: '1' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
});
test('should print output', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
process.stdout.write('one');
process.stdout.write('two');
console.log('full-line');
});
`
}, { reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.output).toContain([
'a.spec.ts:3:11 › foobar',
'one',
'',
'two',
'',
'full-line',
].join('\n'));
});
test('should render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
}); });
}); `,
}); }, { retries: 3, reporter: 'line' });
`, const text = result.output;
}, { reporter: 'line' }); expect(text).toContain('[1/1] a.test.js:3:11 › one');
const text = result.output; expect(text).toContain('[2/1] (retries) a.test.js:3:11 › one (retry #1)');
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); expect(text).toContain('[3/1] (retries) a.test.js:3:11 › one (retry #2)');
expect(result.exitCode).toBe(1); expect(text).toContain('[4/1] (retries) a.test.js:3:11 › one (retry #3)');
}); expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
expect(text).toContain('Retry #1 ────');
expect(text).toContain('Retry #2 ────');
expect(text).toContain('Retry #3 ────');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => { test('render flaky', async ({ runInlineTest }) => {
const result = await runInlineTest({ const result = await runInlineTest({
'a.test.ts': ` 'a.test.js': `
import { test, expect } from '@playwright/test'; import { test, expect } from '@playwright/test';
test('passes', async ({}) => { test('one', async ({}, testInfo) => {
await test.step('outer 1.0', async () => { expect(testInfo.retry).toBe(3);
await test.step('inner 1.1', async () => { });
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('1 flaky');
expect(result.exitCode).toBe(0);
});
test('should print flaky failures', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`
}, { retries: '1', reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.flaky).toBe(1);
expect(result.output).toContain('expect(testInfo.retry).toBe(1)');
});
test('should work on CI', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
test('one', async ({}) => {
expect(1).toBe(0);
});
`,
}, { reporter: 'line' }, { CI: '1' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:11 › one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
});
test('should print output', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('foobar', async ({}, testInfo) => {
process.stdout.write('one');
process.stdout.write('two');
console.log('full-line');
});
`
}, { reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.output).toContain([
'a.spec.ts:3:15 › foobar',
'one',
'',
'two',
'',
'full-line',
].join('\n'));
});
test('should render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
await test.step('inner 1.2', async () => {
expect.soft(1).toBe(2);
});
});
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
});
expect.soft(1).toBe(2); expect.soft(1).toBe(2);
}); });
await test.step('inner 1.2', async () => { `,
expect.soft(1).toBe(2); }, { reporter: 'line' });
}); const text = result.output;
}); expect(text).toContain('1) a.test.ts:3:15 › passes ──');
}); expect(result.exitCode).toBe(1);
`, });
}, { reporter: 'line' }); });
const text = result.output; }
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──');
expect(result.exitCode).toBe(1);
});
test('should not render more than one failed test steps in header (2)', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect.soft(1).toBe(2);
});
});
expect.soft(1).toBe(2);
});
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes ──');
expect(result.exitCode).toBe(1);
});

411
tests/playwright-test/reporter-list.spec.ts
View File

@ -20,212 +20,217 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces
const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ '; const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ '; const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
test('render each test with project name', async ({ runInlineTest }) => { for (const useIntermediateMergeReport of [false, true] as const) {
const result = await runInlineTest({ test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
'playwright.config.ts': ` test.use({ useIntermediateMergeReport });
module.exports = { projects: [
{ name: 'foo' },
{ name: 'bar' },
] };
`,
'a.test.ts': `
const { test, expect } = require('@playwright/test');
test('fails', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
expect(0).toBe(0);
});
test.skip('skipped', async () => {
});
`,
}, { reporter: 'list', workers: '1' });
const text = result.output;
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:7 › fails`); test('render each test with project name', async ({ runInlineTest }) => {
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:7 › passes`); const result = await runInlineTest({
expect(text).toContain(`- 3 [foo] › a.test.ts:9:12 › skipped`); 'playwright.config.ts': `
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:7 › fails`); module.exports = { projects: [
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:7 › passes`); { name: 'foo' },
expect(text).toContain(`- 6 [bar] › a.test.ts:9:12 › skipped`); { name: 'bar' },
expect(result.exitCode).toBe(1); ] };
}); `,
'a.test.ts': `
test('render steps', async ({ runInlineTest }) => { const { test, expect } = require('@playwright/test');
const result = await runInlineTest({ test('fails', async ({}) => {
'a.test.ts': ` expect(1).toBe(0);
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)',
]);
});
test('render steps inline', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:3:11 › passes',
]);
});
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
const TTY_WIDTH = 80;
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
console.log('a'.repeat(80) + 'b'.repeat(20));
});
`,
}, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' });
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
if (process.platform === 'win32')
expect(renderedText).toContain(' ok 1 a.test.ts:3:11 › passes');
else
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:11 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:11 › passes');
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
});
test('render retries', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('flaky', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:3:11 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:11 › flaky (XXms)`,
`1 : 2 a.test.ts:3:11 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:11 › flaky (retry #1) (XXms)`,
]);
});
test('should truncate long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
] };
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('failure in very long name', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
});
test('passes 2 long name', async () => {
});
test.skip('skipped very long name', async () => {
});
`,
}, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' });
expect(result.exitCode).toBe(1);
const lines = result.output.split('\n').slice(3, 11);
expect(lines.every(line => line.length <= 50)).toBe(true);
expect(lines[0]).toBe(` 1 …a.test.ts:3:11 › failure in very long name`);
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
expect(lines[1]).toContain(`:3:11 › failure in very long name (`);
expect(lines[1].length).toBe(50);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:11 › passes`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes (`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:11 › passes 2 long name`);
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
expect(lines[5]).toContain(`test.ts:8:11 › passes 2 long name (`);
expect(lines[5].length).toBe(50);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:12 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`);
});
test('render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
}); });
test('passes', async ({}) => {
expect(0).toBe(0);
});
test.skip('skipped', async () => {
});
`,
}, { reporter: 'list', workers: '1' });
const text = result.output;
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 3 [foo] › a.test.ts:9:16 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 6 [bar] › a.test.ts:9:16 › skipped`);
expect(result.exitCode).toBe(1);
});
test('render steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:15 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)',
]);
});
test('render steps inline', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
}); });
}); await test.step('outer 2.0', async () => {
`, await test.step('inner 2.1', async () => {});
}, { reporter: 'list' }); await test.step('inner 2.2', async () => {});
const text = result.output; });
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──'); });`,
expect(result.exitCode).toBe(1); }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
}); const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:3:11 › passes',
]);
});
test('very long console line should not mess terminal', async ({ runInlineTest }) => {
const TTY_WIDTH = 80;
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
console.log('a'.repeat(80) + 'b'.repeat(20));
});
`,
}, { reporter: 'list' }, { PWTEST_TTY_WIDTH: TTY_WIDTH + '' });
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
if (process.platform === 'win32')
expect(renderedText).toContain(' ok 1 a.test.ts:3:15 › passes');
else
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:15 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:15 › passes');
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
});
test('render retries', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('flaky', async ({}, testInfo) => {
expect(testInfo.retry).toBe(1);
});
`,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:3:15 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
]);
});
test('should truncate long test names', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { projects: [
{ name: 'foo' },
] };
`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('failure in very long name', async ({}) => {
expect(1).toBe(0);
});
test('passes', async ({}) => {
});
test('passes 2 long name', async () => {
});
test.skip('skipped very long name', async () => {
});
`,
}, { reporter: 'list', retries: 0 }, { PWTEST_TTY_WIDTH: '50' });
expect(result.exitCode).toBe(1);
const lines = result.output.split('\n').slice(3, 11);
expect(lines.every(line => line.length <= 50)).toBe(true);
expect(lines[0]).toBe(` 1 …a.test.ts:3:15 › failure in very long name`);
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
expect(lines[1]).toContain(`:3:15 › failure in very long name (`);
expect(lines[1].length).toBe(50);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:15 › passes`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:15 › passes (`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:15 › passes 2 long name`);
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
expect(lines[5]).toContain(`test.ts:8:15 › passes 2 long name (`);
expect(lines[5].length).toBe(50);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:16 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:16 › skipped very long name`);
});
test('render failed test steps', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {
expect(1).toBe(2);
});
});
});
`,
}, { reporter: 'list' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
});
}
function simpleAnsiRenderer(text, ttyWidth) { function simpleAnsiRenderer(text, ttyWidth) {
let lineNumber = 0; let lineNumber = 0;

1307
tests/playwright-test/reporter.spec.ts
View File

File diff suppressed because it is too large Load Diff