mirrors
/
playwright
mirror of https://mirror.osredm.com/root/playwright.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: run more reporter tests through blob report, some fixes (#23765)

This commit is contained in:
Yury Semikhatsky 2023-06-16 21:30:55 -07:00 committed by GitHub
parent 09b1e3ffa9
commit f6d86c20f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1953 additions and 1908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/playwright-test/src/isomorphic/teleReceiver.ts
View File

@ -232,6 +232,7 @@ export class TeleReporterReceiver {
result.status = payload.status;
result.statusEx = payload.status;
result.errors = payload.errors;
result.error = result.errors?.[0];
result.attachments = this._parseAttachments(payload.attachments);
this._reporter.onTestEnd?.(test, result);
}
@ -242,7 +243,10 @@ export class TeleReporterReceiver {
const parentStep = payload.parentStepId ? result.stepMap.get(payload.parentStepId) : undefined;
const step: TestStep = {
titlePath: () => [],
titlePath: () => {
const parentPath = parentStep?.titlePath() || [];
return [...parentPath, payload.title];
},
title: payload.title,
category: payload.category,
location: this._absoluteLocation(payload.location),

2
packages/playwright-test/src/reporters/blob.ts
View File

@ -92,7 +92,7 @@ export class BlobReporter extends TeleReporterEmitter {
override _serializeAttachments(attachments: TestResult['attachments']): JsonAttachment[] {
return super._serializeAttachments(attachments).map(attachment => {
if (!attachment.path || !fs.statSync(attachment.path).isFile())
if (!attachment.path || !fs.statSync(attachment.path, { throwIfNoEntry: false })?.isFile())
return attachment;
// Add run guid to avoid clashes between shards.
const sha1 = calculateSha1(attachment.path + this._salt);

9
tests/playwright-test/playwright-test-fixtures.ts
View File

@ -88,7 +88,7 @@ export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-tes
const configFile = (baseDir: string, files: Files): string | undefined => {
for (const [name, content] of Object.entries(files)) {
if (name.includes('playwright.config')) {
if (content.includes('reporter:'))
if (content.includes('reporter:') || content.includes('reportSlowTests:'))
return path.resolve(baseDir, name);
}
}
@ -349,12 +349,7 @@ export const test = base
const cwd = options.cwd ? path.resolve(test.info().outputDir, options.cwd) : test.info().outputDir;
const testProcess = childProcess({
command,
env: cleanEnv({
PW_TEST_DEBUG_REPORTERS: '1',
PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1',
PWTEST_TTY_WIDTH: '80',
...env
}),
env: cleanEnv(env),
cwd,
});
const { exitCode } = await testProcess.exited;

10
tests/playwright-test/reporter-base.spec.ts
View File

@ -17,6 +17,10 @@
import { test, expect } from './playwright-test-fixtures';
import * as path from 'path';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('handle long test names', async ({ runInlineTest }) => {
const title = 'title'.repeat(30);
const result = await runInlineTest({
@ -106,12 +110,12 @@ test('should filter out node_modules error in a codeframe', async ({ runInlineTe
expect(result.failed).toBe(1);
const output = result.output;
expect(output).toContain('Error: Assertion error');
expect(output).toContain('a.spec.ts:4:11 › fail');
expect(output).toContain('a.spec.ts:4:15 › fail');
expect(output).toContain(` 4 | test('fail', async ({}) => {`);
expect(output).toContain(`> 5 | assert(false);`);
expect(output).toContain(` | ^`);
expect(output).toContain(`utils.js:4`);
expect(output).toContain(`a.spec.ts:5:9`);
expect(output).toContain(`a.spec.ts:5:13`);
});
test('should print codeframe from a helper', async ({ runInlineTest }) => {
@ -360,3 +364,5 @@ test('should contain at most 1 decimal for humanized timing', async ({ runInline
expect(result.passed).toBe(1);
expect(result.output).toMatch(/\d+ passed \(\d+(\.\d)?(ms|s)\)/);
});
});
}

2
tests/playwright-test/reporter-blob.spec.ts
View File

@ -361,7 +361,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', 'list'] });
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0);
const text = stripAnsi(output);

6
tests/playwright-test/reporter-dot.spec.ts
View File

@ -17,6 +17,10 @@
import colors from 'colors/safe';
import { test, expect } from './playwright-test-fixtures';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
@ -107,3 +111,5 @@ test('render 243 tests in rows by 80', async ({ runInlineTest }) => {
colors.green('·').repeat(80) + '\n' +
colors.green('·').repeat(3));
});
});
}

12
tests/playwright-test/reporter-github.spec.ts
View File

@ -23,6 +23,10 @@ function relativeFilePath(file: string): string {
return path.relative(process.cwd(), file);
}
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('print GitHub annotations for success', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
@ -49,9 +53,9 @@ test('print GitHub annotations for failed tests', async ({ runInlineTest }, test
}, { retries: 3, reporter: 'github' }, { GITHUB_WORKSPACE: process.cwd() });
const text = result.output;
const testPath = relativeFilePath(testInfo.outputPath('a.test.js'));
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:7 › example,line=4,col=23:: 1) a.test.js:3:7 › example ───────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #1`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #2`);
expect(text).toContain(`::error file=${testPath},title=a.test.js:3:11 › example,line=4,col=27:: 1) a.test.js:3:11 › example ──────────────────────────────────────────────────────────────────────%0A%0A Retry #3`);
expect(result.exitCode).toBe(1);
});
@ -93,3 +97,5 @@ test('print GitHub annotations for global error', async ({ runInlineTest }) => {
expect(text).toContain('::error ::Error: Oh my!%0A%0A');
expect(result.exitCode).toBe(1);
});
});
}

8
tests/playwright-test/reporter-junit.spec.ts
View File

@ -19,6 +19,10 @@ import path from 'path';
import { test, expect } from './playwright-test-fixtures';
import fs from 'fs';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('should render expected', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
@ -277,6 +281,7 @@ test('should render projects', async ({ runInlineTest }) => {
});
test('should render existing attachments, but not missing ones', async ({ runInlineTest }) => {
test.skip(useIntermediateMergeReport, 'Blob report hashes attachment paths');
const result = await runInlineTest({
'a.test.js': `
import { test, expect } from '@playwright/test';
@ -404,6 +409,7 @@ test('should not embed attachments to a custom testcase property, if not explici
test.describe('report location', () => {
test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
test.skip(useIntermediateMergeReport);
const result = await runInlineTest({
'nested/project/playwright.config.ts': `
module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
@ -440,3 +446,5 @@ test.describe('report location', () => {
expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
});
});
});
}

24
tests/playwright-test/reporter-line.spec.ts
View File

@ -16,6 +16,10 @@
import { test, expect } from './playwright-test-fixtures';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('render unexpected after retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.js': `
@ -26,10 +30,10 @@ test('render unexpected after retry', async ({ runInlineTest }) => {
`,
}, { retries: 3, reporter: 'line' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('[2/1] (retries) a.test.js:3:7 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:3:7 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:3:7 › one (retry #3)');
expect(text).toContain('[1/1] a.test.js:3:11 › one');
expect(text).toContain('[2/1] (retries) a.test.js:3:11 › one (retry #1)');
expect(text).toContain('[3/1] (retries) a.test.js:3:11 › one (retry #2)');
expect(text).toContain('[4/1] (retries) a.test.js:3:11 › one (retry #3)');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(text).not.toContain('2) a.test');
@ -77,7 +81,7 @@ test('should work on CI', async ({ runInlineTest }) => {
`,
}, { reporter: 'line' }, { CI: '1' });
const text = result.output;
expect(text).toContain('[1/1] a.test.js:3:7 › one');
expect(text).toContain('[1/1] a.test.js:3:11 › one');
expect(text).toContain('1 failed');
expect(text).toContain('1) a.test');
expect(result.exitCode).toBe(1);
@ -96,7 +100,7 @@ test('should print output', async ({ runInlineTest }) => {
}, { reporter: 'line' });
expect(result.exitCode).toBe(0);
expect(result.output).toContain([
'a.spec.ts:3:11 › foobar',
'a.spec.ts:3:15 › foobar',
'one',
'',
'two',
@ -119,7 +123,7 @@ test('should render failed test steps', async ({ runInlineTest }) => {
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
@ -140,7 +144,7 @@ test('should not render more than one failed test steps in header', async ({ run
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 ──');
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 ──');
expect(result.exitCode).toBe(1);
});
@ -159,6 +163,8 @@ test('should not render more than one failed test steps in header (2)', async ({
`,
}, { reporter: 'line' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes ──');
expect(text).toContain('1) a.test.ts:3:15 › passes ──');
expect(result.exitCode).toBe(1);
});
});
}

55
tests/playwright-test/reporter-list.spec.ts
View File

@ -20,6 +20,10 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces
const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓ ';
const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x ' : '✘ ';
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('render each test with project name', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
@ -42,12 +46,12 @@ test('render each test with project name', async ({ runInlineTest }) => {
}, { reporter: 'list', workers: '1' });
const text = result.output;
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:7 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:7 › passes`);
expect(text).toContain(`- 3 [foo] › a.test.ts:9:12 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:7 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:7 › passes`);
expect(text).toContain(`- 6 [bar] › a.test.ts:9:12 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 1 [foo] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 3 [foo] › a.test.ts:9:16 › skipped`);
expect(text).toContain(`${NEGATIVE_STATUS_MARK} 4 [bar] › a.test.ts:3:11 › fails`);
expect(text).toContain(`${POSITIVE_STATUS_MARK} 5 [bar] › a.test.ts:6:11 › passes`);
expect(text).toContain(`- 6 [bar] › a.test.ts:9:16 › skipped`);
expect(result.exitCode).toBe(1);
});
@ -71,7 +75,7 @@ test('render steps', async ({ runInlineTest }) => {
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:3:15 › passes',
'1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
@ -100,8 +104,7 @@ test('render steps inline', async ({ runInlineTest }) => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
});`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PWTEST_TTY_WIDTH: '80' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/\d+ms/, 'Xms'));
@ -136,10 +139,10 @@ test('very long console line should not mess terminal', async ({ runInlineTest }
const renderedText = simpleAnsiRenderer(result.rawOutput, TTY_WIDTH);
if (process.platform === 'win32')
expect(renderedText).toContain(' ok 1 a.test.ts:3:11 › passes');
expect(renderedText).toContain(' ok 1 a.test.ts:3:15 › passes');
else
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:11 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:11 › passes');
expect(renderedText).toContain(' ✓ 1 a.test.ts:3:15 › passes');
expect(renderedText).not.toContain(' 1 a.test.ts:3:15 › passes');
expect(renderedText).toContain('a'.repeat(80) + '\n' + 'b'.repeat(20));
});
@ -156,10 +159,10 @@ test('render retries', async ({ runInlineTest }) => {
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([
`0 : 1 a.test.ts:3:11 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:11 › flaky (XXms)`,
`1 : 2 a.test.ts:3:11 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:11 › flaky (retry #1) (XXms)`,
`0 : 1 a.test.ts:3:15 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
]);
});
@ -188,25 +191,25 @@ test('should truncate long test names', async ({ runInlineTest }) => {
const lines = result.output.split('\n').slice(3, 11);
expect(lines.every(line => line.length <= 50)).toBe(true);
expect(lines[0]).toBe(` 1 …a.test.ts:3:11 › failure in very long name`);
expect(lines[0]).toBe(` 1 …a.test.ts:3:15 › failure in very long name`);
expect(lines[1]).toContain(`${NEGATIVE_STATUS_MARK} 1 …`);
expect(lines[1]).toContain(`:3:11 › failure in very long name (`);
expect(lines[1]).toContain(`:3:15 › failure in very long name (`);
expect(lines[1].length).toBe(50);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:11 › passes`);
expect(lines[2]).toBe(` 2 [foo] › a.test.ts:6:15 › passes`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:11 › passes (`);
expect(lines[3]).toContain(`${POSITIVE_STATUS_MARK} 2 [foo] › a.test.ts:6:15 › passes (`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:11 › passes 2 long name`);
expect(lines[4]).toBe(` 3 [foo] › a.test.ts:8:15 › passes 2 long name`);
expect(lines[5]).toContain(`${POSITIVE_STATUS_MARK} 3 …`);
expect(lines[5]).toContain(`test.ts:8:11 › passes 2 long name (`);
expect(lines[5]).toContain(`test.ts:8:15 › passes 2 long name (`);
expect(lines[5].length).toBe(50);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:12 › skipped very long name`);
expect(lines[6]).toBe(` 4 …› a.test.ts:10:16 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:12 › skipped very long name`);
expect(lines[7]).toBe(` - 4 …› a.test.ts:10:16 › skipped very long name`);
});
test('render failed test steps', async ({ runInlineTest }) => {
@ -223,9 +226,11 @@ test('render failed test steps', async ({ runInlineTest }) => {
`,
}, { reporter: 'list' });
const text = result.output;
expect(text).toContain('1) a.test.ts:3:11 › passes › outer 1.0 › inner 1.1 ──');
expect(text).toContain('1) a.test.ts:3:15 › passes › outer 1.0 › inner 1.1 ──');
expect(result.exitCode).toBe(1);
});
});
}
function simpleAnsiRenderer(text, ttyWidth) {
let lineNumber = 0;

15
tests/playwright-test/reporter.spec.ts
View File

@ -74,6 +74,10 @@ class Reporter {
module.exports = Reporter;
`;
for (const useIntermediateMergeReport of [false, true] as const) {
test.describe(`${useIntermediateMergeReport ? 'merged' : 'created'}`, () => {
test.use({ useIntermediateMergeReport });
test('should work with custom reporter', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': `
@ -186,6 +190,7 @@ test('should work without a file extension', async ({ runInlineTest }) => {
});
test('should report onEnd after global teardown', async ({ runInlineTest }) => {
test.skip(useIntermediateMergeReport);
const result = await runInlineTest({
'reporter.ts': smallReporterJS,
'globalSetup.ts': `
@ -712,10 +717,12 @@ test('test and step error should have code snippet', async ({ runInlineTest }) =
class Reporter {
onStepEnd(test, result, step) {
console.log('\\n%%onStepEnd: ' + step.error?.snippet?.length);
if (step.error?.snippet)
fs.writeFileSync('${stepErrorFile.replace(/\\/g, '\\\\')}', step.error?.snippet);
}
onTestEnd(test, result) {
console.log('\\n%%onTestEnd: ' + result.error?.snippet?.length);
if (result.error)
fs.writeFileSync('${testErrorFile.replace(/\\/g, '\\\\')}', result.error?.snippet);
}
onError(error) {
@ -738,8 +745,8 @@ test('test and step error should have code snippet', async ({ runInlineTest }) =
`,
}, { 'reporter': '', 'workers': 1 });
expect(result.output).toContain('onTestEnd: 522');
expect(result.output).toContain('onStepEnd: 522');
expect(result.output).toContain('onTestEnd: 550');
expect(result.output).toContain('onStepEnd: 550');
expect(stripAnsi(fs.readFileSync(testErrorFile, 'utf8'))).toBe(` 3 | test('test', async () => {
4 | await test.step('step', async () => {
> 5 | expect(1).toBe(2);
@ -779,7 +786,7 @@ test('onError should have code snippet', async ({ runInlineTest }) => {
`,
}, { 'reporter': '', 'workers': 1 });
expect(result.output).toContain('onError: 396');
expect(result.output).toContain('onError: 412');
expect(stripAnsi(fs.readFileSync(errorFile, 'utf8'))).toBe(` at a.spec.js:3
1 |
@ -788,3 +795,5 @@ test('onError should have code snippet', async ({ runInlineTest }) => {
| ^
4 | `);
});
});
}