651 lines
17 KiB
JavaScript
651 lines
17 KiB
JavaScript
/* eslint-disable node-core/require-common-first, node-core/required-modules */
|
|
'use strict';
|
|
|
|
const assert = require('assert');
|
|
const fixtures = require('../common/fixtures');
|
|
const fs = require('fs');
|
|
const fsPromises = fs.promises;
|
|
const path = require('path');
|
|
const vm = require('vm');
|
|
const { inspect } = require('util');
|
|
|
|
// https://github.com/w3c/testharness.js/blob/master/testharness.js
|
|
// TODO: get rid of this half-baked harness in favor of the one
|
|
// pulled from WPT
|
|
const harnessMock = {
|
|
test: (fn, desc) => {
|
|
try {
|
|
fn();
|
|
} catch (err) {
|
|
console.error(`In ${desc}:`);
|
|
throw err;
|
|
}
|
|
},
|
|
assert_equals: assert.strictEqual,
|
|
assert_true: (value, message) => assert.strictEqual(value, true, message),
|
|
assert_false: (value, message) => assert.strictEqual(value, false, message),
|
|
assert_throws: (code, func, desc) => {
|
|
assert.throws(func, function(err) {
|
|
return typeof err === 'object' &&
|
|
'name' in err &&
|
|
err.name.startsWith(code.name);
|
|
}, desc);
|
|
},
|
|
assert_array_equals: assert.deepStrictEqual,
|
|
assert_unreached(desc) {
|
|
assert.fail(`Reached unreachable code: ${desc}`);
|
|
}
|
|
};
|
|
|
|
class ResourceLoader {
|
|
constructor(path) {
|
|
this.path = path;
|
|
}
|
|
|
|
toRealFilePath(from, url) {
|
|
// We need to patch this to load the WebIDL parser
|
|
url = url.replace(
|
|
'/resources/WebIDLParser.js',
|
|
'/resources/webidl2/lib/webidl2.js'
|
|
);
|
|
const base = path.dirname(from);
|
|
return url.startsWith('/') ?
|
|
fixtures.path('wpt', url) :
|
|
fixtures.path('wpt', base, url);
|
|
}
|
|
|
|
/**
|
|
* Load a resource in test/fixtures/wpt specified with a URL
|
|
* @param {string} from the path of the file loading this resource,
|
|
* relative to thw WPT folder.
|
|
* @param {string} url the url of the resource being loaded.
|
|
* @param {boolean} asPromise if true, return the resource in a
|
|
* pseudo-Response object.
|
|
*/
|
|
read(from, url, asFetch = true) {
|
|
const file = this.toRealFilePath(from, url);
|
|
if (asFetch) {
|
|
return fsPromises.readFile(file)
|
|
.then((data) => {
|
|
return {
|
|
ok: true,
|
|
json() { return JSON.parse(data.toString()); },
|
|
text() { return data.toString(); }
|
|
};
|
|
});
|
|
} else {
|
|
return fs.readFileSync(file, 'utf8');
|
|
}
|
|
}
|
|
}
|
|
|
|
class StatusRule {
|
|
constructor(key, value, pattern = undefined) {
|
|
this.key = key;
|
|
this.requires = value.requires || [];
|
|
this.fail = value.fail;
|
|
this.skip = value.skip;
|
|
if (pattern) {
|
|
this.pattern = this.transformPattern(pattern);
|
|
}
|
|
// TODO(joyeecheung): implement this
|
|
this.scope = value.scope;
|
|
this.comment = value.comment;
|
|
}
|
|
|
|
/**
|
|
* Transform a filename pattern into a RegExp
|
|
* @param {string} pattern
|
|
* @returns {RegExp}
|
|
*/
|
|
transformPattern(pattern) {
|
|
const result = path.normalize(pattern).replace(/[-/\\^$+?.()|[\]{}]/g, '\\$&');
|
|
return new RegExp(result.replace('*', '.*'));
|
|
}
|
|
}
|
|
|
|
class StatusRuleSet {
|
|
constructor() {
|
|
// We use two sets of rules to speed up matching
|
|
this.exactMatch = {};
|
|
this.patternMatch = [];
|
|
}
|
|
|
|
/**
|
|
* @param {object} rules
|
|
*/
|
|
addRules(rules) {
|
|
for (const key of Object.keys(rules)) {
|
|
if (key.includes('*')) {
|
|
this.patternMatch.push(new StatusRule(key, rules[key], key));
|
|
} else {
|
|
this.exactMatch[key] = new StatusRule(key, rules[key]);
|
|
}
|
|
}
|
|
}
|
|
|
|
match(file) {
|
|
const result = [];
|
|
const exact = this.exactMatch[file];
|
|
if (exact) {
|
|
result.push(exact);
|
|
}
|
|
for (const item of this.patternMatch) {
|
|
if (item.pattern.test(file)) {
|
|
result.push(item);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
// A specification of WPT test
|
|
class WPTTestSpec {
|
|
/**
|
|
* @param {string} mod name of the WPT module, e.g.
|
|
* 'html/webappapis/microtask-queuing'
|
|
* @param {string} filename path of the test, relative to mod, e.g.
|
|
* 'test.any.js'
|
|
* @param {StatusRule[]} rules
|
|
*/
|
|
constructor(mod, filename, rules) {
|
|
this.module = mod;
|
|
this.filename = filename;
|
|
|
|
this.requires = new Set();
|
|
this.failReasons = [];
|
|
this.skipReasons = [];
|
|
for (const item of rules) {
|
|
if (item.requires.length) {
|
|
for (const req of item.requires) {
|
|
this.requires.add(req);
|
|
}
|
|
}
|
|
if (item.fail) {
|
|
this.failReasons.push(item.fail);
|
|
}
|
|
if (item.skip) {
|
|
this.skipReasons.push(item.skip);
|
|
}
|
|
}
|
|
}
|
|
|
|
getRelativePath() {
|
|
return path.join(this.module, this.filename);
|
|
}
|
|
|
|
getAbsolutePath() {
|
|
return fixtures.path('wpt', this.getRelativePath());
|
|
}
|
|
|
|
getContent() {
|
|
return fs.readFileSync(this.getAbsolutePath(), 'utf8');
|
|
}
|
|
}
|
|
|
|
const kIntlRequirement = {
|
|
none: 0,
|
|
small: 1,
|
|
full: 2,
|
|
// TODO(joyeecheung): we may need to deal with --with-intl=system-icu
|
|
};
|
|
|
|
class IntlRequirement {
|
|
constructor() {
|
|
this.currentIntl = kIntlRequirement.none;
|
|
if (process.config.variables.v8_enable_i18n_support === 0) {
|
|
this.currentIntl = kIntlRequirement.none;
|
|
return;
|
|
}
|
|
// i18n enabled
|
|
if (process.config.variables.icu_small) {
|
|
this.currentIntl = kIntlRequirement.small;
|
|
} else {
|
|
this.currentIntl = kIntlRequirement.full;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @param {Set} requires
|
|
* @returns {string|false} The config that the build is lacking, or false
|
|
*/
|
|
isLacking(requires) {
|
|
const current = this.currentIntl;
|
|
if (requires.has('full-icu') && current !== kIntlRequirement.full) {
|
|
return 'full-icu';
|
|
}
|
|
if (requires.has('small-icu') && current < kIntlRequirement.small) {
|
|
return 'small-icu';
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
|
|
const intlRequirements = new IntlRequirement();
|
|
|
|
|
|
class StatusLoader {
|
|
/**
|
|
* @param {string} path relative path of the WPT subset
|
|
*/
|
|
constructor(path) {
|
|
this.path = path;
|
|
this.loaded = false;
|
|
this.rules = new StatusRuleSet();
|
|
/** @type {WPTTestSpec[]} */
|
|
this.specs = [];
|
|
}
|
|
|
|
/**
|
|
* Grep for all .*.js file recursively in a directory.
|
|
* @param {string} dir
|
|
*/
|
|
grep(dir) {
|
|
let result = [];
|
|
const list = fs.readdirSync(dir);
|
|
for (const file of list) {
|
|
const filepath = path.join(dir, file);
|
|
const stat = fs.statSync(filepath);
|
|
if (stat.isDirectory()) {
|
|
const list = this.grep(filepath);
|
|
result = result.concat(list);
|
|
} else {
|
|
if (!(/\.\w+\.js$/.test(filepath))) {
|
|
continue;
|
|
}
|
|
result.push(filepath);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
load() {
|
|
const dir = path.join(__dirname, '..', 'wpt');
|
|
const statusFile = path.join(dir, 'status', `${this.path}.json`);
|
|
const result = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
|
|
this.rules.addRules(result);
|
|
|
|
const subDir = fixtures.path('wpt', this.path);
|
|
const list = this.grep(subDir);
|
|
for (const file of list) {
|
|
const relativePath = path.relative(subDir, file);
|
|
const match = this.rules.match(relativePath);
|
|
this.specs.push(new WPTTestSpec(this.path, relativePath, match));
|
|
}
|
|
this.loaded = true;
|
|
}
|
|
}
|
|
|
|
const kPass = 'pass';
|
|
const kFail = 'fail';
|
|
const kSkip = 'skip';
|
|
const kTimeout = 'timeout';
|
|
const kIncomplete = 'incomplete';
|
|
const kUncaught = 'uncaught';
|
|
const NODE_UNCAUGHT = 100;
|
|
|
|
class WPTRunner {
|
|
constructor(path) {
|
|
this.path = path;
|
|
this.resource = new ResourceLoader(path);
|
|
this.sandbox = null;
|
|
this.context = null;
|
|
|
|
this.globals = new Map();
|
|
|
|
this.status = new StatusLoader(path);
|
|
this.status.load();
|
|
this.specMap = new Map(
|
|
this.status.specs.map((item) => [item.filename, item])
|
|
);
|
|
|
|
this.results = {};
|
|
this.inProgress = new Set();
|
|
this.unexpectedFailures = [];
|
|
}
|
|
|
|
/**
|
|
* Specify that certain global descriptors from the object
|
|
* should be defined in the vm
|
|
* @param {object} obj
|
|
* @param {string[]} names
|
|
*/
|
|
copyGlobalsFromObject(obj, names) {
|
|
for (const name of names) {
|
|
const desc = Object.getOwnPropertyDescriptor(obj, name);
|
|
if (!desc) {
|
|
assert.fail(`${name} does not exist on the object`);
|
|
}
|
|
this.globals.set(name, desc);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Specify that certain global descriptors should be defined in the vm
|
|
* @param {string} name
|
|
* @param {object} descriptor
|
|
*/
|
|
defineGlobal(name, descriptor) {
|
|
this.globals.set(name, descriptor);
|
|
}
|
|
|
|
// TODO(joyeecheung): work with the upstream to port more tests in .html
|
|
// to .js.
|
|
runJsTests() {
|
|
let queue = [];
|
|
|
|
// If the tests are run as `node test/wpt/test-something.js subset.any.js`,
|
|
// only `subset.any.js` will be run by the runner.
|
|
if (process.argv[2]) {
|
|
const filename = process.argv[2];
|
|
if (!this.specMap.has(filename)) {
|
|
throw new Error(`${filename} not found!`);
|
|
}
|
|
queue.push(this.specMap.get(filename));
|
|
} else {
|
|
queue = this.buildQueue();
|
|
}
|
|
|
|
this.inProgress = new Set(queue.map((spec) => spec.filename));
|
|
|
|
for (const spec of queue) {
|
|
const testFileName = spec.filename;
|
|
const content = spec.getContent();
|
|
const meta = spec.title = this.getMeta(content);
|
|
|
|
const absolutePath = spec.getAbsolutePath();
|
|
const context = this.generateContext(spec);
|
|
const relativePath = spec.getRelativePath();
|
|
const scriptsToRun = [];
|
|
// Scripts specified with the `// META: script=` header
|
|
if (meta.script) {
|
|
for (const script of meta.script) {
|
|
scriptsToRun.push({
|
|
filename: this.resource.toRealFilePath(relativePath, script),
|
|
code: this.resource.read(relativePath, script, false)
|
|
});
|
|
}
|
|
}
|
|
// The actual test
|
|
scriptsToRun.push({
|
|
code: content,
|
|
filename: absolutePath
|
|
});
|
|
|
|
for (const { code, filename } of scriptsToRun) {
|
|
try {
|
|
vm.runInContext(code, context, { filename });
|
|
} catch (err) {
|
|
this.fail(
|
|
testFileName,
|
|
{
|
|
status: NODE_UNCAUGHT,
|
|
name: 'evaluation in WPTRunner.runJsTests()',
|
|
message: err.message,
|
|
stack: inspect(err)
|
|
},
|
|
kUncaught
|
|
);
|
|
this.inProgress.delete(filename);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
process.on('exit', () => {
|
|
const total = this.specMap.size;
|
|
if (this.inProgress.size > 0) {
|
|
for (const filename of this.inProgress) {
|
|
this.fail(filename, { name: 'Unknown' }, kIncomplete);
|
|
}
|
|
}
|
|
inspect.defaultOptions.depth = Infinity;
|
|
console.log(this.results);
|
|
|
|
const failures = [];
|
|
let expectedFailures = 0;
|
|
let skipped = 0;
|
|
for (const key of Object.keys(this.results)) {
|
|
const item = this.results[key];
|
|
if (item.fail && item.fail.unexpected) {
|
|
failures.push(key);
|
|
}
|
|
if (item.fail && item.fail.expected) {
|
|
expectedFailures++;
|
|
}
|
|
if (item.skip) {
|
|
skipped++;
|
|
}
|
|
}
|
|
const ran = total - skipped;
|
|
const passed = ran - expectedFailures - failures.length;
|
|
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
|
|
`${passed} passed, ${expectedFailures} expected failures,`,
|
|
`${failures.length} unexpected failures`);
|
|
if (failures.length > 0) {
|
|
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
|
|
throw new Error(
|
|
`Found ${failures.length} unexpected failures. ` +
|
|
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
|
|
}
|
|
});
|
|
}
|
|
|
|
mock(testfile) {
|
|
const resource = this.resource;
|
|
const result = {
|
|
// This is a mock, because at the moment fetch is not implemented
|
|
// in Node.js, but some tests and harness depend on this to pull
|
|
// resources.
|
|
fetch(file) {
|
|
return resource.read(testfile, file, true);
|
|
},
|
|
GLOBAL: {
|
|
isWindow() { return false; }
|
|
},
|
|
Object
|
|
};
|
|
|
|
return result;
|
|
}
|
|
|
|
// Note: this is how our global space for the WPT test should look like
|
|
getSandbox(filename) {
|
|
const result = this.mock(filename);
|
|
for (const [name, desc] of this.globals) {
|
|
Object.defineProperty(result, name, desc);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
generateContext(test) {
|
|
const filename = test.filename;
|
|
const sandbox = this.sandbox = this.getSandbox(test.getRelativePath());
|
|
const context = this.context = vm.createContext(sandbox);
|
|
|
|
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
|
|
const harness = fs.readFileSync(harnessPath, 'utf8');
|
|
vm.runInContext(harness, context, {
|
|
filename: harnessPath
|
|
});
|
|
|
|
sandbox.add_result_callback(
|
|
this.resultCallback.bind(this, filename)
|
|
);
|
|
sandbox.add_completion_callback(
|
|
this.completionCallback.bind(this, filename)
|
|
);
|
|
sandbox.self = sandbox;
|
|
// TODO(joyeecheung): we are not a window - work with the upstream to
|
|
// add a new scope for us.
|
|
return context;
|
|
}
|
|
|
|
getTestTitle(filename) {
|
|
const spec = this.specMap.get(filename);
|
|
const title = spec.meta && spec.meta.title;
|
|
return title ? `${filename} : ${title}` : filename;
|
|
}
|
|
|
|
// Map WPT test status to strings
|
|
getTestStatus(status) {
|
|
switch (status) {
|
|
case 1:
|
|
return kFail;
|
|
case 2:
|
|
return kTimeout;
|
|
case 3:
|
|
return kIncomplete;
|
|
case NODE_UNCAUGHT:
|
|
return kUncaught;
|
|
default:
|
|
return kPass;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Report the status of each specific test case (there could be multiple
|
|
* in one test file).
|
|
*
|
|
* @param {string} filename
|
|
* @param {Test} test The Test object returned by WPT harness
|
|
*/
|
|
resultCallback(filename, test) {
|
|
const status = this.getTestStatus(test.status);
|
|
const title = this.getTestTitle(filename);
|
|
console.log(`---- ${title} ----`);
|
|
if (status !== kPass) {
|
|
this.fail(filename, test, status);
|
|
} else {
|
|
this.succeed(filename, test, status);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Report the status of each WPT test (one per file)
|
|
*
|
|
* @param {string} filename
|
|
* @param {Test[]} test The Test objects returned by WPT harness
|
|
*/
|
|
completionCallback(filename, tests, harnessStatus) {
|
|
// Treat it like a test case failure
|
|
if (harnessStatus.status === 2) {
|
|
const title = this.getTestTitle(filename);
|
|
console.log(`---- ${title} ----`);
|
|
this.resultCallback(filename, { status: 2, name: 'Unknown' });
|
|
}
|
|
this.inProgress.delete(filename);
|
|
}
|
|
|
|
addTestResult(filename, item) {
|
|
let result = this.results[filename];
|
|
if (!result) {
|
|
result = this.results[filename] = {};
|
|
}
|
|
if (item.status === kSkip) {
|
|
// { filename: { skip: 'reason' } }
|
|
result[kSkip] = item.reason;
|
|
} else {
|
|
// { filename: { fail: { expected: [ ... ],
|
|
// unexpected: [ ... ] } }}
|
|
if (!result[item.status]) {
|
|
result[item.status] = {};
|
|
}
|
|
const key = item.expected ? 'expected' : 'unexpected';
|
|
if (!result[item.status][key]) {
|
|
result[item.status][key] = [];
|
|
}
|
|
if (result[item.status][key].indexOf(item.reason) === -1) {
|
|
result[item.status][key].push(item.reason);
|
|
}
|
|
}
|
|
}
|
|
|
|
succeed(filename, test, status) {
|
|
console.log(`[${status.toUpperCase()}] ${test.name}`);
|
|
}
|
|
|
|
fail(filename, test, status) {
|
|
const spec = this.specMap.get(filename);
|
|
const expected = !!(spec.failReasons.length);
|
|
if (expected) {
|
|
console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
|
|
console.log(spec.failReasons.join('; '));
|
|
} else {
|
|
console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
|
|
}
|
|
if (status === kFail || status === kUncaught) {
|
|
console.log(test.message);
|
|
console.log(test.stack);
|
|
}
|
|
const command = `${process.execPath} ${process.execArgv}` +
|
|
` ${require.main.filename} ${filename}`;
|
|
console.log(`Command: ${command}\n`);
|
|
this.addTestResult(filename, {
|
|
expected,
|
|
status: kFail,
|
|
reason: test.message || status
|
|
});
|
|
}
|
|
|
|
skip(filename, reasons) {
|
|
const title = this.getTestTitle(filename);
|
|
console.log(`---- ${title} ----`);
|
|
const joinedReasons = reasons.join('; ');
|
|
console.log(`[SKIPPED] ${joinedReasons}`);
|
|
this.addTestResult(filename, {
|
|
status: kSkip,
|
|
reason: joinedReasons
|
|
});
|
|
}
|
|
|
|
getMeta(code) {
|
|
const matches = code.match(/\/\/ META: .+/g);
|
|
if (!matches) {
|
|
return {};
|
|
} else {
|
|
const result = {};
|
|
for (const match of matches) {
|
|
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
|
|
const key = parts[1];
|
|
const value = parts[2];
|
|
if (key === 'script') {
|
|
if (result[key]) {
|
|
result[key].push(value);
|
|
} else {
|
|
result[key] = [value];
|
|
}
|
|
} else {
|
|
result[key] = value;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
|
|
buildQueue() {
|
|
const queue = [];
|
|
for (const spec of this.specMap.values()) {
|
|
const filename = spec.filename;
|
|
if (spec.skipReasons.length > 0) {
|
|
this.skip(filename, spec.skipReasons);
|
|
continue;
|
|
}
|
|
|
|
const lackingIntl = intlRequirements.isLacking(spec.requires);
|
|
if (lackingIntl) {
|
|
this.skip(filename, [ `requires ${lackingIntl}` ]);
|
|
continue;
|
|
}
|
|
|
|
queue.push(spec);
|
|
}
|
|
return queue;
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
harness: harnessMock,
|
|
WPTRunner
|
|
};
|