Skip to content

Commit 78bf10a

Browse files
rbucktonrbuckton
authored
Merge pull request microsoft#25004 from Microsoft/trackFailedTests
Adds failed test tracking
2 parents 72a00b1 + d3ef20c commit 78bf10a

9 files changed

Lines changed: 308 additions & 23 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,4 +73,5 @@ tests/cases/user/*/**/*.d.ts
7373
!tests/cases/user/zone.js/
7474
!tests/cases/user/bignumber.js/
7575
!tests/cases/user/discord.js/
76-
tests/baselines/reference/dt
76+
tests/baselines/reference/dt
77+
.failed-tests

‎Jakefile.js‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,8 @@ function runConsoleTests(defaultReporter, runInParallel) {
410410
const runners = process.env.runners || process.env.runner || process.env.ru;
411411
const tests = process.env.test || process.env.tests || process.env.t;
412412
const light = process.env.light === undefined || process.env.light !== "false";
413+
const failed = process.env.failed;
414+
const keepFailed = process.env.keepFailed || failed;
413415
const stackTraceLimit = process.env.stackTraceLimit;
414416
const colorsFlag = process.env.color || process.env.colors;
415417
const colors = colorsFlag !== "false" && colorsFlag !== "0";
@@ -440,16 +442,17 @@ function runConsoleTests(defaultReporter, runInParallel) {
440442
testTimeout = 800000;
441443
}
442444

443-
if (tests || runners || light || testTimeout || taskConfigsFolder) {
444-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout);
445+
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
446+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed);
445447
}
446448

447449
// timeout normally isn't necessary but Travis-CI has been timing out on compiler baselines occasionally
448450
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
449451
if (!runInParallel) {
450452
var startTime = Travis.mark();
451453
var args = [];
452-
args.push("-R", reporter);
454+
args.push("-R", "scripts/failed-tests");
455+
args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
453456
if (tests) args.push("-g", `"${tests}"`);
454457
args.push(colors ? "--colors" : "--no-colors");
455458
if (bail) args.push("--bail");
@@ -460,7 +463,14 @@ function runConsoleTests(defaultReporter, runInParallel) {
460463
}
461464
args.push(Paths.builtLocalRun);
462465

463-
var cmd = "mocha " + args.join(" ");
466+
var cmd;
467+
if (failed) {
468+
args.unshift("scripts/run-failed-tests.js");
469+
cmd = host + " " + args.join(" ");
470+
}
471+
else {
472+
cmd = "mocha " + args.join(" ");
473+
}
464474
var savedNodeEnv = process.env.NODE_ENV;
465475
process.env.NODE_ENV = "development";
466476
exec(cmd, function () {
@@ -521,7 +531,7 @@ function runConsoleTests(defaultReporter, runInParallel) {
521531
}
522532

523533
// used to pass data from jake command line directly to run.js
524-
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout) {
534+
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, colors, testTimeout, keepFailed) {
525535
var testConfigContents = JSON.stringify({
526536
runners: runners ? runners.split(",") : undefined,
527537
test: tests ? [tests] : undefined,
@@ -530,7 +540,8 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
530540
taskConfigsFolder: taskConfigsFolder,
531541
stackTraceLimit: stackTraceLimit,
532542
noColor: !colors,
533-
timeout: testTimeout
543+
timeout: testTimeout,
544+
keepFailed: keepFailed
534545
});
535546
fs.writeFileSync('test.config', testConfigContents, { encoding: "utf-8" });
536547
}

‎scripts/build/options.js‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ const os = require("os");
44

55
/** @type {CommandLineOptions} */
66
module.exports = minimist(process.argv.slice(2), {
7-
boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix"],
7+
boolean: ["debug", "inspect", "light", "colors", "lint", "soft", "fix", "failed", "keepFailed"],
88
string: ["browser", "tests", "host", "reporter", "stackTraceLimit", "timeout"],
99
alias: {
1010
"b": "browser",
@@ -32,6 +32,8 @@ module.exports = minimist(process.argv.slice(2), {
3232
lint: process.env.lint || true,
3333
fix: process.env.fix || process.env.f,
3434
workers: process.env.workerCount || os.cpus().length,
35+
failed: false,
36+
keepFailed: false
3537
}
3638
});
3739

@@ -52,6 +54,8 @@ module.exports = minimist(process.argv.slice(2), {
5254
* @property {string} reporter
5355
* @property {string} stackTraceLimit
5456
* @property {string|number} timeout
57+
* @property {boolean} failed
58+
* @property {boolean} keepFailed
5559
*
5660
* @typedef {import("minimist").ParsedArgs & TypedOptions} CommandLineOptions
5761
*/

‎scripts/build/tests.js‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,16 @@ exports.localTest262Baseline = "internal/baselines/test262/local";
2727
*/
2828
function runConsoleTests(runJs, defaultReporter, runInParallel) {
2929
let testTimeout = cmdLineOptions.timeout;
30+
let tests = cmdLineOptions.tests;
3031
const lintFlag = cmdLineOptions.lint;
3132
const debug = cmdLineOptions.debug;
3233
const inspect = cmdLineOptions.inspect;
33-
const tests = cmdLineOptions.tests;
3434
const runners = cmdLineOptions.runners;
3535
const light = cmdLineOptions.light;
3636
const stackTraceLimit = cmdLineOptions.stackTraceLimit;
3737
const testConfigFile = "test.config";
38+
const failed = cmdLineOptions.failed;
39+
const keepFailed = cmdLineOptions.keepFailed || failed;
3840
return cleanTestDirs()
3941
.then(() => {
4042
if (fs.existsSync(testConfigFile)) {
@@ -59,8 +61,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
5961
testTimeout = 400000;
6062
}
6163

62-
if (tests || runners || light || testTimeout || taskConfigsFolder) {
63-
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout);
64+
if (tests || runners || light || testTimeout || taskConfigsFolder || keepFailed) {
65+
writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, testTimeout, keepFailed);
6466
}
6567

6668
const colors = cmdLineOptions.colors;
@@ -75,7 +77,8 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
7577
// timeout normally isn"t necessary but Travis-CI has been timing out on compiler baselines occasionally
7678
// default timeout is 2sec which really should be enough, but maybe we just need a small amount longer
7779
if (!runInParallel) {
78-
args.push("-R", reporter);
80+
args.push("-R", "scripts/failed-tests");
81+
args.push("-O", '"reporter=' + reporter + (keepFailed ? ",keepFailed=true" : "") + '"');
7982
if (tests) {
8083
args.push("-g", `"${tests}"`);
8184
}
@@ -103,7 +106,12 @@ function runConsoleTests(runJs, defaultReporter, runInParallel) {
103106
args.push(runJs);
104107
}
105108
setNodeEnvToDevelopment();
106-
return exec(host, [runJs]);
109+
if (failed) {
110+
return exec(host, ["scripts/run-failed-tests.js"].concat(args));
111+
}
112+
else {
113+
return exec(host, args);
114+
}
107115
})
108116
.then(({ exitCode }) => {
109117
if (exitCode !== 0) return finish(undefined, exitCode);
@@ -148,8 +156,9 @@ exports.cleanTestDirs = cleanTestDirs;
148156
* @param {string | number} [workerCount]
149157
* @param {string} [stackTraceLimit]
150158
* @param {string | number} [timeout]
159+
* @param {boolean} [keepFailed]
151160
*/
152-
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout) {
161+
function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCount, stackTraceLimit, timeout, keepFailed) {
153162
const testConfigContents = JSON.stringify({
154163
test: tests ? [tests] : undefined,
155164
runner: runners ? runners.split(",") : undefined,
@@ -159,6 +168,7 @@ function writeTestConfigFile(tests, runners, light, taskConfigsFolder, workerCou
159168
taskConfigsFolder,
160169
noColor: !cmdLineOptions.colors,
161170
timeout,
171+
keepFailed
162172
});
163173
log.info("Running tests with config: " + testConfigContents);
164174
fs.writeFileSync("test.config", testConfigContents);

‎scripts/failed-tests.d.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import Mocha = require("mocha");
2+
3+
export = FailedTestsReporter;
4+
5+
declare class FailedTestsReporter extends Mocha.reporters.Base {
6+
passes: Mocha.Test[];
7+
failures: Mocha.Test[];
8+
reporterOptions: FailedTestsReporter.ReporterOptions;
9+
reporter?: Mocha.reporters.Base;
10+
constructor(runner: Mocha.Runner, options?: { reporterOptions?: FailedTestsReporter.ReporterOptions });
11+
static writeFailures(file: string, passes: ReadonlyArray<Mocha.Test>, failures: ReadonlyArray<Mocha.Test>, keepFailed: boolean, done: (err?: NodeJS.ErrnoException) => void): void;
12+
done(failures: number, fn?: (failures: number) => void): void;
13+
}
14+
15+
declare namespace FailedTestsReporter {
16+
interface ReporterOptions {
17+
file?: string;
18+
keepFailed?: boolean;
19+
reporter?: string | Mocha.ReporterConstructor;
20+
reporterOptions?: any;
21+
}
22+
}

‎scripts/failed-tests.js‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// @ts-check
2+
const Mocha = require("mocha");
3+
const path = require("path");
4+
const fs = require("fs");
5+
const os = require("os");
6+
7+
/**
8+
* .failed-tests reporter
9+
*
10+
* @typedef {Object} ReporterOptions
11+
* @property {string} [file]
12+
* @property {boolean} [keepFailed]
13+
* @property {string|Mocha.ReporterConstructor} [reporter]
14+
* @property {*} [reporterOptions]
15+
*/
16+
class FailedTestsReporter extends Mocha.reporters.Base {
17+
/**
18+
* @param {Mocha.Runner} runner
19+
* @param {{ reporterOptions?: ReporterOptions }} [options]
20+
*/
21+
constructor(runner, options) {
22+
super(runner, options);
23+
if (!runner) return;
24+
25+
const reporterOptions = this.reporterOptions = options.reporterOptions || {};
26+
if (reporterOptions.file === undefined) reporterOptions.file = ".failed-tests";
27+
if (reporterOptions.keepFailed === undefined) reporterOptions.keepFailed = false;
28+
if (reporterOptions.reporter) {
29+
/** @type {Mocha.ReporterConstructor} */
30+
let reporter;
31+
if (typeof reporterOptions.reporter === "function") {
32+
reporter = reporterOptions.reporter;
33+
}
34+
else if (Mocha.reporters[reporterOptions.reporter]) {
35+
reporter = Mocha.reporters[reporterOptions.reporter];
36+
}
37+
else {
38+
try {
39+
reporter = require(reporterOptions.reporter);
40+
}
41+
catch (_) {
42+
reporter = require(path.resolve(process.cwd(), reporterOptions.reporter));
43+
}
44+
}
45+
46+
const newOptions = Object.assign({}, options, { reporterOptions: reporterOptions.reporterOptions || {} });
47+
this.reporter = new reporter(runner, newOptions);
48+
}
49+
50+
/** @type {Mocha.Test[]} */
51+
this.passes = [];
52+
53+
/** @type {Mocha.Test[]} */
54+
this.failures = [];
55+
56+
runner.on("pass", test => this.passes.push(test));
57+
runner.on("fail", test => this.failures.push(test));
58+
}
59+
60+
/**
61+
* @param {string} file
62+
* @param {ReadonlyArray<Mocha.Test>} passes
63+
* @param {ReadonlyArray<Mocha.Test>} failures
64+
* @param {boolean} keepFailed
65+
* @param {(err?: NodeJS.ErrnoException) => void} done
66+
*/
67+
static writeFailures(file, passes, failures, keepFailed, done) {
68+
const failingTests = new Set(fs.existsSync(file) ? readTests() : undefined);
69+
if (failingTests.size > 0) {
70+
for (const test of passes) {
71+
const title = test.fullTitle().trim();
72+
if (title) failingTests.delete(title);
73+
}
74+
}
75+
for (const test of failures) {
76+
const title = test.fullTitle().trim();
77+
if (title) failingTests.add(title);
78+
}
79+
if (failingTests.size > 0) {
80+
const failed = Array.from(failingTests).join(os.EOL);
81+
fs.writeFile(file, failed, "utf8", done);
82+
}
83+
else if (!keepFailed) {
84+
fs.unlink(file, done);
85+
}
86+
else {
87+
done();
88+
}
89+
90+
function readTests() {
91+
return fs.readFileSync(file, "utf8")
92+
.split(/\r?\n/g)
93+
.map(line => line.trim())
94+
.filter(line => line.length > 0);
95+
}
96+
}
97+
98+
/**
99+
* @param {number} failures
100+
* @param {(failures: number) => void} [fn]
101+
*/
102+
done(failures, fn) {
103+
FailedTestsReporter.writeFailures(this.reporterOptions.file, this.passes, this.failures, this.reporterOptions.keepFailed || this.stats.tests === 0, (err) => {
104+
const reporter = this.reporter;
105+
if (reporter && reporter.done) {
106+
reporter.done(failures, fn);
107+
}
108+
else if (fn) {
109+
fn(failures);
110+
}
111+
112+
if (err) console.error(err);
113+
});
114+
}
115+
}
116+
117+
module.exports = FailedTestsReporter;

0 commit comments

Comments
 (0)