mirror of
https://github.com/facebookarchive/prepack.git
synced 2024-09-11 14:46:37 +03:00
Parameterize test262-runner with the expected counts required (#2228)
Summary: Release note: Added --expectedCounts parameter to test262-runner so that success can depend on the value of the time-out and the version of the test suite that is used. Pull Request resolved: https://github.com/facebook/prepack/pull/2228 Differential Revision: D8775828 Pulled By: hermanventer fbshipit-source-id: 284bdb3526467f634f41a151e3995719af751e49
This commit is contained in:
parent
85b70951d9
commit
dd1f18da0c
@ -60,7 +60,7 @@ jobs:
|
||||
yarn test-sourcemaps
|
||||
yarn test-std-in
|
||||
yarn test-residual
|
||||
yarn test-test262 --statusFile ~/artifacts/test262-status.txt --timeout 120 --cpuScale 0.25 --verbose
|
||||
yarn test-test262 --expectedCounts 11945,5566,0 --statusFile ~/artifacts/test262-status.txt --timeout 120 --cpuScale 0.25 --verbose
|
||||
#yarn test-test262-new --statusFile ~/artifacts/test262-new-status.txt --timeout 120 --verbose
|
||||
- store_artifacts:
|
||||
path: ~/artifacts/
|
||||
|
@ -235,6 +235,9 @@ class MasterProgramArgs {
|
||||
filterString: string;
|
||||
singleThreaded: boolean;
|
||||
relativeTestPath: string;
|
||||
expectedES5: number;
|
||||
expectedES6: number;
|
||||
expectedTimeouts: number;
|
||||
|
||||
constructor(
|
||||
verbose: boolean,
|
||||
@ -244,7 +247,10 @@ class MasterProgramArgs {
|
||||
statusFile: string,
|
||||
filterString: string,
|
||||
singleThreaded: boolean,
|
||||
relativeTestPath: string
|
||||
relativeTestPath: string,
|
||||
expectedES5: number,
|
||||
expectedES6: number,
|
||||
expectedTimeouts: number
|
||||
) {
|
||||
this.verbose = verbose;
|
||||
this.timeout = timeout;
|
||||
@ -254,6 +260,9 @@ class MasterProgramArgs {
|
||||
this.filterString = filterString;
|
||||
this.singleThreaded = singleThreaded;
|
||||
this.relativeTestPath = relativeTestPath;
|
||||
this.expectedES5 = expectedES5;
|
||||
this.expectedES6 = expectedES6;
|
||||
this.expectedTimeouts = expectedTimeouts;
|
||||
}
|
||||
}
|
||||
|
||||
@ -323,7 +332,9 @@ function usage(): string {
|
||||
EOL +
|
||||
`[--verbose] [--timeout <number>] [--bailAfter <number>] ` +
|
||||
EOL +
|
||||
`[--cpuScale <number>] [--statusFile <string>] [--singleThreaded] [--relativeTestPath <string>]`
|
||||
`[--cpuScale <number>] [--statusFile <string>] [--singleThreaded] [--relativeTestPath <string>]` +
|
||||
EOL +
|
||||
`[--expectedCounts <es5pass,es6pass,timeouts>]`
|
||||
);
|
||||
}
|
||||
|
||||
@ -339,39 +350,54 @@ function masterArgsParse(): MasterProgramArgs {
|
||||
bailAfter: Infinity,
|
||||
singleThreaded: false,
|
||||
relativeTestPath: "/../test/test262",
|
||||
expectedCounts: "11944,5566,2",
|
||||
},
|
||||
});
|
||||
let filterString = parsedArgs._[0];
|
||||
if (typeof parsedArgs.verbose !== "boolean") {
|
||||
throw new ArgsParseError("verbose must be a boolean (either --verbose or not)");
|
||||
}
|
||||
let verbose = parsedArgs.verbose;
|
||||
if (typeof parsedArgs.timeout !== "number") {
|
||||
throw new ArgsParseError("timeout must be a number (in seconds) (--timeout 10)");
|
||||
}
|
||||
let timeout = parsedArgs.timeout;
|
||||
if (typeof parsedArgs.bailAfter !== "number") {
|
||||
throw new ArgsParseError("bailAfter must be a number (--bailAfter 10)");
|
||||
}
|
||||
let bailAfter = parsedArgs.bailAfter;
|
||||
if (typeof parsedArgs.cpuScale !== "number") {
|
||||
throw new ArgsParseError("cpuScale must be a number (--cpuScale 0.5)");
|
||||
}
|
||||
let cpuScale = parsedArgs.cpuScale;
|
||||
if (typeof parsedArgs.statusFile !== "string") {
|
||||
throw new ArgsParseError("statusFile must be a string (--statusFile file.txt)");
|
||||
}
|
||||
let statusFile = parsedArgs.statusFile;
|
||||
if (typeof parsedArgs.singleThreaded !== "boolean") {
|
||||
throw new ArgsParseError("singleThreaded must be a boolean (either --singleThreaded or not)");
|
||||
}
|
||||
let singleThreaded = parsedArgs.singleThreaded;
|
||||
if (typeof parsedArgs.relativeTestPath !== "string") {
|
||||
throw new ArgsParseError("relativeTestPath must be a string (--relativeTestPath /../test/test262)");
|
||||
}
|
||||
let relativeTestPath = parsedArgs.relativeTestPath;
|
||||
if (typeof parsedArgs.expectedCounts !== "string") {
|
||||
throw new ArgsParseError("expectedCounts must be a string (--expectedCounts 11944,5566,2");
|
||||
}
|
||||
let expectedCounts = parsedArgs.expectedCounts.split(",").map(x => Number(x));
|
||||
let programArgs = new MasterProgramArgs(
|
||||
parsedArgs.verbose,
|
||||
parsedArgs.timeout,
|
||||
parsedArgs.bailAfter,
|
||||
parsedArgs.cpuScale,
|
||||
parsedArgs.statusFile,
|
||||
verbose,
|
||||
timeout,
|
||||
bailAfter,
|
||||
cpuScale,
|
||||
statusFile,
|
||||
filterString,
|
||||
parsedArgs.singleThreaded,
|
||||
parsedArgs.relativeTestPath
|
||||
singleThreaded,
|
||||
relativeTestPath,
|
||||
expectedCounts[0],
|
||||
expectedCounts[1],
|
||||
expectedCounts[2]
|
||||
);
|
||||
if (programArgs.filterString) {
|
||||
// if filterstring is provided, assume that verbosity is desired
|
||||
@ -637,7 +663,10 @@ function handleFinished(args: MasterProgramArgs, groups: GroupsMap, earlierNumSk
|
||||
}
|
||||
|
||||
// exit status
|
||||
if (!args.filterString && (numPassedES5 < 11944 || numPassedES6 < 5566 || numTimeouts > 2)) {
|
||||
if (
|
||||
!args.filterString &&
|
||||
(numPassedES5 < args.expectedES5 || numPassedES6 < args.expectedES6 || numTimeouts > args.expectedTimeouts)
|
||||
) {
|
||||
console.error(chalk.red("Overall failure. Expected more tests to pass!"));
|
||||
return 1;
|
||||
} else {
|
||||
|
Loading…
Reference in New Issue
Block a user