Parameterize test262-runner with the expected counts required (#2228)

Summary:
Release note: Added --expectedCounts parameter to test262-runner so that success can depend on the value of the time-out and the version of the test suite that is used.
Pull Request resolved: https://github.com/facebook/prepack/pull/2228

Differential Revision: D8775828

Pulled By: hermanventer

fbshipit-source-id: 284bdb3526467f634f41a151e3995719af751e49

.circleci/config.yml

@@ -60,7 +60,7 @@ jobs:
             yarn test-sourcemaps
             yarn test-std-in
             yarn test-residual
-            yarn test-test262 --statusFile ~/artifacts/test262-status.txt --timeout 120 --cpuScale 0.25 --verbose
+            yarn test-test262 --expectedCounts 11945,5566,0 --statusFile ~/artifacts/test262-status.txt --timeout 120 --cpuScale 0.25 --verbose
             #yarn test-test262-new --statusFile ~/artifacts/test262-new-status.txt --timeout 120 --verbose
       - store_artifacts:
           path: ~/artifacts/

scripts/test262-runner.js

@@ -235,6 +235,9 @@ class MasterProgramArgs {
   filterString: string;
   singleThreaded: boolean;
   relativeTestPath: string;
+  expectedES5: number;
+  expectedES6: number;
+  expectedTimeouts: number;
 
   constructor(
     verbose: boolean,
@@ -244,7 +247,10 @@ class MasterProgramArgs {
     statusFile: string,
     filterString: string,
     singleThreaded: boolean,
-    relativeTestPath: string
+    relativeTestPath: string,
+    expectedES5: number,
+    expectedES6: number,
+    expectedTimeouts: number
   ) {
     this.verbose = verbose;
     this.timeout = timeout;
@@ -254,6 +260,9 @@ class MasterProgramArgs {
     this.filterString = filterString;
     this.singleThreaded = singleThreaded;
     this.relativeTestPath = relativeTestPath;
+    this.expectedES5 = expectedES5;
+    this.expectedES6 = expectedES6;
+    this.expectedTimeouts = expectedTimeouts;
   }
 }
 
@@ -323,7 +332,9 @@ function usage(): string {
     EOL +
     `[--verbose] [--timeout <number>] [--bailAfter <number>] ` +
     EOL +
-    `[--cpuScale <number>] [--statusFile <string>] [--singleThreaded] [--relativeTestPath <string>]`
+    `[--cpuScale <number>] [--statusFile <string>] [--singleThreaded] [--relativeTestPath <string>]` +
+    EOL +
+    `[--expectedCounts <es5pass,es6pass,timeouts>]`
   );
 }
 
@@ -339,39 +350,54 @@ function masterArgsParse(): MasterProgramArgs {
       bailAfter: Infinity,
       singleThreaded: false,
       relativeTestPath: "/../test/test262",
+      expectedCounts: "11944,5566,2",
     },
   });
   let filterString = parsedArgs._[0];
   if (typeof parsedArgs.verbose !== "boolean") {
     throw new ArgsParseError("verbose must be a boolean (either --verbose or not)");
   }
+  let verbose = parsedArgs.verbose;
   if (typeof parsedArgs.timeout !== "number") {
     throw new ArgsParseError("timeout must be a number (in seconds) (--timeout 10)");
   }
+  let timeout = parsedArgs.timeout;
   if (typeof parsedArgs.bailAfter !== "number") {
     throw new ArgsParseError("bailAfter must be a number (--bailAfter 10)");
   }
+  let bailAfter = parsedArgs.bailAfter;
   if (typeof parsedArgs.cpuScale !== "number") {
     throw new ArgsParseError("cpuScale must be a number (--cpuScale 0.5)");
   }
+  let cpuScale = parsedArgs.cpuScale;
   if (typeof parsedArgs.statusFile !== "string") {
     throw new ArgsParseError("statusFile must be a string (--statusFile file.txt)");
   }
+  let statusFile = parsedArgs.statusFile;
   if (typeof parsedArgs.singleThreaded !== "boolean") {
     throw new ArgsParseError("singleThreaded must be a boolean (either --singleThreaded or not)");
   }
+  let singleThreaded = parsedArgs.singleThreaded;
   if (typeof parsedArgs.relativeTestPath !== "string") {
     throw new ArgsParseError("relativeTestPath must be a string (--relativeTestPath /../test/test262)");
   }
+  let relativeTestPath = parsedArgs.relativeTestPath;
+  if (typeof parsedArgs.expectedCounts !== "string") {
+    throw new ArgsParseError("expectedCounts must be a string (--expectedCounts 11944,5566,2");
+  }
+  let expectedCounts = parsedArgs.expectedCounts.split(",").map(x => Number(x));
   let programArgs = new MasterProgramArgs(
-    parsedArgs.verbose,
-    parsedArgs.timeout,
-    parsedArgs.bailAfter,
-    parsedArgs.cpuScale,
-    parsedArgs.statusFile,
+    verbose,
+    timeout,
+    bailAfter,
+    cpuScale,
+    statusFile,
     filterString,
-    parsedArgs.singleThreaded,
-    parsedArgs.relativeTestPath
+    singleThreaded,
+    relativeTestPath,
+    expectedCounts[0],
+    expectedCounts[1],
+    expectedCounts[2]
   );
   if (programArgs.filterString) {
     // if filterstring is provided, assume that verbosity is desired
@@ -637,7 +663,10 @@ function handleFinished(args: MasterProgramArgs, groups: GroupsMap, earlierNumSk
   }
 
   // exit status
-  if (!args.filterString && (numPassedES5 < 11944 || numPassedES6 < 5566 || numTimeouts > 2)) {
+  if (
+    !args.filterString &&
+    (numPassedES5 < args.expectedES5 || numPassedES6 < args.expectedES6 || numTimeouts > args.expectedTimeouts)
+  ) {
     console.error(chalk.red("Overall failure. Expected more tests to pass!"));
     return 1;
   } else {