add support for running on tests on multiple browsers

src/compile-testcases.ts

@@ -27,23 +27,35 @@ import { createPassUnwrappedFunctionsTransformer } from './experiments/passUnwra
 import { replaceVDomNode } from './experiments/correctVirtualDom';
 import { inlineNumberToString } from './experiments/inlineNumberToString';
 
+
+type Options = {
+  compile: boolean,
+  minify: boolean,
+  gzip: boolean
+}
+
 export const compileAndTransform = async (
   dir: string,
   file: string,
-  options: Transforms
+  options: Options,
+  transforms: Transforms
 ): Promise<{}> => {
   // Compile examples in `testcases/*` folder as js
   // Run whatever transformations we want on them, saving steps as `elm.{transformation}.js`
-  compileSync([file], {
-    output: 'output/elm.js',
-    cwd: dir,
-  });
 
-  compileSync([file], {
-    output: 'output/elm.opt.js',
-    cwd: dir,
-    optimize: true,
-  });
+  if (options.compile) {
+    compileSync([file], {
+      output: 'output/elm.js',
+      cwd: dir,
+    });
+
+    compileSync([file], {
+      output: 'output/elm.opt.js',
+      cwd: dir,
+      optimize: true,
+    });
+  }
+
 
   const pathInOutput = (p: string) => path.join(dir, 'output', p);
 
@@ -68,38 +80,38 @@ export const compileAndTransform = async (
   );
 
   // We have to ensure that this transformation takes place before everything else
-  if (options.replaceVDomNode) {
+  if (transforms.replaceVDomNode) {
     const results = ts.transform(source, [replaceVDomNode()]);
     source = results.transformed[0];
   }
 
   let inlineCtx: InlineContext | undefined;
   const transformations: any[] = removeDisabled([
-    // [options.replaceVDomNode, replaceVDomNode()],
+    // [transforms.replaceVDomNode, replaceVDomNode()],
 
-    [options.variantShapes, normalizeVariantShapes],
+    [transforms.variantShapes, normalizeVariantShapes],
     [
-      options.inlineFunctions,
+      transforms.inlineFunctions,
       createFunctionInlineTransformer(ctx => {
         inlineCtx = ctx;
         reportInlineTransformResult(ctx);
       }),
     ],
-    [options.inlineEquality, inlineEquality()],
-    [options.inlineNumberToString, inlineNumberToString()],
+    [transforms.inlineEquality, inlineEquality()],
+    [transforms.inlineNumberToString, inlineNumberToString()],
     [
-      options.listLiterals,
+      transforms.listLiterals,
       createInlineListFromArrayTransformer(
         InlineMode.UsingLiteralObjects(Mode.Prod)
       ),
     ],
     [
-      options.passUnwrappedFunctions,
+      transforms.passUnwrappedFunctions,
       createPassUnwrappedFunctionsTransformer(() => inlineCtx),
     ],
-    includeObjectUpdate(options.objectUpdate),
-    [options.arrowFns, convertFunctionExpressionsToArrowFuncs],
-    [options.unusedValues, createRemoveUnusedLocalsTransform()],
+    includeObjectUpdate(transforms.objectUpdate),
+    [transforms.arrowFns, convertFunctionExpressionsToArrowFuncs],
+    [transforms.unusedValues, createRemoveUnusedLocalsTransform()],
   ]);
 
   const {
@@ -123,7 +135,7 @@ export const compileAndTransform = async (
   fs.writeFileSync(pathInOutput('elm.opt.js'), printer.printFile(initialJs));
 
   // Prepack, minify, and gzip
-  if (options.prepack) {
+  if (transforms.prepack) {
     const { code } = prepackFileSync([pathInOutput('elm.opt.transformed.js')], {
       debugNames: true,
       inlineExpressions: true,
@@ -131,20 +143,28 @@ export const compileAndTransform = async (
     });
 
     fs.writeFileSync(pathInOutput('elm.opt.prepack.js'), code);
-    await minify(
-      pathInOutput('elm.opt.prepack.js'),
-      pathInOutput('elm.opt.prepack.min.js')
-    );
-    gzip(pathInOutput('elm.opt.prepack.min.js'));
+    if (options.minify) {
+      await minify(
+        pathInOutput('elm.opt.prepack.js'),
+        pathInOutput('elm.opt.prepack.min.js')
+      );
+    }
+    if (options.gzip) {
+      gzip(pathInOutput('elm.opt.prepack.min.js'));
+    }
   }
 
-  await minify(pathInOutput('elm.opt.js'), pathInOutput('elm.opt.min.js'));
-  gzip(pathInOutput('elm.opt.min.js'));
-  await minify(
-    pathInOutput('elm.opt.transformed.js'),
-    pathInOutput('elm.opt.transformed.min.js')
-  );
-  gzip(pathInOutput('elm.opt.transformed.min.js'));
+  if (options.minify) {
+    await minify(pathInOutput('elm.opt.js'), pathInOutput('elm.opt.min.js'));
+    await minify(
+      pathInOutput('elm.opt.transformed.js'),
+      pathInOutput('elm.opt.transformed.min.js')
+    );
+  }
+  if (options.gzip) {
+    gzip(pathInOutput('elm.opt.min.js'));
+    gzip(pathInOutput('elm.opt.transformed.min.js'));
+  }
 
   return {};
 };

src/generate-report.ts

@@ -5,7 +5,7 @@ Compiles all the test cases and runs them via webdriver to summarize the results
 
 */
 
-import { ObjectUpdate, Transforms } from './types';
+import { ObjectUpdate, Transforms, Browser } from './types';
 import * as Reporting from './reporting';
 
 const defaultOptions: Transforms = {
@@ -36,44 +36,57 @@ const test: Transforms = {
   unusedValues: false,
 };
 
+const options = {
+  compile: true,
+  gzip: false,
+  minify: false,
+  assetSizes: false,
+  runBenchmark: [{
+    browser: Browser.Chrome,
+    headless: false
+  }],
+  transforms: defaultOptions
+}
+
+
 async function go() {
-  const report = await Reporting.run([
+
+  const report = await Reporting.run(options, [
     // Use `runWithBreakdown` if you want the breakdown
     // const report = await Reporting.runWithBreakdown([
     // { name: 'simple',
     //   dir: 'testcases/simple',
     //   elmFile: 'main',
-    //   options: defaultOptions,
+    //   
     // },
     // {
     //   name: 'bench',
     //   dir: 'testcases/bench',
     //   elmFile: 'Main.elm',
-    //   options: defaultOptions,
+    //   
     // },
     // {
     //   name: 'html',
     //   dir: 'testcases/html',
     //   elmFile: 'Main.elm',
-    //   options: defaultOptions,
+    //   
     // },
     // {
     //   name: 'elm-ui',
     //   dir: 'testcases/elm-ui',
     //   elmFile: 'Main.elm',
-    //   options: defaultOptions,
+    //   
     // },
     {
       name: 'elm-ui-2',
       dir: 'testcases/elm-ui-2',
       elmFile: 'Main.elm',
-      options: defaultOptions,
     },
     // {
     //   name: 'elm-markdown',
     //   dir: 'testcases/elm-markdown',
     //   elmFile: 'Run.elm',
-    //   options: defaultOptions,
+    //   
     // },
     // {
     //   name: 'elm-markdown',
@@ -86,7 +99,7 @@ async function go() {
     //   name: 'elm-obj-file',
     //   dir: 'testcases/elm-obj-file',
     //   elmFile: 'Run.elm',
-    //   options: defaultOptions,
+    //   
     // },
   ]);
 

src/reporting.ts

@@ -1,7 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as Compile from './compile-testcases';
-import { Transforms, ObjectUpdate } from './types';
+import { Browser, Transforms, ObjectUpdate, RunTestcaseOptions } from './types';
 import * as Visit from './visit';
 
 export interface Stat {
@@ -12,11 +12,11 @@ export interface Stat {
 // Asset Sizes
 export const assetSizeStats = (dir: string): Stat[] => {
   let stats: Stat[] = [];
-  fs.readdir(dir, function(err, files) {
+  fs.readdir(dir, function (err, files) {
     if (err) {
       console.log('Error getting directory information.');
     } else {
-      files.forEach(function(file) {
+      files.forEach(function (file) {
         const stat = fs.statSync(path.join(dir, file));
         stats.push({
           name: path.basename(file),
@@ -47,12 +47,12 @@ export const markdown = (report: Results): string => {
     report.assets[key].forEach((item: Stat) => {
       buffer.push(
         '    ' +
-          item.name.padEnd(40, ' ') +
-          '' +
-          humanizeNumber(
-            roundToDecimal(1, item.bytes / Math.pow(2, 10))
-          ).padStart(10, ' ') +
-          'kb'
+        item.name.padEnd(40, ' ') +
+        '' +
+        humanizeNumber(
+          roundToDecimal(1, item.bytes / Math.pow(2, 10))
+        ).padStart(10, ' ') +
+        'kb'
       );
     });
     buffer.push('');
@@ -170,11 +170,10 @@ type Testcase = {
   name: string;
   dir: string;
   elmFile: string;
-  options: Transforms;
 };
 
 // Run a list of testcases
-export const run = async function(runnable: Testcase[]) {
+export const run = async function (options: RunTestcaseOptions, runnable: Testcase[]) {
   let results: any[] = [];
   let assets: any = {};
 
@@ -182,24 +181,39 @@ export const run = async function(runnable: Testcase[]) {
     await Compile.compileAndTransform(
       instance.dir,
       instance.elmFile,
-      instance.options
+      {
+        compile: options.compile,
+        minify: options.minify,
+        gzip: options.gzip
+      },
+      options.transforms
     );
-    assets[instance.name] = assetSizeStats(path.join(instance.dir, 'output'));
+    if (options.assetSizes) {
+      assets[instance.name] = assetSizeStats(path.join(instance.dir, 'output'));
+    }
+
+    for (let browser of options.runBenchmark) {
+      results.push(
+        await Visit.benchmark(browser, null, path.join(instance.dir, 'standard.html'))
+      );
+      results.push(
+        await Visit.benchmark(browser,
+          'transformed',
+          path.join(instance.dir, 'transformed.html')
+        )
+      );
+    }
 
-    results.push(
-      await Visit.benchmark(null, path.join(instance.dir, 'standard.html'))
-    );
-    results.push(
-      await Visit.benchmark(
-        'transformed',
-        path.join(instance.dir, 'transformed.html')
-      )
-    );
   }
 
   return { assets: assets, benchmarks: reformat(results) };
 };
 
+
+
+
+
+
 const emptyOpts: Transforms = {
   prepack: false,
   replaceVDomNode: false,
@@ -214,7 +228,7 @@ const emptyOpts: Transforms = {
   unusedValues: false,
 };
 
-const breakdown = function(
+const breakdown = function (
   options: Transforms
 ): { name: string; options: Transforms }[] {
   let transforms: { name: string; options: Transforms }[] = [];
@@ -240,6 +254,11 @@ const breakdown = function(
       name: 'inline equality',
       options: Object.assign({}, emptyOpts, { inlineEquality: true }),
     },
+    {
+      include: options.inlineNumberToString,
+      name: 'inline number-to-string',
+      options: Object.assign({}, emptyOpts, { inlineNumberToString: true }),
+    },
     {
       include: options.arrowFns,
       name: 'arrowize functions',
@@ -273,45 +292,66 @@ const breakdown = function(
 // Run a list of test cases
 // But we'll run each transformation individually to see what the breakdown is.
 // We'll also run a final case with all the requested transformations
-export const runWithBreakdown = async function(runnable: Testcase[]) {
+export const runWithBreakdown = async function (options: RunTestcaseOptions, runnable: Testcase[]) {
   let results: any[] = [];
   let assets: any = {};
 
+  const opts = {
+    browser: Browser.Chrome,
+    headless: false
+  }
+
   for (let instance of runnable) {
     await Compile.compileAndTransform(
       instance.dir,
       instance.elmFile,
-      instance.options
+      {
+        compile: options.compile,
+        minify: options.minify,
+        gzip: options.gzip
+      },
+      options.transforms
     );
     assets[instance.name] = assetSizeStats(path.join(instance.dir, 'output'));
 
-    results.push(
-      await Visit.benchmark(null, path.join(instance.dir, 'standard.html'))
-    );
-    results.push(
-      await Visit.benchmark(
-        'transformed',
-        path.join(instance.dir, 'transformed.html')
-      )
-    );
+    for (let browser of options.runBenchmark) {
+      results.push(
+        await Visit.benchmark(browser, null, path.join(instance.dir, 'standard.html'))
+      );
+      results.push(
+        await Visit.benchmark(browser,
+          'transformed',
+          path.join(instance.dir, 'transformed.html')
+        )
+      );
+    }
 
-    let steps = breakdown(instance.options);
+    let steps = breakdown(options.transforms);
     for (let i in steps) {
       console.log('running', steps[i]);
+
+
       await Compile.compileAndTransform(
         instance.dir,
         instance.elmFile,
+        {
+          compile: false,
+          minify: false,
+          gzip: false
+        },
         steps[i].options
       );
       // TODO: figure out how to capture asset sizes for the breakdown
       // assets[instance.name] = assetSizeStats(path.join(instance.dir, 'output'));
 
-      results.push(
-        await Visit.benchmark(
-          steps[i].name,
-          path.join(instance.dir, 'transformed.html')
-        )
-      );
+      for (let browser of options.runBenchmark) {
+        results.push(
+          await Visit.benchmark(browser,
+            steps[i].name,
+            path.join(instance.dir, 'transformed.html')
+          )
+        );
+      }
     }
   }
 

src/types.ts

@@ -12,6 +12,15 @@ export interface ElmVariant {
   totalTypeSlotCount: number;
 }
 
+export type RunTestcaseOptions = {
+  compile: boolean,
+  gzip: boolean,
+  minify: boolean,
+  assetSizes: boolean,
+  runBenchmark: BrowserOptions[],
+  transforms: Transforms
+}
+
 export type Transforms = {
   prepack: boolean;
   replaceVDomNode: boolean;
@@ -33,3 +42,18 @@ export enum ObjectUpdate {
   InlineAssign = 'inline_assign',
   InlineSpread = 'inline_spread',
 }
+
+
+
+export type BrowserOptions = {
+  browser: Browser,
+  headless: boolean
+}
+
+export enum Browser {
+  Chrome = 'chrome',
+  Firefox = 'firefox',
+  Safari = 'safari',
+  IE = 'ie',
+  Edge = 'edge'
+}

src/visit.ts

@@ -1,12 +1,24 @@
 import * as Webdriver from 'selenium-webdriver';
 import * as chrome from 'selenium-webdriver/chrome';
+import * as firefox from 'selenium-webdriver/firefox';
 import * as Path from 'path';
+import { Browser, BrowserOptions } from './types'
 
-export const benchmark = async (tag: string | null, file: string) => {
-  let driver = new Webdriver.Builder()
-    .forBrowser('chrome')
-    // .setChromeOptions(/* ... */)
-    // .setFirefoxOptions(/* ... */)
+export const benchmark = async (options: BrowserOptions, tag: string | null, file: string) => {
+
+  const firefoxOptions = new firefox.Options();
+  const chromeOptions = new chrome.Options();
+
+
+  if (options.headless) {
+    firefoxOptions.headless();
+    chromeOptions.headless();
+  }
+
+  let driver = await new Webdriver.Builder()
+    .forBrowser(options.browser, 'latest')
+    .setChromeOptions(chromeOptions)
+    .setFirefoxOptions(firefoxOptions)
     .build();
 
   // docs for selenium: