Allow more than one input file for Prepack

Summary:
We usually need at least two files: 1) the environmental model and 2) the bundle to be Prepacked.
In general, there is no good reason to not allow as many files as the caller wants.

To make this not be a breaking chance, I've added a new API for this: prepackSources.

Further complications result from working around a bug in Babel and not increasing the module cycle length for Flow.
Closes https://github.com/facebook/prepack/pull/776

Differential Revision: D5366190

Pulled By: hermanventer

fbshipit-source-id: 74595b1b60e8e8a6d24cb974b5be10210d745266

.gitignore

@@ -7,5 +7,5 @@ npm-debug.log
 build/
 coverage*/
 .vscode
-test/internal
+facebook/test
 prepack.min.js

scripts/generate-sourcemaps-test.js

@@ -69,7 +69,6 @@ function generateTest(name: string, test_path: string, code: string): boolean {
     fs.writeFileSync(name + ".new1.js.map", JSON.stringify(newMap1));
     s = prepackFileSync(name + ".new1.js", {
       compatibility: "node-source-maps",
-      filename: test_path,
       inputSourceMapFilename: name + ".new1.js.map",
       internalDebug: true,
       onError: errorHandler,
@@ -99,6 +98,7 @@ function generateTest(name: string, test_path: string, code: string): boolean {
   console.log(newCode2);
   console.log(chalk.underline("newMap 2"));
   console.log(newMap2);
+
   return false;
 }
 

scripts/test-error-handler.js

@@ -84,6 +84,7 @@ function runTest(name: string, code: string): boolean {
       let expected = expectedErrors[i][prop];
       let actual = (errors[i]: any)[prop];
       if (prop === "location") {
+        delete actual.filename;
         actual = JSON.stringify(actual);
         expected = JSON.stringify(expected);
       }

scripts/test-residual.js

@@ -78,12 +78,13 @@ function exec(code) {
 
 function runTest(name, code) {
   let realmOptions = { residual: true };
+  let sources = [{ filePath: name, fileContents: code }];
   console.log(chalk.inverse(name));
   if (code.includes("// throws introspection error")) {
     try {
       let realm = construct_realm(realmOptions);
       initializeGlobals(realm);
-      let result = realm.$GlobalEnv.executePartialEvaluator(name, code);
+      let result = realm.$GlobalEnv.executePartialEvaluator(sources);
       if (result instanceof IntrospectionThrowCompletion) return true;
       if (result instanceof ThrowCompletion) throw result.value;
     } catch (err) {
@@ -111,7 +112,7 @@ return __result; }).call(this);`);
       for (; i < max; i++) {
         let realm = construct_realm(realmOptions);
         initializeGlobals(realm);
-        let result = realm.$GlobalEnv.executePartialEvaluator(name, code);
+        let result = realm.$GlobalEnv.executePartialEvaluator(sources);
         if (result instanceof ThrowCompletion) throw result.value;
         if (result instanceof AbruptCompletion) throw result;
         let newCode = result.code;

scripts/test-runner.js

@@ -90,7 +90,6 @@ function runTest(name, code, args) {
   let delayUnsupportedRequires = code.includes("// delay unsupported requires");
   let functionCloneCountMatch = code.match(/\/\/ serialized function clone count: (\d+)/);
   let options = {
-    filename: name,
     compatibility,
     speculate,
     delayUnsupportedRequires,
@@ -109,7 +108,8 @@ function runTest(name, code, args) {
       initializeGlobals(realm);
       let serializerOptions = { initializeMoreModules: speculate, delayUnsupportedRequires, internalDebug: true };
       let serializer = new Serializer(realm, serializerOptions);
-      let serialized = serializer.init(name, code, "", false, onError);
+      let sources = [{ filePath: name, fileContents: code }];
+      let serialized = serializer.init(sources, false, onError);
       if (!serialized) {
         console.log(chalk.red("Error during serialization"));
       } else {

src/benchmarker.js

@@ -9,7 +9,7 @@
 
 /* @flow */
 
-import type { Compatibility } from "./types.js";
+import type { Compatibility } from "./options.js";
 import Serializer from "./serializer/index.js";
 import construct_realm from "./construct_realm.js";
 import initializeGlobals from "./globals.js";
@@ -135,7 +135,8 @@ function dump(name: string, raw: string, min: string = raw, compatibility?: "bro
   let realm = construct_realm({ serialize: true, compatibility });
   initializeGlobals(realm);
   let serializer = new Serializer(realm);
-  let serialized = serializer.init(name, raw);
+  let sources = [{ filePath: name, fileContents: raw }];
+  let serialized = serializer.init(sources);
   if (!serialized) {
     process.exit(1);
     invariant(false);

src/construct_realm.js

@@ -12,7 +12,7 @@
 import { Realm } from "./realm.js";
 import { initialize as initializeIntrinsics } from "./intrinsics/index.js";
 import initializeGlobal from "./intrinsics/ecma262/global.js";
-import type { RealmOptions } from "./types.js";
+import type { RealmOptions } from "./options.js";
 import * as evaluators from "./evaluators/index.js";
 import * as partialEvaluators from "./partial-evaluators/index.js";
 import { NewGlobalEnvironment } from "./methods/index.js";

src/environment.js

@@ -11,9 +11,10 @@
 
 import type { BabelNode, BabelNodeFile, BabelNodeStatement } from "babel-types";
 import type { Realm } from "./realm.js";
-import type { SourceMap, SourceType } from "./types.js";
+import type { SourceFile, SourceMap, SourceType } from "./types.js";
 
 import { AbruptCompletion, Completion, JoinedAbruptCompletions, NormalCompletion, PossiblyNormalCompletion, ThrowCompletion } from "./completions.js";
+import { defaultOptions, type Options } from "./options";
 import { ExecutionContext } from "./realm.js";
 import { Value } from "./values/index.js";
 import { AbstractValue, NullValue, SymbolValue, BooleanValue, FunctionValue, NumberValue, ObjectValue, AbstractObjectValue, StringValue, UndefinedValue } from "./values/index.js";
@@ -33,6 +34,7 @@ import {
   IsDataDescriptor,
   ThrowIfMightHaveBeenDeleted,
 } from "./methods/index.js";
+import * as t from "babel-types";
 
 const sourceMap = require('source-map');
 
@@ -1041,6 +1043,7 @@ export class LexicalEnvironment {
       if (onParse) onParse(ast);
       res = this.evaluateCompletion(ast, false);
       if (map.length > 0) this.fixup_source_locations(ast, map);
+      this.fixup_filenames(ast);
     } finally {
       this.realm.popContext(context);
     }
@@ -1050,33 +1053,43 @@ export class LexicalEnvironment {
   }
 
   executePartialEvaluator(
-      filename: string, code: string, sourceMaps: string = "",
-      sourceType: SourceType = "script"): AbruptCompletion | { code: string, map?: SourceMap } {
-    let context = new ExecutionContext();
-    context.lexicalEnvironment = this;
-    context.variableEnvironment = this;
-    context.realm = this.realm;
+    sources: Array<SourceFile>, options: Options = defaultOptions, sourceType: SourceType = "script"
+  ): AbruptCompletion | { code: string, map?: SourceMap } {
+    let body: Array<BabelNodeStatement> = [];
+    let code = {};
+    for (let source of sources) {
+      let context = new ExecutionContext();
+      context.lexicalEnvironment = this;
+      context.variableEnvironment = this;
+      context.realm = this.realm;
 
-    this.realm.pushContext(context);
-
-    try {
-      let ast;
+      this.realm.pushContext(context);
       try {
-        ast = parse(this.realm, code, filename, sourceType);
-      } catch (e) {
-        if (e instanceof ThrowCompletion) return e;
-        throw e;
+        let ast;
+        try {
+          ast = parse(this.realm, source.fileContents, source.filePath, sourceType);
+        } catch (e) {
+          if (e instanceof ThrowCompletion) return e;
+          throw e;
+        }
+        let [res, partialAST] = this.partiallyEvaluateCompletionDeref(ast, false);
+        if (res instanceof AbruptCompletion) return res;
+        invariant(partialAST.type === "File");
+        body = body.concat(((partialAST: any): BabelNodeFile).program.body);
+        if (source.sourceMapContents) {
+          this.fixup_source_locations(partialAST, source.sourceMapContents);
+        }
+        code[source.filePath] = source.fileContents;
+      } finally {
+        this.realm.popContext(context);
       }
-      let [res, partial_ast] = this.partiallyEvaluateCompletionDeref(ast, false);
-      if (res instanceof AbruptCompletion) return res;
-      if (sourceMaps.length > 0) this.fixup_source_locations(partial_ast, sourceMaps);
-      return generate(
-        partial_ast,
-        { sourceMaps: sourceMaps, sourceFileName: filename },
-        code);
-    } finally {
-      this.realm.popContext(context);
     }
+    let prog = t.program(body);
+    this.fixup_filenames(prog);
+    return generate(
+      prog,
+      { sourceMaps: options.sourceMaps },
+      (code: any));
   }
 
   fixup_source_locations(ast: BabelNode, map: string) {
@@ -1095,6 +1108,15 @@ export class LexicalEnvironment {
     });
   }
 
+  fixup_filenames(ast: BabelNode) {
+    traverse(ast, function (node) {
+      let loc = node.loc;
+      if (!!loc && !!loc.source)
+        (loc: any).filename = loc.source;
+      return false;
+    });
+  }
+
   evaluate(ast: BabelNode, strictCode: boolean, metadata?: any): Value | Reference {
     let res = this.evaluateAbstract(ast, strictCode, metadata);
     if (res instanceof PossiblyNormalCompletion)

src/options.js

@@ -10,12 +10,44 @@
 /* @flow */
 
 import type { ErrorHandler } from "./errors.js";
-import type { RealmOptions, Compatibility } from "./types";
-import type { SerializerOptions } from "./serializer/types";
+
+export type Compatibility =
+  | "browser"
+  | "jsc-600-1-4-17"
+  | "node-source-maps"
+  | "node-cli";
+export const CompatibilityValues = [
+  "browser",
+   "jsc-600-1-4-17",
+  "node-source-maps",
+  "node-cli"
+];
+
+export type RealmOptions = {
+  residual?: boolean,
+  serialize?: boolean,
+  debugNames?: boolean,
+  uniqueSuffix?: string,
+  timeout?: number,
+  compatibility?: Compatibility,
+  mathRandomSeed?: string,
+  strictlyMonotonicDateNow?: boolean,
+  errorHandler?: ErrorHandler,
+};
+
+export type SerializerOptions = {
+  initializeMoreModules?: boolean;
+  internalDebug?: boolean;
+  trace?: boolean;
+  singlePass?: boolean;
+  logStatistics?: boolean;
+  logModules?: boolean;
+  delayUnsupportedRequires?: boolean;
+}
 
 export type Options = {|
   onError?: ErrorHandler,
-  filename?: string,
+  outputFilename?: string,
   inputSourceMapFilename?: string,
   sourceMaps?: boolean,
   compatibility?: Compatibility,

src/prepack-cli.js

@@ -12,8 +12,8 @@
 /* eslint-disable no-shadow */
 
 import { CompilerDiagnostics, type ErrorHandlerResult, FatalError } from "./errors.js";
+import { type Compatibility, CompatibilityValues } from "./options.js";
 import { prepackStdin, prepackFileSync } from "./prepack-node.js";
-import { CompatibilityValues, type Compatibility } from './types.js';
 import type { BabelNodeSourceLocation } from "babel-types";
 import fs from "fs";
 

src/prepack-node-environment.js

@@ -108,7 +108,8 @@ export function prepackNodeCLISync(filename: string, options: Options = defaultO
   tickDomainCallback.intrinsicName = 'process._tickDomainCallback';
 
   // Serialize
-  let serialized = serializer.init("", "", "", options.sourceMaps);
+  let sources = [{ filePath: "", fileContents: "" }];
+  let serialized = serializer.init(sources, options.sourceMaps);
   if (!serialized) {
     throw new FatalError("serializer failed");
   }

src/prepack-standalone.js

@@ -18,7 +18,7 @@ import * as t from "babel-types";
 import { getRealmOptions, getSerializerOptions } from "./options";
 import { FatalError } from "./errors.js";
 import { SerializerStatistics } from "./serializer/types.js";
-
+import type { SourceFile } from "./types.js";
 import { AbruptCompletion } from "./completions.js";
 import type { Options } from "./options";
 import { defaultOptions } from "./options";
@@ -36,11 +36,11 @@ Object.setPrototypeOf(InitializationError, Error);
 Object.setPrototypeOf(InitializationError.prototype, Error.prototype);
 Object.setPrototypeOf(FatalError.prototype, InitializationError.prototype);
 
-export function prepackString(
-  filename: string, code: string, sourceMap: string,
-  options: Options = defaultOptions,
+export function prepackSources(
+  sources: Array<SourceFile>, options: Options = defaultOptions
 ): { code: string, map?: SourceMap, statistics?: SerializerStatistics } {
   let realmOptions = getRealmOptions(options);
+  realmOptions.errorHandler = options.onError;
   let realm = construct_realm(realmOptions);
   initializeGlobals(realm);
 
@@ -49,31 +49,62 @@ export function prepackString(
      realm,
      getSerializerOptions(options),
    );
-   let serialized = serializer.init(
-     options.filename || filename,
-     code,
-     sourceMap,
-     options.sourceMaps
-   );
+   let serialized = serializer.init(sources, options.sourceMaps);
    if (!serialized) {
      throw new FatalError("serializer failed");
    }
    if (!options.residual) return serialized;
-   let result = realm.$GlobalEnv.executePartialEvaluator(
-     filename, serialized.code, JSON.stringify(serialized.map));
+   let residualSources = [{ filePath: options.outputFilename || "unknown", fileContents: serialized.code,
+     sourceMapContents: JSON.stringify(serialized.map) }];
+   let result = realm.$GlobalEnv.executePartialEvaluator(residualSources, options);
    if (result instanceof AbruptCompletion) throw result;
-   return (result: any);
+   // $FlowFixMe This looks like a Flow bug
+   return result;
   } else {
    invariant(options.residual);
-   let result = realm.$GlobalEnv.executePartialEvaluator(filename, code, sourceMap);
+   let result = realm.$GlobalEnv.executePartialEvaluator(sources);
    if (result instanceof AbruptCompletion) throw result;
-   return (result: any);
+   // $FlowFixMe This looks like a Flow bug
+   return result;
+  }
+}
+
+export function prepackString(
+  filename: string, code: string, sourceMap: string,
+  options: Options = defaultOptions,
+): { code: string, map?: SourceMap, statistics?: SerializerStatistics } {
+  let sources = [{ filePath: filename, fileContents: code, sourceMapContents: sourceMap }];
+  let realmOptions = getRealmOptions(options);
+  let realm = construct_realm(realmOptions);
+  initializeGlobals(realm);
+
+  if (options.serialize || !options.residual) {
+    let serializer = new Serializer(
+     realm,
+     getSerializerOptions(options),
+    );
+    let serialized = serializer.init(sources, options.sourceMaps);
+    if (!serialized) {
+     throw new FatalError();
+    }
+    if (!options.residual) return serialized;
+    let residualSources = [{ filePath: options.outputFilename || "unknown", fileContents: serialized.code,
+     sourceMapContents: JSON.stringify(serialized.map) }];
+    let result = realm.$GlobalEnv.executePartialEvaluator(residualSources, options);
+    if (result instanceof AbruptCompletion) throw result;
+    return (result: any);
+  } else {
+    invariant(options.residual);
+    let result = realm.$GlobalEnv.executePartialEvaluator(sources, options);
+    if (result instanceof AbruptCompletion) throw result;
+    return (result: any);
   }
 }
 
 /* deprecated: please use prepackString instead. */
 export function prepack(code: string, options: Options = defaultOptions) {
   let filename = options.filename || 'unknown';
+  let sources = [{ filePath: filename, fileContents: code }];
 
   let realmOptions = getRealmOptions(options);
   realmOptions.errorHandler = options.onError;
@@ -81,20 +112,22 @@ export function prepack(code: string, options: Options = defaultOptions) {
   initializeGlobals(realm);
 
   let serializer = new Serializer(realm, getSerializerOptions(options));
-  let serialized = serializer.init(filename, code, "", options.sourceMaps);
+  let serialized = serializer.init(sources, options.sourceMaps);
   if (!serialized) {
     throw new FatalError("serializer failed");
   }
   return serialized;
 }
 
-/* deprecated: pelase use prepackString instead. */
+/* deprecated: please use prepackString instead. */
 export function prepackFromAst(ast: BabelNodeFile | BabelNodeProgram, code: string, options: Options = defaultOptions) {
   if (ast && ast.type === "Program") {
     ast = t.file(ast, [], []);
   } else if (!ast || ast.type !== "File") {
     throw new Error("Not a valid ast?");
   }
+  let filename = options.filename || (ast.loc && ast.loc.source) || "unknown";
+  let sources = [{ filePath: filename, fileContents: code }];
 
   // TODO: Expose an option to wire an already parsed ast all the way through
   // to the execution environment. For now, we just reparse.
@@ -102,7 +135,7 @@ export function prepackFromAst(ast: BabelNodeFile | BabelNodeProgram, code: stri
   let realm = construct_realm(getRealmOptions(options));
   initializeGlobals(realm);
   let serializer = new Serializer(realm, getSerializerOptions(options));
-  let serialized = serializer.init("", code, "", options.sourceMaps);
+  let serialized = serializer.init(sources, options.sourceMaps);
   if (!serialized) {
     throw new FatalError("serializer failed");
   }

src/realm.js

@@ -9,7 +9,7 @@
 
 /* @flow */
 
-import type { RealmOptions, Intrinsics, Compatibility, PropertyBinding, Descriptor } from "./types.js";
+import type { Intrinsics, PropertyBinding, Descriptor } from "./types.js";
 import { CompilerDiagnostics, type ErrorHandlerResult, type ErrorHandler } from "./errors.js";
 import type { NativeFunctionValue, FunctionValue } from "./values/index.js";
 import { Value, ObjectValue, AbstractValue, AbstractObjectValue, StringValue, ConcreteValue, UndefinedValue } from "./values/index.js";
@@ -19,6 +19,7 @@ import type { Binding } from "./environment.js";
 import { cloneDescriptor, GetValue, Construct, ThrowIfMightHaveBeenDeleted } from "./methods/index.js";
 import type { NormalCompletion } from "./completions.js";
 import { Completion, IntrospectionThrowCompletion, ThrowCompletion, AbruptCompletion, PossiblyNormalCompletion } from "./completions.js";
+import type { Compatibility, RealmOptions } from "./options.js";
 import invariant from "./invariant.js";
 import seedrandom from "seedrandom";
 import { Generator, PreludeGenerator } from "./utils/generator.js";

src/serializer/serializer.js

@@ -10,7 +10,7 @@
 /* @flow */
 
 import { Realm, ExecutionContext } from "../realm.js";
-import type { Descriptor, PropertyBinding } from "../types.js";
+import type { Descriptor, PropertyBinding, SourceFile } from "../types.js";
 import { ToLength, IsArray, Get } from "../methods/index.js";
 import { Completion } from "../completions.js";
 import { BoundFunctionValue, ProxyValue, SymbolValue, AbstractValue, EmptyValue, FunctionValue, Value, ObjectValue, NativeFunctionValue, UndefinedValue } from "../values/index.js";
@@ -23,7 +23,8 @@ import type SourceMap from "babel-generator";
 // import { transform } from "babel-core";
 import traverse from "babel-traverse";
 import invariant from "../invariant.js";
-import type { SerializedBinding, VisitedBinding, FunctionInfo, FunctionInstance, SerializerOptions } from "./types.js";
+import type { SerializerOptions } from "../options.js";
+import type { SerializedBinding, VisitedBinding, FunctionInfo, FunctionInstance } from "./types.js";
 import { BodyReference, SerializerStatistics, type VisitedBindings } from "./types.js";
 import { IdentifierCollector } from "./visitors.js";
 import { Logger } from "./logger.js";
@@ -1360,7 +1361,7 @@ export class Serializer {
     return ast;
   }
 
-  init(filename: string, code: string, map?: string = "",
+  init(sources: Array<SourceFile>,
       sourceMaps?: boolean = false, onError?: (Realm, Value) => void): void | {
     code: string,
     map: void | SourceMap,
@@ -1368,7 +1369,11 @@ export class Serializer {
   } {
     // Phase 1: Let's interpret.
     if (this.options.profile) console.time("[Profiling] Interpreting Global Code");
-    this.execute(filename, code, map, onError);
+    let code = {};
+    for (let source of sources) {
+      this.execute(source.filePath, source.fileContents, source.sourceMapContents || "", onError);
+      code[source.filePath] = source.fileContents;
+    }
     if (this.options.profile) console.timeEnd("[Profiling] Interpreting Global Code");
     if (this.logger.hasErrors()) return undefined;
     if (this.options.profile) console.time("[Profiling] Resolving Initial Modules");
@@ -1408,8 +1413,8 @@ export class Serializer {
     let ast = this.serialize();
     let generated = generate(
         ast,
-        { sourceMaps: sourceMaps, sourceFileName: filename },
-        code);
+        { sourceMaps: sourceMaps },
+        (code: any));
     if (this.options.profile) console.timeEnd("[Profiling] Serialize Pass");
     invariant(!this.logger.hasErrors());
     if (this.options.logStatistics) this.statistics.log();

src/serializer/types.js

@@ -80,16 +80,6 @@ export class BodyReference {
   index: number;
 }
 
-export type SerializerOptions = {
-  initializeMoreModules?: boolean;
-  internalDebug?: boolean;
-  trace?: boolean;
-  singlePass?: boolean;
-  logStatistics?: boolean;
-  logModules?: boolean;
-  delayUnsupportedRequires?: boolean;
-}
-
 export class SerializerStatistics {
   constructor() {
     this.objects = 0;

src/types.js

@@ -11,7 +11,6 @@
 
 import type { NumberValue, AbstractValue, BooleanValue, NativeFunctionValue, FunctionValue, StringValue, SymbolValue, UndefinedValue, NullValue, EmptyValue, Value, AbstractObjectValue } from "./values/index.js";
 import { ObjectValue } from "./values/index.js";
-import type { ErrorHandler } from "./errors.js";
 
 export const ElementSize = {
   Float32: 4,
@@ -29,6 +28,12 @@ export type IterationKind = "key+value" | "value" | "key";
 
 export type SourceType = "module" | "script";
 
+export type SourceFile = {
+  filePath: string,
+  fileContents: string,
+  sourceMapContents?: string
+}
+
 export type SourceMap = {
   sources: Array<string>,
   names: Array<string>,
@@ -36,30 +41,6 @@ export type SourceMap = {
   sourcesContent: Array<string>
 };
 
-export type Compatibility =
-  | "browser"
-  | "jsc-600-1-4-17"
-  | "node-source-maps"
-  | "node-cli";
-export const CompatibilityValues = [
-  "browser",
-   "jsc-600-1-4-17",
-  "node-source-maps",
-  "node-cli"
-];
-
-export type RealmOptions = {
-  residual?: boolean,
-  serialize?: boolean,
-  debugNames?: boolean,
-  uniqueSuffix?: string,
-  timeout?: number,
-  compatibility?: Compatibility,
-  mathRandomSeed?: string,
-  strictlyMonotonicDateNow?: boolean,
-  errorHandler?: ErrorHandler,
-};
-
 export type AbstractTime = "early" | "late";
 
 export type ElementType = "Float32" | "Float64" | "Int8" | "Int16" | "Int32" | "Uint8" | "Uint16" | "Uint32" | "Uint8Clamped";