Release source code and source map data after parsing and source map fixup. (#2377)

Summary:
Release notes: Reduce memory usage of running Prepack by 3% in some scenarios

This fixes #2365.

This is realized via a stateful SourceFileCollection.
On a small internal benchmark, this saved around 3% of total node memory usage
(the parsed AST that we keep around uses an order of magnitude more memory
than the original source file and source map,
11MB of source files + source map vs 88MB of parsed AST).
Pull Request resolved: https://github.com/facebook/prepack/pull/2377

Differential Revision: D9150888

Pulled By: NTillmann

fbshipit-source-id: b38a8176c4f1e4633366bd48b7d396aec023e7c3

scripts/test-runner.js

@@ -30,6 +30,7 @@ function SerialPromises(promises: Array<() => Promise<void>>) {
 }
 
 let Serializer = require("../lib/serializer/index.js").default;
+let SourceFileCollection = require("../lib/types.js").SourceFileCollection;
 let SerializerStatistics = require("../lib/serializer/statistics.js").SerializerStatistics;
 let construct_realm = require("../lib/construct_realm.js").default;
 let initializeGlobals = require("../lib/globals.js").default;
@@ -381,8 +382,8 @@ function runTest(name, code, options: PrepackOptions, args) {
         lazyObjectsRuntime: options.lazyObjectsRuntime,
       };
       let serializer = new Serializer(realm, serializerOptions);
-      let sources = [{ filePath: name, fileContents: code }];
-      let serialized = serializer.init(sources, false);
+      let sourceFileCollection = new SourceFileCollection([{ filePath: name, fileContents: code }]);
+      let serialized = serializer.init(sourceFileCollection, false);
       if (!serialized) {
         console.error(chalk.red("Error during serialization"));
       } else {

src/benchmarker.js

@@ -10,6 +10,7 @@
 /* @flow strict-local */
 
 import type { Compatibility } from "./options.js";
+import { SourceFileCollection } from "./types.js";
 import Serializer from "./serializer/index.js";
 import construct_realm from "./construct_realm.js";
 import initializeGlobals from "./globals.js";
@@ -161,8 +162,8 @@ function dump(
   let realm = construct_realm({ serialize: true, compatibility });
   initializeGlobals(realm);
   let serializer = new Serializer(realm);
-  let sources = [{ filePath: name, fileContents: raw }];
-  let serialized = serializer.init(sources);
+  let sourceFileCollection = new SourceFileCollection([{ filePath: name, fileContents: raw }]);
+  let serialized = serializer.init(sourceFileCollection);
   if (!serialized) {
     process.exit(1);
     invariant(false);

src/prepack-node.js

@@ -16,7 +16,7 @@ import { defaultOptions } from "./options";
 import { FatalError } from "./errors.js";
 import { type PrepackOptions } from "./prepack-options";
 import { prepackSources } from "./prepack-standalone.js";
-import { type SourceMap } from "./types.js";
+import { type SourceMap, SourceFileCollection } from "./types.js";
 import { DebugChannel } from "./debugger/server/channel/DebugChannel.js";
 import { FileIOWrapper } from "./debugger/common/channel/FileIOWrapper.js";
 import { type SerializedResult } from "./serializer/types.js";
@@ -131,7 +131,7 @@ export function prepackFile(
   });
 }
 
-export function prepackFileSync(filenames: Array<string>, options: PrepackOptions = defaultOptions): SerializedResult {
+function getSourceFileCollection(filenames: Array<string>, options: PrepackOptions) {
   const sourceFiles = filenames.map(filename => {
     let code = fs.readFileSync(filename, "utf8");
     let sourceMap = "";
@@ -151,19 +151,27 @@ export function prepackFileSync(filenames: Array<string>, options: PrepackOption
     };
   });
 
-  // Don't include sourcemaps that weren't found
-  let validSourceFiles = sourceFiles.filter(sf => sf.sourceMapContents !== "");
+  return new SourceFileCollection(sourceFiles);
+}
+
+export function prepackFileSync(filenames: Array<string>, options: PrepackOptions = defaultOptions): SerializedResult {
+  let sourceFileCollection = getSourceFileCollection(filenames, options);
+
+  // Filter to not include sourcemaps that weren't found
+  let filterValidSourceMaps = a => a.filter(sf => sf.sourceMapContents !== "");
 
   // The existence of debug[In/Out]FilePath represents the desire to use the debugger.
   if (options.debugInFilePath !== undefined && options.debugOutFilePath !== undefined) {
     if (options.debuggerConfigArgs === undefined) options.debuggerConfigArgs = {};
+    let debuggerConfigArgs = options.debuggerConfigArgs;
 
     let ioWrapper = new FileIOWrapper(false, options.debugInFilePath, options.debugOutFilePath);
-    options.debuggerConfigArgs.debugChannel = new DebugChannel(ioWrapper);
-    options.debuggerConfigArgs.sourcemaps = validSourceFiles;
+    debuggerConfigArgs.debugChannel = new DebugChannel(ioWrapper);
+    debuggerConfigArgs.sourcemaps = filterValidSourceMaps(sourceFileCollection.toArray());
   }
 
-  if (options.debugReproArgs) options.debugReproArgs.sourcemaps = validSourceFiles;
+  let debugReproArgs = options.debugReproArgs;
+  if (debugReproArgs) debugReproArgs.sourcemaps = filterValidSourceMaps(sourceFileCollection.toArray());
 
-  return prepackSources(sourceFiles, options, createStatistics(options));
+  return prepackSources(sourceFileCollection, options, createStatistics(options));
 }

src/prepack-standalone.js

@@ -17,7 +17,7 @@ import initializeGlobals from "./globals.js";
 import { EvaluateDirectCallWithArgList } from "./methods/index.js";
 import { getRealmOptions, getSerializerOptions } from "./prepack-options";
 import { FatalError } from "./errors.js";
-import type { SourceFile } from "./types.js";
+import { SourceFileCollection, type SourceFile } from "./types.js";
 import { AbruptCompletion } from "./completions.js";
 import type { PrepackOptions } from "./prepack-options";
 import { defaultOptions } from "./options";
@@ -32,10 +32,12 @@ import { Generator } from "./utils/generator.js";
 import { AbstractObjectValue, AbstractValue, ObjectValue } from "./values/index.js";
 
 export function prepackSources(
-  sources: Array<SourceFile>,
+  sourceFileCollection: SourceFileCollection | Array<SourceFile>,
   options: PrepackOptions = defaultOptions,
   statistics: SerializerStatistics | void = undefined
 ): SerializedResult {
+  if (Array.isArray(sourceFileCollection)) sourceFileCollection = new SourceFileCollection(sourceFileCollection);
+
   let realmOptions = getRealmOptions(options);
   realmOptions.errorHandler = options.errorHandler;
   let realm = construct_realm(
@@ -59,14 +61,14 @@ export function prepackSources(
       !!options.delayUnsupportedRequires,
       !!options.accelerateUnsupportedRequires
     );
-    let [result] = realm.$GlobalEnv.executeSources(sources);
+    let [result] = realm.$GlobalEnv.executeSources(sourceFileCollection.toArray());
     if (result instanceof AbruptCompletion) throw result;
     invariant(options.check);
     checkResidualFunctions(modules, options.check[0], options.check[1]);
     return { code: "", map: undefined };
   } else if (options.serialize === true || options.residual !== true) {
     let serializer = new Serializer(realm, getSerializerOptions(options));
-    let serialized = serializer.init(sources, options.sourceMaps, options.onParse);
+    let serialized = serializer.init(sourceFileCollection, options.sourceMaps, options.onParse);
 
     //Turn off the debugger if there is one
     if (realm.debuggerInstance) {
@@ -107,7 +109,7 @@ export function prepackSources(
   } else {
     invariant(options.residual);
     realm.generator = new Generator(realm, "main", realm.pathConditions);
-    let result = realm.$GlobalEnv.executePartialEvaluator(sources, options);
+    let result = realm.$GlobalEnv.executePartialEvaluator(sourceFileCollection.toArray(), options);
     if (result instanceof AbruptCompletion) throw result;
     return { ...result };
   }

src/serializer/serializer.js

@@ -12,7 +12,7 @@
 import { EnvironmentRecord } from "../environment.js";
 import { Realm, ExecutionContext } from "../realm.js";
 import { CompilerDiagnostic, FatalError } from "../errors.js";
-import type { SourceFile } from "../types.js";
+import { SourceFileCollection } from "../types.js";
 import { AbruptCompletion } from "../completions.js";
 import { Generator } from "../utils/generator.js";
 import generate from "@babel/generator";
@@ -69,12 +69,12 @@ export class Serializer {
   options: SerializerOptions;
 
   _execute(
-    sources: Array<SourceFile>,
+    sourceFileCollection: SourceFileCollection,
     sourceMaps?: boolean = false,
     onParse?: BabelNodeFile => void
   ): { [string]: string } {
     let realm = this.realm;
-    let [res, code] = realm.$GlobalEnv.executeSources(sources, "script", ast => {
+    let [res, code] = realm.$GlobalEnv.executeSources(sourceFileCollection.toArray(), "script", ast => {
       let realmPreludeGenerator = realm.preludeGenerator;
       invariant(realmPreludeGenerator);
       let forbiddenNames = realmPreludeGenerator.nameGenerator.forbiddenNames;
@@ -87,6 +87,9 @@ export class Serializer {
       if (onParse) onParse(ast);
     });
 
+    // Release memory of source files and their source maps
+    sourceFileCollection.destroy();
+
     if (res instanceof AbruptCompletion) {
       let context = new ExecutionContext();
       realm.pushContext(context);
@@ -125,7 +128,7 @@ export class Serializer {
   }
 
   init(
-    sources: Array<SourceFile>,
+    sourceFileCollection: SourceFileCollection,
     sourceMaps?: boolean = false,
     onParse?: BabelNodeFile => void
   ): void | SerializedResult {
@@ -139,7 +142,7 @@ export class Serializer {
         this.logger.logInformation(`Evaluating initialization path...`);
       }
 
-      let code = this._execute(sources, sourceMaps, onParse);
+      let code = this._execute(sourceFileCollection, sourceMaps, onParse);
       let environmentRecordIdAfterGlobalCode = EnvironmentRecord.nextId;
 
       if (this.logger.hasErrors()) return undefined;

src/types.js

@@ -52,6 +52,7 @@ import type { Bindings, Effects, EvaluationResult, PropertyBindings, CreatedObje
 import { CompilerDiagnostic } from "./errors.js";
 import type { Severity } from "./errors.js";
 import type { DebugChannel } from "./debugger/server/channel/DebugChannel.js";
+import invariant from "./invariant.js";
 
 export const ElementSize = {
   Float32: 4,
@@ -94,6 +95,19 @@ export type SourceFile = {
   sourceMapFilename?: string,
 };
 
+export class SourceFileCollection {
+  constructor(sourceFiles: Array<SourceFile>) {
+    this._sourceFiles = sourceFiles;
+  }
+  _sourceFiles: void | Array<SourceFile>;
+  toArray(): Array<SourceFile> {
+    invariant(this._sourceFiles !== undefined);
+    return this._sourceFiles;
+  }
+  destroy(): void {
+    this._sourceFiles = undefined;
+  }
+}
 export type SourceMap = {
   sources: Array<string>,
   names: Array<string>,