mirror of
https://github.com/facebookarchive/prepack.git
synced 2024-09-11 14:46:37 +03:00
Release source code and source map data after parsing and source map fixup. (#2377)
Summary: Release notes: Reduce memory usage of running Prepack by 3% in some scenarios This fixes #2365. This is realized via a stateful SourceFileCollection. On a small internal benchmark, this saved around 3% of total node memory usage (the parsed AST that we keep around uses an order of magnitude more memory than the original source file and source map, 11MB of source files + source map vs 88MB of parsed AST). Pull Request resolved: https://github.com/facebook/prepack/pull/2377 Differential Revision: D9150888 Pulled By: NTillmann fbshipit-source-id: b38a8176c4f1e4633366bd48b7d396aec023e7c3
This commit is contained in:
parent
c89f511102
commit
4103d8e9fe
@ -30,6 +30,7 @@ function SerialPromises(promises: Array<() => Promise<void>>) {
|
||||
}
|
||||
|
||||
let Serializer = require("../lib/serializer/index.js").default;
|
||||
let SourceFileCollection = require("../lib/types.js").SourceFileCollection;
|
||||
let SerializerStatistics = require("../lib/serializer/statistics.js").SerializerStatistics;
|
||||
let construct_realm = require("../lib/construct_realm.js").default;
|
||||
let initializeGlobals = require("../lib/globals.js").default;
|
||||
@ -381,8 +382,8 @@ function runTest(name, code, options: PrepackOptions, args) {
|
||||
lazyObjectsRuntime: options.lazyObjectsRuntime,
|
||||
};
|
||||
let serializer = new Serializer(realm, serializerOptions);
|
||||
let sources = [{ filePath: name, fileContents: code }];
|
||||
let serialized = serializer.init(sources, false);
|
||||
let sourceFileCollection = new SourceFileCollection([{ filePath: name, fileContents: code }]);
|
||||
let serialized = serializer.init(sourceFileCollection, false);
|
||||
if (!serialized) {
|
||||
console.error(chalk.red("Error during serialization"));
|
||||
} else {
|
||||
|
@ -10,6 +10,7 @@
|
||||
/* @flow strict-local */
|
||||
|
||||
import type { Compatibility } from "./options.js";
|
||||
import { SourceFileCollection } from "./types.js";
|
||||
import Serializer from "./serializer/index.js";
|
||||
import construct_realm from "./construct_realm.js";
|
||||
import initializeGlobals from "./globals.js";
|
||||
@ -161,8 +162,8 @@ function dump(
|
||||
let realm = construct_realm({ serialize: true, compatibility });
|
||||
initializeGlobals(realm);
|
||||
let serializer = new Serializer(realm);
|
||||
let sources = [{ filePath: name, fileContents: raw }];
|
||||
let serialized = serializer.init(sources);
|
||||
let sourceFileCollection = new SourceFileCollection([{ filePath: name, fileContents: raw }]);
|
||||
let serialized = serializer.init(sourceFileCollection);
|
||||
if (!serialized) {
|
||||
process.exit(1);
|
||||
invariant(false);
|
||||
|
@ -16,7 +16,7 @@ import { defaultOptions } from "./options";
|
||||
import { FatalError } from "./errors.js";
|
||||
import { type PrepackOptions } from "./prepack-options";
|
||||
import { prepackSources } from "./prepack-standalone.js";
|
||||
import { type SourceMap } from "./types.js";
|
||||
import { type SourceMap, SourceFileCollection } from "./types.js";
|
||||
import { DebugChannel } from "./debugger/server/channel/DebugChannel.js";
|
||||
import { FileIOWrapper } from "./debugger/common/channel/FileIOWrapper.js";
|
||||
import { type SerializedResult } from "./serializer/types.js";
|
||||
@ -131,7 +131,7 @@ export function prepackFile(
|
||||
});
|
||||
}
|
||||
|
||||
export function prepackFileSync(filenames: Array<string>, options: PrepackOptions = defaultOptions): SerializedResult {
|
||||
function getSourceFileCollection(filenames: Array<string>, options: PrepackOptions) {
|
||||
const sourceFiles = filenames.map(filename => {
|
||||
let code = fs.readFileSync(filename, "utf8");
|
||||
let sourceMap = "";
|
||||
@ -151,19 +151,27 @@ export function prepackFileSync(filenames: Array<string>, options: PrepackOption
|
||||
};
|
||||
});
|
||||
|
||||
// Don't include sourcemaps that weren't found
|
||||
let validSourceFiles = sourceFiles.filter(sf => sf.sourceMapContents !== "");
|
||||
return new SourceFileCollection(sourceFiles);
|
||||
}
|
||||
|
||||
export function prepackFileSync(filenames: Array<string>, options: PrepackOptions = defaultOptions): SerializedResult {
|
||||
let sourceFileCollection = getSourceFileCollection(filenames, options);
|
||||
|
||||
// Filter to not include sourcemaps that weren't found
|
||||
let filterValidSourceMaps = a => a.filter(sf => sf.sourceMapContents !== "");
|
||||
|
||||
// The existence of debug[In/Out]FilePath represents the desire to use the debugger.
|
||||
if (options.debugInFilePath !== undefined && options.debugOutFilePath !== undefined) {
|
||||
if (options.debuggerConfigArgs === undefined) options.debuggerConfigArgs = {};
|
||||
let debuggerConfigArgs = options.debuggerConfigArgs;
|
||||
|
||||
let ioWrapper = new FileIOWrapper(false, options.debugInFilePath, options.debugOutFilePath);
|
||||
options.debuggerConfigArgs.debugChannel = new DebugChannel(ioWrapper);
|
||||
options.debuggerConfigArgs.sourcemaps = validSourceFiles;
|
||||
debuggerConfigArgs.debugChannel = new DebugChannel(ioWrapper);
|
||||
debuggerConfigArgs.sourcemaps = filterValidSourceMaps(sourceFileCollection.toArray());
|
||||
}
|
||||
|
||||
if (options.debugReproArgs) options.debugReproArgs.sourcemaps = validSourceFiles;
|
||||
let debugReproArgs = options.debugReproArgs;
|
||||
if (debugReproArgs) debugReproArgs.sourcemaps = filterValidSourceMaps(sourceFileCollection.toArray());
|
||||
|
||||
return prepackSources(sourceFiles, options, createStatistics(options));
|
||||
return prepackSources(sourceFileCollection, options, createStatistics(options));
|
||||
}
|
||||
|
@ -17,7 +17,7 @@ import initializeGlobals from "./globals.js";
|
||||
import { EvaluateDirectCallWithArgList } from "./methods/index.js";
|
||||
import { getRealmOptions, getSerializerOptions } from "./prepack-options";
|
||||
import { FatalError } from "./errors.js";
|
||||
import type { SourceFile } from "./types.js";
|
||||
import { SourceFileCollection, type SourceFile } from "./types.js";
|
||||
import { AbruptCompletion } from "./completions.js";
|
||||
import type { PrepackOptions } from "./prepack-options";
|
||||
import { defaultOptions } from "./options";
|
||||
@ -32,10 +32,12 @@ import { Generator } from "./utils/generator.js";
|
||||
import { AbstractObjectValue, AbstractValue, ObjectValue } from "./values/index.js";
|
||||
|
||||
export function prepackSources(
|
||||
sources: Array<SourceFile>,
|
||||
sourceFileCollection: SourceFileCollection | Array<SourceFile>,
|
||||
options: PrepackOptions = defaultOptions,
|
||||
statistics: SerializerStatistics | void = undefined
|
||||
): SerializedResult {
|
||||
if (Array.isArray(sourceFileCollection)) sourceFileCollection = new SourceFileCollection(sourceFileCollection);
|
||||
|
||||
let realmOptions = getRealmOptions(options);
|
||||
realmOptions.errorHandler = options.errorHandler;
|
||||
let realm = construct_realm(
|
||||
@ -59,14 +61,14 @@ export function prepackSources(
|
||||
!!options.delayUnsupportedRequires,
|
||||
!!options.accelerateUnsupportedRequires
|
||||
);
|
||||
let [result] = realm.$GlobalEnv.executeSources(sources);
|
||||
let [result] = realm.$GlobalEnv.executeSources(sourceFileCollection.toArray());
|
||||
if (result instanceof AbruptCompletion) throw result;
|
||||
invariant(options.check);
|
||||
checkResidualFunctions(modules, options.check[0], options.check[1]);
|
||||
return { code: "", map: undefined };
|
||||
} else if (options.serialize === true || options.residual !== true) {
|
||||
let serializer = new Serializer(realm, getSerializerOptions(options));
|
||||
let serialized = serializer.init(sources, options.sourceMaps, options.onParse);
|
||||
let serialized = serializer.init(sourceFileCollection, options.sourceMaps, options.onParse);
|
||||
|
||||
//Turn off the debugger if there is one
|
||||
if (realm.debuggerInstance) {
|
||||
@ -107,7 +109,7 @@ export function prepackSources(
|
||||
} else {
|
||||
invariant(options.residual);
|
||||
realm.generator = new Generator(realm, "main", realm.pathConditions);
|
||||
let result = realm.$GlobalEnv.executePartialEvaluator(sources, options);
|
||||
let result = realm.$GlobalEnv.executePartialEvaluator(sourceFileCollection.toArray(), options);
|
||||
if (result instanceof AbruptCompletion) throw result;
|
||||
return { ...result };
|
||||
}
|
||||
|
@ -12,7 +12,7 @@
|
||||
import { EnvironmentRecord } from "../environment.js";
|
||||
import { Realm, ExecutionContext } from "../realm.js";
|
||||
import { CompilerDiagnostic, FatalError } from "../errors.js";
|
||||
import type { SourceFile } from "../types.js";
|
||||
import { SourceFileCollection } from "../types.js";
|
||||
import { AbruptCompletion } from "../completions.js";
|
||||
import { Generator } from "../utils/generator.js";
|
||||
import generate from "@babel/generator";
|
||||
@ -69,12 +69,12 @@ export class Serializer {
|
||||
options: SerializerOptions;
|
||||
|
||||
_execute(
|
||||
sources: Array<SourceFile>,
|
||||
sourceFileCollection: SourceFileCollection,
|
||||
sourceMaps?: boolean = false,
|
||||
onParse?: BabelNodeFile => void
|
||||
): { [string]: string } {
|
||||
let realm = this.realm;
|
||||
let [res, code] = realm.$GlobalEnv.executeSources(sources, "script", ast => {
|
||||
let [res, code] = realm.$GlobalEnv.executeSources(sourceFileCollection.toArray(), "script", ast => {
|
||||
let realmPreludeGenerator = realm.preludeGenerator;
|
||||
invariant(realmPreludeGenerator);
|
||||
let forbiddenNames = realmPreludeGenerator.nameGenerator.forbiddenNames;
|
||||
@ -87,6 +87,9 @@ export class Serializer {
|
||||
if (onParse) onParse(ast);
|
||||
});
|
||||
|
||||
// Release memory of source files and their source maps
|
||||
sourceFileCollection.destroy();
|
||||
|
||||
if (res instanceof AbruptCompletion) {
|
||||
let context = new ExecutionContext();
|
||||
realm.pushContext(context);
|
||||
@ -125,7 +128,7 @@ export class Serializer {
|
||||
}
|
||||
|
||||
init(
|
||||
sources: Array<SourceFile>,
|
||||
sourceFileCollection: SourceFileCollection,
|
||||
sourceMaps?: boolean = false,
|
||||
onParse?: BabelNodeFile => void
|
||||
): void | SerializedResult {
|
||||
@ -139,7 +142,7 @@ export class Serializer {
|
||||
this.logger.logInformation(`Evaluating initialization path...`);
|
||||
}
|
||||
|
||||
let code = this._execute(sources, sourceMaps, onParse);
|
||||
let code = this._execute(sourceFileCollection, sourceMaps, onParse);
|
||||
let environmentRecordIdAfterGlobalCode = EnvironmentRecord.nextId;
|
||||
|
||||
if (this.logger.hasErrors()) return undefined;
|
||||
|
14
src/types.js
14
src/types.js
@ -52,6 +52,7 @@ import type { Bindings, Effects, EvaluationResult, PropertyBindings, CreatedObje
|
||||
import { CompilerDiagnostic } from "./errors.js";
|
||||
import type { Severity } from "./errors.js";
|
||||
import type { DebugChannel } from "./debugger/server/channel/DebugChannel.js";
|
||||
import invariant from "./invariant.js";
|
||||
|
||||
export const ElementSize = {
|
||||
Float32: 4,
|
||||
@ -94,6 +95,19 @@ export type SourceFile = {
|
||||
sourceMapFilename?: string,
|
||||
};
|
||||
|
||||
export class SourceFileCollection {
|
||||
constructor(sourceFiles: Array<SourceFile>) {
|
||||
this._sourceFiles = sourceFiles;
|
||||
}
|
||||
_sourceFiles: void | Array<SourceFile>;
|
||||
toArray(): Array<SourceFile> {
|
||||
invariant(this._sourceFiles !== undefined);
|
||||
return this._sourceFiles;
|
||||
}
|
||||
destroy(): void {
|
||||
this._sourceFiles = undefined;
|
||||
}
|
||||
}
|
||||
export type SourceMap = {
|
||||
sources: Array<string>,
|
||||
names: Array<string>,
|
||||
|
Loading…
Reference in New Issue
Block a user