Transitive materialization for Array operators (#2456)

Summary:
This PR implements a step of the way to getting leaked value analysis working for optimized Array operators. It is desirable to leak as little as possible, so that the operators can be specialized to take into account values in the environment in which they run. In the beginning, we are focusing on the narrow range of scenarios in which this is possible. We will start by enforcing the assumptions that we rely on, and make sure that the code that we generate is correct. Once we have correct code, we will start progressively relaxing the assumptions to increase coverage. The overall plan can be found here: #2452.

More specifically, this PR transitively materializes objects reachable via reads to bindings in the optimized function. This is necessary to snapshot the contents of those objects at specialization time.

Fixes #2405.
Pull Request resolved: https://github.com/facebook/prepack/pull/2456

Differential Revision: D9498939

Pulled By: sb98052

fbshipit-source-id: 16853f97dc781505dba29dce7f28996a0a4e7749

src/types.js

@@ -393,7 +393,8 @@ export type LeakType = {
 };
 
 export type MaterializeType = {
-  materialize(realm: Realm, object: ObjectValue): void,
+  materializeObject(realm: Realm, object: ObjectValue): void,
+  materializeObjectsTransitive(realm: Realm, value: FunctionValue): void,
 };
 
 export type PropertiesType = {

src/utils/leak.js

@@ -608,7 +608,313 @@ export class LeakImplementation {
 }
 
 export class MaterializeImplementation {
-  materialize(realm: Realm, val: ObjectValue) {
+  // TODO: Understand relation to snapshots: #2441
+  materializeObject(realm: Realm, val: ObjectValue): void {
     materializeObject(realm, val);
   }
+
+  // This routine materializes objects reachable from non-local bindings read
+  // by a function. It does this for the purpose of outlining calls to that function.
+  //
+  // Notes:
+  // - Locations that are only read need not materialize because their values are up-to-date
+  // at optimization time,
+  // - Locations that are written to are ignored, because we make the assumption, for now,
+  // that the function being outlined is pure.
+  // - Previously havoced locations (#2446) should be reloaded, but are currently rejected.
+  // - Specialization depends on the assumption that the Array op will only be used once.
+  // First, we will enforce it: #2448. Later we will relax it: #2454
+  materializeObjectsTransitive(realm: Realm, outlinedFunction: FunctionValue): void {
+    invariant(realm.isInPureScope());
+    let objectsToMaterialize: Set<ObjectValue> = new Set();
+    let visitedValues: Set<Value> = new Set();
+    computeFromValue(outlinedFunction);
+
+    if (objectsToMaterialize.size !== 0 && realm.instantRender.enabled) {
+      let error = new CompilerDiagnostic(
+        "Instant Render does not support array operators that reference objects via non-local bindings",
+        outlinedFunction.expressionLocation,
+        "PP0042",
+        "FatalError"
+      );
+      realm.handleError(error);
+      throw new FatalError();
+    }
+
+    for (let object of objectsToMaterialize) {
+      if (!TestIntegrityLevel(realm, object, "frozen")) materializeObject(realm, object);
+    }
+
+    return;
+    function computeFromBindings(func: FunctionValue, nonLocalReadBindings: Set<string>): void {
+      invariant(func instanceof ECMAScriptSourceFunctionValue);
+      let environment = func.$Environment;
+      while (environment) {
+        let record = environment.environmentRecord;
+        if (record instanceof ObjectEnvironmentRecord) computeFromValue(record.object);
+        else if (record instanceof DeclarativeEnvironmentRecord || record instanceof FunctionEnvironmentRecord)
+          computeFromDeclarativeEnvironmentRecord(record, nonLocalReadBindings);
+        else if (record instanceof GlobalEnvironmentRecord) {
+          // TODO: #2484
+          break;
+        }
+        environment = environment.parent;
+      }
+    }
+    function computeFromDeclarativeEnvironmentRecord(
+      record: DeclarativeEnvironmentRecord,
+      nonLocalReadBindings: Set<string>
+    ): void {
+      let environmentBindings = record.bindings;
+      for (let bindingName of Object.keys(environmentBindings)) {
+        let binding = environmentBindings[bindingName];
+        invariant(binding !== undefined);
+        let found = nonLocalReadBindings.delete(bindingName);
+        // Check what undefined could mean here, besides absent binding
+        // #2446
+        if (found && binding.value !== undefined) {
+          computeFromValue(binding.value);
+        }
+      }
+    }
+    function computeFromAbstractValue(value: AbstractValue): void {
+      if (value.values.isTop()) {
+        for (let arg of value.args) {
+          computeFromValue(arg);
+        }
+      } else {
+        // If we know which object this might be, then leak each of them.
+        for (let element of value.values.getElements()) {
+          computeFromValue(element);
+        }
+      }
+    }
+    function computeFromProxyValue(value: ProxyValue): void {
+      computeFromValue(value.$ProxyTarget);
+      computeFromValue(value.$ProxyHandler);
+    }
+    function computeFromValue(value: Value): void {
+      if (value.isIntrinsic() || value instanceof EmptyValue || value instanceof PrimitiveValue) {
+        visit(value);
+      } else if (value instanceof AbstractValue) {
+        ifNotVisited(value, computeFromAbstractValue);
+      } else if (value instanceof FunctionValue) {
+        ifNotVisited(value, computeFromFunctionValue);
+      } else if (value instanceof ObjectValue) {
+        ifNotVisited(value, computeFromObjectValue);
+      } else if (value instanceof ProxyValue) {
+        ifNotVisited(value, computeFromProxyValue);
+      }
+    }
+    function computeFromObjectValue(value: Value): void {
+      invariant(value instanceof ObjectValue);
+      let kind = value.getKind();
+      computeFromObjectProperties(value, kind);
+
+      switch (kind) {
+        case "RegExp":
+        case "Number":
+        case "String":
+        case "Boolean":
+        case "ReactElement":
+        case "ArrayBuffer":
+        case "Array":
+          break;
+        case "Date":
+          let dateValue = value.$DateValue;
+          invariant(dateValue !== undefined);
+          computeFromValue(dateValue);
+          break;
+        case "Float32Array":
+        case "Float64Array":
+        case "Int8Array":
+        case "Int16Array":
+        case "Int32Array":
+        case "Uint8Array":
+        case "Uint16Array":
+        case "Uint32Array":
+        case "Uint8ClampedArray":
+        case "DataView":
+          let buf = value.$ViewedArrayBuffer;
+          invariant(buf !== undefined);
+          computeFromValue(buf);
+          break;
+        case "Map":
+        case "WeakMap":
+          ifNotVisited(value, computeFromMap);
+          break;
+        case "Set":
+        case "WeakSet":
+          ifNotVisited(value, computeFromSet);
+          break;
+        default:
+          invariant(kind === "Object", `Object of kind ${kind} is not supported in calls to abstract functions.`);
+          invariant(
+            value.$ParameterMap === undefined,
+            `Arguments object is not supported in calls to abstract functions.`
+          );
+          break;
+      }
+      if (!objectsToMaterialize.has(value)) objectsToMaterialize.add(value);
+    }
+    function computeFromDescriptor(descriptor: Descriptor): void {
+      if (descriptor === undefined) {
+      } else if (descriptor instanceof PropertyDescriptor) {
+        if (descriptor.value !== undefined) computeFromValue(descriptor.value);
+        if (descriptor.get !== undefined) computeFromValue(descriptor.get);
+        if (descriptor.set !== undefined) computeFromValue(descriptor.set);
+      } else if (descriptor instanceof AbstractJoinedDescriptor) {
+        computeFromValue(descriptor.joinCondition);
+        if (descriptor.descriptor1 !== undefined) computeFromDescriptor(descriptor.descriptor1);
+        if (descriptor.descriptor2 !== undefined) computeFromDescriptor(descriptor.descriptor2);
+      } else {
+        invariant(false, "unknown descriptor");
+      }
+    }
+    function computeFromObjectPropertyBinding(binding: PropertyBinding): void {
+      let descriptor = binding.descriptor;
+      if (descriptor === undefined) return; //deleted
+      computeFromDescriptor(descriptor);
+    }
+
+    function computeFromObjectProperties(obj: ObjectValue, kind?: ObjectKind): void {
+      // symbol properties
+      for (let [, propertyBindingValue] of obj.symbols) {
+        invariant(propertyBindingValue);
+        computeFromObjectPropertyBinding(propertyBindingValue);
+      }
+
+      // string properties
+      for (let [, propertyBindingValue] of obj.properties) {
+        invariant(propertyBindingValue);
+        computeFromObjectPropertyBinding(propertyBindingValue);
+      }
+
+      // inject properties with computed names
+      if (obj.unknownProperty !== undefined) {
+        let descriptor = obj.unknownProperty.descriptor;
+        computeFromObjectPropertiesWithComputedNamesDescriptor(descriptor);
+      }
+
+      // prototype
+      computeFromObjectPrototype(obj);
+    }
+    function computeFromObjectPrototype(obj: ObjectValue) {
+      computeFromValue(obj.$Prototype);
+    }
+    function computeFromFunctionValue(fn: FunctionValue) {
+      computeFromObjectProperties(fn);
+
+      if (fn instanceof BoundFunctionValue) {
+        computeFromValue(fn.$BoundTargetFunction);
+        computeFromValue(fn.$BoundThis);
+        for (let boundArg of fn.$BoundArguments) computeFromValue(boundArg);
+        return;
+      }
+
+      invariant(
+        !(fn instanceof NativeFunctionValue),
+        "all native function values should have already been created outside this pure function"
+      );
+
+      let nonLocalReadBindings = nonLocalReadBindingsOfFunction(fn);
+      computeFromBindings(fn, nonLocalReadBindings);
+    }
+
+    function computeFromObjectPropertiesWithComputedNamesDescriptor(descriptor: void | Descriptor): void {
+      // TODO: #2484
+      notSupportedForTransitiveMaterialization();
+    }
+
+    function computeFromMap(val: ObjectValue): void {
+      let kind = val.getKind();
+
+      let entries;
+      if (kind === "Map") {
+        entries = val.$MapData;
+      } else {
+        invariant(kind === "WeakMap");
+        entries = val.$WeakMapData;
+      }
+      invariant(entries !== undefined);
+      let len = entries.length;
+
+      for (let i = 0; i < len; i++) {
+        let entry = entries[i];
+        let key = entry.$Key;
+        let value = entry.$Value;
+        if (key === undefined || value === undefined) continue;
+        computeFromValue(key);
+        computeFromValue(value);
+      }
+    }
+
+    function computeFromSet(val: ObjectValue): void {
+      let kind = val.getKind();
+
+      let entries;
+      if (kind === "Set") {
+        entries = val.$SetData;
+      } else {
+        invariant(kind === "WeakSet");
+        entries = val.$WeakSetData;
+      }
+      invariant(entries !== undefined);
+      let len = entries.length;
+
+      for (let i = 0; i < len; i++) {
+        let entry = entries[i];
+        if (entry === undefined) continue;
+
+        computeFromValue(entry);
+      }
+    }
+
+    function nonLocalReadBindingsOfFunction(func: FunctionValue) {
+      // unboundWrites is currently not used, but we leave it in place
+      // to reuse the function closure visitor implemented for leaking
+      let functionInfo = {
+        unboundReads: new Set(),
+        unboundWrites: new Set(),
+      };
+
+      invariant(func instanceof ECMAScriptSourceFunctionValue);
+
+      let formalParameters = func.$FormalParameters;
+      invariant(formalParameters != null);
+
+      let code = func.$ECMAScriptCode;
+      invariant(code != null);
+
+      traverse(
+        t.file(t.program([t.expressionStatement(t.functionExpression(null, formalParameters, code))])),
+        LeakedClosureRefVisitor,
+        null,
+        functionInfo
+      );
+      traverse.cache.clear();
+
+      // TODO #2478: add invariant that there are no write bindings
+      return functionInfo.unboundReads;
+    }
+    function ifNotVisited<T>(value: T, computeFrom: T => void): void {
+      if (!visitedValues.has(value)) {
+        visitedValues.add(value);
+        computeFrom(value);
+      }
+    }
+    function visit(value: Value): void {
+      visitedValues.add(value);
+    }
+    function notSupportedForTransitiveMaterialization() {
+      let error = new CompilerDiagnostic(
+        "Not supported for transitive materialization",
+        outlinedFunction.expressionLocation,
+        "PP0041",
+        "FatalError"
+      );
+      realm.handleError(error);
+      throw new FatalError();
+    }
+  }
 }

src/values/ArrayValue.js

@@ -21,7 +21,7 @@ import {
   Value,
 } from "./index.js";
 import { IsAccessorDescriptor, IsPropertyKey, IsArrayIndex } from "../methods/is.js";
-import { Leak, Properties, To, Utils } from "../singletons.js";
+import { Leak, Materialize, Properties, To, Utils } from "../singletons.js";
 import { type OperationDescriptor } from "../utils/generator.js";
 import invariant from "../invariant.js";
 import { NestedOptimizedFunctionSideEffect } from "../errors.js";
@@ -64,6 +64,28 @@ function evaluatePossibleNestedOptimizedFunctionsAndStoreEffects(
       }
       throw e;
     }
+
+    // This is an incremental step from this list aimed to resolve a particular issue: #2452
+    //
+    // Assumptions:
+    // 1. We are here because the array op is pure, havocing of bindings is not needed.
+    // 2. The array op is only used once. To be enforced: #2448
+    // 3. Aliasing effects will lead to a fatal error. To be enforced: #2449
+    // 4. Indices of a widened array are not backed by locations
+    //
+    // Transitive materialization is needed to unblock this issue: #2405
+    //
+    // The bindings themselves do not have to materialize, since the values in them
+    // are used to optimize the nested optimized function. We compute the set of
+    // objects that are transitively reachable from read bindings and materialize them.
+
+    Materialize.materializeObjectsTransitive(realm, func);
+
+    // We assume that we do not have to materialize widened arrays because they are intrinsic.
+    // If somebody changes the underlying design in a major way, then materialization could be
+    // needed, and this check will fail.
+    invariant(abstractArrayValue.isIntrinsic());
+
     // Check if effects were pure then add them
     if (abstractArrayValue.nestedOptimizedFunctionEffects === undefined) {
       abstractArrayValue.nestedOptimizedFunctionEffects = new Map();

test/serializer/additional-functions/NestedOptimizedFunction18.js

@@ -0,0 +1,39 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function Component(x) {
+  this.val = x;
+}
+
+function foo(a, b, c, d) {
+  if (!a) {
+    return null;
+  }
+  var arr = Array.from(c);
+  var _ref11;
+  var x = (_ref11 = b) != null ? ((_ref11 = _ref11.feedback) != null ? _ref11.display_comments : _ref11) : _ref11;
+
+  var a = new Component(x);
+  var mappedArr = arr.map(function() {
+    return a;
+  });
+  return d(mappedArr);
+}
+
+global.__optimize && __optimize(foo);
+
+inspect = function() {
+  function func(arr) {
+    return arr.map(item => item.val).join();
+  }
+  var val = foo(
+    true,
+    {
+      feedback: {
+        display_comments: 5,
+      },
+    },
+    [, , ,],
+    func
+  );
+  return JSON.stringify(val);
+};

test/serializer/optimized-functions/TransitiveMaterializationDirect.js

@@ -0,0 +1,20 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c) {
+  var arr = Array.from(c);
+  let obj = { foo: 1 };
+
+  function op(x) {
+    return obj;
+  }
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+global.__optimize && __optimize(f);
+
+inspect = () => f([0]);

test/serializer/optimized-functions/TransitiveMaterializationViaAbstractValue.js

@@ -0,0 +1,21 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c, b) {
+  var arr = Array.from(c);
+  let obj = b ? { foo: 1 } : { foo: 2 };
+
+  function op(x) {
+    return obj;
+  }
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+
+global.__optimize && __optimize(f);
+
+inspect = () => f([0], true);

test/serializer/optimized-functions/TransitiveMaterializationViaFunctionCall.js

@@ -0,0 +1,26 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c) {
+  var arr = Array.from(c);
+  let obj = { foo: 1 };
+
+  function nested(x) {
+    let mapped_inner = arr.map(x => obj);
+    return mapped_inner[0];
+  }
+
+  function op(x) {
+    return nested(x);
+  }
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+
+global.__optimize && __optimize(f);
+
+inspect = () => f([0]);

test/serializer/optimized-functions/TransitiveMaterializationViaFunctionProperty.js

@@ -0,0 +1,22 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c) {
+  var arr = Array.from(c);
+  let obj = { foo: 1 };
+
+  function op(x) {
+    return op.obj;
+  }
+
+  op.obj = obj;
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+global.__optimize && __optimize(f);
+
+inspect = () => f([0]);

test/serializer/optimized-functions/TransitiveMaterializationViaLeakedBinding.js

@@ -0,0 +1,27 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c, g) {
+  var arr = Array.from(c);
+  var leaked = undefined;
+  let obj = { foo: 1 };
+
+  function leak() {
+    return leaked;
+  }
+  g(leak);
+  leaked = obj;
+
+  function op(x) {
+    return leaked;
+  }
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+global.__optimize && __optimize(f);
+
+inspect = () => f([0], () => {});

test/serializer/optimized-functions/TransitiveMaterializationViaObjectProp.js

@@ -0,0 +1,21 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c) {
+  var arr = Array.from(c);
+  let obj = { foo: 1 };
+  let obj2 = { bar: obj };
+
+  function op(x) {
+    return obj2;
+  }
+
+  let mapped = arr.map(op);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+global.__optimize && __optimize(f);
+
+inspect = () => f([0]);

test/serializer/optimized-functions/TransitiveMaterializationViaThisBinding.js

@@ -0,0 +1,21 @@
+// arrayNestedOptimizedFunctionsEnabled
+
+function f(c) {
+  var arr = Array.from(c);
+  let obj = { foo: 1 };
+
+  function op(x) {
+    return this;
+  }
+
+  let bop = op.bind(obj);
+  let mapped = arr.map(bop);
+  let val = arr[0].foo;
+  let ret = mapped[0].foo;
+  obj.foo = 2;
+
+  return ret;
+}
+global.__optimize && __optimize(f);
+
+inspect = () => f([0]);