Re-enable native module cache

spec/file-system-blob-store-spec.coffee

@@ -0,0 +1,109 @@
+temp = require('temp').track()
+path = require 'path'
+fs = require 'fs-plus'
+FileSystemBlobStore = require '../src/file-system-blob-store'
+
+describe "FileSystemBlobStore", ->
+  [storageDirectory, blobStore] = []
+
+  beforeEach ->
+    storageDirectory = temp.path('atom-spec-filesystemblobstore')
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+  afterEach ->
+    fs.removeSync(storageDirectory)
+
+  it "is empty when the file doesn't exist", ->
+    expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
+    expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined()
+
+  it "allows to read and write buffers from/to memory without persisting them", ->
+    blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
+    blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
+
+    expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo"))
+    expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar"))
+
+    expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
+    expect(blobStore.get("bar", "unexisting-key")).toBeUndefined()
+
+  it "persists buffers when saved and retrieves them on load, giving priority to in-memory ones", ->
+    blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
+    blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo"))
+    expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar"))
+    expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
+    expect(blobStore.get("bar", "unexisting-key")).toBeUndefined()
+
+    blobStore.set("foo", "new-key", new Buffer("changed"))
+
+    expect(blobStore.get("foo", "new-key")).toEqual(new Buffer("changed"))
+    expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
+
+  it "persists in-memory and previously stored buffers, and deletes unused keys when saved", ->
+    blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
+    blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+    blobStore.set("bar", "invalidation-key-3", new Buffer("changed"))
+    blobStore.set("qux", "invalidation-key-4", new Buffer("qux"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
+    expect(blobStore.get("bar", "invalidation-key-3")).toEqual(new Buffer("changed"))
+    expect(blobStore.get("qux", "invalidation-key-4")).toEqual(new Buffer("qux"))
+    expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
+    expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined()
+    expect(blobStore.get("qux", "unexisting-key")).toBeUndefined()
+
+  it "allows to delete keys from both memory and stored buffers", ->
+    blobStore.set("a", "invalidation-key-1", new Buffer("a"))
+    blobStore.set("b", "invalidation-key-2", new Buffer("b"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    blobStore.get("a", "invalidation-key-1") # prevent the key from being deleted on save
+    blobStore.set("b", "invalidation-key-3", new Buffer("b"))
+    blobStore.set("c", "invalidation-key-4", new Buffer("c"))
+    blobStore.delete("b")
+    blobStore.delete("c")
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("a"))
+    expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined()
+    expect(blobStore.get("b", "invalidation-key-3")).toBeUndefined()
+    expect(blobStore.get("c", "invalidation-key-4")).toBeUndefined()
+
+  it "ignores errors when loading an invalid blob store", ->
+    blobStore.set("a", "invalidation-key-1", new Buffer("a"))
+    blobStore.set("b", "invalidation-key-2", new Buffer("b"))
+    blobStore.save()
+
+    # Simulate corruption
+    fs.writeFileSync(path.join(storageDirectory, "MAP"), new Buffer([0]))
+    fs.writeFileSync(path.join(storageDirectory, "INVKEYS"), new Buffer([0]))
+    fs.writeFileSync(path.join(storageDirectory, "BLOB"), new Buffer([0]))
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("a", "invalidation-key-1")).toBeUndefined()
+    expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined()
+
+    blobStore.set("a", "invalidation-key-1", new Buffer("x"))
+    blobStore.set("b", "invalidation-key-2", new Buffer("y"))
+    blobStore.save()
+
+    blobStore = FileSystemBlobStore.load(storageDirectory)
+
+    expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("x"))
+    expect(blobStore.get("b", "invalidation-key-2")).toEqual(new Buffer("y"))

spec/fixtures/native-cache/file-1.js

@@ -0,0 +1 @@
+module.exports = function () { return 1; }

spec/fixtures/native-cache/file-2.js

@@ -0,0 +1 @@
+module.exports = function () { return 2; }

spec/fixtures/native-cache/file-3.js

@@ -0,0 +1 @@
+module.exports = function () { return 3; }

spec/fixtures/native-cache/file-4.js

@@ -0,0 +1 @@
+module.exports = function () { return "file-4" }

spec/native-compile-cache-spec.coffee

@@ -0,0 +1,104 @@
+fs = require 'fs'
+path = require 'path'
+Module = require 'module'
+
+describe "NativeCompileCache", ->
+  nativeCompileCache = require '../src/native-compile-cache'
+  [fakeCacheStore, cachedFiles] = []
+
+  beforeEach ->
+    cachedFiles = []
+    fakeCacheStore = jasmine.createSpyObj("cache store", ["set", "get", "has", "delete"])
+    fakeCacheStore.has.andCallFake (cacheKey, invalidationKey) ->
+      fakeCacheStore.get(cacheKey, invalidationKey)?
+    fakeCacheStore.get.andCallFake (cacheKey, invalidationKey) ->
+      for entry in cachedFiles by -1
+        continue if entry.cacheKey isnt cacheKey
+        continue if entry.invalidationKey isnt invalidationKey
+        return entry.cacheBuffer
+      return
+    fakeCacheStore.set.andCallFake (cacheKey, invalidationKey, cacheBuffer) ->
+      cachedFiles.push({cacheKey, invalidationKey, cacheBuffer})
+
+    nativeCompileCache.setCacheStore(fakeCacheStore)
+    nativeCompileCache.setV8Version("a-v8-version")
+    nativeCompileCache.install()
+
+  it "writes and reads from the cache storage when requiring files", ->
+    fn1 = require('./fixtures/native-cache/file-1')
+    fn2 = require('./fixtures/native-cache/file-2')
+
+    expect(cachedFiles.length).toBe(2)
+
+    expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-1'))
+    expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
+    expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
+    expect(fn1()).toBe(1)
+
+    expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-2'))
+    expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
+    expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
+    expect(fn2()).toBe(2)
+
+    delete Module._cache[require.resolve('./fixtures/native-cache/file-1')]
+    fn1 = require('./fixtures/native-cache/file-1')
+    expect(cachedFiles.length).toBe(2)
+    expect(fn1()).toBe(1)
+
+  describe "when v8 version changes", ->
+    it "updates the cache of previously required files", ->
+      nativeCompileCache.setV8Version("version-1")
+      fn4 = require('./fixtures/native-cache/file-4')
+
+      expect(cachedFiles.length).toBe(1)
+      expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4'))
+      expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
+      expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
+      expect(fn4()).toBe("file-4")
+
+      nativeCompileCache.setV8Version("version-2")
+      delete Module._cache[require.resolve('./fixtures/native-cache/file-4')]
+      fn4 = require('./fixtures/native-cache/file-4')
+
+      expect(cachedFiles.length).toBe(2)
+      expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4'))
+      expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey)
+      expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
+      expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
+
+  describe "when a previously required and cached file changes", ->
+    beforeEach ->
+      fs.writeFileSync path.resolve(__dirname + '/fixtures/native-cache/file-5'), """
+      module.exports = function () { return "file-5" }
+      """
+
+    afterEach ->
+      fs.unlinkSync path.resolve(__dirname + '/fixtures/native-cache/file-5')
+
+    it "removes it from the store and re-inserts it with the new cache", ->
+      fn5 = require('./fixtures/native-cache/file-5')
+
+      expect(cachedFiles.length).toBe(1)
+      expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5'))
+      expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
+      expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
+      expect(fn5()).toBe("file-5")
+
+      delete Module._cache[require.resolve('./fixtures/native-cache/file-5')]
+      fs.appendFileSync(require.resolve('./fixtures/native-cache/file-5'), "\n\n")
+      fn5 = require('./fixtures/native-cache/file-5')
+
+      expect(cachedFiles.length).toBe(2)
+      expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5'))
+      expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey)
+      expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
+      expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
+
+  it "deletes previously cached code when the cache is an invalid file", ->
+    fakeCacheStore.has.andReturn(true)
+    fakeCacheStore.get.andCallFake -> new Buffer("an invalid cache")
+
+    fn3 = require('./fixtures/native-cache/file-3')
+
+    expect(fakeCacheStore.delete).toHaveBeenCalledWith(require.resolve('./fixtures/native-cache/file-3'))
+    expect(fn3()).toBe(3)

src/atom-environment.coffee

@@ -131,7 +131,7 @@ class AtomEnvironment extends Model
 
   # Call .loadOrCreate instead
   constructor: (params={}) ->
-    {@applicationDelegate, @window, @document, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params
+    {@applicationDelegate, @window, @document, @blobStore, @clipboard, @configDirPath, @enablePersistence, onlyLoadBaseStyleSheets} = params
 
     @unloaded = false
     @loadTime = null
@@ -733,8 +733,13 @@ class AtomEnvironment extends Model
 
     @storeWindowBackground()
     @packages.deactivatePackages()
+    @saveBlobStoreSync()
     @unloaded = true
 
+  saveBlobStoreSync: ->
+    if @enablePersistence
+      @blobStore.save()
+
   openInitialEmptyEditorIfNecessary: ->
     return unless @config.get('core.openEmptyEditorOnStart')
     if @getLoadSettings().initialPaths?.length is 0 and @workspace.getPaneItems().length is 0

src/file-system-blob-store.js

@@ -0,0 +1,138 @@
+'use strict'
+
+const fs = require('fs-plus')
+const path = require('path')
+
+module.exports =
+class FileSystemBlobStore {
+  static load (directory) {
+    let instance = new FileSystemBlobStore(directory)
+    instance.load()
+    return instance
+  }
+
+  constructor (directory) {
+    this.blobFilename = path.join(directory, 'BLOB')
+    this.blobMapFilename = path.join(directory, 'MAP')
+    this.invalidationKeysFilename = path.join(directory, 'INVKEYS')
+    this.lockFilename = path.join(directory, 'LOCK')
+    this.reset()
+  }
+
+  reset () {
+    this.inMemoryBlobs = new Map()
+    this.invalidationKeys = {}
+    this.storedBlob = new Buffer(0)
+    this.storedBlobMap = {}
+    this.usedKeys = new Set()
+  }
+
+  load () {
+    if (!fs.existsSync(this.blobMapFilename)) {
+      return
+    }
+    if (!fs.existsSync(this.blobFilename)) {
+      return
+    }
+    if (!fs.existsSync(this.invalidationKeysFilename)) {
+      return
+    }
+
+    try {
+      this.storedBlob = fs.readFileSync(this.blobFilename)
+      this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename))
+      this.invalidationKeys = JSON.parse(fs.readFileSync(this.invalidationKeysFilename))
+    } catch (e) {
+      this.reset()
+    }
+  }
+
+  save () {
+    let dump = this.getDump()
+    let blobToStore = Buffer.concat(dump[0])
+    let mapToStore = JSON.stringify(dump[1])
+    let invalidationKeysToStore = JSON.stringify(this.invalidationKeys)
+
+    let acquiredLock = false
+    try {
+      fs.writeFileSync(this.lockFilename, 'LOCK', {flag: 'wx'})
+      acquiredLock = true
+
+      fs.writeFileSync(this.blobFilename, blobToStore)
+      fs.writeFileSync(this.blobMapFilename, mapToStore)
+      fs.writeFileSync(this.invalidationKeysFilename, invalidationKeysToStore)
+    } catch (error) {
+      // Swallow the exception silently only if we fail to acquire the lock.
+      if (error.code !== 'EEXIST') {
+        throw error
+      }
+    } finally {
+      if (acquiredLock) {
+        fs.unlinkSync(this.lockFilename)
+      }
+    }
+  }
+
+  has (key, invalidationKey) {
+    let containsKey = this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key)
+    let isValid = this.invalidationKeys[key] === invalidationKey
+    return containsKey && isValid
+  }
+
+  get (key, invalidationKey) {
+    if (this.has(key, invalidationKey)) {
+      this.usedKeys.add(key)
+      return this.getFromMemory(key) || this.getFromStorage(key)
+    }
+  }
+
+  set (key, invalidationKey, buffer) {
+    this.usedKeys.add(key)
+    this.invalidationKeys[key] = invalidationKey
+    return this.inMemoryBlobs.set(key, buffer)
+  }
+
+  delete (key) {
+    this.inMemoryBlobs.delete(key)
+    delete this.storedBlobMap[key]
+  }
+
+  getFromMemory (key) {
+    return this.inMemoryBlobs.get(key)
+  }
+
+  getFromStorage (key) {
+    if (!this.storedBlobMap[key]) {
+      return
+    }
+
+    return this.storedBlob.slice.apply(this.storedBlob, this.storedBlobMap[key])
+  }
+
+  getDump () {
+    let buffers = []
+    let blobMap = {}
+    let currentBufferStart = 0
+
+    function dump (key, getBufferByKey) {
+      let buffer = getBufferByKey(key)
+      buffers.push(buffer)
+      blobMap[key] = [currentBufferStart, currentBufferStart + buffer.length]
+      currentBufferStart += buffer.length
+    }
+
+    for (let key of this.inMemoryBlobs.keys()) {
+      if (this.usedKeys.has(key)) {
+        dump(key, this.getFromMemory.bind(this))
+      }
+    }
+
+    for (let key of Object.keys(this.storedBlobMap)) {
+      if (!blobMap[key] && this.usedKeys.has(key)) {
+        dump(key, this.getFromStorage.bind(this))
+      }
+    }
+
+    return [buffers, blobMap]
+  }
+}

src/initialize-application-window.coffee

@@ -3,6 +3,7 @@ ApplicationDelegate = require './application-delegate'
 Clipboard = require './clipboard'
 TextEditor = require './text-editor'
 TextEditorComponent = require './text-editor-component'
+FileSystemBlobStore = require './file-system-blob-store'
 CompileCache = require './compile-cache'
 ModuleCache = require './module-cache'
 
@@ -54,7 +55,7 @@ require('whitespace')
 require('wrap-guide')
 
 # Like sands through the hourglass, so are the days of our lives.
-module.exports = ->
+module.exports = ({blobStore}) ->
   {updateProcessEnv} = require('./update-process-env')
   path = require 'path'
   require './window'
@@ -75,7 +76,7 @@ module.exports = ->
   TextEditor.setClipboard(clipboard)
 
   window.atom = new AtomEnvironment({
-    window, document, clipboard,
+    window, document, clipboard, blobStore,
     applicationDelegate: new ApplicationDelegate,
     configDirPath: process.env.ATOM_HOME,
     enablePersistence: true,

src/initialize-test-window.coffee

@@ -5,7 +5,7 @@ cloneObject = (object) ->
   clone[key] = value for key, value of object
   clone
 
-module.exports = ->
+module.exports = ({blobStore}) ->
   startCrashReporter = require('./crash-reporter-start')
   {remote} = require 'electron'
 
@@ -77,6 +77,7 @@ module.exports = ->
     buildAtomEnvironment = (params) ->
       params = cloneObject(params)
       params.clipboard = clipboard unless params.hasOwnProperty("clipboard")
+      params.blobStore = blobStore unless params.hasOwnProperty("blobStore")
       params.onlyLoadBaseStyleSheets = true unless params.hasOwnProperty("onlyLoadBaseStyleSheets")
       new AtomEnvironment(params)
 

src/native-compile-cache.js

@@ -0,0 +1,116 @@
+const Module = require('module')
+const path = require('path')
+const cachedVm = require('cached-run-in-this-context')
+const crypto = require('crypto')
+
+function computeHash (contents) {
+  return crypto.createHash('sha1').update(contents, 'utf8').digest('hex')
+}
+
+class NativeCompileCache {
+  constructor () {
+    this.cacheStore = null
+    this.previousModuleCompile = null
+  }
+
+  setCacheStore (store) {
+    this.cacheStore = store
+  }
+
+  setV8Version (v8Version) {
+    this.v8Version = v8Version.toString()
+  }
+
+  install () {
+    this.savePreviousModuleCompile()
+    this.overrideModuleCompile()
+  }
+
+  uninstall () {
+    this.restorePreviousModuleCompile()
+  }
+
+  savePreviousModuleCompile () {
+    this.previousModuleCompile = Module.prototype._compile
+  }
+
+  overrideModuleCompile () {
+    let self = this
+    let resolvedArgv = null
+    // Here we override Node's module.js
+    // (https://github.com/atom/node/blob/atom/lib/module.js#L378), changing
+    // only the bits that affect compilation in order to use the cached one.
+    Module.prototype._compile = function (content, filename) {
+      let moduleSelf = this
+      // remove shebang
+      content = content.replace(/^#!.*/, '')
+      function require (path) {
+        return moduleSelf.require(path)
+      }
+      require.resolve = function (request) {
+        return Module._resolveFilename(request, moduleSelf)
+      }
+      require.main = process.mainModule
+
+      // Enable support to add extra extension types
+      require.extensions = Module._extensions
+      require.cache = Module._cache
+
+      let dirname = path.dirname(filename)
+
+      // create wrapper function
+      let wrapper = Module.wrap(content)
+
+      let cacheKey = filename
+      let invalidationKey = computeHash(wrapper + self.v8Version)
+      let compiledWrapper = null
+      if (self.cacheStore.has(cacheKey, invalidationKey)) {
+        let buffer = self.cacheStore.get(cacheKey, invalidationKey)
+        let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer)
+        compiledWrapper = compilationResult.result
+        if (compilationResult.wasRejected) {
+          self.cacheStore.delete(cacheKey)
+        }
+      } else {
+        let compilationResult
+        try {
+          compilationResult = cachedVm.runInThisContext(wrapper, filename)
+        } catch (err) {
+          console.error(`Error running script ${filename}`)
+          throw err
+        }
+        if (compilationResult.cacheBuffer) {
+          self.cacheStore.set(cacheKey, invalidationKey, compilationResult.cacheBuffer)
+        }
+        compiledWrapper = compilationResult.result
+      }
+      if (global.v8debug) {
+        if (!resolvedArgv) {
+          // we enter the repl if we're not given a filename argument.
+          if (process.argv[1]) {
+            resolvedArgv = Module._resolveFilename(process.argv[1], null)
+          } else {
+            resolvedArgv = 'repl'
+          }
+        }
+
+        // Set breakpoint on module start
+        if (filename === resolvedArgv) {
+          // Installing this dummy debug event listener tells V8 to start
+          // the debugger.  Without it, the setBreakPoint() fails with an
+          // 'illegal access' error.
+          global.v8debug.Debug.setListener(function () {})
+          global.v8debug.Debug.setBreakPoint(compiledWrapper, 0, 0)
+        }
+      }
+      let args = [moduleSelf.exports, require, moduleSelf, filename, dirname, process, global]
+      return compiledWrapper.apply(moduleSelf.exports, args)
+    }
+  }
+
+  restorePreviousModuleCompile () {
+    Module.prototype._compile = this.previousModuleCompile
+  }
+}
+
+module.exports = new NativeCompileCache()

static/index.js

@@ -4,6 +4,7 @@
   const Module = require('module')
   const getWindowLoadSettings = require('../src/get-window-load-settings')
   const entryPointDirPath = __dirname
+  let blobStore = null
   let useSnapshot = false
 
   window.onload = function () {
@@ -46,6 +47,14 @@
         snapshotResult.entryPointDirPath = __dirname
       }
 
+      const FileSystemBlobStore = useSnapshot ? snapshotResult.customRequire('../src/file-system-blob-store.js') : require('../src/file-system-blob-store')
+      blobStore = FileSystemBlobStore.load(path.join(process.env.ATOM_HOME, 'blob-store'))
+
+      const NativeCompileCache = useSnapshot ? snapshotResult.customRequire('../src/native-compile-cache.js') : require('../src/native-compile-cache')
+      NativeCompileCache.setCacheStore(blobStore)
+      NativeCompileCache.setV8Version(process.versions.v8)
+      NativeCompileCache.install()
+
       if (getWindowLoadSettings().profileStartup) {
         profileStartup(Date.now() - startTime)
       } else {
@@ -88,7 +97,7 @@
 
     const initScriptPath = path.relative(entryPointDirPath, getWindowLoadSettings().windowInitializationScript)
     const initialize = useSnapshot ? snapshotResult.customRequire(initScriptPath) : require(initScriptPath)
-    return initialize().then(function () {
+    return initialize({blobStore: blobStore}).then(function () {
       electron.ipcRenderer.send('window-command', 'window:loaded')
     })
   }