mirror of
https://github.com/pulsar-edit/pulsar.git
synced 2024-09-19 23:17:16 +03:00
Speed up FileSystemBlobStore.load by not storing invalidation keys
This was unneeded because we can simply compute the cache key by concatenating the v8 version and the file's contents.
This commit is contained in:
parent
4b98843fae
commit
0d29004723
@ -14,79 +14,75 @@ describe "FileSystemBlobStore", ->
|
||||
fs.removeSync(storageDirectory)
|
||||
|
||||
it "is empty when the file doesn't exist", ->
|
||||
expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
|
||||
expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined()
|
||||
expect(blobStore.get("foo")).toBeUndefined()
|
||||
expect(blobStore.get("bar")).toBeUndefined()
|
||||
|
||||
it "allows to read and write buffers from/to memory without persisting them", ->
|
||||
blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
|
||||
blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
|
||||
blobStore.set("foo", new Buffer("foo"))
|
||||
blobStore.set("bar", new Buffer("bar"))
|
||||
|
||||
expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo"))
|
||||
expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar"))
|
||||
expect(blobStore.get("foo")).toEqual(new Buffer("foo"))
|
||||
expect(blobStore.get("bar")).toEqual(new Buffer("bar"))
|
||||
|
||||
expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("bar", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("baz")).toBeUndefined()
|
||||
expect(blobStore.get("qux")).toBeUndefined()
|
||||
|
||||
it "persists buffers when saved and retrieves them on load, giving priority to in-memory ones", ->
|
||||
blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
|
||||
blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
|
||||
blobStore.set("foo", new Buffer("foo"))
|
||||
blobStore.set("bar", new Buffer("bar"))
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
expect(blobStore.get("foo", "invalidation-key-1")).toEqual(new Buffer("foo"))
|
||||
expect(blobStore.get("bar", "invalidation-key-2")).toEqual(new Buffer("bar"))
|
||||
expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("bar", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("foo")).toEqual(new Buffer("foo"))
|
||||
expect(blobStore.get("bar")).toEqual(new Buffer("bar"))
|
||||
expect(blobStore.get("baz")).toBeUndefined()
|
||||
expect(blobStore.get("qux")).toBeUndefined()
|
||||
|
||||
blobStore.set("foo", "new-key", new Buffer("changed"))
|
||||
blobStore.set("foo", new Buffer("changed"))
|
||||
|
||||
expect(blobStore.get("foo", "new-key")).toEqual(new Buffer("changed"))
|
||||
expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
|
||||
expect(blobStore.get("foo")).toEqual(new Buffer("changed"))
|
||||
|
||||
it "persists in-memory and previously stored buffers, and deletes unused keys when saved", ->
|
||||
blobStore.set("foo", "invalidation-key-1", new Buffer("foo"))
|
||||
blobStore.set("bar", "invalidation-key-2", new Buffer("bar"))
|
||||
blobStore.set("foo", new Buffer("foo"))
|
||||
blobStore.set("bar", new Buffer("bar"))
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
blobStore.set("bar", "invalidation-key-3", new Buffer("changed"))
|
||||
blobStore.set("qux", "invalidation-key-4", new Buffer("qux"))
|
||||
blobStore.set("bar", new Buffer("changed"))
|
||||
blobStore.set("qux", new Buffer("qux"))
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
expect(blobStore.get("foo", "invalidation-key-1")).toBeUndefined()
|
||||
expect(blobStore.get("bar", "invalidation-key-3")).toEqual(new Buffer("changed"))
|
||||
expect(blobStore.get("qux", "invalidation-key-4")).toEqual(new Buffer("qux"))
|
||||
expect(blobStore.get("foo", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("bar", "invalidation-key-2")).toBeUndefined()
|
||||
expect(blobStore.get("qux", "unexisting-key")).toBeUndefined()
|
||||
expect(blobStore.get("foo")).toBeUndefined()
|
||||
expect(blobStore.get("bar")).toEqual(new Buffer("changed"))
|
||||
expect(blobStore.get("qux")).toEqual(new Buffer("qux"))
|
||||
|
||||
it "allows to delete keys from both memory and stored buffers", ->
|
||||
blobStore.set("a", "invalidation-key-1", new Buffer("a"))
|
||||
blobStore.set("b", "invalidation-key-2", new Buffer("b"))
|
||||
blobStore.set("a", new Buffer("a"))
|
||||
blobStore.set("b", new Buffer("b"))
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
blobStore.get("a", "invalidation-key-1") # prevent the key from being deleted on save
|
||||
blobStore.set("b", "invalidation-key-3", new Buffer("b"))
|
||||
blobStore.set("c", "invalidation-key-4", new Buffer("c"))
|
||||
blobStore.get("a") # prevent the key from being deleted on save
|
||||
blobStore.set("b", new Buffer("b"))
|
||||
blobStore.set("c", new Buffer("c"))
|
||||
blobStore.delete("b")
|
||||
blobStore.delete("c")
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("a"))
|
||||
expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined()
|
||||
expect(blobStore.get("b", "invalidation-key-3")).toBeUndefined()
|
||||
expect(blobStore.get("c", "invalidation-key-4")).toBeUndefined()
|
||||
expect(blobStore.get("a")).toEqual(new Buffer("a"))
|
||||
expect(blobStore.get("b")).toBeUndefined()
|
||||
expect(blobStore.get("b")).toBeUndefined()
|
||||
expect(blobStore.get("c")).toBeUndefined()
|
||||
|
||||
it "ignores errors when loading an invalid blob store", ->
|
||||
blobStore.set("a", "invalidation-key-1", new Buffer("a"))
|
||||
blobStore.set("b", "invalidation-key-2", new Buffer("b"))
|
||||
blobStore.set("a", new Buffer("a"))
|
||||
blobStore.set("b", new Buffer("b"))
|
||||
blobStore.save()
|
||||
|
||||
# Simulate corruption
|
||||
@ -96,14 +92,14 @@ describe "FileSystemBlobStore", ->
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
expect(blobStore.get("a", "invalidation-key-1")).toBeUndefined()
|
||||
expect(blobStore.get("b", "invalidation-key-2")).toBeUndefined()
|
||||
expect(blobStore.get("a")).toBeUndefined()
|
||||
expect(blobStore.get("b")).toBeUndefined()
|
||||
|
||||
blobStore.set("a", "invalidation-key-1", new Buffer("x"))
|
||||
blobStore.set("b", "invalidation-key-2", new Buffer("y"))
|
||||
blobStore.set("a", new Buffer("x"))
|
||||
blobStore.set("b", new Buffer("y"))
|
||||
blobStore.save()
|
||||
|
||||
blobStore = FileSystemBlobStore.load(storageDirectory)
|
||||
|
||||
expect(blobStore.get("a", "invalidation-key-1")).toEqual(new Buffer("x"))
|
||||
expect(blobStore.get("b", "invalidation-key-2")).toEqual(new Buffer("y"))
|
||||
expect(blobStore.get("a")).toEqual(new Buffer("x"))
|
||||
expect(blobStore.get("b")).toEqual(new Buffer("y"))
|
||||
|
@ -9,16 +9,18 @@ describe "NativeCompileCache", ->
|
||||
beforeEach ->
|
||||
cachedFiles = []
|
||||
fakeCacheStore = jasmine.createSpyObj("cache store", ["set", "get", "has", "delete"])
|
||||
fakeCacheStore.has.andCallFake (cacheKey, invalidationKey) ->
|
||||
fakeCacheStore.get(cacheKey, invalidationKey)?
|
||||
fakeCacheStore.get.andCallFake (cacheKey, invalidationKey) ->
|
||||
|
||||
fakeCacheStore.has.andCallFake (cacheKey) ->
|
||||
fakeCacheStore.get(cacheKey)?
|
||||
|
||||
fakeCacheStore.get.andCallFake (cacheKey) ->
|
||||
for entry in cachedFiles by -1
|
||||
continue if entry.cacheKey isnt cacheKey
|
||||
continue if entry.invalidationKey isnt invalidationKey
|
||||
return entry.cacheBuffer
|
||||
return
|
||||
fakeCacheStore.set.andCallFake (cacheKey, invalidationKey, cacheBuffer) ->
|
||||
cachedFiles.push({cacheKey, invalidationKey, cacheBuffer})
|
||||
|
||||
fakeCacheStore.set.andCallFake (cacheKey, cacheBuffer) ->
|
||||
cachedFiles.push({cacheKey, cacheBuffer})
|
||||
|
||||
nativeCompileCache.setCacheStore(fakeCacheStore)
|
||||
nativeCompileCache.setV8Version("a-v8-version")
|
||||
@ -29,13 +31,10 @@ describe "NativeCompileCache", ->
|
||||
fn2 = require('./fixtures/native-cache/file-2')
|
||||
|
||||
expect(cachedFiles.length).toBe(2)
|
||||
|
||||
expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-1'))
|
||||
expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
|
||||
expect(fn1()).toBe(1)
|
||||
|
||||
expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-2'))
|
||||
expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
|
||||
expect(fn2()).toBe(2)
|
||||
@ -51,7 +50,6 @@ describe "NativeCompileCache", ->
|
||||
fn4 = require('./fixtures/native-cache/file-4')
|
||||
|
||||
expect(cachedFiles.length).toBe(1)
|
||||
expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4'))
|
||||
expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
|
||||
expect(fn4()).toBe("file-4")
|
||||
@ -61,8 +59,6 @@ describe "NativeCompileCache", ->
|
||||
fn4 = require('./fixtures/native-cache/file-4')
|
||||
|
||||
expect(cachedFiles.length).toBe(2)
|
||||
expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-4'))
|
||||
expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey)
|
||||
expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
|
||||
|
||||
@ -79,7 +75,6 @@ describe "NativeCompileCache", ->
|
||||
fn5 = require('./fixtures/native-cache/file-5')
|
||||
|
||||
expect(cachedFiles.length).toBe(1)
|
||||
expect(cachedFiles[0].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5'))
|
||||
expect(cachedFiles[0].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[0].cacheBuffer.length).toBeGreaterThan(0)
|
||||
expect(fn5()).toBe("file-5")
|
||||
@ -89,8 +84,6 @@ describe "NativeCompileCache", ->
|
||||
fn5 = require('./fixtures/native-cache/file-5')
|
||||
|
||||
expect(cachedFiles.length).toBe(2)
|
||||
expect(cachedFiles[1].cacheKey).toBe(require.resolve('./fixtures/native-cache/file-5'))
|
||||
expect(cachedFiles[1].invalidationKey).not.toBe(cachedFiles[0].invalidationKey)
|
||||
expect(cachedFiles[1].cacheBuffer).toBeInstanceOf(Uint8Array)
|
||||
expect(cachedFiles[1].cacheBuffer.length).toBeGreaterThan(0)
|
||||
|
||||
@ -100,5 +93,5 @@ describe "NativeCompileCache", ->
|
||||
|
||||
fn3 = require('./fixtures/native-cache/file-3')
|
||||
|
||||
expect(fakeCacheStore.delete).toHaveBeenCalledWith(require.resolve('./fixtures/native-cache/file-3'))
|
||||
expect(fakeCacheStore.delete).toHaveBeenCalled()
|
||||
expect(fn3()).toBe(3)
|
||||
|
@ -14,14 +14,12 @@ class FileSystemBlobStore {
|
||||
constructor (directory) {
|
||||
this.blobFilename = path.join(directory, 'BLOB')
|
||||
this.blobMapFilename = path.join(directory, 'MAP')
|
||||
this.invalidationKeysFilename = path.join(directory, 'INVKEYS')
|
||||
this.lockFilename = path.join(directory, 'LOCK')
|
||||
this.reset()
|
||||
}
|
||||
|
||||
reset () {
|
||||
this.inMemoryBlobs = new Map()
|
||||
this.invalidationKeys = {}
|
||||
this.storedBlob = new Buffer(0)
|
||||
this.storedBlobMap = {}
|
||||
this.usedKeys = new Set()
|
||||
@ -34,14 +32,10 @@ class FileSystemBlobStore {
|
||||
if (!fs.existsSync(this.blobFilename)) {
|
||||
return
|
||||
}
|
||||
if (!fs.existsSync(this.invalidationKeysFilename)) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
this.storedBlob = fs.readFileSync(this.blobFilename)
|
||||
this.storedBlobMap = JSON.parse(fs.readFileSync(this.blobMapFilename))
|
||||
this.invalidationKeys = JSON.parse(fs.readFileSync(this.invalidationKeysFilename))
|
||||
} catch (e) {
|
||||
this.reset()
|
||||
}
|
||||
@ -51,7 +45,6 @@ class FileSystemBlobStore {
|
||||
let dump = this.getDump()
|
||||
let blobToStore = Buffer.concat(dump[0])
|
||||
let mapToStore = JSON.stringify(dump[1])
|
||||
let invalidationKeysToStore = JSON.stringify(this.invalidationKeys)
|
||||
|
||||
let acquiredLock = false
|
||||
try {
|
||||
@ -60,7 +53,6 @@ class FileSystemBlobStore {
|
||||
|
||||
fs.writeFileSync(this.blobFilename, blobToStore)
|
||||
fs.writeFileSync(this.blobMapFilename, mapToStore)
|
||||
fs.writeFileSync(this.invalidationKeysFilename, invalidationKeysToStore)
|
||||
} catch (error) {
|
||||
// Swallow the exception silently only if we fail to acquire the lock.
|
||||
if (error.code !== 'EEXIST') {
|
||||
@ -73,22 +65,19 @@ class FileSystemBlobStore {
|
||||
}
|
||||
}
|
||||
|
||||
has (key, invalidationKey) {
|
||||
let containsKey = this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key)
|
||||
let isValid = this.invalidationKeys[key] === invalidationKey
|
||||
return containsKey && isValid
|
||||
has (key) {
|
||||
return this.inMemoryBlobs.has(key) || this.storedBlobMap.hasOwnProperty(key)
|
||||
}
|
||||
|
||||
get (key, invalidationKey) {
|
||||
if (this.has(key, invalidationKey)) {
|
||||
get (key) {
|
||||
if (this.has(key)) {
|
||||
this.usedKeys.add(key)
|
||||
return this.getFromMemory(key) || this.getFromStorage(key)
|
||||
}
|
||||
}
|
||||
|
||||
set (key, invalidationKey, buffer) {
|
||||
set (key, buffer) {
|
||||
this.usedKeys.add(key)
|
||||
this.invalidationKeys[key] = invalidationKey
|
||||
return this.inMemoryBlobs.set(key, buffer)
|
||||
}
|
||||
|
||||
|
@ -60,11 +60,10 @@ class NativeCompileCache {
|
||||
// create wrapper function
|
||||
let wrapper = Module.wrap(content)
|
||||
|
||||
let cacheKey = filename
|
||||
let invalidationKey = computeHash(wrapper + self.v8Version)
|
||||
let cacheKey = computeHash(wrapper + self.v8Version)
|
||||
let compiledWrapper = null
|
||||
if (self.cacheStore.has(cacheKey, invalidationKey)) {
|
||||
let buffer = self.cacheStore.get(cacheKey, invalidationKey)
|
||||
if (self.cacheStore.has(cacheKey)) {
|
||||
let buffer = self.cacheStore.get(cacheKey)
|
||||
let compilationResult = cachedVm.runInThisContextCached(wrapper, filename, buffer)
|
||||
compiledWrapper = compilationResult.result
|
||||
if (compilationResult.wasRejected) {
|
||||
@ -79,7 +78,7 @@ class NativeCompileCache {
|
||||
throw err
|
||||
}
|
||||
if (compilationResult.cacheBuffer) {
|
||||
self.cacheStore.set(cacheKey, invalidationKey, compilationResult.cacheBuffer)
|
||||
self.cacheStore.set(cacheKey, compilationResult.cacheBuffer)
|
||||
}
|
||||
compiledWrapper = compilationResult.result
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user