mirror of
https://github.com/facebook/sapling.git
synced 2024-10-09 08:18:15 +03:00
c08890f849
Summary: We're seeing that this is always set to true for eden, which is causing buck to run slower than it should. To make this work correctly, I've augmented our journal data structure so that it can track create, change and remove events for the various paths. I've also plumbed rename events into the journal. This requires a slightly more complex merge routine, so I've refactored the two call sites that were merging in slightly different contexts so that they can now share the same guts of the merge routine. Perhaps slightly counterintuitive in the merge code is that we merge a record from the past into the state for now and this is a bit backwards compared to how people think. I've expanded the eden integration test to check that we don't mix up create/change/removes for the same path in a given window. On the watchman side, we use the presence of the filename in the createdPaths set as a hint that the file is new. In that case we will set the watchman `ctime` (which is not the state ctime but is really the *created clock time*) to match the current journal position if the file is new, or leave it set to 0 if the file is not known to be new. This will cause the `is_new` flag to be set appropriately by the code in `watchman/query/eval.cpp`; if the sequence is 0 then it should never be set to true. Otherwise (when the file was in the `createPaths` set) it will be set to the current journal position and this will be seen as newer than the `since` constraint on the query and cause the file to show as `new`. Reviewed By: bolinfest Differential Revision: D5608538 fbshipit-source-id: 8d78f7da05e5e53110108aca220c3a97794f8cc2
172 lines
7.6 KiB
Python
172 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright (c) 2016-present, Facebook, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the BSD-style license found in the
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
import errno
|
|
import os
|
|
import stat
|
|
from .lib import testcase
|
|
from facebook.eden import EdenService
|
|
from facebook.eden.ttypes import FileInformationOrError
|
|
|
|
|
|
INITIAL_SEQ = 5
|
|
|
|
@testcase.eden_repo_test
|
|
class MaterializedQueryTest:
|
|
'''Check that materialization is represented correctly.'''
|
|
|
|
def populate_repo(self):
|
|
self.repo.write_file('hello', 'hola\n')
|
|
self.repo.write_file('adir/file', 'foo!\n')
|
|
self.repo.write_file('bdir/test.sh', '#!/bin/bash\necho test\n',
|
|
mode=0o755)
|
|
self.repo.write_file('bdir/noexec.sh', '#!/bin/bash\necho test\n')
|
|
self.repo.symlink('slink', 'hello')
|
|
self.repo.commit('Initial commit.')
|
|
|
|
def edenfs_logging_settings(self):
|
|
return {'eden.fs.fuse.RequestData': 'DBG5'}
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.client = self.get_thrift_client()
|
|
self.client.open()
|
|
|
|
def tearDown(self):
|
|
self.client.close()
|
|
super().tearDown()
|
|
|
|
def test_noEntries(self):
|
|
pos = self.client.getCurrentJournalPosition(self.mount)
|
|
# setting up the .eden dir bumps the sequence number
|
|
self.assertEqual(INITIAL_SEQ, pos.sequenceNumber)
|
|
self.assertNotEqual(0, pos.mountGeneration)
|
|
|
|
changed = self.client.getFilesChangedSince(self.mount, pos)
|
|
self.assertEqual(set(), set(changed.changedPaths))
|
|
self.assertEqual(set(), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.removedPaths))
|
|
self.assertEqual(pos, changed.fromPosition)
|
|
self.assertEqual(pos, changed.toPosition)
|
|
|
|
def test_getFileInformation(self):
|
|
""" verify that getFileInformation is consistent with the VFS """
|
|
|
|
paths = ['', 'not-exist', 'hello', 'adir',
|
|
'adir/file', 'bdir/test.sh', 'slink']
|
|
info_list = self.client.getFileInformation(self.mount, paths)
|
|
self.assertEqual(len(paths), len(info_list))
|
|
|
|
for idx, path in enumerate(paths):
|
|
try:
|
|
st = os.lstat(os.path.join(self.mount, path))
|
|
self.assertEqual(FileInformationOrError.INFO, info_list[
|
|
idx].getType(),
|
|
msg='have non-error result for ' + path)
|
|
info = info_list[idx].get_info()
|
|
self.assertEqual(st.st_mode, info.mode,
|
|
msg='mode matches for ' + path)
|
|
self.assertEqual(st.st_size, info.size,
|
|
msg='size matches for ' + path)
|
|
self.assertEqual(int(st.st_mtime), info.mtime.seconds)
|
|
if not stat.S_ISDIR(st.st_mode):
|
|
self.assertNotEqual(0, st.st_mtime)
|
|
self.assertNotEqual(0, st.st_ctime)
|
|
self.assertNotEqual(0, st.st_atime)
|
|
except OSError as e:
|
|
self.assertEqual(FileInformationOrError.ERROR, info_list[
|
|
idx].getType(),
|
|
msg='have error result for ' + path)
|
|
err = info_list[idx].get_error()
|
|
self.assertEqual(e.errno, err.errorCode,
|
|
msg='error code matches for ' + path)
|
|
|
|
def test_invalidProcessGeneration(self):
|
|
# Get a candidate position
|
|
pos = self.client.getCurrentJournalPosition(self.mount)
|
|
|
|
# poke the generation to a value that will never manifest in practice
|
|
pos.mountGeneration = 0
|
|
|
|
with self.assertRaises(EdenService.EdenError) as context:
|
|
self.client.getFilesChangedSince(self.mount, pos)
|
|
self.assertEqual(errno.ERANGE, context.exception.errorCode,
|
|
msg='Must return ERANGE')
|
|
|
|
def test_removeFile(self):
|
|
initial_pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
|
|
|
|
os.unlink(os.path.join(self.mount, 'adir', 'file'))
|
|
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
|
|
self.assertEqual(set(), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.changedPaths))
|
|
self.assertEqual(set(['adir/file']), set(changed.removedPaths))
|
|
|
|
def test_renameFile(self):
|
|
initial_pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
|
|
|
|
os.rename(os.path.join(self.mount, 'hello'), os.path.join(self.mount, 'bye'))
|
|
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
|
|
self.assertEqual(set(['bye']), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.changedPaths))
|
|
self.assertEqual(set(['hello']), set(changed.removedPaths))
|
|
|
|
def test_addFile(self):
|
|
initial_pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
|
|
|
|
name = os.path.join(self.mount, 'overlaid')
|
|
with open(name, 'w+') as f:
|
|
pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ + 1, pos.sequenceNumber,
|
|
msg='creating a file bumps the journal')
|
|
|
|
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
|
|
self.assertEqual(set(['overlaid']), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.changedPaths))
|
|
self.assertEqual(set(), set(changed.removedPaths))
|
|
self.assertEqual(initial_pos.sequenceNumber + 1,
|
|
changed.fromPosition.sequenceNumber,
|
|
msg='changes start AFTER initial_pos')
|
|
|
|
f.write('NAME!\n')
|
|
|
|
pos_after_overlaid = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ + 2, pos_after_overlaid.sequenceNumber,
|
|
msg='writing bumps the journal')
|
|
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
|
|
self.assertEqual(set(['overlaid']), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.changedPaths))
|
|
self.assertEqual(set(), set(changed.removedPaths))
|
|
self.assertEqual(initial_pos.sequenceNumber + 1,
|
|
changed.fromPosition.sequenceNumber,
|
|
msg='changes start AFTER initial_pos')
|
|
|
|
name = os.path.join(self.mount, 'adir', 'file')
|
|
with open(name, 'a') as f:
|
|
pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ + 2, pos.sequenceNumber,
|
|
msg='journal still in same place for append')
|
|
f.write('more stuff on the end\n')
|
|
|
|
pos = self.client.getCurrentJournalPosition(self.mount)
|
|
self.assertEqual(INITIAL_SEQ + 3, pos.sequenceNumber,
|
|
msg='appending bumps the journal')
|
|
|
|
changed = self.client.getFilesChangedSince(
|
|
self.mount, pos_after_overlaid)
|
|
self.assertEqual(set(['adir/file']), set(changed.changedPaths))
|
|
self.assertEqual(set(), set(changed.createdPaths))
|
|
self.assertEqual(set(), set(changed.removedPaths))
|
|
self.assertEqual(pos_after_overlaid.sequenceNumber + 1,
|
|
changed.fromPosition.sequenceNumber,
|
|
msg='changes start AFTER pos_after_overlaid')
|