mirror of
https://github.com/facebook/sapling.git
synced 2024-10-07 07:17:55 +03:00
fastmanifest: add cache limit dependant on the system's free space
Summary: This adds a new strategy for the cache limit that depends on the free space availabe in the system. For system with more than 100GB of free space we allocate 5GB for the cache. With more than 20 GB of free space we allocate 2GB of free space and otherwise 10% of the free space at most. Test Plan: Add a new test Reviewers: ttung, durham, rmcelroy Differential Revision: https://phabricator.intern.facebook.com/D3330894
This commit is contained in:
parent
37b0fb6f72
commit
0aa4b1c9a0
@ -449,6 +449,91 @@ class fixedcachelimit(object):
|
||||
return self._bytes
|
||||
|
||||
|
||||
GB = 1024**3
|
||||
MB = 1024**2
|
||||
DEFAULT_LOWGROWTH_TRESHOLDGB = 20
|
||||
DEFAULT_MAXCACHESIZEGB = 6
|
||||
DEFAULT_LOWGROWTH_SLOPE = 0.1
|
||||
DEFAULT_HIGHGROWTHSLOPE = 0.2
|
||||
|
||||
class systemawarecachelimit(object):
|
||||
"""A limit that will be tighter as the free disk space reduces"""
|
||||
def __init__(self, repo):
|
||||
# Probe the system root partition to know what is available
|
||||
st = os.statvfs(repo.root)
|
||||
self.free = st.f_bavail * st.f_frsize
|
||||
self.total = st.f_blocks * st.f_frsize
|
||||
# Read parameters from config
|
||||
self.lowgrowththresholdgb = repo.ui.config("fastmanifest",
|
||||
"lowgrowththresholdgb",
|
||||
DEFAULT_LOWGROWTH_TRESHOLDGB)
|
||||
self.lowgrowslope = repo.ui.config("fastmanifest",
|
||||
"lowgrowthslope",
|
||||
DEFAULT_LOWGROWTH_SLOPE)
|
||||
self.maxcachesizegb = repo.ui.config("fastmanifest",
|
||||
"maxcachesizegb",
|
||||
DEFAULT_MAXCACHESIZEGB)
|
||||
self.highgrowthslope = repo.ui.config("fastmanifest",
|
||||
"highgrowthslope",
|
||||
DEFAULT_HIGHGROWTHSLOPE)
|
||||
try:
|
||||
self.lowgrowththresholdgb = float(self.lowgrowththresholdgb)
|
||||
self.lowgrowslope = float(self.lowgrowslope)
|
||||
self.maxcachesizegb = float(self.maxcachesize)
|
||||
self.highgrowthslope = float(self.highgrowthslope)
|
||||
except ValueError:
|
||||
self.lowgrowththresholdgb = DEFAULT_LOWGROWTH_TRESHOLDGB
|
||||
self.lowgrowslope = DEFAULT_LOWGROWTH_SLOPE
|
||||
self.maxcachesizegb = DEFAULT_MAXCACHESIZEGB
|
||||
self.highgrowthslope = DEFAULT_HIGHGROWTHSLOPE
|
||||
|
||||
def bytes(self):
|
||||
return systemawarecachelimit.cacheallocation(
|
||||
self.free,
|
||||
lowgrowththresholdgb=self.lowgrowththresholdgb,
|
||||
highgrowthslope=self.highgrowthslope,
|
||||
maxcachesizegb=self.maxcachesizegb,
|
||||
lowgrowthslope=self.lowgrowthslope)
|
||||
|
||||
@staticmethod
|
||||
def cacheallocation(freespace,
|
||||
lowgrowththresholdgb=DEFAULT_LOWGROWTH_TRESHOLDGB,
|
||||
lowgrowslope=DEFAULT_LOWGROWTH_SLOPE,
|
||||
maxcachesizegb=DEFAULT_MAXCACHESIZEGB,
|
||||
highgrowthslope=DEFAULT_HIGHGROWTHSLOPE):
|
||||
"""Given the free space available in bytes, return the size of the cache
|
||||
|
||||
When disk space is limited (less than lowgrowththreshold), we increase
|
||||
the cache size linearly: lowgrowthslope * freespace. Over
|
||||
lowgrowththreshold, we increase the cache size linearly but faster:
|
||||
highgrowthslope * freespace until we hit maxcachesize.
|
||||
|
||||
These values are configurable, default values are:
|
||||
|
||||
[fastmanifest]
|
||||
lowgrowththresholdgb = 20
|
||||
lowgrowthslope = 0.1
|
||||
highgrowthslope = 0.2
|
||||
maxcachesizegb = 6
|
||||
|
||||
^ Cache Size
|
||||
|
|
||||
| /------------------- <- maxcachesize
|
||||
| |
|
||||
| / <- slope is highgrowthslope
|
||||
| | <- lowgrowththreshold
|
||||
| /
|
||||
| / <- slope is lowgrowslope
|
||||
|/
|
||||
-------------------------> Free Space
|
||||
"""
|
||||
|
||||
if freespace < lowgrowththresholdgb * GB:
|
||||
return min(maxcachesizegb * GB, lowgrowslope * freespace)
|
||||
else:
|
||||
return min(maxcachesizegb * GB, highgrowthslope * freespace)
|
||||
|
||||
|
||||
def _cachemanifestpruneall(ui, repo, background):
|
||||
if background:
|
||||
daemonize(ui, repo)
|
||||
|
@ -23,6 +23,14 @@ class HybridManifest(unittest.TestCase):
|
||||
assert util.safehasattr(hd, method),\
|
||||
"%s missing in hybrid manifest" % method
|
||||
|
||||
def test_cachelimit(self):
|
||||
cachealloc = fastmanifest.systemawarecachelimit.cacheallocation
|
||||
GB = fastmanifest.GB
|
||||
MB = fastmanifest.MB
|
||||
assert cachealloc(0) == 0
|
||||
assert cachealloc(120 * GB) == 6 * GB
|
||||
assert abs(cachealloc(28 * GB) - 5.6 * GB) < 5 * MB
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.path.insert(0, os.path.join(os.environ["TESTDIR"], ".."))
|
||||
import fastmanifest
|
||||
|
Loading…
Reference in New Issue
Block a user