remotefilelog: wrap lz4 imports for compatibility

Differential Revision: https://phabricator.intern.facebook.com/D4707115

remotefilelog/cacheclient.py

@@ -49,13 +49,13 @@ def writefile(path, content):
 def compress(value):
     # Real world implementations will want to compress values.
     # Insert your favorite compression here, ex:
-    # return lz4.compressHC(value)
+    # return lz4wrapper.lzcompresshc(value)
     return value
 
 def decompress(value):
     # Real world implementations will want to compress values.
     # Insert your favorite compression here, ex:
-    # return lz4.decompress(value)
+    # return lz4wrapper.lz4decompress(value)
     return value
 
 def generateKey(id):

remotefilelog/datapack.py

@@ -1,8 +1,9 @@
-import lz4, struct
+import struct
 from mercurial import util
 from mercurial.node import nullid, hex
 from mercurial.i18n import _
 import basepack, constants
+from lz4wrapper import lz4compress, lz4decompress
 try:
     import cstore
     cstore.datapack
@@ -115,7 +116,7 @@ class datapack(basepack.basepack):
             deltalen = struct.unpack('!Q', rawdeltalen)[0]
 
             delta = rawentry[deltastart + 8:deltastart + 8 + deltalen]
-            delta = lz4.decompress(delta)
+            delta = lz4decompress(delta)
 
             deltachain.append((filename, node, filename, deltabasenode, delta))
 
@@ -361,7 +362,7 @@ class mutabledatapack(basepack.mutablebasepack):
             return
 
         # TODO: allow configurable compression
-        delta = lz4.compress(delta)
+        delta = lz4compress(delta)
         rawdata = "%s%s%s%s%s%s" % (
             struct.pack('!H', len(name)), # unsigned 2 byte int
             name,

remotefilelog/debugcommands.py

@@ -9,7 +9,8 @@ from mercurial import error, filelog, revlog
 from mercurial.node import bin, hex, nullid, short
 from mercurial.i18n import _
 import datapack, historypack, shallowrepo, fileserverclient
-import hashlib, os, lz4
+from lz4wrapper import lz4decompress
+import hashlib, os
 
 def debugremotefilelog(ui, path, **opts):
     decompress = opts.get('decompress')
@@ -163,7 +164,7 @@ def parsefileblob(path, decompress):
         f.close()
 
     if decompress:
-        raw = lz4.decompress(raw)
+        raw = lz4decompress(raw)
 
     index = raw.index('\0')
     size = int(raw[:index])

remotefilelog/fileserverclient.py

@@ -9,10 +9,11 @@ from mercurial.i18n import _
 from mercurial.node import hex, bin, nullid
 from mercurial import util, sshpeer, hg, error, util, wireproto, httppeer
 from mercurial import extensions
-import hashlib, os, lz4, time, io, struct
+import hashlib, os, time, io, struct
 import itertools
 
 import constants, datapack, historypack, shallowutil
+from lz4wrapper import lz4decompress
 from shallowutil import readexactly, readunpack
 
 # Statistics for debugging
@@ -427,7 +428,7 @@ class fileserverclient(object):
                                       % (len(data), size))
 
         self.writedata.addremotefilelognode(filename, bin(node),
-                                             lz4.decompress(data))
+                                             lz4decompress(data))
 
     def requestpack(self, fileids):
         """Requests the given file revisions from the server in a pack format.

remotefilelog/lz4wrapper.py

@@ -0,0 +1,26 @@
+from mercurial import error
+from mercurial.i18n import _
+
+import lz4
+
+def missing(*args, **kwargs):
+    raise error.Abort(_('remotefilelog extension requires lz4 support'))
+
+lz4compress = lzcompresshc = lz4decompress = missing
+
+try:
+    # newer python-lz4 has these functions deprecated as top-level ones,
+    # so we are trying to import from lz4.block first
+    def _compressHC(*args, **kwargs):
+        return lz4.block.compress(*args, mode='high_compression', **kwargs)
+    lzcompresshc = _compressHC
+    lz4compress = lz4.block.compress
+    lz4decompress = lz4.block.decompress
+except (AttributeError, ImportError):
+    # ImportError is possible due to DemandImport
+    try:
+        lzcompresshc = lz4.compressHC
+        lz4compress = lz4.compress
+        lz4decompress = lz4.decompress
+    except (AttributeError, ImportError):
+        pass

remotefilelog/remotefilelogserver.py

@@ -12,7 +12,8 @@ from mercurial.hgweb import protocol as httpprotocol
 from mercurial.node import bin, hex, nullid, nullrev
 from mercurial.i18n import _
 import constants, shallowrepo
-import errno, stat, os, lz4, struct, time
+from lz4wrapper import lzcompresshc
+import errno, stat, os, struct, time
 from shallowutil import readexactly, readunpack
 
 try:
@@ -210,7 +211,7 @@ def _loadfileblob(repo, cachepath, path, node):
             filectx = repo.filectx(path, fileid=node)
 
         text = createfileblob(filectx)
-        text = lz4.compressHC(text)
+        text = lzcompresshc(text)
 
         # everything should be user & group read/writable
         oldumask = os.umask(0o002)