cstore: implement UnionDatapackStore.get()

Summary:
This implements the get function of UnionDatapackStore which iterates over
deltas and produces a full text. Unfortunately the delta chains currently come
out in reverse order (full text last), so we have to read all of them before we
can produce the delta.

Since mpatch takes a char*, and we want to avoid copying the array into a string
array for every read, we event a ref counted string reference type which let's
us transfer ownership of the char* to the caller.

Test Plan: Added a test

Reviewers: #mercurial, simonfar

Reviewed By: simonfar

Subscribers: stash, simonfar, mjpieters

Differential Revision: https://phabricator.intern.facebook.com/D4556547

Signature: t1:4556547:1487200302:c73b8f464f6ce413f1c22c0125a91fbece701f08

cstore/py-datapackstore.h

@@ -279,6 +279,32 @@ static void uniondatapackstore_dealloc(py_uniondatapackstore *self) {
   PyObject_Del(self);
 }
 
+static PyObject *uniondatapackstore_get(py_uniondatapackstore *self, PyObject *args) {
+  try {
+    char *name;
+    Py_ssize_t namelen;
+    char *node;
+    Py_ssize_t nodelen;
+    if (!PyArg_ParseTuple(args, "s#s#", &name, &namelen, &node, &nodelen)) {
+      return NULL;
+    }
+
+    Key key(name, namelen, node, nodelen);
+
+    ConstantStringRef fulltext = self->uniondatapackstore.get(key);
+
+    return PyString_FromStringAndSize(fulltext.content(), fulltext.size());
+  } catch (const pyexception &ex) {
+    return NULL;
+  } catch (const MissingKeyError &ex) {
+    PyErr_SetString(PyExc_KeyError, ex.what());
+    return NULL;
+  } catch (const std::exception &ex) {
+    PyErr_SetString(PyExc_RuntimeError, ex.what());
+    return NULL;
+  }
+}
+
 static PyObject *uniondatapackstore_getdeltachain(py_uniondatapackstore *self, PyObject *args) {
   try {
     char *name;
@@ -355,6 +381,7 @@ static PyObject *uniondatapackstore_getmissing(py_uniondatapackstore *self, PyOb
 // --------- UnionDatapackStore Declaration ---------
 
 static PyMethodDef uniondatapackstore_methods[] = {
+  {"get", (PyCFunction)uniondatapackstore_get, METH_VARARGS, ""},
   {"getdeltachain", (PyCFunction)uniondatapackstore_getdeltachain, METH_VARARGS, ""},
   {"getmissing", (PyCFunction)uniondatapackstore_getmissing, METH_O, ""},
   {NULL, NULL}

cstore/uniondatapackstore.cpp

@@ -8,9 +8,14 @@
 // no-check-code
 
 #include <algorithm>
+#include <memory>
 
 #include "uniondatapackstore.h"
 
+extern "C" {
+#include "mpatch.h"
+}
+
 UnionDatapackStore::UnionDatapackStore(std::vector<DatapackStore*> stores) :
   _stores(stores) {
 }
@@ -21,6 +26,67 @@ UnionDatapackStore::~UnionDatapackStore() {
   // refcount in the py_uniondatapackstore type.
 }
 
+mpatch_flist* getNextLink(void* container, ssize_t index) {
+  std::vector<delta_chain_link_t*> *links = (std::vector<delta_chain_link_t*>*)container;
+
+  if (index < 0 || (size_t)index >= links->size()) {
+    return NULL;
+  }
+
+  delta_chain_link_t *link = links->at(index);
+
+  struct mpatch_flist *res;
+  if ((mpatch_decode((const char*)link->delta, link->delta_sz, &res)) < 0) {
+    throw std::logic_error("invalid patch during patch application");
+  }
+
+  return res;
+}
+
+ConstantStringRef UnionDatapackStore::get(const Key &key) {
+  UnionDeltaChainIterator chain = this->getDeltaChain(key);
+
+  std::vector<delta_chain_link_t*> links;
+
+  delta_chain_link_t *link;
+  while ((link = chain.next()) != NULL) {
+    links.push_back(link);
+  }
+
+  delta_chain_link_t *fulltextLink = links.back();
+  links.pop_back();
+
+  // Short circuit and just return the full text if it's one long
+  if (links.size() == 0) {
+    char * finalText = new char[fulltextLink->delta_sz];
+    memcpy(finalText, fulltextLink->delta, fulltextLink->delta_sz);
+    return ConstantStringRef(finalText, fulltextLink->delta_sz);
+  }
+
+  std::reverse(links.begin(), links.end());
+
+  mpatch_flist *patch = mpatch_fold(&links, getNextLink, 0, links.size());
+  if (!patch) { /* error already set or memory error */
+    throw std::logic_error("mpatch failed to fold patches");
+  }
+
+  ssize_t outlen = mpatch_calcsize(fulltextLink->delta_sz, patch);
+  if (outlen < 0) {
+    mpatch_lfree(patch);
+    throw std::logic_error("mpatch failed to calculate size");
+  }
+
+  char *result= new char[outlen];
+  if (mpatch_apply(result, (const char*)fulltextLink->delta, fulltextLink->delta_sz, patch) < 0) {
+    delete[] result;
+    mpatch_lfree(patch);
+    throw std::logic_error("mpatch failed to apply patches");
+  }
+
+  mpatch_lfree(patch);
+  return ConstantStringRef(result, outlen);
+}
+
 delta_chain_t UnionDeltaChainIterator::getNextChain(const Key &key) {
   for(std::vector<DatapackStore*>::iterator it = _store._stores.begin();
       it != _store._stores.end();

cstore/uniondatapackstore.h

@@ -21,6 +21,79 @@ extern "C" {
 #include <vector>
 #include <stdexcept>
 
+class ConstantString {
+  friend class ConstantStringRef;
+  private:
+    char *_content;
+    size_t _size;
+    size_t _refCount;
+
+    ConstantString(char *content, size_t size) :
+      _content(content),
+      _size(size),
+      _refCount(1) {}
+  public:
+    ~ConstantString() {
+      delete _content;
+    }
+
+    char *content() {
+      return _content;
+    }
+
+    size_t size() {
+      return _size;
+    }
+
+    void incref() {
+      _refCount++;
+    }
+
+    size_t decref() {
+      if (_refCount > 0) {
+        _refCount--;
+      }
+      return _refCount;
+    }
+};
+
+class ConstantStringRef {
+  private:
+    ConstantString *_str;
+  public:
+    ConstantStringRef(char *str, size_t size) :
+      _str(new ConstantString(str, size)) {
+    }
+
+    ConstantStringRef(const ConstantStringRef &other) {
+      other._str->incref();
+      _str = other._str;
+    }
+
+    ~ConstantStringRef() {
+      if (_str->decref() == 0) {
+        delete _str;
+      }
+    }
+
+    ConstantStringRef& operator=(const ConstantStringRef &other) {
+      if (_str->decref() == 0) {
+        delete _str;
+      }
+      _str = other._str;
+      _str->incref();
+      return *this;
+    }
+
+    char *content() {
+      return _str->content();
+    }
+
+    size_t size() {
+      return _str->size();
+    }
+};
+
 class UnionDatapackStore;
 class UnionDatapackStoreKeyIterator : public KeyIterator {
   private:
@@ -56,6 +129,8 @@ class UnionDatapackStore {
 
     ~UnionDatapackStore();
 
+    ConstantStringRef get(const Key &key);
+
     UnionDeltaChainIterator getDeltaChain(const Key &key);
 
     bool contains(const Key &key);

tests/cstore-uniondatapackstore.py

@@ -24,6 +24,7 @@ from remotefilelog.datapack import (
     mutabledatapack,
 )
 
+from mercurial import mdiff
 from mercurial.node import nullid
 import mercurial.ui
 
@@ -59,6 +60,35 @@ class uniondatapackstoretests(unittest.TestCase):
         path = packer.close()
         return fastdatapack(path)
 
+    def testGetFromSingleDelta(self):
+        packdir = self.makeTempDir()
+
+        revisions = [("foo", self.getFakeHash(), nullid, "content")]
+        self.createPack(packdir, revisions=revisions)
+
+        unionstore = uniondatapackstore([datapackstore(packdir)])
+
+        text = unionstore.get(revisions[0][0], revisions[0][1])
+        self.assertEquals("content", text)
+
+    def testGetFromChainDeltas(self):
+        packdir = self.makeTempDir()
+
+        rev1 = "content"
+        rev2 = "content2"
+        firsthash = self.getFakeHash()
+        revisions = [
+            ("foo", firsthash, nullid, rev1),
+            ("foo", self.getFakeHash(), firsthash,
+             mdiff.textdiff(rev1, rev2)),
+        ]
+        self.createPack(packdir, revisions=revisions)
+
+        unionstore = uniondatapackstore([datapackstore(packdir)])
+
+        text = unionstore.get(revisions[1][0], revisions[1][1])
+        self.assertEquals(rev2, text)
+
     def testGetDeltaChainSingleRev(self):
         """Test getting a 1-length delta chain."""
         packdir = self.makeTempDir()