lfs: add retry logic for transferring a single object

Summary:
We got TCP RESETs frequently. It's hard to pinpoint the root cause so let's
just add retry for now.

Test Plan: eyes

Reviewers: #mercurial, #ovrsource_warroom, steaphan

Reviewed By: steaphan

Subscribers: steaphan, mjpieters, medson

Differential Revision: https://phabricator.intern.facebook.com/D5265928

Tasks: 19419154

Signature: t1:5265928:1497635499:17cf5d5cb69b406330326d693a9eceb1d22861f8

hgext3rd/lfs/__init__.py

@@ -21,6 +21,9 @@ Configs::
 
     # size of a file to make it use LFS
     threshold = 10M
+
+    # how many times to retry before giving up on transferring an object
+    retry = 5
 """
 
 from __future__ import absolute_import

hgext3rd/lfs/blobstore.py

@@ -90,6 +90,7 @@ class _gitlfsremote(object):
         baseurl, authinfo = url.authinfo()
         self.baseurl = baseurl.rstrip('/')
         self.urlopener = urlmod.opener(ui, authinfo)
+        self.retry = ui.configint('lfs', 'retry', 5)
 
     def writebatch(self, pointers, fromstore):
         """Batch upload from local to remote blobstore."""
@@ -223,7 +224,23 @@ class _gitlfsremote(object):
                 elif action == 'upload':
                     msg = _('lfs: uploading %s (%s)\n')
                 self.ui.write(msg % (obj.get('oid'), util.bytecount(objsize)))
-            self._basictransfer(obj, action, localstore, progress=progress)
+            origrunningsize = prunningsize[0]
+            retry = self.retry
+            while True:
+                prunningsize[0] = origrunningsize
+                try:
+                    self._basictransfer(obj, action, localstore,
+                                        progress=progress)
+                    break
+                except Exception as ex:
+                    if retry > 0:
+                        if self.ui.verbose:
+                            self.ui.write(
+                                _('lfs: failed: %r (remaining retry %d)\n')
+                                % (ex, retry))
+                        retry -= 1
+                        continue
+                    raise
 
         self.ui.progress(topic, pos=None, total=total)