sapling/eden/fs/service/client.py

120 lines
4.2 KiB
Python
Raw Normal View History

# Copyright (c) 2016-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
from __future__ import absolute_import, division, print_function, unicode_literals
import os
from typing import Any, cast
from facebook.eden import EdenService
from thrift.protocol.THeaderProtocol import THeaderProtocol
from thrift.Thrift import TApplicationException
from thrift.transport.THeaderTransport import THeaderTransport
from thrift.transport.TSocket import TSocket
from thrift.transport.TTransport import TTransportException
SOCKET_PATH = "socket"
class EdenNotRunningError(Exception):
def __init__(self, eden_dir):
msg = "edenfs daemon does not appear to be running: tried %s" % eden_dir
super(EdenNotRunningError, self).__init__(msg)
self.eden_dir = eden_dir
# Monkey-patch EdenService.EdenError's __str__() behavior to just return the
# error message. By default it returns the same data as __repr__(), which is
# ugly to show to users.
def _eden_thrift_error_str(ex):
return ex.message
# TODO: https://github.com/python/mypy/issues/2427
cast(Any, EdenService.EdenError).__str__ = _eden_thrift_error_str
class EdenClient(EdenService.Client):
"""
EdenClient is a subclass of EdenService.Client that provides
a few additional conveniences:
- Smarter constructor
- Implement the context manager __enter__ and __exit__ methods, so it can
be used in with statements.
"""
def __init__(self, eden_dir=None, socket_path=None):
if socket_path is not None:
self._socket_path = socket_path
elif eden_dir is not None:
self._socket_path = os.path.join(eden_dir, SOCKET_PATH)
else:
raise TypeError("one of eden_dir or socket_path is required")
self._socket = TSocket(unix_socket=self._socket_path)
deal with idle timeouts for thrift clients Summary: I noticed that when running `hg push --to master`, the implicit pull took about a minute to collect all the commits from the remote, then we tried to make a thrift call to set the parents only to fail with a python EPIPE stack trace. I opted to simply make a new client for each call; this is similar to how things were running with the lame thrift client but doesn't have as terrible an impact on rebase performance as lame thrift client did. It feels like things might be better if we made the client retry on transport errors, but this is quite fiddly so let's defer that until we really need to cut out the unix domain socket connection overheads. ``` $ hg ci note: commit message saved in ../.hg/last-message.txt Traceback (most recent call last): File "/usr/bin/hg.real", line 47, in <module> dispatch.run() File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 81, in run status = (dispatch(req) or 0) & 255 File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 163, in dispatch ret = _runcatch(req) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 314, in _runcatch return _callcatch(ui, _runcatchfunc) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 365, in _callcatch if not handlecommandexception(ui): File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 993, in handlecommandexception ui.log("commandexception", "%s\n%s\n", warning, traceback.format_exc()) File "/usr/lib64/python2.7/site-packages/hgext3rd/sampling.py", line 79, in log return super(logtofile, self).log(event, *msg, **opts) File "/usr/lib64/python2.7/site-packages/hgext/blackbox.py", line 163, in log parents = ctx.parents() File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 286, in parents return self._parents File "/usr/lib64/python2.7/site-packages/mercurial/util.py", line 862, in __get__ result = self.func(obj) File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 1546, in _parents p = self._repo.dirstate.parents() File "hgext3rd/eden/eden_dirstate.py", line 163, in parents File "hgext3rd/eden/eden_dirstate.py", line 157, in _getparents File "/usr/local/fb-mercurial/eden/hgext3rd/eden/EdenThriftClient.py", line 145, in getParentCommits parents = self._client.getParentCommits(self._root) File "facebook/eden/EdenService.py", line 7330, in getParentCommits File "facebook/eden/EdenService.py", line 7339, in send_getParentCommits File "thrift/transport/THeaderTransport.py", line 411, in flush File "thrift/transport/THeaderTransport.py", line 516, in flushImpl File "thrift/transport/TSocket.py", line 315, in write thrift.transport.TTransport.TTransportException: Socket send failed with error 32 (Broken pipe) ``` Reviewed By: bolinfest Differential Revision: D5942205 fbshipit-source-id: 464e5035075476f47232ca975e107e165057c912
2017-10-03 04:13:43 +03:00
# We used to set a timeout here, but picking the right duration is hard,
# and safely retrying an arbitrary thrift call may not be safe. So we
# just leave the client with no timeout.
# self._socket.setTimeout(60000) # in milliseconds
self._transport = THeaderTransport(self._socket)
self._protocol = THeaderProtocol(self._transport)
super(EdenClient, self).__init__(self._protocol)
def __enter__(self):
self.open()
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
self.close()
def open(self):
try:
self._transport.open()
except TTransportException as ex:
deal with idle timeouts for thrift clients Summary: I noticed that when running `hg push --to master`, the implicit pull took about a minute to collect all the commits from the remote, then we tried to make a thrift call to set the parents only to fail with a python EPIPE stack trace. I opted to simply make a new client for each call; this is similar to how things were running with the lame thrift client but doesn't have as terrible an impact on rebase performance as lame thrift client did. It feels like things might be better if we made the client retry on transport errors, but this is quite fiddly so let's defer that until we really need to cut out the unix domain socket connection overheads. ``` $ hg ci note: commit message saved in ../.hg/last-message.txt Traceback (most recent call last): File "/usr/bin/hg.real", line 47, in <module> dispatch.run() File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 81, in run status = (dispatch(req) or 0) & 255 File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 163, in dispatch ret = _runcatch(req) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 314, in _runcatch return _callcatch(ui, _runcatchfunc) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 365, in _callcatch if not handlecommandexception(ui): File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 993, in handlecommandexception ui.log("commandexception", "%s\n%s\n", warning, traceback.format_exc()) File "/usr/lib64/python2.7/site-packages/hgext3rd/sampling.py", line 79, in log return super(logtofile, self).log(event, *msg, **opts) File "/usr/lib64/python2.7/site-packages/hgext/blackbox.py", line 163, in log parents = ctx.parents() File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 286, in parents return self._parents File "/usr/lib64/python2.7/site-packages/mercurial/util.py", line 862, in __get__ result = self.func(obj) File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 1546, in _parents p = self._repo.dirstate.parents() File "hgext3rd/eden/eden_dirstate.py", line 163, in parents File "hgext3rd/eden/eden_dirstate.py", line 157, in _getparents File "/usr/local/fb-mercurial/eden/hgext3rd/eden/EdenThriftClient.py", line 145, in getParentCommits parents = self._client.getParentCommits(self._root) File "facebook/eden/EdenService.py", line 7330, in getParentCommits File "facebook/eden/EdenService.py", line 7339, in send_getParentCommits File "thrift/transport/THeaderTransport.py", line 411, in flush File "thrift/transport/THeaderTransport.py", line 516, in flushImpl File "thrift/transport/TSocket.py", line 315, in write thrift.transport.TTransport.TTransportException: Socket send failed with error 32 (Broken pipe) ``` Reviewed By: bolinfest Differential Revision: D5942205 fbshipit-source-id: 464e5035075476f47232ca975e107e165057c912
2017-10-03 04:13:43 +03:00
self.close()
if ex.type == TTransportException.NOT_OPEN:
raise EdenNotRunningError(self._socket_path)
deal with idle timeouts for thrift clients Summary: I noticed that when running `hg push --to master`, the implicit pull took about a minute to collect all the commits from the remote, then we tried to make a thrift call to set the parents only to fail with a python EPIPE stack trace. I opted to simply make a new client for each call; this is similar to how things were running with the lame thrift client but doesn't have as terrible an impact on rebase performance as lame thrift client did. It feels like things might be better if we made the client retry on transport errors, but this is quite fiddly so let's defer that until we really need to cut out the unix domain socket connection overheads. ``` $ hg ci note: commit message saved in ../.hg/last-message.txt Traceback (most recent call last): File "/usr/bin/hg.real", line 47, in <module> dispatch.run() File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 81, in run status = (dispatch(req) or 0) & 255 File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 163, in dispatch ret = _runcatch(req) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 314, in _runcatch return _callcatch(ui, _runcatchfunc) File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 365, in _callcatch if not handlecommandexception(ui): File "/usr/lib64/python2.7/site-packages/mercurial/dispatch.py", line 993, in handlecommandexception ui.log("commandexception", "%s\n%s\n", warning, traceback.format_exc()) File "/usr/lib64/python2.7/site-packages/hgext3rd/sampling.py", line 79, in log return super(logtofile, self).log(event, *msg, **opts) File "/usr/lib64/python2.7/site-packages/hgext/blackbox.py", line 163, in log parents = ctx.parents() File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 286, in parents return self._parents File "/usr/lib64/python2.7/site-packages/mercurial/util.py", line 862, in __get__ result = self.func(obj) File "/usr/lib64/python2.7/site-packages/mercurial/context.py", line 1546, in _parents p = self._repo.dirstate.parents() File "hgext3rd/eden/eden_dirstate.py", line 163, in parents File "hgext3rd/eden/eden_dirstate.py", line 157, in _getparents File "/usr/local/fb-mercurial/eden/hgext3rd/eden/EdenThriftClient.py", line 145, in getParentCommits parents = self._client.getParentCommits(self._root) File "facebook/eden/EdenService.py", line 7330, in getParentCommits File "facebook/eden/EdenService.py", line 7339, in send_getParentCommits File "thrift/transport/THeaderTransport.py", line 411, in flush File "thrift/transport/THeaderTransport.py", line 516, in flushImpl File "thrift/transport/TSocket.py", line 315, in write thrift.transport.TTransport.TTransportException: Socket send failed with error 32 (Broken pipe) ``` Reviewed By: bolinfest Differential Revision: D5942205 fbshipit-source-id: 464e5035075476f47232ca975e107e165057c912
2017-10-03 04:13:43 +03:00
raise
def close(self):
if self._transport is not None:
self._transport.close()
self._transport = None
def shutdown(self):
self.initiateShutdown(
"EdenClient.shutdown() invoked with no reason by pid=%s uid=%s"
% (os.getpid(), os.getuid())
)
def initiateShutdown(self, reason):
"""Helper for stopping the server.
To swing through the transition from calling the base shutdown() method
with context to the initiateShutdown() method with a reason, we want to
try the latter method first, falling back to the old way to handle the
case where we deploy a newer client while an older server is still
running on the local system."""
try:
super().initiateShutdown(reason)
except TApplicationException as ex:
if ex.type == TApplicationException.UNKNOWN_METHOD:
# Running an older server build, fall back to the old shutdown
# method with no context
super().shutdown()
else:
raise
def create_thrift_client(eden_dir=None, socket_path=None):
"""Construct a thrift client to speak to the running eden server
instance associated with the specified mount point.
@return Returns a context manager for EdenService.Client.
"""
return EdenClient(eden_dir=eden_dir, socket_path=socket_path)