2016-04-27 19:27:56 +03:00
|
|
|
# conduit.py
|
|
|
|
#
|
|
|
|
# A library function to call a phabricator conduit RPC.
|
|
|
|
# It's different from fbconduit in that this is an authenticated
|
|
|
|
# conduit client.
|
|
|
|
|
2017-10-18 13:47:16 +03:00
|
|
|
from __future__ import absolute_import
|
2016-04-27 19:27:56 +03:00
|
|
|
|
2017-07-13 06:00:00 +03:00
|
|
|
import contextlib
|
2017-10-18 13:47:16 +03:00
|
|
|
import hashlib
|
2016-04-27 19:27:56 +03:00
|
|
|
import json
|
|
|
|
import os
|
|
|
|
import time
|
2017-10-20 20:27:43 +03:00
|
|
|
import warnings
|
2017-10-18 13:47:16 +03:00
|
|
|
|
|
|
|
import urllib3
|
|
|
|
|
2016-04-27 19:27:56 +03:00
|
|
|
from mercurial import util
|
2017-10-18 13:47:16 +03:00
|
|
|
|
|
|
|
from . import arcconfig
|
2016-04-27 19:27:56 +03:00
|
|
|
|
2017-03-23 16:02:24 +03:00
|
|
|
urlreq = util.urlreq
|
2016-06-30 13:41:45 +03:00
|
|
|
|
2017-09-26 14:04:22 +03:00
|
|
|
DEFAULT_URL = 'https://phabricator.intern.facebook.com/api/'
|
2016-04-27 19:27:56 +03:00
|
|
|
DEFAULT_TIMEOUT = 60
|
|
|
|
mocked_responses = None
|
|
|
|
|
|
|
|
class ClientError(Exception):
|
|
|
|
def __init__(self, code, msg):
|
|
|
|
Exception.__init__(self, msg)
|
|
|
|
self.code = code
|
|
|
|
|
|
|
|
class Client(object):
|
2017-10-19 16:03:48 +03:00
|
|
|
def __init__(self, url=None, user=None, cert=None, act_as=None,
|
2017-11-09 19:43:57 +03:00
|
|
|
ca_certs=None, timeout=None):
|
2017-09-26 14:04:22 +03:00
|
|
|
self._url = url or DEFAULT_URL
|
2016-04-27 19:27:56 +03:00
|
|
|
self._user = user
|
|
|
|
self._cert = cert
|
2017-11-22 13:54:20 +03:00
|
|
|
self._oauth = None
|
2016-04-27 19:27:56 +03:00
|
|
|
self._actas = act_as or self._user
|
|
|
|
self._connection = None
|
2017-10-19 16:03:48 +03:00
|
|
|
self._ca_certs = ca_certs
|
2017-11-09 19:43:57 +03:00
|
|
|
self._timeout = timeout
|
2016-04-27 19:27:56 +03:00
|
|
|
|
|
|
|
def apply_arcconfig(self, config):
|
2017-09-26 14:04:22 +03:00
|
|
|
self._url = config.get('conduit_uri', DEFAULT_URL)
|
|
|
|
if self._url == 'https://phabricator.fb.com/api/':
|
|
|
|
self._url = 'https://phabricator.intern.facebook.com/api/'
|
phabstatus: fail gracefully if necessary arcrc settings are missing
Summary:
If the user does not have necesary credentials defined in their arc
configuration, catch the KeyError and convert it into an ArcConfigError.
The existing call sites in the phabstatus and arcdiff extensions catch and
handle ArcConfigError, but not generic KeyErrors.
This also fixes the phabstatus warning messages to end with a newline.
Test Plan: Added a unit test.
Reviewers: #sourcecontrol, quark, simonfar, wez, rmcelroy
Reviewed By: wez, rmcelroy
Subscribers: rmcelroy, net-systems-diffs@fb.com, yogeshwer, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4800977
Tasks: 17002914
Signature: t1:4800977:1490847078:e18bba042e3ff57100e0a7b25c610b5cad17fa2e
2017-03-30 21:55:39 +03:00
|
|
|
try:
|
2017-09-26 14:04:22 +03:00
|
|
|
hostconfig = config['hosts'][self._url]
|
phabstatus: fail gracefully if necessary arcrc settings are missing
Summary:
If the user does not have necesary credentials defined in their arc
configuration, catch the KeyError and convert it into an ArcConfigError.
The existing call sites in the phabstatus and arcdiff extensions catch and
handle ArcConfigError, but not generic KeyErrors.
This also fixes the phabstatus warning messages to end with a newline.
Test Plan: Added a unit test.
Reviewers: #sourcecontrol, quark, simonfar, wez, rmcelroy
Reviewed By: wez, rmcelroy
Subscribers: rmcelroy, net-systems-diffs@fb.com, yogeshwer, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4800977
Tasks: 17002914
Signature: t1:4800977:1490847078:e18bba042e3ff57100e0a7b25c610b5cad17fa2e
2017-03-30 21:55:39 +03:00
|
|
|
self._user = hostconfig['user']
|
2017-11-22 13:54:20 +03:00
|
|
|
if 'oauth' in hostconfig:
|
|
|
|
self._oauth = hostconfig['oauth']
|
|
|
|
else:
|
|
|
|
self._cert = hostconfig['cert']
|
phabstatus: fail gracefully if necessary arcrc settings are missing
Summary:
If the user does not have necesary credentials defined in their arc
configuration, catch the KeyError and convert it into an ArcConfigError.
The existing call sites in the phabstatus and arcdiff extensions catch and
handle ArcConfigError, but not generic KeyErrors.
This also fixes the phabstatus warning messages to end with a newline.
Test Plan: Added a unit test.
Reviewers: #sourcecontrol, quark, simonfar, wez, rmcelroy
Reviewed By: wez, rmcelroy
Subscribers: rmcelroy, net-systems-diffs@fb.com, yogeshwer, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4800977
Tasks: 17002914
Signature: t1:4800977:1490847078:e18bba042e3ff57100e0a7b25c610b5cad17fa2e
2017-03-30 21:55:39 +03:00
|
|
|
except KeyError:
|
2017-04-27 23:45:50 +03:00
|
|
|
try:
|
|
|
|
hostconfig = config['hosts'][config['hosts'].keys()[0]]
|
|
|
|
self._user = hostconfig['user']
|
2017-11-22 13:54:20 +03:00
|
|
|
if 'oauth' in hostconfig:
|
|
|
|
self._oauth = hostconfig['oauth']
|
|
|
|
else:
|
|
|
|
self._cert = hostconfig['cert']
|
2017-04-27 23:45:50 +03:00
|
|
|
except KeyError:
|
|
|
|
raise arcconfig.ArcConfigError(
|
|
|
|
'arcrc is missing user '
|
|
|
|
'credentials for host %s. use '
|
2017-09-26 14:04:22 +03:00
|
|
|
'"arc install-certificate" to fix.' % self._url)
|
2016-04-27 19:27:56 +03:00
|
|
|
self._actas = self._user
|
|
|
|
self._connection = None
|
|
|
|
|
2017-11-09 19:43:57 +03:00
|
|
|
def call(self, method, args, timeout=None):
|
|
|
|
if timeout is None:
|
|
|
|
if self._timeout is None:
|
|
|
|
timeout = DEFAULT_TIMEOUT
|
|
|
|
else:
|
|
|
|
timeout = self._timeout
|
2016-04-27 19:27:56 +03:00
|
|
|
args['__conduit__'] = {
|
|
|
|
'authUser': self._user,
|
|
|
|
'actAsUser': self._actas,
|
2017-10-12 00:04:42 +03:00
|
|
|
'caller': 'hg',
|
2016-04-27 19:27:56 +03:00
|
|
|
}
|
2017-11-22 13:54:20 +03:00
|
|
|
if self._oauth is not None:
|
|
|
|
args['__conduit__']['accessToken'] = self._oauth
|
|
|
|
else:
|
|
|
|
token = '%d' % time.time()
|
|
|
|
sig = token + self._cert
|
|
|
|
args['__conduit__'].update({
|
|
|
|
'authToken': token,
|
|
|
|
'authSignature': hashlib.sha1(sig.encode('utf-8')).hexdigest()
|
|
|
|
})
|
2017-10-18 13:47:16 +03:00
|
|
|
req_data = {
|
2016-04-27 19:27:56 +03:00
|
|
|
'params': json.dumps(args),
|
|
|
|
'output': 'json',
|
2017-10-18 13:47:16 +03:00
|
|
|
}
|
2017-09-26 14:04:22 +03:00
|
|
|
headers = (
|
|
|
|
('Connection', 'Keep-Alive'),
|
|
|
|
)
|
|
|
|
url = self._url + method
|
2017-10-18 13:47:16 +03:00
|
|
|
|
|
|
|
if self._connection is None:
|
2017-10-19 16:03:48 +03:00
|
|
|
self._connection = urllib3.PoolManager(ca_certs=self._ca_certs)
|
2017-09-26 14:04:22 +03:00
|
|
|
try:
|
2017-10-20 20:27:43 +03:00
|
|
|
with warnings.catch_warnings():
|
|
|
|
if not self._ca_certs:
|
|
|
|
# ignore the urllib3 certificate verification warnings
|
|
|
|
warnings.simplefilter(
|
|
|
|
'ignore', urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
response = self._connection.request(
|
|
|
|
'POST', url, headers=headers, fields=req_data,
|
|
|
|
timeout=timeout)
|
2017-10-18 13:47:16 +03:00
|
|
|
except urllib3.exceptions.HTTPError as ex:
|
2017-10-19 16:03:48 +03:00
|
|
|
errno = -1
|
|
|
|
if ex.args and util.safehasattr(ex.args[0], 'errno'):
|
|
|
|
errno = ex.args[0].errno
|
|
|
|
raise ClientError(errno, str(ex))
|
2016-04-27 19:27:56 +03:00
|
|
|
|
2017-09-26 14:04:22 +03:00
|
|
|
try:
|
2017-10-18 13:47:16 +03:00
|
|
|
response = json.loads(response.data)
|
2017-09-26 14:04:22 +03:00
|
|
|
except ValueError:
|
|
|
|
# Can't decode the data, not valid JSON (html error page perhaps?)
|
|
|
|
raise ClientError(-1, 'did not receive a valid JSON response')
|
|
|
|
|
2016-04-27 19:27:56 +03:00
|
|
|
if response['error_code'] is not None:
|
|
|
|
raise ClientError(response['error_code'], response['error_info'])
|
|
|
|
return response['result']
|
|
|
|
|
|
|
|
class MockClient(object):
|
|
|
|
def __init__(self, **kwargs):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def apply_arcconfig(self, config):
|
|
|
|
pass
|
|
|
|
|
|
|
|
def call(self, method, args, timeout=DEFAULT_TIMEOUT):
|
|
|
|
global mocked_responses
|
|
|
|
|
|
|
|
cmd = json.dumps([method, args], sort_keys=True)
|
|
|
|
try:
|
|
|
|
response = mocked_responses.pop(0)
|
|
|
|
# Check expectations via a deep compare of the json representation.
|
|
|
|
# We need this because child objects and values are compared by
|
|
|
|
# address rather than value.
|
|
|
|
expect = json.dumps(response.get('cmd', None), sort_keys=True)
|
|
|
|
if cmd != expect:
|
|
|
|
raise ClientError(None,
|
|
|
|
'mock mismatch got %s expected %s' % (
|
|
|
|
cmd, expect))
|
|
|
|
if 'error_info' in response:
|
|
|
|
raise ClientError(response.get('error_code', None),
|
|
|
|
response['error_info'])
|
|
|
|
return response['result']
|
|
|
|
except IndexError:
|
|
|
|
raise ClientError(None,
|
|
|
|
'No more mocked responses available for call to %s' % cmd)
|
|
|
|
|
|
|
|
|
|
|
|
if 'HG_ARC_CONDUIT_MOCK' in os.environ:
|
|
|
|
# To facilitate testing, we replace the client object with this
|
|
|
|
# fake implementation that returns responses from a file that
|
|
|
|
# contains a series of json serialized object values.
|
|
|
|
with open(os.environ['HG_ARC_CONDUIT_MOCK'], 'r') as f:
|
|
|
|
mocked_responses = json.load(f)
|
|
|
|
Client = MockClient
|
|
|
|
|
2017-07-13 06:00:00 +03:00
|
|
|
class ClientCache(object):
|
|
|
|
def __init__(self):
|
|
|
|
self.max_idle_seconds = 10
|
2017-10-19 16:03:48 +03:00
|
|
|
self.client = {}
|
|
|
|
self.lastuse = {}
|
2017-07-13 06:00:00 +03:00
|
|
|
|
|
|
|
@contextlib.contextmanager
|
2017-10-19 16:03:48 +03:00
|
|
|
def getclient(self, ca_certs=None):
|
2017-07-13 06:00:00 +03:00
|
|
|
# Use the existing client if we have one and it hasn't been idle too
|
|
|
|
# long.
|
|
|
|
#
|
|
|
|
# We reconnect if we have been idle for too long just in case the
|
|
|
|
# server might have closed our connection while we were idle. (We
|
|
|
|
# could potentially check the socket for readability, but that might
|
|
|
|
# still race with the server currently closing our socket.)
|
2017-10-19 16:03:48 +03:00
|
|
|
client = self.client.get(ca_certs),
|
|
|
|
lastuse = self.lastuse.get(ca_certs, 0)
|
|
|
|
if client and time.time() <= (lastuse + self.max_idle_seconds):
|
|
|
|
# Remove self.client for this ca_certs config while we are using
|
|
|
|
# it. If our caller throws an exception during the yield this
|
|
|
|
# ensures that we do not continue to use this client later.
|
|
|
|
del self.client.pop[ca_certs], self.lastuse[ca_certs]
|
2017-07-13 06:00:00 +03:00
|
|
|
else:
|
|
|
|
# We have to make a new connection
|
2017-10-19 16:03:48 +03:00
|
|
|
client = Client(ca_certs=ca_certs)
|
2017-07-13 06:00:00 +03:00
|
|
|
client.apply_arcconfig(arcconfig.load_for_path(os.getcwd()))
|
|
|
|
|
|
|
|
yield client
|
|
|
|
|
|
|
|
# Our caller used this client successfully and did not throw an
|
|
|
|
# exception. Store it to use again next time getclient() is called.
|
2017-10-19 16:03:48 +03:00
|
|
|
self.lastuse[ca_certs] = time.time()
|
|
|
|
self.client[ca_certs] = client
|
2017-07-13 06:00:00 +03:00
|
|
|
|
|
|
|
_clientcache = ClientCache()
|
|
|
|
|
2017-10-19 16:03:48 +03:00
|
|
|
def call_conduit(method, args, ca_certs=None, timeout=DEFAULT_TIMEOUT):
|
|
|
|
with _clientcache.getclient(ca_certs=ca_certs) as client:
|
2017-07-13 06:00:00 +03:00
|
|
|
return client.call(method, args, timeout=timeout)
|