mirror of
https://github.com/facebook/sapling.git
synced 2024-10-08 15:57:43 +03:00
e47f7dc2fa
This is done by a script [2] using RedBaron [1], a tool designed for doing code refactoring. All "default" values are decided by the script and are strongly consistent with the existing code. There are 2 changes done manually to fix tests: [warn] mercurial/exchange.py: experimental.bundle2-output-capture: default needs manual removal [warn] mercurial/localrepo.py: experimental.hook-track-tags: default needs manual removal Since RedBaron is not confident about how to indent things [2]. [1]: https://github.com/PyCQA/redbaron [2]: https://github.com/PyCQA/redbaron/issues/100 [3]: #!/usr/bin/env python # codemod_configitems.py - codemod tool to fill configitems # # Copyright 2017 Facebook, Inc. # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. from __future__ import absolute_import, print_function import os import sys import redbaron def readpath(path): with open(path) as f: return f.read() def writepath(path, content): with open(path, 'w') as f: f.write(content) _configmethods = {'config', 'configbool', 'configint', 'configbytes', 'configlist', 'configdate'} def extractstring(rnode): """get the string from a RedBaron string or call_argument node""" while rnode.type != 'string': rnode = rnode.value return rnode.value[1:-1] # unquote, "'str'" -> "str" def uiconfigitems(red): """match *.ui.config* pattern, yield (node, method, args, section, name)""" for node in red.find_all('atomtrailers'): entry = None try: obj = node[-3].value method = node[-2].value args = node[-1] section = args[0].value name = args[1].value if (obj in ('ui', 'self') and method in _configmethods and section.type == 'string' and name.type == 'string'): entry = (node, method, args, extractstring(section), extractstring(name)) except Exception: pass else: if entry: yield entry def coreconfigitems(red): """match coreconfigitem(...) pattern, yield (node, args, section, name)""" for node in red.find_all('atomtrailers'): entry = None try: args = node[1] section = args[0].value name = args[1].value if (node[0].value == 'coreconfigitem' and section.type == 'string' and name.type == 'string'): entry = (node, args, extractstring(section), extractstring(name)) except Exception: pass else: if entry: yield entry def registercoreconfig(cfgred, section, name, defaultrepr): """insert coreconfigitem to cfgred AST section and name are plain string, defaultrepr is a string """ # find a place to insert the "coreconfigitem" item entries = list(coreconfigitems(cfgred)) for node, args, nodesection, nodename in reversed(entries): if (nodesection, nodename) < (section, name): # insert after this entry node.insert_after( 'coreconfigitem(%r, %r,\n' ' default=%s,\n' ')' % (section, name, defaultrepr)) return def main(argv): if not argv: print('Usage: codemod_configitems.py FILES\n' 'For example, FILES could be "{hgext,mercurial}/*/**.py"') dirname = os.path.dirname reporoot = dirname(dirname(dirname(os.path.abspath(__file__)))) # register configitems to this destination cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py') cfgred = redbaron.RedBaron(readpath(cfgpath)) # state about what to do registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred)) toregister = {} # {(section, name): defaultrepr} coreconfigs = set() # {(section, name)}, whether it's used in core # first loop: scan all files before taking any action for i, path in enumerate(argv): print('(%d/%d) scanning %s' % (i + 1, len(argv), path)) iscore = ('mercurial' in path) and ('hgext' not in path) red = redbaron.RedBaron(readpath(path)) # find all repo.ui.config* and ui.config* calls, and collect their # section, name and default value information. for node, method, args, section, name in uiconfigitems(red): if section == 'web': # [web] section has some weirdness, ignore them for now continue defaultrepr = None key = (section, name) if len(args) == 2: if key in registered: continue if method == 'configlist': defaultrepr = 'list' elif method == 'configbool': defaultrepr = 'False' else: defaultrepr = 'None' elif len(args) >= 3 and (args[2].target is None or args[2].target.value == 'default'): # try to understand the "default" value dnode = args[2].value if dnode.type == 'name': if dnode.value in {'None', 'True', 'False'}: defaultrepr = dnode.value elif dnode.type == 'string': defaultrepr = repr(dnode.value[1:-1]) elif dnode.type in ('int', 'float'): defaultrepr = dnode.value # inconsistent default if key in toregister and toregister[key] != defaultrepr: defaultrepr = None # interesting to rewrite if key not in registered: if defaultrepr is None: print('[note] %s: %s.%s: unsupported default' % (path, section, name)) registered.add(key) # skip checking it again else: toregister[key] = defaultrepr if iscore: coreconfigs.add(key) # second loop: rewrite files given "toregister" result for path in argv: # reconstruct redbaron - trade CPU for memory red = redbaron.RedBaron(readpath(path)) changed = False for node, method, args, section, name in uiconfigitems(red): key = (section, name) defaultrepr = toregister.get(key) if defaultrepr is None or key not in coreconfigs: continue if len(args) >= 3 and (args[2].target is None or args[2].target.value == 'default'): try: del args[2] changed = True except Exception: # redbaron fails to do the rewrite due to indentation # see https://github.com/PyCQA/redbaron/issues/100 print('[warn] %s: %s.%s: default needs manual removal' % (path, section, name)) if key not in registered: print('registering %s.%s' % (section, name)) registercoreconfig(cfgred, section, name, defaultrepr) registered.add(key) if changed: print('updating %s' % path) writepath(path, red.dumps()) if toregister: print('updating configitems.py') writepath(cfgpath, cfgred.dumps()) if __name__ == "__main__": sys.exit(main(sys.argv[1:]))
370 lines
12 KiB
Python
370 lines
12 KiB
Python
# sshpeer.py - ssh repository proxy class for mercurial
|
|
#
|
|
# Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
|
|
#
|
|
# This software may be used and distributed according to the terms of the
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import re
|
|
|
|
from .i18n import _
|
|
from . import (
|
|
error,
|
|
pycompat,
|
|
util,
|
|
wireproto,
|
|
)
|
|
|
|
class remotelock(object):
|
|
def __init__(self, repo):
|
|
self.repo = repo
|
|
def release(self):
|
|
self.repo.unlock()
|
|
self.repo = None
|
|
def __enter__(self):
|
|
return self
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
if self.repo:
|
|
self.release()
|
|
def __del__(self):
|
|
if self.repo:
|
|
self.release()
|
|
|
|
def _serverquote(s):
|
|
if not s:
|
|
return s
|
|
'''quote a string for the remote shell ... which we assume is sh'''
|
|
if re.match('[a-zA-Z0-9@%_+=:,./-]*$', s):
|
|
return s
|
|
return "'%s'" % s.replace("'", "'\\''")
|
|
|
|
def _forwardoutput(ui, pipe):
|
|
"""display all data currently available on pipe as remote output.
|
|
|
|
This is non blocking."""
|
|
s = util.readpipe(pipe)
|
|
if s:
|
|
for l in s.splitlines():
|
|
ui.status(_("remote: "), l, '\n')
|
|
|
|
class doublepipe(object):
|
|
"""Operate a side-channel pipe in addition of a main one
|
|
|
|
The side-channel pipe contains server output to be forwarded to the user
|
|
input. The double pipe will behave as the "main" pipe, but will ensure the
|
|
content of the "side" pipe is properly processed while we wait for blocking
|
|
call on the "main" pipe.
|
|
|
|
If large amounts of data are read from "main", the forward will cease after
|
|
the first bytes start to appear. This simplifies the implementation
|
|
without affecting actual output of sshpeer too much as we rarely issue
|
|
large read for data not yet emitted by the server.
|
|
|
|
The main pipe is expected to be a 'bufferedinputpipe' from the util module
|
|
that handle all the os specific bits. This class lives in this module
|
|
because it focus on behavior specific to the ssh protocol."""
|
|
|
|
def __init__(self, ui, main, side):
|
|
self._ui = ui
|
|
self._main = main
|
|
self._side = side
|
|
|
|
def _wait(self):
|
|
"""wait until some data are available on main or side
|
|
|
|
return a pair of boolean (ismainready, issideready)
|
|
|
|
(This will only wait for data if the setup is supported by `util.poll`)
|
|
"""
|
|
if getattr(self._main, 'hasbuffer', False): # getattr for classic pipe
|
|
return (True, True) # main has data, assume side is worth poking at.
|
|
fds = [self._main.fileno(), self._side.fileno()]
|
|
try:
|
|
act = util.poll(fds)
|
|
except NotImplementedError:
|
|
# non supported yet case, assume all have data.
|
|
act = fds
|
|
return (self._main.fileno() in act, self._side.fileno() in act)
|
|
|
|
def write(self, data):
|
|
return self._call('write', data)
|
|
|
|
def read(self, size):
|
|
r = self._call('read', size)
|
|
if size != 0 and not r:
|
|
# We've observed a condition that indicates the
|
|
# stdout closed unexpectedly. Check stderr one
|
|
# more time and snag anything that's there before
|
|
# letting anyone know the main part of the pipe
|
|
# closed prematurely.
|
|
_forwardoutput(self._ui, self._side)
|
|
return r
|
|
|
|
def readline(self):
|
|
return self._call('readline')
|
|
|
|
def _call(self, methname, data=None):
|
|
"""call <methname> on "main", forward output of "side" while blocking
|
|
"""
|
|
# data can be '' or 0
|
|
if (data is not None and not data) or self._main.closed:
|
|
_forwardoutput(self._ui, self._side)
|
|
return ''
|
|
while True:
|
|
mainready, sideready = self._wait()
|
|
if sideready:
|
|
_forwardoutput(self._ui, self._side)
|
|
if mainready:
|
|
meth = getattr(self._main, methname)
|
|
if data is None:
|
|
return meth()
|
|
else:
|
|
return meth(data)
|
|
|
|
def close(self):
|
|
return self._main.close()
|
|
|
|
def flush(self):
|
|
return self._main.flush()
|
|
|
|
class sshpeer(wireproto.wirepeer):
|
|
def __init__(self, ui, path, create=False):
|
|
self._url = path
|
|
self.ui = ui
|
|
self.pipeo = self.pipei = self.pipee = None
|
|
|
|
u = util.url(path, parsequery=False, parsefragment=False)
|
|
if u.scheme != 'ssh' or not u.host or u.path is None:
|
|
self._abort(error.RepoError(_("couldn't parse location %s") % path))
|
|
|
|
self.user = u.user
|
|
if u.passwd is not None:
|
|
self._abort(error.RepoError(_("password in URL not supported")))
|
|
self.host = u.host
|
|
self.port = u.port
|
|
self.path = u.path or "."
|
|
|
|
sshcmd = self.ui.config("ui", "ssh")
|
|
remotecmd = self.ui.config("ui", "remotecmd")
|
|
|
|
args = util.sshargs(sshcmd,
|
|
_serverquote(self.host),
|
|
_serverquote(self.user),
|
|
_serverquote(self.port))
|
|
|
|
if create:
|
|
cmd = '%s %s %s' % (sshcmd, args,
|
|
util.shellquote("%s init %s" %
|
|
(_serverquote(remotecmd), _serverquote(self.path))))
|
|
ui.debug('running %s\n' % cmd)
|
|
res = ui.system(cmd, blockedtag='sshpeer')
|
|
if res != 0:
|
|
self._abort(error.RepoError(_("could not create remote repo")))
|
|
|
|
self._validaterepo(sshcmd, args, remotecmd)
|
|
|
|
def url(self):
|
|
return self._url
|
|
|
|
def _validaterepo(self, sshcmd, args, remotecmd):
|
|
# cleanup up previous run
|
|
self.cleanup()
|
|
|
|
cmd = '%s %s %s' % (sshcmd, args,
|
|
util.shellquote("%s -R %s serve --stdio" %
|
|
(_serverquote(remotecmd), _serverquote(self.path))))
|
|
self.ui.debug('running %s\n' % cmd)
|
|
cmd = util.quotecommand(cmd)
|
|
|
|
# while self.subprocess isn't used, having it allows the subprocess to
|
|
# to clean up correctly later
|
|
#
|
|
# no buffer allow the use of 'select'
|
|
# feel free to remove buffering and select usage when we ultimately
|
|
# move to threading.
|
|
sub = util.popen4(cmd, bufsize=0)
|
|
self.pipeo, self.pipei, self.pipee, self.subprocess = sub
|
|
|
|
self.pipei = util.bufferedinputpipe(self.pipei)
|
|
self.pipei = doublepipe(self.ui, self.pipei, self.pipee)
|
|
self.pipeo = doublepipe(self.ui, self.pipeo, self.pipee)
|
|
|
|
# skip any noise generated by remote shell
|
|
self._callstream("hello")
|
|
r = self._callstream("between", pairs=("%s-%s" % ("0"*40, "0"*40)))
|
|
lines = ["", "dummy"]
|
|
max_noise = 500
|
|
while lines[-1] and max_noise:
|
|
l = r.readline()
|
|
self.readerr()
|
|
if lines[-1] == "1\n" and l == "\n":
|
|
break
|
|
if l:
|
|
self.ui.debug("remote: ", l)
|
|
lines.append(l)
|
|
max_noise -= 1
|
|
else:
|
|
self._abort(error.RepoError(_('no suitable response from '
|
|
'remote hg')))
|
|
|
|
self._caps = set()
|
|
for l in reversed(lines):
|
|
if l.startswith("capabilities:"):
|
|
self._caps.update(l[:-1].split(":")[1].split())
|
|
break
|
|
|
|
def _capabilities(self):
|
|
return self._caps
|
|
|
|
def readerr(self):
|
|
_forwardoutput(self.ui, self.pipee)
|
|
|
|
def _abort(self, exception):
|
|
self.cleanup()
|
|
raise exception
|
|
|
|
def cleanup(self):
|
|
if self.pipeo is None:
|
|
return
|
|
self.pipeo.close()
|
|
self.pipei.close()
|
|
try:
|
|
# read the error descriptor until EOF
|
|
for l in self.pipee:
|
|
self.ui.status(_("remote: "), l)
|
|
except (IOError, ValueError):
|
|
pass
|
|
self.pipee.close()
|
|
|
|
__del__ = cleanup
|
|
|
|
def _submitbatch(self, req):
|
|
rsp = self._callstream("batch", cmds=wireproto.encodebatchcmds(req))
|
|
available = self._getamount()
|
|
# TODO this response parsing is probably suboptimal for large
|
|
# batches with large responses.
|
|
toread = min(available, 1024)
|
|
work = rsp.read(toread)
|
|
available -= toread
|
|
chunk = work
|
|
while chunk:
|
|
while ';' in work:
|
|
one, work = work.split(';', 1)
|
|
yield wireproto.unescapearg(one)
|
|
toread = min(available, 1024)
|
|
chunk = rsp.read(toread)
|
|
available -= toread
|
|
work += chunk
|
|
yield wireproto.unescapearg(work)
|
|
|
|
def _callstream(self, cmd, **args):
|
|
args = pycompat.byteskwargs(args)
|
|
self.ui.debug("sending %s command\n" % cmd)
|
|
self.pipeo.write("%s\n" % cmd)
|
|
_func, names = wireproto.commands[cmd]
|
|
keys = names.split()
|
|
wireargs = {}
|
|
for k in keys:
|
|
if k == '*':
|
|
wireargs['*'] = args
|
|
break
|
|
else:
|
|
wireargs[k] = args[k]
|
|
del args[k]
|
|
for k, v in sorted(wireargs.iteritems()):
|
|
self.pipeo.write("%s %d\n" % (k, len(v)))
|
|
if isinstance(v, dict):
|
|
for dk, dv in v.iteritems():
|
|
self.pipeo.write("%s %d\n" % (dk, len(dv)))
|
|
self.pipeo.write(dv)
|
|
else:
|
|
self.pipeo.write(v)
|
|
self.pipeo.flush()
|
|
|
|
return self.pipei
|
|
|
|
def _callcompressable(self, cmd, **args):
|
|
return self._callstream(cmd, **args)
|
|
|
|
def _call(self, cmd, **args):
|
|
self._callstream(cmd, **args)
|
|
return self._recv()
|
|
|
|
def _callpush(self, cmd, fp, **args):
|
|
r = self._call(cmd, **args)
|
|
if r:
|
|
return '', r
|
|
for d in iter(lambda: fp.read(4096), ''):
|
|
self._send(d)
|
|
self._send("", flush=True)
|
|
r = self._recv()
|
|
if r:
|
|
return '', r
|
|
return self._recv(), ''
|
|
|
|
def _calltwowaystream(self, cmd, fp, **args):
|
|
r = self._call(cmd, **args)
|
|
if r:
|
|
# XXX needs to be made better
|
|
raise error.Abort(_('unexpected remote reply: %s') % r)
|
|
for d in iter(lambda: fp.read(4096), ''):
|
|
self._send(d)
|
|
self._send("", flush=True)
|
|
return self.pipei
|
|
|
|
def _getamount(self):
|
|
l = self.pipei.readline()
|
|
if l == '\n':
|
|
self.readerr()
|
|
msg = _('check previous remote output')
|
|
self._abort(error.OutOfBandError(hint=msg))
|
|
self.readerr()
|
|
try:
|
|
return int(l)
|
|
except ValueError:
|
|
self._abort(error.ResponseError(_("unexpected response:"), l))
|
|
|
|
def _recv(self):
|
|
return self.pipei.read(self._getamount())
|
|
|
|
def _send(self, data, flush=False):
|
|
self.pipeo.write("%d\n" % len(data))
|
|
if data:
|
|
self.pipeo.write(data)
|
|
if flush:
|
|
self.pipeo.flush()
|
|
self.readerr()
|
|
|
|
def lock(self):
|
|
self._call("lock")
|
|
return remotelock(self)
|
|
|
|
def unlock(self):
|
|
self._call("unlock")
|
|
|
|
def addchangegroup(self, cg, source, url, lock=None):
|
|
'''Send a changegroup to the remote server. Return an integer
|
|
similar to unbundle(). DEPRECATED, since it requires locking the
|
|
remote.'''
|
|
d = self._call("addchangegroup")
|
|
if d:
|
|
self._abort(error.RepoError(_("push refused: %s") % d))
|
|
for d in iter(lambda: cg.read(4096), ''):
|
|
self.pipeo.write(d)
|
|
self.readerr()
|
|
|
|
self.pipeo.flush()
|
|
|
|
self.readerr()
|
|
r = self._recv()
|
|
if not r:
|
|
return 1
|
|
try:
|
|
return int(r)
|
|
except ValueError:
|
|
self._abort(error.ResponseError(_("unexpected response:"), r))
|
|
|
|
instance = sshpeer
|