This is done by a script [2] using RedBaron [1], a tool designed for doing
code refactoring. All "default" values are decided by the script and are
strongly consistent with the existing code.
There are 2 changes done manually to fix tests:
[warn] mercurial/exchange.py: experimental.bundle2-output-capture: default needs manual removal
[warn] mercurial/localrepo.py: experimental.hook-track-tags: default needs manual removal
Since RedBaron is not confident about how to indent things [2].
[1]: https://github.com/PyCQA/redbaron
[2]: https://github.com/PyCQA/redbaron/issues/100
[3]:
#!/usr/bin/env python
# codemod_configitems.py - codemod tool to fill configitems
#
# Copyright 2017 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import, print_function
import os
import sys
import redbaron
def readpath(path):
with open(path) as f:
return f.read()
def writepath(path, content):
with open(path, 'w') as f:
f.write(content)
_configmethods = {'config', 'configbool', 'configint', 'configbytes',
'configlist', 'configdate'}
def extractstring(rnode):
"""get the string from a RedBaron string or call_argument node"""
while rnode.type != 'string':
rnode = rnode.value
return rnode.value[1:-1] # unquote, "'str'" -> "str"
def uiconfigitems(red):
"""match *.ui.config* pattern, yield (node, method, args, section, name)"""
for node in red.find_all('atomtrailers'):
entry = None
try:
obj = node[-3].value
method = node[-2].value
args = node[-1]
section = args[0].value
name = args[1].value
if (obj in ('ui', 'self') and method in _configmethods
and section.type == 'string' and name.type == 'string'):
entry = (node, method, args, extractstring(section),
extractstring(name))
except Exception:
pass
else:
if entry:
yield entry
def coreconfigitems(red):
"""match coreconfigitem(...) pattern, yield (node, args, section, name)"""
for node in red.find_all('atomtrailers'):
entry = None
try:
args = node[1]
section = args[0].value
name = args[1].value
if (node[0].value == 'coreconfigitem' and section.type == 'string'
and name.type == 'string'):
entry = (node, args, extractstring(section),
extractstring(name))
except Exception:
pass
else:
if entry:
yield entry
def registercoreconfig(cfgred, section, name, defaultrepr):
"""insert coreconfigitem to cfgred AST
section and name are plain string, defaultrepr is a string
"""
# find a place to insert the "coreconfigitem" item
entries = list(coreconfigitems(cfgred))
for node, args, nodesection, nodename in reversed(entries):
if (nodesection, nodename) < (section, name):
# insert after this entry
node.insert_after(
'coreconfigitem(%r, %r,\n'
' default=%s,\n'
')' % (section, name, defaultrepr))
return
def main(argv):
if not argv:
print('Usage: codemod_configitems.py FILES\n'
'For example, FILES could be "{hgext,mercurial}/*/**.py"')
dirname = os.path.dirname
reporoot = dirname(dirname(dirname(os.path.abspath(__file__))))
# register configitems to this destination
cfgpath = os.path.join(reporoot, 'mercurial', 'configitems.py')
cfgred = redbaron.RedBaron(readpath(cfgpath))
# state about what to do
registered = set((s, n) for n, a, s, n in coreconfigitems(cfgred))
toregister = {} # {(section, name): defaultrepr}
coreconfigs = set() # {(section, name)}, whether it's used in core
# first loop: scan all files before taking any action
for i, path in enumerate(argv):
print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
iscore = ('mercurial' in path) and ('hgext' not in path)
red = redbaron.RedBaron(readpath(path))
# find all repo.ui.config* and ui.config* calls, and collect their
# section, name and default value information.
for node, method, args, section, name in uiconfigitems(red):
if section == 'web':
# [web] section has some weirdness, ignore them for now
continue
defaultrepr = None
key = (section, name)
if len(args) == 2:
if key in registered:
continue
if method == 'configlist':
defaultrepr = 'list'
elif method == 'configbool':
defaultrepr = 'False'
else:
defaultrepr = 'None'
elif len(args) >= 3 and (args[2].target is None or
args[2].target.value == 'default'):
# try to understand the "default" value
dnode = args[2].value
if dnode.type == 'name':
if dnode.value in {'None', 'True', 'False'}:
defaultrepr = dnode.value
elif dnode.type == 'string':
defaultrepr = repr(dnode.value[1:-1])
elif dnode.type in ('int', 'float'):
defaultrepr = dnode.value
# inconsistent default
if key in toregister and toregister[key] != defaultrepr:
defaultrepr = None
# interesting to rewrite
if key not in registered:
if defaultrepr is None:
print('[note] %s: %s.%s: unsupported default'
% (path, section, name))
registered.add(key) # skip checking it again
else:
toregister[key] = defaultrepr
if iscore:
coreconfigs.add(key)
# second loop: rewrite files given "toregister" result
for path in argv:
# reconstruct redbaron - trade CPU for memory
red = redbaron.RedBaron(readpath(path))
changed = False
for node, method, args, section, name in uiconfigitems(red):
key = (section, name)
defaultrepr = toregister.get(key)
if defaultrepr is None or key not in coreconfigs:
continue
if len(args) >= 3 and (args[2].target is None or
args[2].target.value == 'default'):
try:
del args[2]
changed = True
except Exception:
# redbaron fails to do the rewrite due to indentation
# see https://github.com/PyCQA/redbaron/issues/100
print('[warn] %s: %s.%s: default needs manual removal'
% (path, section, name))
if key not in registered:
print('registering %s.%s' % (section, name))
registercoreconfig(cfgred, section, name, defaultrepr)
registered.add(key)
if changed:
print('updating %s' % path)
writepath(path, red.dumps())
if toregister:
print('updating configitems.py')
writepath(cfgpath, cfgred.dumps())
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
smtp.verifycert was accidentally broken by 799db3fe9866. And,
I believe the "loose" value has been broken for longer than that.
The current code refuses to talk to a remote server unless the
CA is trusted or the fingerprint is validated. In other words,
we lost the ability for smtp.verifycert to lower/disable security.
There are special considerations for smtp.verifycert in
sslutil.validatesocket() (the "strict" argument). This violates
the direction sslutil is evolving towards, which has all security
options determined at wrapsocket() time and a unified code path and
configs for determining security options.
Since smtp.verifycert is broken and since we'll soon have new
security defaults and new mechanisms for controlling host security,
this patch formally deprecates smtp.verifycert. With this patch,
the socket security code in mail.py now effectively mirrors code
in url.py and other places we're doing socket security.
For the record, removing smtp.verifycert because it was accidentally
broken is a poor excuse to remove it. However, I would have done this
anyway because smtp.verifycert is a one-off likely used by few people
(users of the patchbomb extension) and I don't think the existence
of this seldom-used one-off in security code can be justified,
especially when you consider that better mechanisms are right around
the corner.
Arguments to sslutil.wrapsocket() are partially determined by
calling sslutil.sslkwargs(). This function receives a ui and
a hostname and determines what settings, if any, need to be
applied when the socket is wrapped.
Both the ui and hostname are passed into wrapsocket(). The
other arguments to wrapsocket() provided by sslkwargs() (ca_certs
and cert_reqs) are not looked at or modified anywhere outside
of sslutil.py. So, sslkwargs() doesn't need to exist as a
separate public API called before wrapsocket().
This commit starts the process of removing external consumers of
sslkwargs() by removing the "ui" key/argument from its return.
All callers now pass the ui argument explicitly.
Now that the socket validator doesn't have any instance state,
we can make it a generic function.
The "validator" class has been converted into the "validatesocket"
function and all consumers have been updated.
The home of 'Abort' is 'error' not 'util' however, a lot of code seems to be
confused about that and gives all the credit to 'util' instead of the
hardworking 'error'. In a spirit of equity, we break the cycle of injustice and
give back to 'error' the respect it deserves. And screw that 'util' poser.
For great justice.
Python 2.6 introduced the "except type as instance" syntax, replacing
the "except type, instance" syntax that came before. Python 3 dropped
support for the latter syntax. Since we no longer support Python 2.4 or
2.5, we have no need to continue supporting the "except type, instance".
This patch mass rewrites the exception syntax to be Python 2.6+ and
Python 3 compatible.
This patch was produced by running `2to3 -f except -w -n .`.
64e4c6bb733d made 'ui' argument is passed via sslutil.sslkwargs(), but mailer
doesn't call sslkwargs() if smtp.verifycert is off. So we have to put it in
sslkwargs manually.
The mail module only verifies the smtp ssl certificate if 'verifycert' is enabled
(the default). The 'verifycert' can take three possible values:
- 'strict'
- 'loose'
- any "False" value, eg: 'false' or '0'
We tested the validity of the third value, but never converted it to actual
falseness, making 'False' an equivalent for 'loose'.
This changeset fixes it.
Before this patch, port 25 (wellknown port of SMTP) is used as default
port, even if "[smtp] tls" is configured as "smtps".
This patch uses port 465 (wellknown port of SMTPS) as default port, if
"[smtp] tls" is configured as "smtps".
Before this patch, the certificate of the SMTP server for STARTTLS or
SMTPS isn't verified.
This may cause man-in-the-middle security problem (stealing
authentication information), even though SMTP channel itself is
encrypted by SSL.
When "[smtp] tls" is configured as "smtps" or "starttls", this patch:
- uses classes introduced by preceding patches instead of "SMTP" or
"SMTP_SSL" of smtplib, and
- verifies the certificate of the SMTP server, if "[smtp]
verifycert" is configured as other than False
"[smtp] verifycert" can be configured in 3 levels:
- "strict":
This verifies peer certificate, and aborts if:
- peer certification is not valid, or
- no configuration in "[hostfingerprints]" and "[web] cacerts"
This is default value of "[smtp] verifycert" for security.
- "loose":
This verifies peer certificate, and aborts if peer certification is
not valid.
This just shows warning message ("certificate not verified"), if
there is no configuration in "[hostfingerprints]" and "[web]
cacerts".
This is as same as verification for HTTPS connection.
- False(no verification):
Peer certificate is not verified.
This is as same as the behavior before this patch series.
"hg email --insecure" uses "loose" level, and ignores "[web] cacerts"
as same as push/pull/etc... with --insecure.
Ignoring "[web] cacerts" configuration for "hg email --insecure" is
already done in "dispatch._dispatch()" by looking "insecure" up in the
table of command options.
Original "smtplib.SMTP_SSL" has no route to pass "ca_certs" and
"cert_reqs" arguments to underlying SSL socket creation. This causes
that "getpeercert()" on SSL socket returns empty dict, so the peer
certificate for SMTPS can't be verified.
This patch introduces the "SMTPS" class derived from "smtplib.SMTP" to
pass "ca_certs" and "cert_reqs" arguments to underlying SSL socket
creation.
"SMTPS" class is derived directly from "smtplib.SMTP", because amount
of "smtplib.SMTP_SSL" definition derived from "smtplib.SMTP" is as
same as one needed to override it.
This patch defines "SMTPS" class, only when "smtplib.SMTP" class has
"_get_socket()" method, because this makes using SSL socket instead of
normal socket easy.
"smtplib.SMTP" class of Python 2.5.x or earlier doesn't have this
method. Omitting SMTPS support for them is reasonable, because
"smtplib.SMTP_SSL" is already unavailable for them before this patch.
Almost all code of "SMTPS" class is imported from "smtplib.SMTP_SSL"
of Python 2.7.3, but it differs from original code in point below:
- "ssl.wrap_socket()" is replaced by "sslutil.ssl_wrap_socket()" for
compatibility between Python versions
Original "smtplib.SMTP" has no route to pass "ca_certs" and
"cert_reqs" arguments to underlying SSL socket creation. This causes
that "getpeercert()" on SSL socket returns empty dict, so the peer
certificate for STARTTLS can't be verified.
This patch introduces the "STARTTLS" class derived from "smtplib.SMTP"
to pass "ca_certs" and "cert_reqs" arguments to underlying SSL socket
creation.
Almost all code of "starttls()" in this class is imported from
"smtplib.SMTP" of Python 2.7.3, but it differs from original code in
points below:
- "self.ehlo_or_helo_if_needed()" invocation is omitted, because:
- "ehlo_or_helo_if_needed()" is available with Python 2.6 or later, and
- "ehlo()" is explicitly invoked in "mercurial.mail._smtp()"
- "if not _have_ssl:" check is omitted, because:
- "_have_ssl" is available with Python 2.6 or later, and
- same checking is done in "mercurial.sslutil.ssl_wrap_socket()"
- "ssl.wrap_socket()" is replaced by "sslutil.ssl_wrap_socket()" for
compatibility between Python versions
- use "sock.recv()" also as "sock.read()", if "sock" doesn't have
"read()" method
with Python 2.5.x or earlier, "sslutil.ssl_wrap_socket()" returns
"httplib.FakeSocket"-ed object, and it doesn't have "read()"
method, which is invoked via "smtplib.SSLFakeFile".
Currently we only support enabling TLS by using SMTP STARTTLS extension. But
not all the servers support it.
With this patch, user can choose which way to enable TLS:
* Default:
tls = none
port = 25
* To use STARTTLS:
tls = starttls
port = 465
* To use SMTP over SSL:
tls = smtps
port = 465
To keep backward compatibility, when tls = true, we use STARTTLS to enable TLS.
Signed-off-by: Zhigang Wang <w1z2g3@gmail.com>
This adds util.getport(port) which tries to parse port as an int, and
failing that, looks it up using socket.getservbyname(). Thus, the
following will work:
[smtp]
port = submission
[web]
port = http
This does not apply to ports in URLs used in clone, pull, etc.
