2008-08-14 05:18:40 +04:00
|
|
|
# win32mbcs.py -- MBCS filename support for Mercurial
|
2008-01-09 16:41:30 +03:00
|
|
|
#
|
|
|
|
# Copyright (c) 2008 Shun-ichi Goto <shunichi.goto@gmail.com>
|
|
|
|
#
|
2009-12-07 13:18:03 +03:00
|
|
|
# Version: 0.3
|
2008-01-09 16:41:30 +03:00
|
|
|
# Author: Shun-ichi Goto <shunichi.goto@gmail.com>
|
|
|
|
#
|
2009-04-26 03:08:54 +04:00
|
|
|
# This software may be used and distributed according to the terms of the
|
2010-01-20 07:20:08 +03:00
|
|
|
# GNU General Public License version 2 or any later version.
|
2008-01-09 16:41:30 +03:00
|
|
|
#
|
2009-04-26 03:25:53 +04:00
|
|
|
|
2009-06-24 14:25:56 +04:00
|
|
|
'''allow the use of MBCS paths with problematic encodings
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2009-07-26 04:03:00 +04:00
|
|
|
Some MBCS encodings are not good for some path operations (i.e.
|
|
|
|
splitting path, case conversion, etc.) with its encoded bytes. We call
|
|
|
|
such a encoding (i.e. shift_jis and big5) as "problematic encoding".
|
|
|
|
This extension can be used to fix the issue with those encodings by
|
|
|
|
wrapping some functions to convert to Unicode string before path
|
|
|
|
operation.
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2009-05-31 16:55:51 +04:00
|
|
|
This extension is useful for:
|
2009-07-23 02:25:54 +04:00
|
|
|
|
|
|
|
- Japanese Windows users using shift_jis encoding.
|
|
|
|
- Chinese Windows users using big5 encoding.
|
|
|
|
- All users who use a repository with one of problematic encodings on
|
|
|
|
case-insensitive file system.
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2008-08-14 05:18:40 +04:00
|
|
|
This extension is not needed for:
|
2009-07-23 02:25:54 +04:00
|
|
|
|
|
|
|
- Any user who use only ASCII chars in path.
|
|
|
|
- Any user who do not use any of problematic encodings.
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2008-08-14 05:18:40 +04:00
|
|
|
Note that there are some limitations on using this extension:
|
2009-07-23 02:25:54 +04:00
|
|
|
|
|
|
|
- You should use single encoding in one repository.
|
2010-11-19 12:07:15 +03:00
|
|
|
- If the repository path ends with 0x5c, .hg/hgrc cannot be read.
|
2011-02-01 10:29:11 +03:00
|
|
|
- win32mbcs is not compatible with fixutf8 extension.
|
2009-12-07 13:18:03 +03:00
|
|
|
|
2009-12-14 02:26:28 +03:00
|
|
|
By default, win32mbcs uses encoding.encoding decided by Mercurial.
|
|
|
|
You can specify the encoding by config option::
|
2009-12-07 13:18:03 +03:00
|
|
|
|
|
|
|
[win32mbcs]
|
|
|
|
encoding = sjis
|
|
|
|
|
2009-12-14 02:26:28 +03:00
|
|
|
It is useful for the users who want to commit with UTF-8 log message.
|
2009-06-22 17:48:08 +04:00
|
|
|
'''
|
2016-03-03 00:50:35 +03:00
|
|
|
from __future__ import absolute_import
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2016-03-03 00:50:35 +03:00
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
|
2016-05-14 08:03:12 +03:00
|
|
|
from mercurial.i18n import _
|
2016-03-03 00:50:35 +03:00
|
|
|
from mercurial import (
|
|
|
|
encoding,
|
|
|
|
error,
|
2016-12-17 17:54:46 +03:00
|
|
|
pycompat,
|
2017-06-30 04:45:54 +03:00
|
|
|
registrar,
|
2016-03-03 00:50:35 +03:00
|
|
|
)
|
|
|
|
|
2016-08-23 18:26:08 +03:00
|
|
|
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
|
2015-04-28 23:44:37 +03:00
|
|
|
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
|
|
|
|
# be specifying the version(s) of Mercurial they are tested with, or
|
|
|
|
# leave the attribute unspecified.
|
2016-08-23 18:26:08 +03:00
|
|
|
testedwith = 'ships-with-hg-core'
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2017-06-30 04:45:54 +03:00
|
|
|
configtable = {}
|
|
|
|
configitem = registrar.configitem(configtable)
|
|
|
|
|
|
|
|
# Encoding.encoding may be updated by --encoding option.
|
|
|
|
# Use a lambda do delay the resolution.
|
|
|
|
configitem('win32mbcs', 'encoding',
|
|
|
|
default=lambda: encoding.encoding,
|
|
|
|
)
|
|
|
|
|
2010-11-19 12:07:15 +03:00
|
|
|
_encoding = None # see extsetup
|
2009-12-07 13:18:03 +03:00
|
|
|
|
2008-08-14 05:18:40 +04:00
|
|
|
def decode(arg):
|
2009-03-23 15:13:27 +03:00
|
|
|
if isinstance(arg, str):
|
2009-12-07 13:18:03 +03:00
|
|
|
uarg = arg.decode(_encoding)
|
|
|
|
if arg == uarg.encode(_encoding):
|
2009-03-23 15:13:27 +03:00
|
|
|
return uarg
|
|
|
|
raise UnicodeError("Not local encoding")
|
|
|
|
elif isinstance(arg, tuple):
|
|
|
|
return tuple(map(decode, arg))
|
|
|
|
elif isinstance(arg, list):
|
|
|
|
return map(decode, arg)
|
2009-07-10 10:52:01 +04:00
|
|
|
elif isinstance(arg, dict):
|
|
|
|
for k, v in arg.items():
|
|
|
|
arg[k] = decode(v)
|
2009-03-23 15:13:27 +03:00
|
|
|
return arg
|
2008-08-14 05:18:40 +04:00
|
|
|
|
|
|
|
def encode(arg):
|
flake8: enable F821 check
Summary:
This check is useful and detects real errors (ex. fbconduit). Unfortunately
`arc lint` will run it with both py2 and py3 so a lot of py2 builtins will
still be warned.
I didn't find a clean way to disable py3 check. So this diff tries to fix them.
For `xrange`, the change was done by a script:
```
import sys
import redbaron
headertypes = {'comment', 'endl', 'from_import', 'import', 'string',
'assignment', 'atomtrailers'}
xrangefix = '''try:
xrange(0)
except NameError:
xrange = range
'''
def isxrange(x):
try:
return x[0].value == 'xrange'
except Exception:
return False
def main(argv):
for i, path in enumerate(argv):
print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
content = open(path).read()
try:
red = redbaron.RedBaron(content)
except Exception:
print(' warning: failed to parse')
continue
hasxrange = red.find('atomtrailersnode', value=isxrange)
hasxrangefix = 'xrange = range' in content
if hasxrangefix or not hasxrange:
print(' no need to change')
continue
# find a place to insert the compatibility statement
changed = False
for node in red:
if node.type in headertypes:
continue
# node.insert_before is an easier API, but it has bugs changing
# other "finally" and "except" positions. So do the insert
# manually.
# # node.insert_before(xrangefix)
line = node.absolute_bounding_box.top_left.line - 1
lines = content.splitlines(1)
content = ''.join(lines[:line]) + xrangefix + ''.join(lines[line:])
changed = True
break
if changed:
# "content" is faster than "red.dumps()"
open(path, 'w').write(content)
print(' updated')
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
```
For other py2 builtins that do not have a py3 equivalent, some `# noqa`
were added as a workaround for now.
Reviewed By: DurhamG
Differential Revision: D6934535
fbshipit-source-id: 546b62830af144bc8b46788d2e0fd00496838939
2018-02-10 04:31:44 +03:00
|
|
|
if isinstance(arg, unicode): # noqa
|
2009-12-07 13:18:03 +03:00
|
|
|
return arg.encode(_encoding)
|
2009-03-23 15:13:27 +03:00
|
|
|
elif isinstance(arg, tuple):
|
|
|
|
return tuple(map(encode, arg))
|
|
|
|
elif isinstance(arg, list):
|
|
|
|
return map(encode, arg)
|
2009-07-10 10:52:01 +04:00
|
|
|
elif isinstance(arg, dict):
|
|
|
|
for k, v in arg.items():
|
|
|
|
arg[k] = encode(v)
|
2009-03-23 15:13:27 +03:00
|
|
|
return arg
|
2008-08-14 05:18:40 +04:00
|
|
|
|
2009-07-10 14:34:08 +04:00
|
|
|
def appendsep(s):
|
|
|
|
# ensure the path ends with os.sep, appending it if necessary.
|
|
|
|
try:
|
|
|
|
us = decode(s)
|
|
|
|
except UnicodeError:
|
|
|
|
us = s
|
|
|
|
if us and us[-1] not in ':/\\':
|
2016-12-17 17:54:46 +03:00
|
|
|
s += pycompat.ossep
|
2009-07-10 14:34:08 +04:00
|
|
|
return s
|
|
|
|
|
2012-10-17 13:09:00 +04:00
|
|
|
def basewrapper(func, argtype, enc, dec, args, kwds):
|
|
|
|
# check check already converted, then call original
|
2009-03-23 15:13:27 +03:00
|
|
|
for arg in args:
|
2012-10-17 13:09:00 +04:00
|
|
|
if isinstance(arg, argtype):
|
2009-07-10 10:52:01 +04:00
|
|
|
return func(*args, **kwds)
|
2009-03-23 15:13:27 +03:00
|
|
|
|
|
|
|
try:
|
2012-10-17 13:09:00 +04:00
|
|
|
# convert string arguments, call func, then convert back the
|
|
|
|
# return value.
|
|
|
|
return enc(func(*dec(args), **dec(kwds)))
|
2009-03-23 15:13:27 +03:00
|
|
|
except UnicodeError:
|
2015-10-08 22:55:45 +03:00
|
|
|
raise error.Abort(_("[win32mbcs] filename conversion failed with"
|
2009-12-07 13:18:03 +03:00
|
|
|
" %s encoding\n") % (_encoding))
|
2008-08-14 05:18:40 +04:00
|
|
|
|
2012-10-17 13:09:00 +04:00
|
|
|
def wrapper(func, args, kwds):
|
flake8: enable F821 check
Summary:
This check is useful and detects real errors (ex. fbconduit). Unfortunately
`arc lint` will run it with both py2 and py3 so a lot of py2 builtins will
still be warned.
I didn't find a clean way to disable py3 check. So this diff tries to fix them.
For `xrange`, the change was done by a script:
```
import sys
import redbaron
headertypes = {'comment', 'endl', 'from_import', 'import', 'string',
'assignment', 'atomtrailers'}
xrangefix = '''try:
xrange(0)
except NameError:
xrange = range
'''
def isxrange(x):
try:
return x[0].value == 'xrange'
except Exception:
return False
def main(argv):
for i, path in enumerate(argv):
print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
content = open(path).read()
try:
red = redbaron.RedBaron(content)
except Exception:
print(' warning: failed to parse')
continue
hasxrange = red.find('atomtrailersnode', value=isxrange)
hasxrangefix = 'xrange = range' in content
if hasxrangefix or not hasxrange:
print(' no need to change')
continue
# find a place to insert the compatibility statement
changed = False
for node in red:
if node.type in headertypes:
continue
# node.insert_before is an easier API, but it has bugs changing
# other "finally" and "except" positions. So do the insert
# manually.
# # node.insert_before(xrangefix)
line = node.absolute_bounding_box.top_left.line - 1
lines = content.splitlines(1)
content = ''.join(lines[:line]) + xrangefix + ''.join(lines[line:])
changed = True
break
if changed:
# "content" is faster than "red.dumps()"
open(path, 'w').write(content)
print(' updated')
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
```
For other py2 builtins that do not have a py3 equivalent, some `# noqa`
were added as a workaround for now.
Reviewed By: DurhamG
Differential Revision: D6934535
fbshipit-source-id: 546b62830af144bc8b46788d2e0fd00496838939
2018-02-10 04:31:44 +03:00
|
|
|
return basewrapper(func, unicode, encode, decode, args, kwds) # noqa
|
2012-10-17 13:09:00 +04:00
|
|
|
|
|
|
|
def reversewrapper(func, args, kwds):
|
|
|
|
return basewrapper(func, str, decode, encode, args, kwds)
|
|
|
|
|
2009-07-10 14:34:08 +04:00
|
|
|
def wrapperforlistdir(func, args, kwds):
|
|
|
|
# Ensure 'path' argument ends with os.sep to avoids
|
|
|
|
# misinterpreting last 0x5c of MBCS 2nd byte as path separator.
|
|
|
|
if args:
|
|
|
|
args = list(args)
|
|
|
|
args[0] = appendsep(args[0])
|
2009-08-24 23:00:34 +04:00
|
|
|
if 'path' in kwds:
|
2009-07-10 14:34:08 +04:00
|
|
|
kwds['path'] = appendsep(kwds['path'])
|
|
|
|
return func(*args, **kwds)
|
|
|
|
|
|
|
|
def wrapname(name, wrapper):
|
2009-07-08 17:48:48 +04:00
|
|
|
module, name = name.rsplit('.', 1)
|
|
|
|
module = sys.modules[module]
|
2009-03-23 15:13:27 +03:00
|
|
|
func = getattr(module, name)
|
2009-07-10 10:52:01 +04:00
|
|
|
def f(*args, **kwds):
|
|
|
|
return wrapper(func, args, kwds)
|
2016-11-22 01:48:13 +03:00
|
|
|
f.__name__ = func.__name__
|
2009-03-23 15:13:27 +03:00
|
|
|
setattr(module, name, f)
|
2008-08-14 05:18:40 +04:00
|
|
|
|
|
|
|
# List of functions to be wrapped.
|
|
|
|
# NOTE: os.path.dirname() and os.path.basename() are safe because
|
|
|
|
# they use result of os.path.split()
|
|
|
|
funcs = '''os.path.join os.path.split os.path.splitext
|
2016-08-30 19:22:53 +03:00
|
|
|
os.path.normpath os.makedirs mercurial.util.endswithsep
|
|
|
|
mercurial.util.splitpath mercurial.util.fscasesensitive
|
2011-07-06 13:28:42 +04:00
|
|
|
mercurial.util.fspath mercurial.util.pconvert mercurial.util.normpath
|
2013-07-04 18:05:59 +04:00
|
|
|
mercurial.util.checkwinfilename mercurial.util.checkosfilename
|
|
|
|
mercurial.util.split'''
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2012-10-17 13:09:00 +04:00
|
|
|
# These functions are required to be called with local encoded string
|
|
|
|
# because they expects argument is local encoded string and cause
|
|
|
|
# problem with unicode string.
|
2017-05-12 15:46:14 +03:00
|
|
|
rfuncs = '''mercurial.encoding.upper mercurial.encoding.lower
|
win32mbcs: avoid unintentional failure at colorization
Since 1d07d9da84a0, pycompat.bytestr() wrapped by win32mbcs returns
unicode object, if an argument is not byte-str object. And this causes
unexpected failure at colorization.
pycompat.bytestr() is used to convert from color effect "int" value to
byte-str object in color module. Wrapped pycompat.bytestr() returns
unicode object for such "int" value, because it isn't byte-str.
If this returned unicode object is used to colorize non-ASCII byte-str
in cases below, UnicodeDecodeError is raised at an operation between
them.
- colorization uses "ansi" color mode, or
Even though this isn't default on Windows, user might use this
color mode for third party pager.
- ui.write() is buffered with labeled=True
Buffering causes "ansi" color mode internally, regardless of
actual color mode. With "win32" color mode, extra escape sequences
are omitted at writing data out.
For example, with "win32" color mode, "hg status" doesn't fail for
non-ASCII filenames, but "hg log" does for non-ASCII text, because
the latter implies buffered formatter.
There are many "color effect" value lines in color.py, and making them
byte-str objects isn't suitable for fixing on stable. In addition to
it, pycompat.bytestr will be used to get byte-str object from any
types other than int, too.
To resolve this issue, this patch does:
- replace pycompat.bytestr in checkwinfilename() with newly added
hook point util._filenamebytestr, and
- make win32mbcs reverse-wrap util._filenamebytestr
(this is a replacement of 1d07d9da84a0)
This patch does two things above at same time, because separately
applying the former change adds broken revision (from point of view of
win32mbcs) to stable branch.
"_" prefix is added to "filenamebytestr", because it is win32mbcs
specific hook point.
2017-05-31 17:44:33 +03:00
|
|
|
mercurial.util._filenamebytestr'''
|
2012-10-17 13:09:00 +04:00
|
|
|
|
2011-12-25 15:32:48 +04:00
|
|
|
# List of Windows specific functions to be wrapped.
|
|
|
|
winfuncs = '''os.path.splitunc'''
|
|
|
|
|
2008-01-09 16:41:30 +03:00
|
|
|
# codec and alias names of sjis and big5 to be faked.
|
2008-08-14 05:18:40 +04:00
|
|
|
problematic_encodings = '''big5 big5-tw csbig5 big5hkscs big5-hkscs
|
|
|
|
hkscs cp932 932 ms932 mskanji ms-kanji shift_jis csshiftjis shiftjis
|
|
|
|
sjis s_jis shift_jis_2004 shiftjis2004 sjis_2004 sjis2004
|
2009-06-04 16:23:31 +04:00
|
|
|
shift_jisx0213 shiftjisx0213 sjisx0213 s_jisx0213 950 cp950 ms950 '''
|
2008-01-09 16:41:30 +03:00
|
|
|
|
2010-11-19 12:07:15 +03:00
|
|
|
def extsetup(ui):
|
2009-03-23 15:13:27 +03:00
|
|
|
# TODO: decide use of config section for this extension
|
2011-12-25 15:32:48 +04:00
|
|
|
if ((not os.path.supports_unicode_filenames) and
|
2016-12-18 23:56:41 +03:00
|
|
|
(pycompat.sysplatform != 'cygwin')):
|
2009-03-23 15:13:27 +03:00
|
|
|
ui.warn(_("[win32mbcs] cannot activate on this platform.\n"))
|
|
|
|
return
|
2009-12-07 13:18:03 +03:00
|
|
|
# determine encoding for filename
|
|
|
|
global _encoding
|
2017-06-30 04:45:54 +03:00
|
|
|
_encoding = ui.config('win32mbcs', 'encoding')
|
2009-03-23 15:13:27 +03:00
|
|
|
# fake is only for relevant environment.
|
2009-12-07 13:18:03 +03:00
|
|
|
if _encoding.lower() in problematic_encodings.split():
|
2009-03-23 15:13:27 +03:00
|
|
|
for f in funcs.split():
|
2009-07-10 14:34:08 +04:00
|
|
|
wrapname(f, wrapper)
|
2017-10-13 09:30:46 +03:00
|
|
|
if pycompat.iswindows:
|
2011-12-25 15:32:48 +04:00
|
|
|
for f in winfuncs.split():
|
|
|
|
wrapname(f, wrapper)
|
2017-04-26 16:26:28 +03:00
|
|
|
wrapname("mercurial.util.listdir", wrapperforlistdir)
|
|
|
|
wrapname("mercurial.windows.listdir", wrapperforlistdir)
|
2012-10-17 13:09:00 +04:00
|
|
|
# wrap functions to be called with local byte string arguments
|
|
|
|
for f in rfuncs.split():
|
|
|
|
wrapname(f, reversewrapper)
|
2010-11-19 12:07:15 +03:00
|
|
|
# Check sys.args manually instead of using ui.debug() because
|
|
|
|
# command line options is not yet applied when
|
|
|
|
# extensions.loadall() is called.
|
|
|
|
if '--debug' in sys.argv:
|
check-code: detect "missing _() in ui message" more exactly
Before this patch, "missing _() in ui message" rule overlooks
translatable message, which starts with other than alphabet.
To detect "missing _() in ui message" more exactly, this patch
improves the regexp with assumptions below.
- sequence consisting of below might precede "translatable message"
in same string token
- formatting string, which starts with '%'
- escaped character, which starts with 'b' (as replacement of '\\'), or
- characters other than '%', 'b' and 'x' (as replacement of alphabet)
- any string tokens might precede a string token, which contains
"translatable message"
This patch builds an input file, which is used to examine "missing _()
in ui message" detection, before '"$check_code" stringjoin.py' in
test-contrib-check-code.t, because this reduces amount of change churn
in subsequent patch.
This patch also applies "()" instead of "_()" on messages below to
hide false-positives:
- messages for ui.debug() or debug commands/tools
- contrib/debugshell.py
- hgext/win32mbcs.py (ui.write() is used, though)
- mercurial/commands.py
- _debugchangegroup
- debugindex
- debuglocks
- debugrevlog
- debugrevspec
- debugtemplate
- untranslatable messages
- doc/gendoc.py (ReST specific text)
- hgext/hgk.py (permission string)
- hgext/keyword.py (text written into configuration file)
- mercurial/cmdutil.py (formatting strings for JSON)
2016-06-20 18:50:39 +03:00
|
|
|
ui.write(("[win32mbcs] activated with encoding: %s\n")
|
2010-11-19 12:07:15 +03:00
|
|
|
% _encoding)
|