mirror of
https://github.com/facebook/sapling.git
synced 2024-10-07 23:38:50 +03:00
d64d4dc9f0
Prior to version 2.7, calling locale.getpreferredencoding() would always return 'mac-roman' on Mac OS X. Previously, this was handled by a call to locale.setlocale(). Unfortunately, Python 2.6.5 and older have a bug where isspace() would incorrectly report True for 0x85 and 0xa0 after such a call. In order to fix this, we replace the previous _encodingfixup mapping to an _encodingfixers mapping. Rather than mapping encodings to their replacement, it maps them to a function returning the replacement. This allows us to provide an simplified implementation of getpreferredencoding() which extracts the expected encoding and restores the locale. This fix is based on a patch originally submitted by Martijn Pieters as well as feedback from Brodie Rao.
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
# encoding.py - character transcoding support for Mercurial
|
|
#
|
|
# Copyright 2005-2009 Matt Mackall <mpm@selenic.com> and others
|
|
#
|
|
# This software may be used and distributed according to the terms of the
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
import error
|
|
import sys, unicodedata, locale, os
|
|
|
|
def _getpreferredencoding():
|
|
'''
|
|
On darwin, getpreferredencoding ignores the locale environment and
|
|
always returns mac-roman. http://bugs.python.org/issue6202 fixes this
|
|
for Python 2.7 and up. This is the same corrected code for earlier
|
|
Python versions.
|
|
|
|
However, we can't use a version check for this method, as some distributions
|
|
patch Python to fix this. Instead, we use it as a 'fixer' for the mac-roman
|
|
encoding, as it is unlikely that this encoding is the actually expected.
|
|
'''
|
|
try:
|
|
locale.CODESET
|
|
except AttributeError:
|
|
# Fall back to parsing environment variables :-(
|
|
return locale.getdefaultlocale()[1]
|
|
|
|
oldloc = locale.setlocale(locale.LC_CTYPE)
|
|
locale.setlocale(locale.LC_CTYPE, "")
|
|
result = locale.nl_langinfo(locale.CODESET)
|
|
locale.setlocale(locale.LC_CTYPE, oldloc)
|
|
|
|
return result
|
|
|
|
_encodingfixers = {
|
|
'646': lambda: 'ascii',
|
|
'ANSI_X3.4-1968': lambda: 'ascii',
|
|
'mac-roman': _getpreferredencoding
|
|
}
|
|
|
|
try:
|
|
encoding = os.environ.get("HGENCODING")
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding() or 'ascii'
|
|
encoding = _encodingfixers.get(encoding, lambda: encoding)()
|
|
except locale.Error:
|
|
encoding = 'ascii'
|
|
encodingmode = os.environ.get("HGENCODINGMODE", "strict")
|
|
fallbackencoding = 'ISO-8859-1'
|
|
|
|
def tolocal(s):
|
|
"""
|
|
Convert a string from internal UTF-8 to local encoding
|
|
|
|
All internal strings should be UTF-8 but some repos before the
|
|
implementation of locale support may contain latin1 or possibly
|
|
other character sets. We attempt to decode everything strictly
|
|
using UTF-8, then Latin-1, and failing that, we use UTF-8 and
|
|
replace unknown characters.
|
|
"""
|
|
for e in ('UTF-8', fallbackencoding):
|
|
try:
|
|
u = s.decode(e) # attempt strict decoding
|
|
return u.encode(encoding, "replace")
|
|
except LookupError, k:
|
|
raise error.Abort("%s, please check your locale settings" % k)
|
|
except UnicodeDecodeError:
|
|
pass
|
|
u = s.decode("utf-8", "replace") # last ditch
|
|
return u.encode(encoding, "replace")
|
|
|
|
def fromlocal(s):
|
|
"""
|
|
Convert a string from the local character encoding to UTF-8
|
|
|
|
We attempt to decode strings using the encoding mode set by
|
|
HGENCODINGMODE, which defaults to 'strict'. In this mode, unknown
|
|
characters will cause an error message. Other modes include
|
|
'replace', which replaces unknown characters with a special
|
|
Unicode character, and 'ignore', which drops the character.
|
|
"""
|
|
try:
|
|
return s.decode(encoding, encodingmode).encode("utf-8")
|
|
except UnicodeDecodeError, inst:
|
|
sub = s[max(0, inst.start - 10):inst.start + 10]
|
|
raise error.Abort("decoding near '%s': %s!" % (sub, inst))
|
|
except LookupError, k:
|
|
raise error.Abort("%s, please check your locale settings" % k)
|
|
|
|
def colwidth(s):
|
|
"Find the column width of a UTF-8 string for display"
|
|
d = s.decode(encoding, 'replace')
|
|
if hasattr(unicodedata, 'east_asian_width'):
|
|
w = unicodedata.east_asian_width
|
|
return sum([w(c) in 'WFA' and 2 or 1 for c in d])
|
|
return len(d)
|
|
|