2006-12-13 22:27:09 +03:00
|
|
|
# demandimport.py - global demand-loading of modules for Mercurial
|
|
|
|
#
|
2007-06-19 10:51:34 +04:00
|
|
|
# Copyright 2006, 2007 Matt Mackall <mpm@selenic.com>
|
2006-12-13 22:27:09 +03:00
|
|
|
#
|
2009-04-26 03:08:54 +04:00
|
|
|
# This software may be used and distributed according to the terms of the
|
2010-01-20 07:20:08 +03:00
|
|
|
# GNU General Public License version 2 or any later version.
|
2006-12-13 22:27:09 +03:00
|
|
|
|
|
|
|
'''
|
|
|
|
demandimport - automatic demandloading of modules
|
|
|
|
|
|
|
|
To enable this module, do:
|
|
|
|
|
|
|
|
import demandimport; demandimport.enable()
|
|
|
|
|
|
|
|
Imports of the following forms will be demand-loaded:
|
|
|
|
|
|
|
|
import a, b.c
|
|
|
|
import a.b as c
|
|
|
|
from a import b,c # a will be loaded immediately
|
|
|
|
|
|
|
|
These imports will not be delayed:
|
|
|
|
|
|
|
|
from a import *
|
|
|
|
b = __import__(a)
|
|
|
|
'''
|
|
|
|
|
2015-08-09 05:05:28 +03:00
|
|
|
from __future__ import absolute_import
|
|
|
|
|
|
|
|
import contextlib
|
|
|
|
import os
|
|
|
|
import sys
|
2015-06-28 03:31:06 +03:00
|
|
|
|
|
|
|
# __builtin__ in Python 2, builtins in Python 3.
|
|
|
|
try:
|
|
|
|
import __builtin__ as builtins
|
|
|
|
except ImportError:
|
|
|
|
import builtins
|
2015-05-28 23:11:26 +03:00
|
|
|
|
2015-08-09 05:05:28 +03:00
|
|
|
contextmanager = contextlib.contextmanager
|
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
_origimport = __import__
|
|
|
|
|
2011-07-26 06:15:48 +04:00
|
|
|
nothing = object()
|
|
|
|
|
2015-08-09 01:01:27 +03:00
|
|
|
# Python 3 doesn't have relative imports nor level -1.
|
|
|
|
level = -1
|
|
|
|
if sys.version_info[0] >= 3:
|
|
|
|
level = 0
|
|
|
|
_import = _origimport
|
2011-08-23 00:50:52 +04:00
|
|
|
|
2015-08-09 03:07:34 +03:00
|
|
|
def _hgextimport(importfunc, name, globals, *args, **kwargs):
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
try:
|
2015-08-09 03:07:34 +03:00
|
|
|
return importfunc(name, globals, *args, **kwargs)
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
except ImportError:
|
|
|
|
if not globals:
|
|
|
|
raise
|
|
|
|
# extensions are loaded with "hgext_" prefix
|
|
|
|
hgextname = 'hgext_%s' % name
|
|
|
|
nameroot = hgextname.split('.', 1)[0]
|
|
|
|
contextroot = globals.get('__name__', '').split('.', 1)[0]
|
|
|
|
if nameroot != contextroot:
|
|
|
|
raise
|
|
|
|
# retry to import with "hgext_" prefix
|
2015-08-09 03:07:34 +03:00
|
|
|
return importfunc(hgextname, globals, *args, **kwargs)
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
class _demandmod(object):
|
2016-08-06 16:24:33 +03:00
|
|
|
"""module demand-loader and proxy
|
|
|
|
|
|
|
|
Specify 1 as 'level' argument at construction, to import module
|
|
|
|
relatively.
|
|
|
|
"""
|
|
|
|
def __init__(self, name, globals, locals, level):
|
2006-12-13 22:27:09 +03:00
|
|
|
if '.' in name:
|
|
|
|
head, rest = name.split('.', 1)
|
|
|
|
after = [rest]
|
|
|
|
else:
|
|
|
|
head = name
|
|
|
|
after = []
|
2017-03-26 10:33:12 +03:00
|
|
|
object.__setattr__(self, r"_data",
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
(head, globals, locals, after, level, set()))
|
2017-03-26 10:33:12 +03:00
|
|
|
object.__setattr__(self, r"_module", None)
|
2006-12-13 22:27:09 +03:00
|
|
|
def _extend(self, name):
|
|
|
|
"""add to the list of submodules to load"""
|
|
|
|
self._data[3].append(name)
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
|
|
|
|
def _addref(self, name):
|
|
|
|
"""Record that the named module ``name`` imports this module.
|
|
|
|
|
|
|
|
References to this proxy class having the name of this module will be
|
|
|
|
replaced at module load time. We assume the symbol inside the importing
|
|
|
|
module is identical to the "head" name of this module. We don't
|
|
|
|
actually know if "as X" syntax is being used to change the symbol name
|
|
|
|
because this information isn't exposed to __import__.
|
|
|
|
"""
|
|
|
|
self._data[5].add(name)
|
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
def _load(self):
|
|
|
|
if not self._module:
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
head, globals, locals, after, level, modrefs = self._data
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
mod = _hgextimport(_import, head, globals, locals, None, level)
|
demandimport: avoid infinite recursion at actual module importing (issue5304)
Before this patch, importing C module on Windows environment causes
infinite recursion call, if py2exe is used with -b2 option.
At importing C module "a.b", extra hooking by zipextimporter of py2exe
causes:
0. assumption before accessing "b" of "a":
- built-in module object is created for "a",
(= "a" is actually imported)
- _demandmod is created for "a.b" as a proxy object, and
(= "a.b" is not yet imported)
- an attribute "b" of "a" is initialized by the latter
1. invocation of __import__ via _hgextimport() in _demandmod._load()
for "a.b" implies _demandimport() for "a.b"
This is unintentional, because _demandmod might be returned by
_hgextimport() instead of built-in module object.
2. _demandimport() at (1) is invoked with not context of "a", but
context of zipextimporter
Just after invocation of _hgextimport() in _demandimport(), an
attribute "b" of the built-in module object for "a" is still
bound to the proxy object for "a.b", because context of "a" isn't
updated by actual importing "a.b". even though the built-in
module object for "a.b" already appears in sys.modules.
Therefore, chainmodules() returns _demandmod for "a.b", which is
gotten from the attribute "b" of "a".
3. processfromitem() on "a.b" causes _demandmod._load() for "a.b"
again
_demandimport() takes context of "a" in this case.
Therefore, attributes below are bound to built-in module object
for "a.b", as expected:
- "b" of built-in module object for "a"
- _module of _demandmod for "a.b"
4. but _demandimport() invoked at (1) returns _demandmod object
because _demandimport() just returns the object returned by
chainmodules() at (3) above.
5. then, _demandmod._load() causes infinite recursion call
_demandimport() returns _demandmod for "a.b", and it is "self" at
_demandmod._load().
To avoid infinite recursion at actual module importing, this patch
uses self._module, if _hgextimport() returns _demandmod itself. If
_demandmod._module isn't yet bound at this point, execution should be
aborted, because actual importing failed.
In this patch, _demandmod._module is examined not on _demandimport()
side, but on _demandmod._load() side, because:
- the former has some exit points
- only the latter uses _hgextimport(), except for _demandimport()
BTW, this issue occurs only in the code path for non .py/.pyc files in
zipextimporter (strictly speaking, in _memimporter) of py2exe.
Even if zipextimporter is enabled, .py/.pyc files are handled by
zipimporter, and it doesn't imply unintentional _demandimport() at
invocation of __import__ via _hgextimport().
2016-07-30 23:39:59 +03:00
|
|
|
if mod is self:
|
|
|
|
# In this case, _hgextimport() above should imply
|
|
|
|
# _demandimport(). Otherwise, _hgextimport() never
|
|
|
|
# returns _demandmod. This isn't intentional behavior,
|
|
|
|
# in fact. (see also issue5304 for detail)
|
|
|
|
#
|
|
|
|
# If self._module is already bound at this point, self
|
|
|
|
# should be already _load()-ed while _hgextimport().
|
|
|
|
# Otherwise, there is no way to import actual module
|
|
|
|
# as expected, because (re-)invoking _hgextimport()
|
|
|
|
# should cause same result.
|
|
|
|
# This is reason why _load() returns without any more
|
|
|
|
# setup but assumes self to be already bound.
|
|
|
|
mod = self._module
|
|
|
|
assert mod and mod is not self, "%s, %s" % (self, mod)
|
|
|
|
return
|
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
# load submodules
|
2006-12-17 23:56:12 +03:00
|
|
|
def subload(mod, p):
|
|
|
|
h, t = p, None
|
|
|
|
if '.' in p:
|
|
|
|
h, t = p.split('.', 1)
|
2011-07-26 06:15:48 +04:00
|
|
|
if getattr(mod, h, nothing) is nothing:
|
2016-08-06 16:24:33 +03:00
|
|
|
setattr(mod, h, _demandmod(p, mod.__dict__, mod.__dict__,
|
|
|
|
level=1))
|
2006-12-18 05:34:42 +03:00
|
|
|
elif t:
|
2006-12-17 23:56:12 +03:00
|
|
|
subload(getattr(mod, h), t)
|
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
for x in after:
|
2006-12-17 23:56:12 +03:00
|
|
|
subload(mod, x)
|
|
|
|
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
# Replace references to this proxy instance with the actual module.
|
2006-12-13 22:27:09 +03:00
|
|
|
if locals and locals.get(head) == self:
|
|
|
|
locals[head] = mod
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
|
|
|
|
for modname in modrefs:
|
|
|
|
modref = sys.modules.get(modname, None)
|
|
|
|
if modref and getattr(modref, head, None) == self:
|
|
|
|
setattr(modref, head, mod)
|
|
|
|
|
2017-03-26 10:33:12 +03:00
|
|
|
object.__setattr__(self, r"_module", mod)
|
2007-06-19 04:43:26 +04:00
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
def __repr__(self):
|
2007-06-19 04:43:26 +04:00
|
|
|
if self._module:
|
|
|
|
return "<proxied module '%s'>" % self._data[0]
|
2006-12-13 22:27:09 +03:00
|
|
|
return "<unloaded module '%s'>" % self._data[0]
|
|
|
|
def __call__(self, *args, **kwargs):
|
2007-12-10 19:24:47 +03:00
|
|
|
raise TypeError("%s object is not callable" % repr(self))
|
2006-12-16 07:16:20 +03:00
|
|
|
def __getattribute__(self, attr):
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
if attr in ('_data', '_extend', '_load', '_module', '_addref'):
|
2006-12-16 07:16:20 +03:00
|
|
|
return object.__getattribute__(self, attr)
|
2006-12-13 22:27:09 +03:00
|
|
|
self._load()
|
|
|
|
return getattr(self._module, attr)
|
|
|
|
def __setattr__(self, attr, val):
|
|
|
|
self._load()
|
|
|
|
setattr(self._module, attr, val)
|
|
|
|
|
2015-12-24 03:22:20 +03:00
|
|
|
_pypy = '__pypy__' in sys.builtin_module_names
|
|
|
|
|
2014-05-11 01:57:25 +04:00
|
|
|
def _demandimport(name, globals=None, locals=None, fromlist=None, level=level):
|
2006-12-13 22:27:09 +03:00
|
|
|
if not locals or name in ignore or fromlist == ('*',):
|
|
|
|
# these cases we can't really delay
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
return _hgextimport(_import, name, globals, locals, fromlist, level)
|
2006-12-13 22:27:09 +03:00
|
|
|
elif not fromlist:
|
|
|
|
# import a [as b]
|
|
|
|
if '.' in name: # a.b
|
|
|
|
base, rest = name.split('.', 1)
|
2006-12-16 07:16:20 +03:00
|
|
|
# email.__init__ loading email.mime
|
|
|
|
if globals and globals.get('__name__', None) == base:
|
2011-08-23 00:50:52 +04:00
|
|
|
return _import(name, globals, locals, fromlist, level)
|
2006-12-13 22:27:09 +03:00
|
|
|
# if a is already demand-loaded, add b to its submodule list
|
|
|
|
if base in locals:
|
|
|
|
if isinstance(locals[base], _demandmod):
|
|
|
|
locals[base]._extend(rest)
|
|
|
|
return locals[base]
|
2013-10-04 20:02:22 +04:00
|
|
|
return _demandmod(name, globals, locals, level)
|
2006-12-13 22:27:09 +03:00
|
|
|
else:
|
2015-08-09 02:24:57 +03:00
|
|
|
# There is a fromlist.
|
2010-10-30 20:32:04 +04:00
|
|
|
# from a import b,c,d
|
2015-08-09 02:24:57 +03:00
|
|
|
# from . import b,c,d
|
|
|
|
# from .a import b,c,d
|
|
|
|
|
|
|
|
# level == -1: relative and absolute attempted (Python 2 only).
|
|
|
|
# level >= 0: absolute only (Python 2 w/ absolute_import and Python 3).
|
|
|
|
# The modern Mercurial convention is to use absolute_import everywhere,
|
|
|
|
# so modern Mercurial code will have level >= 0.
|
|
|
|
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
# The name of the module the import statement is located in.
|
|
|
|
globalname = globals.get('__name__')
|
|
|
|
|
2015-11-01 15:19:09 +03:00
|
|
|
def processfromitem(mod, attr):
|
2015-10-04 01:30:17 +03:00
|
|
|
"""Process an imported symbol in the import statement.
|
|
|
|
|
2016-09-26 17:28:57 +03:00
|
|
|
If the symbol doesn't exist in the parent module, and if the
|
|
|
|
parent module is a package, it must be a module. We set missing
|
|
|
|
modules up as _demandmod instances.
|
2015-10-04 01:30:17 +03:00
|
|
|
"""
|
2015-10-04 20:36:54 +03:00
|
|
|
symbol = getattr(mod, attr, nothing)
|
2016-09-26 17:28:57 +03:00
|
|
|
nonpkg = getattr(mod, '__path__', nothing) is nothing
|
2015-10-04 20:36:54 +03:00
|
|
|
if symbol is nothing:
|
2016-09-26 17:28:57 +03:00
|
|
|
if nonpkg:
|
2016-12-19 16:46:00 +03:00
|
|
|
# do not try relative import, which would raise ValueError,
|
|
|
|
# and leave unknown attribute as the default __import__()
|
|
|
|
# would do. the missing attribute will be detected later
|
|
|
|
# while processing the import statement.
|
|
|
|
return
|
2016-02-06 13:09:10 +03:00
|
|
|
mn = '%s.%s' % (mod.__name__, attr)
|
|
|
|
if mn in ignore:
|
|
|
|
importfunc = _origimport
|
|
|
|
else:
|
|
|
|
importfunc = _demandmod
|
|
|
|
symbol = importfunc(attr, mod.__dict__, locals, level=1)
|
2015-10-04 20:36:54 +03:00
|
|
|
setattr(mod, attr, symbol)
|
2015-10-04 01:30:17 +03:00
|
|
|
|
demandimport: replace more references to _demandmod instances
_demandmod instances may be referenced by multiple importing modules.
Before this patch, the _demandmod instance only maintained a reference
to its first consumer when using the "from X import Y" syntax. This is
because we only created a single _demandmod instance (attached to the
parent X module). If multiple modules A and B performed
"from X import Y", we'd produce a single _demandmod instance
"demandmod" with the following references:
X.Y = <demandmod>
A.Y = <demandmod>
B.Y = <demandmod>
The locals from the first consumer (A) would be stored in <demandmod1>.
When <demandmod1> was loaded, we'd look at the locals for the first
consumer and replace the symbol, if necessary. This resulted in state:
X.Y = <module>
A.Y = <module>
B.Y = <demandmod>
B's reference to Y wasn't updated and was still using the proxy object
because we just didn't record that B had a reference to <demandmod> that
needed updating!
With this patch, we add support for tracking which modules in addition
to the initial importer have a reference to the _demandmod instance and
we replace those references at module load time.
In the case of posix.py, this fixes an issue where the "encoding" module
was being proxied, resulting in hundreds of thousands of
__getattribute__ lookups on the _demandmod instance during dirstate
operations on mozilla-central, speeding up execution by many
milliseconds. There are likely several other operation that benefit from
this change as well.
The new mechanism isn't perfect: references in locals (not globals) may
likely linger. So, if there is an import inside a function and a symbol
from that module is used in a hot loop, we could have unwanted overhead
from proxying through _demandmod. Non-global imports are discouraged
anyway. So hopefully this isn't a big deal in practice. We could
potentially deploy a code checker that bans use of attribute lookups of
function-level-imported modules inside loops.
This deficiency in theory could be avoided by storing the set of globals
and locals dicts to update in the _demandmod instance. However, I tried
this and it didn't work. One reason is that some globals are _demandmod
instances. We could work around this, but it's a bit more work. There
also might be other module import foo at play. The solution as
implemented is better than what we had and IMO is good enough for the
time being.
It's worth noting that this sub-optimal behavior was made worse by the
introduction of absolute_import and its recommended "from . import X"
syntax for importing modules from the "mercurial" package. If we ever
wrote performance tests, measuring the amount of module imports and
__getattribute__ proxy calls through _demandmod instances would be
something I'd have it check.
2015-10-04 21:17:43 +03:00
|
|
|
# Record the importing module references this symbol so we can
|
|
|
|
# replace the symbol with the actual module instance at load
|
|
|
|
# time.
|
|
|
|
if globalname and isinstance(symbol, _demandmod):
|
|
|
|
symbol._addref(globalname)
|
|
|
|
|
demandimport: delay loading for "from a import b" with absolute_import
Before this patch, "from a import b" doesn't delay loading module "b",
if absolute_import is enabled, even though "from . import b" does.
For example:
- it is assumed that extension X has "from P import M" for module M
under package P with absolute_import feature
- if importing module M is already delayed before loading extension
X, loading module M in extension X is delayed until actually
referring
util, cmdutil, scmutil or so of Mercurial itself should be
imported by "from . import M" style before loading extension X
- otherwise, module M is loaded immediately at loading extension X,
even if extension X itself isn't used at that "hg" command invocation
Some minor modules (e.g. filemerge or so) of Mercurial itself
aren't imported by "from . import M" style before loading
extension X. And of course, external libraries aren't, too.
This might cause startup performance problem of hg command, because
many bundled extensions already enable absolute_import feature.
To delay loading module for "from a import b" with absolute_import
feature, this patch does below in "from a (or .a) import b" with
absolute_import case:
1. import root module of "name" by system built-in __import__
(referred as _origimport)
2. recurse down the module chain for hierarchical "name"
This logic can be shared with non absolute_import
case. Therefore, this patch also centralizes it into chainmodules().
3. and fall through to process elements in "fromlist" for the leaf
module of "name"
Processing elements in "fromlist" is executed in the code path
after "if _pypy: .... else: ..." clause. Therefore, this patch
replaces "if _pypy:" with "elif _pypy:" to share it.
At faecf59a4184 introducing original "work around" for "from a import
b" case, elements in "fromlist" were imported with "level=level". But
"level" might be grater than 1 (e.g. level=2 in "from .. import b"
case) at demandimport() invocation, and importing direct sub-module in
"fromlist" with level grater than 1 causes unexpected result.
IMHO, this seems main reason of "errors for unknown reason" described
in faecf59a4184, and we don't have to worry about it, because this
issue was already fixed by 2711f50242cf.
This is reason why this patch removes "errors for unknown reasons"
comment.
2016-06-18 20:17:33 +03:00
|
|
|
def chainmodules(rootmod, modname):
|
|
|
|
# recurse down the module chain, and return the leaf module
|
|
|
|
mod = rootmod
|
|
|
|
for comp in modname.split('.')[1:]:
|
|
|
|
if getattr(mod, comp, nothing) is nothing:
|
2016-08-06 16:24:33 +03:00
|
|
|
setattr(mod, comp, _demandmod(comp, mod.__dict__,
|
|
|
|
mod.__dict__, level=1))
|
demandimport: delay loading for "from a import b" with absolute_import
Before this patch, "from a import b" doesn't delay loading module "b",
if absolute_import is enabled, even though "from . import b" does.
For example:
- it is assumed that extension X has "from P import M" for module M
under package P with absolute_import feature
- if importing module M is already delayed before loading extension
X, loading module M in extension X is delayed until actually
referring
util, cmdutil, scmutil or so of Mercurial itself should be
imported by "from . import M" style before loading extension X
- otherwise, module M is loaded immediately at loading extension X,
even if extension X itself isn't used at that "hg" command invocation
Some minor modules (e.g. filemerge or so) of Mercurial itself
aren't imported by "from . import M" style before loading
extension X. And of course, external libraries aren't, too.
This might cause startup performance problem of hg command, because
many bundled extensions already enable absolute_import feature.
To delay loading module for "from a import b" with absolute_import
feature, this patch does below in "from a (or .a) import b" with
absolute_import case:
1. import root module of "name" by system built-in __import__
(referred as _origimport)
2. recurse down the module chain for hierarchical "name"
This logic can be shared with non absolute_import
case. Therefore, this patch also centralizes it into chainmodules().
3. and fall through to process elements in "fromlist" for the leaf
module of "name"
Processing elements in "fromlist" is executed in the code path
after "if _pypy: .... else: ..." clause. Therefore, this patch
replaces "if _pypy:" with "elif _pypy:" to share it.
At faecf59a4184 introducing original "work around" for "from a import
b" case, elements in "fromlist" were imported with "level=level". But
"level" might be grater than 1 (e.g. level=2 in "from .. import b"
case) at demandimport() invocation, and importing direct sub-module in
"fromlist" with level grater than 1 causes unexpected result.
IMHO, this seems main reason of "errors for unknown reason" described
in faecf59a4184, and we don't have to worry about it, because this
issue was already fixed by 2711f50242cf.
This is reason why this patch removes "errors for unknown reasons"
comment.
2016-06-18 20:17:33 +03:00
|
|
|
mod = getattr(mod, comp)
|
|
|
|
return mod
|
|
|
|
|
2015-08-09 02:24:57 +03:00
|
|
|
if level >= 0:
|
2015-08-09 02:13:27 +03:00
|
|
|
if name:
|
demandimport: delay loading for "from a import b" with absolute_import
Before this patch, "from a import b" doesn't delay loading module "b",
if absolute_import is enabled, even though "from . import b" does.
For example:
- it is assumed that extension X has "from P import M" for module M
under package P with absolute_import feature
- if importing module M is already delayed before loading extension
X, loading module M in extension X is delayed until actually
referring
util, cmdutil, scmutil or so of Mercurial itself should be
imported by "from . import M" style before loading extension X
- otherwise, module M is loaded immediately at loading extension X,
even if extension X itself isn't used at that "hg" command invocation
Some minor modules (e.g. filemerge or so) of Mercurial itself
aren't imported by "from . import M" style before loading
extension X. And of course, external libraries aren't, too.
This might cause startup performance problem of hg command, because
many bundled extensions already enable absolute_import feature.
To delay loading module for "from a import b" with absolute_import
feature, this patch does below in "from a (or .a) import b" with
absolute_import case:
1. import root module of "name" by system built-in __import__
(referred as _origimport)
2. recurse down the module chain for hierarchical "name"
This logic can be shared with non absolute_import
case. Therefore, this patch also centralizes it into chainmodules().
3. and fall through to process elements in "fromlist" for the leaf
module of "name"
Processing elements in "fromlist" is executed in the code path
after "if _pypy: .... else: ..." clause. Therefore, this patch
replaces "if _pypy:" with "elif _pypy:" to share it.
At faecf59a4184 introducing original "work around" for "from a import
b" case, elements in "fromlist" were imported with "level=level". But
"level" might be grater than 1 (e.g. level=2 in "from .. import b"
case) at demandimport() invocation, and importing direct sub-module in
"fromlist" with level grater than 1 causes unexpected result.
IMHO, this seems main reason of "errors for unknown reason" described
in faecf59a4184, and we don't have to worry about it, because this
issue was already fixed by 2711f50242cf.
This is reason why this patch removes "errors for unknown reasons"
comment.
2016-06-18 20:17:33 +03:00
|
|
|
# "from a import b" or "from .a import b" style
|
|
|
|
rootmod = _hgextimport(_origimport, name, globals, locals,
|
|
|
|
level=level)
|
|
|
|
mod = chainmodules(rootmod, name)
|
|
|
|
elif _pypy:
|
2015-12-24 03:22:20 +03:00
|
|
|
# PyPy's __import__ throws an exception if invoked
|
|
|
|
# with an empty name and no fromlist. Recreate the
|
|
|
|
# desired behaviour by hand.
|
|
|
|
mn = globalname
|
|
|
|
mod = sys.modules[mn]
|
|
|
|
if getattr(mod, '__path__', nothing) is nothing:
|
|
|
|
mn = mn.rsplit('.', 1)[0]
|
|
|
|
mod = sys.modules[mn]
|
|
|
|
if level > 1:
|
|
|
|
mn = mn.rsplit('.', level - 1)[0]
|
|
|
|
mod = sys.modules[mn]
|
|
|
|
else:
|
|
|
|
mod = _hgextimport(_origimport, name, globals, locals,
|
|
|
|
level=level)
|
2015-10-04 01:30:17 +03:00
|
|
|
|
2015-08-09 02:13:27 +03:00
|
|
|
for x in fromlist:
|
2015-11-01 15:19:09 +03:00
|
|
|
processfromitem(mod, x)
|
2015-08-09 02:13:27 +03:00
|
|
|
|
|
|
|
return mod
|
2015-08-09 02:24:57 +03:00
|
|
|
|
|
|
|
# But, we still need to support lazy loading of standard library and 3rd
|
|
|
|
# party modules. So handle level == -1.
|
demandimport: allow extensions to import own modules by absolute name
Before this patch, python modules of each extensions can't import
another one in own extension by absolute name, because root modules of
each extensions are loaded with "hgext_" prefix.
For example, "import extroot.bar" in "extroot/foo.py" of "extroot"
extension fails, even though "import bar" in it succeeds.
Installing extensions into site-packages of python library path can
avoid this problem, but this solution is not reasonable in some cases:
using binary package of Mercurial on Windows, for example.
This patch retries to import with "hgext_" prefix after ImportError,
if the module in the extension may try to import another one in own
extension.
This patch doesn't change some "_import()"/"_origimport()" invocations
below, because ordinary extensions shouldn't cause such invocations.
- invocation of "_import()" when root module imports sub-module by
absolute path without "fromlist"
for example, "import a.b" in "a.__init__.py".
extensions are loaded with "hgext_" prefix, and this causes
execution of another (= fixed by this patch) code path.
- invocation of "_origimport()" when "level != -1" with "fromlist"
for example, importing after "from __future__ import
absolute_import" (level == 0), or "from . import b" or "from .a
import b" (0 < level),
for portability between python versions and environments,
extensions shouldn't cause "level != -1".
2013-10-04 20:02:22 +04:00
|
|
|
mod = _hgextimport(_origimport, name, globals, locals)
|
demandimport: delay loading for "from a import b" with absolute_import
Before this patch, "from a import b" doesn't delay loading module "b",
if absolute_import is enabled, even though "from . import b" does.
For example:
- it is assumed that extension X has "from P import M" for module M
under package P with absolute_import feature
- if importing module M is already delayed before loading extension
X, loading module M in extension X is delayed until actually
referring
util, cmdutil, scmutil or so of Mercurial itself should be
imported by "from . import M" style before loading extension X
- otherwise, module M is loaded immediately at loading extension X,
even if extension X itself isn't used at that "hg" command invocation
Some minor modules (e.g. filemerge or so) of Mercurial itself
aren't imported by "from . import M" style before loading
extension X. And of course, external libraries aren't, too.
This might cause startup performance problem of hg command, because
many bundled extensions already enable absolute_import feature.
To delay loading module for "from a import b" with absolute_import
feature, this patch does below in "from a (or .a) import b" with
absolute_import case:
1. import root module of "name" by system built-in __import__
(referred as _origimport)
2. recurse down the module chain for hierarchical "name"
This logic can be shared with non absolute_import
case. Therefore, this patch also centralizes it into chainmodules().
3. and fall through to process elements in "fromlist" for the leaf
module of "name"
Processing elements in "fromlist" is executed in the code path
after "if _pypy: .... else: ..." clause. Therefore, this patch
replaces "if _pypy:" with "elif _pypy:" to share it.
At faecf59a4184 introducing original "work around" for "from a import
b" case, elements in "fromlist" were imported with "level=level". But
"level" might be grater than 1 (e.g. level=2 in "from .. import b"
case) at demandimport() invocation, and importing direct sub-module in
"fromlist" with level grater than 1 causes unexpected result.
IMHO, this seems main reason of "errors for unknown reason" described
in faecf59a4184, and we don't have to worry about it, because this
issue was already fixed by 2711f50242cf.
This is reason why this patch removes "errors for unknown reasons"
comment.
2016-06-18 20:17:33 +03:00
|
|
|
mod = chainmodules(mod, name)
|
2015-10-04 01:30:17 +03:00
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
for x in fromlist:
|
2015-10-04 01:30:17 +03:00
|
|
|
processfromitem(mod, x)
|
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
return mod
|
|
|
|
|
2007-08-14 18:03:15 +04:00
|
|
|
ignore = [
|
2015-08-09 03:12:37 +03:00
|
|
|
'__future__',
|
2007-08-14 18:03:15 +04:00
|
|
|
'_hashlib',
|
2016-02-26 09:35:11 +03:00
|
|
|
# ImportError during pkg_resources/__init__.py:fixup_namespace_package
|
|
|
|
'_imp',
|
2007-08-14 18:03:15 +04:00
|
|
|
'_xmlplus',
|
|
|
|
'fcntl',
|
2016-09-27 15:56:00 +03:00
|
|
|
'nt', # pathlib2 tests the existence of built-in 'nt' module
|
2007-08-14 18:03:15 +04:00
|
|
|
'win32com.gen_py',
|
2017-04-14 22:34:26 +03:00
|
|
|
'win32com.shell', # 'appdirs' tries to import win32com.shell
|
2010-01-13 13:57:32 +03:00
|
|
|
'_winreg', # 2.7 mimetypes needs immediate ImportError
|
2009-03-10 05:00:37 +03:00
|
|
|
'pythoncom',
|
2007-08-14 18:03:15 +04:00
|
|
|
# imported by tarfile, not available under Windows
|
|
|
|
'pwd',
|
|
|
|
'grp',
|
|
|
|
# imported by profile, itself imported by hotshot.stats,
|
|
|
|
# not available under Windows
|
|
|
|
'resource',
|
2009-09-18 02:39:43 +04:00
|
|
|
# this trips up many extension authors
|
|
|
|
'gtk',
|
2010-01-21 04:23:36 +03:00
|
|
|
# setuptools' pkg_resources.py expects "from __main__ import x" to
|
|
|
|
# raise ImportError if x not defined
|
|
|
|
'__main__',
|
2010-03-09 18:03:57 +03:00
|
|
|
'_ssl', # conditional imports in the stdlib, issue1964
|
2015-10-28 18:27:09 +03:00
|
|
|
'_sre', # issue4920
|
2011-03-02 08:35:22 +03:00
|
|
|
'rfc822',
|
|
|
|
'mimetools',
|
2016-02-06 13:16:12 +03:00
|
|
|
'sqlalchemy.events', # has import-time side effects (issue5085)
|
2014-12-23 01:27:31 +03:00
|
|
|
# setuptools 8 expects this module to explode early when not on windows
|
2016-09-21 05:46:59 +03:00
|
|
|
'distutils.msvc9compiler',
|
2016-10-14 04:03:39 +03:00
|
|
|
'__builtin__',
|
|
|
|
'builtins',
|
2017-05-04 04:26:57 +03:00
|
|
|
'urwid.command_map', # for pudb
|
2007-08-14 18:03:15 +04:00
|
|
|
]
|
2006-12-13 22:27:09 +03:00
|
|
|
|
2016-09-27 16:36:00 +03:00
|
|
|
if _pypy:
|
|
|
|
ignore.extend([
|
|
|
|
# _ctypes.pointer is shadowed by "from ... import pointer" (PyPy 5)
|
|
|
|
'_ctypes.pointer',
|
|
|
|
])
|
|
|
|
|
2014-02-11 02:51:06 +04:00
|
|
|
def isenabled():
|
2015-06-28 03:29:15 +03:00
|
|
|
return builtins.__import__ == _demandimport
|
2014-02-11 02:51:06 +04:00
|
|
|
|
2006-12-13 22:27:09 +03:00
|
|
|
def enable():
|
|
|
|
"enable global demand-loading of modules"
|
2014-04-08 03:35:13 +04:00
|
|
|
if os.environ.get('HGDEMANDIMPORT') != 'disable':
|
2015-06-28 03:29:15 +03:00
|
|
|
builtins.__import__ = _demandimport
|
2006-12-13 22:27:09 +03:00
|
|
|
|
|
|
|
def disable():
|
|
|
|
"disable global demand-loading of modules"
|
2015-06-28 03:29:15 +03:00
|
|
|
builtins.__import__ = _origimport
|
2015-05-28 23:11:26 +03:00
|
|
|
|
|
|
|
@contextmanager
|
|
|
|
def deactivated():
|
|
|
|
"context manager for disabling demandimport in 'with' blocks"
|
|
|
|
demandenabled = isenabled()
|
|
|
|
if demandenabled:
|
|
|
|
disable()
|
|
|
|
|
|
|
|
try:
|
|
|
|
yield
|
|
|
|
finally:
|
|
|
|
if demandenabled:
|
|
|
|
enable()
|