"""
|
modulegraph.find_modules - High-level module dependency finding interface
|
=========================================================================
|
|
History
|
........
|
|
Originally (loosely) based on code in py2exe's build_exe.py by Thomas Heller.
|
"""
|
import sys
|
import os
|
import imp
|
import warnings
|
import pkgutil
|
|
from . import modulegraph
|
from .modulegraph import Alias, Script, Extension
|
from .util import imp_find_module
|
|
__all__ = [
|
'find_modules', 'parse_mf_results'
|
]
|
|
_PLATFORM_MODULES = {'posix', 'nt', 'os2', 'mac', 'ce', 'riscos'}
|
|
|
def get_implies():
|
result = {
|
# imports done from builtin modules in C code
|
# (untrackable by modulegraph)
|
"_curses": ["curses"],
|
"posix": ["resource"],
|
"gc": ["time"],
|
"time": ["_strptime"],
|
"datetime": ["time"],
|
"MacOS": ["macresource"],
|
"cPickle": ["copy_reg", "cStringIO"],
|
"parser": ["copy_reg"],
|
"codecs": ["encodings"],
|
"cStringIO": ["copy_reg"],
|
"_sre": ["copy", "string", "sre"],
|
"zipimport": ["zlib"],
|
|
# Python 3.2:
|
"_datetime": ["time", "_strptime"],
|
"_json": ["json.decoder"],
|
"_pickle": ["codecs", "copyreg", "_compat_pickle"],
|
"_posixsubprocess": ["gc"],
|
"_ssl": ["socket"],
|
|
# Python 3.3:
|
"_elementtree": ["copy", "xml.etree.ElementPath"],
|
|
# mactoolboxglue can do a bunch more of these
|
# that are far harder to predict, these should be tracked
|
# manually for now.
|
|
# this isn't C, but it uses __import__
|
"anydbm": ["dbhash", "gdbm", "dbm", "dumbdbm", "whichdb"],
|
# package aliases
|
"wxPython.wx": Alias('wx'),
|
|
}
|
|
if sys.version_info[0] == 3:
|
result["_sre"] = ["copy", "re"]
|
result["parser"] = ["copyreg"]
|
|
# _frozen_importlib is part of the interpreter itself
|
result["_frozen_importlib"] = None
|
|
if sys.version_info[0] == 2 and sys.version_info[1] >= 5:
|
result.update({
|
"email.base64MIME": Alias("email.base64mime"),
|
"email.Charset": Alias("email.charset"),
|
"email.Encoders": Alias("email.encoders"),
|
"email.Errors": Alias("email.errors"),
|
"email.Feedparser": Alias("email.feedParser"),
|
"email.Generator": Alias("email.generator"),
|
"email.Header": Alias("email.header"),
|
"email.Iterators": Alias("email.iterators"),
|
"email.Message": Alias("email.message"),
|
"email.Parser": Alias("email.parser"),
|
"email.quopriMIME": Alias("email.quoprimime"),
|
"email.Utils": Alias("email.utils"),
|
"email.MIMEAudio": Alias("email.mime.audio"),
|
"email.MIMEBase": Alias("email.mime.base"),
|
"email.MIMEImage": Alias("email.mime.image"),
|
"email.MIMEMessage": Alias("email.mime.message"),
|
"email.MIMEMultipart": Alias("email.mime.multipart"),
|
"email.MIMENonMultipart": Alias("email.mime.nonmultipart"),
|
"email.MIMEText": Alias("email.mime.text"),
|
})
|
|
if sys.version_info[:2] >= (2, 5):
|
result["_elementtree"] = ["pyexpat"]
|
|
import xml.etree
|
for _, module_name, is_package in pkgutil.iter_modules(xml.etree.__path__):
|
if not is_package:
|
result["_elementtree"].append("xml.etree.%s" % (module_name,))
|
|
if sys.version_info[:2] >= (2, 6):
|
result['future_builtins'] = ['itertools']
|
|
# os.path is an alias for a platform specific submodule,
|
# ensure that the graph shows this.
|
result['os.path'] = Alias(os.path.__name__)
|
|
return result
|
|
|
def parse_mf_results(mf):
|
"""
|
Return two lists: the first one contains the python files in the graph,
|
the second the C extensions.
|
|
:param mf: a :class:`modulegraph.modulegraph.ModuleGraph` instance
|
"""
|
# Retrieve modules from modulegraph
|
py_files = []
|
extensions = []
|
|
for item in mf.iter_graph():
|
# There may be __main__ modules (from mf.run_script), but
|
# we don't need it in the zipfile we build.
|
if item.identifier == "__main__":
|
continue
|
src = item.filename
|
if src and src != '-':
|
if isinstance(item, Script):
|
# Scripts are python files
|
py_files.append(item)
|
|
elif isinstance(item, Extension):
|
extensions.append(item)
|
|
else:
|
py_files.append(item)
|
|
# sort on the file names, the output is nicer to read
|
py_files.sort(key=lambda v: v.filename)
|
extensions.sort(key=lambda v: v.filename)
|
return py_files, extensions
|
|
|
def plat_prepare(includes, packages, excludes):
|
# used by Python itself
|
includes.update(["warnings", "unicodedata", "weakref"])
|
|
if not sys.platform.startswith('irix'):
|
excludes.update([
|
'AL',
|
'sgi',
|
'vms_lib',
|
])
|
|
if sys.platform not in ('mac', 'darwin'):
|
# XXX - this doesn't look nearly complete
|
excludes.update([
|
'Audio_mac',
|
'Carbon.File',
|
'Carbon.Folder',
|
'Carbon.Folders',
|
'EasyDialogs',
|
'MacOS',
|
'macfs',
|
'macostools',
|
'_scproxy',
|
])
|
|
if not sys.platform == 'win32':
|
# only win32
|
excludes.update([
|
'nturl2path',
|
'win32api',
|
'win32con',
|
'win32ctypes',
|
'win32event',
|
'win32evtlogutil',
|
'win32evtlog',
|
'win32file',
|
'win32gui',
|
'win32pipe',
|
'win32process',
|
'win32security',
|
'pywintypes',
|
'winsound',
|
'win32',
|
'_winreg',
|
'_winapi',
|
'msvcrt',
|
'winreg',
|
'_subprocess',
|
])
|
|
if not sys.platform == 'riscos':
|
excludes.update([
|
'riscosenviron',
|
'rourl2path',
|
])
|
|
if not sys.platform == 'dos' or sys.platform.startswith('ms-dos'):
|
excludes.update([
|
'dos',
|
])
|
|
if not sys.platform == 'os2emx':
|
excludes.update([
|
'_emx_link',
|
])
|
|
excludes.update(_PLATFORM_MODULES - set(sys.builtin_module_names))
|
|
# Carbon.Res depends on this, but the module hasn't been present
|
# for a while...
|
excludes.add('OverrideFrom23')
|
excludes.add('OverrideFrom23._Res')
|
|
# import trickery in the dummy_threading module (stdlib)
|
excludes.add('_dummy_threading')
|
|
try:
|
imp_find_module('poll')
|
except ImportError:
|
excludes.update([
|
'poll',
|
])
|
|
|
def find_needed_modules(
|
mf=None, scripts=(), includes=(), packages=(), warn=warnings.warn):
|
if mf is None:
|
mf = modulegraph.ModuleGraph()
|
# feed Modulefinder with everything, and return it.
|
|
for path in scripts:
|
mf.add_script(path)
|
|
for mod in includes:
|
try:
|
if mod[-2:] == '.*':
|
mf.import_hook(mod[:-2], None, ['*'])
|
else:
|
mf.import_hook(mod)
|
except ImportError:
|
warn("No module named %s" % (mod,))
|
|
for f in packages:
|
# If modulegraph has seen a reference to the package, then
|
# we prefer to believe that (imp_find_module doesn't seem to locate
|
# sub-packages)
|
m = mf.find_node(f)
|
if m is not None:
|
path = m.packagepath[0]
|
else:
|
# Find path of package
|
# TODO: use imp_find_module_or_importer
|
try:
|
path = imp_find_module(f, mf.path)[1]
|
except ImportError:
|
warn("No package named %s" % f)
|
continue
|
|
# walk the path to find subdirs containing __init__.py files
|
# scan the results (directory of __init__.py files)
|
# first trim the path (of the head package),
|
# then convert directory name in package name,
|
# finally push into modulegraph.
|
# FIXME:
|
# 1) Needs to be adjusted for namespace packages in python 3.3
|
# 2) Code is fairly dodgy and needs better tests
|
for (dirpath, dirnames, filenames) in os.walk(path):
|
if '__init__.py' in filenames and dirpath.startswith(path):
|
package = f + '.' + dirpath[len(path)+1:].replace(os.sep, '.')
|
if package.endswith('.'):
|
package = package[:-1]
|
m = mf.import_hook(package, None, ["*"])
|
else:
|
# Exclude subtrees that aren't packages
|
dirnames[:] = []
|
|
return mf
|
|
#
|
# resource constants
|
#
|
|
|
PY_SUFFIXES = ['.py', '.pyw', '.pyo', '.pyc']
|
C_SUFFIXES = [
|
_triple[0] for _triple in imp.get_suffixes()
|
if _triple[2] == imp.C_EXTENSION
|
]
|
|
|
#
|
# side-effects
|
#
|
|
|
def _replacePackages():
|
REPLACEPACKAGES = {
|
'_xmlplus': 'xml',
|
}
|
for k, v in REPLACEPACKAGES.items():
|
modulegraph.replacePackage(k, v)
|
|
|
_replacePackages()
|
|
|
def find_modules(
|
scripts=(), includes=(), packages=(), excludes=(), path=None, debug=0):
|
"""
|
High-level interface, takes iterables for:
|
scripts, includes, packages, excludes
|
|
And returns a :class:`modulegraph.modulegraph.ModuleGraph` instance,
|
python_files, and extensions
|
|
python_files is a list of pure python dependencies as modulegraph.Module
|
objects, extensions is a list of platform-specific C extension dependencies
|
as modulegraph.Module objects
|
"""
|
scripts = set(scripts)
|
includes = set(includes)
|
packages = set(packages)
|
excludes = set(excludes)
|
plat_prepare(includes, packages, excludes)
|
mf = modulegraph.ModuleGraph(
|
path=path,
|
excludes=(excludes - includes),
|
implies=get_implies(),
|
debug=debug,
|
)
|
find_needed_modules(mf, scripts, includes, packages)
|
return mf
|
