# Copyright (c) 2010 ActiveState Software Inc. All rights reserved. """ pypm.common.python ~~~~~~~~~~~~~~~~~~ Functions and classes dealing with the actual Python interpreters """ from __future__ import print_function import os import sys from glob import glob from abc import ABCMeta from abc import abstractmethod from applib import sh from applib.misc import xjoin from pypm.common.util import lazyproperty from pypm.common.util import memoize from pypm.common.util import concise_path from pypm.common.supported import PLATNAME WIN = sys.platform.startswith('win') class NoWriteAccess(EnvironmentError): pass class BasePythonEnvironment(object): """Represent a generic Python installation""" __metaclass__ = ABCMeta def __init__(self, root_dir): """Initialize a local environment based in current Python interpreter""" self.root_dir = os.path.abspath(root_dir) self._info = None def _collect_python_info(self): """Collect essential info by eval'ing Python expressions The need for this function is to avoid the overhead of launching the Python interpreter multiple times. We do this by eval'ing multiple expressions at once. """ if self._info is None: if self.root_dir == sys.prefix: # optimize for current python environment by directly using # the current interpreter import site import activestate pyver = '%d.%d' % sys.version_info[:2] apyver = activestate.version user_site = site.USER_SITE user_base = site.USER_BASE else: expr = [ '"%d.%d" % sys.version_info[:2]', 'activestate.version', 'site.USER_SITE', 'site.USER_BASE',] with_modules = ['sys', 'site', 'activestate'] output = self.eval(expr, with_modules).strip().splitlines() pyver, apyver, user_site, user_base = output self._info = dict( pyver = pyver, apyver = apyver, user_site = user_site, user_base = user_base, ) def ensure_write_access(self): """Ensure that this Python environment can be modified Typically, this is expected to raise ``NoWriteAccess`` for global installations like /opt/ActivePython-2.6. """ if not os.access(self.root_dir, os.W_OK): raise NoWriteAccess('no write access to "%s"' % self.root_dir) def eval(self, expr, with_modules=None, preimport_stmts=[], python_args=None): """Evaluate `expr` and return the *printed* value (utf8 string) `expr` can be a string, or list of strings. For the later, each expr is printed in separate line. Note that `expr` supports only single-quote; and double-quotes will be appropriate escaped. Before evaluating `expr` execute `preimport_stmts` and import the mentioned modules. If `python_args` is None, use the default args (-B), otherwise use the argument value as Python args. Note that the returned value will include a newline at the end - which is the default behavior of `print`. XXX: we should change this? """ statements = [] if preimport_stmts: assert isinstance(preimport_stmts, (list, tuple)) statements.extend(preimport_stmts) if with_modules: assert isinstance(with_modules, (list, tuple)) statements.extend( ['import %s' % mod for mod in with_modules] ) if python_args is None: python_args = ['-B'] # no bytecode generation assert isinstance(python_args, (tuple, list)) if not isinstance(expr, (list, tuple)): expr = [expr] for e in expr: statements.append('print({0})'.format(e)) cmd = [self.python_exe] + python_args + [ '-s', '-c', ';'.join(statements) ] return sh.run(cmd)[0].decode('utf8') def has_module(self, module, python_args=None): # remove PWD from sys.path to avoid conflict with modules in PWD preimport_stmts = ["import sys", "sys.path.remove('')"] try: self.eval('', with_modules=[module], preimport_stmts=preimport_stmts, python_args=python_args) return True except sh.RunNonZeroReturn as e: if 'ImportError' in e.stderr.decode('utf8'): return False else: raise def get_abspath(self, relpath): """Get absolute path to a file described by `relpath` `relpath` is supposed to be inside the Python installation. """ return xjoin(self.base_dir, relpath) @lazyproperty def apyver(self): """Return ActivePython version""" self._collect_python_info() return self._info['apyver'] @lazyproperty def pyver(self): self._collect_python_info() return self._info['pyver'] @lazyproperty def pyversion(self): """Short and concise string-representation of Python version >>> pyenv.pyversion '2.6.4' """ return self.eval("platform.python_version()", ['platform']).strip() @lazyproperty def pyversion_info(self): """``sys.version_info``""" return eval(self.eval("tuple(sys.version_info)", ['sys']).strip()) @abstractmethod def python_exe(self): """Return the path to the Python executable""" @abstractmethod def base_dir(self): """Return the base directory under which Python packages should be installed; DO NOT USE `self.root_dir` - which differs for user site area! """ @abstractmethod def site_packages_dir(self): """Return the path to site-packages directory""" @abstractmethod def get_install_scheme_path(self, path): """Return the distutils/sysconfig install scheme path for this Python environment This method should use `distutils.sysconfig` on 2.6/3.1 and `sysconfig` on 2.7+, to detect the install scheme. """ @property def printable_location(self): """A printable string for this Python's location A concise representation of the path to this Python's directory that is meant for displaying to the end user. """ return '"{0}" ({1})'.format( concise_path(self.base_dir), self.pyver) def __str__(self): return '{0.__class__.__name__}<{0.python_exe}>'.format(self) __repr__ = __str__ # make it appear in py.test errors class PythonEnvironment(BasePythonEnvironment): """A Python environment representing ordinary Python installations Examples: /usr /opt/ActivePython-2.6 /tmp/Python-2.6.2/buildimage # ./configure --prefix=./buildimage """ @lazyproperty def python_exe(self): # NOTE: we CANNOT use get_install_scheme_path() in `script_dirs`, as # that would lead to infinite loop (stack overflow). This is why the # method call is commented out below; and we must resort to heuristics. if WIN: executable = 'Python.exe' executable_alt = [ 'Python[23456789].[0123456789].exe', # eg: python3.1.exe 'Python[23456789].exe', # eg: python3.exe 'Python[23456789][0123456789].exe', # eg: python31.exe (old apy) ] script_dirs = [ xjoin(self.root_dir), xjoin(self.root_dir, 'Scripts'), # virtualenv xjoin(self.root_dir, 'bin'), # some, like VanL, do this # self.get_install_scheme_path('scripts') ] else: executable = 'python' executable_alt = [ 'python[23456789].[0123456789]', # eg: python3.1 'python[23456789]', # eg: python3 ] script_dirs = [ xjoin(self.root_dir, 'bin'), # self.get_install_scheme_path('scripts') ] for script_dir in script_dirs: python_exe = os.path.join(script_dir, executable) if os.path.exists(python_exe): return python_exe # In one last attempt, try finding executables with pyver in them. eg: # python31.exe or python3.1 for script_dir in script_dirs: for pat in executable_alt: for python_exe in glob(os.path.join(script_dir, pat)): return python_exe raise IOError( 'cannot find the Python executable; ' + \ 'searched in these directories:\n\t%s' % \ '\n\t'.join(script_dirs)) @lazyproperty def base_dir(self): return self.root_dir @lazyproperty def site_packages_dir(self): # There is also platlib; let's juse assume that purelib and platlib have # same values ... and that they both point to the site-packages/ # directory. return self.get_install_scheme_path('purelib') @memoize def get_install_scheme_path(self, path, usersite=False): if self.pyversion_info[:2] not in ((2,6), (3,1)): # use the `sysconfig` module that was introduced in Python 2.7 # Find the desired scheme; alas, there is no standard function for # this in the stdlib. See http://bugs.python.org/issue8772 # The following should work on default ActivePython installation. scheme = os.name if usersite: if PLATNAME.startswith('mac'): scheme = 'osx_framework_user' else: scheme += '_user' elif scheme == 'posix': scheme = 'posix_prefix' return self.eval( "sysconfig.get_paths('%s')['%s']" % (scheme, path), with_modules=['sysconfig'] ).strip() else: # backward compat: use the limited `distutils.sysconfig` module on # 2.6. Unlike 2.7's sysconfig, this does not have the convenient # `get_paths` function ... forcing us to do the template # substitution ourselves, which may be error prone. :-/ # HACK: 2.6 does not have a path named 'stdlib'; let's just use # purelib, which is usually site-packages/ and go one-level up to # get the intended stdlib. if path == 'stdlib': joinwith = '..' path = 'purelib' else: joinwith = None if PLATNAME.startswith('win'): scheme = 'nt' else: scheme = 'unix' scheme += ('_user' if usersite else ( '_prefix' if scheme == 'unix' else '')) template = self.eval( "INSTALL_SCHEMES['%s']['%s']" % (scheme, path), preimport_stmts=[ "import os", "from distutils.command.install import INSTALL_SCHEMES", ]).strip() # hardcoded template substitution (distutils API does not provide # this) template = template.replace('$base', self.base_dir) template = template.replace('$py_version_short', self.pyver) if '$user' in template: assert isinstance(self, UserLocalPythonEnvironment) template = template.replace('$usersite', self.user_site_dir) template = template.replace('$userbase', self.user_base_dir) assert '$' not in template, \ 'Unsubstituded variables left in "%s"' % template if joinwith: template = os.path.join(template, joinwith) return template # # Commonly used Python environments # class GlobalPythonEnvironment(PythonEnvironment): """Represent the environment of the global Python process By 'global', we mean current Python interpreter but surpassing any mediating virtualenv. """ def __init__(self): """Initialize a local environment based in current Python interpreter""" super(GlobalPythonEnvironment, self).__init__( root_dir = get_real_prefix()) class VirtualPythonEnvironment(PythonEnvironment): """A Python environment based on a virtualenv directory""" pass # # Support for PEP 370 # class UserLocalPythonEnvironment(PythonEnvironment): """Represent PEP 370 style like environment for current Python PEP 370 made it possible to install packages into $HOME using the --user option to 'setup.py install': ~/.local on Unix ~/Library/Python on MacOSX $APPDATA/Python on Windows """ def __init__(self): super(PythonEnvironment, self).__init__(get_real_prefix()) @lazyproperty def global_env(self): return GlobalPythonEnvironment() def ensure_write_access(self): if not os.path.exists(self.user_base_dir): # user may not have a user base directory *yet* sh.mkdirs(self.user_base_dir) # This is unlikely to happen if not os.access(self.user_base_dir, os.W_OK): raise NoWriteAccess( 'no write access to user base "%s"' % self.user_base_dir) @lazyproperty def base_dir(self): return self.user_base_dir @lazyproperty def user_base_dir(self): self._collect_python_info() return self._info['user_base'] @lazyproperty def user_site_dir(self): self._collect_python_info() return self._info['user_site'] site_packages_dir = user_site_dir def get_install_scheme_path(self, path): return super(UserLocalPythonEnvironment, self).get_install_scheme_path( path, True) def __str__(self): return '{0.__class__.__name__}<{0.python_exe},{0.user_base_dir}>'.format( self) def get_real_prefix(): """Return the real sys.prefix of current Python bypassing virtualenvs""" return getattr(sys, 'real_prefix', sys.prefix) if __name__ == '__main__': # debug for pyenv in [UserLocalPythonEnvironment(), GlobalPythonEnvironment()]: print(str(pyenv)) print('=' * 40) print('pyver = %s' % pyenv.pyver) print('python_exe = %s' % pyenv.python_exe) print('site_packages_dir = %s' % pyenv.site_packages_dir) print()