# Copyright (c) 2010 ActiveState Software Inc. All rights reserved. """ pypm.client.fs ~~~~~~~~~~~~~~ File system related functionality, which includes: - Downloading packages from remote location to local cache - Extracting packages into the appropriate directory structure (user site layout) """ import os import logging import six.moves from applib import sh from applib.misc import xjoin from pypm.common import net, licensing from pypm.common.util import wrapped, concise_path from pypm.common.package import PackageFile from pypm.common.repository import RepoPackage from pypm.common.supported import PLATNAME from pypm.client.base import application from pypm.client import error LOG = logging.getLogger(__name__) # TODO: we are not actually utilizing the download "cache" yet. DOWNLOAD_CACHE = xjoin(application.locations.user_cache_dir, 'download-cache') class Downloader: def __init__(self, pypmenv): self.pypmenv = pypmenv def download_packages(self, packages): """Download the given list of packages We first download the BE packages first in order to catch license related errors early. This does not, however, prevent late errors occuring due to missing/expired license. Return a dictionary of location to downloaded packages. """ # reorder packages to keep BE at top paid, free = [], [] for pkg in packages: (paid if pkg.requires_be_license else free).append(pkg) packages = paid + free locations = {} for pkg in packages: locations[pkg] = self.download_package(pkg) return locations def download_package(self, package): assert type(package) is RepoPackage sh.mkdirs(DOWNLOAD_CACHE) auth = licensing.get_be_license_auth() send_license = package.requires_be_license license_installed = auth is not None # A license is required for this package, but no license is installed if not license_installed and send_license: msg = '\n'.join([ wrapped('If you have purchased ActivePython Business Edition, ' 'please login to your account at:'), ' https://account.activestate.com/', wrapped('and download and run the license installer for your ' 'platform.'), '', wrapped('Please visit <%s> to learn more about the ' 'ActivePython Business Edition offering.' % \ licensing.BE_HOME_PAGE)]) raise error.PackageAccessError( package, 'requires Business Edition subscription', msg) try: # At this point, the user is already known to have a BE license file_location, _ = net.download_file( package.download_url, DOWNLOAD_CACHE, dict( auth=auth, use_cache=True, # XXX: this introduces network delay # (If-None-Match) despite having the file # in download cache # TODO: abstract client.store...autosync save_properties=True, start_info='{{status}}: [{0}] {1} {2}'.format( six.moves.urlparse(package.download_url).netloc, package.name, package.printable_version)), interactive=self.pypmenv.options['interactive']) except six.moves.HTTPError as e: reason = str(e) LOG.debug("HTTPError while accessing URL: %s -- reason: %s", package.download_url, reason) if send_license and e.code in (401, 402, 403): msg = wrapped( 'Your ActivePython Business Edition subscription seems to ' 'have expired. Please visit your account at ' 'https://account.activestate.com/ to renew your subscription.' ) else: msg = '' raise error.PackageAccessError(package, reason, msg) return file_location class Extractor: """Extracts the binary package to Python directory This is not as simple as it may sound. While we build all packages in a simple Python directory structure (including virtualenv) and store that very same directory structure in the created binary packages, the *user* may be using a different directory structure. PEP 370, for example, uses $APPDATA/Python/Python26 as LIB directory on Windows; ~/Library/Python/lib/python/ on OSX/2.7. But as far as the binary package file is concerned, when extracted - as it is - over $APPDATA/Python, it implicitly expects the LIB directory to be $APPDATA/Python/Lib. Therefore we 'rewrite' the paths in tarball (.pypm/data.tar.gz) to the corresponding install scheme path[1] in local ``pyenv``. - [1] See ``pypm.common.python...get_install_scheme_path`` function """ def __init__(self, pypmenv): self.pypmenv = pypmenv def extract_package(self, pkg_filename, name): bpkgfile = PackageFile(pkg_filename) pyenv = self.pypmenv.pyenv return self._extract_to_install_scheme(bpkgfile, name) def _pyenv_scheme_path(self, path): pyenv = self.pypmenv.pyenv fullpath = pyenv.get_install_scheme_path(path) assert fullpath.startswith(pyenv.base_dir), \ "'%s' is not based on '%s' (%s)" % ( fullpath, pyenv.base_dir, pyenv.root_dir) p = os.path.relpath(fullpath, pyenv.base_dir) if PLATNAME.startswith('win'): p = p.replace('\\', '/') return p def _extract_to_install_scheme(self, bpkgfile, name): pyenv = self.pypmenv.pyenv # Install scheme used by the build environment (i.e., pyenv used by # pypm-builder on our build machines). as_build_scheme = { 'win': { 'purelib': 'lib/site-packages', 'stdlib': 'lib', 'scripts': 'scripts', }, 'unix': { 'purelib': 'lib/python{0}/site-packages'.format(pyenv.pyver), 'stdlib': 'lib/python{0}'.format(pyenv.pyver), 'scripts': 'bin', }, } plat = PLATNAME.startswith('win') and 'win' or 'unix' # Scheme used by pyenv pyenv_scheme = { 'purelib': self._pyenv_scheme_path('purelib'), 'stdlib': self._pyenv_scheme_path('stdlib'), 'scripts': self._pyenv_scheme_path('scripts'), } files_to_overwrite = [] force_overwrite = self.pypmenv.options['force'] # Hack #1: Don't check for distribute and pip, as virtualenvs usually # already have a copy of them installed. if name in ('distribute', 'setuptools', 'pip'): force_overwrite = True with bpkgfile.extract_over2(pyenv.base_dir) as tf: for tinfo in tf.getmembers(): # Replace AS build virtualenv scheme with the user's scheme # Eg: lib/site-packages/XYZ -> %APPDATA%/Python/Python26/XYZ for name, prefix in as_build_scheme[plat].items(): if tinfo.name.lower().startswith(prefix): old = tinfo.name new = pyenv_scheme[name] + old[len(prefix):] if new != old: LOG.debug('fs:extract: transforming "%s" to "%s"', old, new) tinfo.name = new # Check for overwrites if os.path.lexists(tinfo.name) and not os.path.isdir(tinfo.name): # Hack #2: allow overwriting of *.pth files (setuptools # hackishness) eg: [...]/site-packages/setuptools.pth if not tinfo.name.endswith('.pth'): files_to_overwrite.append(tinfo.name) if files_to_overwrite: LOG.debug( 'install requires overwriting of %d files:\n%s', len(files_to_overwrite), '\n'.join([os.path.join(pyenv.base_dir, f) for f in files_to_overwrite])) if force_overwrite: LOG.warn('overwriting %d files' % len(files_to_overwrite)) else: errmsg = ['cannot overwrite "%s"' % concise_path(os.path.join( pyenv.base_dir, files_to_overwrite[0]))] if len(files_to_overwrite) > 1: errmsg.append(' (and %d other files)' % (len(files_to_overwrite)-1,)) errmsg.append('; run pypm as "pypm --force ..." to overwrite anyway') if len(files_to_overwrite) > 1: errmsg.append('; run "pypm log" to see the full list of files to be overwritten') raise IOError(wrapped(''.join(errmsg))) return tf.getnames() def undo_extract(self, files_list): """Undo whatever self.extract_package did""" # sort in descending order so that children of a directory # get removed before the directory itself files_list.sort() files_list.reverse() for path in files_list: path = self.pypmenv.pyenv.get_abspath(path) if not os.path.lexists(path): LOG.warn('no longer exists: %s', path) else: if os.path.isfile(path) or os.path.islink(path): sh.rm(path) # remove the corresponding .pyc and .pyo files if path.endswith('.py'): sh.rm(path+'c') sh.rm(path+'o') elif os.path.isdir(path): if len(os.listdir(path)) > 0: # cannot delete directory with files added # after the installation LOG.debug( 'non-empty directory: %s - hence skipping', path) else: # directory `path` is empty sh.rm(path) else: raise TypeError( "don't know what to do with this type of file: " + path)