import os import shutil import stat import sys from tempfile import mkdtemp from distutils2.database import get_distribution, get_distributions from distutils2.depgraph import generate_graph from distutils2.errors import InstallationConflict, InstallationException from distutils2.index import Client as IndexClient from distutils2.version import get_version_predicate from distutils2.pypi.errors import ProjectNotFound, ReleaseNotFound from distutils2 import logger def run_command(name, cwd, executable=None, **kwargs): if executable is None: executable = sys.executable # we can run a variety of commands, select the one we want here commands = { 'distutils_install': 'install --record={record_file}', 'setuptools_install': 'install --record={record_file}'\ ' --single-version-externally-managed', 'egg_info': 'egg_info -e {output}', } old_dir = os.getcwd() try: os.chdir(cwd) cmd = executable + "setup.py " + commands.get(name) return os.system(cmd.format(**kwargs)) == 0 finally: os.chdir(old_dir) class DependencyHandler(object): def __init__(self, index, distributions): self.mover = DirectoryMover() self.index = index self.distributions = distributions def project_exists_locally(self, predicate, installed=None): if installed is None: installed = list(get_distributions(use_egg_info=True)) # check that the project isn't already installed if predicate.name.lower() in [p.name.lower() for p in installed]: index = installed.index(predicate.name.lower()) installed_project = installed[index] logger.info('Found %r %s' % (installed_project.name.lower(), installed_project.version)) if predicate.match(installed_project.version): return True return False def compute_dependencies(self, requirements, paths, installed=None): """Get the missing dependencies for the specified requirements. :param requirements: The requirements, as specified in PEP 375 :param paths: A list of paths to look for the installed distributions :param installed: a list of distributions installed (optional. If None is passed, will build the list from the given paths) Returns a tuple containing lists for (missing, spare, conflict) distributions. """ empty_set = [], [], [] if not installed: logger.debug('Reading installed distributions') installed = list(get_distributions(use_egg_info=True)) predicate = get_version_predicate(requirements) if self.project_exists(predicate): return empty_set try: release = self.index.get_release(requirements) except (ReleaseNotFound, ProjectNotFound): raise InstallationException('Release not found: %r' % requirements) if release is None: logger.info('Could not find a matching project') return empty_set self.get_dist_dependencies(release.dist) # build the dependency graph with local and required dependencies dists = list(installed) dists.append(release) depgraph = generate_graph(dists) # Get what the missing deps are missing_dists = depgraph.missing[release] if missing_dists: logger.info("Missing dependencies found, retrieving metadata") # we have missing deps for missing in missing_dists: self.compute_dependencies(missing) # how to update the list of distributions to install/remove? # Fill in the info existing = [d for d in installed if d.name == release.name] # Finish this if existing: info['remove'].append(existing[0]) info['conflict'].extend(depgraph.reverse_list[existing[0]]) info['install'].append(release) return info def get_dist_dependencies(self, dist): """Get a list of dependencies for the given distribution. This method takes care of all the packaging tools, for instance it is able to deal with setuptools dependencies as well as PEP 345 dependencies. """ # first, try to get the dependencies from the metadata. If the metadata # is not of the right version, then fallback on setuptools and try to # get the dependencies using egginfo metadata = self.index.get_metadata(dist.name, dist.version) dependencies = [] if metadata['METADATA-VERSION'] >= 1.2: dependencies = metadata['requires-dist'] else: dependencies = self.get_setuptools_dependencies(dist) return dependencies def get_setuptools_dependencies(self, dist): """return a list of dependencies given a setuptools release""" logger.info('Fetching %s dependencies from setuptools egg_info' % dist.name) dist_location = self.distributions.unpack(dist) dependencies = [] dirname = mkdtemp(prefix='%s-egginfo' % dist.name) try: if run_command('egginfo', dist_location, output=dirname): # get_distribution is finding the right egg-info directory and # read/convert the metadata it finds into PEP 345 compatible # metadata (and that's how we can get the requires_dist field) dist = get_distribution(name=dist.name, use_egg_info=True, paths=(dirname,), use_cache=False) dependencies = dist.metadata['requires_dist'] finally: shutil.rmtree(dirname) return dependencies class DirectoryMover(object): def __init__(self, temp=None, dry_run=False, count_files=False): if temp == None: temp = mkdtemp(prefix="directory-mover") self.temp = temp self.transitions = [] self.dry_run = dry_run self.count_files = count_files if count_files: self.counter = {'deleted_files': 0, 'deleted_dirs': 0, 'moved_files': 0, 'moved_dirs': 0} def _incr_counter(self, counter): if self.count_files: self.counter[counter] += 1 def commit(self): if not self.dry_run: for _, current, destination in self.transitions: # move current to destination if destination is "/dev/null": if os.path.isfile(current): os.remove(current) self.incr_counter('deleted_files') elif os.path.isdir(current): if not os.path.exists(current): continue # that's just possible. Ignore this. for _, _, files in os.walk(current): self._incr_counter('deleted_files') # empty dirs with only empty dirs if os.stat(current).st_mode & stat.S_IWUSR: shutil.rmtree(current) self._incr_counter('deleted_dirs') else: # it's a move if os.path.isdir(current): self._incr_counter('moved_dirs') shutil.move(current, destination) def rollback(self): if not self.dry_run: for origin, current, _ in self.transitions: # move current to origin pass def delete(self, source): self.move(source, "/dev/null") def move(self, source, destination): if not self.dry_run: # do the move and add it to self.transitions. if os.path.isfile(source): self.transition.append((source, self.temp, destination)) elif os.path.isdir(source): os.remove(source) class DistributionCache(object): """Aimed to only contain downloaded and extracted distribution archives from the indexes, so we avoid downloading them multiple times. """ def __init__(self, location=None): if location is None: location = mkdtemp(prefix="distribution-cache") self.location = location self.distributions = {} def unpack(self, dist): """Return the location of the unpacked distribution""" if (dist.name, dist.version) not in self.distributions: location = dist.download(path=self.location) # try to find the path it unpacked to (the root), which should by # of the form name-version name = "%s-%s" % (dist.name, dist.version) potential_location = os.path.join(location, name) if os.path.isdir(potential_location): location = potential_location self.distribution[(dist.name, dist.version)] = location return self.distributions[(dist.name, dist.version)] class Installer(object): def __init__(self, index=None, dependency_handler=None, dry_run=False): if index is None: index = IndexClient() if dependency_handler is None: dependency_handler = DependencyHandler() # share a chache between the dependency handler and the installer self._cache = DistributionCache() self._dep_handler = DependencyHandler(self._cache) self._index = index self._mover = DirectoryMover(dry_run=dry_run) def install(self, requirements, destination=None, pythonpath=None): """Resolve the given requirements and install the appropriate distributions in :param destination:. :param requirements: PEP 375 requirement (for instance "foobar == 2.0") :param destination: top level directory where the distributions should be installed :param pythonpath: the list of paths to look distributions into. (defaults to os.path) """ if pythonpath is None: pythonpath = os.path dist = self.index.get_distribution(requirements, prefer_final=True) missing, spare, conflict =\ self._dep_handler.compute_dependencies(dist, pythonpath) self.proceed_installation(missing, spare, conflict) def install_dist(self, dist, destination): # detect the installation method # then install the distribution using this method. # use the distribution cache to check that the distributions hadn't # been extracted yet during the dependency resolving. pass def remove_dist(self, dist): # find the distribution and remove it. pass def proceed_installation(self, destination, missing, spare, conflict): """Proceed to the actual installation. Install the missing distribution, remove the spare ones and if there is a conflict, don't do anything. All the operations are done in such a way that it's not possible to let the file system in an inconsistant state (if there is a problem during the installation, a rollback is done). :param destination: where the distribution should be installed. :param missing: the list of distributions to install :param spare: the list of distributions to remove :param conflict: the list of distributions that are conflicting with the installation scenario requested. Passing conflict so that installers that want to deal with conflicts can do so. The default behaviour in distutils2 is to not take care of this. """ if conflict: raise InstallationConflict(conflict) try: for dist in missing: self.install_dist(dist, destination) for dist in spare: self.remove_dist(dist) except: self._mover.rollback() else: self._mover.commit() # and here are some high level apis for the 80% def install(requirements, destination=None, pythonpath=None, index_url=None, index=None, dry_run=False, dependency_handler=None): if index is None and index_url is not None: index = IndexClient(index_url=index_url) installer = Installer(index=index, dependency_handler=dependency_handler, dry_run=dry_run) installer.install(requirements, destination, pythonpath) def remove(project_name, pythonpath=None, dry_run=False): """Removes a single project. Returns True on success """ dist = get_distribution(project_name, use_egg_info=True, paths=pythonpath) files = dist.list_installed_files(local=True) mover = DirectoryMover(mkdtemp(prefix=project_name + '-uninstall')) try: for f in files: mover.remove(f) except Exception: mover.rollback() else: mover.commit()