Skip to content

Commit

Permalink
Speed up unpack_file_url
Browse files Browse the repository at this point in the history
E.g.: `pip install /path/to/dir`

by building an sdist and then unpacking it instead of doing
`shutil.copytree`. `shutil.copytree` may copy files that aren't included
in the sdist, which means that:

1. If those files are large, a `pip install` could take much longer than
it should.

2. Folks that are using `python setup.py install` to test their package
are not fully testing it, because it may copy over files that wouldn't
be there if they built and sdist and installed that.

So this method building an sdist is both more accurate and faster.

Fixes #2195
  • Loading branch information
msabramo committed Mar 13, 2015
1 parent 5a32a4d commit 4e6d935
Showing 1 changed file with 42 additions and 1 deletion.
43 changes: 42 additions & 1 deletion pip/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@
from pip.exceptions import InstallationError, HashMismatch
from pip.models import PyPI
from pip.utils import (splitext, rmtree, format_size, display_path,
backup_dir, ask_path_exists, unpack_file)
backup_dir, ask_path_exists, unpack_file,
call_subprocess)
from pip.utils.filesystem import check_path_owner
from pip.utils.logging import indent_log
from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner
from pip.locations import write_delete_marker_file
from pip.vcs import vcs
Expand Down Expand Up @@ -725,6 +727,45 @@ def unpack_file_url(link, location, download_dir=None):
_copy_file(from_path, download_dir, content_type, link)


def _copy_dist_from_dir(link_path, location):
"""Copy distribution files in `link_path` to `location`.
Invoked when user requests to install a local directory. E.g.:
pip install .
pip install ~/dev/git-repos/python-prompt-toolkit
"""

# Note: This is currently VERY SLOW if you have a lot of data in the
# directory, because it copies everything with `shutil.copytree`.
# What it should really do is build an sdist and install that.
# See https://github.com/pypa/pip/issues/2195

if os.path.isdir(location):
rmtree(location)

# build an sdist
setup_py = 'setup.py'
sdist_args = [sys.executable]
sdist_args.append('-c')
sdist_args.append(
"import setuptools, tokenize;__file__=%r;"
"exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py)
sdist_args.append('sdist')
sdist_args += ['--dist-dir', location]
logger.info('Running setup.py sdist for %s', link_path)

with indent_log():
call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

# unpack sdist into `location`
sdist = os.path.join(location, os.listdir(location)[0])
logger.info('Unpacking sdist %s into %s', sdist, location)
unpack_file(sdist, location, content_type=None, link=None)


class PipXmlrpcTransport(xmlrpc_client.Transport):
"""Provide a `xmlrpclib.Transport` implementation via a `PipSession`
object.
Expand Down

1 comment on commit 4e6d935

@erikrose
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like you forgot to commit the bit that actually calls your new function. Do you still have that lying around anywhere?

Please sign in to comment.