From efd7094a361effdf5c86d72125b4bceed64d6808 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Mon, 2 Jan 2023 19:11:55 +0100 Subject: [PATCH] Deprecate `-c` and `-m` command line options and use `ROSDEP_CACHE_PATH` instead. Signed-off-by: Martin Pecka --- src/rosdep2/main.py | 29 ++++--- ...06d8ea9d8eec9db440c6e9937664e0159b5.pickle | 4 + .../0a12d6e7b0d47be9b76e7726720e4cb79528cbaa | 0 .../82cbc7008b5117bcc6ce794832659e4f7763d2db | 0 .../f6f4ef95664e373cd4754501337fa217f5b55d91 | 0 test/{sources_cache => sources.cache}/index | 0 test/sources.list.d.good/20-default.list | 4 + test/test_rosdep_catkin_support.py | 32 ++++++++ test/test_rosdep_installers.py | 2 +- test/test_rosdep_lookup.py | 2 +- test/test_rosdep_main.py | 27 ++++--- test/test_rosdep_sources_list.py | 80 +++++++++++++++++++ 12 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle rename test/{sources_cache => sources.cache}/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa (100%) rename test/{sources_cache => sources.cache}/82cbc7008b5117bcc6ce794832659e4f7763d2db (100%) rename test/{sources_cache => sources.cache}/f6f4ef95664e373cd4754501337fa217f5b55d91 (100%) rename test/{sources_cache => sources.cache}/index (100%) create mode 100644 test/sources.list.d.good/20-default.list diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 7deaa34ca..a20f54afb 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -53,7 +53,7 @@ from urllib2 import URLError import warnings -from optparse import OptionParser +from optparse import OptionParser, SUPPRESS_HELP import rospkg @@ -63,7 +63,7 @@ from .installers import normalize_uninstalled_to_list from .installers import RosdepInstaller from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages -from .meta import MetaDatabase, get_meta_cache_dir +from .meta import MetaDatabase from .rospkg_loader import DEFAULT_VIEW_KEY from .sources_list import update_sources_list, get_sources_cache_dir,\ download_default_sources_list, SourcesListLoader, CACHE_INDEX,\ @@ -122,6 +122,16 @@ class UsageError(Exception): Recursively change the permissions of the user's ros home directory. May require sudo. Can be useful to fix permissions after calling "rosdep update" with sudo accidentally. + +Environment variables: + +ROSDEP_SOURCE_PATH + Overrides path to the sources list directory (by default /etc/ros/rosdep/sources.list.d). + Applies to init and update commands. + +ROSDEP_CACHE_PATH + Overrides path to the cache directory (by default $HOME/.ros/rosdep). + Applies to all commands except init. """ @@ -301,16 +311,13 @@ def setup_environment_variables(ros_distro, meta_cache_dir=None): def _rosdep_main(args): # sources cache dir is our local database. - default_sources_cache = get_sources_cache_dir() - default_meta_cache = get_meta_cache_dir() - parser = OptionParser(usage=_usage, prog='rosdep') parser.add_option('--os', dest='os_override', default=None, metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid') - parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache, - metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache)) - parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache, - metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache)) + parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=None, + metavar='SOURCES_CACHE_DIR', help=SUPPRESS_HELP) # deprecated + parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=None, + metavar='META_CACHE_DIR', help=SUPPRESS_HELP) # deprecated parser.add_option('--verbose', '-v', dest='verbose', default=False, action='store_true', help='verbose display') parser.add_option('--version', dest='print_version', default=False, @@ -657,7 +664,6 @@ def update_error_handler(data_source, exc): try: if not options.quiet: print('reading in sources list data from %s' % (sources_list_dir)) - sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() try: if os.geteuid() == 0: print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr) @@ -665,7 +671,7 @@ def update_error_handler(data_source, exc): except AttributeError: # nothing we wanna do under Windows pass - update_sources_list(sources_cache_dir=sources_cache_dir, + update_sources_list(sources_cache_dir=options.sources_cache_dir, success_handler=update_success_handler, error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, @@ -673,6 +679,7 @@ def update_error_handler(data_source, exc): quiet=options.quiet, meta_cache_dir=options.meta_cache_dir) if not options.quiet: + sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() print('updated cache in %s' % (sources_cache_dir)) except InvalidData as e: print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr) diff --git a/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle b/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle new file mode 100644 index 000000000..3c37def2d --- /dev/null +++ b/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle @@ -0,0 +1,4 @@ +crosdep2.meta +CacheWrapper +q)q}q(Urosdep_versionqU0.22.1qU_CacheWrapper__dataq}q(UfoxyqKUmelodicq KUhumbleq +KUrollingq KUnoeticq KuU category_nameq UROS_PYTHON_VERSIONqub. \ No newline at end of file diff --git a/test/sources_cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa b/test/sources.cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa similarity index 100% rename from test/sources_cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa rename to test/sources.cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa diff --git a/test/sources_cache/82cbc7008b5117bcc6ce794832659e4f7763d2db b/test/sources.cache/82cbc7008b5117bcc6ce794832659e4f7763d2db similarity index 100% rename from test/sources_cache/82cbc7008b5117bcc6ce794832659e4f7763d2db rename to test/sources.cache/82cbc7008b5117bcc6ce794832659e4f7763d2db diff --git a/test/sources_cache/f6f4ef95664e373cd4754501337fa217f5b55d91 b/test/sources.cache/f6f4ef95664e373cd4754501337fa217f5b55d91 similarity index 100% rename from test/sources_cache/f6f4ef95664e373cd4754501337fa217f5b55d91 rename to test/sources.cache/f6f4ef95664e373cd4754501337fa217f5b55d91 diff --git a/test/sources_cache/index b/test/sources.cache/index similarity index 100% rename from test/sources_cache/index rename to test/sources.cache/index diff --git a/test/sources.list.d.good/20-default.list b/test/sources.list.d.good/20-default.list new file mode 100644 index 000000000..1ac6041d1 --- /dev/null +++ b/test/sources.list.d.good/20-default.list @@ -0,0 +1,4 @@ +#autogenerated by rosdep, do not edit. use 'rosdep update' instead +yaml https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml +yaml https://github.com/ros/rosdistro/raw/master/rosdep/python.yaml +gbpdistro https://github.com/ros/rosdistro/raw/master/releases/fuerte.yaml fuerte diff --git a/test/test_rosdep_catkin_support.py b/test/test_rosdep_catkin_support.py index d117c4445..3c9ec4918 100644 --- a/test/test_rosdep_catkin_support.py +++ b/test/test_rosdep_catkin_support.py @@ -1,12 +1,35 @@ from rosdep2.catkin_support import get_installer, get_catkin_view, ValidationFailed, resolve_for_os from rosdep2.platforms.debian import APT_INSTALLER +from rosdep2.cache_tools import CACHE_PATH_ENV +from rosdep2.sources_list import SOURCE_PATH_ENV import pytest +import os +from tempfile import mkdtemp + + +def get_test_dir(): + return os.path.abspath(os.path.dirname(__file__)) + + +def get_cache_dir(): + # get_catkin_view calls update(), so we need a writable location + return mkdtemp() + + +def get_source_list_dir(): + p = os.path.join(get_test_dir(), "sources.list.d.good") + assert os.path.isdir(p) + return p @pytest.mark.online def test_workflow(): + old_cpe = os.getenv(CACHE_PATH_ENV, None) + old_spe = os.getenv(SOURCE_PATH_ENV, None) try: + os.environ[CACHE_PATH_ENV] = get_cache_dir() + os.environ[SOURCE_PATH_ENV] = get_source_list_dir() installer = get_installer(APT_INSTALLER) view = get_catkin_view('fuerte', 'ubuntu', 'lucid') resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'lucid') @@ -16,3 +39,12 @@ def test_workflow(): except ValidationFailed: # tests fail on the server because 'rosdep init' has not been run pass + finally: + if old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = old_cpe + if old_spe is None: + del os.environ[SOURCE_PATH_ENV] + else: + os.environ[SOURCE_PATH_ENV] = old_spe diff --git a/test/test_rosdep_installers.py b/test/test_rosdep_installers.py index 4b91a3f22..9eb9825e6 100644 --- a/test/test_rosdep_installers.py +++ b/test/test_rosdep_installers.py @@ -47,7 +47,7 @@ def get_test_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = os.path.join(get_test_dir(), 'sources.cache') assert os.path.isdir(p) return p diff --git a/test/test_rosdep_lookup.py b/test/test_rosdep_lookup.py index 176abd0d0..6179ea74e 100644 --- a/test/test_rosdep_lookup.py +++ b/test/test_rosdep_lookup.py @@ -45,7 +45,7 @@ def get_test_tree_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = os.path.join(get_test_dir(), 'sources.cache') assert os.path.isdir(p) return p diff --git a/test/test_rosdep_main.py b/test/test_rosdep_main.py index c94143c24..532dee5d2 100644 --- a/test/test_rosdep_main.py +++ b/test/test_rosdep_main.py @@ -47,6 +47,7 @@ from rosdep2.ament_packages import AMENT_PREFIX_PATH_ENV_VAR from rosdep2.main import rosdep_main from rosdep2.main import setup_proxy_opener +from rosdep2.cache_tools import CACHE_PATH_ENV GITHUB_BASE_URL = 'https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml' @@ -66,7 +67,7 @@ def get_test_catkin_tree_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = get_test_dir() assert os.path.isdir(p) return p @@ -95,6 +96,7 @@ def setUp(self): self.old_rr = rospkg.get_ros_root() self.old_rpp = rospkg.get_ros_package_path() self.old_app = os.getenv(AMENT_PREFIX_PATH_ENV_VAR, None) + self.old_cpe = os.getenv(CACHE_PATH_ENV, None) if 'ROS_ROOT' in os.environ: del os.environ['ROS_ROOT'] os.environ['ROS_PACKAGE_PATH'] = os.path.join(get_test_tree_dir()) @@ -102,6 +104,7 @@ def setUp(self): if 'ROS_PYTHON_VERSION' not in os.environ: # avoid `test_check` failure due to warning on stderr os.environ['ROS_PYTHON_VERSION'] = sys.version[0] + os.environ[CACHE_PATH_ENV] = get_cache_dir() def tearDown(self): if self.old_rr is not None: @@ -110,10 +113,13 @@ def tearDown(self): os.environ['ROS_PACKAGE_PATH'] = self.old_rpp if self.old_app is not None: os.environ[AMENT_PREFIX_PATH_ENV_VAR] = self.old_app + if self.old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = self.old_cpe def test_bad_commands(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] for commands in [[], ['fake', 'something'], ['check'], ['install', '-a', 'rospack_fake'], ['check', 'rospack_fake', '--os', 'ubuntulucid'], ]: @@ -124,8 +130,7 @@ def test_bad_commands(self): pass def test_check(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] with fakeout() as b: try: @@ -163,8 +168,7 @@ def test_check(self): @patch('rosdep2.platforms.debian.read_stdout') @patch('rosdep2.installers.os.geteuid', return_value=1) def test_install(self, mock_geteuid, mock_read_stdout): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] catkin_tree = get_test_catkin_tree_dir() def read_stdout(cmd, capture_stderr=False): @@ -221,12 +225,11 @@ def read_stdout(cmd, capture_stderr=False): def test_where_defined(self): try: - sources_cache = get_cache_dir() expected = GITHUB_PYTHON_URL for command in (['where_defined', 'testpython'], ['where_defined', 'testpython']): with fakeout() as b: # set os to ubuntu so this test works on different platforms - rosdep_main(command + ['-c', sources_cache, '--os=ubuntu:lucid']) + rosdep_main(command + ['--os=ubuntu:lucid']) stdout, stderr = b output = stdout.getvalue().strip() assert output == expected, output @@ -235,8 +238,7 @@ def test_where_defined(self): def test_what_needs(self): try: - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] expected = ['python_dep'] with fakeout() as b: rosdep_main(['what-needs', 'testpython'] + cmd_extras) @@ -253,8 +255,7 @@ def test_what_needs(self): assert False, 'system exit occurred' def test_keys(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] try: with fakeout() as b: diff --git a/test/test_rosdep_sources_list.py b/test/test_rosdep_sources_list.py index 960bf10b0..a270b96d6 100644 --- a/test/test_rosdep_sources_list.py +++ b/test/test_rosdep_sources_list.py @@ -38,6 +38,9 @@ import pytest import rospkg.distro import rosdep2.sources_list +from rosdep2.sources_list import SOURCES_CACHE_DIR +from rosdep2.cache_tools import CACHE_PATH_ENV +from rosdep2.meta import META_CACHE_DIR GITHUB_BASE_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/base.yaml' @@ -499,6 +502,83 @@ def test_SourcesListLoader_create_default(): loader.load_view(GITHUB_URL, rosdep_db) +@pytest.mark.online +def test_SourcesListLoader_cache_dir_from_env_var(): + from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher + # create temp dir for holding sources cache + tempdir = tempfile.mkdtemp() + old_cpe = os.getenv(CACHE_PATH_ENV, None) + os.environ[CACHE_PATH_ENV] = tempdir + + try: + # pull in cache data + sources_list_dir = get_test_dir() + retval = update_sources_list(sources_list_dir=sources_list_dir, error_handler=None) + assert retval + sources_cache_dir = os.path.join(tempdir, SOURCES_CACHE_DIR) + assert os.path.exists(sources_cache_dir) + assert len(os.listdir(sources_cache_dir)) > 0 + meta_cache_dir = os.path.join(tempdir, META_CACHE_DIR) + assert os.path.exists(meta_cache_dir) + assert len(os.listdir(meta_cache_dir)) > 0 + + # now test with cached data + matcher = rosdep2.sources_list.DataSourceMatcher(['ubuntu', 'lucid']) + loader = SourcesListLoader.create_default(matcher) + assert loader.sources + assert not any([s for s in loader.sources if not matcher.matches(s)]) + + # test API + + # very simple, always raises RNF + try: + loader.get_rosdeps('foo') + except rospkg.ResourceNotFound: + pass + try: + loader.get_view_key('foo') + except rospkg.ResourceNotFound: + pass + + assert [] == loader.get_loadable_resources() + all_sources = [x.url for x in loader.sources] + assert all_sources == loader.get_loadable_views() + + # test get_source early to make sure model matches expected + try: + loader.get_source('foo') + assert False, 'should have raised' + except rospkg.ResourceNotFound: + pass + s = loader.get_source(GITHUB_URL) + assert s.url == GITHUB_URL + + # get_view_dependencies + # - loader doesn't new view name, so assume everything + assert all_sources == loader.get_view_dependencies('foo') + # - actual views don't depend on anything + assert [] == loader.get_view_dependencies(GITHUB_URL) + + # load_view + from rosdep2.model import RosdepDatabase + for verbose in [True, False]: + rosdep_db = RosdepDatabase() + loader.load_view(GITHUB_URL, rosdep_db, verbose=verbose) + assert rosdep_db.is_loaded(GITHUB_URL) + assert [] == rosdep_db.get_view_dependencies(GITHUB_URL) + entry = rosdep_db.get_view_data(GITHUB_URL) + assert 'cmake' in entry.rosdep_data + assert GITHUB_URL == entry.origin + + # - coverage, repeat loader, should noop + loader.load_view(GITHUB_URL, rosdep_db) + finally: + if old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = old_cpe + + def test_unpickle_same_results(): try: import cPickle as pickle