diff --git a/README.md b/README.md index a71da8cb..380a7219 100644 --- a/README.md +++ b/README.md @@ -1072,6 +1072,7 @@ Using **CONAN_CLANG_VERSIONS** env variable in Travis ci or Appveyor: - "missing": Build only missing packages. - "outdated": Build only missing or if the available package is not built with the current recipe. Useful to upload new configurations, e.j packages for a new compiler without rebuild all packages. +- **remove_outdated_packages**: Remove all outdated packages from remote after to upload a package. Default [False] - **test_folder**: Custom test folder consumed by Conan create, e.j .conan/test_package - **conanfile**: Custom conanfile consumed by Conan create. e.j. conanfile.py - **config_url**: Conan config URL be installed before to build e.j https://github.com/bincrafters/conan-config.git @@ -1218,6 +1219,7 @@ This is especially useful for CI integration. - **CONAN_CONFIG_ARGS**: Conan config arguments used when installing conan config - **CONAN_BASE_PROFILE**: Apply options, settings, etc. to this profile instead of `default`. - **CONAN_IGNORE_SKIP_CI**: Ignore `[skip ci]` in commit message. +- **CONAN_REMOVE_OUTDATED_PACKAGES**: Remove all outdated packages from remote after to upload a package. Default [False] - **CONAN_CONANFILE**: Custom conanfile consumed by Conan create. e.j. conanfile.py - **CPT_TEST_FOLDER**: Custom test_package path, e.j .conan/test_package - **CONAN_FORCE_SELINUX**: Force docker to relabel file objects on the shared volumes diff --git a/cpt/eraser.py b/cpt/eraser.py new file mode 100644 index 00000000..efb8855b --- /dev/null +++ b/cpt/eraser.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +class Eraser(object): + """ Helper to connect on remove and remove outdated packages + """ + + def __init__(self, conan_api, remote_manager, auth_manager, printer, remove): + """ Initialize Eraser instance + :param conan_api: Conan API instance + :param remote_manager: Remote manager instance + :param auth_manager: Authication manager to access the remote + :param printer: CPT output + :param remove: True if should remove outdated packages from remote. Otherwise, False. + """ + self.conan_api = conan_api + self.remote_manager = remote_manager + self.auth_manager = auth_manager + self.printer = printer + self.remove = remove + + def remove_outdated_packages(self, reference): + """ Remove outdated packages from remote + :param reference: Package reference e.g. foo/0.1.0@user/channel + """ + if not self.remote_manager or not self.remote_manager.upload_remote_name: + self.printer.print_message("Remove outdated skipped, no remote available") + return + remote_name = self.remote_manager.upload_remote_name + + if not self.auth_manager or not self.auth_manager.credentials_ready(remote_name): + self.printer.print_message("Remove outdated skipped, credentials for remote '%s' not available" % remote_name) + return + + if self.remove: + self.printer.print_message("Removing outdated packages for '%s'" % str(reference)) + self.auth_manager.login(remote_name) + self.conan_api.remove(pattern=str(reference), + force=True, + remote_name=remote_name, + outdated=True) diff --git a/cpt/packager.py b/cpt/packager.py index 7be650d9..9c456e62 100644 --- a/cpt/packager.py +++ b/cpt/packager.py @@ -22,6 +22,7 @@ from cpt.tools import get_bool_from_env from cpt.tools import split_colon_env from cpt.uploader import Uploader +from cpt.eraser import Eraser def load_cf_class(path, conan_api): @@ -54,6 +55,7 @@ def load_cf_class(path, conan_api): else: return conan_api.app.loader.load_basic(path) + class PlatformInfo(object): """Easy mockable for testing""" @staticmethod @@ -121,6 +123,7 @@ def __init__(self, username=None, channel=None, runner=None, docker_shell=None, pip_install=None, build_policy=None, + remove_outdated_packages=False, always_update_conan_in_docker=False, conan_api=None, client_cache=None, @@ -165,6 +168,11 @@ def __init__(self, username=None, channel=None, runner=None, default_username=self.username, skip_check_credentials=self.skip_check_credentials) + self._remove_outdated_packages = remove_outdated_packages or \ + get_bool_from_env("CONAN_REMOVE_OUTDATED_PACKAGES") + self.eraser = Eraser(self.conan_api, self.remotes_manager, self.auth_manager, self.printer, + self._remove_outdated_packages) + # Upload related variables self.upload_retry = upload_retry or os.getenv("CONAN_UPLOAD_RETRY", 3) @@ -583,6 +591,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): profile_abs_path = save_profile_to_tmp(profile_text) r = CreateRunner(profile_abs_path, build.reference, self.conan_api, self.uploader, + eraser=self.eraser, exclude_vcvars_precommand=self.exclude_vcvars_precommand, build_policy=self.build_policy, runner=self.runner, @@ -610,6 +619,7 @@ def run_builds(self, curpage=None, total_pages=None, base_profile_name=None): docker_image_skip_pull=self._docker_image_skip_pull, build_policy=self.build_policy, always_update_conan_in_docker=self._update_conan_in_docker, + remove_outdated_packages=self._remove_outdated_packages, upload=self._upload_enabled(), upload_retry=self.upload_retry, upload_only_recipe=self.upload_only_recipe, diff --git a/cpt/run_in_docker.py b/cpt/run_in_docker.py index 18c9b403..9e8963e9 100644 --- a/cpt/run_in_docker.py +++ b/cpt/run_in_docker.py @@ -10,6 +10,7 @@ from cpt.remotes import RemotesManager from cpt.runner import CreateRunner, unscape_env from cpt.uploader import Uploader +from cpt.eraser import Eraser def run(): @@ -29,6 +30,9 @@ def run(): test_folder = unscape_env(os.getenv("CPT_TEST_FOLDER")) reference = ConanFileReference.loads(os.getenv("CONAN_REFERENCE")) + remove_outdated_packages = unscape_env(os.getenv("CPT_REMOVE_OUTDATED_PACKAGES")) + eraser = Eraser(conan_api, remotes_manager, auth_manager, printer, remove_outdated_packages) + profile_text = unscape_env(os.getenv("CPT_PROFILE")) abs_profile_path = save_profile_to_tmp(profile_text) base_profile_text = unscape_env(os.getenv("CPT_BASE_PROFILE")) @@ -43,7 +47,7 @@ def run(): base_profile_text) upload = os.getenv("CPT_UPLOAD_ENABLED") - runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, + runner = CreateRunner(abs_profile_path, reference, conan_api, uploader, eraser=eraser, build_policy=build_policy, printer=printer, upload=upload, upload_only_recipe=upload_only_recipe, test_folder=test_folder, config_url=config_url, diff --git a/cpt/runner.py b/cpt/runner.py index 8b6bb884..c76c2490 100644 --- a/cpt/runner.py +++ b/cpt/runner.py @@ -15,7 +15,7 @@ class CreateRunner(object): - def __init__(self, profile_abs_path, reference, conan_api, uploader, + def __init__(self, profile_abs_path, reference, conan_api, uploader, eraser, exclude_vcvars_precommand=False, build_policy=None, runner=None, cwd=None, printer=None, upload=False, upload_only_recipe=None, test_folder=None, config_url=None, config_args=None, @@ -25,6 +25,7 @@ def __init__(self, profile_abs_path, reference, conan_api, uploader, self.printer = printer or Printer() self._cwd = cwd or os.getcwd() self._uploader = uploader + self._eraser = eraser self._upload = upload self._conan_api = conan_api self._profile_abs_path = profile_abs_path @@ -127,21 +128,23 @@ def run(self): self.printer.print_rule() return for installed in r['installed']: + str_ref = str(self._reference) reference = installed["recipe"]["id"] if client_version >= Version("1.10.0"): reference = ConanFileReference.loads(reference) reference = str(reference.copy_clear_rev()) - if ((reference == str(self._reference)) or \ + if ((reference == str_ref) or \ (reference in self._upload_dependencies) or \ ("all" in self._upload_dependencies)) and \ installed['packages']: package_id = installed['packages'][0]['id'] - if installed['packages'][0]["built"]: + if installed['packages'][0]["built"]: if self._upload_only_recipe: self._uploader.upload_recipe(reference, self._upload) else: self._uploader.upload_packages(reference, self._upload, package_id) + self._eraser.remove_outdated_packages(str_ref) else: self.printer.print_message("Skipping upload for %s, " "it hasn't been built" % package_id) @@ -154,6 +157,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference docker_image_skip_update=False, build_policy=None, docker_image_skip_pull=False, always_update_conan_in_docker=False, + remove_outdated_packages=False, upload=False, upload_retry=None, upload_only_recipe=None, runner=None, docker_shell="", docker_conan_home="", @@ -179,6 +183,7 @@ def __init__(self, profile_text, base_profile_text, base_profile_name, reference self._build_policy = build_policy self._docker_image = docker_image self._always_update_conan_in_docker = always_update_conan_in_docker + self._remove_outdated_packages = remove_outdated_packages self._docker_image_skip_update = docker_image_skip_update self._docker_image_skip_pull = docker_image_skip_pull self._sudo_docker_command = sudo_docker_command or "" @@ -316,6 +321,7 @@ def get_env_vars(self): ret["CONAN_TEMP_TEST_FOLDER"] = "1" # test package folder to a temp one ret["CPT_UPLOAD_ENABLED"] = self._upload ret["CPT_UPLOAD_RETRY"] = self._upload_retry + ret["CPT_REMOVE_OUTDATED_PACKAGES"] = self._remove_outdated_packages ret["CPT_UPLOAD_ONLY_RECIPE"] = self._upload_only_recipe ret["CPT_BUILD_POLICY"] = escape_env(self._build_policy) ret["CPT_TEST_FOLDER"] = escape_env(self._test_folder) diff --git a/cpt/test/test_client/erase_checks_test.py b/cpt/test/test_client/erase_checks_test.py new file mode 100644 index 00000000..f58776e9 --- /dev/null +++ b/cpt/test/test_client/erase_checks_test.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +import unittest +from parameterized import parameterized + +from conans.client.tools import environment_append +from conans.test.utils.tools import TestClient, TestServer + +from cpt.test.test_client.tools import get_patched_multipackager + + +class EraseTest(unittest.TestCase): + + old_conanfile = """from conans import ConanFile +class Pkg(ConanFile): + name = "lib" + version = "1.0" + options = {"shared": [True, False]} + default_options = "shared=False" + + def build(self): + self.output.warn("OLD") +""" + + @parameterized.expand([ + ("1", "assertIn"), + ("0", "assertNotIn") + ]) + def test_remove_updated_packages_env_var(self, remove_packages, assert_func): + ts = TestServer(users={"user": "password"}) + tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) + tc.save({"conanfile.py": self.old_conanfile}) + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user", + "CONAN_REMOVE_OUTDATED_PACKAGES": remove_packages}): + mulitpackager = get_patched_multipackager(tc, build_policy="missing", + exclude_vcvars_precommand=True) + mulitpackager.add({}, {"shared": True}) + mulitpackager.add({}, {"shared": False}) + mulitpackager.run() + self.assertIn("Uploading package 1/2", tc.out) + self.assertIn("Uploading package 2/2", tc.out) + self.assertIn("OLD", tc.out) + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'", + tc.out) + + @parameterized.expand([ + (True, "assertIn"), + (False, "assertNotIn") + ]) + def test_remove_updated_packages_params(self, remove_packages, assert_func): + ts = TestServer(users={"user": "password"}) + tc = TestClient(servers={"default": ts}, users={"default": [("user", "password")]}) + tc.save({"conanfile.py": self.old_conanfile}) + with environment_append({"CONAN_UPLOAD": ts.fake_url, "CONAN_LOGIN_USERNAME": "user", + "CONAN_PASSWORD": "password", "CONAN_USERNAME": "user"}): + mulitpackager = get_patched_multipackager(tc, build_policy="missing", + exclude_vcvars_precommand=True, + remove_outdated_packages=remove_packages) + mulitpackager.add({}, {"shared": True}) + mulitpackager.add({}, {"shared": False}) + mulitpackager.run() + self.assertIn("Uploading package 1/2", tc.out) + self.assertIn("Uploading package 2/2", tc.out) + self.assertIn("OLD", tc.out) + getattr(self, assert_func)("Removing outdated packages for 'lib/1.0@user/mychannel'", + tc.out) diff --git a/cpt/test/unit/eraser_test.py b/cpt/test/unit/eraser_test.py new file mode 100644 index 00000000..41fa0aa1 --- /dev/null +++ b/cpt/test/unit/eraser_test.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +import unittest + +from collections import namedtuple +from conans.test.utils.tools import TestBufferConanOutput +from cpt.eraser import Eraser +from cpt.printer import Printer +from cpt.test.unit.packager_test import MockConanAPI + + +class AuthTest(unittest.TestCase): + + def setUp(self): + self.conan_api = MockConanAPI() + self.output = TestBufferConanOutput() + self.printer = Printer(self.output.write) + + def test_invalid_remote(self): + eraser = Eraser(self.conan_api, None, None, self.printer, True) + eraser.remove_outdated_packages("foo/0.1.0@user/channel") + self.assertIn("Remove outdated skipped, no remote available", self.output) + self.assertFalse(self.conan_api.calls) + + def test_invalid_authentication(self): + FakeRemoteManager = namedtuple("FakeRemoteManager", "upload_remote_name") + remote_manager = FakeRemoteManager(upload_remote_name="default") + eraser = Eraser(self.conan_api, remote_manager, None, self.printer, True) + eraser.remove_outdated_packages("foo/0.1.0@user/channel") + self.assertIn("Remove outdated skipped, credentials for remote 'default' not available", self.output) + self.assertFalse(self.conan_api.calls)