From ced8c08ffe902a01a631dd4ac343c8b4505af075 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 30 May 2024 17:58:44 +0200 Subject: [PATCH 01/19] WIP: methods to run gh create issue Signed-off-by: Jose Luis Rivero --- release.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/release.py b/release.py index 265796308..27023033a 100755 --- a/release.py +++ b/release.py @@ -2,6 +2,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from tempfile import NamedTemporaryFile import subprocess import sys import tempfile @@ -515,9 +516,51 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') +def get_vendor_repo_name(args): + canonical_name = get_canonical_package_name(args.package) + return canonical_name.replace('-', '_') + "_vendor" + + +def create_issue_in_repo(github_repo, title, body): + _out = "" + with NamedTemporaryFile("w", delete=True) as f: + f.write(body) + f.flush() + cmd = ['gh', 'issue', 'create', + '--repo', github_repo, + '--title', title, + '--body-file', f.name] + _out, _err = check_call(cmd) + if _err: + print(f"An error happened running gh issue cmd: {_err}") + sys.exit(1) + return _out + + + +def create_issue_in_gz_vendor_repo(args, ros_distro): + gz_vendor_repo = 'gazebo-release/' + get_vendor_repo_name(args) + print(gz_vendor_repo) + gz_vendor_repo = 'j-rivero/test' + title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' + body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ + 'This repository needs to be updated accordingly for the branch ${ros_distro}:\n'\ + ' * Sync to the new version in CMakelists.txt \n'\ + ' * Bump the patch version in package.xml \n'\ + ' * Run the release process for this ROS package' + + return create_issue_in_repo(gz_vendor_repo, title, body) + + + def go(argv): args = parse_args(argv) + ros_distro = 'rolling' + out = create_issue_in_gz_vendor_repo(args, ros_distro) + print(out) + sys.exit(2) + # Default to release 1 if not present if not args.release_version: args.release_version = 1 From b68bd353397740b26d3dba319aafff9ca6c4237b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 13:24:50 +0200 Subject: [PATCH 02/19] Fully working version Signed-off-by: Jose Luis Rivero --- release.py | 49 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/release.py b/release.py index 27023033a..784736b4b 100755 --- a/release.py +++ b/release.py @@ -23,6 +23,7 @@ LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' +ROS_VENDOR = {'harmonic': 'rolling'} OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none" @@ -516,9 +517,25 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') -def get_vendor_repo_name(args): - canonical_name = get_canonical_package_name(args.package) - return canonical_name.replace('-', '_') + "_vendor" +def get_collections_for_package(package_name, version): + script_directory = os.path.dirname(os.path.abspath(sys.argv[0])) + helper_script = f'{script_directory}/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py' + collection_yaml = f'{script_directory}/jenkins-scripts/dsl/gz-collections.yaml' + cmd = [helper_script, + get_canonical_package_name(package_name), + version, + collection_yaml] + _out, _err = check_call(cmd, IGNORE_DRY_RUN) + if _err: + print(f"An error happened running get_collections_from_package_and_version: {_err}") + sys.exit(1) + collection_list = _out.decode().strip().split(' ') + return collection_list + + +def get_vendor_github_repo(package_name): + canonical_name = get_canonical_package_name(package_name) + return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" def create_issue_in_repo(github_repo, title, body): @@ -534,33 +551,26 @@ def create_issue_in_repo(github_repo, title, body): if _err: print(f"An error happened running gh issue cmd: {_err}") sys.exit(1) - return _out - + if DRY_RUN: + return f"http://github.com/{github_repo}/issues/#dry-run-no-number" + return _out.decode().replace('\n', '') def create_issue_in_gz_vendor_repo(args, ros_distro): - gz_vendor_repo = 'gazebo-release/' + get_vendor_repo_name(args) - print(gz_vendor_repo) + gz_vendor_repo = get_vendor_github_repo(args.package) gz_vendor_repo = 'j-rivero/test' title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ - 'This repository needs to be updated accordingly for the branch ${ros_distro}:\n'\ + 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ ' * Sync to the new version in CMakelists.txt \n'\ ' * Bump the patch version in package.xml \n'\ ' * Run the release process for this ROS package' - return create_issue_in_repo(gz_vendor_repo, title, body) - def go(argv): args = parse_args(argv) - ros_distro = 'rolling' - out = create_issue_in_gz_vendor_repo(args, ros_distro) - print(out) - sys.exit(2) - # Default to release 1 if not present if not args.release_version: args.release_version = 1 @@ -684,6 +694,15 @@ def go(argv): args.version) display_help_job_chain_for_source_calls(args) + print("ROS vendor packages that can be updated:") + for collection in get_collections_for_package(args.package, args.version): + if collection in ROS_VENDOR: + ros_distro = ROS_VENDOR[collection] + print(f" * Github {get_vendor_github_repo(args.package)} " + f"part of {collection} in ROS 2 {ros_distro}") + issue_url = create_issue_in_gz_vendor_repo(args, ros_distro) + print(f" + Issue created: {issue_url}") + if __name__ == '__main__': go(sys.argv) From db115270e12bc2e18c1151aa2024eb70794a977b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 16:12:00 +0200 Subject: [PATCH 03/19] Missing helper file Signed-off-by: Jose Luis Rivero --- ...et_collections_from_package_and_version.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py diff --git a/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py b/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py new file mode 100755 index 000000000..f7840304a --- /dev/null +++ b/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import sys +import yaml + + +# Function to find the collection name based on lib name and major version +def find_collection(data, lib_name, major_version): + instances = [] + + for collection in data['collections']: + for lib in collection['libs']: + if lib['name'] == lib_name and lib['major_version'] == major_version: + instances.append(collection['name']) + return instances + + +def get_major_version(version): + elements = version.split('.') + return int(elements[0]) + + +if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + +lib_name = sys.argv[1] +version = sys.argv[2] +yaml_file = sys.argv[3] + +with open(yaml_file, 'r') as file: + data = yaml.safe_load(file) + +collection_names = find_collection(data, lib_name, get_major_version(version)) +print(f"{' '.join(collection_names)}") From 67f4a2b904a8fb62957d9f8f0f4df9a3ca355c93 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 18:09:49 +0200 Subject: [PATCH 04/19] Improve the error reporting on gh Signed-off-by: Jose Luis Rivero --- release.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/release.py b/release.py index 784736b4b..2e5e3f5d6 100755 --- a/release.py +++ b/release.py @@ -539,21 +539,21 @@ def get_vendor_github_repo(package_name): def create_issue_in_repo(github_repo, title, body): - _out = "" + _out = b"" with NamedTemporaryFile("w", delete=True) as f: f.write(body) f.flush() - cmd = ['gh', 'issue', 'create', + cmd = ['ghxx', 'issue', 'create', '--repo', github_repo, '--title', title, '--body-file', f.name] - _out, _err = check_call(cmd) - if _err: - print(f"An error happened running gh issue cmd: {_err}") - sys.exit(1) + try: + _out, _err = check_call(cmd) + except Exception as e: + _err = str(e) if DRY_RUN: - return f"http://github.com/{github_repo}/issues/#dry-run-no-number" - return _out.decode().replace('\n', '') + return f"http://github.com/{github_repo}/issues/#dry-run-no-number", "" + return _out.decode().replace('\n', ''), _err def create_issue_in_gz_vendor_repo(args, ros_distro): @@ -565,7 +565,13 @@ def create_issue_in_gz_vendor_repo(args, ros_distro): ' * Sync to the new version in CMakelists.txt \n'\ ' * Bump the patch version in package.xml \n'\ ' * Run the release process for this ROS package' - return create_issue_in_repo(gz_vendor_repo, title, body) + _out, _err = create_issue_in_repo(gz_vendor_repo, title, body) + if _err: + print(' !! An error happened running the "gh issue" cmd. Do not run this script again.\n' + ' Please create the issue manually. The error reported was:\n' + f' {_err}') + sys.exit(1) + return _out def go(argv): From 3236d73a33aabbf22c9163ac716e5eb46fa5c72f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 18:39:32 +0200 Subject: [PATCH 05/19] Implement a basic testing Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 46 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index 921153017..c623d9e38 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -12,7 +12,7 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - gz-foo 1.2.3 token ${test_params}"" + gz-foo 1.2.3 token ${test_params} } exec_ignition_releasepy_test() @@ -22,7 +22,7 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-foo 1.2.3 token ${test_params}"" + ign-foo 1.2.3 token ${test_params} } exec_ignition_gazebo_releasepy_test() @@ -32,7 +32,18 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-gazebo 1.2.3 token ${test_params}"" + ign-gazebo 1.2.3 token ${test_params} +} + +exec_releasepy_with_real_gz() +{ + gz_pkg=${1} major_version=${2} + ./release.py \ + --dry-run \ + --no-sanity-checks \ + --source-repo-uri http://github.com/gazebosim/gz-common \ + --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ + "${gz_pkg}" "${major_version}.x.y" token } expect_job_run() @@ -73,7 +84,26 @@ expect_param() echo "${param} not found in test output" exit 1 fi +} +expect_vendor_repo() +{ + output="${1}" repo="${2}" + + if ! grep "Github ${repo}" <<< "${output}"; then + echo "${repo} not found in test output" + exit 1 + fi +} + +expect_no_vendor() +{ + output="${1}" + + if grep -q 'in ROS 2' <<< "${output}"; then + echo "ROS 2 string found in output" + exit 1 + fi } source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") @@ -81,6 +111,7 @@ expect_job_run "${source_repo_uri_test}" "gz-foo-source" expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" expect_number_of_jobs "${source_repo_uri_test}" "1" expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" +expect_no_vendor "${source_repo_uri_test}" # non existing package source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" @@ -88,6 +119,7 @@ expect_job_run "${source_tarball_uri_test}" "generic-release-homebrew_pull_reque expect_job_not_run "${source_tarball_uri_test}" "gz-foo-source" expect_number_of_jobs "${source_tarball_uri_test}" "7" expect_param "${source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" +expect_no_vendor "${source_tarball_uri_test}" nightly_test=$(exec_releasepy_test "--nightly-src-branch my-nightly-branch3 --upload-to-repo nightly") expect_job_run "${nightly_test}" "gz-foo-debbuilder" @@ -95,6 +127,7 @@ expect_job_not_run "${nightly_test}" "generic-release-homebrew_pull_request_upda expect_job_not_run "${nightly_test}" "gz-foo-source" expect_number_of_jobs "${nightly_test}" "2" expect_param "${nightly_test}" "SOURCE_TARBALL_URI=my-nightly-branch3" +expect_no_vendor "${nightly_test}" bump_linux_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz --only-bump-revision-linux -r 2") expect_job_run "${bump_linux_test}" "gz-foo-debbuilder" @@ -102,6 +135,7 @@ expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_u expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" +expect_no_vendor "${bump_linux_test}" ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${ignition_test}" "gz-foo-source" @@ -128,3 +162,9 @@ expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7" expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo" + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-fuel-tools 9) +expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2) +expect_no_vendor "${bump_linux_test}" From 8ed28301c45bcc1016946d77de85f3df7988b790 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 3 Jun 2024 12:48:42 +0200 Subject: [PATCH 06/19] Cleanup Signed-off-by: Jose Luis Rivero --- release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.py b/release.py index 2e5e3f5d6..0fc79b5b0 100755 --- a/release.py +++ b/release.py @@ -540,6 +540,8 @@ def get_vendor_github_repo(package_name): def create_issue_in_repo(github_repo, title, body): _out = b"" + # For parsing the \n correctly we need to store the content in a temp + # file and pass it as --body-file with NamedTemporaryFile("w", delete=True) as f: f.write(body) f.flush() @@ -558,7 +560,6 @@ def create_issue_in_repo(github_repo, title, body): def create_issue_in_gz_vendor_repo(args, ros_distro): gz_vendor_repo = get_vendor_github_repo(args.package) - gz_vendor_repo = 'j-rivero/test' title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ From 0087404ca01c13117f9e20105881af8044e04812 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 7 Jun 2024 19:45:16 +0200 Subject: [PATCH 07/19] WIP: create PR for vendor_repositories Signed-off-by: Jose Luis Rivero --- release.py | 84 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/release.py b/release.py index 0fc79b5b0..a21d47e1b 100755 --- a/release.py +++ b/release.py @@ -3,6 +3,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory import subprocess import sys import tempfile @@ -538,46 +539,61 @@ def get_vendor_github_repo(package_name): return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" -def create_issue_in_repo(github_repo, title, body): - _out = b"" - # For parsing the \n correctly we need to store the content in a temp - # file and pass it as --body-file - with NamedTemporaryFile("w", delete=True) as f: - f.write(body) - f.flush() - cmd = ['ghxx', 'issue', 'create', - '--repo', github_repo, - '--title', title, - '--body-file', f.name] - try: - _out, _err = check_call(cmd) - except Exception as e: - _err = str(e) - if DRY_RUN: - return f"http://github.com/{github_repo}/issues/#dry-run-no-number", "" - return _out.decode().replace('\n', ''), _err - - -def create_issue_in_gz_vendor_repo(args, ros_distro): - gz_vendor_repo = get_vendor_github_repo(args.package) - title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' - body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ - 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ - ' * Sync to the new version in CMakelists.txt \n'\ - ' * Bump the patch version in package.xml \n'\ - ' * Run the release process for this ROS package' - _out, _err = create_issue_in_repo(gz_vendor_repo, title, body) - if _err: - print(' !! An error happened running the "gh issue" cmd. Do not run this script again.\n' - ' Please create the issue manually. The error reported was:\n' - f' {_err}') +def get_vendor_repo_url(package_name): + return f"https://github.com/{get_vendor_github_repo(package_name)}" + + +def prepare_vendor_pr_temp_workspace(package_name): + pr_ws_dir = tempfile.mkdtemp() + gz_vendor_tool = os.path.join(pr_ws_dir, "gz_vendor") + cmd = ['git', 'clone', '-q', + 'https://github.com/gazebo-tooling/gz_vendor/', + gz_vendor_tool] + _, _err_tool = check_call(cmd) + gz_vendor_repo = os.path.join(pr_ws_dir, 'gz_vendor_repo') + cmd = ['git', 'clone', '-q', + get_vendor_repo_url(package_name), + gz_vendor_repo] + _, _err_repo = check_call(cmd) + if _err_tool or _err_repo: + print("Problems with cloning vendor and tool repos:") + print(f"{_err_tool} {_err_repo}") + sys.exit(1) + + return gz_vendor_tool, gz_vendor_repo + + +def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path): + run_cmd = ['python3', + f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", + 'package.xml', + '--output_dir', vendor_repo_path] + _, _err_run = check_call(run_cmd) + if _err_run: + print("Problems running the create_vendor_package.py script:") sys.exit(1) - return _out + + +def create_pr_in_gz_vendor_repo(args): + vendor_tool_path, vendor_repo_path = \ + prepare_vendor_pr_temp_workspace(args.package) + execute_update_vendor_package_tool( + vendor_tool_path, vendor_repo_path) + + cmd_diff = ['git', "-C", vendor_repo_path, 'diff'] + _out, _ = check_call(cmd_diff) + print(vendor_repo_path) + print(_out.decode()) + + def go(argv): args = parse_args(argv) + create_pr_in_gz_vendor_repo(args) + sys.exit(0) + # Default to release 1 if not present if not args.release_version: args.release_version = 1 From a777cd64f5e39c0c6880fb0396c734e354800373 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 11 Jun 2024 19:40:58 +0200 Subject: [PATCH 08/19] Implement the PR creation Signed-off-by: Jose Luis Rivero --- release.py | 101 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/release.py b/release.py index a21d47e1b..34945f11d 100755 --- a/release.py +++ b/release.py @@ -2,8 +2,6 @@ from __future__ import print_function from argparse import RawTextHelpFormatter -from tempfile import NamedTemporaryFile -from tempfile import TemporaryDirectory import subprocess import sys import tempfile @@ -51,6 +49,10 @@ class ErrorNoOutput(Exception): pass +class ErrorAlreadyExists(Exception): + pass + + def error(msg): print("\n !! " + msg + "\n") sys.exit(1) @@ -378,13 +380,13 @@ def discover_distros(repo_dir): return distro_arch_list -def check_call(cmd, ignore_dry_run=False): +def check_call(cmd, ignore_dry_run=False, cwd=None): if DRY_RUN and not ignore_dry_run: print_only_dbg('Dry-run running:\n %s\n' % (' '.join(cmd))) return b'', b'' else: print_only_dbg('Running:\n %s' % (' '.join(cmd))) - po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) out, err = po.communicate() if po.returncode != 0: # bitbucket for the first one, github for the second @@ -394,6 +396,8 @@ def check_call(cmd, ignore_dry_run=False): raise ErrorNoPermsRepo() if b"abort: no username supplied" in err: raise ErrorNoUsernameSupplied() + if b"already exists:" in err: + raise ErrorAlreadyExists() if not out and not err: # assume that call is only for getting return code raise ErrorNoOutput() @@ -518,7 +522,7 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') -def get_collections_for_package(package_name, version): +def get_collections_for_package(package_name, version) -> list: script_directory = os.path.dirname(os.path.abspath(sys.argv[0])) helper_script = f'{script_directory}/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py' collection_yaml = f'{script_directory}/jenkins-scripts/dsl/gz-collections.yaml' @@ -534,27 +538,27 @@ def get_collections_for_package(package_name, version): return collection_list -def get_vendor_github_repo(package_name): +def get_vendor_github_repo(package_name) -> str: canonical_name = get_canonical_package_name(package_name) return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" -def get_vendor_repo_url(package_name): +def get_vendor_repo_url(package_name) -> str: return f"https://github.com/{get_vendor_github_repo(package_name)}" -def prepare_vendor_pr_temp_workspace(package_name): - pr_ws_dir = tempfile.mkdtemp() - gz_vendor_tool = os.path.join(pr_ws_dir, "gz_vendor") +def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> tuple[str, str]: + print(ws_dir) + gz_vendor_tool = os.path.join(ws_dir, "gz_vendor") cmd = ['git', 'clone', '-q', 'https://github.com/gazebo-tooling/gz_vendor/', gz_vendor_tool] - _, _err_tool = check_call(cmd) - gz_vendor_repo = os.path.join(pr_ws_dir, 'gz_vendor_repo') + _, _err_tool = check_call(cmd, IGNORE_DRY_RUN) + gz_vendor_repo = os.path.join(ws_dir, 'gz_vendor_repo') cmd = ['git', 'clone', '-q', get_vendor_repo_url(package_name), gz_vendor_repo] - _, _err_repo = check_call(cmd) + _, _err_repo = check_call(cmd, IGNORE_DRY_RUN) if _err_tool or _err_repo: print("Problems with cloning vendor and tool repos:") print(f"{_err_tool} {_err_repo}") @@ -563,7 +567,7 @@ def prepare_vendor_pr_temp_workspace(package_name): return gz_vendor_tool, gz_vendor_repo -def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path): +def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path) -> None: run_cmd = ['python3', f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", 'package.xml', @@ -571,28 +575,67 @@ def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path): _, _err_run = check_call(run_cmd) if _err_run: print("Problems running the create_vendor_package.py script:") + print(_err_run.decode()) sys.exit(1) -def create_pr_in_gz_vendor_repo(args): - vendor_tool_path, vendor_repo_path = \ - prepare_vendor_pr_temp_workspace(args.package) - execute_update_vendor_package_tool( - vendor_tool_path, vendor_repo_path) +def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: + cmd_diff = ['git', "-C", repo_path, 'diff'] + _out, _ = check_call(cmd_diff, IGNORE_DRY_RUN) + + branch_name = f'releasepy/{args.version}' + vendor_repo = get_vendor_repo_url(args.package) + branch_cmd = ['git', "-C", repo_path, + 'checkout', '-b', branch_name] + _, _ = check_call(branch_cmd, IGNORE_DRY_RUN) + commit_cmd = ['git', "-C", repo_path, + 'commit', + '-m', f'Bump version to {args.version}', + 'CMakeLists.txt', 'package.xml'] + _, _ = check_call(commit_cmd) + push_cmd = ['git', "-C", repo_path, + 'push', '--force', + vendor_repo, branch_name] + _, _ = check_call(push_cmd) + pr_cmd = ['gh', 'pr', 'create', + '--base', base_branch, + '--head', branch_name, + '--title', f'Bump version to {args.version}', + '--body', 'PR automatically created by release.py'] + try: + _out, _err = check_call(pr_cmd, cwd=repo_path) + except ErrorAlreadyExists: + return 'there is already a PR for the branch.'\ + 'Please check it out manuallly.' + + if _err: + print("Problems creating the PR for the vendor package:") + print(_err.decode()) + sys.exit(1) + + return _out.decode() - cmd_diff = ['git', "-C", vendor_repo_path, 'diff'] - _out, _ = check_call(cmd_diff) - print(vendor_repo_path) - print(_out.decode()) +def create_pr_in_gz_vendor_repo(args, ros_distro) -> str: + pr_msg = '' + with tempfile.TemporaryDirectory() as ws_dir: + ws_dir = tempfile.mkdtemp() + # Prepare the temporary workspace + vendor_tool_path, vendor_repo_path = \ + prepare_vendor_pr_temp_workspace(args.package, ws_dir) + # Run updating script on the temporary workspace + execute_update_vendor_package_tool( + vendor_tool_path, vendor_repo_path) + # Commits and PR creation + pr_msg = create_pr_for_vendor_package( + args, vendor_repo_path, ros_distro) + return pr_msg def go(argv): args = parse_args(argv) - create_pr_in_gz_vendor_repo(args) - sys.exit(0) # Default to release 1 if not present if not args.release_version: @@ -718,13 +761,15 @@ def go(argv): display_help_job_chain_for_source_calls(args) print("ROS vendor packages that can be updated:") - for collection in get_collections_for_package(args.package, args.version): + for collection in get_collections_for_package(args.package, + args.version): if collection in ROS_VENDOR: ros_distro = ROS_VENDOR[collection] print(f" * Github {get_vendor_github_repo(args.package)} " f"part of {collection} in ROS 2 {ros_distro}") - issue_url = create_issue_in_gz_vendor_repo(args, ros_distro) - print(f" + Issue created: {issue_url}") + print(" + Preparing a PR: ", end='', flush=True) + pr_url = create_pr_in_gz_vendor_repo(args, ros_distro) + print(pr_url) if __name__ == '__main__': From 6610c742df050e89fad65608d3bb3ed22eb1bed8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 11 Jun 2024 20:06:03 +0200 Subject: [PATCH 09/19] Remove debug Signed-off-by: Jose Luis Rivero --- release.py | 1 - 1 file changed, 1 deletion(-) diff --git a/release.py b/release.py index 34945f11d..9076daceb 100755 --- a/release.py +++ b/release.py @@ -548,7 +548,6 @@ def get_vendor_repo_url(package_name) -> str: def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> tuple[str, str]: - print(ws_dir) gz_vendor_tool = os.path.join(ws_dir, "gz_vendor") cmd = ['git', 'clone', '-q', 'https://github.com/gazebo-tooling/gz_vendor/', From 641899bbb29a1e8e5c6ce0b9bbe8c0d529d55c89 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 11 Jun 2024 20:27:46 +0200 Subject: [PATCH 10/19] Dealing with error code coming from get_collections_from_package_and_version Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 2 +- release.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index c623d9e38..b9a06701e 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -90,7 +90,7 @@ expect_vendor_repo() { output="${1}" repo="${2}" - if ! grep "Github ${repo}" <<< "${output}"; then + if ! grep -q "Github ${repo}" <<< "${output}"; then echo "${repo} not found in test output" exit 1 fi diff --git a/release.py b/release.py index 9076daceb..19408b0c0 100755 --- a/release.py +++ b/release.py @@ -530,10 +530,17 @@ def get_collections_for_package(package_name, version) -> list: get_canonical_package_name(package_name), version, collection_yaml] - _out, _err = check_call(cmd, IGNORE_DRY_RUN) - if _err: - print(f"An error happened running get_collections_from_package_and_version: {_err}") - sys.exit(1) + try: + _out, _err = check_call(cmd, IGNORE_DRY_RUN) + except ErrorNoOutput: + # no output is a valid result + _out = b"" + _err = "" + else: + if _err: + print(f"An error happened running get_collections_from_package_and_version: {_err}") + sys.exit(1) + collection_list = _out.decode().strip().split(' ') return collection_list From 80475de9dd9471d98ac8d7fcda625c87734da452 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Jul 2024 13:01:38 +0200 Subject: [PATCH 11/19] Implement venv creation for vendor Signed-off-by: Jose Luis Rivero --- release.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/release.py b/release.py index 19408b0c0..f3c44ec3a 100755 --- a/release.py +++ b/release.py @@ -2,6 +2,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from typing import Tuple import subprocess import sys import tempfile @@ -10,6 +11,7 @@ import urllib.request import argparse import shutil +import venv USAGE = 'release.py ' try: @@ -554,8 +556,15 @@ def get_vendor_repo_url(package_name) -> str: return f"https://github.com/{get_vendor_github_repo(package_name)}" -def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> tuple[str, str]: +def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, str]: gz_vendor_tool = os.path.join(ws_dir, "gz_vendor") + # Create virtualenv for vendor dependencies + venv_dir = os.path.join(ws_dir, "venv") + venv.create(venv_dir, system_site_packages=True, with_pip=True) + subprocess.run([os.path.join(venv_dir, 'bin', 'pip3'), 'install', '-q', + 'jinja2==3.1.2', + 'catkin_pkg==1.0.0', + 'argparse']) cmd = ['git', 'clone', '-q', 'https://github.com/gazebo-tooling/gz_vendor/', gz_vendor_tool] @@ -570,11 +579,13 @@ def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> tuple[str, str]: print(f"{_err_tool} {_err_repo}") sys.exit(1) - return gz_vendor_tool, gz_vendor_repo + return gz_vendor_tool, gz_vendor_repo, venv_dir -def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path) -> None: - run_cmd = ['python3', +def execute_update_vendor_package_tool(vendor_tool_path, + vendor_repo_path, + vendor_venv) -> None: + run_cmd = [os.path.join(vendor_venv, 'bin', 'python3'), f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", 'package.xml', '--output_dir', vendor_repo_path] @@ -588,6 +599,8 @@ def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path) -> No def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: cmd_diff = ['git', "-C", repo_path, 'diff'] _out, _ = check_call(cmd_diff, IGNORE_DRY_RUN) + if not _out.decode(): + return 'vendor tool did not produce any change, avoid the PR' branch_name = f'releasepy/{args.version}' vendor_repo = get_vendor_repo_url(args.package) @@ -627,11 +640,11 @@ def create_pr_in_gz_vendor_repo(args, ros_distro) -> str: with tempfile.TemporaryDirectory() as ws_dir: ws_dir = tempfile.mkdtemp() # Prepare the temporary workspace - vendor_tool_path, vendor_repo_path = \ + vendor_tool_path, vendor_repo_path, venv_dir = \ prepare_vendor_pr_temp_workspace(args.package, ws_dir) # Run updating script on the temporary workspace execute_update_vendor_package_tool( - vendor_tool_path, vendor_repo_path) + vendor_tool_path, vendor_repo_path, venv_dir) # Commits and PR creation pr_msg = create_pr_for_vendor_package( args, vendor_repo_path, ros_distro) From 82344430add9c96f3bf4de19a74d3a788dee0394 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 19 Jul 2024 16:32:22 +0200 Subject: [PATCH 12/19] Implement then --only-bump-ros-vendor-package option Signed-off-by: Jose Luis Rivero --- release.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/release.py b/release.py index f3c44ec3a..ac7a16578 100755 --- a/release.py +++ b/release.py @@ -158,6 +158,11 @@ def parse_args(argv): parser.add_argument('--only-bump-revision-linux', dest='bump_rev_linux_only', action='store_true', default=False, help='Bump only revision number. Do not upload new tarball.') + parser.add_argument('--only-bump-ros-vendor-package', dest='bump_ros_vendor_only', + action='store_true', default=False, + help='Only process the ROS vendor package (if any).') + + args = parser.parse_args() @@ -624,8 +629,8 @@ def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: try: _out, _err = check_call(pr_cmd, cwd=repo_path) except ErrorAlreadyExists: - return 'there is already a PR for the branch.'\ - 'Please check it out manuallly.' + return f'there is already a PR for the branch: {branch_name} .'\ + 'Please check it out manuallly.' if _err: print("Problems creating the PR for the vendor package:") @@ -652,9 +657,26 @@ def create_pr_in_gz_vendor_repo(args, ros_distro) -> str: return pr_msg +def process_ros_vendor_package(args): + print("ROS vendor packages that can be updated:") + for collection in get_collections_for_package(args.package, + args.version): + if collection in ROS_VENDOR: + ros_distro = ROS_VENDOR[collection] + print(f" * Github {get_vendor_github_repo(args.package)} " + f"part of {collection} in ROS 2 {ros_distro}") + print(" + Preparing a PR: ", end='', flush=True) + pr_url = create_pr_in_gz_vendor_repo(args, ros_distro) + print(pr_url) + + def go(argv): args = parse_args(argv) + # If only the process of ROS vendor package is set, just do it + if args.bump_ros_vendor_only: + process_ros_vendor_package(args) + sys.exit(0) # Default to release 1 if not present if not args.release_version: @@ -778,18 +800,8 @@ def go(argv): 'Source', args.version) display_help_job_chain_for_source_calls(args) - - print("ROS vendor packages that can be updated:") - for collection in get_collections_for_package(args.package, - args.version): - if collection in ROS_VENDOR: - ros_distro = ROS_VENDOR[collection] - print(f" * Github {get_vendor_github_repo(args.package)} " - f"part of {collection} in ROS 2 {ros_distro}") - print(" + Preparing a PR: ", end='', flush=True) - pr_url = create_pr_in_gz_vendor_repo(args, ros_distro) - print(pr_url) - + # Process the possible update of an associated ROS vendor package + process_ros_vendor_package(args) if __name__ == '__main__': go(sys.argv) From f049c453b48394d34ed2256d3236ec9c6011aa6a Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Wed, 25 Sep 2024 19:16:56 -0500 Subject: [PATCH 13/19] Add ionic->rolling, harmonic->jazzy Signed-off-by: Addisu Z. Taddese --- release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.py b/release.py index ac7a16578..346a035d3 100755 --- a/release.py +++ b/release.py @@ -24,7 +24,7 @@ LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' -ROS_VENDOR = {'harmonic': 'rolling'} +ROS_VENDOR = {'harmonic': 'jazzy', 'ionic': 'rolling'} OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none" From 807070d47b358621957312ac09ec3e6178c0744c Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Thu, 26 Sep 2024 08:23:40 -0500 Subject: [PATCH 14/19] Ignore dry-run when updating vendor package. Otherwise, the dry-run will report there were no changes in the vendor package Signed-off-by: Addisu Z. Taddese --- release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.py b/release.py index 346a035d3..4da6594ca 100755 --- a/release.py +++ b/release.py @@ -594,7 +594,7 @@ def execute_update_vendor_package_tool(vendor_tool_path, f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", 'package.xml', '--output_dir', vendor_repo_path] - _, _err_run = check_call(run_cmd) + _, _err_run = check_call(run_cmd, IGNORE_DRY_RUN) if _err_run: print("Problems running the create_vendor_package.py script:") print(_err_run.decode()) From 1a84b587a3fdc2a7343e5b19b4dbd88b4ea6ac2f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Sep 2024 16:01:34 +0200 Subject: [PATCH 15/19] Do not need to make argparse explicit Signed-off-by: Jose Luis Rivero --- release.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release.py b/release.py index 4da6594ca..c345f67d4 100755 --- a/release.py +++ b/release.py @@ -568,8 +568,7 @@ def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, st venv.create(venv_dir, system_site_packages=True, with_pip=True) subprocess.run([os.path.join(venv_dir, 'bin', 'pip3'), 'install', '-q', 'jinja2==3.1.2', - 'catkin_pkg==1.0.0', - 'argparse']) + 'catkin_pkg==1.0.0']) cmd = ['git', 'clone', '-q', 'https://github.com/gazebo-tooling/gz_vendor/', gz_vendor_tool] From c2da471d7fdb6a8c5c9770242ea806098099ffdb Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Sep 2024 16:03:24 +0200 Subject: [PATCH 16/19] Update all files after running the create_vendor_package script Signed-off-by: Jose Luis Rivero --- release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.py b/release.py index c345f67d4..ad9157d4a 100755 --- a/release.py +++ b/release.py @@ -614,7 +614,7 @@ def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: commit_cmd = ['git', "-C", repo_path, 'commit', '-m', f'Bump version to {args.version}', - 'CMakeLists.txt', 'package.xml'] + '--all'] _, _ = check_call(commit_cmd) push_cmd = ['git', "-C", repo_path, 'push', '--force', From 6c1e3c0c7b00eb75225f8c09e2e3e34fad0e9751 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 26 Sep 2024 18:34:41 +0200 Subject: [PATCH 17/19] Fix test suite by injecting testing data Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 21 +++++++++++++++++---- release.py | 14 +++++++++++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index b9a06701e..2b2af0895 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -1,9 +1,22 @@ #!/bin/bash -e -test_dir=$(mktemp -d) -mkdir -p ${test_dir}/{focal,jammy,ubuntu}/debian -export _RELEASEPY_TEST_RELEASE_REPO=${test_dir} export _RELEASEPY_DEBUG=1 +test_dir=$(mktemp -d) +export _RELEASEPY_TEST_RELEASE_REPO="${test_dir}/test-release" +mkdir -p ${_RELEASEPY_TEST_RELEASE_REPO}/{focal,jammy,ubuntu}/debian +export _RELEASEPY_TEST_SOURCE_REPO="${test_dir}/src" +mkdir -p ${_RELEASEPY_TEST_SOURCE_REPO} +# Fake packages.xml to make the vendor package script happy +cat > "${_RELEASEPY_TEST_SOURCE_REPO}/package.xml" <<-EOF + + + gz-foo + 0.0.0 + test + Testing maintainer + Foo License + +EOF exec_releasepy_test() { @@ -167,4 +180,4 @@ ros_vendor_test=$(exec_releasepy_with_real_gz gz-fuel-tools 9) expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2) -expect_no_vendor "${bump_linux_test}" +expect_no_vendor "${ros_vendor_test}" diff --git a/release.py b/release.py index ad9157d4a..84dc8b01b 100755 --- a/release.py +++ b/release.py @@ -162,8 +162,6 @@ def parse_args(argv): action='store_true', default=False, help='Only process the ROS vendor package (if any).') - - args = parser.parse_args() args.package_alias = args.package @@ -589,9 +587,16 @@ def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, st def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path, vendor_venv) -> None: + # The source repository when releasing matches the + src_repo = os.getcwd() + try: + src_repo = os.environ['_RELEASEPY_TEST_SOURCE_REPO'] + except KeyError: + pass + run_cmd = [os.path.join(vendor_venv, 'bin', 'python3'), f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", - 'package.xml', + f"{os.path.join(src_repo, 'package.xml')}", '--output_dir', vendor_repo_path] _, _err_run = check_call(run_cmd, IGNORE_DRY_RUN) if _err_run: @@ -636,6 +641,9 @@ def create_pr_for_vendor_package(args, repo_path, base_branch) -> str: print(_err.decode()) sys.exit(1) + if DRY_RUN: + return ' (skipped the creation on --dry-run)' + return _out.decode() From 4180170f05d81205da6abfde1e213521c2b80295 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 27 Sep 2024 11:03:17 +0200 Subject: [PATCH 18/19] Use ssh protocol when cloning Signed-off-by: Jose Luis Rivero --- release.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release.py b/release.py index 84dc8b01b..aaa2b79d7 100755 --- a/release.py +++ b/release.py @@ -556,7 +556,10 @@ def get_vendor_github_repo(package_name) -> str: def get_vendor_repo_url(package_name) -> str: - return f"https://github.com/{get_vendor_github_repo(package_name)}" + # Clone needs ssh for real pushing operations. In simulation prefer https to avoid + # unexpected pushes and facilitate testing + protocol = 'https://github.com/' if DRY_RUN else 'ssh://git@github.com:' + return f"{protocol}{get_vendor_github_repo(package_name)}" def prepare_vendor_pr_temp_workspace(package_name, ws_dir) -> Tuple[str, str, str]: From 24d6a8325fc03359de06c08fbd03902ea3ec62ba Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 27 Sep 2024 11:10:02 +0200 Subject: [PATCH 19/19] Avoid to release metapackages Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 3 +++ release.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/check_releasepy.bash b/check_releasepy.bash index 2b2af0895..9d5c100c4 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -181,3 +181,6 @@ expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2) expect_no_vendor "${ros_vendor_test}" + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-ionic 3) +expect_no_vendor "${ros_vendor_test}" diff --git a/release.py b/release.py index aaa2b79d7..262851709 100755 --- a/release.py +++ b/release.py @@ -669,6 +669,9 @@ def create_pr_in_gz_vendor_repo(args, ros_distro) -> str: def process_ros_vendor_package(args): print("ROS vendor packages that can be updated:") + if args.package.replace('gz-','') in ROS_VENDOR: + print(" - There are no gz metapackages in ROS") + return for collection in get_collections_for_package(args.package, args.version): if collection in ROS_VENDOR: