From 154a50241f032563804b4ba49a556c012def633b Mon Sep 17 00:00:00 2001 From: Dean Roehrich Date: Wed, 11 Sep 2024 15:44:40 -0500 Subject: [PATCH] A tool to vendor a new API into a consumer repo (#209) The vendor-new-api.py tool is a companion tool to crd-bumper. After crd-bumper has been used to create a new API version, the vendor-new-api tool can be used to vendor that into other repositories that consume the API. Signed-off-by: Dean Roehrich --- tools/crd-bumper/README.md | 17 +++ tools/crd-bumper/pkg/go_cli.py | 77 ++++++++++++ tools/crd-bumper/pkg/make_cmd.py | 5 +- tools/crd-bumper/pkg/vendoring.py | 192 +++++++++++++++++++++++++++++ tools/crd-bumper/vendor-new-api.py | 176 ++++++++++++++++++++++++++ 5 files changed, 465 insertions(+), 2 deletions(-) create mode 100644 tools/crd-bumper/pkg/go_cli.py create mode 100644 tools/crd-bumper/pkg/vendoring.py create mode 100755 tools/crd-bumper/vendor-new-api.py diff --git a/tools/crd-bumper/README.md b/tools/crd-bumper/README.md index 7821f87..64d02a7 100644 --- a/tools/crd-bumper/README.md +++ b/tools/crd-bumper/README.md @@ -57,6 +57,23 @@ crd-bumper.py --repo $REPO --most-recent-spoke v1alpha1 --prev-ver v1beta1 --new Do not attempt to run `make vet` or `make test` between steps. The individual commits do not build. +## Vendor the New API + +Vendor this new API into another repository using the `vendor-new-api` tool. This tool will update that repo to change its Go code references and its Kustomize config references to point at the new API. + +### Executing the Tool + +The following example will vendor the new `v1beta2` API we created above for lustre-fs-operator into the nnf-sos repository. The module representing lustre-fs-operator is specified in the same form that it would appear in the `go.mod` file in nnf-sos. It begins by creating a new branch in nnf-sos off "master" named `api-lustre-fs-operator-v1beta2`, where it will do all of its work. + +```console +DEST_REPO=git@github.com:NearNodeFlash/nnf-sos.git +vendor-new-api.py -r $DEST_REPO --hub-ver v1beta2 --module github.com/NearNodeFlash/lustre-fs-operator +``` + +The repository with its new API will be found under a directory named `workingspace/nnf-sos`. + +The new `api-lustre-fs-operator-v1beta2` branch will have a commit containing the newly-vendored API and adjusted code. This commit message will have **ACTION** comments describing something that must be manually verified, and possibly adjusted, before the tests will succeed. + ## Library and Tool Support The library and tool support is taken from the [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. See [release v1.6.6](https://github.com/kubernetes-sigs/cluster-api/tree/release-1.6) for a version that contains multi-version support for CRDs where they have a hub with one spoke. (Note: In v1.7.0 they removed the old API--the old spoke--and their repo contains only one version, the hub.) diff --git a/tools/crd-bumper/pkg/go_cli.py b/tools/crd-bumper/pkg/go_cli.py new file mode 100644 index 0000000..639e98d --- /dev/null +++ b/tools/crd-bumper/pkg/go_cli.py @@ -0,0 +1,77 @@ +# Copyright 2024 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shlex +import subprocess + + +class GoCLI: + """Go Commands""" + + def __init__(self, dryrun): + self._dryrun = dryrun + + def get(self, module, ver): + """Execute a go-get.""" + + cmd = f"go get {module}@{ver}" + if self._dryrun: + print(f"Dryrun: {cmd}") + else: + print(f"Run: {cmd}") + res = subprocess.run( + shlex.split(cmd), + capture_output=True, + text=True, + check=False, + ) + if res.returncode != 0: + raise RuntimeError(f"Failure in command: {res.stderr}") + + def tidy(self): + """Execute a go-mod-tidy.""" + + cmd = "go mod tidy" + if self._dryrun: + print(f"Dryrun: {cmd}") + else: + print(f"Run: {cmd}") + res = subprocess.run( + shlex.split(cmd), + capture_output=True, + text=True, + check=False, + ) + if res.returncode != 0: + raise RuntimeError(f"Failure in command: {res.stderr}") + + def vendor(self): + """Execute a go-mod-vendor.""" + + cmd = "go mod vendor" + if self._dryrun: + print(f"Dryrun: {cmd}") + else: + print(f"Run: {cmd}") + res = subprocess.run( + shlex.split(cmd), + capture_output=True, + text=True, + check=False, + ) + if res.returncode != 0: + raise RuntimeError(f"Failure in command: {res.stderr}") diff --git a/tools/crd-bumper/pkg/make_cmd.py b/tools/crd-bumper/pkg/make_cmd.py index 02ec347..8cec468 100644 --- a/tools/crd-bumper/pkg/make_cmd.py +++ b/tools/crd-bumper/pkg/make_cmd.py @@ -27,8 +27,6 @@ class MakeCmd: """Run make commands or updating the Makefile.""" def __init__(self, dryrun, project, prev_ver, new_ver): - if not isinstance(project, Project): - raise TypeError("need a Project") self._dryrun = dryrun self._project = project self._prev_ver = prev_ver @@ -87,6 +85,9 @@ def generate(self): def generate_go_conversions(self): """Execute 'make generate-go-conversions'""" + fu = FileUtil(self._dryrun, "Makefile") + if fu.find_with_pattern("^generate-go-conversions:") is None: + return cmd = "make generate-go-conversions" if self._dryrun: print(f"Dryrun: {cmd}") diff --git a/tools/crd-bumper/pkg/vendoring.py b/tools/crd-bumper/pkg/vendoring.py new file mode 100644 index 0000000..becaaf0 --- /dev/null +++ b/tools/crd-bumper/pkg/vendoring.py @@ -0,0 +1,192 @@ +# Copyright 2024 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +from .fileutil import FileUtil + + +class Vendor: + """Tools for vendoring a new API version.""" + + def __init__(self, dryrun, module, hub_ver): + self._dryrun = dryrun + self._module = module + self._hub_ver = hub_ver + self._current_ver = None + self._preferred_alias = None + + def current_api_version(self): + """Return the current in-use API version.""" + return self._current_ver + + def uses_module(self): + """Determine whether the specified module is vendored here.""" + path = "go.mod" + if not os.path.isfile(path): + return False + fu = FileUtil(self._dryrun, path) + line = fu.find_with_pattern(f"^\t{self._module} v") + if line is None: + return False + return True + + def set_current_api_version(self): + """Determine the version of the API currently being vendored.""" + + path = f"vendor/{self._module}/api" + for _, dir_names, _ in os.walk(path, followlinks=False): + if len(dir_names) == 1: + # Only one API vendored here. + self._current_ver = dir_names[0] + break + raise ValueError( + f"Expected to find one API at {path}, but found {len(dir_names)}." + ) + if self._current_ver is None: + raise ValueError(f"Unable to find API at {path}.") + + def verify_one_api_version(self): + """Verify that only one API version is being vendored.""" + + self._current_ver = None + self.set_current_api_version() + + def set_preferred_api_alias(self): + """ + What is this repo using as the alias for this module's API? + + In other words, which of the following import statements is preferred in + this module, where the group, unfortunately, is "dataworkflowservices"? + + dwsv1alpha1 "github.com/DataWorkflowServices/dws/api/v1alpha" + dataworkflowservicesv1alpha1 "github.com/DataWorkflowServices/dws/api/v1alpha" + + We'll look at cmd/main.go to get an answer. + """ + + fname = "cmd/main.go" + fu = FileUtil(self._dryrun, fname) + # Find the import. + line = fu.find_in_file( + f'{self._current_ver} "{self._module}/api/{self._current_ver}"' + ) + if line is not None: + pat = rf'^\s+(.+){self._current_ver}\s+"{self._module}/api/{self._current_ver}"' + m = re.search(pat, line) + if m is not None: + self._preferred_alias = m.group(1) + if self._preferred_alias is None: + raise ValueError(f"Expected to find the module's alias in {fname}.") + + def update_go_files(self, top=None): + """Walk over Go files, bumping them to point at the new hub + If top=None then this walks over the cmd/, internal/, and api/ directories + that kubebuilder would have put in place. + """ + + if top is not None: + if os.path.isdir(top) is False: + raise NotADirectoryError(f"{top} is not a directory.") + top = [top] + else: + top = ["cmd", "api", "internal/controller", "controllers"] + + for dname in top: + if os.path.isdir(dname): + self._walk_go_files(dname) + + def _walk_go_files(self, dirname): + """Walk the files in the given directory, and update them to point at the new hub.""" + + if os.path.isdir(dirname) is False: + raise NotADirectoryError(f"{dirname} is not a directory") + + for root, _, f_names in os.walk(dirname, followlinks=False): + for fname in f_names: + full_path = os.path.join(root, fname) + if fname.endswith(".go"): + self._point_go_files_at_new_hub(full_path) + + def _point_go_files_at_new_hub(self, path): + """Update the given file to point it at the new hub.""" + + fu = FileUtil(self._dryrun, path) + group = self._preferred_alias + + # Find the import. + pat = f'{group}{self._current_ver} "{self._module}/api/{self._current_ver}"' + line = fu.find_in_file(pat) + if line is not None: + # Rewrite the import statement. + # Before: '\tdwsv1alpha1 "github.com/hewpack/dws/api/v1alpha1"' + # After: '\tdwsv1alpha2 "github.com/hewpack/dws/api/v1alpha2"' + line2 = line.replace(self._current_ver, self._hub_ver) + fu.replace_in_file(line, line2) + # This matches: dwsv1alpha1. (yes, dot) + fu.replace_in_file(f"{group}{self._current_ver}.", f"{group}{self._hub_ver}.") + fu.store() + + def update_config_files(self, top): + """Walk over Kustomize config files, bumping them to point at the new hub.""" + + if os.path.isdir(top) is False: + raise NotADirectoryError(f"{top} is not a directory.") + + self._walk_config_files(top) + + def _walk_config_files(self, dirname): + """Walk the files in the given directory, and update them to point at the new hub.""" + + if os.path.isdir(dirname) is False: + raise NotADirectoryError(f"{dirname} is not a directory") + + for root, _, f_names in os.walk(dirname, followlinks=False): + for fname in f_names: + full_path = os.path.join(root, fname) + if fname.endswith(".yaml"): + self._point_config_files_at_new_hub(full_path) + + def _point_config_files_at_new_hub(self, path): + """Update the given file to point it at the new hub.""" + + fu = FileUtil(self._dryrun, path) + group = self._preferred_alias + + pat = f"apiVersion: {group}" + line = fu.find_in_file(pat) + if line is not None: + line2 = line.replace(self._current_ver, self._hub_ver) + fu.replace_in_file(line, line2) + fu.store() + + def commit(self, git, stage): + """Create a commit message.""" + + msg = f"""Vendor {self._hub_ver} API from {self._module}. + +ACTION: If any of the code in this repo was referencing non-local + APIs, the references to them may have been inadvertently + modified. Verify that any non-local APIs are being referenced + by their correct versions. + +ACTION: Begin by running "make vet". Repair any issues that it finds. + Then run "make test" and continue repairing issues until the tests + pass. +""" + git.commit_stage(stage, msg) diff --git a/tools/crd-bumper/vendor-new-api.py b/tools/crd-bumper/vendor-new-api.py new file mode 100755 index 0000000..013cf92 --- /dev/null +++ b/tools/crd-bumper/vendor-new-api.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Hewlett Packard Enterprise Development LP +# Other additional copyright holders may be indicated within. +# +# The entirety of this work is licensed under the Apache License, +# Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. +# +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Vendor the specified CRD version, updating Go code to point at it.""" + +import argparse +import os +import sys +import yaml + +from pkg.git_cli import GitCLI +from pkg.make_cmd import MakeCmd +from pkg.vendoring import Vendor +from pkg.go_cli import GoCLI + +WORKING_DIR = "workingspace" + +PARSER = argparse.ArgumentParser() +PARSER.add_argument( + "--hub-ver", + type=str, + required=True, + help="Version of the hub API.", +) +PARSER.add_argument( + "--module", + "-m", + type=str, + required=True, + help="Go module which has the versioned API, specified the way it is found in go.mod.", +) +PARSER.add_argument( + "--repo", + "-r", + type=str, + required=True, + help="Git repository URL which has the Go code that consumes the APIs.", +) +PARSER.add_argument( + "--branch", + "-b", + type=str, + required=False, + help="Branch name to create. Default is 'api--'", +) +PARSER.add_argument( + "--this-branch", + action="store_true", + dest="this_branch", + help="Continue working in the current branch. Use when stepping through with 'step'.", +) +PARSER.add_argument( + "--dry-run", + "-n", + action="store_true", + dest="dryrun", + help="Dry run. Implies only one step.", +) +PARSER.add_argument( + "--no-commit", + "-C", + action="store_true", + dest="nocommit", + help="Skip git-commit. Implies only one step.", +) +PARSER.add_argument( + "--workdir", + type=str, + required=False, + default=WORKING_DIR, + help=f"Name for working directory. All repos will be cloned below this directory. Default: {WORKING_DIR}.", +) + + +def main(): + """main""" + + args = PARSER.parse_args() + + gitcli = GitCLI(args.dryrun, args.nocommit) + gitcli.clone_and_cd(args.repo, args.workdir) + + gocli = GoCLI(args.dryrun) + + # Load any repo-specific local config. + bumper_cfg = None + if os.path.isfile("crd-bumper.yaml"): + with open("crd-bumper.yaml", "r", encoding="utf-8") as fi: + bumper_cfg = yaml.safe_load(fi) + + makecmd = MakeCmd(args.dryrun, None, None, None) + + if args.branch is None: + bn = os.path.basename(args.module) + args.branch = f"api-{bn}-{args.hub_ver}" + if args.this_branch: + print("Continuing work in current branch") + else: + print(f"Creating branch {args.branch}") + try: + gitcli.checkout_branch(args.branch) + except RuntimeError as ex: + print(str(ex)) + print( + "If you are continuing in an existing branch, then specify `--this-branch`." + ) + sys.exit(1) + + vendor_new_api(args, makecmd, gitcli, gocli, bumper_cfg) + + +def vendor_new_api(args, makecmd, git, gocli, bumper_cfg): + """Vendor the new API into the repo.""" + + vendor = Vendor(args.dryrun, args.module, args.hub_ver) + + if vendor.uses_module() is False: + print( + f"Module {args.module} is not found in go.mod in {args.repo}. Nothing to do." + ) + sys.exit(0) + try: + vendor.set_current_api_version() + vendor.set_preferred_api_alias() + except ValueError as ex: + print(str(ex)) + sys.exit(1) + + print(f"Updating files from {vendor.current_api_version()} to {args.hub_ver}") + + # Update the Go files that are in the usual kubebuilder locations. + vendor.update_go_files() + + # Bump any other, non-controller, directories of code. + if bumper_cfg is not None and "extra_go_dirs" in bumper_cfg: + for extra_dir in bumper_cfg["extra_go_dirs"].split(","): + vendor.update_go_files(extra_dir) + # Bump any necessary references in the config/ dir. + if bumper_cfg is not None and "extra_config_dirs" in bumper_cfg: + for extra_dir in bumper_cfg["extra_config_dirs"].split(","): + vendor.update_config_files(extra_dir) + + gocli.get(args.module, "master") + gocli.tidy() + gocli.vendor() + vendor.verify_one_api_version() + + makecmd.manifests() + makecmd.generate() + makecmd.generate_go_conversions() + makecmd.fmt() + makecmd.clean_bin() + + vendor.commit(git, "vendor-new-api") + + +if __name__ == "__main__": + main() + +sys.exit(0)