A tool to vendor a new API into a consumer repo (#209)

The vendor-new-api.py tool is a companion tool to crd-bumper. After crd-bumper
has been used to create a new API version, the vendor-new-api tool can be used
to vendor that into other repositories that consume the API.

Signed-off-by: Dean Roehrich <[email protected]>

tools/crd-bumper/README.md

@@ -57,6 +57,23 @@ crd-bumper.py --repo $REPO --most-recent-spoke v1alpha1 --prev-ver v1beta1 --new
 
 Do not attempt to run `make vet` or `make test` between steps. The individual commits do not build.
 
+## Vendor the New API
+
+Vendor this new API into another repository using the `vendor-new-api` tool. This tool will update that repo to change its Go code references and its Kustomize config references to point at the new API.
+
+### Executing the Tool
+
+The following example will vendor the new `v1beta2` API we created above for lustre-fs-operator into the nnf-sos repository. The module representing lustre-fs-operator is specified in the same form that it would appear in the `go.mod` file in nnf-sos. It begins by creating a new branch in nnf-sos off "master" named `api-lustre-fs-operator-v1beta2`, where it will do all of its work.
+
+```console
+[email protected]:NearNodeFlash/nnf-sos.git
+vendor-new-api.py -r $DEST_REPO --hub-ver v1beta2 --module github.com/NearNodeFlash/lustre-fs-operator
+```
+
+The repository with its new API will be found under a directory named `workingspace/nnf-sos`.
+
+The new `api-lustre-fs-operator-v1beta2` branch will have a commit containing the newly-vendored API and adjusted code. This commit message will have **ACTION** comments describing something that must be manually verified, and possibly adjusted, before the tests will succeed.
+
 ## Library and Tool Support
 
 The library and tool support is taken from the [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. See [release v1.6.6](https://github.com/kubernetes-sigs/cluster-api/tree/release-1.6) for a version that contains multi-version support for CRDs where they have a hub with one spoke. (Note: In v1.7.0 they removed the old API--the old spoke--and their repo contains only one version, the hub.)

tools/crd-bumper/pkg/go_cli.py

@@ -0,0 +1,77 @@
+# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import shlex
+import subprocess
+
+
+class GoCLI:
+    """Go Commands"""
+
+    def __init__(self, dryrun):
+        self._dryrun = dryrun
+
+    def get(self, module, ver):
+        """Execute a go-get."""
+
+        cmd = f"go get {module}@{ver}"
+        if self._dryrun:
+            print(f"Dryrun: {cmd}")
+        else:
+            print(f"Run: {cmd}")
+            res = subprocess.run(
+                shlex.split(cmd),
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            if res.returncode != 0:
+                raise RuntimeError(f"Failure in command: {res.stderr}")
+
+    def tidy(self):
+        """Execute a go-mod-tidy."""
+
+        cmd = "go mod tidy"
+        if self._dryrun:
+            print(f"Dryrun: {cmd}")
+        else:
+            print(f"Run: {cmd}")
+            res = subprocess.run(
+                shlex.split(cmd),
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            if res.returncode != 0:
+                raise RuntimeError(f"Failure in command: {res.stderr}")
+
+    def vendor(self):
+        """Execute a go-mod-vendor."""
+
+        cmd = "go mod vendor"
+        if self._dryrun:
+            print(f"Dryrun: {cmd}")
+        else:
+            print(f"Run: {cmd}")
+            res = subprocess.run(
+                shlex.split(cmd),
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            if res.returncode != 0:
+                raise RuntimeError(f"Failure in command: {res.stderr}")

tools/crd-bumper/pkg/make_cmd.py

@@ -27,8 +27,6 @@ class MakeCmd:
     """Run make commands or updating the Makefile."""
 
     def __init__(self, dryrun, project, prev_ver, new_ver):
-        if not isinstance(project, Project):
-            raise TypeError("need a Project")
         self._dryrun = dryrun
         self._project = project
         self._prev_ver = prev_ver
@@ -87,6 +85,9 @@ def generate(self):
     def generate_go_conversions(self):
         """Execute 'make generate-go-conversions'"""
 
+        fu = FileUtil(self._dryrun, "Makefile")
+        if fu.find_with_pattern("^generate-go-conversions:") is None:
+            return
         cmd = "make generate-go-conversions"
         if self._dryrun:
             print(f"Dryrun: {cmd}")

tools/crd-bumper/pkg/vendoring.py

@@ -0,0 +1,192 @@
+# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+
+from .fileutil import FileUtil
+
+
+class Vendor:
+    """Tools for vendoring a new API version."""
+
+    def __init__(self, dryrun, module, hub_ver):
+        self._dryrun = dryrun
+        self._module = module
+        self._hub_ver = hub_ver
+        self._current_ver = None
+        self._preferred_alias = None
+
+    def current_api_version(self):
+        """Return the current in-use API version."""
+        return self._current_ver
+
+    def uses_module(self):
+        """Determine whether the specified module is vendored here."""
+        path = "go.mod"
+        if not os.path.isfile(path):
+            return False
+        fu = FileUtil(self._dryrun, path)
+        line = fu.find_with_pattern(f"^\t{self._module} v")
+        if line is None:
+            return False
+        return True
+
+    def set_current_api_version(self):
+        """Determine the version of the API currently being vendored."""
+
+        path = f"vendor/{self._module}/api"
+        for _, dir_names, _ in os.walk(path, followlinks=False):
+            if len(dir_names) == 1:
+                # Only one API vendored here.
+                self._current_ver = dir_names[0]
+                break
+            raise ValueError(
+                f"Expected to find one API at {path}, but found {len(dir_names)}."
+            )
+        if self._current_ver is None:
+            raise ValueError(f"Unable to find API at {path}.")
+
+    def verify_one_api_version(self):
+        """Verify that only one API version is being vendored."""
+
+        self._current_ver = None
+        self.set_current_api_version()
+
+    def set_preferred_api_alias(self):
+        """
+        What is this repo using as the alias for this module's API?
+
+        In other words, which of the following import statements is preferred in
+        this module, where the group, unfortunately, is "dataworkflowservices"?
+
+           dwsv1alpha1 "github.com/DataWorkflowServices/dws/api/v1alpha"
+           dataworkflowservicesv1alpha1 "github.com/DataWorkflowServices/dws/api/v1alpha"
+
+        We'll look at cmd/main.go to get an answer.
+        """
+
+        fname = "cmd/main.go"
+        fu = FileUtil(self._dryrun, fname)
+        # Find the import.
+        line = fu.find_in_file(
+            f'{self._current_ver} "{self._module}/api/{self._current_ver}"'
+        )
+        if line is not None:
+            pat = rf'^\s+(.+){self._current_ver}\s+"{self._module}/api/{self._current_ver}"'
+            m = re.search(pat, line)
+            if m is not None:
+                self._preferred_alias = m.group(1)
+        if self._preferred_alias is None:
+            raise ValueError(f"Expected to find the module's alias in {fname}.")
+
+    def update_go_files(self, top=None):
+        """Walk over Go files, bumping them to point at the new hub
+        If top=None then this walks over the cmd/, internal/, and api/ directories
+        that kubebuilder would have put in place.
+        """
+
+        if top is not None:
+            if os.path.isdir(top) is False:
+                raise NotADirectoryError(f"{top} is not a directory.")
+            top = [top]
+        else:
+            top = ["cmd", "api", "internal/controller", "controllers"]
+
+        for dname in top:
+            if os.path.isdir(dname):
+                self._walk_go_files(dname)
+
+    def _walk_go_files(self, dirname):
+        """Walk the files in the given directory, and update them to point at the new hub."""
+
+        if os.path.isdir(dirname) is False:
+            raise NotADirectoryError(f"{dirname} is not a directory")
+
+        for root, _, f_names in os.walk(dirname, followlinks=False):
+            for fname in f_names:
+                full_path = os.path.join(root, fname)
+                if fname.endswith(".go"):
+                    self._point_go_files_at_new_hub(full_path)
+
+    def _point_go_files_at_new_hub(self, path):
+        """Update the given file to point it at the new hub."""
+
+        fu = FileUtil(self._dryrun, path)
+        group = self._preferred_alias
+
+        # Find the import.
+        pat = f'{group}{self._current_ver} "{self._module}/api/{self._current_ver}"'
+        line = fu.find_in_file(pat)
+        if line is not None:
+            # Rewrite the import statement.
+            # Before: '\tdwsv1alpha1 "github.com/hewpack/dws/api/v1alpha1"'
+            # After:  '\tdwsv1alpha2 "github.com/hewpack/dws/api/v1alpha2"'
+            line2 = line.replace(self._current_ver, self._hub_ver)
+            fu.replace_in_file(line, line2)
+        # This matches: dwsv1alpha1. (yes, dot)
+        fu.replace_in_file(f"{group}{self._current_ver}.", f"{group}{self._hub_ver}.")
+        fu.store()
+
+    def update_config_files(self, top):
+        """Walk over Kustomize config files, bumping them to point at the new hub."""
+
+        if os.path.isdir(top) is False:
+            raise NotADirectoryError(f"{top} is not a directory.")
+
+        self._walk_config_files(top)
+
+    def _walk_config_files(self, dirname):
+        """Walk the files in the given directory, and update them to point at the new hub."""
+
+        if os.path.isdir(dirname) is False:
+            raise NotADirectoryError(f"{dirname} is not a directory")
+
+        for root, _, f_names in os.walk(dirname, followlinks=False):
+            for fname in f_names:
+                full_path = os.path.join(root, fname)
+                if fname.endswith(".yaml"):
+                    self._point_config_files_at_new_hub(full_path)
+
+    def _point_config_files_at_new_hub(self, path):
+        """Update the given file to point it at the new hub."""
+
+        fu = FileUtil(self._dryrun, path)
+        group = self._preferred_alias
+
+        pat = f"apiVersion: {group}"
+        line = fu.find_in_file(pat)
+        if line is not None:
+            line2 = line.replace(self._current_ver, self._hub_ver)
+            fu.replace_in_file(line, line2)
+        fu.store()
+
+    def commit(self, git, stage):
+        """Create a commit message."""
+
+        msg = f"""Vendor {self._hub_ver} API from {self._module}.
+
+ACTION: If any of the code in this repo was referencing non-local
+  APIs, the references to them may have been inadvertently
+  modified. Verify that any non-local APIs are being referenced
+  by their correct versions.
+
+ACTION: Begin by running "make vet". Repair any issues that it finds.
+  Then run "make test" and continue repairing issues until the tests
+  pass.
+"""
+        git.commit_stage(stage, msg)