Allow vendoring into nnf-integration-test repo (#231)

Adjust vendor-new-api.py so it can vendor into the nnf-integration-test repo.
This relies on a new crd-bumper.yaml file in that repo to give some hints.

Refactor code that checks whether an API is a hub or a spoke.

Refactor code that references the crd-bumper.yaml file. This simplifies the
use of the object that holds the content of that file.

Reformat code with 'black'.

Signed-off-by: Dean Roehrich <[email protected]>

tools/crd-bumper/crd-bumper.py

@@ -31,6 +31,7 @@
 from pkg.git_cli import GitCLI
 from pkg.conversion_gen import ConversionGen
 from pkg.make_cmd import MakeCmd
+from pkg.hub_spoke_util import HubSpokeUtil
 
 WORKING_DIR = "workingspace"
 
@@ -148,7 +149,7 @@ def main():
     project = Project(args.dryrun)
 
     # Load any repo-specific local config.
-    bumper_cfg = None
+    bumper_cfg = {}
     if os.path.isfile("crd-bumper.yaml"):
         with open("crd-bumper.yaml", "r", encoding="utf-8") as fi:
             bumper_cfg = yaml.safe_load(fi)
@@ -158,7 +159,7 @@ def main():
     )
     makecmd = MakeCmd(args.dryrun, project, args.prev_ver, args.new_ver)
 
-    validate_args(args, cgen)
+    validate_args(args)
 
     if args.branch is None:
         args.branch = f"api-{args.new_ver}"
@@ -335,11 +336,11 @@ def bump_controllers(cgen, makecmd, git, stage, project, args, bumper_cfg):
     controllers.edit_util_conversion_test()
 
     # Bump any other, non-controller, directories of code.
-    if bumper_cfg is not None and "extra_go_dirs" in bumper_cfg:
+    if "extra_go_dirs" in bumper_cfg:
         for extra_dir in bumper_cfg["extra_go_dirs"].split(","):
             controllers.update_extras(extra_dir)
     # Bump any necessary references in the config/ dir.
-    if bumper_cfg is not None and "extra_config_dirs" in bumper_cfg:
+    if "extra_config_dirs" in bumper_cfg:
         for extra_dir in bumper_cfg["extra_config_dirs"].split(","):
             controllers.update_extra_config(extra_dir)
 
@@ -423,11 +424,11 @@ def prologue(git, cmd_elem, operation_order):
     git.expect_previous(cmd_elem[0], op_cmd_list)
 
 
-def validate_args(args, cgen):
+def validate_args(args):
     """Validate the commandline args"""
 
     if args.most_recent_spoke:
-        if not cgen.is_spoke(args.most_recent_spoke):
+        if not HubSpokeUtil.is_spoke(args.most_recent_spoke):
             print(f"API --most-recent-spoke {args.most_recent_spoke} is not a spoke.")
             sys.exit(1)
         if args.most_recent_spoke in [args.prev_ver, args.new_ver]:

tools/crd-bumper/pkg/conversion_gen.py

@@ -38,19 +38,6 @@ def __init__(self, dryrun, project, prev_ver, new_ver, most_recent_spoke):
         self.module()
         self._preferred_alias = self.set_preferred_api_alias()
 
-    def is_spoke(self, ver):
-        """
-        Determine whether or not the 'ver' API is a spoke.
-        """
-        path = f"api/{ver}/conversion.go"
-        if not os.path.isfile(path):
-            return False
-        fu = FileUtil(self._dryrun, path)
-        line = fu.find_in_file(" ConvertTo(dstRaw conversion.Hub) ")
-        if line is None:
-            return False
-        return True
-
     def preferred_api_alias(self):
         """Return the preferred alias."""
         return self._preferred_alias

tools/crd-bumper/pkg/create_apis.py

@@ -22,6 +22,7 @@
 
 from .project import Project
 from .fileutil import FileUtil
+from .hub_spoke_util import HubSpokeUtil
 
 
 class CreateApis:
@@ -45,14 +46,7 @@ def prev_is_hub(self):
 
         for _, dir_names, _ in os.walk("api", followlinks=False):
             if len(dir_names) > 1:
-                path = f"api/{self._prev_ver}/conversion.go"
-                if not os.path.isfile(path):
-                    return False
-                fu = FileUtil(self._dryrun, path)
-                line = fu.find_in_file(" Hub() ")
-                if line is None:
-                    return False
-                break
+                return HubSpokeUtil.is_hub(self._prev_ver)
         return True
 
     def create(self):

tools/crd-bumper/pkg/hub_spoke_util.py

@@ -0,0 +1,52 @@
+# Copyright 2024 Hewlett Packard Enterprise Development LP
+# Other additional copyright holders may be indicated within.
+#
+# The entirety of this work is licensed under the Apache License,
+# Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.
+#
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from .fileutil import FileUtil
+
+
+class HubSpokeUtil:
+    """Simple utilities related to hub and spokes."""
+
+    @staticmethod
+    def is_spoke(ver):
+        """
+        Determine whether or not the 'ver' API is a spoke.
+        """
+        path = f"api/{ver}/conversion.go"
+        if not os.path.isfile(path):
+            return False
+        fu = FileUtil(False, path)
+        line = fu.find_in_file(" ConvertTo(dstRaw conversion.Hub) ")
+        if line is None:
+            return False
+        return True
+
+    @staticmethod
+    def is_hub(ver):
+        """
+        Determine whether or not the 'ver' API is a hub.
+        """
+        path = f"api/{ver}/conversion.go"
+        if not os.path.isfile(path):
+            return False
+        fu = FileUtil(False, path)
+        line = fu.find_in_file(" Hub() ")
+        if line is None:
+            return False
+        return True

tools/crd-bumper/pkg/vendoring.py

@@ -68,7 +68,7 @@ def verify_one_api_version(self):
         self._current_ver = None
         self.set_current_api_version()
 
-    def set_preferred_api_alias(self):
+    def set_preferred_api_alias(self, main_file=None):
         """
         What is this repo using as the alias for this module's API?
 
@@ -81,7 +81,10 @@ def set_preferred_api_alias(self):
         We'll look at cmd/main.go to get an answer.
         """
 
-        fname = "cmd/main.go"
+        if main_file is not None:
+            fname = main_file
+        else:
+            fname = "cmd/main.go"
         fu = FileUtil(self._dryrun, fname)
         # Find the import.
         line = fu.find_in_file(
@@ -95,6 +98,10 @@ def set_preferred_api_alias(self):
         if self._preferred_alias is None:
             raise ValueError(f"Expected to find the module's alias in {fname}.")
 
+    def update_go_file(self, full_path):
+        """Bump the given Go file to point at the new hub."""
+        self._point_go_files_at_new_hub(full_path)
+
     def update_go_files(self, top=None):
         """Walk over Go files, bumping them to point at the new hub
         If top=None then this walks over the cmd/, internal/, and api/ directories
@@ -106,7 +113,9 @@ def update_go_files(self, top=None):
                 raise NotADirectoryError(f"{top} is not a directory.")
             top = [top]
         else:
-            top = ["cmd", "api/" + self._hub_ver, "internal/controller", "controllers"]
+            top = ["cmd", "internal/controller", "controllers"]
+            if self._hub_ver is not None:
+                top.append("api/" + self._hub_ver)
 
         for dname in top:
             if os.path.isdir(dname):
@@ -140,7 +149,9 @@ def _point_go_files_at_new_hub(self, path):
             line2 = line.replace(self._current_ver, self._vendor_hub_ver)
             fu.replace_in_file(line, line2)
         # This matches: dwsv1alpha1. (yes, dot)
-        fu.replace_in_file(f"{group}{self._current_ver}.", f"{group}{self._vendor_hub_ver}.")
+        fu.replace_in_file(
+            f"{group}{self._current_ver}.", f"{group}{self._vendor_hub_ver}."
+        )
         fu.store()
 
     def update_config_files(self, top):

tools/crd-bumper/unserve.py

@@ -27,6 +27,7 @@
 from pkg.make_cmd import MakeCmd
 from pkg.project import Project
 from pkg.unserve import Unserve
+from pkg.hub_spoke_util import HubSpokeUtil
 
 WORKING_DIR = "workingspace"
 BRANCH_SUFFIX = "unserve"
@@ -109,7 +110,7 @@ def main():
             )
             sys.exit(1)
 
-    if not cgen.is_spoke(args.spoke_ver):
+    if not HubSpokeUtil.is_spoke(args.spoke_ver):
         print(f"API --spoke-ver {args.spoke_ver} is not a spoke.")
         sys.exit(1)
 

tools/crd-bumper/vendor-new-api.py

@@ -24,19 +24,21 @@
 import sys
 import yaml
 
+from pkg.conversion_gen import ConversionGen
 from pkg.git_cli import GitCLI
 from pkg.make_cmd import MakeCmd
 from pkg.vendoring import Vendor
 from pkg.go_cli import GoCLI
+from pkg.hub_spoke_util import HubSpokeUtil
 
 WORKING_DIR = "workingspace"
 
 PARSER = argparse.ArgumentParser()
 PARSER.add_argument(
     "--hub-ver",
     type=str,
-    required=True,
-    help="Version of the hub API.",
+    required=False,
+    help="Version of the hub API of the target repo.",
 )
 PARSER.add_argument(
     "--vendor-hub-ver",
@@ -77,7 +79,7 @@
     "--this-branch",
     action="store_true",
     dest="this_branch",
-    help="Continue working in the current branch. Use when stepping through with 'step'.",
+    help="Continue working in the current branch.",
 )
 PARSER.add_argument(
     "--dry-run",
@@ -112,8 +114,13 @@ def main():
 
     gocli = GoCLI(args.dryrun)
 
+    if args.hub_ver:
+        if not HubSpokeUtil.is_hub(args.hub_ver):
+            print(f"API --hub-ver {args.hub_ver} is not a hub.")
+            sys.exit(1)
+
     # Load any repo-specific local config.
-    bumper_cfg = None
+    bumper_cfg = {}
     if os.path.isfile("crd-bumper.yaml"):
         with open("crd-bumper.yaml", "r", encoding="utf-8") as fi:
             bumper_cfg = yaml.safe_load(fi)
@@ -150,35 +157,55 @@ def vendor_new_api(args, makecmd, git, gocli, bumper_cfg):
         )
         sys.exit(0)
     try:
+        if not os.path.isdir("vendor"):
+            # This repo doesn't normally vendor, so go get it.
+            # The .gitignore file should already be covering it.
+            gocli.tidy()
+            gocli.vendor()
         vendor.set_current_api_version()
-        vendor.set_preferred_api_alias()
+        main_file = None
+        if "alternate_main" in bumper_cfg:
+            main_file = bumper_cfg["alternate_main"]
+        vendor.set_preferred_api_alias(main_file)
     except ValueError as ex:
         print(str(ex))
         sys.exit(1)
 
-    print(f"Updating files from {vendor.current_api_version()} to {args.hub_ver}")
+    print(
+        f"Updating files from {vendor.current_api_version()} to {args.vendor_hub_ver}"
+    )
 
     # Update the Go files that are in the usual kubebuilder locations.
     vendor.update_go_files()
 
     # Bump any other, non-controller, directories of code.
-    if bumper_cfg is not None and "extra_go_dirs" in bumper_cfg:
+    if "extra_go_dirs" in bumper_cfg:
         for extra_dir in bumper_cfg["extra_go_dirs"].split(","):
             vendor.update_go_files(extra_dir)
+    if "extra_go_files" in bumper_cfg:
+        for full_path in bumper_cfg["extra_go_files"].split(","):
+            vendor.update_go_file(full_path)
     # Bump any necessary references in the config/ dir.
-    if bumper_cfg is not None and "extra_config_dirs" in bumper_cfg:
+    if "extra_config_dirs" in bumper_cfg:
         for extra_dir in bumper_cfg["extra_config_dirs"].split(","):
             vendor.update_config_files(extra_dir)
 
     gocli.get(args.module, args.version)
     gocli.tidy()
     gocli.vendor()
 
-    makecmd.manifests()
-    makecmd.generate()
-    makecmd.generate_go_conversions()
+    have_controller_gen = True
+    if "skip_controller_gen" in bumper_cfg:
+        if bumper_cfg["skip_controller_gen"] is True:
+            have_controller_gen = False
+
+    if have_controller_gen:
+        makecmd.manifests()
+        makecmd.generate()
+        makecmd.generate_go_conversions()
     makecmd.fmt()
-    makecmd.clean_bin()
+    if have_controller_gen:
+        makecmd.clean_bin()
 
     vendor.commit(git, "vendor-new-api")