Skip to content

Commit

Permalink
Implement Init plugin subcommand
Browse files Browse the repository at this point in the history
  • Loading branch information
Fedosin committed Nov 24, 2023
1 parent a09edab commit 5839bb1
Show file tree
Hide file tree
Showing 6 changed files with 5,067 additions and 14 deletions.
278 changes: 267 additions & 11 deletions cmd/plugin/cmd/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,31 @@ package cmd

import (
"context"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/go-errors/errors"
"github.com/spf13/cobra"

appsv1 "k8s.io/api/apps/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
configclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/yamlprocessor"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"

operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
"sigs.k8s.io/cluster-api-operator/internal/controller/genericprovider"
"sigs.k8s.io/cluster-api-operator/util"
)

type initOptions struct {
kubeconfig string
kubeconfigContext string
operatorVersion string
coreProvider string
bootstrapProviders []string
controlPlaneProviders []string
Expand All @@ -39,8 +56,19 @@ type initOptions struct {
waitProviderTimeout int
}

const (
capiOperatorLatestVersion = "v0.7.0"
capiOperatorProviderName = "capi-operator"
capiOperatorManifestsURL = "https://github.com/kubernetes-sigs/cluster-api-operator/releases/latest/operator-components.yaml"
)

var initOpts = &initOptions{}

var capiOperatorLabels = map[string]string{
"clusterctl.cluster.x-k8s.io/core": "capi-operator",
"control-plane": "controller-manager",
}

var initCmd = &cobra.Command{
Use: "init",
GroupID: groupManagement,
Expand Down Expand Up @@ -79,7 +107,7 @@ var initCmd = &cobra.Command{
capioperator init --kubeconfig=foo.yaml --infrastructure=aws
# Initialize a management cluster with multiple infrastructure providers.
capioperator init --infrastructure=aws;vsphere
capioperator init --infrastructure=aws --infrastructure=vsphere
# Initialize a management cluster with a custom target namespace for the operator.
capioperator init --infrastructure aws --target-namespace foo`),
Expand All @@ -90,23 +118,25 @@ var initCmd = &cobra.Command{
}

func init() {
initCmd.PersistentFlags().StringVar(&initOpts.kubeconfig, "kubeconfig", "",
initCmd.PersistentFlags().StringVar(&initOpts.kubeconfig, "kubeconfig", filepath.Join(os.Getenv("HOME"), ".kube", "config"),
"Path to the kubeconfig for the management cluster. If unspecified, default discovery rules apply.")
initCmd.PersistentFlags().StringVar(&initOpts.kubeconfigContext, "kubeconfig-context", "",
"Context to be used within the kubeconfig file. If empty, current context will be used.")
initCmd.PersistentFlags().StringVar(&initOpts.coreProvider, "core", "",
initCmd.PersistentFlags().StringVar(&initOpts.operatorVersion, "operator-version", "",
"CAPI Operator version (e.g. v0.7.0) to install on the management cluster. If unspecified, the latest release is used.")
initCmd.PersistentFlags().StringVar(&initOpts.coreProvider, "core", "cluster-api",
"Core provider version (e.g. cluster-api:v1.1.5) to add to the management cluster. If unspecified, Cluster API's latest release is used.")
initCmd.PersistentFlags().StringSliceVarP(&initOpts.infrastructureProviders, "infrastructure", "i", nil,
initCmd.PersistentFlags().StringSliceVarP(&initOpts.infrastructureProviders, "infrastructure", "i", []string{},
"Infrastructure providers and versions (e.g. aws:v0.5.0) to add to the management cluster.")
initCmd.PersistentFlags().StringSliceVarP(&initOpts.bootstrapProviders, "bootstrap", "b", nil,
initCmd.PersistentFlags().StringSliceVarP(&initOpts.bootstrapProviders, "bootstrap", "b", []string{"kubeadm"},
"Bootstrap providers and versions (e.g. kubeadm:v1.1.5) to add to the management cluster. If unspecified, Kubeadm bootstrap provider's latest release is used.")
initCmd.PersistentFlags().StringSliceVarP(&initOpts.controlPlaneProviders, "control-plane", "c", nil,
initCmd.PersistentFlags().StringSliceVarP(&initOpts.controlPlaneProviders, "control-plane", "c", []string{"kubeadm"},
"Control plane providers and versions (e.g. kubeadm:v1.1.5) to add to the management cluster. If unspecified, the Kubeadm control plane provider's latest release is used.")
// initCmd.PersistentFlags().StringSliceVar(&initOpts.ipamProviders, "ipam", nil,
// "IPAM providers and versions (e.g. infoblox:v0.0.1) to add to the management cluster.")
// initCmd.PersistentFlags().StringSliceVar(&initOpts.runtimeExtensionProviders, "runtime-extension", nil,
// "Runtime extension providers and versions (e.g. test:v0.0.1) to add to the management cluster.")
initCmd.PersistentFlags().StringSliceVar(&initOpts.addonProviders, "addon", nil,
initCmd.PersistentFlags().StringSliceVar(&initOpts.addonProviders, "addon", []string{},
"Add-on providers and versions (e.g. helm:v0.1.0) to add to the management cluster.")
initCmd.Flags().StringVarP(&initOpts.targetNamespace, "target-namespace", "n", "capi-operator-system",
"The target namespace where the operator should be deployed. If unspecified, the 'capi-operator-system' namespace is used.")
Expand All @@ -123,9 +153,235 @@ func init() {
func runInit() error {
ctx := context.Background()

return initProvider(ctx, initOpts)
return initProviders(ctx, initOpts)
}

func initProviders(ctx context.Context, opts *initOptions) error {
client, err := CreateKubeClient(initOpts.kubeconfig, initOpts.kubeconfigContext)
if err != nil {
return fmt.Errorf("cannot create a client: %w", err)
}

// Checking if CAPI operator deployment exists.
deploymentExists, err := checkCAPIOpearatorAvailability(ctx, client)
if err != nil {
return fmt.Errorf("cannot check CAPI operator availability: %w", err)
}

if deploymentExists && initOpts.operatorVersion != "" {
return fmt.Errorf("cannot specify an operator version when the CAPI operator is already installed")
}

// Ensure that cert manager is installed.
if err := ensureCertManager(opts); err != nil {
return fmt.Errorf("cannot ensure that cert manager is installed: %w", err)
}

// Deploy CAPI operator if it doesn't exist.
if !deploymentExists {
if err := deployCAPIOperator(opts); err != nil {
return fmt.Errorf("cannot deploy CAPI operator: %w", err)
}
}

// Deploy Core Provider.
if err := createGenericProvider(ctx, client, newGenericProvider(clusterctlv1.CoreProviderType), opts.coreProvider, opts.targetNamespace); err != nil {
return fmt.Errorf("cannot create core provider: %w", err)
}

// Deploy Bootstrap Providers.
for _, bootstrapProvider := range opts.bootstrapProviders {
if err := createGenericProvider(ctx, client, newGenericProvider(clusterctlv1.BootstrapProviderType), bootstrapProvider, opts.targetNamespace); err != nil {
return fmt.Errorf("cannot create bootstrap provider: %w", err)
}
}

// Deploy Infrastructure Providers.
for _, infrastructureProvider := range opts.infrastructureProviders {
if err := createGenericProvider(ctx, client, newGenericProvider(clusterctlv1.InfrastructureProviderType), infrastructureProvider, opts.targetNamespace); err != nil {
return fmt.Errorf("cannot create infrastructure provider: %w", err)
}
}

// Deploy Control Plane Providers.
for _, controlPlaneProvider := range opts.controlPlaneProviders {
if err := createGenericProvider(ctx, client, newGenericProvider(clusterctlv1.ControlPlaneProviderType), controlPlaneProvider, opts.targetNamespace); err != nil {
return fmt.Errorf("cannot create control plane provider: %w", err)
}
}

// Deploy Add-on Providers.
for _, addonProvider := range opts.addonProviders {
if err := createGenericProvider(ctx, client, newGenericProvider(clusterctlv1.AddonProviderType), addonProvider, opts.targetNamespace); err != nil {
return fmt.Errorf("cannot create addon provider: %w", err)
}
}

return nil
}

// checkCAPIOpearatorAvailability checks if the CAPI operator is available on the management cluster.
func checkCAPIOpearatorAvailability(ctx context.Context, client ctrlclient.Client) (bool, error) {
var deploymentList appsv1.DeploymentList

// Search deployments with desired labels in all namespaces.
capiOperatorLabels := map[string]string{
"clusterctl.cluster.x-k8s.io/core": "capi-operator",
"control-plane": "controller-manager",
}

if err := client.List(ctx, &deploymentList, ctrlclient.MatchingLabels(capiOperatorLabels)); err != nil {
return false, fmt.Errorf("cannot get a list of deployments from the server: %w", err)
}

if len(deploymentList.Items) > 1 {
return false, fmt.Errorf("more than one CAPI Operator deployments were found")
}

return len(deploymentList.Items) == 1, nil
}

func ensureCertManager(opts *initOptions) error {
configClient, err := configclient.New("")
if err != nil {
return fmt.Errorf("cannot create config client: %w", err)
}

clusterKubeconfig := cluster.Kubeconfig{
Path: opts.kubeconfig,
Context: opts.kubeconfigContext,
}

clusterClient := cluster.New(clusterKubeconfig, configClient)

// Before installing the operator, ensure the cert-manager Webhook is in place.
certManager := clusterClient.CertManager()
if err := certManager.EnsureInstalled(); err != nil {
return fmt.Errorf("cannot install cert-manager Webhook: %w", err)
}

return nil
}

// deployCAPIOperator deploys the CAPI operator on the management cluster.
func deployCAPIOperator(opts *initOptions) error {
configClient, err := configclient.New("")
if err != nil {
return fmt.Errorf("cannot create config client: %w", err)
}

providerConfig := configclient.NewProvider(capiOperatorProviderName, capiOperatorManifestsURL, clusterctlv1.ProviderTypeUnknown)

repo, err := util.RepositoryFactory(providerConfig, configClient.Variables())
if err != nil {
return fmt.Errorf("cannot create repository: %w", err)
}

if opts.operatorVersion == "" {
// TODO(mfedosin): support DefaultVersion for the operator
// opts.operatorVersion = repo.DefaultVersion()
opts.operatorVersion = capiOperatorLatestVersion
}

componentsFile, err := repo.GetFile(opts.operatorVersion, repo.ComponentsPath())
if err != nil {
return fmt.Errorf("cannot get components file: %w", err)
}

options := repository.ComponentsOptions{
TargetNamespace: opts.targetNamespace,
SkipTemplateProcess: false,
Version: opts.operatorVersion,
}

components, err := repository.NewComponents(repository.ComponentsInput{
Provider: providerConfig,
ConfigClient: configClient,
Processor: yamlprocessor.NewSimpleProcessor(),
RawYaml: componentsFile,
Options: options,
})
if err != nil {
return fmt.Errorf("cannot generate CAPI operator components: %w", err)
}

clusterKubeconfig := cluster.Kubeconfig{
Path: opts.kubeconfig,
Context: opts.kubeconfigContext,
}

clusterClient := cluster.New(clusterKubeconfig, configClient)

// TODO(mfedosin): we have to ignore wait-providers and wait-providers-timeout options until we upgrade
// to CAPI v1.6.0 where clusterctl functions have Context support.

if err := clusterClient.ProviderComponents().Create(components.Objs()); err != nil {
return fmt.Errorf("cannot create CAPI operator components: %w", err)
}

return nil
}

func initProvider(ctx context.Context, opts *initOptions) error {
return errors.New("Not implemented")
func newGenericProvider(providerType clusterctlv1.ProviderType) genericprovider.GenericProvider {
switch providerType {
case clusterctlv1.CoreProviderType:
return &genericprovider.CoreProviderWrapper{CoreProvider: &operatorv1.CoreProvider{}}
case clusterctlv1.BootstrapProviderType:
return &genericprovider.BootstrapProviderWrapper{BootstrapProvider: &operatorv1.BootstrapProvider{}}
case clusterctlv1.ControlPlaneProviderType:
return &genericprovider.ControlPlaneProviderWrapper{ControlPlaneProvider: &operatorv1.ControlPlaneProvider{}}
case clusterctlv1.InfrastructureProviderType:
return &genericprovider.InfrastructureProviderWrapper{InfrastructureProvider: &operatorv1.InfrastructureProvider{}}
case clusterctlv1.AddonProviderType:
return &genericprovider.AddonProviderWrapper{AddonProvider: &operatorv1.AddonProvider{}}
case clusterctlv1.IPAMProviderType, clusterctlv1.RuntimeExtensionProviderType, clusterctlv1.ProviderTypeUnknown:
panic(fmt.Sprintf("unsupported provider type %s", providerType))
default:
panic(fmt.Sprintf("unknown provider type %s", providerType))
}
}

// createGenericProvider creates a generic provider.
func createGenericProvider(ctx context.Context, client ctrlclient.Client, provider genericprovider.GenericProvider, providerInput string, defaultNamespace string) error {
// Parse the provider string
// Format is <optional-namespace>:<provider-name>:<optional-version>
var name, namespace, version string

parts := strings.Split(providerInput, ":")
switch len(parts) {
case 1:
name = parts[0]
case 2:
name = parts[0]
version = parts[1]
case 3:
namespace = parts[0]
name = parts[1]
version = parts[2]
default:
return fmt.Errorf("invalid provider format: %s", provider)
}

// Set name and namespace
provider.SetName(name)

if namespace == "" {
namespace = defaultNamespace
}

provider.SetNamespace(namespace)

// Set version
if version != "" {
spec := provider.GetSpec()
spec.Version = version
provider.SetSpec(spec)
}

// Create the provider
if err := client.Create(ctx, provider.GetObject()); err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("cannot create provider: %w", err)
}

return nil
}
Loading

0 comments on commit 5839bb1

Please sign in to comment.