Skip to content

Commit

Permalink
Revert "cluster: update kind to use new containerd patch format (#314)…
Browse files Browse the repository at this point in the history
…" (#320)

This reverts commit 5d03bf0.

Signed-off-by: Nick Santos <[email protected]>
  • Loading branch information
nicks authored Nov 20, 2023
1 parent 0a01b9a commit 1eebafa
Show file tree
Hide file tree
Showing 7 changed files with 22 additions and 116 deletions.
3 changes: 1 addition & 2 deletions pkg/api/types.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
package api

import (
"github.com/tilt-dev/ctlptl/pkg/api/k3dv1alpha4"
"github.com/tilt-dev/localregistry-go"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"

"github.com/tilt-dev/ctlptl/pkg/api/k3dv1alpha4"
)

// TypeMeta partially copies apimachinery/pkg/apis/meta/v1.TypeMeta
Expand Down
12 changes: 0 additions & 12 deletions pkg/cluster/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,3 @@ type Admin interface {
type AdminInContainer interface {
ModifyConfigInContainer(ctx context.Context, cluster *api.Cluster, containerID string, dockerClient dctr.Client, configWriter configWriter) error
}

// Containerd made major changes to their config format for
// configuring registries. Each cluster has its own way
// of detecting this.

type containerdRegistryAPI int

const (
containerdRegistryV1 containerdRegistryAPI = iota
containerdRegistryV2
containerdRegistryBroken
)
1 change: 1 addition & 0 deletions pkg/cluster/admin_docker_desktop.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

// The DockerDesktop manages the Kubernetes cluster for DockerDesktop.
// This is a bit different than the other admins, due to the overlap
//
type dockerDesktopAdmin struct {
os string
host string
Expand Down
106 changes: 15 additions & 91 deletions pkg/cluster/admin_kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (a *kindAdmin) EnsureInstalled(ctx context.Context) error {
return nil
}

func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) *v1alpha4.Cluster {
func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry) *v1alpha4.Cluster {
kindConfig := desired.KindV1Alpha4Cluster
if kindConfig == nil {
kindConfig = &v1alpha4.Cluster{}
Expand All @@ -67,22 +67,13 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist
kindConfig.APIVersion = "kind.x-k8s.io/v1alpha4"

if registry != nil {
if registryAPI == containerdRegistryV2 {
// Point to the registry config path.
// We'll add these files post-creation.
patch := `[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/containerd/certs.d"
`
kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
} else {
patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%d"]
patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:%d"]
endpoint = ["http://%s:%d"]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s:%d"]
endpoint = ["http://%s:%d"]
`, registry.Status.HostPort, registry.Name, registry.Status.ContainerPort,
registry.Name, registry.Status.ContainerPort, registry.Name, registry.Status.ContainerPort)
kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
}
registry.Name, registry.Status.ContainerPort, registry.Name, registry.Status.ContainerPort)
kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
}
return kindConfig
}
Expand Down Expand Up @@ -116,30 +107,21 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *
}
}

kindVersion, err := a.getKindVersion(ctx)
if err != nil {
return errors.Wrap(err, "creating cluster")
}

registryAPI := containerdRegistryV2
kindVersionSemver, err := semver.ParseTolerant(kindVersion)
if err != nil {
return errors.Wrap(err, "parsing kind version")
}
if kindVersionSemver.LT(semver.MustParse("0.20.0")) {
registryAPI = containerdRegistryV1
}

args := []string{"create", "cluster", "--name", kindName}
if desired.KubernetesVersion != "" {
kindVersion, err := a.getKindVersion(ctx)
if err != nil {
return errors.Wrap(err, "creating cluster")
}

node, err := a.getNodeImage(ctx, kindVersion, desired.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "creating cluster")
}
args = append(args, "--image", node)
}

kindConfig := a.kindClusterConfig(desired, registry, registryAPI)
kindConfig := a.kindClusterConfig(desired, registry)
buf := bytes.NewBuffer(nil)
encoder := yaml.NewEncoder(buf)
err = encoder.Encode(kindConfig)
Expand All @@ -158,73 +140,15 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *

networkName := kindNetworkName()

if registry != nil {
if !a.inKindNetwork(registry, networkName) {
_, _ = fmt.Fprintf(a.iostreams.ErrOut, " Connecting kind to registry %s\n", registry.Name)
err := a.dockerClient.NetworkConnect(ctx, networkName, registry.Name, nil)
if err != nil {
return errors.Wrap(err, "connecting registry")
}
}

if registryAPI == containerdRegistryV2 {
err = a.applyContainerdPatchRegistryApiV2(ctx, desired, registry)
if err != nil {
return err
}
}
}

return nil
}

// We want to make sure that the image is pullable from either:
// localhost:[registry-port] or
// [registry-name]:5000
// by configuring containerd to rewrite the url.
func (a *kindAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
nodes, err := a.getNodes(ctx, desired.Name)
if err != nil {
return errors.Wrap(err, "configuring registry")
}

for _, node := range nodes {
registryDir := fmt.Sprintf("/etc/containerd/certs.d/localhost:%d", registry.Status.HostPort)
contents := fmt.Sprintf(`[host."http://%s:%d"]
`, registry.Name, registry.Status.ContainerPort)
iostreams := a.iostreams
iostreams.In = strings.NewReader(contents)
err := a.runner.RunIO(ctx,
iostreams,
"docker", "exec", "-i", node, "sh", "-c",
fmt.Sprintf("mkdir -p %s && cp /dev/stdin %s/hosts.toml", registryDir, registryDir))
if registry != nil && !a.inKindNetwork(registry, networkName) {
_, _ = fmt.Fprintf(a.iostreams.ErrOut, " Connecting kind to registry %s\n", registry.Name)
err := a.dockerClient.NetworkConnect(ctx, networkName, registry.Name, nil)
if err != nil {
return errors.Wrap(err, "configuring registry")
return errors.Wrap(err, "connecting registry")
}
}
return nil
}

func (a *kindAdmin) getNodes(ctx context.Context, cluster string) ([]string, error) {
kindName := strings.TrimPrefix(cluster, "kind-")
buf := bytes.NewBuffer(nil)
iostreams := a.iostreams
iostreams.Out = buf
err := a.runner.RunIO(ctx, iostreams,
"kind", "get", "nodes", "--name", kindName)
if err != nil {
return nil, errors.Wrap(err, "kind get nodes")
}

scanner := bufio.NewScanner(buf)
result := []string{}
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
result = append(result, line)
}
}
return result, nil
return nil
}

func (a *kindAdmin) clusterExists(ctx context.Context, cluster string) (bool, error) {
Expand Down
3 changes: 1 addition & 2 deletions pkg/cluster/admin_kind_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ import (
"testing"

"github.com/stretchr/testify/assert"
"k8s.io/cli-runtime/pkg/genericclioptions"

"github.com/tilt-dev/ctlptl/internal/exec"
"k8s.io/cli-runtime/pkg/genericclioptions"
)

func TestNodeImage(t *testing.T) {
Expand Down
12 changes: 4 additions & 8 deletions pkg/cluster/admin_minikube.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,8 @@ func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, regist
if err != nil {
return err
}
registryAPI := containerdRegistryV1
if v.GTE(v1_26) && v.LT(v1_27) {
registryAPI = containerdRegistryBroken
} else if v.GTE(v1_27) {
registryAPI = containerdRegistryV2
}
isRegistryApiBroken := v.GTE(v1_26) && v.LT(v1_27)
isRegistryApiV2 := v.GTE(v1_26)

clusterName := desired.Name
if registry != nil {
Expand Down Expand Up @@ -143,7 +139,7 @@ func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, regist

// https://github.com/tilt-dev/ctlptl/issues/239
if registry != nil {
if registryAPI == containerdRegistryBroken {
if isRegistryApiBroken {
return fmt.Errorf(
"Error: Local registries are broken in minikube v1.26.\n" +
"See: https://github.com/kubernetes/minikube/issues/14480 .\n" +
Expand Down Expand Up @@ -172,7 +168,7 @@ func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, regist
return err
}

if registryAPI == containerdRegistryV2 {
if isRegistryApiV2 {
err = a.applyContainerdPatchRegistryApiV2(ctx, desired, registry, networkMode)
if err != nil {
return err
Expand Down
1 change: 0 additions & 1 deletion pkg/cluster/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"os"

"github.com/docker/docker/pkg/stringid"

"github.com/tilt-dev/ctlptl/internal/dctr"
)

Expand Down

0 comments on commit 1eebafa

Please sign in to comment.