Skip to content
This repository has been archived by the owner on Oct 10, 2023. It is now read-only.

Commit

Permalink
Add antrea-interworking 0.9.0
Browse files Browse the repository at this point in the history
Signed-off-by: Wenqi Qiu <[email protected]>
  • Loading branch information
wenqiq committed Apr 5, 2023
1 parent 731e6d4 commit 1b6ac9b
Show file tree
Hide file tree
Showing 15 changed files with 1,522 additions and 150 deletions.
19 changes: 1 addition & 18 deletions addons/controllers/antrea/antreaconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,13 +202,6 @@ func (r *AntreaConfigReconciler) ReconcileAntreaConfigNormal(
return err
}

if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil && antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil {
err := fmt.Errorf("providerRef and inline should not be both set in AntreaConfig.spec.antreaNsx.bootstrapFrom")
antreaConfig.Status.Message = err.Error()
} else {
// clear the message here.
antreaConfig.Status.Message = ""
}
// update status.secretRef
dataValueSecretName := util.GenerateDataValueSecretName(cluster.Name, constants.AntreaAddonName)
antreaConfig.Status.SecretRef = dataValueSecretName
Expand Down Expand Up @@ -329,21 +322,11 @@ func (r *AntreaConfigReconciler) ensureProviderServiceAccount(ctx context.Contex
}

func (r *AntreaConfigReconciler) registerAntreaNSX(ctx context.Context, antreaConfig *cniv1alpha2.AntreaConfig, cluster *clusterapiv1beta1.Cluster) error {
if !antreaConfig.Spec.AntreaNsx.Enable || antreaConfig.Spec.AntreaNsx.BootstrapFrom.Inline != nil {
if !antreaConfig.Spec.AntreaNsx.Enable || antreaConfig.Spec.AntreaNsx.AntreaNsxConfig.BootstrapFrom == bootstrapFromInline {
r.Log.Info("antreaNsx is not enabled or inline is set, there is no ProviderServiceAccount or NsxServiceAccount to be created")
r.deregisterAntreaNSX(ctx, antreaConfig, cluster)
return nil
}
if antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef != nil {
if strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind) != nsxServiceAccountKind ||
strings.ToLower(antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup) != nsxServiceAccountAPIGroup {
err := fmt.Errorf("either ProviderRef.Kind(%s) or ProviderRef.ApiGroup(%s) is invalid, expcted:ProviderRef.Kind(%s) ProviderRef.ApiGroup(%s)",
antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.Kind, antreaConfig.Spec.AntreaNsx.BootstrapFrom.ProviderRef.ApiGroup,
nsxServiceAccountKind, nsxServiceAccountAPIGroup)
antreaConfig.Status.Message = err.Error()
return err
}
}
antreaConfig.Status.Message = ""
err := r.ensureProviderServiceAccount(ctx, antreaConfig, cluster)
if err != nil {
Expand Down
151 changes: 122 additions & 29 deletions addons/controllers/antrea/antreaconfig_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@ package controllers
import (
"context"
"fmt"
"reflect"
"strings"

"github.com/pkg/errors"

"golang.org/x/mod/semver"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clusterv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1"
clusterapiutil "sigs.k8s.io/cluster-api/util"
ctrl "sigs.k8s.io/controller-runtime"
Expand All @@ -23,6 +25,11 @@ import (
cniv1alpha2 "github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha2"
)

const (
bootstrapFromInline = "Inline"
bootstrapFromSupervisorCluster = "SupervisorCluster"
)

// AntreaConfigSpec defines the desired state of AntreaConfig
type AntreaConfigSpec struct {
InfraProvider string `yaml:"infraProvider"`
Expand All @@ -35,42 +42,65 @@ type antrea struct {
}

type antreaNsx struct {
Enable bool `yaml:"enable,omitempty"`
BootstrapFrom antreaNsxBootstrapFrom `yaml:"bootstrapFrom,omitempty"`
AntreaNsxConfig antreaNsxConfig `yaml:"config,omitempty"`
Enable bool `yaml:"enable,omitempty"`
AntreaNsxConfig antreaNsxConfig `yaml:"config,omitempty"`
}

type antreaNsxBootstrapFrom struct {
// ProviderRef is used with uTKG, which will be filled by NCP operator
ProviderRef *antreaNsxProvider `yaml:"providerRef,omitempty"`
// Inline is used with TKGm, user need to fill in manually
Inline *antreaNsxInline `yaml:"inline,omitempty"`
}

type antreaNsxProvider struct {
// Api version for nsxServiceAccount, its value is "nsx.vmware.com/v1alpha1" now
ApiVersion string `yaml:"apiVersion,omitempty"`
// Its value is NsxServiceAccount
Kind string `yaml:"kind,omitempty"`
// Name is the name for NsxServiceAccount
Name string `yaml:"name,omitempty"`
type antreaNsxConfig struct {
InfraType string `yaml:"infraType,omitempty"`
BootstrapFrom string `yaml:"bootstrapFrom,omitempty"`
BootstrapSupervisorResourceName string `yaml:"bootstrapSupervisorResourceName,omitempty"`
NSXCert string `yaml:"nsxCert,omitempty"`
NSXKey string `yaml:"nsxKey,omitempty"`
NSXUser string `yaml:"nsxUser,omitempty"`
NSXPassword string `yaml:"nsxPassword,omitempty"`
ClusterName string `yaml:"clusterName,omitempty"`
NSXManagers []string `yaml:"NSXManagers,omitempty"`
VPCPath []string `yaml:"vpcPath,omitempty"`
ProxyEndpoints proxyEndpoints `yaml:"proxyEndpoints,omitempty"`
MpAdapterConf mpAdapterConf `yaml:"mp_adapter_conf,omitempty"`
CcpAdapterConf ccpAdapterConf `yaml:"ccp_adapter_conf,omitempty"`
}

type nsxCertRef struct {
// TLSCert is cert file to access nsx manager
TLSCert string `yaml:"tls.crt,omitempty"`
// TLSKey is key file to access nsx manager
TLSKey string `yaml:"tls.key,omitempty"`
type proxyEndpoints struct {
RestApi []string `yaml:"rest_api,omitempty"`
NSXRpcFwdProxy []string `yaml:"nsx_rpc_fwd_proxy,omitempty"`
}

type antreaNsxInline struct {
NsxManagers []string `yaml:"nsxManagers,omitempty"`
ClusterName string `yaml:"clusterName,omitempty"`
NsxCertRef nsxCertRef `yaml:"NsxCert,omitempty"`
type mpAdapterConf struct {
NSXClientAuthCertFile string `yaml:"NSXClientAuthCertFile,omitempty"`
NSXClientAuthKeyFile string `yaml:"NSXClientAuthKeyFile,omitempty"`
NSXRemoteAuth bool `yaml:"NSXRemoteAuth,omitempty"`
NSXCAFile string `yaml:"NSXCAFile,omitempty"`
NSXInsecure bool `yaml:"NSXInsecure,omitempty"`
NSXRPCConnType string `yaml:"NSXRPCConnType,omitempty"`
ClusterType string `yaml:"clusterType,omitempty"`
NSXClientTimeout int `yaml:"NSXClientTimeout,omitempty"`
InventoryBatchSize int `yaml:"InventoryBatchSize,omitempty"`
InventoryBatchPeriod int `yaml:"InventoryBatchPeriod,omitempty"`
EnableDebugServer bool `yaml:"EnableDebugServer,omitempty"`
APIServerPort int `yaml:"APIServerPort,omitempty"`
DebugServerPort int `yaml:"DebugServerPort,omitempty"`
NSXRPCDebug bool `yaml:"NSXRPCDebug,omitempty"`
ConditionTimeout int `yaml:"ConditionTimeout,omitempty"`
}

type antreaNsxConfig struct {
InfraType string `yaml:"infraType,omitempty"`
type ccpAdapterConf struct {
EnableDebugServer bool `yaml:"EnableDebugServer,omitempty"`
APIServerPort int `yaml:"APIServerPort,omitempty"`
DebugServerPort int `yaml:"DebugServerPort,omitempty"`
NSXRPCDebug bool `yaml:"NSXRPCDebug,omitempty"`
// Time to wait for realization
RealizeTimeoutSeconds int `yaml:"RealizeTimeoutSeconds,omitempty"`
// An interval for regularly report latest realization error in background
RealizeErrorSyncIntervalSeconds int `yaml:"RealizeErrorSyncIntervalSeconds,omitempty"`
ReconcilerWorkerCount int `yaml:"ReconcilerWorkerCount,omitempty"`
// Average QPS = ReconcilerWorkerCount * ReconcilerQPS
ReconcilerQPS int `yaml:"ReconcilerQPS,omitempty"`
// Peak QPS = ReconcilerWorkerCount * ReconcilerBurst
ReconcilerBurst int `yaml:"ReconcilerBurst,omitempty"`
// #! 24 Hours
ReconcilerResyncSeconds int `yaml:"ReconcilerResyncSeconds,omitempty"`
}

type antreaEgress struct {
Expand Down Expand Up @@ -295,6 +325,69 @@ func mapAntreaConfigSpec(cluster *clusterv1beta1.Cluster, config *cniv1alpha2.An
if semver.Compare(version, "v1.9.0") >= 0 {
configSpec.Antrea.AntreaConfigDataValue.FeatureGates.TopologyAwareHints = &config.Spec.Antrea.AntreaConfigDataValue.FeatureGates.TopologyAwareHints
}
// NSX related
if semver.Compare(version, "1.9.0") >= 0 && config.Spec.AntreaNsx.Enable {
configSpec.AntreaNsx.Enable = config.Spec.AntreaNsx.Enable
switch config.Spec.AntreaNsx.AntreaNsxConfig.BootstrapFrom {
case bootstrapFromInline:
configSpec.AntreaNsx.AntreaNsxConfig.NSXManagers = config.Spec.AntreaNsx.AntreaNsxConfig.NSXManagers
configSpec.AntreaNsx.AntreaNsxConfig.ClusterName = config.Spec.AntreaNsx.AntreaNsxConfig.ClusterName
// NSX cert
secret, err := getNSXCert(client, config.Name, config.Namespace)
if err != nil {
return configSpec, err
}
configSpec.AntreaNsx.AntreaNsxConfig.NSXCert = string(secret.Data["tls.crt"])
configSpec.AntreaNsx.AntreaNsxConfig.NSXKey = string(secret.Data["tls.key"])
configSpec.AntreaNsx.AntreaNsxConfig.VPCPath = config.Spec.AntreaNsx.AntreaNsxConfig.VPCPath
case bootstrapFromSupervisorCluster:
configSpec.AntreaNsx.AntreaNsxConfig.BootstrapSupervisorResourceName = config.Spec.AntreaNsx.AntreaNsxConfig.BootstrapSupervisorResourceName
}
configSpec.AntreaNsx.AntreaNsxConfig.ProxyEndpoints.NSXRpcFwdProxy = config.Spec.AntreaNsx.AntreaNsxConfig.ProxyEndpoints.NSXRpcFwdProxy
configSpec.AntreaNsx.AntreaNsxConfig.ProxyEndpoints.RestApi = config.Spec.AntreaNsx.AntreaNsxConfig.ProxyEndpoints.RestApi

ccpConf := config.Spec.AntreaNsx.AntreaNsxConfig.CcpAdapterConf
if err := copyStructAtoB(ccpConf, &configSpec.AntreaNsx.AntreaNsxConfig.CcpAdapterConf); err != nil {
return configSpec, err
}
mpConf := config.Spec.AntreaNsx.AntreaNsxConfig.MpAdapterConf
if err := copyStructAtoB(mpConf, &configSpec.AntreaNsx.AntreaNsxConfig.MpAdapterConf); err != nil {
return configSpec, err
}
}

return configSpec, nil
}

func copyStructAtoB(a interface{}, b interface{}) error {
va := reflect.ValueOf(a)
vb := reflect.ValueOf(b).Elem()
for i := 0; i < va.NumField(); i++ {
fieldA := va.Field(i)
fieldB := vb.FieldByName(va.Type().Field(i).Name)
if fieldB.IsValid() && fieldA.Type() == fieldB.Type() {
fieldB.Set(fieldA)
}
}
return nil
}

func getNSXCert(client client.Client, secretName, secretNamespace string) (secret *corev1.Secret, err error) {
secret = &corev1.Secret{}
if err := client.Get(context.TODO(), types.NamespacedName{
Namespace: secretNamespace,
Name: secretName,
}, secret); err != nil {
return nil, err
}
if secret.Data == nil {
return nil, fmt.Errorf("missing secret data")
}
if _, ok := secret.Data["tls.crt"]; !ok {
return nil, fmt.Errorf("missing tls.crt")
}
if _, ok := secret.Data["tls.key"]; !ok {
return nil, fmt.Errorf("missing tls.key")
}
return secret, nil
}
78 changes: 78 additions & 0 deletions addons/controllers/antrea/antreaconfig_util_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package controllers

import (
"context"
"testing"

"github.com/vmware-tanzu/tanzu-framework/apis/addonconfigs/cni/v1alpha2"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)

func TestGetNSXCert(t *testing.T) {
fakeClient := fake.NewFakeClientWithScheme(scheme.Scheme)

secretName := "test-secret"
secretNamespace := "test-namespace"
testSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: secretNamespace,
},
Data: map[string][]byte{
"tls.crt": []byte("fake-certificate-data"),
"tls.key": []byte("fake-key-data"),
},
}
if err := fakeClient.Create(context.Background(), testSecret); err != nil {
t.Fatalf("failed to create test secret: %v", err)
}

returnedSecret, err := getNSXCert(fakeClient, secretName, secretNamespace)
require.NoError(t, err, "getNSXCert returned an unexpected error")
require.NotNil(t, returnedSecret, "getNSXCert returned a nil secret")

assert.Equal(t, secretName, returnedSecret.Name, "returned secret has unexpected name")
assert.Equal(t, secretNamespace, returnedSecret.Namespace, "returned secret has unexpected namespace")
assert.Equal(t, 2, len(returnedSecret.Data), "returned secret has unexpected number of data fields")

if _, ok := returnedSecret.Data["tls.crt"]; !ok {
t.Error("returned secret missing tls.crt field")
}
if _, ok := returnedSecret.Data["tls.key"]; !ok {
t.Error("returned secret missing tls.key field")
}
}

func TestCopyStruct(t *testing.T) {
ccpConf := v1alpha2.CcpAdapterConf{
EnableDebugServer: true,
APIServerPort: 1234,
}
descCcpAdapterConf := ccpAdapterConf{
EnableDebugServer: false,
APIServerPort: 0,
}
err := copyStructAtoB(ccpConf, &descCcpAdapterConf)
require.NoError(t, err, "copy CcpAdapterConf values error")
assert.Equal(t, 1234, descCcpAdapterConf.APIServerPort)
assert.Equal(t, true, descCcpAdapterConf.EnableDebugServer)

mpConf := v1alpha2.MpAdapterConf{
NSXClientAuthCertFile: "fake-cert-file",
ConditionTimeout: 150,
}
descMpAdapterConf := mpAdapterConf{
NSXClientAuthCertFile: "",
ConditionTimeout: 0,
}
err = copyStructAtoB(mpConf, &descMpAdapterConf)
require.NoError(t, err, "copy MpAdapterConf values error")
assert.Equal(t, "fake-cert-file", descMpAdapterConf.NSXClientAuthCertFile)
assert.Equal(t, 150, descMpAdapterConf.ConditionTimeout)
}
2 changes: 2 additions & 0 deletions addons/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/onsi/gomega v1.20.2
github.com/oracle/cluster-api-provider-oci v0.6.0
github.com/pkg/errors v0.9.1
github.com/stretchr/testify v1.8.0
github.com/vmware-tanzu/carvel-kapp-controller v0.35.0
github.com/vmware-tanzu/carvel-secretgen-controller v0.5.0
github.com/vmware-tanzu/carvel-vendir v0.26.0
Expand Down Expand Up @@ -91,6 +92,7 @@ require (
github.com/nxadm/tail v1.4.8 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.18.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.13.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions addons/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
Expand All @@ -865,6 +866,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
Expand Down
Loading

0 comments on commit 1b6ac9b

Please sign in to comment.