Skip to content

Commit

Permalink
feat(virtualization-api): first implementation (#11)
Browse files Browse the repository at this point in the history
Description
Added a new component virtualization-api - extension API server.

Why do we need it, and what problem does it solve?
Using the api server, we can connect to virtual machines and perform various operations. Currently implemented: console, vnc, port forwarding
---------
Signed-off-by: Yaroslav Borbat <[email protected]>
Signed-off-by: Yaroslav Borbat <[email protected]>
Co-authored-by: Ivan Mikheykin <[email protected]>
Signed-off-by: Nikita Korolev <[email protected]>
  • Loading branch information
yaroslavborbat authored and universal-itengineer committed Mar 20, 2024
1 parent 4a3574f commit 828836c
Show file tree
Hide file tree
Showing 190 changed files with 38,081 additions and 243 deletions.
48 changes: 48 additions & 0 deletions crds/virtualmachine.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,18 @@ spec:
topologyKey:
type: string
description: ""
matchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
mismatchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
required:
- topologyKey
type: object
Expand Down Expand Up @@ -471,6 +483,18 @@ spec:
topologyKey:
type: string
description: ""
matchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
mismatchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
required:
- topologyKey
type: object
Expand Down Expand Up @@ -559,6 +583,18 @@ spec:
topologyKey:
type: string
description: ""
matchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
mismatchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
required:
- topologyKey
type: object
Expand Down Expand Up @@ -652,6 +688,18 @@ spec:
topologyKey:
type: string
description: ""
matchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
mismatchLabelKeys:
items:
type: string
description: ""
type: array
description: ""
required:
- topologyKey
type: object
Expand Down
25 changes: 22 additions & 3 deletions hooks/generate_certificates.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# limitations under the License.


from lib.hooks.internal_tls import GenerateCertificatesHook, CertitifacteRequest, CACertitifacteRequest, default_sans
from lib.hooks.internal_tls import *
from lib.module import values as module_values
from deckhouse.hook import Context
from typing import Callable
Expand All @@ -25,7 +25,7 @@
def main():
hook = GenerateCertificatesHook(
CertitifacteRequest(
cn=f"virtualization-controller",
cn="virtualization-controller",
sansGenerator=default_sans([
"virtualization-controller-admission-webhook",
f"virtualization-controller-admission-webhook.{common.NAMESPACE}",
Expand All @@ -46,11 +46,30 @@ def main():
before_gen_check=dvcr_before_check
),

CertitifacteRequest(
cn="virtualization-api",
sansGenerator=default_sans([
"virtualization-api",
f"virtualization-api.{common.NAMESPACE}",
f"virtualization-api.{common.NAMESPACE}.svc"],
),
tls_secret_name="virtualization-api-tls",
values_path_prefix=f"{common.MODULE_NAME}.internal.apiserver.cert"
),

CertitifacteRequest(
cn="virtualization-api-proxy",
sansGenerator=empty_sans(),
tls_secret_name="virtualization-api-proxy-tls",
values_path_prefix=f"{common.MODULE_NAME}.internal.apiserver.proxyCert",
extended_key_usages = [EXTENDED_KEY_USAGES[1]]
),

namespace=common.NAMESPACE,
module_name=common.MODULE_NAME,

ca_request=CACertitifacteRequest(
cn=f"virtualization.deckhouse.io",
cn="virtualization.deckhouse.io",
ca_secret_name="virtualization-ca",
values_path_prefix=f"{common.MODULE_NAME}.internal.rootCA",
))
Expand Down
3 changes: 2 additions & 1 deletion hooks/lib/certificate/certificate.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ def with_hosts(self, *hosts: str):
if not is_valid_hostname(h):
continue
alt_names.append(f"DNS:{h}")
self.add_extension("subjectAltName", False, ", ".join(alt_names))
if len(alt_names) > 0:
self.add_extension("subjectAltName", False, ", ".join(alt_names))
return self

def __sign(self, ca_subj: crypto.X509Name, ca_key: crypto.PKey) -> None:
Expand Down
10 changes: 7 additions & 3 deletions hooks/lib/certificate/parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,17 @@ def parse_key(key: str) -> crypto.PKey:


def get_certificate_san(crt: crypto.X509) -> list[str]:
san = ''
san = ""
ext_count = crt.get_extension_count()
for i in range(0, ext_count):
ext = crt.get_extension(i)
if 'subjectAltName' in str(ext.get_short_name()):
if "subjectAltName" in str(ext.get_short_name()):
san = ext.__str__()
return san.split(', ')
break
if len(san) > 0:
return san.split(', ')
return []



def is_outdated_ca(ca: str, cert_outdated_duration: timedelta) -> bool:
Expand Down
8 changes: 7 additions & 1 deletion hooks/lib/hooks/internal_tls.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ def r(ctx: hook.Context):
snaps[s["filterResult"]["name"]] = TLSSecretData(
s["filterResult"]["data"])
ca_data = TLSSecretData()
if self.__with_common_ca:
if self.__with_common_ca():
tls_value_data = self.__sync_ca(self.ca_request,
snaps.get(self.ca_request.ca_secret_name, TLSSecretData()))
ca_data = convert_to_TLSSecretData(tls_value_data)
Expand Down Expand Up @@ -456,3 +456,9 @@ def generate_sans(ctx: hook.Context) -> list[str]:
res.append(san)
return res
return generate_sans


def empty_sans() -> Callable[[hook.Context], list[str]]:
def generator(ctx: hook.Context) -> list:
return []
return generator
130 changes: 130 additions & 0 deletions images/virt-artifact/patches/011-virt-api-authentication.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
diff --git a/pkg/controller/virtinformers.go b/pkg/controller/virtinformers.go
index 5cbb8197f..82f6f9238 100644
--- a/pkg/controller/virtinformers.go
+++ b/pkg/controller/virtinformers.go
@@ -300,6 +300,8 @@ type KubeInformerFactory interface {
ResourceQuota() cache.SharedIndexInformer

K8SInformerFactory() informers.SharedInformerFactory
+
+ VirtualizationCA() cache.SharedIndexInformer
}

type kubeInformerFactory struct {
@@ -1293,3 +1295,12 @@ func VolumeSnapshotClassInformer(clientSet kubecli.KubevirtClient, resyncPeriod
lw := cache.NewListWatchFromClient(restClient, "volumesnapshotclasses", k8sv1.NamespaceAll, fields.Everything())
return cache.NewSharedIndexInformer(lw, &vsv1.VolumeSnapshotClass{}, resyncPeriod, cache.Indexers{})
}
+
+func (f *kubeInformerFactory) VirtualizationCA() cache.SharedIndexInformer {
+ return f.getInformer("extensionsVirtualizationCAConfigMapInformer", func() cache.SharedIndexInformer {
+ restClient := f.clientSet.CoreV1().RESTClient()
+ fieldSelector := fields.OneTermEqualSelector("metadata.name", "virtualization-ca")
+ lw := cache.NewListWatchFromClient(restClient, "configmaps", f.kubevirtNamespace, fieldSelector)
+ return cache.NewSharedIndexInformer(lw, &k8sv1.ConfigMap{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
+ })
+}
diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go
index e9e140548..e2a349012 100644
--- a/pkg/util/tls/tls.go
+++ b/pkg/util/tls/tls.go
@@ -132,6 +132,55 @@ func SetupTLSWithCertManager(caManager ClientCAManager, certManager certificate.
return tlsConfig
}

+func SetupTLSWithVirtualizationCAManager(caManager, virtualizationCAManager ClientCAManager, certManager certificate.Manager, clientAuth tls.ClientAuthType, clusterConfig *virtconfig.ClusterConfig) *tls.Config {
+ tlsConfig := &tls.Config{
+ GetCertificate: func(info *tls.ClientHelloInfo) (certificate *tls.Certificate, err error) {
+ cert := certManager.Current()
+ if cert == nil {
+ return nil, fmt.Errorf(noSrvCertMessage)
+ }
+ return cert, nil
+ },
+ GetConfigForClient: func(hi *tls.ClientHelloInfo) (*tls.Config, error) {
+ cert := certManager.Current()
+ if cert == nil {
+ return nil, fmt.Errorf(noSrvCertMessage)
+ }
+
+ clientCAPool, err := caManager.GetCurrent()
+ if err != nil {
+ log.Log.Reason(err).Error("Failed to get requestheader client CA")
+ return nil, err
+ }
+
+ virtualizationCA, err := virtualizationCAManager.GetCurrentRaw()
+ if err != nil {
+ log.Log.Reason(err).Error("Failed to get CA from config-map virtualization-ca")
+ return nil, err
+ }
+
+ clientCAPool.AppendCertsFromPEM(virtualizationCA)
+
+ kv := clusterConfig.GetConfigFromKubeVirtCR()
+ tlsConfig := getTLSConfiguration(kv)
+ ciphers := CipherSuiteIds(tlsConfig.Ciphers)
+ minTLSVersion := TLSVersion(tlsConfig.MinTLSVersion)
+ config := &tls.Config{
+ CipherSuites: ciphers,
+ MinVersion: minTLSVersion,
+ Certificates: []tls.Certificate{*cert},
+ ClientCAs: clientCAPool,
+ ClientAuth: clientAuth,
+ }
+
+ config.BuildNameToCertificate()
+ return config, nil
+ },
+ }
+ tlsConfig.BuildNameToCertificate()
+ return tlsConfig
+}
+
func SetupTLSForVirtHandlerServer(caManager ClientCAManager, certManager certificate.Manager, externallyManaged bool, clusterConfig *virtconfig.ClusterConfig) *tls.Config {
// #nosec cause: InsecureSkipVerify: true
// resolution: Neither the client nor the server should validate anything itself, `VerifyPeerCertificate` is still executed
diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go
index 120f2d68f..4b82edd13 100644
--- a/pkg/virt-api/api.go
+++ b/pkg/virt-api/api.go
@@ -884,7 +884,7 @@ func (app *virtAPIApp) registerMutatingWebhook(informers *webhooks.Informers) {
})
}

-func (app *virtAPIApp) setupTLS(k8sCAManager kvtls.ClientCAManager, kubevirtCAManager kvtls.ClientCAManager) {
+func (app *virtAPIApp) setupTLS(k8sCAManager, kubevirtCAManager, virtualizationCAManager kvtls.ClientCAManager) {

// A VerifyClientCertIfGiven request means we're not guaranteed
// a client has been authenticated unless they provide a peer
@@ -901,7 +901,7 @@ func (app *virtAPIApp) setupTLS(k8sCAManager kvtls.ClientCAManager, kubevirtCAMa
// response is given. That status request won't send a peer cert regardless
// if the TLS handshake requests it. As a result, the TLS handshake fails
// and our aggregated endpoint never becomes available.
- app.tlsConfig = kvtls.SetupTLSWithCertManager(k8sCAManager, app.certmanager, tls.VerifyClientCertIfGiven, app.clusterConfig)
+ app.tlsConfig = kvtls.SetupTLSWithVirtualizationCAManager(k8sCAManager, virtualizationCAManager, app.certmanager, tls.VerifyClientCertIfGiven, app.clusterConfig)
app.handlerTLSConfiguration = kvtls.SetupTLSForVirtHandlerClients(kubevirtCAManager, app.handlerCertManager, app.externallyManaged)
}

@@ -919,10 +919,12 @@ func (app *virtAPIApp) startTLS(informerFactory controller.KubeInformerFactory)

authConfigMapInformer := informerFactory.ApiAuthConfigMap()
kubevirtCAConfigInformer := informerFactory.KubeVirtCAConfigMap()
+ virtualizationCAConfigInformer := informerFactory.VirtualizationCA()

k8sCAManager := kvtls.NewKubernetesClientCAManager(authConfigMapInformer.GetStore())
kubevirtCAInformer := kvtls.NewCAManager(kubevirtCAConfigInformer.GetStore(), app.namespace, app.caConfigMapName)
- app.setupTLS(k8sCAManager, kubevirtCAInformer)
+ virtualizationCAInformer := kvtls.NewCAManager(virtualizationCAConfigInformer.GetStore(), app.namespace, "virtualization-ca")
+ app.setupTLS(k8sCAManager, kubevirtCAInformer, virtualizationCAInformer)

app.Compose()

@@ -1007,6 +1009,7 @@ func (app *virtAPIApp) Run() {

kubeInformerFactory.ApiAuthConfigMap()
kubeInformerFactory.KubeVirtCAConfigMap()
+ kubeInformerFactory.VirtualizationCA()
crdInformer := kubeInformerFactory.CRD()
vmiPresetInformer := kubeInformerFactory.VirtualMachinePreset()
vmRestoreInformer := kubeInformerFactory.VirtualMachineRestore()
2 changes: 2 additions & 0 deletions images/virt-artifact/patches/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,5 @@ Rename group name for all kubevirt CRDs to override them with deckhouse virtuali

Also, remove short names and change categories. Just in case.

#### `011-virt-api-authentication.patch`
Added the ability for virt-api to authenticate clients with certificates signed by our rootCA located in the config-map virtualization-ca.
12 changes: 12 additions & 0 deletions images/virtualization-api/werf.inc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
image: {{ $.ImageName }}
fromImage: base-ubuntu-jammy
import:
- image: virtualization-artifact
add: /usr/local/go/src/virtualization-controller/virtualization-api
to: /app/virtualization-api
after: install
docker:
USER: "65532:65532"
WORKDIR: "/app"
ENTRYPOINT: ["/app/virtualization-api"]
Loading

0 comments on commit 828836c

Please sign in to comment.