MGMT-18684: WIP

src/assisted_installer_controller/assisted_installer_controller.go

@@ -422,10 +422,19 @@ func (c *controller) ApproveCsrs(ctx context.Context) {
 	}
 }
 
+func (c *controller) mayApproveCSR(csr *certificatesv1.CertificateSigningRequest) (bool, error) {
+	// TODO: Write an implementation that will check the public key and ensure it matches what we have on record.
+	return true, nil
+}
+
 func (c *controller) approveCsrs(csrs *certificatesv1.CertificateSigningRequestList) {
 	for i := range csrs.Items {
 		csr := csrs.Items[i]
-		if !isCsrApproved(&csr) {
+		mayApproveCSR, err := c.mayApproveCSR(&csr)
+		if err != nil {
+			continue
+		}
+		if mayApproveCSR && !isCsrApproved(&csr) {
 			c.log.Infof("Approving CSR %s", csr.Name)
 			// We can fail and it is ok, we will retry on the next time
 			_ = c.kc.ApproveCsr(&csr)

src/common/common.go

@@ -3,6 +3,10 @@ package common
 import (
 	"bytes"
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io"
 	"os"
@@ -16,6 +20,8 @@ import (
 	"github.com/openshift/assisted-installer/src/utils"
 	"github.com/openshift/assisted-service/models"
 
+	cryptorand "crypto/rand"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/thoas/go-funk"
@@ -35,6 +41,8 @@ const (
 	installConfigMapAttribute      = "invoker"
 	InvokerAssisted                = "assisted-service"
 	InvokerAgent                   = "agent-installer"
+	// ECPrivateKeyBlockType is a possible value for pem.Block.Type.
+	ECPrivateKeyBlockType = "EC PRIVATE KEY"
 )
 
 func GetHostsInStatus(hosts map[string]inventory_client.HostData, status []string, isMatch bool) map[string]inventory_client.HostData {
@@ -306,3 +314,22 @@ func DownloadKubeconfigNoingress(ctx context.Context, dir string, ic inventory_c
 
 	return kubeconfigPath, nil
 }
+
+// MakeEllipticPrivateKeyPEM creates an ECDSA private key
+func MakeEllipticPrivateKeyPEM() ([]byte, error) {
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	if err != nil {
+		return nil, err
+	}
+
+	derBytes, err := x509.MarshalECPrivateKey(privateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	privateKeyPemBlock := &pem.Block{
+		Type:  ECPrivateKeyBlockType,
+		Bytes: derBytes,
+	}
+	return pem.EncodeToMemory(privateKeyPemBlock), nil
+}

src/ignition/ignition.go

@@ -1,14 +1,22 @@
 package ignition
 
 import (
+	"crypto/ecdsa"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
+	"fmt"
 	"os"
 
+	"encoding/base64"
+
 	ignitionConfigPrevVersion "github.com/coreos/ignition/v2/config/v3_1"
 	ignitionConfig "github.com/coreos/ignition/v2/config/v3_2"
 	"github.com/coreos/ignition/v2/config/v3_2/translate"
 	"github.com/coreos/ignition/v2/config/v3_2/types"
 	"github.com/coreos/ignition/v2/config/validate"
+	"github.com/go-openapi/swag"
+	"github.com/openshift/assisted-installer/src/common"
 	"github.com/pkg/errors"
 )
 
@@ -21,6 +29,7 @@ type Ignition interface {
 	ParseIgnitionFile(path string) (*types.Config, error)
 	WriteIgnitionFile(path string, config *types.Config) error
 	MergeIgnitionConfig(base *types.Config, overrides *types.Config) (*types.Config, error)
+	InjectKubeletTempPrivateKey(pathToSourceIgnition string, privateKeyBytes []byte, certPathToInject string) error
 }
 
 type ignition struct{}
@@ -78,3 +87,73 @@ func (i *ignition) MergeIgnitionConfig(base *types.Config, overrides *types.Conf
 	}
 	return &config, nil
 }
+
+// parseECPublicKey parses a single ECDSA public key from the provided data
+func parseECPublicKey(data []byte) (*ecdsa.PublicKey, error) {
+	var err error
+
+	// Parse the key
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKIXPublicKey(data); err != nil {
+		if cert, err := x509.ParseCertificate(data); err == nil {
+			parsedKey = cert.PublicKey
+		} else {
+			return nil, err
+		}
+	}
+
+	// Test if parsed key is an ECDSA Public Key
+	var pubKey *ecdsa.PublicKey
+	var ok bool
+	if pubKey, ok = parsedKey.(*ecdsa.PublicKey); !ok {
+		return nil, fmt.Errorf("data doesn't contain valid ECDSA Public Key")
+	}
+
+	return pubKey, nil
+}
+
+func MakePrivatePublicKeyPairForKubelet() ([]byte, []byte, error) {
+	var privateKeyBytes []byte
+	var publicKeyBytes []byte
+	privateKeyBytes, err := common.MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "error generating private key")
+	}
+	pemBlock, _ := pem.Decode(privateKeyBytes)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "could not decode private key")
+	}
+	if publicKey, err := parseECPublicKey(pemBlock.Bytes); err == nil {
+		publicKeyBytes = publicKey.X.Bytes()
+	}
+	return privateKeyBytes, publicKeyBytes, nil
+}
+
+func (i *ignition) InjectKubeletTempPrivateKey(pathToSourceIgnition string, privateKeyBytes []byte, certPathToInject string) error {
+	sourceIgnition, err := i.ParseIgnitionFile(pathToSourceIgnition)
+	if err != nil {
+		return errors.Wrapf(err, "unable to parse ignition file %s", pathToSourceIgnition)
+	}
+	privateKeyFile := types.File{
+		Node: types.Node{
+			Overwrite: swag.Bool(true),
+			Path: certPathToInject,
+			User: types.NodeUser{
+				Name: swag.String("root"),
+			},
+		},
+		FileEmbedded1: types.FileEmbedded1{
+			Contents: types.Resource{
+				Source: swag.String(
+					fmt.Sprintf(
+						"data:text/plain;charset=utf-8;base64,%s",
+						base64.StdEncoding.EncodeToString([]byte(privateKeyBytes)),
+					),
+				),
+			},
+			Mode: swag.Int(420),
+		},
+	}
+	sourceIgnition.Storage.Files = append(sourceIgnition.Storage.Files, privateKeyFile)
+	return nil
+}

src/installer/installer.go

@@ -34,6 +34,7 @@ import (
 	"github.com/openshift/assisted-installer/src/utils"
 	"github.com/openshift/assisted-service/models"
 	preinstallUtils "github.com/rh-ecosystem-edge/preinstall-utils/pkg"
+
 )
 
 // In dry run mode we prefer to get quick feedback about errors rather
@@ -51,6 +52,7 @@ const (
 	singleNodeMasterIgnitionPath = "/opt/openshift/master.ign"
 	waitingForMastersStatusInfo  = "Waiting for masters to join bootstrap control plane"
 	waitingForBootstrapToPrepare = "Waiting for bootstrap node preparation"
+	ECPrivateKeyBlockType        = "EC PRIVATE KEY"
 )
 
 var generalWaitTimeout = 30 * time.Second
@@ -96,6 +98,19 @@ func (i *installer) FormatDisks() {
 	}
 }
 
+func (i *installer) StorePublicKeyInControlPlane(publicKey []byte, client k8s_client.K8SClient) error {
+	err := client.CreateNamespace("node-public-keys")
+	if err != nil {
+		i.log.Error("unable to create node-public-keys namespace on control plane", err)
+	}
+	err = client.CreateConfigMap(fmt.Sprintf("node-key-%s", i.HostID), "node-public-keys", map[string]string{"content": string(publicKey[:])})
+	if err != nil {
+		i.log.WithError(err).Errorf("unable to create config map for host ID %s for on control plane", i.HostID)
+	}
+	return nil
+}
+
+
 func (i *installer) InstallNode() error {
 	i.log.Infof("Installing node with role: %s", i.Config.Role)
 
@@ -141,15 +156,32 @@ func (i *installer) InstallNode() error {
 
 	}
 
+	// privateKeyBytes, publicKeyBytes, err := ignition.MakePrivatePublicKeyPairForKubelet()
+	_, publicKeyBytes, err := ignition.MakePrivatePublicKeyPairForKubelet()
+	if err != nil {
+		return errors.Wrap(err, "could not generate private/public key pair for temporary kubelet certificate")
+	}
+	// ign := ignition.NewIgnition()
+	// ign.InjectKubeletTempPrivateKey(ignitionPath, privateKeyBytes, "/var/lib/kubelet/pki/kubelet-client.key.tmp")
+
 	if err = i.writeImageToDisk(ignitionPath); err != nil {
 		return err
 	}
 
+	kc, err := i.kcBuilder(KubeconfigPath, i.log)
+	if err != nil {
+		i.log.Error(err)
+		return err
+	}
+
 	if i.Config.Role == string(models.HostRoleWorker) {
 		// Wait for 2 masters to be ready before rebooting
 		if err = i.workerWaitFor2ReadyMasters(ctx); err != nil {
 			return err
 		}
+		if err := i.StorePublicKeyInControlPlane(publicKeyBytes, kc); err != nil {
+			return errors.Wrap(err, "unable to store public key for node in control plane")
+		}
 	}
 
 	if i.EnableSkipMcoReboot {
@@ -171,6 +203,9 @@ func (i *installer) InstallNode() error {
 		if err = i.waitForControlPlane(ctx); err != nil {
 			return err
 		}
+		if err := i.StorePublicKeyInControlPlane(publicKeyBytes, kc); err != nil {
+			return errors.Wrap(err, "unable to store public key for node in control plane")
+		}
 		i.log.Info("Setting bootstrap node new role to master")
 	}
 
@@ -522,6 +557,21 @@ func (i *installer) waitForControlPlane(ctx context.Context) error {
 		return err
 	}
 
+	// // Build a namespace and config map to store some gathered information on hosts
+	// // So that this information may be used post reboot of the bootstrap to verify CSRs
+	// hostNames, err := i.getInventoryHostNames()
+	// if err != nil {
+	// 	i.log.Warnf("Failed to get hosts info from inventory, err %s", err)
+	// }
+	// err = kc.CreateNamespace("bootstrap-host-info")
+	// if err != nil {
+	// 	i.log.Error("unable to create bootstrap-host-info namespace on control plane", err)
+	// }
+	// err = kc.CreateConfigMap("known-hosts", "bootstrap-host-info", map[string]string{"content": strings.Join(hostNames, "\n")})
+	// if err != nil {
+	// 	i.log.Error("unable to create known-hosts cm on control plane", err)
+	// }
+
 	// waiting for controller pod to be running
 	if err = i.waitForController(kc); err != nil {
 		i.log.Error(err)
@@ -531,6 +581,19 @@ func (i *installer) waitForControlPlane(ctx context.Context) error {
 	return nil
 }
 
+// func (i *installer) getInventoryHostNames() ([]string, error) {
+// 	var keys []string
+// 	var inventoryHostsMap map[string]inventory_client.HostData
+// 	inventoryHostsMap, err := i.getInventoryHostsMap(inventoryHostsMap)
+// 	if err != nil {
+// 		return nil, errors.Wrap(err, "failed to get hosts info from inventory")
+// 	}
+// 	for k := range inventoryHostsMap {
+// 		keys = append(keys, k)
+// 	}
+// 	return keys, nil
+// }
+
 func (i *installer) waitForETCDBootstrap(ctx context.Context) error {
 	i.UpdateHostInstallProgress(models.HostStageWaitingForBootkube, "waiting for ETCD bootstrap to be complete")
 	i.log.Infof("Started waiting for ETCD bootstrap to complete")

src/installer/installer_test.go

@@ -310,6 +310,8 @@ var _ = Describe("installer HostRoleMaster role", func() {
 				for _, version := range []string{"4.7", "4.7.1", "4.7-pre-release", "4.8"} {
 					Context(version, func() {
 						BeforeEach(func() {
+							mockk8sclient.EXPECT().CreateNamespace("node-public-keys").Times(1)
+							mockk8sclient.EXPECT().CreateConfigMap(gomock.Any(), "node-public-keys", gomock.Any())
 							conf.OpenshiftVersion = version
 						})
 						AfterEach(func() {
@@ -442,6 +444,8 @@ var _ = Describe("installer HostRoleMaster role", func() {
 					Expect(ret).To(HaveOccurred())
 				})
 				It("bootstrap role extract ignition retry", func() {
+					mockk8sclient.EXPECT().CreateNamespace("node-public-keys").Times(1)
+					mockk8sclient.EXPECT().CreateConfigMap(gomock.Any(), "node-public-keys", gomock.Any())
 					updateProgressSuccess([][]string{{string(models.HostStageStartingInstallation), conf.Role},
 						{string(models.HostStageWaitingForControlPlane), waitingForBootstrapToPrepare},
 						{string(models.HostStageWaitingForControlPlane), waitingForMastersStatusInfo},
@@ -554,6 +558,8 @@ var _ = Describe("installer HostRoleMaster role", func() {
 						})
 
 						It(fmt.Sprintf("for platform type %v is expected to remove uninitialized taint = %v", platformType, expectedRemoveUninitializedTaint), func() {
+							mockk8sclient.EXPECT().CreateNamespace("node-public-keys").Times(1)
+							mockk8sclient.EXPECT().CreateConfigMap(gomock.Any(), "node-public-keys", gomock.Any())
 							updateProgressSuccess([][]string{{string(models.HostStageStartingInstallation), conf.Role},
 								{string(models.HostStageWaitingForControlPlane), waitingForBootstrapToPrepare},
 								{string(models.HostStageWaitingForControlPlane), waitingForMastersStatusInfo},
@@ -894,6 +900,8 @@ var _ = Describe("installer HostRoleMaster role", func() {
 					evaluateDiskSymlinkSuccess()
 				})
 				It("worker role happy flow", func() {
+					mockk8sclient.EXPECT().CreateNamespace("node-public-keys").Times(1)
+					mockk8sclient.EXPECT().CreateConfigMap(gomock.Any(), "node-public-keys", gomock.Any())
 					updateProgressSuccess([][]string{{string(models.HostStageStartingInstallation), conf.Role},
 						{string(models.HostStageInstalling), conf.Role},
 						{string(models.HostStageWritingImageToDisk)},

src/k8s_client/k8s_client.go

@@ -88,6 +88,8 @@ type K8SClient interface {
 	IsClusterCapabilityEnabled(configv1.ClusterVersionCapability) (bool, error)
 	UntaintNode(name string) error
 	PatchMachineConfigPoolPaused(pause bool, mcpName string) error
+	CreateNamespace(name string) error
+	CreateConfigMap(name string, namespace string, data map[string]string) error
 }
 
 type K8SClientBuilder func(configPath string, logger logrus.FieldLogger) (K8SClient, error)
@@ -691,3 +693,27 @@ func (c *k8sClient) PatchMachineConfigPoolPaused(pause bool, mcpName string) err
 	c.log.Infof("Setting pause MCP %s to %t", mcpName, pause)
 	return c.runtimeClient.Patch(context.TODO(), mcp, runtimeclient.RawPatch(types.MergePatchType, pausePatch))
 }
+
+func (c *k8sClient) CreateNamespace(name string) error {
+	if _, err := c.client.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}, metav1.CreateOptions{}); err != nil {
+		return errors.Wrap(err, "unable to create namespace for known hosts")
+	}
+	return nil
+}
+
+func (c *k8sClient) CreateConfigMap(name string, namespace string, data map[string]string) error {
+	if _, err := c.client.CoreV1().ConfigMaps(namespace).Create(context.TODO(), &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Data: data,
+	}, metav1.CreateOptions{}); err != nil {
+		return errors.Wrap(err, "unable to create config map for known hosts")
+	}
+	return nil
+}