diff --git a/_data.tf b/_data.tf new file mode 100644 index 0000000..90d881e --- /dev/null +++ b/_data.tf @@ -0,0 +1,8 @@ +variable "node_name_command" { + type = "map" + + default = { + "" = "hostname -f" + "aws" = "curl -s http://169.254.169.254/latest/meta-data/local-hostname" + } +} diff --git a/etcd.tf b/etcd.tf index 06e2b7a..30332fb 100644 --- a/etcd.tf +++ b/etcd.tf @@ -1,27 +1,29 @@ data "template_file" "etcd-cfssl-new-cert" { + count = "${length(var.etcd_addresses)}" template = "${file("${path.module}/resources/cfssl-new-cert.sh")}" vars { user = "etcd" group = "etcd" - role = "k8s-etcd" profile = "client-server" path = "/etc/etcd/ssl" + cn = "${count.index}.etcd.${var.dns_domain}" + org = "" - hosts = "${join(",", list( + extra_names = "${join(",", list( "etcd.${var.dns_domain}", - "*.etcd.${var.dns_domain}", ))}" } } data "ignition_file" "etcd-cfssl-new-cert" { + count = "${length(var.etcd_addresses)}" mode = 0755 filesystem = "root" path = "/opt/bin/cfssl-new-cert" content { - content = "${data.template_file.etcd-cfssl-new-cert.rendered}" + content = "${element(data.template_file.etcd-cfssl-new-cert.*.rendered, count.index)}" } } @@ -157,7 +159,7 @@ data "ignition_config" "etcd" { data.ignition_file.cfssl.id, data.ignition_file.cfssljson.id, data.ignition_file.cfssl-client-config.id, - data.ignition_file.etcd-cfssl-new-cert.id, + element(data.ignition_file.etcd-cfssl-new-cert.*.id, count.index), data.ignition_file.etcd-prom-machine-role.id, element(data.ignition_file.etcdctl-wrapper.*.id, count.index), ), diff --git a/master.tf b/master.tf index 1a48436..20508f5 100644 --- a/master.tf +++ b/master.tf @@ -4,11 +4,12 @@ data "template_file" "master-cfssl-new-cert" { vars { user = "root" group = "root" - role = "k8s-apiserver" profile = "client-server" path = "/etc/kubernetes/ssl" + cn = "system:node:$(${var.node_name_command[var.cloud_provider]})" + org = "system:nodes" - hosts = "${join(",", list( + extra_names = "${join(",", list( "10.3.0.1", "kubernetes", "kubernetes.default", diff --git a/resources/cfssl-new-cert.sh b/resources/cfssl-new-cert.sh index 391b080..babc8c0 100644 --- a/resources/cfssl-new-cert.sh +++ b/resources/cfssl-new-cert.sh @@ -11,8 +11,8 @@ _hostname="$(hostname)" /opt/bin/cfssl gencert \ -config=/etc/cfssl/config.json \ -profile=${profile} \ - -hostname="$${_ip},$${_hostname},${hosts}" - << EOF | /opt/bin/cfssljson -bare "${role}" -{"CN":"${role}","key":{"algo":"ecdsa","size":384}} + -hostname="$${_ip},$${_hostname}${extra_names != "" ? ",${extra_names}" : "" }" - << EOF | /opt/bin/cfssljson -bare node +{"CN":"${cn}",${org != "" ? "\"names\":[{\"O\":\"${org}\"}]," : ""}"key":{"algo":"ecdsa","size":384}} EOF /opt/bin/cfssl info -config=/etc/cfssl/config.json | /opt/bin/cfssljson -bare ca diff --git a/resources/etcd-member-dropin.conf b/resources/etcd-member-dropin.conf index 21b4685..484c2e0 100644 --- a/resources/etcd-member-dropin.conf +++ b/resources/etcd-member-dropin.conf @@ -12,12 +12,12 @@ Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${private_ipv4}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${private_ipv4}:2380" Environment="ETCD_CLIENT_CERT_AUTH=true" Environment="ETCD_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.pem" -Environment="ETCD_CERT_FILE=/etc/etcd/ssl/k8s-etcd.pem" -Environment="ETCD_KEY_FILE=/etc/etcd/ssl/k8s-etcd-key.pem" +Environment="ETCD_CERT_FILE=/etc/etcd/ssl/node.pem" +Environment="ETCD_KEY_FILE=/etc/etcd/ssl/node-key.pem" Environment="ETCD_PEER_CLIENT_CERT_AUTH=true" Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.pem" -Environment="ETCD_PEER_CERT_FILE=/etc/etcd/ssl/k8s-etcd.pem" -Environment="ETCD_PEER_KEY_FILE=/etc/etcd/ssl/k8s-etcd-key.pem" +Environment="ETCD_PEER_CERT_FILE=/etc/etcd/ssl/node.pem" +Environment="ETCD_PEER_KEY_FILE=/etc/etcd/ssl/node-key.pem" Environment="RKT_RUN_ARGS=\ --uuid-file-save=/var/lib/coreos/etcd-member-wrapper.uuid \ --volume etc-etcd,kind=host,source=/etc/etcd,readOnly=true \ diff --git a/resources/etcd-metrics-proxy.service b/resources/etcd-metrics-proxy.service index e87036b..8314b48 100644 --- a/resources/etcd-metrics-proxy.service +++ b/resources/etcd-metrics-proxy.service @@ -12,8 +12,8 @@ ExecStart=/bin/sh -c "\ -v /etc/etcd/ssl:/etc/etcd/ssl \ quay.io/utilitywarehouse/etcd-metrics-proxy:v0.6.1 \ -etcd-ca /etc/etcd/ssl/ca.pem \ - -etcd-cert /etc/etcd/ssl/k8s-etcd.pem \ - -etcd-key /etc/etcd/ssl/k8s-etcd-key.pem \ + -etcd-cert /etc/etcd/ssl/node.pem \ + -etcd-key /etc/etcd/ssl/node-key.pem \ -upstream-host ${etcd_ip} \ -upstream-server-name ${etcd_ip}" ExecStop=-/bin/sh -c 'docker stop -t 3 "$(docker ps -q --filter=name=%p_)"' diff --git a/resources/etcdctl-wrapper b/resources/etcdctl-wrapper index 91b09e8..d147b19 100644 --- a/resources/etcdctl-wrapper +++ b/resources/etcdctl-wrapper @@ -5,7 +5,7 @@ docker run --rm \ --entrypoint /usr/local/bin/etcdctl \ ${etcd_image_url}:${etcd_image_tag} \ --ca-file /etc/etcd/ssl/ca.pem \ - --cert-file /etc/etcd/ssl/k8s-etcd.pem \ - --key-file /etc/etcd/ssl/k8s-etcd-key.pem \ + --cert-file /etc/etcd/ssl/node.pem \ + --key-file /etc/etcd/ssl/node-key.pem \ --endpoint https://${private_ipv4}:2379 \ "$@" diff --git a/resources/kube-apiserver.yaml b/resources/kube-apiserver.yaml index c345549..5dcd79a 100644 --- a/resources/kube-apiserver.yaml +++ b/resources/kube-apiserver.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Pod metadata: @@ -13,14 +14,14 @@ spec: - apiserver - --etcd-servers=${etcd_endpoints} - --etcd-cafile=/etc/kubernetes/ssl/ca.pem - - --etcd-certfile=/etc/kubernetes/ssl/k8s-apiserver.pem - - --etcd-keyfile=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --etcd-certfile=/etc/kubernetes/ssl/node.pem + - --etcd-keyfile=/etc/kubernetes/ssl/node-key.pem - --allow-privileged=true - --service-cluster-ip-range=${service_network} - --secure-port=443 - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota - - --tls-cert-file=/etc/kubernetes/ssl/k8s-apiserver.pem - - --tls-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem + - --admission-control=NodeRestriction,NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota + - --tls-cert-file=/etc/kubernetes/ssl/node.pem + - --tls-private-key-file=/etc/kubernetes/ssl/node-key.pem - --client-ca-file=/etc/kubernetes/ssl/ca.pem - --service-account-key-file=/etc/kubernetes/ssl/signing-key.pem - --service-account-lookup=true @@ -29,8 +30,7 @@ spec: - --oidc-issuer-url=${oidc_issuer_url} - --oidc-username-claim=email - --oidc-client-id=${oidc_client_id} - - --authorization-rbac-super-user=k8s-admin - - --authorization-mode=RBAC + - --authorization-mode=Node,RBAC - --apiserver-count=${master_instance_count} - --audit-log-path=/var/log/kube-api-server/audit - --v=0 diff --git a/resources/kube-controller-manager.yaml b/resources/kube-controller-manager.yaml index fc4420b..5aaf199 100644 --- a/resources/kube-controller-manager.yaml +++ b/resources/kube-controller-manager.yaml @@ -14,6 +14,7 @@ spec: - controller-manager - --master=http://127.0.0.1:8080 - --leader-elect=true + - --use-service-account-credentials - --service-account-private-key-file=/etc/kubernetes/ssl/signing-key.pem - --root-ca-file=/etc/kubernetes/ssl/ca.pem ${cloud_provider == "" ? "" : "- --cloud-provider=${cloud_provider}"} diff --git a/resources/master-kubeconfig b/resources/master-kubeconfig index 886d085..8b835e4 100644 --- a/resources/master-kubeconfig +++ b/resources/master-kubeconfig @@ -8,8 +8,8 @@ clusters: users: - name: kubelet user: - client-certificate: /etc/kubernetes/ssl/k8s-apiserver.pem - client-key: /etc/kubernetes/ssl/k8s-apiserver-key.pem + client-certificate: /etc/kubernetes/ssl/node.pem + client-key: /etc/kubernetes/ssl/node-key.pem contexts: - context: cluster: local diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service index 948698d..1ab1de1 100644 --- a/resources/master-kubelet.service +++ b/resources/master-kubelet.service @@ -45,3 +45,4 @@ Restart=always RestartSec=10 [Install] WantedBy=multi-user.target +After=systemd-resolved.service diff --git a/resources/worker-kubeconfig b/resources/worker-kubeconfig index 40fae4d..aee733d 100644 --- a/resources/worker-kubeconfig +++ b/resources/worker-kubeconfig @@ -8,8 +8,8 @@ clusters: users: - name: kubelet user: - client-certificate: /etc/kubernetes/ssl/k8s-worker.pem - client-key: /etc/kubernetes/ssl/k8s-worker-key.pem + client-certificate: /etc/kubernetes/ssl/node.pem + client-key: /etc/kubernetes/ssl/node-key.pem contexts: - context: cluster: local diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service index 269d2a8..6ce186e 100644 --- a/resources/worker-kubelet.service +++ b/resources/worker-kubelet.service @@ -50,3 +50,4 @@ Restart=always RestartSec=10 [Install] WantedBy=multi-user.target +After=systemd-resolved.service diff --git a/worker.tf b/worker.tf index 0afd019..27739ce 100644 --- a/worker.tf +++ b/worker.tf @@ -2,15 +2,13 @@ data "template_file" "worker-cfssl-new-cert" { template = "${file("${path.module}/resources/cfssl-new-cert.sh")}" vars { - user = "root" - group = "root" - role = "k8s-worker" - profile = "client" - path = "/etc/kubernetes/ssl" - - hosts = "${join(",", list( - "*.worker.${var.dns_domain}", - ))}" + user = "root" + group = "root" + profile = "client" + path = "/etc/kubernetes/ssl" + cn = "system:node:$(${var.node_name_command[var.cloud_provider]})" + org = "system:nodes" + extra_names = "" } }