Merge pull request #7 from utilitywarehouse/node-auth

kubernetes node authentication
utilitywarehouse · Oct 10, 2017 · eb3dc3c · eb3dc3c
2 parents e1fb215 + c20fd9c
commit eb3dc3c
Show file tree

Hide file tree

Showing 14 changed files with 49 additions and 37 deletions.
diff --git a/_data.tf b/_data.tf
@@ -0,0 +1,8 @@
+variable "node_name_command" {
+  type = "map"
+
+  default = {
+    ""    = "hostname -f"
+    "aws" = "curl -s http://169.254.169.254/latest/meta-data/local-hostname"
+  }
+}
diff --git a/etcd.tf b/etcd.tf
@@ -1,27 +1,29 @@
 data "template_file" "etcd-cfssl-new-cert" {
+  count    = "${length(var.etcd_addresses)}"
   template = "${file("${path.module}/resources/cfssl-new-cert.sh")}"
 
   vars {
     user    = "etcd"
     group   = "etcd"
-    role    = "k8s-etcd"
     profile = "client-server"
     path    = "/etc/etcd/ssl"
+    cn      = "${count.index}.etcd.${var.dns_domain}"
+    org     = ""
 
-    hosts = "${join(",", list(
+    extra_names = "${join(",", list(
       "etcd.${var.dns_domain}",
-      "*.etcd.${var.dns_domain}",
     ))}"
   }
 }
 
 data "ignition_file" "etcd-cfssl-new-cert" {
+  count      = "${length(var.etcd_addresses)}"
   mode       = 0755
   filesystem = "root"
   path       = "/opt/bin/cfssl-new-cert"
 
   content {
-    content = "${data.template_file.etcd-cfssl-new-cert.rendered}"
+    content = "${element(data.template_file.etcd-cfssl-new-cert.*.rendered, count.index)}"
   }
 }
 
@@ -157,7 +159,7 @@ data "ignition_config" "etcd" {
         data.ignition_file.cfssl.id,
         data.ignition_file.cfssljson.id,
         data.ignition_file.cfssl-client-config.id,
-        data.ignition_file.etcd-cfssl-new-cert.id,
+        element(data.ignition_file.etcd-cfssl-new-cert.*.id, count.index),
         data.ignition_file.etcd-prom-machine-role.id,
         element(data.ignition_file.etcdctl-wrapper.*.id, count.index),
     ),

diff --git a/master.tf b/master.tf
@@ -4,11 +4,12 @@ data "template_file" "master-cfssl-new-cert" {
   vars {
     user    = "root"
     group   = "root"
-    role    = "k8s-apiserver"
     profile = "client-server"
     path    = "/etc/kubernetes/ssl"
+    cn      = "system:node:$(${var.node_name_command[var.cloud_provider]})"
+    org     = "system:nodes"
 
-    hosts = "${join(",", list(
+    extra_names = "${join(",", list(
       "10.3.0.1",
       "kubernetes",
       "kubernetes.default",

diff --git a/resources/cfssl-new-cert.sh b/resources/cfssl-new-cert.sh
@@ -11,8 +11,8 @@ _hostname="$(hostname)"
 /opt/bin/cfssl gencert \
   -config=/etc/cfssl/config.json \
   -profile=${profile} \
-  -hostname="$${_ip},$${_hostname},${hosts}" - << EOF | /opt/bin/cfssljson -bare "${role}"
-{"CN":"${role}","key":{"algo":"ecdsa","size":384}}
+  -hostname="$${_ip},$${_hostname}${extra_names != "" ? ",${extra_names}" : "" }" - << EOF | /opt/bin/cfssljson -bare node
+{"CN":"${cn}",${org != "" ? "\"names\":[{\"O\":\"${org}\"}]," : ""}"key":{"algo":"ecdsa","size":384}}
 EOF
 
 /opt/bin/cfssl info -config=/etc/cfssl/config.json | /opt/bin/cfssljson -bare ca

diff --git a/resources/etcd-member-dropin.conf b/resources/etcd-member-dropin.conf
@@ -12,12 +12,12 @@ Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${private_ipv4}:2379"
 Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${private_ipv4}:2380"
 Environment="ETCD_CLIENT_CERT_AUTH=true"
 Environment="ETCD_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.pem"
-Environment="ETCD_CERT_FILE=/etc/etcd/ssl/k8s-etcd.pem"
-Environment="ETCD_KEY_FILE=/etc/etcd/ssl/k8s-etcd-key.pem"
+Environment="ETCD_CERT_FILE=/etc/etcd/ssl/node.pem"
+Environment="ETCD_KEY_FILE=/etc/etcd/ssl/node-key.pem"
 Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
 Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/etcd/ssl/ca.pem"
-Environment="ETCD_PEER_CERT_FILE=/etc/etcd/ssl/k8s-etcd.pem"
-Environment="ETCD_PEER_KEY_FILE=/etc/etcd/ssl/k8s-etcd-key.pem"
+Environment="ETCD_PEER_CERT_FILE=/etc/etcd/ssl/node.pem"
+Environment="ETCD_PEER_KEY_FILE=/etc/etcd/ssl/node-key.pem"
 Environment="RKT_RUN_ARGS=\
   --uuid-file-save=/var/lib/coreos/etcd-member-wrapper.uuid \
   --volume etc-etcd,kind=host,source=/etc/etcd,readOnly=true \

diff --git a/resources/etcd-metrics-proxy.service b/resources/etcd-metrics-proxy.service
@@ -12,8 +12,8 @@ ExecStart=/bin/sh -c "\
       -v /etc/etcd/ssl:/etc/etcd/ssl \
       quay.io/utilitywarehouse/etcd-metrics-proxy:v0.6.1 \
       -etcd-ca /etc/etcd/ssl/ca.pem \
-      -etcd-cert /etc/etcd/ssl/k8s-etcd.pem \
-      -etcd-key /etc/etcd/ssl/k8s-etcd-key.pem \
+      -etcd-cert /etc/etcd/ssl/node.pem \
+      -etcd-key /etc/etcd/ssl/node-key.pem \
       -upstream-host ${etcd_ip} \
       -upstream-server-name ${etcd_ip}"
 ExecStop=-/bin/sh -c 'docker stop -t 3 "$(docker ps -q --filter=name=%p_)"'

diff --git a/resources/etcdctl-wrapper b/resources/etcdctl-wrapper
@@ -5,7 +5,7 @@ docker run --rm \
     --entrypoint /usr/local/bin/etcdctl \
     ${etcd_image_url}:${etcd_image_tag} \
     --ca-file /etc/etcd/ssl/ca.pem \
-    --cert-file /etc/etcd/ssl/k8s-etcd.pem \
-    --key-file /etc/etcd/ssl/k8s-etcd-key.pem \
+    --cert-file /etc/etcd/ssl/node.pem \
+    --key-file /etc/etcd/ssl/node-key.pem \
     --endpoint https://${private_ipv4}:2379 \
     "$@"
diff --git a/resources/kube-apiserver.yaml b/resources/kube-apiserver.yaml
@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Pod
 metadata:
@@ -13,14 +14,14 @@ spec:
     - apiserver
     - --etcd-servers=${etcd_endpoints}
     - --etcd-cafile=/etc/kubernetes/ssl/ca.pem
-    - --etcd-certfile=/etc/kubernetes/ssl/k8s-apiserver.pem
-    - --etcd-keyfile=/etc/kubernetes/ssl/k8s-apiserver-key.pem
+    - --etcd-certfile=/etc/kubernetes/ssl/node.pem
+    - --etcd-keyfile=/etc/kubernetes/ssl/node-key.pem
     - --allow-privileged=true
     - --service-cluster-ip-range=${service_network}
     - --secure-port=443
-    - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
-    - --tls-cert-file=/etc/kubernetes/ssl/k8s-apiserver.pem
-    - --tls-private-key-file=/etc/kubernetes/ssl/k8s-apiserver-key.pem
+    - --admission-control=NodeRestriction,NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
+    - --tls-cert-file=/etc/kubernetes/ssl/node.pem
+    - --tls-private-key-file=/etc/kubernetes/ssl/node-key.pem
     - --client-ca-file=/etc/kubernetes/ssl/ca.pem
     - --service-account-key-file=/etc/kubernetes/ssl/signing-key.pem
     - --service-account-lookup=true
@@ -29,8 +30,7 @@ spec:
     - --oidc-issuer-url=${oidc_issuer_url}
     - --oidc-username-claim=email
     - --oidc-client-id=${oidc_client_id}
-    - --authorization-rbac-super-user=k8s-admin
-    - --authorization-mode=RBAC
+    - --authorization-mode=Node,RBAC
     - --apiserver-count=${master_instance_count}
     - --audit-log-path=/var/log/kube-api-server/audit
     - --v=0

diff --git a/resources/kube-controller-manager.yaml b/resources/kube-controller-manager.yaml
@@ -14,6 +14,7 @@ spec:
     - controller-manager
     - --master=http://127.0.0.1:8080
     - --leader-elect=true
+    - --use-service-account-credentials
     - --service-account-private-key-file=/etc/kubernetes/ssl/signing-key.pem
     - --root-ca-file=/etc/kubernetes/ssl/ca.pem
     ${cloud_provider == "" ? "" : "- --cloud-provider=${cloud_provider}"}

diff --git a/resources/master-kubeconfig b/resources/master-kubeconfig
@@ -8,8 +8,8 @@ clusters:
 users:
 - name: kubelet
   user:
-    client-certificate: /etc/kubernetes/ssl/k8s-apiserver.pem
-    client-key: /etc/kubernetes/ssl/k8s-apiserver-key.pem
+    client-certificate: /etc/kubernetes/ssl/node.pem
+    client-key: /etc/kubernetes/ssl/node-key.pem
 contexts:
 - context:
     cluster: local

diff --git a/resources/master-kubelet.service b/resources/master-kubelet.service
@@ -45,3 +45,4 @@ Restart=always
 RestartSec=10
 [Install]
 WantedBy=multi-user.target
+After=systemd-resolved.service
diff --git a/resources/worker-kubeconfig b/resources/worker-kubeconfig
@@ -8,8 +8,8 @@ clusters:
 users:
 - name: kubelet
   user:
-    client-certificate: /etc/kubernetes/ssl/k8s-worker.pem
-    client-key: /etc/kubernetes/ssl/k8s-worker-key.pem
+    client-certificate: /etc/kubernetes/ssl/node.pem
+    client-key: /etc/kubernetes/ssl/node-key.pem
 contexts:
 - context:
     cluster: local

diff --git a/resources/worker-kubelet.service b/resources/worker-kubelet.service
@@ -50,3 +50,4 @@ Restart=always
 RestartSec=10
 [Install]
 WantedBy=multi-user.target
+After=systemd-resolved.service
diff --git a/worker.tf b/worker.tf
@@ -2,15 +2,13 @@ data "template_file" "worker-cfssl-new-cert" {
   template = "${file("${path.module}/resources/cfssl-new-cert.sh")}"
 
   vars {
-    user    = "root"
-    group   = "root"
-    role    = "k8s-worker"
-    profile = "client"
-    path    = "/etc/kubernetes/ssl"
-
-    hosts = "${join(",", list(
-      "*.worker.${var.dns_domain}",
-    ))}"
+    user        = "root"
+    group       = "root"
+    profile     = "client"
+    path        = "/etc/kubernetes/ssl"
+    cn          = "system:node:$(${var.node_name_command[var.cloud_provider]})"
+    org         = "system:nodes"
+    extra_names = ""
   }
 }