sys: Configure the aggregation layer (#42)

https://kubernetes.io/docs/tasks/access-kubernetes-api/configure-aggregation-layer/

Using https://github.com/Azure/acs-engine/pull/1406/files for reference

Required by: https://github.com/kubernetes-incubator/metrics-server

cfssl.tf

@@ -76,6 +76,36 @@ data "ignition_file" "cfssl-init-ca" {
   }
 }
 
+data "ignition_file" "cfssl-init-proxy-pki" {
+  mode       = 0755
+  filesystem = "root"
+  path       = "/opt/bin/cfssl-init-proxy-pki"
+
+  content {
+    content = "${file("${path.module}/resources/cfssl-init-proxy-pki")}"
+  }
+}
+
+data "ignition_file" "cfssl-proxy-ca-csr-json" {
+  mode       = 0644
+  filesystem = "root"
+  path       = "/etc/cfssl/proxy-ca-csr.json"
+
+  content {
+    content = "${file("${path.module}/resources/cfssl-proxy-ca-csr.json")}"
+  }
+}
+
+data "ignition_file" "cfssl-proxy-csr-json" {
+  mode       = 0644
+  filesystem = "root"
+  path       = "/etc/cfssl/proxy-csr.json"
+
+  content {
+    content = "${file("${path.module}/resources/cfssl-proxy-csr.json")}"
+  }
+}
+
 data "template_file" "cfssl-server-config" {
   template = "${file("${path.module}/resources/cfssl-server-config.json")}"
 
@@ -166,6 +196,9 @@ data "ignition_config" "cfssl" {
         data.ignition_file.cfssl-ca-csr.id,
         data.ignition_file.cfssl-init-ca.id,
         data.ignition_file.cfssl-sk-csr.id,
+        data.ignition_file.cfssl-init-proxy-pki.id,
+        data.ignition_file.cfssl-proxy-ca-csr-json.id,
+        data.ignition_file.cfssl-proxy-csr-json.id,
         data.ignition_file.cfssl-nginx-conf.id,
         data.ignition_file.cfssl-nginx-auth.id,
         data.ignition_file.format-and-mount.id,

master.tf

@@ -37,22 +37,22 @@ data "ignition_file" "master-cfssl-new-cert" {
   }
 }
 
-data "template_file" "master-cfssl-sk-get" {
-  template = "${file("${path.module}/resources/cfssl-sk-get.sh")}"
+data "template_file" "master-cfssl-keys-and-certs-get" {
+  template = "${file("${path.module}/resources/cfssl-keys-and-certs-get")}"
 
   vars {
     path = "/etc/kubernetes/ssl"
     auth = "${base64encode("apiserver:${random_id.cfssl-auth-key-apiserver.hex}")}"
   }
 }
 
-data "ignition_file" "master-cfssl-sk-get" {
+data "ignition_file" "master-cfssl-keys-and-certs-get" {
   mode       = 0755
   filesystem = "root"
-  path       = "/opt/bin/cfssl-sk-get"
+  path       = "/opt/bin/cfssl-keys-and-certs-get"
 
   content {
-    content = "${data.template_file.master-cfssl-sk-get.rendered}"
+    content = "${data.template_file.master-cfssl-keys-and-certs-get.rendered}"
   }
 }
 
@@ -215,7 +215,7 @@ data "ignition_config" "master" {
         data.ignition_file.cfssljson.id,
         data.ignition_file.cfssl-client-config.id,
         data.ignition_file.master-cfssl-new-cert.id,
-        data.ignition_file.master-cfssl-sk-get.id,
+        data.ignition_file.master-cfssl-keys-and-certs-get.id,
         data.ignition_file.master-prom-machine-role.id,
         data.ignition_file.master-kubeconfig.id,
         data.ignition_file.kubelet-kubeconfig.id,

resources/cfssl-init-proxy-pki

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+_ca_args="/etc/cfssl/proxy-ca-csr.json"
+_args="/etc/cfssl/proxy-csr.json"
+
+if [ ! -f "${_ca_args}" ]; then
+    echo "${_ca_args} not found"
+    exit 1
+fi
+
+if [ ! -f "${_args}" ]; then
+    echo "${_args} not found"
+    exit 1
+fi
+
+[ -f proxy-ca-key.pem ] && _ca_args="-ca-key=proxy-ca-key.pem ${_ca_args}"
+
+[ -f proxy-ca-key.pem ] && [ -f proxy-ca.pem ] \
+    && (( "$(date +%s)" < "$(date -d "$(/opt/bin/cfssl certinfo -cert=/var/lib/cfssl/proxy-ca.pem | jq -r '.not_after')" +%s)" - 7 * 24 * 3600 )) \
+    || /opt/bin/cfssl gencert -initca ${_ca_args} | /opt/bin/cfssljson -bare proxy-ca -
+
+[ -f proxy-key.pem ] && [ -f proxy.pem ] \
+    && (( "$(date +%s)" < "$(date -d "$(/opt/bin/cfssl certinfo -cert=/var/lib/cfssl/proxy.pem | jq -r '.not_after')" +%s)" - 7 * 24 * 3600 )) \
+    || /opt/bin/cfssl gencert -ca proxy-ca.pem -ca-key proxy-ca-key.pem ${_args} | /opt/bin/cfssljson -bare proxy

resources/cfssl-keys-and-certs-get

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+set -o errexit
+
+key() {
+  local key=$$1
+  /usr/bin/curl -Ls -o $${key} \
+      -H 'Authorization: Basic ${auth}' \
+      http://$(jq -r '.remotes.server | split(":")[0]' /etc/cfssl/config.json):8889/$${key}
+  set +e
+  /usr/bin/openssl ec -in ${path}/$${key} -noout
+  if [ $? -ne 0 ]; then
+    echo "Failed to get $${key} from cfssl server";
+    exit 1;
+  fi
+  set -e
+  /usr/bin/chmod 0600 $${key}
+}
+
+cert () {
+  local cert=$$1
+  /usr/bin/curl -Ls -o $${cert} \
+      -H 'Authorization: Basic ${auth}' \
+      http://$(jq -r '.remotes.server | split(":")[0]' /etc/cfssl/config.json):8889/$${cert}
+  set +e
+  /opt/bin/cfssl certinfo -cert $${cert}
+  if [ $? -ne 0 ]; then
+    echo "Failed to get $${cert} from cfssl server";
+    exit 1;
+  fi
+  set -e
+  /usr/bin/chmod 0600 $${cert}
+}
+
+mkdir -p ${path}
+cd ${path}
+
+key signing-key.pem
+cert proxy-ca.pem
+cert proxy.pem
+key proxy-key.pem

resources/cfssl-nginx.conf

@@ -19,6 +19,9 @@ http {
     auth_basic_user_file nginx.htpasswd;
     server {
         listen 8889 default_server;
-        location =/signing-key { alias /data/sk-key.pem; }
+        location =/signing-key.pem { alias /data/sk-key.pem; }
+        location =/proxy-ca.pem { alias /data/proxy-ca.pem; }
+        location =/proxy.pem { alias /data/proxy.pem; }
+        location =/proxy-key.pem { alias /data/proxy-key.pem; }
     }
 }

resources/cfssl-nginx.service

@@ -9,6 +9,9 @@ ExecStart=/bin/sh -c "\
     /usr/bin/docker run --rm \
       --name %p_$(uuidgen) \
       -v /var/lib/cfssl/sk-key.pem:/data/sk-key.pem \
+      -v /var/lib/cfssl/proxy-ca.pem:/data/proxy-ca.pem \
+      -v /var/lib/cfssl/proxy.pem:/data/proxy.pem \
+      -v /var/lib/cfssl/proxy-key.pem:/data/proxy-key.pem \
       -v /etc/cfssl/sk-nginx.conf:/etc/nginx/nginx.conf \
       -v /etc/cfssl/sk-nginx.htpasswd:/etc/nginx/nginx.htpasswd \
       -p 8889:8889 \

resources/cfssl-proxy-ca-csr.json

@@ -0,0 +1,7 @@
+{
+    "key": {
+        "algo": "ecdsa",
+        "size": 256
+    },
+    "CN": "proxyClientCA"
+}

resources/cfssl-proxy-csr.json

@@ -0,0 +1,12 @@
+{
+    "key": {
+        "algo": "ecdsa",
+        "size": 256
+    },
+    "CN": "aggregator",
+    "names": [
+        {
+            "O":  "system:masters"
+        }
+    ]
+}

resources/cfssl.service

@@ -6,6 +6,7 @@ Requires=disk-mounter.service
 WorkingDirectory=/var/lib/cfssl
 ExecStartPre=/bin/sh -c 'if [ ! -f sk-key.pem ]; then /opt/bin/cfssl genkey /etc/cfssl/sk-csr.json | /opt/bin/cfssljson -bare sk && rm sk.csr; fi'
 ExecStartPre=/opt/bin/cfssl-init-ca
+ExecStartPre=/opt/bin/cfssl-init-proxy-pki
 ExecStart=/opt/bin/cfssl serve \
     -address=0.0.0.0 \
     -port=8888 \

resources/kube-apiserver.yaml

@@ -1,4 +1,3 @@
----
 apiVersion: v1
 kind: Pod
 metadata:
@@ -39,6 +38,13 @@ spec:
     - --audit-log-maxsize=100
     - --audit-log-maxbackup=20
     - --external-hostname=${master_address}
+    - --requestheader-client-ca-file=/etc/kubernetes/ssl/proxy-ca.pem
+    - --requestheader-allowed-names=aggregator
+    - --requestheader-extra-headers-prefix=X-Remote-Extra-
+    - --requestheader-group-headers=X-Remote-Group
+    - --requestheader-username-headers=X-Remote-User
+    - --proxy-client-cert-file=/etc/kubernetes/ssl/proxy.pem
+    - --proxy-client-key-file=/etc/kubernetes/ssl/proxy-key.pem
     - --v=0
     livenessProbe:
       httpGet:

resources/master-kubelet.service

@@ -24,7 +24,7 @@ ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
 # This is a partial workaround to this upstream Kubernetes issue:
 #  https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731
 ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8
-ExecStartPre=/opt/bin/cfssl-sk-get
+ExecStartPre=/opt/bin/cfssl-keys-and-certs-get
 ExecStartPre=/opt/bin/cfssl-new-cert
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-controller-manager' | awk '{ print $1; }')"
 ExecStartPre=-/bin/sh -c "docker restart $(docker ps --no-trunc | grep 'kube-apiserver' | awk '{ print $1; }')"