Skip to content

Commit

Permalink
Update dhall-kubernetes version to v4.0.0
Browse files Browse the repository at this point in the history 
This change update the kubernetes binding to use the new
Optional types. The main consequence is that all the fields that
are optional needs to be prefixed with Some. This let us remove
the `--omit-empty` parameter resulting in cleaner resources where
we don't need to set a dummy emptyDir medium value.

See this issue for the details:
dhall-lang/dhall-kubernetes#86

Change-Id: I23a0a028909208cd58f57a6f07ee93090b3f3a1a
  • Loading branch information
@TristanCacqueray
TristanCacqueray committed Apr 6, 2020
1 parent 27f9126 commit 0f75963
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 82 deletions.
4 changes: 2 additions & 2 deletions CONTRIBUTE.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,13 @@ INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/c
Then you can evaluate the resources function, for example to get the scheduler service:

```bash
dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
```

Or get all the kubernetes resources:

```bash
dhall-to-yaml --omit-empty <<< "(./conf/zuul/resources.dhall ($INPUT)).List"
dhall-to-yaml <<< "(./conf/zuul/resources.dhall ($INPUT)).List"
```

## Run the ansible roles locally
Expand Down
2 changes: 1 addition & 1 deletion build/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ ARG DHALL_VERSION=1.30.0
ARG DHALL_JSON_VERSION=1.6.2
ARG DHALL_JSON_DIGEST=ea37627c4e19789af33def099d4cb145b874c03b4d5b98cb33ce06be1debf4f3
ARG DHALL_LANG_REF=v14.0.0
ARG DHALL_KUBE_REF=v3.0.0
ARG DHALL_KUBE_REF=v4.0.0

# kubectl versions and digests
ARG KUBECTL_VERSION=v1.17.0
Expand Down
2 changes: 1 addition & 1 deletion conf/Kubernetes.dhall
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{- Import the kubernetes types, see the ./Prelude.dhall file for documentation -}
env:DHALL_KUBERNETES
? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/3c6d09a9409977cdde58a091d76a6d20509ca4b0/package.dhall sha256:e9c55c7ff71f901314129e7ef100c3af5ec7a918dce25e06d83fa8c5472cb680
? https://raw.githubusercontent.com/dhall-lang/dhall-kubernetes/v4.0.0/package.dhall sha256:d9eac5668d5ed9cb3364c0a39721d4694e4247dad16d8a82827e4619ee1d6188
160 changes: 85 additions & 75 deletions conf/zuul/resources.dhall
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,11 +264,11 @@ in \(input : Input)
let mkObjectMeta =
\(name : Text)
-> \(labels : Labels)
-> Kubernetes.ObjectMeta::{ name = name, labels = labels }
-> Kubernetes.ObjectMeta::{ name = name, labels = Some labels }

let mkSelector =
\(labels : Labels)
-> Kubernetes.LabelSelector::{ matchLabels = labels }
-> Kubernetes.LabelSelector::{ matchLabels = Some labels }

let mkService =
\(name : Text)
Expand All @@ -280,8 +280,8 @@ in \(input : Input)
, metadata = mkObjectMeta name labels
, spec = Some Kubernetes.ServiceSpec::{
, type = Some "ClusterIP"
, selector = labels
, ports =
, selector = Some labels
, ports = Some
[ Kubernetes.ServicePort::{
, name = Some port-name
, protocol = Some "TCP"
Expand All @@ -300,9 +300,7 @@ in \(input : Input)
( \(volume : Volume.Type)
-> Kubernetes.Volume::{
, name = volume.name
, emptyDir = Kubernetes.EmptyDirVolumeSource::{
, medium = Some ""
}
, emptyDir = Some Kubernetes.EmptyDirVolumeSource::{=}
}
)

Expand All @@ -326,9 +324,10 @@ in \(input : Input)
-> Kubernetes.PodTemplateSpec::{
, metadata = mkObjectMeta component.name labels
, spec = Some Kubernetes.PodSpec::{
, volumes =
mkVolumeSecret component.volumes
# mkVolumeEmptyDir component.data-dir
, volumes = Some
( mkVolumeSecret component.volumes
# mkVolumeEmptyDir component.data-dir
)
, containers = [ component.container ]
, automountServiceAccountToken = Some False
}
Expand All @@ -351,14 +350,16 @@ in \(input : Input)
, metadata =
mkObjectMeta component-name ([] : Labels)
, spec = Some Kubernetes.PersistentVolumeClaimSpec::{
, accessModes = [ "ReadWriteOnce" ]
, accessModes = Some [ "ReadWriteOnce" ]
, resources = Some Kubernetes.ResourceRequirements::{
, requests = toMap
{ storage =
Natural/show
component.claim-size
++ "Gi"
}
, requests = Some
( toMap
{ storage =
Natural/show
component.claim-size
++ "Gi"
}
)
}
}
}
Expand All @@ -371,7 +372,7 @@ in \(input : Input)
, replicas = Some component.count
, selector = mkSelector labels
, template = mkPodTemplateSpec component labels
, volumeClaimTemplates = claim
, volumeClaimTemplates = Some claim
}
}

Expand Down Expand Up @@ -408,7 +409,7 @@ in \(input : Input)
( \(env : EnvSecret)
-> Kubernetes.EnvVar::{
, name = env.name
, valueFrom = Kubernetes.EnvVarSource::{
, valueFrom = Some Kubernetes.EnvVarSource::{
, secretKeyRef = Some Kubernetes.SecretKeySelector::{
, key = env.key
, name = Some env.secret
Expand All @@ -433,16 +434,17 @@ in \(input : Input)
-> Kubernetes.Resource.Secret
Kubernetes.Secret::{
, metadata = Kubernetes.ObjectMeta::{ name = volume.name }
, stringData =
Prelude.List.map
File
{ mapKey : Text, mapValue : Text }
( \(config : File)
-> { mapKey = config.path
, mapValue = config.content
}
)
volume.files
, stringData = Some
( Prelude.List.map
File
{ mapKey : Text, mapValue : Text }
( \(config : File)
-> { mapKey = config.path
, mapValue = config.content
}
)
volume.files
)
}

let zk-hosts =
Expand Down Expand Up @@ -546,23 +548,25 @@ in \(input : Input)
, image = Some
"docker.io/library/postgres:12.1"
, imagePullPolicy = Some "IfNotPresent"
, ports =
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "pg"
, containerPort = 5432
}
]
, env =
mkEnvVarValue
( toMap
{ POSTGRES_USER = "zuul"
, POSTGRES_PASSWORD =
default-db-password
, PGDATA = "/var/lib/pg/data"
}
)
, volumeMounts =
mkVolumeMount db-volumes
, env = Some
( mkEnvVarValue
( toMap
{ POSTGRES_USER = "zuul"
, POSTGRES_PASSWORD =
default-db-password
, PGDATA =
"/var/lib/pg/data"
}
)
)
, volumeMounts = Some
(mkVolumeMount db-volumes)
}
}
)
Expand All @@ -588,14 +592,14 @@ in \(input : Input)
, image = Some
"docker.io/library/zookeeper"
, imagePullPolicy = Some "IfNotPresent"
, ports =
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "zk"
, containerPort = 2181
}
]
, volumeMounts =
mkVolumeMount zk-volumes
, volumeMounts = Some
(mkVolumeMount zk-volumes)
}
}
)
Expand Down Expand Up @@ -672,21 +676,23 @@ in \(input : Input)
, container = Kubernetes.Container::{
, name = "scheduler"
, image = zuul-image "scheduler"
, args = [ "zuul-scheduler", "-d" ]
, args = Some [ "zuul-scheduler", "-d" ]
, imagePullPolicy = Some "IfNotPresent"
, ports =
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "gearman"
, containerPort = 4730
}
]
, env =
zuul-env
# db-uri-secret-env
# zk-hosts-secret-env
, volumeMounts =
mkVolumeMount
(scheduler-volumes # zuul-data-dir)
, env = Some
( zuul-env
# db-uri-secret-env
# zk-hosts-secret-env
)
, volumeMounts = Some
( mkVolumeMount
(scheduler-volumes # zuul-data-dir)
)
}
}
)
Expand All @@ -704,18 +710,19 @@ in \(input : Input)
, container = Kubernetes.Container::{
, name = "executor"
, image = zuul-image "executor"
, args = [ "zuul-executor", "-d" ]
, args = Some [ "zuul-executor", "-d" ]
, imagePullPolicy = Some "IfNotPresent"
, ports =
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "finger"
, containerPort = 7900
}
]
, env = zuul-env
, volumeMounts =
mkVolumeMount
(executor-volumes # zuul-data-dir)
, env = Some zuul-env
, volumeMounts = Some
( mkVolumeMount
(executor-volumes # zuul-data-dir)
)
, securityContext = Some Kubernetes.SecurityContext::{
, privileged = Some True
}
Expand All @@ -735,18 +742,19 @@ in \(input : Input)
, container = Kubernetes.Container::{
, name = "web"
, image = zuul-image "web"
, args = [ "zuul-web", "-d" ]
, args = Some [ "zuul-web", "-d" ]
, imagePullPolicy = Some "IfNotPresent"
, ports =
, ports = Some
[ Kubernetes.ContainerPort::{
, name = Some "api"
, containerPort = 9000
}
]
, env = zuul-env
, volumeMounts =
mkVolumeMount
(web-volumes # zuul-data-dir)
, env = Some zuul-env
, volumeMounts = Some
( mkVolumeMount
(web-volumes # zuul-data-dir)
)
}
}
)
Expand All @@ -762,12 +770,13 @@ in \(input : Input)
, container = Kubernetes.Container::{
, name = "merger"
, image = zuul-image "merger"
, args = [ "zuul-merger", "-d" ]
, args = Some [ "zuul-merger", "-d" ]
, imagePullPolicy = Some "IfNotPresent"
, env = zuul-env
, volumeMounts =
mkVolumeMount
(merger-volumes # zuul-data-dir)
, env = Some zuul-env
, volumeMounts = Some
( mkVolumeMount
(merger-volumes # zuul-data-dir)
)
}
}
)
Expand Down Expand Up @@ -852,17 +861,18 @@ in \(input : Input)
, container = Kubernetes.Container::{
, name = "launcher"
, image = nodepool-image "launcher"
, args =
, args = Some
[ "sh"
, "-c"
, shard-config
++ "nodepool-launcher -d -c /var/lib/nodepool/config.yaml"
]
, imagePullPolicy = Some "IfNotPresent"
, env = nodepool-env
, volumeMounts =
mkVolumeMount
(nodepool-volumes # nodepool-data-dir)
, env = Some nodepool-env
, volumeMounts = Some
( mkVolumeMount
(nodepool-volumes # nodepool-data-dir)
)
}
}
)
Expand Down
4 changes: 2 additions & 2 deletions playbooks/files/cr_spec.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Render kubernetes resources using:
# INPUT=$(yaml-to-dhall "(./conf/zuul/input.dhall).Input.Type" < playbooks/files/cr_spec.yaml)
# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).Components.Zuul.Scheduler"
# Or
# dhall-to-yaml --omit-empty --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List"
# dhall-to-yaml --explain <<< "(./conf/zuul/resources.dhall ($INPUT)).List"

executor:
count: 1
Expand Down
2 changes: 1 addition & 1 deletion roles/zuul/library/dhall_to_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

def run(expression: str) -> Any:
proc = subprocess.Popen(
['dhall-to-json', '--omit-empty', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
['dhall-to-json', '--explain'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate(expression.encode('utf-8'))
if stderr:
return dict(failed=True, msg=stderr.decode('utf-8'))
Expand Down

0 comments on commit 0f75963

Please sign in to comment.