From 426be56d496005b4618b667cec20bdb426b2d0bd Mon Sep 17 00:00:00 2001 From: sbuliarca Date: Fri, 16 Feb 2024 17:50:58 +0200 Subject: [PATCH] kafka-shared prod: update debugging Makefile & readmes (#154) --- dev-aws/kafka-shared/{ => customer-billing}/Makefile | 0 .../kafka-shared/dev-enablement}/Makefile | 0 dev-aws/kafka-shared/iam/Makefile | 1 + dev-aws/kafka-shared/otel/Makefile | 1 + dev-aws/kafka-shared/pubsub/Makefile | 1 + lib/kafka-shared/Makefile | 5 +++-- prod-aws/kafka-shared/README.md | 7 +++++++ prod-aws/kafka-shared/TERRAFORM_LOCALLY.md | 2 +- prod-aws/kafka-shared/customer-billing/Makefile | 1 + prod-aws/kafka-shared/iam/Makefile | 1 + prod-aws/kafka-shared/otel/Makefile | 1 + prod-aws/kafka-shared/pubsub/Makefile | 1 + 12 files changed, 18 insertions(+), 3 deletions(-) rename dev-aws/kafka-shared/{ => customer-billing}/Makefile (100%) rename {prod-aws/kafka-shared => dev-aws/kafka-shared/dev-enablement}/Makefile (100%) create mode 100644 dev-aws/kafka-shared/iam/Makefile create mode 100644 dev-aws/kafka-shared/otel/Makefile create mode 100644 dev-aws/kafka-shared/pubsub/Makefile create mode 100644 prod-aws/kafka-shared/customer-billing/Makefile create mode 100644 prod-aws/kafka-shared/iam/Makefile create mode 100644 prod-aws/kafka-shared/otel/Makefile create mode 100644 prod-aws/kafka-shared/pubsub/Makefile diff --git a/dev-aws/kafka-shared/Makefile b/dev-aws/kafka-shared/customer-billing/Makefile similarity index 100% rename from dev-aws/kafka-shared/Makefile rename to dev-aws/kafka-shared/customer-billing/Makefile diff --git a/prod-aws/kafka-shared/Makefile b/dev-aws/kafka-shared/dev-enablement/Makefile similarity index 100% rename from prod-aws/kafka-shared/Makefile rename to dev-aws/kafka-shared/dev-enablement/Makefile diff --git a/dev-aws/kafka-shared/iam/Makefile b/dev-aws/kafka-shared/iam/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/dev-aws/kafka-shared/iam/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/dev-aws/kafka-shared/otel/Makefile b/dev-aws/kafka-shared/otel/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/dev-aws/kafka-shared/otel/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/dev-aws/kafka-shared/pubsub/Makefile b/dev-aws/kafka-shared/pubsub/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/dev-aws/kafka-shared/pubsub/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/lib/kafka-shared/Makefile b/lib/kafka-shared/Makefile index 4feec69e..ae392d19 100644 --- a/lib/kafka-shared/Makefile +++ b/lib/kafka-shared/Makefile @@ -3,7 +3,8 @@ KAFKA_CA_CERT := $(shell pwd)/certs-0/ca.crt KAFKA_CLIENT_KEY := $(shell pwd)/certs-0/tls.key KAFKA_CLIENT_CERT := $(shell pwd)/certs-0/tls.crt -CONTEXT := $(shell basename $(shell dirname ${PWD})) +SUBPROJECT := $(shell basename ${PWD}) +CONTEXT := $(shell basename $(shell dirname $(shell dirname ${PWD}))) ENV := $(shell echo ${CONTEXT} | cut -d"-" -f1) PROVIDER := $(shell echo ${CONTEXT} | rev | cut -d"-" -f1 | rev) @@ -21,7 +22,7 @@ init: certs terraform init \ -backend-config region=eu-west-1 \ -backend-config bucket=uw-${ENV}-pubsub-tf-applier-state \ - -backend-config key="${CONTEXT}"/kafka-shared \ + -backend-config key="${CONTEXT}"/kafka-shared-${SUBPROJECT} \ -backend-config encrypt=true \ -reconfigure \ -upgrade diff --git a/prod-aws/kafka-shared/README.md b/prod-aws/kafka-shared/README.md index d5ca359f..476e90ff 100644 --- a/prod-aws/kafka-shared/README.md +++ b/prod-aws/kafka-shared/README.md @@ -12,6 +12,13 @@ Please follow these guidelines for the resources: This will make it easier to define access in the kafka-ui ([dev](https://kafka-ui.dev.merit.uw.systems/) and [prod](https://kafka-ui.prod.merit.uw.systems/) ) 2. consider the number of partitions for a topic. In Amazon MSK the total number of partitions influences the cost. See [RFC](https://wiki.uw.systems/posts/amazon-msk-managed-kafka-9kjst9t5#h97za-msk) +## Structure +Each child folder contains a Terraform module for a team with the shared kafka resources for that team. + +For each of these modules there is a [terraform applier](https://github.com/utilitywarehouse/terraform-applier) entry under: +- [dev](https://github.com/utilitywarehouse/kubernetes-manifests/tree/master/dev-aws/pubsub/kafka) +- [prod](https://github.com/utilitywarehouse/kubernetes-manifests/tree/master/prod-aws/pubsub/kafka) + ## pubsub admins: debugging terraform locally See [instructions](TERRAFORM_LOCALLY.md) diff --git a/prod-aws/kafka-shared/TERRAFORM_LOCALLY.md b/prod-aws/kafka-shared/TERRAFORM_LOCALLY.md index a605f49e..1f5f084f 100644 --- a/prod-aws/kafka-shared/TERRAFORM_LOCALLY.md +++ b/prod-aws/kafka-shared/TERRAFORM_LOCALLY.md @@ -26,7 +26,7 @@ source_profile = uw-prod-jump-role 1. Reference AWS profile necessary to perform next actions-
`export AWS_PROFILE=pubsub-dev`
or
`export AWS_PROFILE=pubsub-prod`
in each terminal window. 2. Login to AWS- run from the `terraform` repo dir
`make okta-aws-cli-login`
More info [here](https://github.com/utilitywarehouse/terraform/tree/master/aws#okta-login) -3. Go to directory with the terraform state- e.g.
`cd dev-aws/kafka-shared` +3. Go to directory with the terraform state- e.g.
`cd dev-aws/kafka-shared/dev-enablement` 4. Run
`make init`
to initialize terraform state. This needs to happen only once per directory. 5. In a separate window but in the same directory run
`make connect`
in order to connect to the Kafka cluster.
6. Run
`make plan`
to see which changes will be applied. diff --git a/prod-aws/kafka-shared/customer-billing/Makefile b/prod-aws/kafka-shared/customer-billing/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/prod-aws/kafka-shared/customer-billing/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/prod-aws/kafka-shared/iam/Makefile b/prod-aws/kafka-shared/iam/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/prod-aws/kafka-shared/iam/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/prod-aws/kafka-shared/otel/Makefile b/prod-aws/kafka-shared/otel/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/prod-aws/kafka-shared/otel/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile diff --git a/prod-aws/kafka-shared/pubsub/Makefile b/prod-aws/kafka-shared/pubsub/Makefile new file mode 100644 index 00000000..33493275 --- /dev/null +++ b/prod-aws/kafka-shared/pubsub/Makefile @@ -0,0 +1 @@ +include ../../lib/kafka-shared/Makefile