diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 7b66543e..809442c7 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -3,6 +3,8 @@ name: Checkov on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] workflow_dispatch: permissions: read-all @@ -24,14 +26,14 @@ jobs: id: checkov uses: bridgecrewio/checkov-action@0549dc60bddd4c55cb85c6c3a07072e3cf2ca48e with: - skip_check: CKV_DOCKER_2,CKV_DOCKER_3 + skip_check: CKV_DOCKER_2,CKV_DOCKER_3,CKV_SECRET_6 quiet: true output_format: cli,sarif output_file_path: console,results.sarif download_external_modules: true - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3 if: success() || failure() with: sarif_file: results.sarif diff --git a/.github/workflows/helm_lint.yaml b/.github/workflows/helm_lint.yaml new file mode 100644 index 00000000..6777d7c1 --- /dev/null +++ b/.github/workflows/helm_lint.yaml @@ -0,0 +1,47 @@ +name: Lint and Test Chart + +on: pull_request + +permissions: read-all + +jobs: + lint-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@20d2b4f98d41febe2bbca46408499dbb535b6258 # v3 + with: + version: v3.14.0 + + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Run chart-testing (lint) + if: steps.list-changed.outputs.changed == 'true' + run: ct lint --target-branch ${{ github.event.repository.default_branch }} + + - name: Create kind cluster + if: steps.list-changed.outputs.changed == 'true' + uses: helm/kind-action@v1.8.0 + + - name: Run chart-testing (install) + if: steps.list-changed.outputs.changed == 'true' + run: ct install --target-branch ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/publish-and-deploy.yaml b/.github/workflows/publish-and-deploy.yaml index 4901cee1..392f7d33 100644 --- a/.github/workflows/publish-and-deploy.yaml +++ b/.github/workflows/publish-and-deploy.yaml @@ -51,3 +51,19 @@ jobs: if_key_exists: fail # replace / ignore / fail; optional (defaults to fail) - name: Pull new Docker image run: ssh ${{ secrets.GRANDSVC_SSH_TARGET }} "cd ${{ secrets.GRANDSVC_PROJECT_PATH }} && git pull && docker compose pull && docker compose up -d" + + deploy_to_dev0: + name: Deploy to dev0 + needs: push_to_registry + runs-on: ubuntu-latest + steps: + - name: Install SSH key + uses: shimataro/ssh-key-action@d4fffb50872869abe2d9a9098a6d9c5aa7d16be4 # v2 + with: + key: ${{ secrets.DEV0_KEY }} + name: id_ed25519 # optional + known_hosts: ${{ secrets.DEV0_KNOWN_HOSTS }} + #config: ${{ secrets.CONFIG }} # ssh_config; optional + if_key_exists: fail # replace / ignore / fail; optional (defaults to fail) + - name: Pull new Docker image + run: ssh ${{ secrets.DEV0_SSH_TARGET }} "cd ${{ secrets.DEV0_PROJECT_PATH }} && git pull && cd infra/helm/meshdb && helm template . -f ../../../../values.yaml -f ../../../../secret.values.yaml | kubectl apply -f -" diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 00000000..20a4973d --- /dev/null +++ b/infra/README.md @@ -0,0 +1,33 @@ +# Meshdb Environment Setup + +These instructions will set up a 4 node k3s cluster on proxmox. +- 1 "manager" node for control plane and to be used for deployments. +- 3 "agent" nodes to run services. + +1. Clone this repository +2. Set up tfvars. See [proxmox provider](https://registry.terraform.io/providers/Telmate/proxmox/latest/docs). Create an API key in Proxmox, and disable Privilege Separation. +``` +cd meshdb/infra/tf/ +cp example.tfvars your_env.tfvars +# Modify your_env.tfvars to meet your needs +``` +3. Create the VMs that will host k3s +``` +terraform init --var-file=your_env.tfvars +terraform plan --var-file=your_env.tfvars +terraform apply --var-file=your_env.tfvars +``` +4. SSH into the manager node +5. Update values + secrets in `/opt/meshdb_mgmt/values.yaml` and `/opt/meshdb_mgmt/secret.values.yaml` + +6. Deploy helm chart. Create the namespace you indicated in `/opt/meshdb_mgmt/values.yaml` + +``` +your_ns="meshdbdev0" +cd /opt/meshdb_mgmt/meshdb/infra/helm/meshdb/ +kubectl create namespace $your_ns +helm template . -f ../../../../values.yaml -f ../../../../secret.values.yaml | kubectl apply -f - +kubectl get all -n $your_ns +``` + +7. If you need a superuser: `kubectl exec -it -n meshdbdev0 service/meshdb-meshweb bash` and `python manage.py createsuperuser` diff --git a/infra/cluster/longhorn.yaml b/infra/cluster/longhorn.yaml new file mode 100644 index 00000000..da07719d --- /dev/null +++ b/infra/cluster/longhorn.yaml @@ -0,0 +1,9 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: longhorn + namespace: longhorn-system +spec: + repo: https://charts.longhorn.io + chart: longhorn + targetNamespace: longhorn-system \ No newline at end of file diff --git a/infra/cluster/main.tf b/infra/cluster/main.tf new file mode 100644 index 00000000..8f387d91 --- /dev/null +++ b/infra/cluster/main.tf @@ -0,0 +1,33 @@ +provider "kubernetes" { + config_path = "/etc/rancher/k3s/k3s.yaml" +} + +# Create metallb-system +resource "kubernetes_namespace" "metallb-system-ns" { + metadata { + name = "metallb-system" + } +} + +# Create metallb with the manifest +resource "kubernetes_manifest" "metallb" { + manifest = yamldecode(file("./metallb.yaml")) + depends_on = [ + kubernetes_namespace.metallb-system-ns + ] +} + +# Create longhorn-system +resource "kubernetes_namespace" "longhorn-system-ns" { + metadata { + name = "longhorn-system" + } +} + +# Create longhorn with the manifest +resource "kubernetes_manifest" "longhorn" { + manifest = yamldecode(file("./longhorn.yaml")) + depends_on = [ + kubernetes_namespace.longhorn-system-ns + ] +} diff --git a/infra/cluster/metallb.yaml b/infra/cluster/metallb.yaml new file mode 100644 index 00000000..72e8a1e0 --- /dev/null +++ b/infra/cluster/metallb.yaml @@ -0,0 +1,9 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: metallb + namespace: metallb-system +spec: + repo: https://metallb.github.io/metallb + chart: metallb + targetNamespace: metallb-system diff --git a/infra/cluster/metallb_extra.yaml b/infra/cluster/metallb_extra.yaml new file mode 100644 index 00000000..b32cf5fc --- /dev/null +++ b/infra/cluster/metallb_extra.yaml @@ -0,0 +1,17 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: pool-1 + namespace: metallb-system +spec: + addresses: + - "METALLB_ADDR_RANGE" +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: k3s-l2 + namespace: metallb-system +spec: + ipAddressPools: + - pool-1 diff --git a/infra/helm/meshdb/Chart.yaml b/infra/helm/meshdb/Chart.yaml new file mode 100644 index 00000000..b618e60f --- /dev/null +++ b/infra/helm/meshdb/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: meshdb +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/infra/helm/meshdb/README.md b/infra/helm/meshdb/README.md new file mode 100644 index 00000000..01649577 --- /dev/null +++ b/infra/helm/meshdb/README.md @@ -0,0 +1,83 @@ +# meshdb + +A Helm chart for Kubernetes + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| autoscaling.enabled | bool | `false` | | +| autoscaling.maxReplicas | int | `100` | | +| autoscaling.minReplicas | int | `1` | | +| autoscaling.targetCPUUtilizationPercentage | int | `80` | | +| aws.access_key_id | string | `"the_real_value"` | AWS access key id for S3 | +| aws.secret_access_key | string | `"the_real_value"` | AWS secret access key for S3 | +| fullnameOverride | string | `"meshdb"` | App name | +| image.pullPolicy | string | `"Always"` | pullPolicy for all images, should be `Always` | +| map.base_url | string | `"http://admin-map.grandsvc.mesh.nycmesh.net"` | Map url | +| meshdb_app_namespace | string | `"meshdbdev0"` | K8s namespace used for all resources | +| meshweb.affinity | object | `{}` | | +| meshweb.backup_s3_base_folder | string | `"meshdb-backups/development/"` | Base folder for django postgres backups | +| meshweb.backup_s3_bucket_name | string | `"meshdb-data-backups"` | Bucket used for django postgres backups | +| meshweb.disable_pano_edits | string | `"True"` | Feature flag for disabling panorama edits | +| meshweb.disable_profiling | string | `"True"` | Disable profiling in meshweb | +| meshweb.django_secret_key | string | `"the_real_value"` | Django secret key | +| meshweb.enable_debug | string | `"False"` | Enable `DEBUG` in meshweb | +| meshweb.image.repository | string | `"willnilges/meshdb"` | Docker image repo for meshweb | +| meshweb.image.tag | string | `"main"` | Docker image tag for meshweb | +| meshweb.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it | +| meshweb.nn_assign_psk | string | `"the_real_value"` | Legacy NN assign form password | +| meshweb.nodeSelector | object | `{}` | | +| meshweb.pano_github_token | string | `"the_real_value"` | Github token for downloading panorama | +| meshweb.podSecurityContext | object | `{}` | | +| meshweb.port | int | `8081` | Port used by meshweb (internally) | +| meshweb.query_psk | string | `"the_real_value"` | Legacy query form password | +| meshweb.resources | object | `{}` | | +| meshweb.securityContext | object | `{}` | | +| meshweb.static_pvc_name | string | `"meshdb-static-pvc"` | Name of the PVC for static content | +| meshweb.static_pvc_size | string | `"1Gi"` | Size of the PVC for static content | +| meshweb.tolerations | list | `[]` | | +| nameOverride | string | `""` | | +| nginx.affinity | object | `{}` | | +| nginx.nodeSelector | object | `{}` | | +| nginx.podSecurityContext | object | `{}` | | +| nginx.port | int | `80` | Nginx port | +| nginx.resources | object | `{}` | | +| nginx.securityContext | object | `{}` | | +| nginx.server_name | string | `"db.nycmesh.net"` | `server_name` used by nginx | +| nginx.tolerations | list | `[]` | | +| pelias.affinity | object | `{}` | | +| pelias.nodeSelector | object | `{}` | | +| pelias.podSecurityContext | object | `{}` | | +| pelias.port | int | `6800` | Pelias port (internal) | +| pelias.resources | object | `{}` | | +| pelias.securityContext | object | `{}` | | +| pelias.tolerations | list | `[]` | | +| pg.affinity | object | `{}` | | +| pg.dbname | string | `"meshdb"` | Postgres database name | +| pg.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it | +| pg.nodeSelector | object | `{}` | | +| pg.password | string | `"the_real_value"` | Password for postgres | +| pg.podSecurityContext | object | `{}` | | +| pg.port | string | `"5432"` | Postgres port (internal) | +| pg.pvc_name | string | `"meshdb-postgres-pvc"` | Name of the PVC for postgres | +| pg.pvc_size | string | `"20Gi"` | Size of the PVC for postgres | +| pg.resources | object | `{}` | | +| pg.securityContext | object | `{}` | | +| pg.tolerations | list | `[]` | | +| pg.user | string | `"meshdb"` | Postgres user | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| redis.affinity | object | `{}` | | +| redis.liveness_probe | string | `"true"` | Enable liveness probe with `true` all other values will disable it | +| redis.nodeSelector | object | `{}` | | +| redis.podSecurityContext | object | `{}` | | +| redis.port | int | `6379` | Redis port (internal) | +| redis.resources | object | `{}` | | +| redis.securityContext | object | `{}` | | +| redis.tolerations | list | `[]` | | +| uisp.psk | string | `"the_real_value"` | Password for UISP | +| uisp.url | string | `"https://uisp.mesh.nycmesh.net/nms"` | UISP url | +| uisp.user | string | `"nycmesh_readonly"` | Username for UISP | + +---------------------------------------------- diff --git a/infra/helm/meshdb/secret.values.yaml b/infra/helm/meshdb/secret.values.yaml new file mode 100644 index 00000000..bd92eab2 --- /dev/null +++ b/infra/helm/meshdb/secret.values.yaml @@ -0,0 +1,15 @@ +pg: + password: the_real_value + +aws: + access_key_id: the_real_value + secret_access_key: the_real_value + +meshweb: + django_secret_key: the_real_value + nn_assign_psk: the_real_value + query_psk: the_real_value + pano_github_token: the_real_value + +uisp: + psk: the_real_value diff --git a/infra/helm/meshdb/templates/_helpers.tpl b/infra/helm/meshdb/templates/_helpers.tpl new file mode 100644 index 00000000..a1381dd7 --- /dev/null +++ b/infra/helm/meshdb/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "meshdb.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "meshdb.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "meshdb.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "meshdb.labels" -}} +helm.sh/chart: {{ include "meshdb.chart" . }} +{{ include "meshdb.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "meshdb.selectorLabels" -}} +app.kubernetes.io/name: {{ include "meshdb.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "meshdb.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "meshdb.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/infra/helm/meshdb/templates/configmap.yaml b/infra/helm/meshdb/templates/configmap.yaml new file mode 100644 index 00000000..03847993 --- /dev/null +++ b/infra/helm/meshdb/templates/configmap.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshdbconfig + namespace: {{ .Values.meshdb_app_namespace }} +data: + DB_NAME: {{ .Values.pg.dbname }} + DB_USER: {{ .Values.pg.user | quote }} + DB_HOST: {{ include "meshdb.fullname" . }}-postgres.{{ .Values.meshdb_app_namespace }}.svc.cluster.local + DB_PORT: {{ .Values.pg.port | quote }} + # Backups + BACKUP_S3_BUCKET_NAME: {{ .Values.meshweb.backup_s3_bucket_name | quote }} + BACKUP_S3_BASE_FOLDER: {{ .Values.meshweb.backup_s3_base_folder | quote }} + + CELERY_BROKER: "redis://{{ include "meshdb.fullname" . }}-redis.{{ .Values.meshdb_app_namespace }}.svc.cluster.local:{{ .Values.redis.port }}/0" + + # Change to pelias:3000 when using full docker-compose + PELIAS_ADDRESS_PARSER_URL: {{ include "meshdb.fullname" . }}-pelias.{{ .Values.meshdb_app_namespace }}.svc.cluster.local + + # Comment this out to enter prod mode + DEBUG: {{ .Values.meshweb.enable_debug | quote }} + DISABLE_PROFILING: {{ .Values.meshweb.disable_profiling | quote }} + + # Comment this out to allow edits to the panoramas in the admin panel + DISABLE_PANO_EDITS: {{ .Values.meshweb.disable_pano_edits | quote }} + + UISP_URL: {{ .Values.uisp.url | quote }} + UISP_USER: {{ .Values.uisp.user | quote }} + + ADMIN_MAP_BASE_URL: {{ .Values.map.base_url | quote }} diff --git a/infra/helm/meshdb/templates/meshweb.yaml b/infra/helm/meshdb/templates/meshweb.yaml new file mode 100644 index 00000000..e6e298fb --- /dev/null +++ b/infra/helm/meshdb/templates/meshweb.yaml @@ -0,0 +1,181 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "meshdb.fullname" . }}-meshweb + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "meshdb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: meshdb-meshweb-app + {{- include "meshdb.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.meshweb.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-meshweb + securityContext: + {{- toYaml .Values.meshweb.securityContext | nindent 12 }} + image: "{{ .Values.meshweb.image.repository }}:{{ .Values.meshweb.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.meshweb.port }} + protocol: TCP + env: + - name: DB_NAME + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_NAME + - name: DB_USER + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_USER + - name: DB_HOST + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_HOST + - name: DB_PORT + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_PORT + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: postgres-password + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: aws-secret-access-key + - name: BACKUP_S3_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: BACKUP_S3_BUCKET_NAME + - name: BACKUP_S3_BASE_FOLDER + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: BACKUP_S3_BASE_FOLDER + - name: CELERY_BROKER + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: CELERY_BROKER + - name: DJANGO_SECRET_KEY + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: django-secret-key + - name: PELIAS_ADDRESS_PARSER_URL + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: PELIAS_ADDRESS_PARSER_URL + - name: NN_ASSIGN_PSK + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: nn-assign-psk + - name: QUERY_PSK + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: query-psk + - name: DEBUG + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DEBUG + - name: DISABLE_PROFILING + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DISABLE_PROFILING + - name: DISABLE_PANO_EDITS + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DISABLE_PANO_EDITS + - name: PANO_GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: pano-github-token + - name: UISP_URL + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: UISP_URL + - name: UISP_USER + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: UISP_USER + - name: UISP_PASS + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: uisp-pass + - name: ADMIN_MAP_BASE_URL + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: ADMIN_MAP_BASE_URL + volumeMounts: + - name: static-content-vol + mountPath: /opt/meshdb/static + {{ if eq .Values.meshweb.liveness_probe "true" }} + livenessProbe: + exec: + command: + - curl + - http://127.0.0.1:{{ .Values.meshweb.port }}/api/v1 + periodSeconds: 3 + initialDelaySeconds: 4 + timeoutSeconds: 3 + {{ end }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.meshweb.resources | nindent 12 }} + volumes: + - name: static-content-vol + persistentVolumeClaim: + claimName: {{ .Values.meshweb.static_pvc_name }} + {{- with .Values.meshweb.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.meshweb.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.meshweb.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/helm/meshdb/templates/meshweb_static_pvc.yaml b/infra/helm/meshdb/templates/meshweb_static_pvc.yaml new file mode 100644 index 00000000..4736196c --- /dev/null +++ b/infra/helm/meshdb/templates/meshweb_static_pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.meshweb.static_pvc_name }} + namespace: {{ .Values.meshdb_app_namespace }} +spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn + resources: + requests: + storage: {{ .Values.meshweb.static_pvc_size }} \ No newline at end of file diff --git a/infra/helm/meshdb/templates/nginx.yaml b/infra/helm/meshdb/templates/nginx.yaml new file mode 100644 index 00000000..b4c0540d --- /dev/null +++ b/infra/helm/meshdb/templates/nginx.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "meshdb.fullname" . }}-nginx + namespace: {{ .Values.meshdb_app_namespace }} + labels: + name: meshdb-nginx + {{- include "meshdb.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "meshdb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: meshdb-nginx-app + {{- include "meshdb.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.nginx.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-nginx + securityContext: + {{- toYaml .Values.nginx.securityContext | nindent 12 }} + image: "nginx:latest" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: nginx + containerPort: {{ .Values.nginx.port }} + protocol: TCP + volumeMounts: + - name: nginx-conf + mountPath: /etc/nginx/conf.d/nginx.conf + subPath: nginx.conf + readOnly: true + - name: static-data-vol + mountPath: /var/www/html/static + readOnly: true + resources: + {{- toYaml .Values.nginx.resources | nindent 12 }} + volumes: + - name: nginx-conf + configMap: + name: nginx-conf + items: + - key: nginx.conf + path: nginx.conf + - name: static-data-vol + persistentVolumeClaim: + claimName: {{ .Values.meshweb.static_pvc_name }} + {{- with .Values.nginx.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nginx.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nginx.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/helm/meshdb/templates/nginx_configmap.yaml b/infra/helm/meshdb/templates/nginx_configmap.yaml new file mode 100644 index 00000000..cf0692da --- /dev/null +++ b/infra/helm/meshdb/templates/nginx_configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf + namespace: {{ .Values.meshdb_app_namespace }} +data: + nginx.conf: | + server { + listen {{ .Values.nginx.port }}; + server_name {{ .Values.nginx.server_name }}; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log debug; + + location = /favicon.ico { access_log off; log_not_found off; } + location /static/ { + root /var/www/html; + } + + location / { + proxy_pass http://{{ include "meshdb.fullname" . }}-meshweb.{{ .Values.meshdb_app_namespace }}.svc.cluster.local:{{ .Values.meshweb.port }}/; + #proxy_set_header Host $host; + proxy_set_header Host {{ .Values.nginx.server_name }}; + #proxy_set_header X-Real-IP $remote_addr; + #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + #proxy_redirect off; + } + } + diff --git a/infra/helm/meshdb/templates/pelias.yaml b/infra/helm/meshdb/templates/pelias.yaml new file mode 100644 index 00000000..c674dc80 --- /dev/null +++ b/infra/helm/meshdb/templates/pelias.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "meshdb.fullname" . }}-pelias + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "meshdb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: meshdb-pelias-app + {{- include "meshdb.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.pelias.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-pelias + securityContext: + {{- toYaml .Values.pelias.securityContext | nindent 12 }} + image: "pelias/parser" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: pelias + containerPort: {{ .Values.pelias.port }} + protocol: TCP + resources: + {{- toYaml .Values.pelias.resources | nindent 12 }} + {{- with .Values.pelias.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pelias.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pelias.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/helm/meshdb/templates/postgres.yaml b/infra/helm/meshdb/templates/postgres.yaml new file mode 100644 index 00000000..9f55c0ef --- /dev/null +++ b/infra/helm/meshdb/templates/postgres.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "meshdb.fullname" . }}-postgres + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "meshdb.selectorLabels" . | nindent 6 }} + strategy: + type: Recreate + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: meshdb-postgres-app + {{- include "meshdb.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.pg.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-postgres + securityContext: + {{- toYaml .Values.pg.securityContext | nindent 12 }} + image: postgres:15-bookworm + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: postgres + containerPort: {{ .Values.pg.port }} + protocol: TCP + resources: + {{- toYaml .Values.pg.resources | nindent 12 }} + {{ if eq .Values.pg.liveness_probe "true" }} + livenessProbe: + exec: + command: + - pg_isready -U ${DB_USER} + periodSeconds: 5 + initialDelaySeconds: 5 + timeoutSeconds: 3 + {{ end }} + env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_NAME + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: meshdbconfig + key: DB_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: meshdb-secrets + key: postgres-password + - name: PGDATA + value: "/var/lib/postgresql/data/meshdb_data" + volumeMounts: + - name: pg-data-vol + mountPath: /var/lib/postgresql/data + volumes: + - name: pg-data-vol + persistentVolumeClaim: + claimName: {{ .Values.pg.pvc_name }} + {{- with .Values.pg.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pg.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.pg.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/helm/meshdb/templates/postgres_pvc.yaml b/infra/helm/meshdb/templates/postgres_pvc.yaml new file mode 100644 index 00000000..78116336 --- /dev/null +++ b/infra/helm/meshdb/templates/postgres_pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.pg.pvc_name }} + namespace: {{ .Values.meshdb_app_namespace }} +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: {{ .Values.pg.pvc_size }} \ No newline at end of file diff --git a/infra/helm/meshdb/templates/redis.yaml b/infra/helm/meshdb/templates/redis.yaml new file mode 100644 index 00000000..44cd5c6f --- /dev/null +++ b/infra/helm/meshdb/templates/redis.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "meshdb.fullname" . }}-redis + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "meshdb.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: meshdb-redis-app + {{- include "meshdb.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + securityContext: + {{- toYaml .Values.redis.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-redis + securityContext: + {{- toYaml .Values.redis.securityContext | nindent 12 }} + image: "redis:latest" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: redis + containerPort: {{ .Values.redis.port }} + protocol: TCP + resources: + {{- toYaml .Values.redis.resources | nindent 12 }} + {{ if eq .Values.redis.liveness_probe "true" }} + livenessProbe: + exec: + command: + - "redis-cli" + - "--raw" + - "incr" + - "ping" + periodSeconds: 3 + initialDelaySeconds: 2 + timeoutSeconds: 3 + {{ end }} + {{- with .Values.redis.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.redis.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.redis.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/helm/meshdb/templates/secrets.yaml b/infra/helm/meshdb/templates/secrets.yaml new file mode 100644 index 00000000..78d1520c --- /dev/null +++ b/infra/helm/meshdb/templates/secrets.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: meshdb-secrets + namespace: {{ .Values.meshdb_app_namespace }} +type: Opaque +data: + postgres-password: {{ .Values.pg.password | b64enc | quote }} + aws-access-key-id: {{ .Values.aws.access_key_id | b64enc | quote }} + aws-secret-access-key: {{ .Values.aws.secret_access_key | b64enc | quote }} + django-secret-key: {{ .Values.meshweb.django_secret_key | b64enc | quote }} + nn-assign-psk: {{ .Values.meshweb.nn_assign_psk | b64enc | quote }} + query-psk: {{ .Values.meshweb.query_psk | b64enc | quote }} + uisp-pass: {{ .Values.uisp.psk | b64enc | quote }} + pano-github-token: {{ .Values.meshweb.pano_github_token | b64enc | quote }} diff --git a/infra/helm/meshdb/templates/service.yaml b/infra/helm/meshdb/templates/service.yaml new file mode 100644 index 00000000..16ec990c --- /dev/null +++ b/infra/helm/meshdb/templates/service.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "meshdb.fullname" . }}-nginx + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + type: LoadBalancer + ports: + - port: {{ .Values.nginx.port }} + targetPort: {{ .Values.nginx.port }} + protocol: TCP + name: nginx + selector: + app: meshdb-nginx-app +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "meshdb.fullname" . }}-postgres + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.pg.port }} + targetPort: {{ .Values.pg.port }} + protocol: TCP + name: postgres + selector: + app: meshdb-postgres-app +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "meshdb.fullname" . }}-redis + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.redis.port }} + targetPort: {{ .Values.redis.port }} + protocol: TCP + name: redis + selector: + app: meshdb-redis-app +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "meshdb.fullname" . }}-pelias + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.pelias.port }} + targetPort: {{ .Values.pelias.port }} + protocol: TCP + name: pelias + selector: + app: meshdb-pelias-app +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "meshdb.fullname" . }}-meshweb + namespace: {{ .Values.meshdb_app_namespace }} + labels: + {{- include "meshdb.labels" . | nindent 4 }} +spec: + ports: + - port: {{ .Values.meshweb.port }} + targetPort: {{ .Values.meshweb.port }} + protocol: TCP + name: meshweb-service + selector: + app: meshdb-meshweb-app + diff --git a/infra/helm/meshdb/values.yaml b/infra/helm/meshdb/values.yaml new file mode 100644 index 00000000..6db1d17e --- /dev/null +++ b/infra/helm/meshdb/values.yaml @@ -0,0 +1,119 @@ +# Default values for meshdb. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +meshdb_app_namespace: "meshdbdev0" + +pg: + dbname: meshdb + user: meshdb + port: "5432" + pvc_name: "meshdb-postgres-pvc" + pvc_size: "20Gi" + liveness_probe: "true" + podSecurityContext: {} + # fsGroup: 2000 + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + affinity: {} + tolerations: [] + +meshweb: + port: 8081 + backup_s3_bucket_name: meshdb-data-backups + backup_s3_base_folder: "meshdb-backups/development/" + enable_debug: "False" + disable_profiling: "True" + disable_pano_edits: "True" + static_pvc_name: "meshdb-static-pvc" + static_pvc_size: "1Gi" + liveness_probe: "true" + image: + repository: willnilges/meshdb + tag: main + podSecurityContext: {} + securityContext: {} + resources: {} + nodeSelector: {} + affinity: {} + tolerations: [] + +nginx: + server_name: "db.nycmesh.net" + port: 80 + podSecurityContext: {} + securityContext: {} + resources: {} + nodeSelector: {} + affinity: {} + tolerations: [] + +uisp: + url: "https://uisp.mesh.nycmesh.net/nms" + user: "nycmesh_readonly" + +redis: + liveness_probe: "true" + port: 6379 + podSecurityContext: {} + securityContext: {} + resources: {} + nodeSelector: {} + affinity: {} + tolerations: [] + +pelias: + port: 6800 + podSecurityContext: {} + securityContext: {} + resources: {} + nodeSelector: {} + affinity: {} + tolerations: [] + +map: + base_url: "http://admin-map.grandsvc.mesh.nycmesh.net" + +image: + pullPolicy: Always + +nameOverride: "" +fullnameOverride: "meshdb" + +#serviceAccount: +# # Specifies whether a service account should be created +# create: true +# # Automatically mount a ServiceAccount's API credentials? +# automount: true +# # Annotations to add to the service account +# annotations: {} +# # The name of the service account to use. +# # If not set and create is true, a name is generated using the fullname template +# name: "" + +podAnnotations: {} +podLabels: {} + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 diff --git a/infra/tf/example.tfvars b/infra/tf/example.tfvars new file mode 100644 index 00000000..4a668bd9 --- /dev/null +++ b/infra/tf/example.tfvars @@ -0,0 +1,16 @@ +meshdb_proxmox_host = "10.70.90.52" +meshdb_proxmox_node = "jon" +meshdb_proxmox_template_image = "debian-cloud" +meshdb_proxmox_storage_location = "local-lvm" +meshdb_env_name = "dev1" +meshdb_local_user = "debian" +meshdb_local_password = "MeshMesh9999" +meshdb_proxmox_token_id = "meshdbdev-terraform-prov@pve!james-test" +meshdb_proxmox_token_secret = "" +meshdb_ips = [ + "10.70.90.X", + "10.70.90.Y", + "10.70.90.Z", + "10.70.90.A", +] +meshdb_metallb_range = "10.70.90.80/29" \ No newline at end of file diff --git a/infra/tf/gen_ssh_key.sh b/infra/tf/gen_ssh_key.sh new file mode 100644 index 00000000..7082a73a --- /dev/null +++ b/infra/tf/gen_ssh_key.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +ENV_NAME="$1" +EXPECTED_KEYFILE_NAME="./meshdb$ENV_NAME" + +if [ ! -f "$EXPECTED_KEYFILE_NAME" ]; then + ssh-keygen -t ed25519 -f $EXPECTED_KEYFILE_NAME -C meshdb${ENV_NAME}@db.mesh.nycmesh.net -q -N "" +else + echo "File $EXPECTED_KEYFILE_NAME already exists" +fi diff --git a/infra/tf/main.tf b/infra/tf/main.tf new file mode 100644 index 00000000..df7d5099 --- /dev/null +++ b/infra/tf/main.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + proxmox = { + source = "telmate/proxmox" + version = "3.0.1-rc1" + } + } +} + +provider "proxmox" { + # Configuration options + pm_api_url = "https://${var.meshdb_proxmox_host}:8006/api2/json" + # TODO: Setup cert + pm_tls_insecure = true + pm_debug = true + pm_api_token_id = "${var.meshdb_proxmox_token_id}" + pm_api_token_secret = "${var.meshdb_proxmox_token_secret}" +} diff --git a/infra/tf/mgr.tf b/infra/tf/mgr.tf new file mode 100644 index 00000000..76b96278 --- /dev/null +++ b/infra/tf/mgr.tf @@ -0,0 +1,109 @@ +resource "null_resource" "ssh_key" { + provisioner "local-exec" { + command = "bash ${path.module}/gen_ssh_key.sh ${var.meshdb_env_name}" + } +} + +resource "proxmox_vm_qemu" "meshdbdevmgr" { + name = "meshdb${var.meshdb_env_name}mgr" + desc = "managment server for meshdb ${var.meshdb_env_name}" + target_node = var.meshdb_proxmox_node + + clone = var.meshdb_proxmox_template_image + + cores = 2 + sockets = 1 + memory = 2560 + os_type = "cloud-init" + agent = 1 + cloudinit_cdrom_storage = var.meshdb_proxmox_storage_location + ciuser = "${var.meshdb_local_user}" + cipassword = "${var.meshdb_local_password}" + + scsihw = "virtio-scsi-pci" + + disks { + scsi { + scsi0 { + disk { + backup = false + size = 50 + storage = var.meshdb_proxmox_storage_location + + } + } + } + } + + network { + bridge = "vmbr0" + model = "virtio" + } + + ipconfig0 = "ip=${var.meshdb_ips[0]}/${var.meshdb_networkrange},gw=${var.meshdb_gateway}" + + ssh_user = "root" + ssh_private_key = file("${path.module}/meshdb${var.meshdb_env_name}") + + sshkeys = file("${path.module}/meshdb${var.meshdb_env_name}.pub") + + serial { + id = 0 + type = "socket" + } + + tags = "meshdb${var.meshdb_env_name}" + + depends_on = [ + null_resource.ssh_key + ] +} + +resource "null_resource" "mgr_config_files" { + connection { + type = "ssh" + user = "${var.meshdb_local_user}" + private_key = file("${path.module}/meshdb${var.meshdb_env_name}") + host = "${var.meshdb_ips[0]}" + } + + provisioner "file" { + source = "${path.module}/mgr_provision.sh" + destination = "/home/${var.meshdb_local_user}/mgr_provision.sh" + } + + provisioner "file" { + source = "${path.module}/setup_node.sh" + destination = "/home/${var.meshdb_local_user}/setup_node.sh" + } + + provisioner "file" { + source = "${path.module}/meshdb${var.meshdb_env_name}" + destination = "/home/${var.meshdb_local_user}/.ssh/id_rsa" + } + + provisioner "file" { + source = "${path.module}/meshdb${var.meshdb_env_name}.pub" + destination = "/home/${var.meshdb_local_user}/.ssh/id_rsa.pub" + } + + provisioner "remote-exec" { + inline = [ + "sudo bash /home/${var.meshdb_local_user}/mgr_provision.sh" + ] + } + + provisioner "remote-exec" { + inline = [<> /etc/rancher/k3s/config.yaml +echo "disable: servicelb" >> /etc/rancher/k3s/config.yaml diff --git a/infra/tf/output.tf b/infra/tf/output.tf new file mode 100644 index 00000000..8ada9a00 --- /dev/null +++ b/infra/tf/output.tf @@ -0,0 +1,10 @@ +output "mgr_ip" { + description = "IP address of the managment node" + value = proxmox_vm_qemu.meshdbdevmgr.default_ipv4_address +} + +output "worker_ips" { + value = { + for k, meshdbnode in proxmox_vm_qemu.meshdbnode : k => meshdbnode.default_ipv4_address + } +} \ No newline at end of file diff --git a/infra/tf/setup_node.sh b/infra/tf/setup_node.sh new file mode 100644 index 00000000..6051070b --- /dev/null +++ b/infra/tf/setup_node.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# setup_node.sh +MASTER_IP="$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" +NODE_TOKEN="$(sudo cat /var/lib/rancher/k3s/server/node-token)" + +ssh_target="$1" + +chmod 600 ~/.ssh/id_rsa +chmod 600 ~/.ssh/id_rsa.pub + +ssh -t $ssh_target -o "StrictHostKeyChecking no" "curl -sfL https://get.k3s.io>k3s; sudo bash k3s agent --server https://${MASTER_IP}:6443 --token $NODE_TOKEN;sudo apt-get update && sudo apt-get install nfs-common -y" diff --git a/infra/tf/stage2.sh b/infra/tf/stage2.sh new file mode 100644 index 00000000..9ba17576 --- /dev/null +++ b/infra/tf/stage2.sh @@ -0,0 +1,21 @@ +metallb_addr="$1" + +cd /opt/meshdb_mgmt/ +cd meshdb +git checkout james/infra +cd infra/cluster + +sleep 10 + +terraform init +terraform plan +terraform apply + +sed -i -e "s/METALLB_ADDR_RANGE/${metallb_addr}/g" metallb_extra.yaml +terraform init +terraform plan +terraform apply -auto-approve + +sleep 30 + +kubectl apply -f /opt/meshdb_mgmt/meshdb/infra/cluster/metallb_extra.yaml \ No newline at end of file diff --git a/infra/tf/stage2config.tf b/infra/tf/stage2config.tf new file mode 100644 index 00000000..d1c67ca4 --- /dev/null +++ b/infra/tf/stage2config.tf @@ -0,0 +1,25 @@ +resource "null_resource" "mgr_stage_two" { + connection { + type = "ssh" + user = "${var.meshdb_local_user}" + private_key = file("${path.module}/meshdb${var.meshdb_env_name}") + host = "${var.meshdb_ips[0]}" + } + + provisioner "file" { + source = "${path.module}/stage2.sh" + destination = "/home/${var.meshdb_local_user}/stage2.sh" + } + + provisioner "remote-exec" { + inline = [ + format("sudo bash /home/%s/stage2.sh \"%s\"", var.meshdb_local_user, replace(var.meshdb_metallb_range, "/","\\/")) + ] + } + + depends_on = [ + proxmox_vm_qemu.meshdbdevmgr, + proxmox_vm_qemu.meshdbnode, + null_resource.mgr_config_files + ] +} diff --git a/infra/tf/vars.tf b/infra/tf/vars.tf new file mode 100644 index 00000000..c60ab552 --- /dev/null +++ b/infra/tf/vars.tf @@ -0,0 +1,68 @@ +variable "meshdb_proxmox_host" { + type = string + description = "ip/domain of the proxmox server" +} + +variable "meshdb_proxmox_token_id" { + type = string + description = "proxmox server token id" +} + +variable "meshdb_proxmox_token_secret" { + type = string + description = "proxmox server token secret" +} + +variable "meshdb_proxmox_node" { + type = string + description = "target node on the proxmox server" + default = "jon" +} + +variable "meshdb_proxmox_template_image" { + type = string + description = "name of the template you have already setup in proxmox" + default = "ubuntu-cloud" +} + +variable "meshdb_proxmox_storage_location" { + type = string + description = "target resource pool on the proxmox server" + default = "local-lvm" +} + +variable "meshdb_env_name" { + type = string + description = "name of the environment(dev0, dev1, stage, prod)" +} + +variable "meshdb_local_user" { + type = string + description = "local user username" + default = "ubuntu" +} + +variable "meshdb_local_password" { + type = string + description = "password for the local user" + sensitive = true +} + +variable "meshdb_ips" { + description = "static IPs to use for nodes" +} + +variable "meshdb_gateway" { + description = "default gateway to use for nodes" + default = "10.70.90.1" +} + +variable "meshdb_networkrange" { + description = "network range to use for nodes" + default = "24" +} + +variable "meshdb_metallb_range" { + type = string + description = "ip range for metallb" +} diff --git a/infra/tf/workers.tf b/infra/tf/workers.tf new file mode 100644 index 00000000..891f5858 --- /dev/null +++ b/infra/tf/workers.tf @@ -0,0 +1,61 @@ +resource "proxmox_vm_qemu" "meshdbnode" { + for_each = { + "node1" = { name = "1" } + "node2" = { name = "2" } + "node3" = { name = "3" } + } + + name = "meshdb${var.meshdb_env_name}node${each.value.name}" + desc = "node ${each.value.name} for meshdb ${var.meshdb_env_name}" + target_node = var.meshdb_proxmox_node + + clone = var.meshdb_proxmox_template_image + + cores = 2 + sockets = 1 + memory = 2560 + os_type = "cloud-init" + agent = 1 + cloudinit_cdrom_storage = var.meshdb_proxmox_storage_location + ciuser = "${var.meshdb_local_user}" + cipassword = "${var.meshdb_local_password}" + + scsihw = "virtio-scsi-pci" + + disks { + scsi { + scsi0 { + disk { + backup = false + size = 50 + storage = var.meshdb_proxmox_storage_location + + } + } + } + } + + network { + bridge = "vmbr0" + model = "virtio" + } + + ipconfig0 = "ip=${var.meshdb_ips[each.value.name]}/${var.meshdb_networkrange},gw=${var.meshdb_gateway}" + + ssh_user = "root" + ssh_private_key = file("${path.module}/meshdb${var.meshdb_env_name}") + + sshkeys = file("${path.module}/meshdb${var.meshdb_env_name}.pub") + + serial { + id = 0 + type = "socket" + } + + tags = "meshdb${var.meshdb_env_name}" + + depends_on = [ + null_resource.ssh_key + ] +} +