Merge remote-tracking branch 'origin/main' into p3-staging

This is to include recent changes like nova_flavors
rackerlabs · Dec 10, 2024 · 1c81d3c · 1c81d3c
2 parents 73479a4 + c5a31c5
commit 1c81d3c
Show file tree

Hide file tree

Showing 44 changed files with 3,733 additions and 1,107 deletions.
diff --git a/.github/workflows/build-container-images.yaml b/.github/workflows/build-container-images.yaml
@@ -12,7 +12,6 @@ on:
     branches:
       - main
     paths:
-      - "containers/argo_utils/**"
       - "containers/bmc-utils/**"
       - "containers/python311_alpine/**"
       - "containers/python312_alpine/**"
@@ -39,7 +38,6 @@ jobs:
         uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3
 
       - name: Login to ghcr.io
-        if: ${{ github.event_name != 'pull_request' }}
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3
         with:
           registry: "ghcr.io"

diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml
@@ -116,6 +116,7 @@ jobs:
       matrix:
         container:
           - name: ironic-nautobot-client
+          - name: nova-flavors
 
     steps:
       - name: setup docker buildx

diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -19,7 +19,7 @@ jobs:
         with:
           python-version: '3.11'
           cache: 'pip'
-      - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4
+      - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}

diff --git a/.github/workflows/yamllint.yaml b/.github/workflows/yamllint.yaml
@@ -52,4 +52,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-      - uses: reviewdog/action-shellcheck@ccaafec556ffa154f112bfcb7b9c9574190b7091 # v1
+      - uses: reviewdog/action-shellcheck@22f96e34e9185b642c5567cc26d1df952f5c9d10 # v1
diff --git a/apps/appsets/openstack.yaml b/apps/appsets/openstack.yaml
@@ -39,7 +39,7 @@ spec:
                 - component: nova
                   openstackRelease: 2024.2
                   # renovate: datasource=custom.openstackhelm depName=nova
-                  chartVersion: 0.3.46
+                  chartVersion: 0.3.47
                 - component: horizon
                   openstackRelease: 2024.2
                   # renovate: datasource=custom.openstackhelm depName=horizon

diff --git a/bootstrap/sealed-secrets/kustomization.yaml b/bootstrap/sealed-secrets/kustomization.yaml
@@ -11,5 +11,5 @@ helmCharts:
   namespace: kube-system
   valuesFile: values.yaml
   releaseName: sealed-secrets
-  version: 2.16.2
+  version: 2.17.0
   repo: https://bitnami-labs.github.io/sealed-secrets
diff --git a/components/glance/aio-values.yaml b/components/glance/aio-values.yaml
@@ -68,7 +68,9 @@ pod:
   lifecycle:
     disruption_budget:
       api:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.api - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
   resources:
     enabled: true
   probes:

diff --git a/components/horizon/aio-values.yaml b/components/horizon/aio-values.yaml
@@ -51,6 +51,14 @@ manifests:
   secret_db: false
   service_ingress: false
 
+pod:
+  lifecycle:
+    disruption_budget:
+      horizon:
+        # this should be set to no more than (pod.replicas.horizon - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
+
 # We don't want to enable OpenStack Helm's
 # helm.sh/hooks because they set them as
 # post-install,post-upgrade which in ArgoCD

diff --git a/components/ironic/aio-values.yaml b/components/ironic/aio-values.yaml
@@ -185,7 +185,9 @@ pod:
   lifecycle:
     disruption_budget:
       api:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.api - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
 
 # we don't want to enable OpenStack Helm's
 # helm.sh/hooks because they set them as

diff --git a/components/keystone/aio-values.yaml b/components/keystone/aio-values.yaml
@@ -19,6 +19,11 @@ bootstrap:
     # give 'argoworkflow' 'admin' over the 'baremetal' project
     openstack role add --user-domain infra --project-domain infra --user argoworkflow --project baremetal admin
 
+    # create 'flavorsync' user to allow synchronization of the flavors to nova
+    openstack user create --or-show --domain service --password abcd1234 flavorsync
+    openstack role create --or-show flavorsync
+    openstack role add --user flavorsync --user-domain service --domain default --inherited  flavorsync
+
     # create 'monitoring' user for monitoring usage
     openstack user create --or-show --domain infra --password monitoring_demo monitoring
     # give 'monitoring' the 'admin' over the 'baremetal' project
@@ -229,7 +234,9 @@ pod:
   lifecycle:
     disruption_budget:
       api:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.api - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
 
 conf:
   keystone:

diff --git a/components/nautobot/dexauth.py b/components/nautobot/dexauth.py
diff --git a/components/nautobot/kustomization.yaml b/components/nautobot/kustomization.yaml
@@ -8,11 +8,6 @@ resources:
   - external-secret-nautobot-sso.yaml
 
 configMapGenerator:
-  - name: dexauth
-    files:
-      - dexauth.py
-    options:
-      disableNameSuffixHash: true
   - name: nautobot-sso
     literals:
       # enables SOCIAL_AUTH_PIPELINE to load the group_sync plugin

diff --git a/components/nautobot/nautobot_config.py b/components/nautobot/nautobot_config.py
@@ -478,20 +478,6 @@ def _read_cred(filename):
 # below *adds* scope.
 SOCIAL_AUTH_OIDC_SCOPE = ["groups"]
 
-# include custom auth pipeline to sync groups
-SOCIAL_AUTH_PIPELINE = (
-    "social_core.pipeline.social_auth.social_details",
-    "social_core.pipeline.social_auth.social_uid",
-    "social_core.pipeline.social_auth.auth_allowed",
-    "social_core.pipeline.social_auth.social_user",
-    "social_core.pipeline.user.get_username",
-    "social_core.pipeline.user.create_user",
-    "social_core.pipeline.social_auth.associate_user",
-    "social_core.pipeline.social_auth.load_extra_data",
-    "social_core.pipeline.user.user_details",
-    "dexauth.group_sync",
-)
-
 if SOCIAL_AUTH_OIDC_OIDC_ENDPOINT and SOCIAL_AUTH_OIDC_SECRET:
     AUTHENTICATION_BACKENDS = [
         "social_core.backends.open_id_connect.OpenIdConnectAuth",

diff --git a/components/neutron/aio-values.yaml b/components/neutron/aio-values.yaml
@@ -52,7 +52,7 @@ conf:
         type_drivers: "vlan,local,understack_vxlan"
   neutron:
     DEFAULT:
-      service_plugins: "router,segments,port_forwarding"
+      service_plugins: "l3_understack,segments,port_forwarding"
       # we don't want HA L3 routers. It's a Python value so we need to quote it in YAML.
       l3_ha: "False"
       # we aren't using availability zones so having calls attempt to add things to
@@ -66,7 +66,9 @@ pod:
   lifecycle:
     disruption_budget:
       server:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.server - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
   mounts:
     neutron_server:
       neutron_server:

diff --git a/components/nova/aio-values.yaml b/components/nova/aio-values.yaml
@@ -134,7 +134,9 @@ pod:
   lifecycle:
     disruption_budget:
       osapi:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.osapi - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
 
 manifests:
   job_db_init: false

diff --git a/components/placement/aio-values.yaml b/components/placement/aio-values.yaml
@@ -27,7 +27,9 @@ pod:
   lifecycle:
     disruption_budget:
       api:
-        min_available: 1
+        # this should be set to no more than (pod.replicas.api - 1)
+        # usually set on per-deployment basis.
+        min_available: 0
 
 manifests:
   job_db_init: false

diff --git a/containers/bmc-utils/Dockerfile.bmc_utils b/containers/bmc-utils/Dockerfile.bmc_utils
@@ -1,22 +1,11 @@
-ARG BASE=ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest
+ARG BASE=ghcr.io/rackerlabs/understack/python3.11.8-alpine3.19:latest
 
 FROM ${BASE} AS builder
 
-ARG APP_PATH=/app
-ARG APP_USER=appuser
-ARG APP_GROUP=appgroup
-ARG APP_USER_UID=1000
-ARG APP_GROUP_GID=1000
-
-COPY --chown=${APP_USER}:${APP_GROUP} requirements.txt /app
+COPY --chown=appuser:appgroup requirements.txt /app
 RUN --mount=type=cache,target=/root/.cache/.pip pip install --no-cache-dir -r /app/requirements.txt
 
 FROM ${BASE} AS prod
-ARG APP_PATH=/app
-ARG APP_USER=appuser
-ARG APP_GROUP=appgroup
-ARG APP_USER_UID=1000
-ARG APP_GROUP_GID=1000
 
 LABEL org.opencontainers.image.title="Python 3.11 image with BMC utils"
 LABEL org.opencontainers.image.base.name="ghcr.io/rackerlabs/understack/argo-bmc-utils-python3.11.8"
@@ -26,8 +15,6 @@ ENV PATH="/opt/venv/bin:$PATH"
 COPY  --from=builder /opt/venv /opt/venv
 
 WORKDIR /app
-
-USER $APP_USER
-
-COPY --chown=${APP_USER}:${APP_GROUP} code/ /app
+USER appuser
+COPY --chown=appuser:appgroup code/ /app
 CMD ["python", "-"]
diff --git a/containers/bmc-utils/requirements.txt b/containers/bmc-utils/requirements.txt
@@ -1,3 +1,3 @@
 requests==2.32.3
-pynautobot==2.4.1
+pynautobot==2.4.2
 sushy==5.3.0
diff --git a/containers/ironic-nautobot-client/Dockerfile.ironic-nautobot-client b/containers/ironic-nautobot-client/Dockerfile.ironic-nautobot-client
@@ -1,12 +1,6 @@
-ARG BASE=ghcr.io/rackerlabs/understack/argo-python3.11.8-alpine3.19:latest
+ARG BASE=ghcr.io/rackerlabs/understack/python3.11.8-alpine3.19:latest
 FROM ${BASE} AS builder
 
-ARG APP_PATH=/app
-ARG APP_USER=appuser
-ARG APP_GROUP=appgroup
-ARG APP_USER_UID=1000
-ARG APP_GROUP_GID=1000
-
 RUN --mount=type=cache,target=/var/cache/apk apk add --virtual build-deps gcc python3-dev musl-dev linux-headers
 RUN --mount=type=cache,target=/root/.cache/.pip pip install 'wheel==0.43.0'
 RUN --mount=type=cache,target=/root/.cache/.pip \
@@ -15,29 +9,21 @@ RUN --mount=type=cache,target=/root/.cache/.pip \
     /opt/poetry/bin/poetry self add 'poetry-dynamic-versioning[plugin]==1.3.0'
 
 # copy in the code
-COPY --chown=${APP_USER}:${APP_GROUP} python/understack-workflows /app
-COPY --chown=${APP_USER}:${APP_GROUP} python/understack-flavor-matcher /understack-flavor-matcher
+COPY --chown=appuser:appgroup python/understack-workflows /app
+COPY --chown=appuser:appgroup python/understack-flavor-matcher /understack-flavor-matcher
 # need netifaces built as a wheel
-RUN --mount=type=cache,target=/root/.cache/.pip pip wheel --wheel-dir /app/dist netifaces
+RUN --mount=type=cache,target=/root/.cache/.pip pip wheel --wheel-dir /app/dist netifaces psutil
 # build wheels and requirements.txt, skip hashes due to building of netifaces above which won't match
 RUN cd /app && /opt/poetry/bin/poetry build -f wheel && /opt/poetry/bin/poetry export --without-hashes -f requirements.txt -o dist/requirements.txt
 
 FROM ${BASE} AS prod
-ARG APP_PATH=/app
-ARG APP_USER=appuser
-ARG APP_GROUP=appgroup
-ARG APP_USER_UID=1000
-ARG APP_GROUP_GID=1000
-
-
 LABEL org.opencontainers.image.description="UnderStack Workflows"
-
 WORKDIR /app
 
 RUN mkdir -p /opt/venv/wheels/
 COPY --from=builder /app/dist/*.whl /app/dist/requirements.txt /opt/venv/wheels/
-COPY --chown=${APP_USER}:${APP_GROUP} python/understack-flavor-matcher /understack-flavor-matcher
+COPY --chown=appuser:appgroup python/understack-flavor-matcher /understack-flavor-matcher
 
-RUN --mount=type=cache,target=/root/.cache/.pip /opt/venv/bin/pip install --find-links /opt/venv/wheels/ --only-binary netifaces -r /opt/venv/wheels/requirements.txt understack-workflows
+RUN --mount=type=cache,target=/root/.cache/.pip /opt/venv/bin/pip install --find-links /opt/venv/wheels/ --only-binary netifaces psutil -r /opt/venv/wheels/requirements.txt understack-workflows
 
-USER $APP_USER
+USER appuser
diff --git a/containers/nova-flavors/Dockerfile.nova-flavors b/containers/nova-flavors/Dockerfile.nova-flavors
@@ -0,0 +1,35 @@
+FROM ghcr.io/rackerlabs/understack/argo-python3.12.2-alpine3.19 AS builder
+
+RUN --mount=type=cache,target=/var/cache/apk apk add --virtual build-deps gcc python3-dev musl-dev linux-headers
+RUN --mount=type=cache,target=/root/.cache/.pip pip install 'wheel==0.43.0'
+RUN --mount=type=cache,target=/root/.cache/.pip \
+    python -m venv /opt/poetry && \
+    /opt/poetry/bin/pip install 'poetry==1.7.1' && \
+    /opt/poetry/bin/poetry self add 'poetry-dynamic-versioning[plugin]==1.3.0'
+
+# copy in the code
+COPY --chown=appuser:appgroup operators/nova-flavors /app
+COPY --chown=appuser:appgroup python/understack-flavor-matcher /understack-flavor-matcher
+# need watchdog and psutil built AS a wheel
+RUN --mount=type=cache,target=/root/.cache/.pip pip wheel --wheel-dir /app/dist watchdog psutil
+CMD ["nova-flavors-sync"]
+
+WORKDIR /app
+RUN cd /app && /opt/poetry/bin/poetry build -f wheel && /opt/poetry/bin/poetry export --without-hashes -f requirements.txt -o dist/requirements.txt
+
+######################## PROD  ########################
+FROM ghcr.io/rackerlabs/understack/argo-python3.12.2-alpine3.19 AS prod
+
+ENV FLAVORS_DIR="/flavors"
+ENV NOVA_FLAVOR_MONITOR_LOGLEVEL="info"
+
+LABEL org.opencontainers.image.description="Nova-Flavors synchronizer"
+
+RUN mkdir -p /opt/venv/wheels/
+COPY --from=builder /app/dist/*.whl /app/dist/requirements.txt /opt/venv/wheels/
+COPY --chown=appuser:appgroup python/understack-flavor-matcher /python/understack-flavor-matcher
+
+RUN --mount=type=cache,target=/root/.cache/.pip cd /app && /opt/venv/bin/pip install --find-links /opt/venv/wheels/ --only-binary watchdog psutil -r /opt/venv/wheels/requirements.txt nova-flavors
+
+USER appuser
+CMD ["nova-flavors-sync"]