Add support for a hosted mode deployment

The "pause" hack no longer works with the latest addon-framework
library. The alternative, which is in this commit, is to return an error
in the `Manifests` method when the pause annotation is set. It does spam
the logs but since this isn't a setting that should be used in
production, this seems okay until the following is addressed:
open-cluster-management-io/addon-framework#80

Relates:
stolostron/backlog#24362

Signed-off-by: mprahl <[email protected]>
(cherry picked from commit 12cfe6e3f51534f0d10edab037bbaf49d7f298e2)

.github/workflows/kind.yml

@@ -26,6 +26,9 @@ jobs:
  kind:
  - 'v1.19.16'
  - 'latest'
+ hosted_mode:
+ - "true"
+ - "false"
  name: KinD tests
  steps:
  - name: Checkout 
@@ -40,60 +43,33 @@ jobs:
  with:
  go-version: ${{ matrix.go }}
 
- - name: Verify modules
- run: |
- go mod verify
-
- - name: Verify format
- run: |
- make fmt
- git diff --exit-code
- make lint
-
  - name: Create KinD Cluster (${{ matrix.kind }}) and deploy components
  env:
  KIND_VERSION: ${{ matrix.kind }}
+ HOSTED_MODE: ${{ matrix.hosted_mode }}
  run: |
  ./build/manage-clusters.sh
 
- - name: Unit and Integration Tests
+ - if: matrix.hosted_mode == 'false'
+ name: E2E tests
  run: |
- make test
+ export GOPATH=$(go env GOPATH)
+ make wait-for-work-agent
+ make e2e-test
 
- - name: E2E Tests
+ - if: matrix.hosted_mode == 'true'
+ name: E2E tests hosted mode
  run: |
  export GOPATH=$(go env GOPATH)
  make wait-for-work-agent
- make e2e-test
- 
+ make e2e-test-hosted-mode
+
  - name: Debug
  if: ${{ failure() }}
  run: |
  RUN_MODE="debug" ./build/manage-clusters.sh
 
- - name: Gosec
- if: |
- github.event_name == 'push' &&
- github.repository_owner == 'stolostron' &&
- matrix.kind == 'latest'
- run: |
- make gosec-scan
-
- - name: SonarCloud Analysis
- if: |
- github.event_name == 'push' &&
- github.repository_owner == 'stolostron' &&
- matrix.kind == 'latest'
- uses: sonarsource/sonarcloud-github-action@master
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
- with:
- projectBaseDir: governance-policy-addon-controller
- args: >
- --debug
-
- - name: Clean up cluster
+ - name: Clean up clusters
  if: ${{ always() }}
  run: |
  RUN_MODE="delete" ./build/manage-clusters.sh

.github/workflows/unit.yml

@@ -0,0 +1,66 @@
+name: Linting and Unit tests
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+defaults:
+ run:
+ shell: bash
+ working-directory: governance-policy-addon-controller
+
+jobs:
+ linting-and-unit-tests:
+ runs-on: ubuntu-latest
+ name: Linting and Unit tests
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ path: governance-policy-addon-controller
+ fetch-depth: 0 # Fetch all history for all tags and branches
+
+ - name: Set up Go
+ uses: actions/setup-go@v3
+ with:
+ go-version-file: governance-policy-addon-controller/go.mod
+
+ - name: Verify modules
+ run: |
+ go mod verify
+
+ - name: Verify format
+ run: |
+ make fmt
+ git diff --exit-code
+ make lint
+
+ - name: Unit and Integration Tests
+ run: |
+ make test
+
+ - name: Gosec
+ if: |
+ github.event_name == 'push' &&
+ github.repository_owner == 'stolostron' &&
+ matrix.kind == 'latest'
+ run: |
+ make gosec-scan
+
+ - name: SonarCloud Analysis
+ if: |
+ github.event_name == 'push' &&
+ github.repository_owner == 'stolostron' &&
+ matrix.kind == 'latest'
+ uses: sonarsource/sonarcloud-github-action@master
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+ with:
+ projectBaseDir: governance-policy-addon-controller
+ args: >
+ --debug

Makefile

@@ -153,7 +153,10 @@ KUBEWAIT ?= $(PWD)/build/common/scripts/kubewait.sh
 
 KIND_NAME ?= policy-addon-ctrl1
 KIND_KUBECONFIG ?= $(PWD)/$(KIND_NAME).kubeconfig
-HUB_KUBECONFIG ?= $(PWD)/$(KIND_NAME).kubeconfig-internal
+KIND_KUBECONFIG_INTERNAL ?= $(PWD)/$(KIND_NAME).kubeconfig-internal
+HUB_KUBECONFIG ?= $(PWD)/policy-addon-ctrl1.kubeconfig
+HUB_KUBECONFIG_INTERNAL ?= $(PWD)/policy-addon-ctrl1.kubeconfig-internal
+HUB_CLUSTER_NAME ?= policy-addon-ctrl1
 MANAGED_CLUSTER_NAME ?= cluster1
 ifneq ($(KIND_VERSION), latest)
  KIND_ARGS = --image kindest/node:$(KIND_VERSION)
@@ -168,33 +171,38 @@ $(KIND_KUBECONFIG):
  @echo "creating cluster"
  kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
  kind get kubeconfig --name $(KIND_NAME) > $(KIND_KUBECONFIG)
+ kind get kubeconfig --name $(KIND_NAME) --internal > $(KIND_KUBECONFIG_INTERNAL)
  KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.57.0/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml
 
-$(HUB_KUBECONFIG):
- @echo "fetching internal kubeconfig"
- kind get kubeconfig --name $(KIND_NAME) --internal > $(HUB_KUBECONFIG)
-
 .PHONY: kind-delete-cluster
 kind-delete-cluster: ## Delete a kind cluster.
  kind delete cluster --name $(KIND_NAME) || true
- rm $(KIND_KUBECONFIG) || true
- rm $(HUB_KUBECONFIG) || true
+ -rm $(KIND_KUBECONFIG)
+ -rm $(KIND_KUBECONFIG_INTERNAL)
 
 REGISTRATION_OPERATOR = $(PWD)/.go/registration-operator
 $(REGISTRATION_OPERATOR):
  @mkdir -p .go
  git clone --depth 1 https://github.com/open-cluster-management-io/registration-operator.git .go/registration-operator
+ # A workaround for https://github.com/open-cluster-management-io/registration-operator/pull/266
+ sed -i 's/-e "s,mode: Default,mode: Detached,"/-e "s,mode: Default,mode: Detached," -e "s,cluster1,$$(MANAGED_CLUSTER_NAME),"/' .go/registration-operator/Makefile
 
 .PHONY: kind-deploy-registration-operator-hub
-kind-deploy-registration-operator-hub: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) $(HUB_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
+kind-deploy-registration-operator-hub: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
  cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) make deploy-hub
  @printf "\n*** Pausing and waiting to let everything deploy ***\n\n"
  KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager -n open-cluster-management -c condition=Available -m 90
  KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager-placement-controller -n open-cluster-management-hub -c condition=Available -m 90
+ @echo installing Policy CRD on hub
+ KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/stolostron/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml
 
 .PHONY: kind-deploy-registration-operator-managed
 kind-deploy-registration-operator-managed: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
- cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG) make deploy-spoke
+ cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) make deploy-spoke
+
+.PHONY: kind-deploy-registration-operator-managed-hosted
+kind-deploy-registration-operator-managed-hosted: $(REGISTRATION_OPERATOR) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster in hosted mode.
+ cd $(REGISTRATION_OPERATOR) && KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(MANAGED_CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) make deploy-spoke-hosted
 
 .PHONY: kind-approve-cluster
 kind-approve-cluster: $(KIND_KUBECONFIG) ## Approve managed cluster in the kind cluster.
@@ -238,7 +246,11 @@ e2e-dependencies: ## Download ginkgo locally if necessary.
 
 .PHONY: e2e-test
 e2e-test: e2e-dependencies
- $(GINKGO) -v --fail-fast --slow-spec-threshold=10s test/e2e
+ $(GINKGO) -v --label-filter="!hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
+
+.PHONY: e2e-test-hosted-mode
+e2e-test-hosted-mode: e2e-dependencies
+ $(GINKGO) -v --label-filter="hosted-mode" --fail-fast --slow-spec-threshold=10s test/e2e
 
 .PHONY: e2e-debug
 e2e-debug: ## Collect debug logs from deployed clusters.

build/crd-sync.sh

@@ -50,13 +50,15 @@ $(cat .go/cert-policy-crd-v1.yaml)
 {{- end }}
 EOF
 
+addLabelsExpression='.metadata.labels += {"addon.open-cluster-management.io/hosted-manifest-location": "hosting"}'
+
 cat > pkg/addon/configpolicy/manifests/managedclusterchart/templates/policy.open-cluster-management.io_configurationpolicies_crd.yaml << EOF
 # Copyright Contributors to the Open Cluster Management project
 
 {{- if semverCompare "< 1.16.0" .Capabilities.KubeVersion.Version }}
-$(cat .go/config-policy-crd-v1beta1.yaml)
+$(yq e "$addLabelsExpression" .go/config-policy-crd-v1beta1.yaml)
 {{ else }}
-$(cat .go/config-policy-crd-v1.yaml)
+$(yq e "$addLabelsExpression" .go/config-policy-crd-v1.yaml)
 {{- end }}
 EOF
 
@@ -74,9 +76,9 @@ cat > pkg/addon/policyframework/manifests/managedclusterchart/templates/policy.o
 # Copyright Contributors to the Open Cluster Management project
 
 {{- if semverCompare "< 1.16.0" .Capabilities.KubeVersion.Version }}
-$(cat .go/policy-crd-v1beta1.yaml)
+$(yq e "$addLabelsExpression" .go/policy-crd-v1beta1.yaml)
 {{ else }}
-$(cat .go/policy-crd-v1.yaml)
+$(yq e "$addLabelsExpression" .go/policy-crd-v1.yaml)
 {{- end }}
 EOF
 

build/manage-clusters.sh

@@ -8,6 +8,14 @@ if [[ -n "${MANAGED_CLUSTER_COUNT//[0-9]}" ]] || [[ "${MANAGED_CLUSTER_COUNT}" =
  echo "error: provided MANAGED_CLUSTER_COUNT is not a nonzero integer"
  exit 1
 fi
+
+if [[ "${MANAGED_CLUSTER_COUNT}" -gt 1 ]] && [[ "${HOSTED_MODE}" == "true" ]]; then
+ # This is a current limitation in the registration operator Makefile where the hosted mode Klusterlet object name
+ # is not customizable.
+ echo "error: provided MANAGED_CLUSTER_COUNT cannot be greater than 1 in hosted mode"
+ exit 1
+fi
+
 KIND_PREFIX=${KIND_PREFIX:-"policy-addon-ctrl"}
 CLUSTER_PREFIX=${CLUSTER_PREFIX:-"cluster"}
 
@@ -30,7 +38,6 @@ esac
 for i in $(seq 2 $((MANAGED_CLUSTER_COUNT+1))); do
  export KIND_NAME="${KIND_PREFIX}${i}"
  export MANAGED_CLUSTER_NAME="${CLUSTER_PREFIX}${i}"
- export HUB_KUBECONFIG="${PWD}/${KIND_PREFIX}1.kubeconfig-internal"
  case ${RUN_MODE} in
  delete)
  make kind-delete-cluster
@@ -39,7 +46,12 @@ for i in $(seq 2 $((MANAGED_CLUSTER_COUNT+1))); do
  make e2e-debug
  ;;
  create)
- make kind-deploy-registration-operator-managed
+ if [[ "${HOSTED_MODE}" == "true" ]]; then
+ make kind-deploy-registration-operator-managed-hosted
+ else
+ make kind-deploy-registration-operator-managed
+ fi
+
  # Approval takes place on the hub
  export KIND_NAME="${KIND_PREFIX}1"
  make kind-approve-cluster

config/rbac/role.yaml

@@ -121,6 +121,17 @@ rules:
  - patch
  - update
  - watch
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
+ - get
+ - list
+ - patch
+ - update
+ - watch
 - apiGroups:
  - ""
  resourceNames:
@@ -203,19 +214,9 @@ rules:
  - manifestworks
  verbs:
  - create
+ - delete
  - get
  - list
- - watch
-- apiGroups:
- - work.open-cluster-management.io
- resourceNames:
- - addon-cert-policy-controller-deploy
- - addon-config-policy-controller-deploy
- - addon-governance-policy-framework-deploy
- - addon-iam-policy-controller-deploy
- resources:
- - manifestworks
- verbs:
- - delete
  - patch
  - update
+ - watch