From fc1fc80f1fc271dec6ee191300297b06ef724f32 Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 12 Nov 2024 11:48:18 +0100 Subject: [PATCH 1/2] Rewrite dumper in Go Differences: - All APIs are fully qualified in both the options (`--must-exist=certificates.cert-manager.io`, `--ignore=deployment.apps`) and the output files (`objects-Certificate.cert-manager.io.json`). This makes it possible to distinguish between objects with the same kind but different groups. See https://github.com/projectsyn/k8s-object-dumper/issues/47. - Resources without a list endpoint are ignored and do not cause and error or need to be explicitly ignored. - Ignore and must-exist options are now command line flags instead of files in `/usr/local/share`. Fixes #47 and #42. --- .dockerignore | 4 - .editorconfig | 26 -- .github/ISSUE_TEMPLATE/01_bug_report.md | 20 - .github/ISSUE_TEMPLATE/02_feature_request.md | 18 - .github/ISSUE_TEMPLATE/bug_report.yml | 49 ++ .github/ISSUE_TEMPLATE/config.yml | 4 - .github/ISSUE_TEMPLATE/feature_request.yml | 67 +++ .github/PULL_REQUEST_TEMPLATE.md | 16 +- .github/changelog-configuration.json | 68 +-- .github/workflows/build.yml | 107 +---- .github/workflows/lint.yml | 27 ++ .github/workflows/release.yml | 63 +++ .github/workflows/test.yml | 32 ++ .gitignore | 18 +- .goreleaser.yml | 54 +++ .yamllint.yml | 7 - CONTRIBUTING.md | 2 - Dockerfile | 50 +- Makefile | 66 +++ Makefile.vars.mk | 19 + README.md | 112 ++--- dump-objects | 456 ------------------- go.mod | 67 +++ go.sum | 194 ++++++++ internal/pkg/discovery/discovery.go | 149 ++++++ internal/pkg/discovery/discovery_test.go | 116 +++++ internal/pkg/dumper/dir.go | 115 +++++ internal/pkg/dumper/dir_test.go | 165 +++++++ internal/pkg/dumper/dumper.go | 19 + internal/pkg/dumper/dumper_test.go | 43 ++ known-to-fail | 13 - main.go | 85 ++++ must-exist | 19 - renovate.json | 6 +- 34 files changed, 1463 insertions(+), 813 deletions(-) delete mode 100644 .dockerignore delete mode 100644 .editorconfig delete mode 100644 .github/ISSUE_TEMPLATE/01_bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/02_feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 .goreleaser.yml delete mode 100644 .yamllint.yml create mode 100644 Makefile create mode 100644 Makefile.vars.mk delete mode 100755 dump-objects create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/pkg/discovery/discovery.go create mode 100644 internal/pkg/discovery/discovery_test.go create mode 100644 internal/pkg/dumper/dir.go create mode 100644 internal/pkg/dumper/dir_test.go create mode 100644 internal/pkg/dumper/dumper.go create mode 100644 internal/pkg/dumper/dumper_test.go delete mode 100644 known-to-fail create mode 100644 main.go delete mode 100644 must-exist diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 5aa79f4..0000000 --- a/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!dump-objects -!known-to-fail -!must-exist diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 98e0bca..0000000 --- a/.editorconfig +++ /dev/null @@ -1,26 +0,0 @@ -; This file is for unifying the coding style for different editors and IDEs. -; More information at https://editorconfig.org - -root = true - -[*] -charset = utf-8 -end_of_line = lf -insert_final_newline = true -trim_trailing_whitespace = true - -[*.{y*ml,*json*,*sonnet}] -indent_style = space -indent_size = 2 - -[*.*sonnet] -# C-style doc comments -block_comment_start = /* -block_comment = * -block_comment_end = */ - -[.gitkeep] -insert_final_newline = false - -[Makefile] -indent_style = tab diff --git a/.github/ISSUE_TEMPLATE/01_bug_report.md b/.github/ISSUE_TEMPLATE/01_bug_report.md deleted file mode 100644 index f074759..0000000 --- a/.github/ISSUE_TEMPLATE/01_bug_report.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: 🐜 Bug report -about: Create a report to help us improve 🔧 -labels: bug ---- - - - -## Steps to Reproduce the Problem - - - 1. - 1. - 1. - -## Actual Behavior - - -## Expected Behavior - diff --git a/.github/ISSUE_TEMPLATE/02_feature_request.md b/.github/ISSUE_TEMPLATE/02_feature_request.md deleted file mode 100644 index bf42f36..0000000 --- a/.github/ISSUE_TEMPLATE/02_feature_request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: 🚀 Feature request -about: Suggest an idea for this project 💡 -labels: enhancement ---- - -## Context - - -## Alternatives - diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..e866d86 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,49 @@ +name: 🐛 Bug report +description: Create a report to help us improve 🎉 +labels: + - bug + +body: + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: context + attributes: + label: Additional Context + description: Add any other context about the problem here. + validations: + required: false + - type: textarea + id: logs + attributes: + label: Logs + description: If applicable, add logs to help explain the bug. + render: shell + validations: + required: false + - type: textarea + id: expected_behavior + attributes: + label: Expected Behavior + description: A clear and concise description of what you expected to happen. + validations: + required: true + - type: textarea + id: reproduction_steps + attributes: + label: Steps To Reproduce + description: Describe steps to reproduce the behavior + validations: + required: false + - type: textarea + id: version + attributes: + label: Versions + placeholder: v1.2.3 [, Kubernetes 1.21] + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index ea8278c..3ba13e0 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1 @@ blank_issues_enabled: false -contact_links: - - name: ❓ Help and Support RocketChat Channel - url: https://community.appuio.ch - about: Please ask and answer questions here. 🏥 diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..401c7aa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,67 @@ +name: 🚀 Feature request +description: Suggest an idea for this project 💡 +labels: + - enhancement + +body: + - type: textarea + id: summary + attributes: + label: Summary + value: | + **As** role name\ + **I want** a feature or functionality\ + **So that** I get certain business value + description: This user story helps us to quickly understand what this idea is about. + validations: + required: true + - type: textarea + id: context + attributes: + label: Context + description: Add more information here. You are completely free regarding form and length. + validations: + required: true + - type: textarea + id: out_of_scope + attributes: + label: Out of Scope + description: List aspects that are explicitly not part of this feature + placeholder: | + - ... + - ... + - ... + validations: + required: false + - type: textarea + id: links + attributes: + label: Further links + description: URLs of relevant Git repositories, PRs, Issues, etc. + placeholder: | + - #567 + - https://kubernetes.io/docs/reference/ + validations: + required: false + - type: textarea + id: acceptance_criteria + attributes: + label: Acceptance Criteria + description: If you already have ideas what the detailed requirements are, please list them below in given-when-then expressions. + placeholder: | + - Given a precondition, when an action happens, then expect a result + + ```gherkin + Given a precondition + When an action happens + Then expect a result + ``` + validations: + required: false + - type: textarea + id: implementation_idea + attributes: + label: Implementation Ideas + description: If applicable, shortly list possible implementation ideas + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4d8d78d..c6d8d79 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,23 +1,19 @@ - +* Short summary of what's included in the PR +* Give special note to breaking changes ## Checklist - -- [ ] Keep pull requests small so they can be easily reviewed. -- [ ] Update the documentation. - [ ] Categorize the PR by setting a good title and adding one of the labels: `bug`, `enhancement`, `documentation`, `change`, `breaking`, `dependency` as they show up in the changelog +- [ ] Update tests. - [ ] Link this PR to related issues. diff --git a/.github/changelog-configuration.json b/.github/changelog-configuration.json index 56a090f..8c93e7b 100644 --- a/.github/changelog-configuration.json +++ b/.github/changelog-configuration.json @@ -1,30 +1,42 @@ { - "pr_template": "- ${{TITLE}} (#${{NUMBER}})", - "categories": [ - { - "title": "## 🚀 Features", - "labels": ["enhancement", "feature"] - }, - { - "title": "## 🛠️ Minor Changes", - "labels": ["change"] - }, - { - "title": "## 🔎 Breaking Changes", - "labels": ["breaking"] - }, - { - "title": "## 🐛 Fixes", - "labels": ["bug", "fix"] - }, - { - "title": "## 📄 Documentation", - "labels": ["documentation"] - }, - { - "title": "## 🔗 Dependency Updates", - "labels": ["dependency"] - } - ], - "template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}" + "pr_template": "- ${{TITLE}} (#${{NUMBER}})", + "categories": [ + { + "title": "## 🚀 Features", + "labels": [ + "enhancement" + ] + }, + { + "title": "## 🛠️ Minor Changes", + "labels": [ + "change" + ] + }, + { + "title": "## 🔎 Breaking Changes", + "labels": [ + "breaking" + ] + }, + { + "title": "## 🐛 Fixes", + "labels": [ + "bug" + ] + }, + { + "title": "## 📄 Documentation", + "labels": [ + "documentation" + ] + }, + { + "title": "## 🔗 Dependency Updates", + "labels": [ + "dependency" + ] + } + ], + "template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}" } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d02b093..911c0f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,105 +1,32 @@ -name: Docker image build +name: Build on: - schedule: - - cron: '0 7 * * *' # everyday at 07:00 - push: - branches: - - 'master' - tags: - - 'v*.*.*' pull_request: branches: - master + push: + branches: + - master jobs: - docker: + go: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=projectsyn/k8s-object-dumper - VERSION=noop - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly - elif [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then - VERSION=edge - fi - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - MINOR=${VERSION%.*} - MAJOR=${MINOR%.*} - TAGS="$TAGS,${DOCKER_IMAGE}:${MINOR},${DOCKER_IMAGE}:${MAJOR},${DOCKER_IMAGE}:latest" - elif [ "${{ github.event_name }}" = "push" ]; then - TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" - fi - echo "version=${VERSION}" >> ${GITHUB_ENV} - echo "tags=${TAGS}" >> ${GITHUB_ENV} - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_ENV} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + - uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Determine Go version from go.mod + run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV - - name: Login to DockerHub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + - uses: actions/setup-go@v5 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + go-version: ${{ env.GO_VERSION }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v6 + - uses: actions/cache@v4 with: - context: . - file: ./Dockerfile - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ env.tags }} - labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} - org.opencontainers.image.description=${{ github.event.repository.description }} - org.opencontainers.image.url=${{ github.event.repository.html_url }} - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.version=${{ env.version }} - org.opencontainers.image.created=${{ env.created }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.licenses=${{ github.event.repository.license.spdx_id }} + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - - name: Build changelog from PRs with labels - if: startsWith(github.ref, 'refs/tags/v') - id: build_changelog - uses: mikepenz/release-changelog-builder-action@v5 - with: - configuration: ".github/changelog-configuration.json" - # PreReleases still get a changelog, but the next full release gets a diff since the last full release, - # combining possible changelogs of all previous PreReleases in between. - # PreReleases show a partial changelog since last PreRelease. - ignorePreReleases: "${{ !contains(github.ref, '-rc') }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Create Release - if: startsWith(github.ref, 'refs/tags/v') - uses: actions/create-release@v1 - with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - body: ${{steps.build_changelog.outputs.changelog}} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Run build + run: make build diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..b83139f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: Lint + +on: + pull_request: {} + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Determine Go version from go.mod + run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run linters + run: make lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7239b5a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: Release + +on: + push: + tags: + - "*" + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine Go version from go.mod + run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build changelog from PRs with labels + id: build_changelog + uses: mikepenz/release-changelog-builder-action@v5 + with: + configuration: ".github/changelog-configuration.json" + # PreReleases still get a changelog, but the next full release gets a diff since the last full release, + # combining possible changelogs of all previous PreReleases in between. + # PreReleases show a partial changelog since last PreRelease. + ignorePreReleases: "${{ !contains(github.ref, '-rc') }}" + outputFile: .github/release-notes.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish releases + uses: goreleaser/goreleaser-action@v6 + with: + args: release --release-notes .github/release-notes.md + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1b806d2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,32 @@ +name: Test + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Determine Go version from go.mod + run: echo "GO_VERSION=$(grep "go 1." go.mod | cut -d " " -f 2)" >> $GITHUB_ENV + + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run tests + run: make test diff --git a/.gitignore b/.gitignore index 8a2ce51..f75c5c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,14 @@ -tmp/ -log/ -redhat/rpms +# Goreleaser +dist/ +.github/release-notes.md -# Data directory used for local testing -data/ +# Tools +bin/ + +# Build +k8s-object-dumper +*.out + +# Docs +.cache/ +.public/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..06a6b6e --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,54 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +builds: +- env: + - CGO_ENABLED=0 # this is needed otherwise the Docker image build is faulty + goarch: + - amd64 + - arm64 + goos: + - linux + goarm: + - 8 + +archives: +- format: binary + name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + +checksum: + name_template: "checksums.txt" + +snapshot: + name_template: "{{ incpatch .Version }}-snapshot" + +dockers: +- goarch: amd64 + use: buildx + build_flag_templates: + - "--platform=linux/amd64" + image_templates: + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64" + +- goarch: arm64 + use: buildx + build_flag_templates: + - "--platform=linux/arm64/v8" + image_templates: + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64" + +docker_manifests: + ## ghcr.io + # For prereleases, updating `latest` does not make sense. + # Only the image for the exact version should be pushed. + - name_template: "{{ if not .Prerelease }}{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:latest{{ end }}" + image_templates: + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64" + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64" + + - name_template: "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}" + image_templates: + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-amd64" + - "{{ .Env.REGISTRY }}/{{ .Env.IMAGE_NAME }}:v{{ .Version }}-arm64" + +release: + prerelease: auto diff --git a/.yamllint.yml b/.yamllint.yml deleted file mode 100644 index addf0aa..0000000 --- a/.yamllint.yml +++ /dev/null @@ -1,7 +0,0 @@ -extends: default - -rules: - # 80 chars should be enough, but don't fail if a line is longer - line-length: - max: 80 - level: warning diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1329548..6276933 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,5 +2,3 @@ This code repository is part of Project Syn and the contribution guide at https://syn.tools/syn/contribution_guide.html does apply. - -Submit Pull Requests at https://github.com/projectsyn/component-k8s-object-dumper/pulls. diff --git a/Dockerfile b/Dockerfile index 1962ec7..890920b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,13 @@ -FROM docker.io/debian:12.7-slim as base +FROM docker.io/library/alpine:3.20 as runtime -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - bash \ - jq \ - less \ - moreutils \ - procps \ - && rm -rf /var/lib/apt/lists/* +RUN \ + apk add --update --no-cache \ + bash \ + curl \ + ca-certificates \ + tzdata -FROM base as downloader +ENTRYPOINT ["k8s-object-dumper"] +COPY k8s-object-dumper /usr/bin/ -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - && rm -rf /var/lib/apt/lists/* - -ARG K8S_VERSION=v1.18.20 - -RUN curl -sLo /tmp/kubectl "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" \ - && chmod +x /tmp/kubectl - -RUN curl -sLo /tmp/krossa.tar.gz https://github.com/appuio/krossa/releases/download/v0.0.4/krossa_0.0.4_linux_amd64.tar.gz \ - && mkdir /tmp/krossa \ - && tar -xzf /tmp/krossa.tar.gz --directory /tmp/krossa \ - && ls /tmp/krossa - -FROM base - -RUN mkdir /data \ - && chown 1001:0 /data - -COPY --from=downloader /tmp/kubectl /usr/local/bin -COPY --from=downloader /tmp/krossa/krossa /usr/local/bin -COPY dump-objects /usr/local/bin -COPY must-exist /usr/local/share/k8s-object-dumper/ -COPY known-to-fail /usr/local/share/k8s-object-dumper/ - -USER 1001 -ENTRYPOINT ["/usr/local/bin/dump-objects"] -CMD ["-d", "/data"] +USER 65536:0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..123090f --- /dev/null +++ b/Makefile @@ -0,0 +1,66 @@ +# Set Shell to bash, otherwise some targets fail with dash/zsh etc. +SHELL := /bin/bash + +# Disable built-in rules +MAKEFLAGS += --no-builtin-rules +MAKEFLAGS += --no-builtin-variables +.SUFFIXES: +.SECONDARY: +.DEFAULT_GOAL := help + +# General variables +include Makefile.vars.mk + +.PHONY: help +help: ## Show this help + @grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +.PHONY: build +build: build-bin build-docker ## All-in-one build + +.PHONY: build-bin +build-bin: export CGO_ENABLED = 0 +build-bin: fmt vet ## Build binary + @go build -o $(BIN_FILENAME) + +.PHONY: build-docker +build-docker: build-bin ## Build docker image + $(DOCKER_CMD) build -t $(CONTAINER_IMG) . + +.PHONY: run +run: + go run . + +.PHONY: test +test: envtest ## Test with envtest + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -race -coverprofile cover.out -covermode atomic ./... + +.PHONY: fmt +fmt: ## Run 'go fmt' against code + go fmt ./... + +.PHONY: vet +vet: ## Run 'go vet' against code + go vet ./... + +.PHONY: lint +lint: fmt vet generate ## All-in-one linting + @echo 'Check for uncommitted changes ...' + git diff --exit-code + +.PHONY: generate +generate: ## Generate additional code and artifacts + @go generate ./... + +.PHONY: clean +clean: ## Cleans local build artifacts + rm -rf dist .cache + +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest diff --git a/Makefile.vars.mk b/Makefile.vars.mk new file mode 100644 index 0000000..bab3237 --- /dev/null +++ b/Makefile.vars.mk @@ -0,0 +1,19 @@ +## These are some common variables for Make + +PROJECT_ROOT_DIR = . +PROJECT_NAME ?= k8s-object-dumper +PROJECT_OWNER ?= projectsyn + +## BUILD:go +BIN_FILENAME ?= $(PROJECT_NAME) + +## BUILD:docker +DOCKER_CMD ?= docker + +IMG_TAG ?= latest +# Image URL to use all building/pushing image targets +CONTAINER_IMG ?= local.dev/$(PROJECT_OWNER)/$(PROJECT_NAME):$(IMG_TAG) + +LOCALBIN ?= $(shell pwd)/bin +ENVTEST ?= $(LOCALBIN)/setup-envtest +ENVTEST_K8S_VERSION = 1.28.3 diff --git a/README.md b/README.md index abfa355..ec2f3a0 100644 --- a/README.md +++ b/README.md @@ -1,92 +1,70 @@ # K8s Object Dumper -K8s Object Dumper allows to collect all objects from Kubernetes and write them into files. -It is written to be used as a pre backup command for [K8up](https://k8up.io). +Discover and dump all listable objects from a Kubernetes cluster into JSON files. +Written to be used as a pre backup command for [K8up](https://k8up.io). -This repository is part of Project Syn. -For documentation on Project Syn, see https://syn.tools. - -K8s Object Dumper consists of a shell script. -This script uses the Kubernetes API to list each and every API and Kind known to the targeted cluster. -It then dumps all those kinds to Json files (one file per kind). -For easier restore, those dumped objects then get split up by namespace and kind. +## Usage -The resulting structure looks like the following: +The project uses controller-runtime's configuration discovery to find the Kubernetes API server. -``` -├─ objects-.json -├─ … -└─ split/ - ├─ / - | ├─ __all__.json - | ├─ .json - | └─ … - └─ … -``` -## Usage -Using Docker +### Dump to STDOUT ```bash -docker run --rm -v "/path/to/kubeconfig:/kubeconfig" -e KUBECONFIG=/kubeconfig -v "${PWD}/data:/data" projectsyn/k8s-object-dumper:latest -d /data > objects.tar.gz +$ k8s-object-dumper +{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"v1", ...}]} +{"apiVersion":"v1","kind":"List","items":[{"apiVersion":"apps/v1", ...}]} ``` -Using Kubernetes (with K8up) - -See [Commodore Component: cluster-backup](https://github.com/projectsyn/component-cluster-backup). - -## Configuration - -The `dump-objects` scripts reads configuration from two files. +### Dump to a directory -`/usr/local/share/k8s-object-dumper/must-exists` contains a list of types that must exist within the list of discovered types. -This is a safeguard helping to detect failure of the discover mechanism. -Types must be all lower case and plural. -One type per line. +```bash +$ k8s-object-dumper -d dir +``` -Example: +Will result in the following directory structure: ``` -configmaps -daemonsets -deployments -endpoints -ingresses -jobs -namespaces -nodes -persistentvolumeclaims -persistentvolumes -replicasets -roles -secrets -serviceaccounts -services -statefulsets +└─ dir/ + ├─ objects-[.].json + ├─ … + └─ split/ + ├─ / + | ├─ __all__.json + | ├─ [.].json + | └─ … + └─ … ``` -Some types can not be exported and the script will return an error for them. -Those errors can be suppressed by placing those types in `/usr/local/share/k8s-object-dumper/known-to-fail`. -Like `must-exist` types are listed line by line but in addition Bash regular expressions can be used. +### Advanced usage + +```bash +# Fail if a Pods, Deployments or AlertingRules are not found +$ k8s-object-dumper \ + -must-exist=pods \ + -must-exist=deployments.apps \ + -must-exist=alertingrules.monitoring.openshift.io +# Ignore all Secrets and all cert-manager objects +$ k8s-object-dumper \ + -ignore=secrets \ + -ignore=.+cert-manager.io +``` +## Development -Example: +The project uses [envtest](https://book.kubebuilder.io/reference/envtest) to run tests against a real Kubernetes API server. +```bash +$ make test ``` -.+mutators -.+reviews -.+validators -bindings -deploymentconfigrollbacks -imagesignatures -imagestream.+ -mutations -useridentitymappings -validations -``` -To enable informative logging output for non-error cases, set the `-D` argument. +## Differences to the original `bash` version `< 0.3.0` + +- All APIs are fully qualified in both the options (`--must-exist=certificates.cert-manager.io`, `--ignore=deployment.apps`) and the output files (`objects-Certificate.cert-manager.io.json`). + This makes it possible to distinguish between objects with the same kind but different groups. See https://github.com/projectsyn/k8s-object-dumper/issues/47. +- Resources without a list endpoint are ignored, do not cause an error, and don't need to be explicitly ignored. +- Ignore and must-exist options are now command line flags instead of files in `/usr/local/share`. ## Contributing and license diff --git a/dump-objects b/dump-objects deleted file mode 100755 index a49c556..0000000 --- a/dump-objects +++ /dev/null @@ -1,456 +0,0 @@ -#!/bin/bash -# -# Export all K8s objects before backup -# -# == Authors -# -# * Manuel Hutter -# -# == License -# -# Copyright (c) 2017, VSHN AG, info@vshn.ch -# Licensed under "BSD 3-Clause". See LICENSE file. - -set -e -u -o pipefail - -# sysexit.h -readonly EX_PROTOCOL=76 - -readonly min_expected_kinds=98 - -default_output_dir=/data/k8s-backup - -if ! kubectl=$(type -p kubectl); then - echo "K8s client \"kubectl\" not found in PATH" >&2 - exit 1 -fi - -usage() { - echo "Usage: $0 [-v] [-d ]" - echo - echo 'Options:' - echo ' -v Verbose output' - echo " -d Destination directory (default: ${default_output_dir})" - echo ' -s Short delays in case of failures' - echo ' -D enable Debug log' -} - -output_dir="$default_output_dir" -opt_verbose= -opt_debug=false -opt_fastretries= - -while getopts 'hvd:Ds' opt; do - case "$opt" in - h) - usage - exit 0 - ;; - v) opt_verbose=yes ;; - d) output_dir="$OPTARG" ;; - s) opt_fastretries=yes ;; - D) opt_debug=true ;; - *) - usage >&2 - exit 1 - ;; - esac -done - -shift $((OPTIND - 1)) - -if [[ "$#" -gt 0 ]]; then - usage >&2 - exit 1 -fi - -# Remove old files -find "$output_dir" -mindepth 1 -maxdepth 1 -type f -delete - -log() { - if [ "$opt_debug" = true ]; then - echo "$@" >&2 - fi -} - -delay_attempt() { - local attempt="$1" - local msg="$2" - local delay - - if [[ -n "$opt_fastretries" ]]; then - delay=1 - else - delay=$(( 1 + ( (1 + attempt) ** 5) )) - fi - log "Pausing for ${delay} seconds: ${msg}" >&2 - sleep "$delay" || : -} - -run_kubectl() { - "$kubectl" ${opt_verbose:+--v=10} "$@" -} - -# -# Capture output of "oc" command in destination file -# -capture_kubectl() { - local destfile="$1"; shift - local attempt - local status - - for ((attempt=0; ; ++attempt)); do - if run_kubectl "$@" > "$destfile"; then - return 0 - else - status=$? - - if (( attempt > 3 )); then - return "$status" - fi - - delay_attempt "$attempt" \ - "kubectl failed with exit status ${status} (arguments: $*)" - fi - done - - return 1 -} - -sanitize_name() { - sed -e 's#[^-.a-zA-Z0-9_]#-#g' -} - -verbose() { - [[ -n "$opt_verbose" ]] -} - -extract_group_versions() { - local base="$1" - - jq --arg base "$base" \ - '.groups[].versions[].groupVersion | ($base + "/" + .)' -} - -extract_versioned_apis() { - local base="$1" - - jq --arg base "$base" '.versions[] | ($base + "/" + .)' -} - -validate_kinds() { - local -a expected - mapfile -t expected < /usr/local/share/k8s-object-dumper/must-exist - local missing=() - for j in "${expected[@]}"; do - local found= - - for i; do - if [[ "$i" == "$j" ]]; then - found=yes - break - fi - done - - if [[ -z "$found" ]]; then - missing+=( "$j" ) - fi - done - - if [[ -n "${missing+${missing[*]}}" ]]; then - echo "Object kind discovery via API did not find the following types: ${missing[*]}" >&2 - return 1 - fi - - return 0 -} - -# -# Peruse Kubernetes API to gather list of known types -# -object_kinds() { - # Kubernetes legacy v1 API - capture_kubectl "${output_dir}/k8s-api.json" get --raw /api - - # Kubernetes APIs - capture_kubectl "${output_dir}/k8s-apis.json" get --raw /apis - - # Process API groups - extract_group_versions /apis \ - < "${output_dir}/k8s-apis.json" \ - > "${output_dir}/k8s-apis.groups.json" - extract_versioned_apis /api \ - < "${output_dir}/k8s-api.json" \ - > "${output_dir}/k8s-api.groups.json" - - # Combine groups - jq --slurp --raw-output 'sort | unique | @sh' \ - "${output_dir}/k8s-apis.groups.json" \ - "${output_dir}/k8s-api.groups.json" \ - > "${output_dir}/api-groups.txt" - - eval "local -a groups=( $(< "${output_dir}/api-groups.txt") )" - - local -a files=() - - # Get all object types for each group - for url in "${groups[@]}"; do - local san_name - san_name=$(sanitize_name <<< "${url##/}") - local name="api-${san_name}.json" - local fname="${output_dir}/${name}" - - capture_kubectl "$fname" get --raw "$url" - - files+=( "$fname" ) - done - - # Build list of types - jq --slurp --raw-output 'map(.resources[].name) | sort | unique | @sh' \ - "${files[@]}" -} - -# -# Determine whether a type can be retrieved from a cluster -# -# -retrievable_kind() { - local -a know_to_fail - mapfile -t known_to_fail < /usr/local/share/k8s-object-dumper/known-to-fail - - for kind in "${known_to_fail[@]}" - do - if [[ "$1" =~ ^${kind}$ ]]; then - return 1 - fi - done - - return 0 -} - -# -# Download objects and parse received JSON structure to determine count -# -# Failures during download or parsing are returned. A small set of errors is -# recognized and leads to $EX_PROTOCOL being returned. -# -fetch_objects() { - local kind="$1" destfile="$2" - local objcount= - local status=1 - local error= - - # Capture stderr - if error=$(run_kubectl get --all-namespaces --output=json "$kind" 2>&1 >"$destfile" | tee -a /dev/stderr); then - if objcount=$(jq --raw-output '.items | length' < "$destfile"); then - log "Received ${objcount} ${kind} objects" - return 0 - fi - else - status=$? - - local errprefix= - - if verbose; then - log "Kubernetes client failed with status ${status}" - errprefix='[^]]+ *[-_a-z0-9]+\.go:[0-9]+] *' - fi - - errprefix+='(|Error from server( \([a-zA-Z]+\)|): *|error: *)' - - if grep -Eiq \ - -e "^${errprefix}the server does not allow this method on the requested resource\$" \ - -e "^${errprefix}Unable to list \"${kind}\": the server could not find the requested resource\$" \ - -e "^${errprefix}Unable to list {\".*\" \"v1\" \"${kind}\"}: the server could not find the requested resource\$" \ - -e "^${errprefix}Unable to list \".*/v1, Resource=${kind}\": the server could not find the requested resource\$" \ - -e "^${errprefix}the server doesn't have a resource type \"${kind}\"\$" \ - -e "^${errprefix}You may not request a new project via this API.\$" \ - <<< "$error" - then - return "$EX_PROTOCOL" - fi - fi - - return 1 -} - -# -# Download all objects of given object kind -# -# In case of failures the download is attempted several times with short -# delays. Recognized errors (EX_PROTOCOL) are returned right away. -# -fetch() { - local kind="$1" destfile="$2" - local attempt - local status - - for ((attempt=0; ; ++attempt)); do - status=0 - ( fetch_objects "$kind" "$destfile"; ) || status=$? - - if (( status == 0 || status == EX_PROTOCOL )); then - return "$status" - fi - - if (( attempt > 3 )); then - break - fi - - delay_attempt "$attempt" "Retrieval of ${kind} objects failed" - done - - echo "Retrieving objects of kind \"${kind}\" failed" >&2 - - return 1 -} - -# -# Split object lists into individual files -# -split_objects() { - local splitdir="${output_dir}/split" - - if [[ ! -d "$splitdir" ]]; then - mkdir "$splitdir" - fi - - # Remove old files - find "$splitdir" -mindepth 1 -type f -name '*.json' -delete - - log "Splitting ${#} JSON files" - - if krossa "$splitdir" "$@"; then - # Remove empty directories - find "$splitdir" -type d -empty -delete - - return 0 - fi - - return 1 -} - -# -# Compare against Kubernetes version -# -# Arguments: -# op: jq comparison operator (==, <, <=, >, >=) -# major, minor: Integers -# -check_version() { - local op="$1" major="$2" minor="$3" - - jq --exit-status \ - --arg major "$major" --arg minor "$minor" \ - "[ - (.major | tonumber), - # Remove non-numeric suffix - (.minor | "'sub("\\D+$"; ""; "il")'" | tonumber) - ] ${op} - [(\$major | tonumber), (\$minor | tonumber)]" \ - < "${output_dir}/version.json" \ - > /dev/null -} - -date > "${output_dir}/timestamp-begin.txt" - -capture_kubectl "${output_dir}/version.txt" version - -capture_kubectl "${output_dir}/version.json" get --raw /version - -if check_version '<' 1 7; then - echo "Kubernetes 1.7 or newer required" >&2 - exit 1 -fi - -object_kinds > "${output_dir}/api-kinds.txt" - -eval "declare -a kinds=( $(< "${output_dir}/api-kinds.txt") )" - -declare -i errors=0 - -if (( "${#kinds[@]}" < min_expected_kinds )); then - echo "Expected at least ${min_expected_kinds} resource kinds" >&2 - (( ++errors )) || true -fi - -if ! validate_kinds "${kinds[@]}"; then - (( ++errors )) || true -fi - -log "Fetching resources for ${#kinds[@]} distinct kinds: ${kinds[*]}" - -declare -i idx=0 -declare -a object_files=() - -for i in "${kinds[@]}"; do - prefix="${i} ($((++idx))/${#kinds[@]}): " - - if [[ "$i" == */* ]]; then - if verbose; then - log "${prefix}Skipping subresource" - fi - continue - fi - - log "${prefix}Downloading" - - if retrievable_kind "$i"; then - retrievable=yes - else - retrievable= - fi - - san_name=$(sanitize_name <<< "$i") - destfile="${output_dir}/objects-${san_name}.json" - - if fetch "$i" "$destfile"; then - status=0 - object_files+=( "$destfile" ) - else - status=$? - fi - - if (( status == EX_PROTOCOL )); then - if [[ -n "$retrievable" ]]; then - echo "Download failed with status ${status} when resource kind \"${i}\" is expected to be retrievable" >&2 - status=1 - else - log "Ignoring error about unretrievable resource kind" - status=0 - fi - elif [[ -z "$retrievable" ]]; then - echo "Download succeeded even though resource kind \"${i}\" isn't expected to be retrievable" >&2 - status=1 - fi - - if (( status != 0 )); then - (( ++errors )) || true - fi - - log >&2 -done - -if ! split_objects ${object_files+"${object_files[@]}"}; then - (( ++errors )) || true -fi - -date > "${output_dir}/timestamp-end.txt" - - -# Output for K8up -( cd "${output_dir}" && tar czf "${output_dir}/k8s-objects.tar.gz" timestamp-*.txt version.* objects-*.json split/ ) - -# dump tar to original stdout -cat "${output_dir}/k8s-objects.tar.gz" >&1 - -if (( errors > 0 )); then - echo "Encountered ${errors} error(s) while backing up data" >&2 - exit 1 -fi - -log 'K8s backup finished without errors' - -exit 0 - -# vim: set sw=2 sts=2 et : diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9574ae7 --- /dev/null +++ b/go.mod @@ -0,0 +1,67 @@ +module github.com/projectsyn/k8s-object-dumper + +go 1.23.2 + +require ( + github.com/stretchr/testify v1.9.0 + go.uber.org/multierr v1.11.0 + k8s.io/api v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 + sigs.k8s.io/controller-runtime v0.19.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5bbc855 --- /dev/null +++ b/go.sum @@ -0,0 +1,194 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= +sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/pkg/discovery/discovery.go b/internal/pkg/discovery/discovery.go new file mode 100644 index 0000000..f97f649 --- /dev/null +++ b/internal/pkg/discovery/discovery.go @@ -0,0 +1,149 @@ +package discovery + +import ( + "context" + "fmt" + "io" + "regexp" + "slices" + "strings" + + "go.uber.org/multierr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" +) + +type DiscoveryOptions struct { + BatchSize int64 + LogWriter io.Writer + + // MustExistResources is a list of resources that must exist in the cluster. + // This can be used as a sanity check to ensure that the discovery process is working as expected. + // If a resource does not exist, the discovery process will fail. + // If the list is empty, no resources are required to exist. + MustExistResources []string + + // IgnoreResources is a list of resources to ignore during discovery. + IgnoreResources []*regexp.Regexp +} + +// GetBatchSize returns the set batch size for listing objects or the default. +func (opts DiscoveryOptions) GetBatchSize() int64 { + if opts.BatchSize == 0 { + return 500 + } + return opts.BatchSize +} + +// GetLogWriter returns the set batch size for listing objects or io.Discard as default. +func (opts DiscoveryOptions) GetLogWriter() io.Writer { + if opts.LogWriter == nil { + return io.Discard + } + return opts.LogWriter +} + +// DiscoverObjects discovers all objects in the cluster and calls the provided callback for each list of objects. +// The callback can be called multiple times with the same object kind. +// Objects are unique in general, but the callback should be able to handle duplicates. +// Some API servers do not implement list batching correctly and thus might introduce duplicates. +func DiscoverObjects(ctx context.Context, conf *rest.Config, cb func(*unstructured.UnstructuredList) error, opts DiscoveryOptions) error { + batchSize := opts.GetBatchSize() + logWriter := opts.GetLogWriter() + + dc, err := discovery.NewDiscoveryClientForConfig(conf) + if err != nil { + return fmt.Errorf("failed to create discovery client: %w", err) + } + dynClient, err := dynamic.NewForConfig(conf) + if err != nil { + return fmt.Errorf("failed to create dynamic client: %w", err) + } + + sprl, err := dc.ServerPreferredResources() + if err != nil { + return fmt.Errorf("failed to get server preferred resources: %w", err) + } + + fmt.Fprintln(logWriter, "Discovered resources:") + for _, re := range sprl { + fmt.Fprintln(logWriter, re.GroupVersion) + for _, r := range re.APIResources { + fmt.Fprintln(logWriter, " ", r.Kind) + } + } + + if len(opts.MustExistResources) > 0 { + want := sets.New(opts.MustExistResources...) + have := sets.New[string]() + for _, re := range sprl { + for _, r := range re.APIResources { + res := formatGVRForComparison(groupVersionFromString(re.GroupVersion).WithResource(r.Name)) + have.Insert(res) + } + } + missing := want.Difference(have) + if missing.Len() > 0 { + return fmt.Errorf("missing resources: %s", sets.List(missing)) + } + } + + var errors []error + for _, re := range sprl { + for _, r := range re.APIResources { + res := groupVersionFromString(re.GroupVersion).WithResource(r.Name) + if !slices.Contains(r.Verbs, "list") { + fmt.Fprintf(logWriter, "skipping %s: no list verb\n", res) + continue + } + + if i := slices.IndexFunc(opts.IgnoreResources, func(re *regexp.Regexp) bool { + return re.MatchString(formatGVRForComparison(res)) + }); i > -1 { + fmt.Fprintf(logWriter, "skipping %s: ignored by regex %q\n", res, opts.IgnoreResources[i].String()) + continue + } + + continueKey := "" + for { + l, err := dynClient.Resource(res).List(ctx, metav1.ListOptions{ + Limit: batchSize, + Continue: continueKey, + }) + if err != nil { + errors = append(errors, fmt.Errorf("failed to list %s: %w", res, err)) + break + } + if err := cb(l); err != nil { + errors = append(errors, fmt.Errorf("failed to dump %s: %w", res, err)) + } + if l.GetContinue() == "" { + break + } + continueKey = l.GetContinue() + } + } + } + + return multierr.Combine(errors...) +} + +func groupVersionFromString(s string) schema.GroupVersion { + parts := strings.Split(s, "/") + if len(parts) == 1 { + return schema.GroupVersion{Version: parts[0]} + } + return schema.GroupVersion{Group: parts[0], Version: parts[1]} +} + +func formatGVRForComparison(gvr schema.GroupVersionResource) string { + if gvr.Group == "" { + return gvr.Resource + } + return fmt.Sprintf("%s.%s", gvr.Resource, gvr.Group) +} diff --git a/internal/pkg/discovery/discovery_test.go b/internal/pkg/discovery/discovery_test.go new file mode 100644 index 0000000..04fd4b7 --- /dev/null +++ b/internal/pkg/discovery/discovery_test.go @@ -0,0 +1,116 @@ +package discovery_test + +import ( + "context" + "fmt" + "regexp" + "testing" + + "github.com/projectsyn/k8s-object-dumper/internal/pkg/discovery" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +type objKey struct { + apiVersion, kind, name, namespace string +} + +func Test_DiscoverObjects(t *testing.T) { + cfg, stop := setupEnvtestEnv(t) + defer stop() + + objs := map[objKey]unstructured.Unstructured{} + objTracker := func(obj *unstructured.UnstructuredList) error { + for _, o := range obj.Items { + objs[objKey{apiVersion: o.GetAPIVersion(), kind: o.GetKind(), name: o.GetName(), namespace: o.GetNamespace()}] = o + } + return nil + } + + c, err := client.New(cfg, client.Options{}) + require.NoError(t, err) + + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ns", + }, + } + sas := make([]client.Object, 0, 10) + for i := 1; i <= cap(sas); i++ { + sas = append(sas, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-service-account-%d", i), + Namespace: "test-ns", + }, + }) + } + cr := rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster-role", + }, + } + r := rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-role", + Namespace: "test-ns", + }, + } + + for _, obj := range append([]client.Object{&ns, &cr, &r}, sas...) { + require.NoError(t, c.Create(context.Background(), obj)) + } + + require.NoError(t, discovery.DiscoverObjects(context.Background(), cfg, objTracker, discovery.DiscoveryOptions{ + BatchSize: int64(cap(sas) / 2), + IgnoreResources: []*regexp.Regexp{ + regexp.MustCompile(`^roles.rbac.authorization.k8s.io$`), + }, + MustExistResources: []string{ + "clusterroles.rbac.authorization.k8s.io", + "deployments.apps", + "namespaces", + }, + })) + + require.Contains(t, objs, objKey{apiVersion: "v1", kind: "Namespace", name: "test-ns", namespace: ""}) + require.Contains(t, objs, objKey{apiVersion: "rbac.authorization.k8s.io/v1", kind: "ClusterRole", name: "test-cluster-role", namespace: ""}) + for i := 1; i <= cap(sas); i++ { + require.Contains(t, objs, objKey{apiVersion: "v1", kind: "ServiceAccount", name: fmt.Sprintf("test-service-account-%d", i), namespace: "test-ns"}) + } + require.NotContains(t, objs, objKey{apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", name: "test-role", namespace: "test-ns"}, "Roles are ignored by regex") +} + +func Test_DiscoverObjects_MustExistResources_NotSatisfied(t *testing.T) { + cfg, stop := setupEnvtestEnv(t) + defer stop() + + discard := func(obj *unstructured.UnstructuredList) error { + return nil + } + + require.ErrorContains(t, discovery.DiscoverObjects(context.Background(), cfg, discard, discovery.DiscoveryOptions{ + MustExistResources: []string{ + "fluxcapacitors.spaceship.io", + "namespaces", + }, + }), "missing resources: [fluxcapacitors.spaceship.io]") +} + +func setupEnvtestEnv(t *testing.T) (cfg *rest.Config, stop func()) { + t.Helper() + + testEnv := &envtest.Environment{} + + cfg, err := testEnv.Start() + require.NoError(t, err) + + return cfg, func() { + require.NoError(t, testEnv.Stop()) + } +} diff --git a/internal/pkg/dumper/dir.go b/internal/pkg/dumper/dir.go new file mode 100644 index 0000000..e752428 --- /dev/null +++ b/internal/pkg/dumper/dir.go @@ -0,0 +1,115 @@ +package dumper + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "go.uber.org/multierr" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// DirDumper writes objects to a directory. +// Must be initialized with newDirDumper. +// Must be closed after use. +type DirDumper struct { + dir string + + openFiles map[string]*os.File + sharedBuf *bytes.Buffer +} + +// NewDirDumper creates a new dirDumper that writes objects to the given directory. +// The directory will be created if it does not exist. +// If the directory cannot be created, an error is returned. +func NewDirDumper(dir string) (*DirDumper, error) { + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory %q: %w", dir, err) + } + return &DirDumper{ + dir: dir, + openFiles: make(map[string]*os.File), + sharedBuf: new(bytes.Buffer), + }, nil +} + +// Close closes the dirDumper and all open files. +// The dirDumper cannot be used after it is closed. +func (d *DirDumper) Close() error { + var errs []error + for _, f := range d.openFiles { + if err := f.Close(); err != nil { + errs = append(errs, err) + } + } + return multierr.Combine(errs...) +} + +// Dump writes the objects in the list to the directory. +// The objects are written to the directory in two ways: +// - All objects are written to a file named objects-.json +// - Objects with a namespace are written to a directory named split/ with two files: +// - __all__.json contains all objects in the namespace +// - .json contains all objects of the kind in the namespace +// +// If an object cannot be written, an error is returned. +// This method is not safe for concurrent use. +func (d *DirDumper) Dump(l *unstructured.UnstructuredList) error { + buf := d.sharedBuf + var errs []error + for _, o := range l.Items { + buf.Reset() + if err := json.NewEncoder(buf).Encode(o.Object); err != nil { + errs = append(errs, fmt.Errorf("failed to encode object: %w", err)) + continue + } + p := buf.Bytes() + gk := o.GroupVersionKind().GroupKind() + + if err := d.writeToFile(fmt.Sprintf("%s/objects-%s.json", d.dir, gk), p); err != nil { + errs = append(errs, err) + } + + if o.GetNamespace() == "" { + continue + } + + if err := d.writeToFile(fmt.Sprintf("%s/split/%s/__all__.json", d.dir, o.GetNamespace()), p); err != nil { + errs = append(errs, err) + } + if err := d.writeToFile(fmt.Sprintf("%s/split/%s/%s.json", d.dir, o.GetNamespace(), gk), p); err != nil { + errs = append(errs, err) + } + } + return multierr.Combine(errs...) +} + +func (d *DirDumper) writeToFile(path string, b []byte) error { + f, err := d.file(path) + if err != nil { + return fmt.Errorf("failed to open file for copying: %w", err) + } + if _, err := f.Write(b); err != nil { + return fmt.Errorf("failed to copy to file: %w", err) + } + return nil +} + +func (d *DirDumper) file(path string) (*os.File, error) { + f, ok := d.openFiles[path] + if ok { + return f, nil + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("failed to create directory %q: %w", dir, err) + } + f, err := os.Create(path) + if err != nil { + return nil, fmt.Errorf("failed to create file %q: %w", path, err) + } + d.openFiles[path] = f + return f, nil +} diff --git a/internal/pkg/dumper/dir_test.go b/internal/pkg/dumper/dir_test.go new file mode 100644 index 0000000..1d715bb --- /dev/null +++ b/internal/pkg/dumper/dir_test.go @@ -0,0 +1,165 @@ +package dumper_test + +import ( + "bytes" + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper" +) + +func Test_DirDumper(t *testing.T) { + tdir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + defer os.RemoveAll(tdir) + + subject, err := dumper.NewDirDumper(tdir) + require.NoError(t, err) + + uls := []*unstructured.UnstructuredList{ + { + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": map[string]interface{}{ + "name": "test-pod", + "namespace": "test-ns", + }, + }, + }, + { + Object: map[string]interface{}{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": map[string]interface{}{ + "name": "test-pod", + "namespace": "test-ns-2", + }, + }, + }, + }, + }, + { + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": map[string]interface{}{ + "name": "test-pod-2", + "namespace": "test-ns", + }, + }, + }, + }, + }, + { + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "Service", + "apiVersion": "v1", + "metadata": map[string]interface{}{ + "name": "test-svc", + "namespace": "test-ns", + }, + }, + }, + }, + }, + { + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "ClusterRole", + "apiVersion": "rbac.authorization.k8s.io/v1", + "metadata": map[string]interface{}{ + "name": "cluster-scoped", + }, + }, + }, + }, + }, + } + + for i, ul := range uls { + require.NoErrorf(t, subject.Dump(ul), "failed to dump list %d", i) + } + defer func() { + require.NoError(t, subject.Close()) + }() + + require.FileExists(t, tdir+"/objects-Pod.json") + requireFileContains(t, tdir+"/objects-Pod.json", []ExpectedObject{ + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns"}, + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"}, + {Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"}, + }) + require.FileExists(t, tdir+"/objects-Service.json") + requireFileContains(t, tdir+"/objects-Service.json", []ExpectedObject{ + {Kind: "Service", Name: "test-svc", Namespace: "test-ns"}, + }) + require.FileExists(t, tdir+"/objects-ClusterRole.rbac.authorization.k8s.io.json") + requireFileContains(t, tdir+"/objects-ClusterRole.rbac.authorization.k8s.io.json", []ExpectedObject{ + {Kind: "ClusterRole", Name: "cluster-scoped", Namespace: ""}, + }) + require.FileExists(t, tdir+"/split/test-ns/__all__.json") + requireFileContains(t, tdir+"/split/test-ns/__all__.json", []ExpectedObject{ + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns"}, + {Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"}, + {Kind: "Service", Name: "test-svc", Namespace: "test-ns"}, + }) + require.FileExists(t, tdir+"/split/test-ns/Pod.json") + requireFileContains(t, tdir+"/split/test-ns/Pod.json", []ExpectedObject{ + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns"}, + {Kind: "Pod", Name: "test-pod-2", Namespace: "test-ns"}, + }) + require.FileExists(t, tdir+"/split/test-ns/Service.json") + requireFileContains(t, tdir+"/split/test-ns/Service.json", []ExpectedObject{ + {Kind: "Service", Name: "test-svc", Namespace: "test-ns"}, + }) + require.FileExists(t, tdir+"/split/test-ns-2/__all__.json") + requireFileContains(t, tdir+"/split/test-ns-2/__all__.json", []ExpectedObject{ + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"}, + }) + require.FileExists(t, tdir+"/split/test-ns-2/Pod.json") + requireFileContains(t, tdir+"/split/test-ns-2/Pod.json", []ExpectedObject{ + {Kind: "Pod", Name: "test-pod", Namespace: "test-ns-2"}, + }) +} + +type ExpectedObject struct { + Kind, Name, Namespace string +} + +func requireFileContains(t *testing.T, path string, expected []ExpectedObject) { + t.Helper() + raw, err := os.ReadFile(path) + require.NoError(t, err) + + var objs []unstructured.Unstructured + raw = bytes.TrimSuffix(raw, []byte("\n")) + rawObjs := bytes.Split(raw, []byte("\n")) + for _, rawObj := range rawObjs { + var obj unstructured.Unstructured + require.NoError(t, json.Unmarshal(rawObj, &obj)) + objs = append(objs, obj) + } + + actualObjects := make([]ExpectedObject, 0, len(objs)) + for _, obj := range objs { + actualObjects = append(actualObjects, ExpectedObject{ + Kind: obj.GetKind(), + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }) + } + + require.ElementsMatch(t, expected, actualObjects) +} diff --git a/internal/pkg/dumper/dumper.go b/internal/pkg/dumper/dumper.go new file mode 100644 index 0000000..3bc9365 --- /dev/null +++ b/internal/pkg/dumper/dumper.go @@ -0,0 +1,19 @@ +// Dumper provides means to dump a UnstructuredList returned by a dynamic client. +package dumper + +import ( + "encoding/json" + "io" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Dumper is an interface for dumping a list of unstructured objects +type DumperFunc func(*unstructured.UnstructuredList) error + +// DumpToWriter dumps the list of unstructured objects to the provided writer as JSON +func DumpToWriter(w io.Writer) DumperFunc { + return func(l *unstructured.UnstructuredList) error { + return json.NewEncoder(w).Encode(l) + } +} diff --git a/internal/pkg/dumper/dumper_test.go b/internal/pkg/dumper/dumper_test.go new file mode 100644 index 0000000..bc0793b --- /dev/null +++ b/internal/pkg/dumper/dumper_test.go @@ -0,0 +1,43 @@ +package dumper_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func Test_DumpToWriter(t *testing.T) { + var b bytes.Buffer + + subject := dumper.DumpToWriter(&b) + + require.NoError(t, + subject(&unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "kind": "List", + }, + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "kind": "Pod", + "apiVersion": "v1", + "metadata": map[string]interface{}{ + "name": "test-pod", + "namespace": "test-ns", + }, + }, + }, + }, + }), + ) + + var got unstructured.UnstructuredList + require.NoError(t, json.NewDecoder(&b).Decode(&got)) + + require.Len(t, got.Items, 1) + require.Equal(t, "Pod", got.Items[0].GetKind()) +} diff --git a/known-to-fail b/known-to-fail deleted file mode 100644 index 39c5d3f..0000000 --- a/known-to-fail +++ /dev/null @@ -1,13 +0,0 @@ -.+reviews -.+validators -.+mutators -bindings -deploymentconfigrollbacks -imagesignatures -imagestreamimages -imagestreamimports -imagestreammappings -mutations -projectrequests -useridentitymappings -validations diff --git a/main.go b/main.go new file mode 100644 index 0000000..0c6079e --- /dev/null +++ b/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/projectsyn/k8s-object-dumper/internal/pkg/discovery" + "github.com/projectsyn/k8s-object-dumper/internal/pkg/dumper" +) + +func main() { + var dir string + var batchSize int64 + mustExistResources := new(repeatableStringFlag) + ignoreResources := new(repeatableRegexpFlag) + + flag.StringVar(&dir, "dir", "", "Directory to dump objects into") + flag.Int64Var(&batchSize, "batch-size", 500, "Batch size for listing objects") + flag.Var(mustExistResources, "must-exist", "Resource that must exist in the cluster. Can be used multiple times.") + flag.Var(ignoreResources, "ignore", "Resource to ignore during discovery. Regexp, anchored by default. Can be used multiple times.") + + flag.Parse() + + df := dumper.DumpToWriter(os.Stdout) + if dir != "" { + if err := os.MkdirAll(dir, 0755); err != nil { + fmt.Fprintf(os.Stderr, "failed to create directory %s: %v\n", dir, err) + os.Exit(1) + } + d, err := dumper.NewDirDumper(dir) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create directory dumper: %v\n", err) + os.Exit(1) + } + defer d.Close() + df = d.Dump + } + + conf, err := ctrl.GetConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to get Kubernetes config: %v", err) + } + + if err := discovery.DiscoverObjects(context.Background(), conf, df, discovery.DiscoveryOptions{ + BatchSize: batchSize, + LogWriter: os.Stderr, + MustExistResources: *mustExistResources, + IgnoreResources: *ignoreResources, + }); err != nil { + fmt.Fprintf(os.Stderr, "failed to dump some or all objects: %+v\n", err) + os.Exit(1) + } +} + +type repeatableStringFlag []string + +func (i *repeatableStringFlag) String() string { + return fmt.Sprintf("%v", *i) +} + +func (i *repeatableStringFlag) Set(value string) error { + *i = append(*i, value) + return nil +} + +type repeatableRegexpFlag []*regexp.Regexp + +func (i *repeatableRegexpFlag) String() string { + return fmt.Sprintf("%v", *i) +} + +func (i *repeatableRegexpFlag) Set(value string) error { + value = fmt.Sprintf("^%s$", value) + r, err := regexp.Compile(value) + if err != nil { + return fmt.Errorf("failed to compile regexp %q: %w", value, err) + } + *i = append(*i, r) + return nil +} diff --git a/must-exist b/must-exist deleted file mode 100644 index a986ddd..0000000 --- a/must-exist +++ /dev/null @@ -1,19 +0,0 @@ -configmaps -daemonsets -deployments -endpoints -horizontalpodautoscalers -ingresses -jobs -limitranges -namespaces -nodes -persistentvolumeclaims -persistentvolumes -replicasets -resourcequotas -roles -secrets -serviceaccounts -services -statefulsets diff --git a/renovate.json b/renovate.json index b44ed76..5db72dd 100644 --- a/renovate.json +++ b/renovate.json @@ -1,8 +1,6 @@ { + "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:base" - ], - "labels": [ - "dependency" + "config:recommended" ] } From f52a602d3005f4ead5e3d86bab144aa629c1f7cf Mon Sep 17 00:00:00 2001 From: Sebastian Widmer Date: Tue, 12 Nov 2024 13:29:57 +0100 Subject: [PATCH 2/2] Rename batch to chunk to have same terminology as in kubectl --- internal/pkg/discovery/discovery.go | 18 +++++++++--------- internal/pkg/discovery/discovery_test.go | 2 +- main.go | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/pkg/discovery/discovery.go b/internal/pkg/discovery/discovery.go index f97f649..cf1086e 100644 --- a/internal/pkg/discovery/discovery.go +++ b/internal/pkg/discovery/discovery.go @@ -19,7 +19,7 @@ import ( ) type DiscoveryOptions struct { - BatchSize int64 + ChunkSize int64 LogWriter io.Writer // MustExistResources is a list of resources that must exist in the cluster. @@ -32,15 +32,15 @@ type DiscoveryOptions struct { IgnoreResources []*regexp.Regexp } -// GetBatchSize returns the set batch size for listing objects or the default. -func (opts DiscoveryOptions) GetBatchSize() int64 { - if opts.BatchSize == 0 { +// GetChunkSize returns the set chunk size for listing objects or the default. +func (opts DiscoveryOptions) GetChunkSize() int64 { + if opts.ChunkSize == 0 { return 500 } - return opts.BatchSize + return opts.ChunkSize } -// GetLogWriter returns the set batch size for listing objects or io.Discard as default. +// GetLogWriter returns the set chunk size for listing objects or io.Discard as default. func (opts DiscoveryOptions) GetLogWriter() io.Writer { if opts.LogWriter == nil { return io.Discard @@ -51,9 +51,9 @@ func (opts DiscoveryOptions) GetLogWriter() io.Writer { // DiscoverObjects discovers all objects in the cluster and calls the provided callback for each list of objects. // The callback can be called multiple times with the same object kind. // Objects are unique in general, but the callback should be able to handle duplicates. -// Some API servers do not implement list batching correctly and thus might introduce duplicates. +// Some API servers do not implement list chunking correctly and thus might introduce duplicates. func DiscoverObjects(ctx context.Context, conf *rest.Config, cb func(*unstructured.UnstructuredList) error, opts DiscoveryOptions) error { - batchSize := opts.GetBatchSize() + chunkSize := opts.GetChunkSize() logWriter := opts.GetLogWriter() dc, err := discovery.NewDiscoveryClientForConfig(conf) @@ -112,7 +112,7 @@ func DiscoverObjects(ctx context.Context, conf *rest.Config, cb func(*unstructur continueKey := "" for { l, err := dynClient.Resource(res).List(ctx, metav1.ListOptions{ - Limit: batchSize, + Limit: chunkSize, Continue: continueKey, }) if err != nil { diff --git a/internal/pkg/discovery/discovery_test.go b/internal/pkg/discovery/discovery_test.go index 04fd4b7..311c0ce 100644 --- a/internal/pkg/discovery/discovery_test.go +++ b/internal/pkg/discovery/discovery_test.go @@ -67,7 +67,7 @@ func Test_DiscoverObjects(t *testing.T) { } require.NoError(t, discovery.DiscoverObjects(context.Background(), cfg, objTracker, discovery.DiscoveryOptions{ - BatchSize: int64(cap(sas) / 2), + ChunkSize: int64(cap(sas) / 2), IgnoreResources: []*regexp.Regexp{ regexp.MustCompile(`^roles.rbac.authorization.k8s.io$`), }, diff --git a/main.go b/main.go index 0c6079e..abe2096 100644 --- a/main.go +++ b/main.go @@ -15,12 +15,12 @@ import ( func main() { var dir string - var batchSize int64 + var chunkSize int64 mustExistResources := new(repeatableStringFlag) ignoreResources := new(repeatableRegexpFlag) flag.StringVar(&dir, "dir", "", "Directory to dump objects into") - flag.Int64Var(&batchSize, "batch-size", 500, "Batch size for listing objects") + flag.Int64Var(&chunkSize, "chunk-size", 500, "Chunk size for listing objects") flag.Var(mustExistResources, "must-exist", "Resource that must exist in the cluster. Can be used multiple times.") flag.Var(ignoreResources, "ignore", "Resource to ignore during discovery. Regexp, anchored by default. Can be used multiple times.") @@ -47,7 +47,7 @@ func main() { } if err := discovery.DiscoverObjects(context.Background(), conf, df, discovery.DiscoveryOptions{ - BatchSize: batchSize, + ChunkSize: chunkSize, LogWriter: os.Stderr, MustExistResources: *mustExistResources, IgnoreResources: *ignoreResources,