From 15a43d253362156b7ba067e36212b0bae18e2179 Mon Sep 17 00:00:00 2001 From: Vadim Bauer Date: Tue, 29 Oct 2024 19:23:57 +0100 Subject: [PATCH] Refactor the Pipeline (#228) Refactor main, pr and release workflow, the Pipeline, Dagger. --------- Signed-off-by: Vadim Bauer --- .github/actions/publish-and-sign/action.yaml | 54 +++++ .github/dependabot.yml | 2 +- .github/workflows/default.yaml | 63 +++-- .github/workflows/release.yaml | 69 ------ .goreleaser.yaml | 24 +- dagger/main.go | 230 +++++++++++-------- 6 files changed, 248 insertions(+), 194 deletions(-) create mode 100644 .github/actions/publish-and-sign/action.yaml delete mode 100644 .github/workflows/release.yaml diff --git a/.github/actions/publish-and-sign/action.yaml b/.github/actions/publish-and-sign/action.yaml new file mode 100644 index 00000000..6422743d --- /dev/null +++ b/.github/actions/publish-and-sign/action.yaml @@ -0,0 +1,54 @@ +name: Publish and Sign Snapshot Image +description: Publishes and signs a snapshot image using Dagger. + +inputs: + IMAGE_TAGS: + description: 'Tags for the image, e.g. "latest, v1.0.0"' + required: true + GITHUB_TOKEN: + description: 'GitHub token' + required: true + REGISTRY_PASSWORD: + description: 'Registry password' + required: true + REGISTRY_ADDRESS: + description: 'Registry address' + required: true + REGISTRY_USERNAME: + description: 'Registry username' + required: true + +runs: + using: "composite" + steps: + - name: Dagger Version + uses: sagikazarmark/dagger-version-action@v0.0.1 + + - name: Install Cosign + uses: sigstore/cosign-installer@v3.7.0 + + - name: Check Env Variables + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} + run: cosign env + + - name: Publish and Sign Snapshot Image + uses: dagger/dagger-for-github@v6 + env: + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} + REGISTRY_ADDRESS: ${{ inputs.REGISTRY_ADDRESS }} + REGISTRY_USERNAME: ${{ inputs.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ inputs.REGISTRY_PASSWORD }} + IMAGE_TAGS: ${{ inputs.IMAGE_TAGS }} + with: + version: ${{ steps.dagger_version.outputs.version }} + verb: call + args: "publish-image-and-sign \ + --registry='${{ env.REGISTRY_ADDRESS }}' \ + --registry-username='${{ env.REGISTRY_USERNAME }}' \ + --registry-password=env:REGISTRY_PASSWORD \ + --image-tags='${{ env.IMAGE_TAGS}}' \ + --github-token=env:GITHUB_TOKEN \ + --actions-id-token-request-url=$ACTIONS_ID_TOKEN_REQUEST_URL \ + --actions-id-token-request-token=env:ACTIONS_ID_TOKEN_REQUEST_TOKEN" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f56c9df6..d8adc852 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,4 +3,4 @@ updates: - package-ecosystem: gomod directory: "/" schedule: - interval: daily \ No newline at end of file + interval: weekly diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml index 2c5d172b..84e11a95 100644 --- a/.github/workflows/default.yaml +++ b/.github/workflows/default.yaml @@ -3,6 +3,8 @@ name: Main and Pull Request Pipeline on: push: branches: [main] + tags: + - "v*.*.*" pull_request: paths-ignore: - '*.md' @@ -31,7 +33,7 @@ jobs: - name: Run Reviewdog env: REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | + run: | reviewdog -f=sarif -name="Golang Linter Report" -reporter=github-check -filter-mode nofilter -fail-level any -tee < golangci-lint-report.sarif test-code: @@ -56,28 +58,63 @@ jobs: verb: call args: build-dev --platform linux/amd64 - push-snapshop-release: - permissions: - contents: write - packages: write - - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest + push-latest-images: + if: github.event.pull_request == null && !startsWith(github.ref, 'refs/tags/v') needs: - lint - test-code + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Dagger Version - uses: sagikazarmark/dagger-version-action@v0.0.1 + - name: Publish and Sign Snapshot Image + uses: ./.github/actions/publish-and-sign + with: + IMAGE_TAGS: latest + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + REGISTRY_ADDRESS: ${{ vars.REGISTRY_ADDRESS }} + REGISTRY_USERNAME: ${{ vars.REGISTRY_USERNAME }} + - - name: Push Release + publish-release: + if: startsWith(github.ref, 'refs/tags/v') + needs: + - lint + - test-code + - push-latest-images + permissions: + contents: write + packages: write + id-token: write + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Create Release uses: dagger/dagger-for-github@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - version: ${{ steps.dagger_version.outputs.version }} + version: "latest" verb: call - args: snapshot-release --github-token=${{ env.GITHUB_TOKEN }} + args: "release --github-token=env:GITHUB_TOKEN" + + - name: Publish and Sign Tagged Image + if: success() + uses: ./.github/actions/publish-and-sign + with: + IMAGE_TAGS: "latest, ${{ github.ref_name }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + REGISTRY_ADDRESS: ${{ vars.REGISTRY_ADDRESS }} + REGISTRY_USERNAME: ${{ vars.REGISTRY_USERNAME }} + diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index ad837985..00000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,69 +0,0 @@ -name: Release Artifacts and Container Images - -on: - push: - tags: - - "v*" - branches: [main] - -permissions: - contents: write - packages: write - -jobs: - publish-release: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Call Dagger Function - uses: dagger/dagger-for-github@v6 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - version: "latest" - verb: call - args: release --github-token='env:${{ env.GITHUB_TOKEN }}' - - publish-images: - runs-on: ubuntu-latest - environment: PROD - env: - COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} - COSIGN_KEY: ${{ secrets.COSIGN_KEY }} - REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} - REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} - REGISTRY_ADDRESS: ${{ vars.REGISTRY_ADDRESS }} - PUBLISH_ADDRESS: ${{ vars.PUBLISH_ADDRESS }} - TAG: ${{ github.ref_name }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Call Docker-Publish Function - uses: dagger/dagger-for-github@v6 - with: - version: "latest" - verb: call - args: "publish-image --cosign-password='env:${{ env.COSIGN_PASSWORD }}' --cosign-key='env:${{ env.COSIGN_KEY }}' --reg-username='${{ env.REGISTRY_USERNAME }}' --reg-password='env:${{ env.REGISTRY_PASSWORD }}' --reg-address='${{ env.REGISTRY_ADDRESS }}' --publish-address='${{ env.PUBLISH_ADDRESS }}' --tag='${{ env.TAG }}'" - - name: Notify on success - if: success() - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - body: "Container image published successfully! 🎉" - }) - - name: Notify on failure - if: failure() - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - body: "Failed to publish Container image. ❌" - }) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 91b7003a..ade3d18b 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,5 +1,5 @@ version: 2 -project_name: harbor-cli +project_name: harbor before: hooks: @@ -7,7 +7,6 @@ before: builds: - main: ./cmd/harbor/main.go - env: - CGO_ENABLED=0 ldflags: @@ -25,17 +24,18 @@ builds: - goos: windows goarch: arm64 mod_timestamp: "{{ .CommitTimestamp }}" + archives: - format: tar.gz format_overrides: - goos: windows format: zip nfpms: - - package_name: harbor + - homepage: https://github.com/goharbor/harbor-cli/ - maintainer: Vadim Bauer + maintainer: Harbor Community description: |- - [Sandbox] Official Harbor CLI + CLI for Harbor Container Registry formats: - rpm - deb @@ -49,13 +49,14 @@ checksum: name_template: 'checksums.txt' snapshot: - name_template: "HarborCLI Snapshot {{.Commit}}" - publish: true + version_template: "{{ incpatch .Version }}-next" release: name_template: "HarborCLI {{.Tag}}" - draft: false # Set to false to ensure that releases are published, not kept as drafts + draft: true # Set to false to ensure that releases are published, not kept as drafts prerelease: auto # Auto-detect prereleases based on tag + replace_existing_draft: true + replace_existing_artifacts: true disable: false # Ensure release publishing is enabled github: owner: goharbor # Your GitHub repository owner @@ -63,16 +64,13 @@ release: changelog: use: github - format: "{{.SHA}}: {{.Message}} (@{{.AuthorUsername}})" - sort: asc filters: exclude: - "^docs:" - "^test:" - "merge conflict" - groups: - - title: Dependency updates + - title: "Dependency updates" regexp: '^.*?(.+)\(deps\)!?:.+$' order: 300 - title: "New Features" @@ -90,5 +88,5 @@ changelog: - title: "Build process updates" regexp: ^.*?(build|ci)(\(.+\))??!?:.+$ order: 400 - - title: Other work + - title: "Other work" order: 9999 diff --git a/dagger/main.go b/dagger/main.go index ca5f6874..a99871ca 100644 --- a/dagger/main.go +++ b/dagger/main.go @@ -16,9 +16,9 @@ const ( ) func New( - // Local or remote directory with source code, defaults to "./" - // +optional - // +defaultPath="./" +// Local or remote directory with source code, defaults to "./" +// +optional +// +defaultPath="./" source *dagger.Directory, ) *HarborCli { return &HarborCli{Source: source} @@ -94,7 +94,7 @@ func (m *HarborCli) build( return builds } -// Run linter golangci-lint and write the linting results to a file golangci-lint-report.txt +// LintReport Executes the Linter and writes the linting results to a file golangci-lint-report.sarif func (m *HarborCli) LintReport(ctx context.Context) *dagger.File { report := "golangci-lint-report.sarif" return m.lint(ctx).WithExec([]string{"golangci-lint", "run", @@ -102,7 +102,7 @@ func (m *HarborCli) LintReport(ctx context.Context) *dagger.File { "--issues-exit-code", "0"}).File(report) } -// Run linter golangci-lint +// Lint Run the linter golangci-lint func (m *HarborCli) Lint(ctx context.Context) (string, error) { return m.lint(ctx).WithExec([]string{"golangci-lint", "run"}).Stderr(ctx) } @@ -118,115 +118,79 @@ func (m *HarborCli) lint(ctx context.Context) *dagger.Container { return linter } -// Create snapshot release with goreleaser -func (m *HarborCli) SnapshotRelease( - ctx context.Context, - githubToken *dagger.Secret, -) { - _, err := m. - goreleaserContainer(githubToken). - WithExec([]string{"release", "--snapshot", "--clean"}). - Stderr(ctx) - if err != nil { - log.Printf("❌ Error occured during snapshot release for the recently merged pull-request: %s", err) - return - } - log.Println("Pull-Request tasks completed successfully 🎉") -} - -// Create release with goreleaser -func (m *HarborCli) Release( - ctx context.Context, - // Github API token - githubToken *dagger.Secret, -) { - goreleaser := m.goreleaserContainer(githubToken). - WithExec([]string{"ls", "-la"}). - WithExec([]string{"goreleaser", "release", "--clean"}) - - _, err := goreleaser.Stderr(ctx) - if err != nil { - log.Printf("Error occured during release: %s", err) - return - } - log.Println("Release tasks completed successfully 🎉") -} - // PublishImage publishes a container image to a registry with a specific tag and signs it using Cosign. func (m *HarborCli) PublishImage( ctx context.Context, - cosignKey *dagger.Secret, - cosignPassword *dagger.Secret, - regUsername string, - regPassword *dagger.Secret, - regAddress string, - publishAddress string, - tag string, -) string { - var container *dagger.Container - var filteredBuilders []*dagger.Container - + registry, registryUsername string, +// +optional +// +default=["latest"] + imageTags []string, + registryPassword *dagger.Secret) []string { builders := m.build(ctx) - if len(builders) > 0 { - fmt.Println(len(builders)) - container = builders[0] - builders = builders[3:6] + releaseImages := []*dagger.Container{} + + for i, tag := range imageTags { + imageTags[i] = strings.TrimSpace(tag) + if strings.HasPrefix(imageTags[i], "v") { + imageTags[i] = strings.TrimPrefix(imageTags[i], "v") + } } - dir := dag.Directory() - dir = dir.WithDirectory(".", container.Directory(".")) - - // Create a minimal cli_runtime container - cli_runtime := dag.Container(). - From("alpine:latest"). - WithWorkdir("/root/"). - WithFile("/root/harbor", dir.File("./harbor")). - WithExec([]string{"ls"}). - WithExec([]string{"./harbor", "--help"}). - WithEntrypoint([]string{"./harbor"}) + fmt.Printf("provided tags: %s\n", imageTags) for _, builder := range builders { - if !(buildPlatform(ctx, builder) == "linux/amd64") { - filteredBuilders = append(filteredBuilders, builder) + os, _ := builder.EnvVariable(ctx, "GOOS") + arch, _ := builder.EnvVariable(ctx, "GOARCH") + + if os != "linux" { + continue } - } - publisher := cli_runtime.WithRegistryAuth(regAddress, regUsername, regPassword) - // Push the versioned tag - versionedAddress := fmt.Sprintf("%s:%s", publishAddress, tag) - addr, err := publisher.Publish(ctx, versionedAddress, dagger.ContainerPublishOpts{PlatformVariants: filteredBuilders}) - if err != nil { - panic(err) - } - // Push the latest tag - latestAddress := fmt.Sprintf("%s:latest", publishAddress) - addr, err = publisher.Publish(ctx, latestAddress) - if err != nil { - panic(err) + ctr := dag.Container(dagger.ContainerOpts{Platform: dagger.Platform(os + "/" + arch)}). + From("alpine:latest"). + WithFile("/harbor", builder.File("./harbor")). + WithEntrypoint([]string{"./harbor"}) + releaseImages = append(releaseImages, ctr) } - _, err = dag.Cosign().Sign(ctx, cosignKey, cosignPassword, []string{addr}, dagger.CosignSignOpts{RegistryUsername: regUsername, RegistryPassword: regPassword}) - if err != nil { - panic(err) + imageAddrs := []string{} + for _, imageTag := range imageTags { + addr, err := dag.Container().WithRegistryAuth(registry, registryUsername, registryPassword). + Publish(ctx, + fmt.Sprintf("%s/%s/harbor-cli:%s", registry, "harbor-cli", imageTag), + dagger.ContainerPublishOpts{PlatformVariants: releaseImages}, + ) + + if err != nil { + panic(err) + } + fmt.Printf("Published image address: %s\n", addr) + imageAddrs = append(imageAddrs, addr) } - fmt.Printf("Successfully published image to %s 🎉\n", addr) + return imageAddrs +} - return addr +// SnapshotRelease Create snapshot non OCI artifacts with goreleaser +func (m *HarborCli) SnapshotRelease(ctx context.Context) *dagger.Directory { + return m.goreleaserContainer(). + WithExec([]string{"goreleaser", "release", "--snapshot", "--clean", "--skip", "validate"}). + Directory("/src/dist") } -// Return the platform of the container -func buildPlatform(ctx context.Context, container *dagger.Container) string { - platform, err := container.Platform(ctx) +// Release Create release with goreleaser +func (m *HarborCli) Release(ctx context.Context, githubToken *dagger.Secret) { + goreleaser := m.goreleaserContainer(). + WithSecretVariable("GITHUB_TOKEN", githubToken). + WithExec([]string{"goreleaser", "release", "--clean"}) + _, err := goreleaser.Stderr(ctx) if err != nil { - log.Fatalf("error getting platform", err) + log.Printf("Error occured during release: %s", err) + return } - return string(platform) + log.Println("Release tasks completed successfully 🎉") } // Return a container with the goreleaser binary mounted and the source directory mounted. -func (m *HarborCli) goreleaserContainer( - // Github API token - githubToken *dagger.Secret, -) *dagger.Container { +func (m *HarborCli) goreleaserContainer() *dagger.Container { // Export the syft binary from the syft container as a file to generate SBOM syft := dag.Container(). From(fmt.Sprintf("anchore/syft:%s", SYFT_VERSION)). @@ -242,11 +206,11 @@ func (m *HarborCli) goreleaserContainer( WithFile("/bin/syft", syft). WithMountedDirectory("/src", m.Source). WithWorkdir("/src"). - WithEnvVariable("TINI_SUBREAPER", "true"). - WithSecretVariable("GITHUB_TOKEN", githubToken) + WithEnvVariable("TINI_SUBREAPER", "true") + } -// Generate CLI Documentation with doc.go and return the directory containing the generated files +// RunDoc Generate CLI Documentation with doc.go and return the directory containing the generated files func (m *HarborCli) RunDoc(ctx context.Context) *dagger.Directory { return dag.Container(). From("golang:"+GO_VERSION+"-alpine"). @@ -260,7 +224,7 @@ func (m *HarborCli) RunDoc(ctx context.Context) *dagger.Directory { WithWorkdir("/src").Directory("/src/doc") } -// Executes Go tests and returns the directory containing the test results +// Test Executes Go tests and returns the directory containing the test results func (m *HarborCli) Test(ctx context.Context) *dagger.Directory { return dag.Container(). From("golang:"+GO_VERSION+"-alpine"). @@ -282,3 +246,73 @@ func parsePlatform(platform string) (string, string, error) { } return parts[0], parts[1], nil } + +// PublishImageAndSign builds and publishes container images to a registry with a specific tags and then signs them using Cosign. +func (m *HarborCli) PublishImageAndSign( + ctx context.Context, + registry string, + registryUsername string, + registryPassword *dagger.Secret, + imageTags []string, +// +optional + githubToken *dagger.Secret, +// +optional + actionsIdTokenRequestToken *dagger.Secret, +// +optional + actionsIdTokenRequestUrl string, +) (string, error) { + + imageAddrs := m.PublishImage(ctx, registry, registryUsername, imageTags, registryPassword) + _, err := m.Sign( + ctx, + githubToken, + actionsIdTokenRequestUrl, + actionsIdTokenRequestToken, + registryUsername, + registryPassword, + imageAddrs[0], + ) + if err != nil { + return "", fmt.Errorf("failed to sign image: %w", err) + } + + fmt.Printf("Signed image: %s\n", imageAddrs) + return imageAddrs[0], nil +} + +// Sign signs a container image using Cosign, works also with GitHub Actions +func (m *HarborCli) Sign(ctx context.Context, +// +optional + githubToken *dagger.Secret, +// +optional + actionsIdTokenRequestUrl string, +// +optional + actionsIdTokenRequestToken *dagger.Secret, + registryUsername string, + registryPassword *dagger.Secret, + imageAddr string, +) (string, error) { + registryPasswordPlain, _ := registryPassword.Plaintext(ctx) + + cosing_ctr := dag.Container().From("cgr.dev/chainguard/cosign") + + // If githubToken is provided, use it to sign the image + if githubToken != nil { + if actionsIdTokenRequestUrl == "" || actionsIdTokenRequestToken == nil { + return "", fmt.Errorf("actionsIdTokenRequestUrl (exist=%s) and actionsIdTokenRequestToken (exist=%t) must be provided when githubToken is provided", actionsIdTokenRequestUrl, actionsIdTokenRequestToken != nil) + } + fmt.Printf("Setting the ENV Vars GITHUB_TOKEN, ACTIONS_ID_TOKEN_REQUEST_URL, ACTIONS_ID_TOKEN_REQUEST_TOKEN to sign with GitHub Token") + cosing_ctr = cosing_ctr.WithSecretVariable("GITHUB_TOKEN", githubToken). + WithEnvVariable("ACTIONS_ID_TOKEN_REQUEST_URL", actionsIdTokenRequestUrl). + WithSecretVariable("ACTIONS_ID_TOKEN_REQUEST_TOKEN", actionsIdTokenRequestToken) + } + + return cosing_ctr.WithSecretVariable("REGISTRY_PASSWORD", registryPassword). + WithExec([]string{"cosign", "env"}). + WithExec([]string{"cosign", "sign", "--yes", "--recursive", + "--registry-username", registryUsername, + "--registry-password", registryPasswordPlain, + imageAddr, + "--timeout", "1m", + }).Stdout(ctx) +}