From 05dc3d010e1a925baf930b3f16421f4ef5f83825 Mon Sep 17 00:00:00 2001 From: "Kumar Singh, Aman [Engineering]" Date: Tue, 23 Apr 2024 10:17:16 +0000 Subject: [PATCH] Setup workflow to build and publish docker image with source tarball oss: Add links to the code and license of where this was built from --- .github/workflows/build_publish.yml | 31 +++++ .gitignore | 3 + Dockerfile.ubi8 | 124 +++++++++++++++++++ Makefile | 24 +++- build.sh | 26 ++++ packages/grafana-data/src/types/config.ts | 1 + pkg/api/dtos/frontend_settings.go | 1 + pkg/api/frontendsettings.go | 1 + pkg/services/licensing/licensingtest/fake.go | 5 + pkg/services/licensing/models.go | 3 + pkg/services/licensing/oss.go | 10 +- public/app/core/components/Footer/Footer.tsx | 7 ++ public/app/core/components/NavBar/utils.ts | 7 ++ 13 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build_publish.yml create mode 100644 Dockerfile.ubi8 create mode 100755 build.sh diff --git a/.github/workflows/build_publish.yml b/.github/workflows/build_publish.yml new file mode 100644 index 0000000000000..98400e605e151 --- /dev/null +++ b/.github/workflows/build_publish.yml @@ -0,0 +1,31 @@ +name: Create and publish a Docker image + +on: [push] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build the container + run: make push_from_dist + shell: bash \ No newline at end of file diff --git a/.gitignore b/.gitignore index d08eafbeaeb7b..2b89c624b7004 100644 --- a/.gitignore +++ b/.gitignore @@ -194,3 +194,6 @@ deployment_tools_config.json # Temporary file for backporting PRs .pr-body.txt + +build_dir/ +grafana-dist/ \ No newline at end of file diff --git a/Dockerfile.ubi8 b/Dockerfile.ubi8 new file mode 100644 index 0000000000000..ff8714a602965 --- /dev/null +++ b/Dockerfile.ubi8 @@ -0,0 +1,124 @@ +# syntax=docker/dockerfile:1 + +ARG GO_SRC=go-builder +ARG JS_SRC=js-builder + +FROM registry.access.redhat.com/ubi8/nodejs-18:1-81 as js-builder + +ENV NODE_OPTIONS=--max_old_space_size=8000 + +USER root +WORKDIR /grafana + +COPY package.json yarn.lock .yarnrc.yml ./ +COPY .yarn .yarn +COPY packages packages +COPY plugins-bundled plugins-bundled +COPY public public + +RUN npm install -g yarn@1.22.11 +RUN yarn install --immutable + +COPY tsconfig.json .eslintrc .editorconfig .browserslistrc .prettierrc.js babel.config.json ./ +COPY public public +COPY scripts scripts +COPY emails emails + +ENV NODE_ENV production +RUN yarn build + +FROM registry.access.redhat.com/ubi8/go-toolset:1.20.10-3 as go-builder + +ARG COMMIT_SHA="" +ARG BUILD_BRANCH="" +ARG GO_BUILD_TAGS="oss" +ARG WIRE_TAGS="oss" +ARG BINGO="true" + +# Install build dependencies +RUN if grep -i -q alpine /etc/issue; then \ + apk add --no-cache gcc g++ make git; \ + fi + +USER root +WORKDIR /grafana + +COPY go.* ./ +COPY .bingo .bingo + +RUN go mod download + +COPY embed.go Makefile build.go package.json ./ +COPY cue.mod cue.mod +COPY kinds kinds +COPY local local +COPY packages/grafana-schema packages/grafana-schema +COPY public/app/plugins public/app/plugins +COPY public/api-merged.json public/api-merged.json +COPY pkg pkg +COPY scripts scripts +COPY conf conf +COPY .github .github +COPY LICENSE ./ + +ENV COMMIT_SHA=${COMMIT_SHA} +ENV BUILD_BRANCH=${BUILD_BRANCH} + +RUN go mod verify +RUN make build-go GO_BUILD_TAGS=${GO_BUILD_TAGS} WIRE_TAGS=${WIRE_TAGS} + +# Final stage +# helpers for COPY --from +FROM ${GO_SRC} as go-src +FROM ${JS_SRC} as js-src + +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6-854 + +LABEL maintainer="Grafana team " + +ARG GF_UID="472" +ARG GF_GID="0" + +ENV PATH="/usr/share/grafana/bin:$PATH" \ + GF_PATHS_CONFIG="/etc/grafana/grafana.ini" \ + GF_PATHS_DATA="/var/lib/grafana" \ + GF_PATHS_HOME="/usr/share/grafana" \ + GF_PATHS_LOGS="/var/log/grafana" \ + GF_PATHS_PLUGINS="/var/lib/grafana/plugins" \ + GF_PATHS_PROVISIONING="/etc/grafana/provisioning" + +WORKDIR $GF_PATHS_HOME +USER root + +RUN microdnf install shadow-utils + +COPY conf ./conf + +RUN export GF_GID_NAME=$(getent group $GF_GID | cut -d':' -f1) && \ + mkdir -p "$GF_PATHS_HOME/.aws" && \ + useradd --system -u $GF_UID -G "$GF_GID_NAME" grafana && \ + mkdir -p "$GF_PATHS_PROVISIONING/datasources" \ + "$GF_PATHS_PROVISIONING/dashboards" \ + "$GF_PATHS_PROVISIONING/notifiers" \ + "$GF_PATHS_PROVISIONING/plugins" \ + "$GF_PATHS_PROVISIONING/access-control" \ + "$GF_PATHS_PROVISIONING/alerting" \ + "$GF_PATHS_LOGS" \ + "$GF_PATHS_PLUGINS" \ + "$GF_PATHS_DATA" && \ + cp "$GF_PATHS_HOME/conf/sample.ini" "$GF_PATHS_CONFIG" && \ + cp "$GF_PATHS_HOME/conf/ldap.toml" /etc/grafana/ldap.toml && \ + chown -R "grafana:$GF_GID_NAME" "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" "$GF_PATHS_PROVISIONING" && \ + chmod -R 777 "$GF_PATHS_DATA" "$GF_PATHS_HOME/.aws" "$GF_PATHS_LOGS" "$GF_PATHS_PLUGINS" "$GF_PATHS_PROVISIONING" + +COPY --from=go-src /grafana/bin/grafana* /grafana/bin/*/grafana* ./bin/ +COPY --from=js-src /grafana/public ./public +COPY --from=go-src /grafana/LICENSE ./ +COPY public/build/static/source*.tar.gz ./public/build/static/ + +EXPOSE 3000 + +COPY ./packaging/docker/run.sh /run.sh + +USER grafana +ENTRYPOINT [ "/run.sh" ] \ No newline at end of file diff --git a/Makefile b/Makefile index 3a41dfd325495..a32d9e48b3989 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ WIRE_TAGS = "oss" -include local/Makefile include .bingo/Variables.mk -.PHONY: all deps-go deps-js deps build-go build-backend build-server build-cli build-js build build-docker-full build-docker-full-ubuntu lint-go golangci-lint test-go test-js gen-ts test run run-frontend clean devenv devenv-down protobuf drone help gen-go gen-cue fix-cue +.PHONY: all deps-go deps-js deps build-go build-backend build-server build-cli build-js build build-docker-full build-docker-full-ubuntu lint-go golangci-lint test-go test-js gen-ts test run run-frontend clean devenv devenv-down protobuf drone help gen-go gen-cue fix-cue dist build_from_dist push_from_dist GO = go GO_FILES ?= ./pkg/... @@ -110,6 +110,7 @@ gen-cue: ## Do all CUE/Thema code generation gen-go: $(WIRE) @echo "generate go files" + @export WIRE_VERBOSE=true $(WIRE) gen -tags $(WIRE_TAGS) ./pkg/server fix-cue: $(CUE) @@ -322,3 +323,24 @@ format-drone: help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +# AGPL helpers +# Show all edits since an upstream commit. +BASE_COMMIT=f4c5a603b25f69127bfe065381ff454e55b334c5 +dist: + rm -rf grafana-dist + git archive --format=tar --prefix=grafana-dist/ $(BASE_COMMIT) | tar -xf - + mkdir grafana-dist/patches + git format-patch -o grafana-dist/patches/ $(BASE_COMMIT) + # Set a default version in our build. + sed -ibak s,unknown-dev,$(shell git rev-parse HEAD), grafana-dist/pkg/build/git.go + + # And produce the output. + mkdir -p public/build/static + tar -czf public/build/static/source_$(shell git rev-parse HEAD).tar.gz grafana-dist/ + +build_from_dist: dist + ./build.sh public/build/static/source_$(shell git rev-parse HEAD).tar.gz $(shell git rev-parse HEAD) + +push_from_dist: build_from_dist + docker push ghcr.io/goldmansachs/grafana:9.5.18-ubi8-$(shell git rev-parse HEAD) \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000000000..4d93d1a7aa3f4 --- /dev/null +++ b/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Builds the container image from the source tarball. + +set -e +set -o pipefail + +INPUT="$(realpath $1)" +COMMIT="$2" + +echo "Attempting to build ${INPUT}" +mkdir build_dir +tar -C build_dir -xf "${INPUT}" + +# My patch failed with out of memory. Let's use git instead. +cd build_dir/grafana-dist +git init +git config user.email "you@example.com" +git config user.name "Your Name" +git add -A +git commit -m "baseline version" +git am patches/*.patch + + +mkdir -p public/build/static +cp "${INPUT}" public/build/static +docker build --tag ghcr.io/goldmansachs/grafana:9.5.18-ubi8-"${COMMIT}" -f ./Dockerfile.ubi8 . \ No newline at end of file diff --git a/packages/grafana-data/src/types/config.ts b/packages/grafana-data/src/types/config.ts index 1db14beb3021b..4b01d3c326b44 100644 --- a/packages/grafana-data/src/types/config.ts +++ b/packages/grafana-data/src/types/config.ts @@ -40,6 +40,7 @@ export enum GrafanaEdition { export interface LicenseInfo { expiry: number; licenseUrl: string; + sourceUrl: string; stateInfo: string; edition: GrafanaEdition; enabledFeatures: { [key: string]: boolean }; diff --git a/pkg/api/dtos/frontend_settings.go b/pkg/api/dtos/frontend_settings.go index 9d960a3b58241..24722d51e1eb2 100644 --- a/pkg/api/dtos/frontend_settings.go +++ b/pkg/api/dtos/frontend_settings.go @@ -35,6 +35,7 @@ type FrontendSettingsLicenseInfoDTO struct { Expiry int64 `json:"expiry"` StateInfo string `json:"stateInfo"` LicenseUrl string `json:"licenseUrl"` + SourceUrl string `json:"sourceUrl"` Edition string `json:"edition"` EnabledFeatures map[string]bool `json:"enabledFeatures"` diff --git a/pkg/api/frontendsettings.go b/pkg/api/frontendsettings.go index 5bfdef5b5f2d1..c529114afc553 100644 --- a/pkg/api/frontendsettings.go +++ b/pkg/api/frontendsettings.go @@ -169,6 +169,7 @@ func (hs *HTTPServer) getFrontendSettings(c *contextmodel.ReqContext) (*dtos.Fro Expiry: hs.License.Expiry(), StateInfo: hs.License.StateInfo(), LicenseUrl: hs.License.LicenseURL(hasAccess(accesscontrol.ReqGrafanaAdmin, licensing.PageAccess)), + SourceUrl: hs.License.SourceURL(), Edition: hs.License.Edition(), EnabledFeatures: hs.License.EnabledFeatures(), }, diff --git a/pkg/services/licensing/licensingtest/fake.go b/pkg/services/licensing/licensingtest/fake.go index 7068217abea68..66d14ed531c75 100644 --- a/pkg/services/licensing/licensingtest/fake.go +++ b/pkg/services/licensing/licensingtest/fake.go @@ -36,6 +36,11 @@ func (f *FakeLicensing) LicenseURL(showAdminLicensingPage bool) string { return mockedArgs.Get(0).(string) } +func (f *FakeLicensing) SourceURL() string { + mockedArgs := f.Called() + return mockedArgs.Get(0).(string) +} + func (f *FakeLicensing) StateInfo() string { mockedArgs := f.Called() return mockedArgs.Get(0).(string) diff --git a/pkg/services/licensing/models.go b/pkg/services/licensing/models.go index 4d2ad518f6bd7..66f3a78091686 100644 --- a/pkg/services/licensing/models.go +++ b/pkg/services/licensing/models.go @@ -12,6 +12,9 @@ type Licensing interface { LicenseURL(showAdminLicensingPage bool) string + // A link to the complete corresponding source + SourceURL() string + StateInfo() string EnabledFeatures() map[string]bool diff --git a/pkg/services/licensing/oss.go b/pkg/services/licensing/oss.go index 8139d63542d37..006f64ea113b1 100644 --- a/pkg/services/licensing/oss.go +++ b/pkg/services/licensing/oss.go @@ -1,6 +1,8 @@ package licensing import ( + "fmt" + "github.com/grafana/grafana/pkg/api/dtos" contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model" "github.com/grafana/grafana/pkg/services/hooks" @@ -34,11 +36,11 @@ func (*OSSLicensingService) ContentDeliveryPrefix() string { } func (l *OSSLicensingService) LicenseURL(showAdminLicensingPage bool) string { - if showAdminLicensingPage { - return l.Cfg.AppSubURL + "/admin/upgrading" - } + return fmt.Sprintf("https://github.com/goldmansachs/grafana/blob/%v/LICENSING.md", setting.BuildCommit) +} - return "https://grafana.com/oss/grafana?utm_source=grafana_footer" +func (l *OSSLicensingService) SourceURL() string { + return fmt.Sprintf("/public/build/static/source_%v.tar.gz", setting.BuildCommit) } func (*OSSLicensingService) EnabledFeatures() map[string]bool { diff --git a/public/app/core/components/Footer/Footer.tsx b/public/app/core/components/Footer/Footer.tsx index 3ebe9e3b73382..f8760231e29fd 100644 --- a/public/app/core/components/Footer/Footer.tsx +++ b/public/app/core/components/Footer/Footer.tsx @@ -73,6 +73,13 @@ export function getVersionLinks(): FooterLink[] { url: hasReleaseNotes ? `https://github.com/grafana/grafana/blob/main/CHANGELOG.md` : undefined, }); + links.push({ + target: '_blank', + id: 'version', + text: `v${buildInfo.version} (${buildInfo.commit})`, + url: licenseInfo.sourceUrl + }); + if (buildInfo.hasUpdate) { links.push({ target: '_blank', diff --git a/public/app/core/components/NavBar/utils.ts b/public/app/core/components/NavBar/utils.ts index 92cc50e577e9c..cd27cebc49156 100644 --- a/public/app/core/components/NavBar/utils.ts +++ b/public/app/core/components/NavBar/utils.ts @@ -183,6 +183,13 @@ export function getEditionAndUpdateLinks(): NavModelItem[] { text: `${buildInfo.edition}${stateInfo}`, url: licenseInfo.licenseUrl, icon: 'external-link-alt', + }, + { + id: 'source', + text: 'Grafana Source Code', + icon: 'license', + url: licenseInfo.sourceUrl, + target: '_blank', }); if (buildInfo.hasUpdate) {