From 1ef83f379544a5416ed777395c3d8a0a919ae8a2 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:20 +0000
Subject: [PATCH 001/127] PMM-12899 Use module and build cache

---
 .github/workflows/main.yml                   |   8 +-
 build/docker/client/Dockerfile               |  18 +-
 build/docker/server/Dockerfile.el9           |  14 +-
 build/docker/server/create_users.sh          |  29 ---
 build/local/README.md                        |  34 ++++
 build/local/build.sh                         | 110 ++++++++++++
 build/local/update.sh                        | 101 +++++++++++
 build/packages/rpm/server/SPECS/grafana.spec |   4 +-
 build/scripts/build-client-binary            |  70 +++++---
 build/scripts/build-client-docker            |  14 +-
 build/scripts/build-client-packages          | 104 +++++------
 build/scripts/build-client-rpm               |  16 +-
 build/scripts/build-client-source            |  39 +++--
 build/scripts/build-client-srpm              |  17 +-
 build/scripts/build-server-rpm               | 175 ++++++++++---------
 build/scripts/build-submodules               |  52 +++---
 build/scripts/vars                           |   6 +-
 17 files changed, 519 insertions(+), 292 deletions(-)
 delete mode 100644 build/docker/server/create_users.sh
 create mode 100644 build/local/README.md
 create mode 100755 build/local/build.sh
 create mode 100755 build/local/update.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9cb7d3927a..019b6bd5d8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -142,8 +142,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -160,8 +160,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile
index 2c03161268..6154ddfd08 100644
--- a/build/docker/client/Dockerfile
+++ b/build/docker/client/Dockerfile
@@ -5,19 +5,13 @@ RUN microdnf install shadow-utils jq -y
 RUN groupadd -g 1002 pmm-agent && \
     useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
             -d /usr/local/percona/pmm \
-            -c "PMM 2.X Client User" pmm-agent
+            -c "PMM 3.x Client User" pmm-agent
 
 FROM redhat/ubi9-micro
 
 ARG VERSION
 ARG BUILD_DATE
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-
 COPY LICENSE /licenses/
 COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
 COPY --from=users --chown=0:0 /etc/group /etc/group
@@ -32,10 +26,18 @@ RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 ADD pmm-client.tar.gz /tmp/
 
 RUN cd /tmp/pmm-client-* \
-    && env PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses Apache-2.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management Client
+LABEL org.opencontainers.image.vendor Percona
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
 ENV PATH /usr/local/percona/pmm/bin/:$PATH
diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 35dc8ac340..28dae9192e 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -8,12 +8,6 @@ ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 EXPOSE 8080 8443
 
 WORKDIR /opt
@@ -40,6 +34,14 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm-docker/update.yml && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml
 
+LABEL org.opencontainers.image.created ${BUILD_DATE}
+LABEL org.opencontainers.image.licenses AGPL-3.0
+LABEL org.opencontainers.image.title Percona Monitoring and Management
+LABEL org.opencontainers.image.vendor Percona LLC
+LABEL org.opencontainers.image.version ${VERSION}
+LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+
 USER pmm
 
 HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -sf http://127.0.0.1:8080/v1/readyz
diff --git a/build/docker/server/create_users.sh b/build/docker/server/create_users.sh
deleted file mode 100644
index e37eae3bf8..0000000000
--- a/build/docker/server/create_users.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-users=(
-  "pmm:1000:/bin/false:/home/pmm:pmm"
-  "nginx:1001:/sbin/nologin:/var/cache/nginx:nginx"
-)
-
-for user in "${users[@]}"; do
-  IFS=: read -r name uid shell home_dir group <<< "$user"
-  group_id="$uid"
-
-  # Check if user already exists
-  if id "$name" >/dev/null 2>&1; then
-    echo "User $name already exists"
-    continue
-  fi
-
-  # Create user with home directory if it doesn't exist
-  if [ ! -d "$home_dir" ]; then
-    mkdir -p "$home_dir"
-  fi
-
-  # Create user with specified UID, GID, and shell
-  groupadd -o -g "$group_id" "$group"
-  useradd -o -u "$uid" -g "$group" -G "$group" -s "$shell" -d "$home_dir" -c "$name" -m "$name"
-  chown "$uid:$group_id" "$home_dir"
-
-done
-
diff --git a/build/local/README.md b/build/local/README.md
new file mode 100644
index 0000000000..1517e9da3f
--- /dev/null
+++ b/build/local/README.md
@@ -0,0 +1,34 @@
+# Local builds
+
+This directory contains a set of scripts aimed at providing a simple way to build PMM locally.
+
+## Background
+
+Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
+
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+
+The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+
+## Prerequisites
+
+Below is a list of prerequisites that are required to build PMM locally.
+
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
+- Docker: 25.0.2+ (tested on 25.0.2)
+- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
+- make: 4.3+ (tested on 4.3)
+- bash (GNU): 5.1+ (tested on 5.1)
+- git: 2.34.1+ (tested on 2.34.1)
+- curl: 7.81.0+ (tested on 7.81.0)
+- yq: 4.42.0+ (tested on 4.42.1)
+- jq: 1.6+ (tested on 1.6)
+
+## Build Steps
+
+1. Install the prerequisites
+2. Clone the PMM repository
+3. Change to the `build/local` directory
+4. Run the `build.sh` script
diff --git a/build/local/build.sh b/build/local/build.sh
new file mode 100755
index 0000000000..f27ec569a8
--- /dev/null
+++ b/build/local/build.sh
@@ -0,0 +1,110 @@
+#!/bin/bash -e
+
+# Update submodules and PR branches
+/bin/bash $(dirname $0)/update.sh
+
+get_branch_name() {
+  local module="${1:-}"
+  local branch_name
+  local path
+
+  path=$(git config -f .gitmodules submodule.${module}.path)
+  cd "$path" || exit 1
+  branch_name=$(git branch --show-current)
+  cd - > /dev/null
+  echo $branch_name
+}
+
+# Define global variables
+pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+echo $pmm_commit > apiCommitSha
+pmm_branch=$(get_branch_name pmm)
+echo $pmm_branch > apiBranch
+pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+echo $pmm_url > apiURL
+pmm_qa_branch=$(get_branch_name pmm-qa)
+echo $pmm_qa_branch > pmmQABranch
+pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+echo $pmm_qa_commit > pmmQACommitSha
+pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+echo $pmm_ui_tests_branch > pmmUITestBranch
+pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+fb_commit_sha=$(git rev-parse HEAD)
+echo $fb_commit_sha > fbCommitSha
+
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# Important: the docker container's user need to be able to write to these directories.
+# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+# that the directories we create on the host are owned by a user with same uid and gid.
+
+# Create cache directories. 
+test -d "${root_dir}/go-path" || mkdir -p "go-path"
+test -d "${root_dir}/go-build" || mkdir -p "go-build"
+
+PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
+# Build client source: 4m39s from scratch, 0m27s using cache
+"$PATH_TO_SCRIPTS/build-client-source"
+
+# Build client binary: ??? from scratch, 0m20s using cache
+"$PATH_TO_SCRIPTS/build-client-binary"
+
+# Building client source rpm takes 13s (caching does not apply)
+"$PATH_TO_SCRIPTS/build-client-srpm"
+
+# Building client rpm takes 1m40s
+"$PATH_TO_SCRIPTS/build-client-rpm"
+
+# Building client docker image takes 17s
+GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+"$PATH_TO_SCRIPTS/build-client-docker"
+
+# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+# total time: 6m26s - build from scratch, no initial cache
+# total time: 2m49s - subsequent build, using cache from prior builds
+
+
+# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+# total time: 8m45s - build from scratch, no initial cache
+# total time: ??? - subsequent build, using cache from prior builds
+
+export RPM_EPOCH=1
+export RPMBUILD_DIST="el9"
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
+"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
+"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
+"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+# 3rd-party
+"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
+"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+
+export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
+export RPMBUILD_DIST=el9
+export DOCKERFILE=Dockerfile.el9.local
+${PATH_TO_SCRIPTS}/build-server-docker
+
+# Clean up temporary files
+rm -f apiBranch \
+  apiCommitSha \
+	apiURL \
+	fbCommitSha \
+	pmmQABranch \
+	pmmQACommitSha \
+	pmmUITestBranch \
+	pmmUITestsCommitSha
diff --git a/build/local/update.sh b/build/local/update.sh
new file mode 100755
index 0000000000..4922dd9bda
--- /dev/null
+++ b/build/local/update.sh
@@ -0,0 +1,101 @@
+#!/bin/bash -e
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+main() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
+main
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index bd16ba3121..85976d18a1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -17,8 +17,8 @@ Version:        %{grafana_version}
 Release:        %{rpm_release}
 Summary:        Grafana is an open source, feature rich metrics dashboard and graph editor
 License:        AGPLv3
-URL:            https://github.com/percona-platform/grafana
-Source0:        https://github.com/percona-platform/grafana/archive/%{commit}.tar.gz
+URL:            https://github.com/percona/grafana
+Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 82d88c645d..295a5a5b02 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -5,6 +5,9 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
+# Important: `bin_dir` and `binary_dir` are different directories. `bin_dir` defines the directory for binary files 
+# inside the container, while `binary_dir` defines the directory for binary files outside the container.
+
 extract_source_tarball() {
     if [ ! -d "${source_dir}" ]; then
         mkdir -p $(dirname ${source_dir})
@@ -18,58 +21,71 @@ gobuild_component() {
     local extract_path=${3:-"github.com/percona/$parent_component"}
     local component_path=$4
     local result_file=${5:-bin/$(basename ${component_path:-$extract_path})}
+    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
+    local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
+    local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
+    local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
+    local build_command
+
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
         result_file=${result_file/_metrics/}
     fi
 
     if [ -x "${binary_dir}/${result_file}" ]; then
-        echo skip build
+        echo Skipping the build...
         return
     fi
-    if [ ${component} = 'pmm-admin' -o ${component} = 'pmm-agent' ]; then
+    if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
         build_command="cd /tmp/go/src/${extract_path} && make build"
-    elif [ ${component} = 'node_exporter' ] || [ ${component} = 'mysqld_exporter' ] || \
-         [ ${component} = 'proxysql_exporter' ] || [ ${component} = 'rds_exporter' ] || \
-         [ ${component} = 'postgres_exporter' ] || [ ${component} = 'azure_metrics_exporter' ]; then
+    elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then
         build_command="cd /tmp/go/src/${extract_path} && make release"
     elif [ ${component} = 'percona-toolkit' ]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o \${binary_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="go build -o \${binary_dir}/${result_file} ./src/${extract_path}/${component_path}"
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
     fi
-    docker run --rm -v ${tmp_dir}:/home/builder/tmp ${rpmbuild_docker_image} sh -c "
+
+    docker run --rm \
+      ${volume_mounts} \
+      ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
         export GOOS=${GOOS:-linux}
         export GOARCH=${GOARCH:-amd64}
         export CGO_ENABLED=0
         export GO111MODULE=auto
+        export GOPATH=/tmp/go
+        export GOMODCACHE=/home/builder/go/pkg/mod
 
-        # for 'make release'
+        # Setting variable for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
-        export PMM_RELEASE_FULLCOMMIT=$(cd sources/${parent_component}/src/${extract_path} && git rev-parse HEAD)
-        export COMPONENT_VERSION=$(cd sources/${parent_component}/src/${extract_path} && git describe --abbrev=0 --always)
-        export COMPONENT_BRANCH=$(cd sources/${parent_component}/src/${extract_path} && git describe --always --contains --all)
         export PMM_RELEASE_BRANCH=""
         sudo chown -R builder:builder /home/builder/tmp
 
-        source_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
-        binary_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
-        export PMM_RELEASE_PATH=\${binary_dir}/bin
-
-        mkdir -p \$binary_dir/bin
+        export PMM_RELEASE_PATH=${bin_dir}/bin
+        if [ "${component}" = vmagent ]; then
+            export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
+        else
+            export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
+        fi
 
+        mkdir -p ${bin_dir}/bin
         mkdir -p /tmp/go/src/${extract_path}
-        archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz")
-        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf \${source_dir}/\$archive
-        pushd /tmp/go
-            export GOPATH=\$(pwd -P)
-            ${build_command}
-        popd
+
+        tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
+
+        ${build_command}
+
         sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
@@ -83,8 +99,7 @@ copy_component() {
 
     if [ ! -d "${component_dir}" ]; then
         mkdir -p ${component_dir}
-        tar \
-            -C ${component_dir} \
+        tar -C ${component_dir} \
             --strip-components=1 \
             -zxpf "$archive"
     fi
@@ -104,7 +119,6 @@ main() {
     gobuild_component "rds_exporter"
     gobuild_component "azure_metrics_exporter"
     gobuild_component "clickhouse_exporter" "" "github.com/Percona-Lab/clickhouse_exporter"
-
     gobuild_component "pmm-admin" "pmm" "" "admin"
     gobuild_component "pmm-agent" "pmm" "" "agent"
 
@@ -121,16 +135,16 @@ main() {
     copy_component "postgres_exporter" "queries-postgres-uptime.yml"
     copy_component "postgres_exporter" "queries-mr.yaml"
     copy_component "postgres_exporter" "queries-lr.yaml"
-
     copy_component "percona-toolkit" "bin/pt-summary"
     copy_component "percona-toolkit" "bin/pt-mysql-summary"
+
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
     gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
     rm -rf ${binary_tarball}
     mkdir -p $(dirname ${binary_tarball}) || :
     
-    # Removes pmm binary in case this is not a PR build - such as RC/GA build
+    # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
     if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
         rm -f "${binary_dir}/bin/pmm"
     fi
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index ea6287703f..119a47f7a3 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -20,13 +20,13 @@ fi
 
 CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-             -t ${DOCKER_CLIENT_TAG} \
-             ${DOCKER_FILE_LOCATION}
+docker buildx build \
+    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+    --progress plain \
+    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+    -t ${DOCKER_CLIENT_TAG} \
+    ${DOCKER_FILE_LOCATION}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_client_tag_file})
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index ae155999aa..97865555b3 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -17,7 +17,7 @@ Usage: $0 [OPTIONS]
         --build_source_deb  If it is 1 source deb package will be built
         --build_rpm         If it is 1 rpm will be built
         --build_deb         If it is 1 deb will be built
-        --install_deps      Install build dependencies(root previlages are required)
+        --install_deps      Install build dependencies(root privileges are required)
         --branch            Branch from which submodules should be taken(default v3)
         --help) usage ;;
 Example $0 --builddir=/tmp/PMM_CLIENT --get_sources=1 --build_src_rpm=1 --build_rpm=1
@@ -90,13 +90,12 @@ check_workdir(){
     if [ "x$WORKDIR" = "x$CURDIR" ]; then
         echo >&2 "Current directory cannot be used for building!"
         exit 1
-    else
-        if ! test -d "$WORKDIR"; then
-            echo >&2 "$WORKDIR is not a directory."
-            exit 1
-        fi
     fi
-    return
+    
+    if ! test -d "$WORKDIR"; then
+        echo >&2 "$WORKDIR is not a directory."
+        exit 1
+    fi
 }
 
 get_sources(){
@@ -108,11 +107,11 @@ get_sources(){
     git clone $REPO
     retval=$?
     if [ $retval != 0 ]; then
-        echo "There were some issues during repo cloning from github. Please retry one more time"
+        echo "There were some issues during repo cloning from github. Please retry."
         exit 1
     fi
     cd pmm-client
-    if [ ! -z $BRANCH ]; then
+    if [ -n $BRANCH ]; then
         git reset --hard
         git clean -xdf
         git checkout $BRANCH
@@ -122,7 +121,6 @@ get_sources(){
     #
     VERSION=$(pmm_version)
     mv Makefile build/
-    #cat VERSION > $VERSION_FILE
     echo "VERSION=${VERSION}" > $VERSION_FILE
     echo "REVISION=${REVISION}" >> $VERSION_FILE
     echo "RPM_RELEASE=${RPM_RELEASE}" >> $VERSION_FILE
@@ -133,7 +131,6 @@ get_sources(){
     echo "MongoExp_BRANCH_NAME=${MongoExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "MysqlExp_BRANCH_NAME=${MysqlExp_BRANCH_NAME}" >> $VERSION_FILE
     echo "ProxysqlExp_BRANCH_NAME=${ProxysqlExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "QAN_BRANCH_NAME=${QAN_BRANCH_NAME}" >> $VERSION_FILE
     echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "TOOLKIT_BRANCH_NAME=${TOOLKIT_BRANCH_NAME}" >> $VERSION_FILE
     PRODUCT=pmm-client
@@ -146,10 +143,8 @@ get_sources(){
     echo "BUILD_ID=${BUILD_ID}" >> $VERSION_FILE
     echo "UPLOAD=UPLOAD/experimental/BUILDS/${PRODUCT_NAME}/${VERSION}/${BRANCH_NAME}/${REVISION}/${BUILD_ID}" >> $VERSION_FILE
     echo "MongoExp_REPO=${MongoExp_REPO}" >> $VERSION_FILE
-    echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
     echo "MysqlExp_REPO=${MysqlExp_REPO}" >> $VERSION_FILE
     echo "ProxysqlExp_REPO=${ProxysqlExp_REPO}" >> $VERSION_FILE
-    echo "QAN_REPO=${QAN_REPO}" >> $VERSION_FILE
     echo "NodeExp_REPO=${NodeExp_REPO}" >> $VERSION_FILE
     cd ../
     mv ${PRODUCT} ${PRODUCT}-${VERSION}
@@ -179,6 +174,7 @@ get_system(){
 }
 
 install_deps() {
+    local RUN_CMD
     if [ $INSTALL = 0 ]; then
         echo "Dependencies will not be installed"
         return;
@@ -188,7 +184,17 @@ install_deps() {
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
-        yum -y install git wget rpmdevtools bison yum-utils rpm-build
+        # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
+        command -v git || RUN_CMD+=" git"
+        command -v curl || RUN_CMD+=" curl"
+        command -v wget || RUN_CMD+=" wget"
+        command -v spectool || RUN_CMD+=" rpmdevtools"
+        command -v bison || RUN_CMD+=" bison"
+        command -v repoquery || RUN_CMD+=" yum-utils"
+        command -v rpmbuild || RUN_CMD+=" rpm-build"
+        if [ -n "$RUN_CMD" ]; then
+            yum -y install "$RUN_CMD"
+        fi
     else
         apt-get update
         apt-get -y install devscripts equivs debhelper
@@ -198,9 +204,9 @@ install_deps() {
 
 get_tar(){
     TARBALL=$1
-    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
     if [ -z $TARFILE ]; then
-        TARFILE=$(basename $(find $CURDIR/$TARBALL -name 'pmm-client*.tar.gz' | sort | tail -n1) || :)
+        TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
         if [ -z $TARFILE ]; then
             echo "There is no $TARBALL for build"
             exit 1
@@ -216,9 +222,9 @@ get_tar(){
 get_deb_sources(){
     param=$1
     echo $param
-    FILE=$(basename $(find $WORKDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+    FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
     if [ -z $FILE ]; then
-        FILE=$(basename $(find $CURDIR/source_deb -name "pmm-client*.$param" | sort | tail -n1) || :)
+        FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
         if [ -z $FILE ]; then
             echo "There is no sources for build"
             exit 1
@@ -232,7 +238,7 @@ get_deb_sources(){
 }
 
 build_srpm(){
-    if [ $SRPM = 0 ]; then
+    if [ "$SRPM" = 0 ]; then
         echo "SRC RPM will not be created"
         return;
     fi
@@ -242,28 +248,24 @@ build_srpm(){
     fi
     cd $WORKDIR
     get_tar "tarball"
-    
-    #
+
     rm -fr rpmbuild
     ls | grep -v tar.gz | xargs rm -rf
-    
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
-    
-    #
+
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-    
-    #
+
     tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
-    #
+
     cd ${WORKDIR}/rpmbuild/SPECS
     cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
     cp ${WORKDIR}/${TARFILE} ../SOURCES/
     cd ${WORKDIR}
-    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" rpmbuild/SPECS/pmm-client.spec
+    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
     mkdir -p ${WORKDIR}/srpm
     mkdir -p ${CURDIR}/srpm
     cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
@@ -271,7 +273,7 @@ build_srpm(){
 }
 
 build_rpm(){
-    if [ $RPM = 0 ]; then
+    if [ "$RPM" = 0 ]; then
         echo "RPM will not be created"
         return;
     fi
@@ -280,9 +282,9 @@ build_rpm(){
         exit 1
     fi
 
-    SRC_RPM=$(basename $(find $WORKDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+    SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
     if [ -z $SRC_RPM ]; then
-        SRC_RPM=$(basename $(find $CURDIR/srpm -name 'pmm-client*.src.rpm' | sort | tail -n1) || :)
+        SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
         if [ -z $SRC_RPM ]; then
             echo "There is no src rpm for build"
             echo "You can create it using key --build_src_rpm=1"
@@ -298,6 +300,10 @@ build_rpm(){
     rm -fr rpmbuild
     mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
     cp $SRC_RPM rpmbuild/SRPMS/
+    if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
+        # Remove older client rpm packages before building a new one
+        rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
+    fi
     rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
              --define "version $pmm_version" \
              --define "release $RPM_RELEASE" \
@@ -305,17 +311,17 @@ build_rpm(){
              --rebuild rpmbuild/SRPMS/$SRC_RPM
 
     return_code=$?
-    if [ $return_code != 0 ]; then
+    if [ "$return_code" -ne 0 ]; then
+        echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n"
         exit $return_code
     fi
-    mkdir -p ${WORKDIR}/rpm
+
     mkdir -p ${CURDIR}/rpm
-    cp rpmbuild/RPMS/*/*.rpm ${WORKDIR}/rpm
     cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
 }
 
 build_source_deb(){
-    if [ $SDEB = 0 ]; then
+    if [ "$SDEB" = 0 ]; then
         echo "source deb package will not be created"
         return;
     fi
@@ -326,17 +332,16 @@ build_source_deb(){
     rm -rf pmm-client*
     get_tar "tarball"
     rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
-    #
-    TARFILE=$(basename $(find . -name 'pmm-client-*.tar.gz' | sort | tail -n1))
+
+    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
     NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
     VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
     VERSION=${VERSION_TMP%.tar.gz}
 
     rm -fr ${NAME}-${VERSION}
-    #
+
     NEWTAR=${NAME}_${VERSION}.orig.tar.gz
     mv ${TARFILE} ${NEWTAR}
-    #
 
     git clone $REPO ${NAME}-${VERSION}_all
     pushd ${NAME}-${VERSION}_all
@@ -363,7 +368,7 @@ build_source_deb(){
     mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/
     mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/
     mv debian ../${NAME}-${VERSION}_all/
-    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/pmm-client.pmm-agent.service
+    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service
     cd ../
 
     rm -rf ${NAME}-${VERSION}
@@ -396,12 +401,12 @@ build_source_deb(){
 }
 
 build_deb(){
-    if [ $DEB = 0 ]; then
-        echo "source deb package will not be created"
+    if [ "$DEB" = 0 ]; then
+        echo "deb package will not be created"
         return;
     fi
     if [ "$OS" = "rpm" ]; then
-        echo "It is not possible to build source deb here"
+        echo "It is not possible to build deb here"
         exit 1
     fi
     for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
@@ -432,6 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
+PACKAGE_NAME=pmm3-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
@@ -446,15 +452,15 @@ ARCH=
 OS=
 SUBMODULE_BRANCH="v3"
 INSTALL=0
-RPM_RELEASE=6
-DEB_RELEASE=6
+RPM_RELEASE=1
+DEB_RELEASE=1
+REVISION=0
 
 if [ -n "$pmm_release" ]; then
     RPM_RELEASE="$RPM_RELEASE.$pmm_release"
     DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
-REVISION=0
 parse_arguments PICK-ARGS-FROM-ARGV "$@"
 get_system
 install_deps
@@ -482,8 +488,6 @@ if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then
 fi
 
 check_workdir
-get_system
-install_deps
 get_sources
 build_srpm
 build_source_deb
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 7c178670d0..ae5780ae24 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -18,14 +15,11 @@ main() {
         export pmm_release=$pmm_release
         export sudo_path=\$(ls /usr/bin/sudo)
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            \$sudo_path /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --install_deps=1
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_rpm=1
+
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index b9867f764b..978f09a1b0 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -7,11 +7,12 @@ set -o xtrace
 
 prepare_tarball() {
     local repo_name=$1
+    local git_dir
     if [ $repo_name = 'pmm-submodules' ]; then
-        mkdir -p ${root_dir}/sources/${repo_name}
         git_dir="${root_dir}/sources/${repo_name}"
+        mkdir -p "${git_dir}"
     else
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+        git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     fi
     local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
     local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
@@ -24,43 +25,43 @@ prepare_tarball() {
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping build
-    else
-        if [ ${repo_name} = 'pmm-submodules' ]; then
-            tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
-            mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${commit}/ \
-                -o ${tarball} \
-                "${commit}"
-        fi
+        return
+    fi
+
+    if [ ${repo_name} = 'pmm-submodules' ]; then
+        tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
+        mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${commit}/ \
+        -o ${tarball} \
+        "${commit}"
 }
 
 prepare_vmagent_tarball() {
     local repo_name=vmagent
     local commit_hash=${vmagent_commit_hash}
-    local commit_short=${commit_hash::6}
+    local commit_short=${commit_hash::7}
     local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
     local repo_var=$(
-        echo "${repo_name}_commit" \
-            | sed -e 's:-:_:g' \
-            | tr '[[:lower:]]' '[[:upper:]]'
+        echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
     )
     echo "${repo_var}=$commit_short" >> ${client_properties}
 
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        wget https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz -O ${tarball}
+        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 
 main() {
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
-    echo "PMM_SUBMODULES_COMMIT=$(git rev-parse --short HEAD)" > ${client_properties}
+
     prepare_vmagent_tarball
     prepare_tarball pmm-submodules
     prepare_tarball pmm
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index aff1082492..20747d0779 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -5,9 +5,6 @@
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
-
 main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
@@ -15,17 +12,13 @@ main() {
         set -o xtrace
 
         export pmm_release=$pmm_release
-        export sudo_path=
-        if [ -e /usr/bin/sudo ]; then
-            export sudo_path=\$(ls /usr/bin/sudo)
-        fi
+        export sudo_path=\$(command -v sudo)
         [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build
         mkdir -p /tmp/pmm
-        pushd /home/builder/results
-            /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_src_rpm=1
-        popd
+        cd /home/builder/results
+        /home/builder/bin/build-client-packages \
+            --builddir=/tmp/pmm \
+            --build_src_rpm=1
         \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm
     "
 }
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 4c2cb97df6..9be0b05f3c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,6 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
+    local cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -31,18 +32,17 @@ is_build_needed() {
     # s3://pmm-build-cache/RELEASE/7 - el7
     # s3://pmm-build-cache/RELEASE/9 - el9
 
-    # RPM_EPOCH is set for Feature Builds
-    if [ -n "$RPM_EPOCH" ]; then
-        aws s3 sync \
-            --region us-east-2 \
-            --no-sign-request \
-            s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
-    else
+    [ "${FORCE_REBUILD}" = 1 ] && return 0
+
+    # RPM_EPOCH is set to 1 for Feature Builds
+    if [ "$RPM_EPOCH" = 1 ]; then
+        cache_dir=PR-BUILDS
+    fi
+    if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -58,103 +58,120 @@ prepare_specs() {
 
     sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
     cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-    if [ -d "${root_dir}/sources/${repo_name}" ]; then
-        local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-        local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
-        local short_commit=${full_commit:0:7}
-
-        sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        if [[ ${spec_name} != "grafana" ]]; then
-            sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
+        return
+    fi
+    
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+    local short_commit=${full_commit:0:7}
+
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ "${spec_name}" != "grafana" ]; then
+        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -z "${full_pmm_version}" ]; then
-            echo 'A full_pmm_version is not specified.'
-            exit 1
-        fi
-        sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    if [ -z "${full_pmm_version}" ]; then
+        echo 'The full_pmm_version is not specified.'
+        exit 1
+    fi
+    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
 
-        if [ -n "$pmm_release" ]; then
-            sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-            grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        fi
+    if [ -n "$pmm_release" ]; then
+        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    fi
 
-        if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
-            echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
-        else
-            git -C "${git_dir}" archive \
-                --format=tar.gz \
-                --prefix=${repo_name}-${full_commit}/ \
-                -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
-                "${full_commit}"
-        fi
+    if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
+        echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
+        return
     fi
+
+    git -C "${git_dir}" archive \
+        --format=tar.gz \
+        --prefix=${repo_name}-${full_commit}/ \
+        -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
+        "${full_commit}"
 }
 
 build() {
     local spec_name=$1
     local repo_name=${2:-$1}
     prepare_specs "${spec_name}" "${repo_name}"
+
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
+    local cache_dir=RELEASE
+    local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
+    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
 
-    if  [[ -n "${FORCE_REBUILD}" ]] || is_build_needed "${spec_name}" "${rpm_version}"; then
-        echo "RPM Build Dir: ${rpmbuild_dir}, Docker image: ${rpmbuild_docker_image}"
-
-        docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-            set -o errexit
-            set -o xtrace
-            echo \"spec_name: ${spec_name}\"
-            echo \"repo_name: ${repo_name}\"
-            echo \"rpm_verison: ${rpm_version}\"
+    if ! is_build_needed "${spec_name}" "${rpm_version}"; then
+        return
+    fi
 
-            sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    if [ -d "${root_dir}/go-build" ]; then
+        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
+    fi
+    if [ -d "${root_dir}/go-path" ]; then
+        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
+    fi
 
-            # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-            if [ -n \"$RPM_EPOCH\" ]; then
-                sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-            fi
+    echo "spec_name: ${spec_name}"
+    echo "repo_name: ${repo_name}"
+    echo "rpm_verison: ${rpm_version}"
 
-            rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+    docker run --rm ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+        set -o errexit
+        set -o xtrace
 
-            printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
-                | sudo tee /etc/yum.repos.d/local.repo
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
-            until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-              echo waiting
-              sleep 1
-            done
+        # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+        if [ \"$RPM_EPOCH\" = 1 ]; then
+            sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+        fi
 
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        rm -rf /home/builder/rpm/RPMS/${spec_name}-*
 
-            spectool -C SOURCES -g SOURCES/${spec_name}.spec
-            rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
-                      --define 'dist .${rpmbuild_dist}' \
-                      --define 'debug_package %{nil}' \
-                      -ba SOURCES/${spec_name}.spec
+        printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+            | sudo tee /etc/yum.repos.d/local.repo
 
-            rm -f SOURCES/${spec_name}.spec*
-            sudo chown -R $(id -u):$(id -g) /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        "
+        until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo waiting
+          sleep 1
+        done
 
-        if [ -n "$RPM_EPOCH" ]; then
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/PR-BUILDS/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
-        else
-            aws s3 sync \
-                --region us-east-2 \
-                ${rpms_dir}/${spec_name}-${rpm_version} \
-                s3://pmm-build-cache/RELEASE/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-                || :
+        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
+          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
-        cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+        spectool -C SOURCES -g SOURCES/${spec_name}.spec
+        rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+                  --define 'dist .${rpmbuild_dist}' \
+                  --define 'debug_package %{nil}' \
+                  -ba SOURCES/${spec_name}.spec
+
+        rm -f SOURCES/${spec_name}.spec*
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    "
+
+    # TODO: see if this is even needed
+    # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+
+    if ! command -v aws &> /dev/null; then
+        echo "AWS CLI is not installed. Skipping upload to S3."
+        return
+    fi
 
+    if [ "$RPM_EPOCH" = 1 ]; then
+      cache_dir=PR-BUILDS
     fi
+    aws s3 sync \
+        --region us-east-2 \
+        ${rpms_dir}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        || :
 }
 
 build "$1" "$2"
diff --git a/build/scripts/build-submodules b/build/scripts/build-submodules
index 34134e794b..e32f4b05a8 100755
--- a/build/scripts/build-submodules
+++ b/build/scripts/build-submodules
@@ -1,42 +1,30 @@
 #!/bin/bash
 set -o errexit
 
+git submodule update --init --jobs 10
+git submodule status
+
 if [ -s ci.yml ]; then
-    sudo rm -rf results tmp || :
     python3 ci.py
+    cat .git-sources
     . ./.git-sources
-    echo $pmm_commit > apiCommitSha
-    echo $pmm_branch > apiBranch
-    echo $pmm_url > apiURL
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit > pmmQACommitSha
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 else
-    sudo rm -rf results tmp || :
-    git reset --hard
-    git clean -fdx
-    git submodule foreach --recursive git reset --hard
-    git submodule foreach --recursive git clean -fdx
-    git submodule status
-    export commit_sha=$(git submodule status | grep 'pmm-managed' | awk -F ' ' '{print $1}')
-    export api_tests_commit_sha=$(git submodule status | grep 'pmm' | awk -F ' ' '{print $1}')
-    export api_tests_branch=$(git config -f .gitmodules submodule.pmm.branch)
-    export api_tests_url=$(git config -f .gitmodules submodule.pmm.url)
-    echo $api_tests_commit_sha > apiCommitSha
-    echo $api_tests_branch > apiBranch
-    echo $api_tests_url > apiURL
-    cat apiBranch
-    cat apiURL
-    export pmm_qa_commit_sha=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-    export pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
-    echo $pmm_qa_branch > pmmQABranch
-    echo $pmm_qa_commit_sha > pmmQACommitSha
-    export pmm_ui_tests_commit_sha=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-    export pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
-    echo $pmm_ui_tests_branch > pmmUITestBranch
-    echo $pmm_ui_tests_commit_sha > pmmUITestsCommitSha
+    # Define variables
+    pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+    pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+    pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+    pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+    pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+    pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+    pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
 fi
 
-export fb_commit_sha=$(git rev-parse HEAD)
+fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
+echo $pmm_commit > apiCommitSha
+echo $pmm_branch > apiBranch
+echo $pmm_url > apiURL
+echo $pmm_qa_branch > pmmQABranch
+echo $pmm_qa_commit > pmmQACommitSha
+echo $pmm_ui_tests_branch > pmmUITestBranch
+echo $pmm_ui_tests_commit > pmmUITestsCommitSha
diff --git a/build/scripts/vars b/build/scripts/vars
index af854a8ffe..ea72294352 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -19,11 +19,7 @@ full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD)
 # Replace '/' with '-' to prevent sed from failing on dependabot-authored PRs
 full_pmm_version=${full_pmm_version//\//-}
 
-# TODO Maybe it makes sense to use variable from job here
-if [[ ${pmm_branch} =~ release-* || \
-      ${pmm_branch} =~ pmm-3.* || \
-      ${pmm_branch} == v3 || \
-      ${pmm_branch} == "main" ]]; then
+if [[ "${pmm_branch}" =~ ^pmm-3.|^v3$|^main$ ]]; then
     full_pmm_version=${pmm_base_version}
 fi
 

From 32c8fa8f90063b01efcda089c32d34b7b65eef4f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 01:41:46 +0000
Subject: [PATCH 002/127] PMM-12899 update grafana spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 85976d18a1..da293bbc1f 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -22,9 +22,6 @@ Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
 ExclusiveArch:  %{ix86} x86_64 %{arm}
 
 BuildRequires: fontconfig
-%if 0%{?rhel} < 9
-BuildRequires: nodejs-grunt-cli
-%endif
 
 %description
 Grafana is an open source, feature rich metrics dashboard and graph editor for
@@ -34,9 +31,7 @@ Graphite, InfluxDB & OpenTSDB.
 %setup -q -n grafana-%{commit}
 rm -rf Godeps
 sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
-%if 0%{?rhel} >= 9
-    sudo npm install -g grunt-cli
-%endif
+sudo npm install -g grunt-cli
 
 %build
 mkdir -p _build/src
@@ -82,6 +77,9 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
+* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
 - PMM-12991 Grafana 10.4.0
 

From 37e005a07ee7c730bbb6f3e42c55620f6522a184 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 02:11:03 +0000
Subject: [PATCH 003/127] PMM-12899 fix the build_command

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 295a5a5b02..f580e3f5d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -51,7 +51,7 @@ gobuild_component() {
     elif [ ${component} = 'percona-toolkit' ]; then
         build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} go build -o ${bin_dir}/${result_file} ."
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
 
     docker run --rm \

From c863f50812f8c198d15610526f9f24c9026113ab Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 03:47:06 +0000
Subject: [PATCH 004/127] PMM-12899 fix builddeps

---
 build/local/build.sh                         |  3 ---
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-packages          |  2 +-
 build/scripts/build-server-rpm               | 12 ++++++------
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index f27ec569a8..938d12f720 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -82,15 +82,12 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-controller
-"$PATH_TO_SCRIPTS/build-server-rpm" dbaas-tools
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
 "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" alertmanager
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index da293bbc1f..95d5b956f4 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -77,7 +77,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Tue Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 97865555b3..e7205ae7bb 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -437,7 +437,7 @@ build_deb(){
 #main
 
 CURDIR=$(pwd)
-PACKAGE_NAME=pmm3-client
+PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties
 args=
 WORKDIR=
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 9be0b05f3c..295924c54f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,7 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    local build_deps=("dbaas-tools.spec" "grafana.spec" "percona-dashboards.spec")
+    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -128,7 +128,7 @@ build() {
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-        if [ \"$RPM_EPOCH\" = 1 ]; then
+        if [ ${RPM_EPOCH} = 1 ]; then
             sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
         fi
 
@@ -138,12 +138,12 @@ build() {
             | sudo tee /etc/yum.repos.d/local.repo
 
         until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-          echo waiting
-          sleep 1
+            echo waiting
+            sleep 1
         done
 
-        if grep ${spec_name}.spec <<< ${build_deps[*]} &>/dev/null; then
-          sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+        if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec

From 853be6ed9076267f039ed860dce8ef1c30168c42 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Mar 2024 04:07:25 +0000
Subject: [PATCH 005/127] PMM-12899 use rpmbuild:3

---
 build/local/build.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 938d12f720..03da4e51eb 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -43,7 +43,7 @@ test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9 
+export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
 # Local reference test environment
 # CPU: 4 cores
@@ -83,7 +83,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
 "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana-db-migrator
 "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
 # 3rd-party
@@ -91,7 +90,6 @@ export RPMBUILD_DIST="el9"
 "$PATH_TO_SCRIPTS/build-server-rpm" grafana
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:ol9
 export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9.local
 ${PATH_TO_SCRIPTS}/build-server-docker

From d57ec55633f133d0fc74f543eca0a67e708e93d8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 26 Mar 2024 09:39:53 +0300
Subject: [PATCH 006/127] PMM-12641 Fix the dashboard upgrade (#2926)

* PMM-12641 Fix the dashboard upgrade

This fixes a bug where, due to a wrong ansible formula, PMM would upgrade the dashboards no matter what :)

* PMM-12641 follow up on review
---
 .../roles/initialization/tasks/main.yml       | 95 ++++++++++---------
 1 file changed, 50 insertions(+), 45 deletions(-)

diff --git a/build/ansible/roles/initialization/tasks/main.yml b/build/ansible/roles/initialization/tasks/main.yml
index be42baf2e8..4cb85312f7 100644
--- a/build/ansible/roles/initialization/tasks/main.yml
+++ b/build/ansible/roles/initialization/tasks/main.yml
@@ -27,56 +27,61 @@
 
 - name: Set need_upgrade fact
   set_fact:
-    need_upgrade: not pmm_current_version is version(pmm_image_version, '>=')
+    need_upgrade: "{{ pmm_current_version is version(pmm_image_version, '<') }}"
 
 - name: Print current PMM and image versions
   debug:
     msg: "Current version: {{ pmm_current_version }} Image Version: {{ pmm_image_version }}"
 
-- name: Enable maintenance mode before upgrade
-  copy:
-    src: maintenance.html
-    dest: /usr/share/pmm-server/maintenance/
-    owner: pmm
-    group: pmm
-    mode: 0644
-
-- name: Upgrade dashboards
-  include_role:
-    name: dashboards
-  when: need_upgrade
-
-- name: Copy file with image version
-  copy:
-    src: /usr/share/percona-dashboards/VERSION
-    dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
-    owner: pmm
-    group: pmm
-    mode: 0644
-    remote_src: yes
-  when: need_upgrade
-
-- name: Create a backup directory
-  file:
-    path: /srv/backup
-    state: directory
-    owner: pmm
-    group: pmm
-    mode: 0775
+- name: Print need_upgrade fact
+  debug:
+    msg: "Need upgrade: {{ need_upgrade }}"
 
-# Note: we want to leave this for some time until we achieve stable builds
-- name: Output pmm-managed logs
-  shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
+- name: Perform upgrade tasks
+  block:
+    - name: Enable maintenance mode before upgrade
+      copy:
+        src: maintenance.html
+        dest: /usr/share/pmm-server/maintenance/
+        owner: pmm
+        group: pmm
+        mode: 0644
+    
+    - name: Upgrade dashboards
+      include_role:
+        name: dashboards
+    
+    - name: Copy file with image version
+      copy:
+        src: /usr/share/percona-dashboards/VERSION
+        dest: /srv/grafana/PERCONA_DASHBOARDS_VERSION
+        owner: pmm
+        group: pmm
+        mode: 0644
+        remote_src: yes
+    
+    - name: Create a backup directory
+      file:
+        path: /srv/backup
+        state: directory
+        owner: pmm
+        group: pmm
+        mode: 0775
 
-- name: Wait for PMM to be ready
-  ansible.builtin.uri:
-    url: "http://127.0.0.1:7772/v1/readyz"
-    status_code: 200
-    method: GET
-  retries: 20
-  delay: 5
+    # Note: we want to leave this for some time until we achieve stable builds
+    - name: Output pmm-managed logs
+      shell: sleep 10 && tail -n 300 /srv/logs/pmm-managed.log
 
-- name: Disable maintenance mode
-  file:
-    state: absent
-    path: /usr/share/pmm-server/maintenance/maintenance.html
+    - name: Wait for PMM to be ready
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:7772/v1/readyz"
+        status_code: 200
+        method: GET
+      retries: 20
+      delay: 5
+    
+    - name: Disable maintenance mode
+      file:
+        state: absent
+        path: /usr/share/pmm-server/maintenance/maintenance.html
+  when: need_upgrade

From 5c7821fc83520d31455ef6ad20a405f162a7669c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:03:42 +0000
Subject: [PATCH 007/127] PMM-12899 prevent grafana from using own GOPATH

---
 build/packages/rpm/server/SPECS/grafana.spec |  4 ++--
 build/scripts/build-server-rpm               | 15 +++++++++------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 95d5b956f4..a535cf69b1 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -34,8 +34,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-mkdir -p _build/src
-export GOPATH="$(pwd)/_build"
+# mkdir -p _build/src
+# export GOPATH="$(pwd)/_build"
 
 make build-go
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 295924c54f..6aa60e4388 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -104,7 +104,6 @@ build() {
     local CH_TAG=${rpm_version#*-}
     local cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    # local build_deps=("grafana.spec" "percona-dashboards.spec")
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -117,6 +116,7 @@ build() {
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi
 
+    echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
     echo "repo_name: ${repo_name}"
     echo "rpm_verison: ${rpm_version}"
@@ -125,7 +125,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
@@ -142,8 +142,10 @@ build() {
             sleep 1
         done
 
+        # Only these two specs have build dependencies
         if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sudo yum-builddep --randomwait=1 -y SOURCES/${spec_name}.spec
+            sleep 5s
+            sudo yum-builddep -y SOURCES/${spec_name}.spec
         fi
 
         spectool -C SOURCES -g SOURCES/${spec_name}.spec
@@ -160,7 +162,7 @@ build() {
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
     if ! command -v aws &> /dev/null; then
-        echo "AWS CLI is not installed. Skipping upload to S3."
+        echo "AWS CLI is not installed. Upload to S3 will be skipped."
         return
     fi
 
@@ -172,9 +174,10 @@ build() {
         ${rpms_dir}/${spec_name}-${rpm_version} \
         s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
+
+    echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
 }
 
-build "$1" "$2"
-echo "build-server-rpm finished, spec_name: $1, repo_name: ${2:-$1}"
+build "$@"
 
 # vim: expandtab shiftwidth=4 tabstop=4

From 42a02a9217d204fac5d490e808b39ae9878c8cbc Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 31 Mar 2024 23:52:37 +0000
Subject: [PATCH 008/127] PMM-12899 use a dedicated folder for yarn cache

---
 build/local/build.sh           | 1 +
 build/scripts/build-server-rpm | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 03da4e51eb..cba72f27df 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -41,6 +41,7 @@ echo $fb_commit_sha > fbCommitSha
 # Create cache directories. 
 test -d "${root_dir}/go-path" || mkdir -p "go-path"
 test -d "${root_dir}/go-build" || mkdir -p "go-build"
+test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
 
 PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6aa60e4388..06ae02326e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -112,6 +112,9 @@ build() {
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
+    if [ -d "${root_dir}/yarn-cache" ]; then
+        volume_mounts+=" -v ${root_dir}/yarn-cache:/home/builder/.yarn"
+    fi
     if [ -d "${root_dir}/go-path" ]; then
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi

From 6d0b02bdcab1d18ef4a91591a31683c8270ebc00 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:19:42 +0000
Subject: [PATCH 009/127] PMM-12899 extract server rpm script

---
 build/local/build.sh                  | 82 ++++++++++++++++++++-------
 build/scripts/build-rpmbuild-docker   | 11 ----
 build/scripts/build-server-docker     | 18 +++---
 build/scripts/build-server-rpm        | 12 ++--
 build/scripts/build-server-rpm-script | 40 +++++++++++++
 5 files changed, 115 insertions(+), 48 deletions(-)
 delete mode 100755 build/scripts/build-rpmbuild-docker
 create mode 100755 build/scripts/build-server-rpm-script

diff --git a/build/local/build.sh b/build/local/build.sh
index cba72f27df..80caaa0d2d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,7 +1,38 @@
 #!/bin/bash -e
 
-# Update submodules and PR branches
-/bin/bash $(dirname $0)/update.sh
+# Define global variables
+NO_UPDATE=0
+NO_CLIENT=0
+NO_CLIENT_DOCKER=0
+NO_SERVER_RPM=0
+
+while test "$#" -gt 0; do
+  case "$1" in
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [ "$NO_UPDATE" -eq 0 ]; then
+  # Update submodules and PR branches
+  /bin/bash $(dirname $0)/update.sh
+fi
 
 get_branch_name() {
   local module="${1:-}"
@@ -33,7 +64,7 @@ echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 fb_commit_sha=$(git rev-parse HEAD)
 echo $fb_commit_sha > fbCommitSha
 
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`.
+# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
 # Important: the docker container's user need to be able to write to these directories.
 # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
 # that the directories we create on the host are owned by a user with same uid and gid.
@@ -51,22 +82,26 @@ export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-# Build client source: 4m39s from scratch, 0m27s using cache
-"$PATH_TO_SCRIPTS/build-client-source"
+if [ "$NO_CLIENT" -eq 0 ]; then
+  # Build client source: 4m39s from scratch, 0m27s using cache
+  "$PATH_TO_SCRIPTS/build-client-source"
 
-# Build client binary: ??? from scratch, 0m20s using cache
-"$PATH_TO_SCRIPTS/build-client-binary"
+  # Build client binary: ??? from scratch, 0m20s using cache
+  "$PATH_TO_SCRIPTS/build-client-binary"
 
-# Building client source rpm takes 13s (caching does not apply)
-"$PATH_TO_SCRIPTS/build-client-srpm"
+  # Building client source rpm takes 13s (caching does not apply)
+  "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client rpm takes 1m40s
-"$PATH_TO_SCRIPTS/build-client-rpm"
+  # Building client rpm takes 1m40s
+  "$PATH_TO_SCRIPTS/build-client-rpm"
+fi
 
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-"$PATH_TO_SCRIPTS/build-client-docker"
+if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-client-docker"
+fi
 
 # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
 # total time: 6m26s - build from scratch, no initial cache
@@ -79,20 +114,23 @@ export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
 
 export RPM_EPOCH=1
 export RPMBUILD_DIST="el9"
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-# 3rd-party
-"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-"$PATH_TO_SCRIPTS/build-server-rpm" grafana
+if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+
+  # 3rd-party
+  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
 export RPMBUILD_DIST=el9
-export DOCKERFILE=Dockerfile.el9.local
+export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
 # Clean up temporary files
diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker
deleted file mode 100755
index 77706f3621..0000000000
--- a/build/scripts/build-rpmbuild-docker
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-. $(dirname $0)/vars
-
-root_dir=$(cd $(dirname $0)/../..; pwd -P)
-docker build --squash -f ${docker_file} -t rpmbuild:local ${root_dir}/build/docker/rpmbuild/
-
-# vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 421a4dd688..8cb2f70bfc 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -32,15 +32,15 @@ if [ -z "${DOCKER_TAG}" ]; then
     DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
 fi
 
-IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2`
-
-docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-             --build-arg VERSION="$IMAGE_VERSION" \
-             --squash \
-             --no-cache \
-             -f ${docker_root}/${docker_file} \
-             -t ${DOCKER_TAG} \
-             ${docker_root}/
+IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+docker buildx build \
+    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg VERSION="$IMAGE_VERSION" \
+    --progress plain \
+    -f ${docker_root}/${docker_file} \
+    -t ${DOCKER_TAG} \
+    ${docker_root}
 
 if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_tag_file})
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 06ae02326e..031aa7707e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,7 +24,7 @@ is_build_needed() {
     local spec_name=$1
     local rpm_version=$2
     local packages=
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
     # s3://pmm-build-cache/PR-BUILDS/7 - el7
@@ -36,13 +36,13 @@ is_build_needed() {
 
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
-        cache_dir=PR-BUILDS
+        s3_cache_dir=PR-BUILDS
     fi
     if command -v aws &> /dev/null; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
-            s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+            s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
             ${rpms_dir}/${spec_name}-${rpm_version}
     fi
 
@@ -102,7 +102,7 @@ build() {
     local rpm_version=$(get_rpm_version "${spec_name}")
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
-    local cache_dir=RELEASE
+    local s3_cache_dir=RELEASE
     local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
@@ -170,12 +170,12 @@ build() {
     fi
 
     if [ "$RPM_EPOCH" = 1 ]; then
-      cache_dir=PR-BUILDS
+      s3_cache_dir=PR-BUILDS
     fi
     aws s3 sync \
         --region us-east-2 \
         ${rpms_dir}/${spec_name}-${rpm_version} \
-        s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
         || :
 
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
new file mode 100755
index 0000000000..cece634cdc
--- /dev/null
+++ b/build/scripts/build-server-rpm-script
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -o errexit
+set -o xtrace
+
+. $(dirname $0)/vars
+
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+
+# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+if [ ${RPM_EPOCH} = 1 ]; then
+    sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
+fi
+
+rm -rf /home/builder/rpm/RPMS/${spec_name}-*
+
+printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+    | sudo tee /etc/yum.repos.d/local.repo
+
+until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+    echo waiting
+    sleep 1
+done
+
+# Only these two specs have build dependencies
+if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+    sleep 5s
+    sudo yum-builddep -y SOURCES/${spec_name}.spec
+fi
+
+spectool -C SOURCES -g SOURCES/${spec_name}.spec
+rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+          --define 'dist .${rpmbuild_dist}' \
+          --define 'debug_package %{nil}' \
+          -ba SOURCES/${spec_name}.spec
+
+rm -f SOURCES/${spec_name}.spec*
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+
+# vim: expandtab shiftwidth=4 tabstop=4

From 098e894628c160cd4805ac5f5feb7bdf743ac024 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 01:31:54 +0000
Subject: [PATCH 010/127] PMM-12899 fix server param evaluation

---
 build/scripts/build-server-docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 8cb2f70bfc..16027c3beb 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -35,7 +35,7 @@ fi
 IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
 
 docker buildx build \
-    --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \
+    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
     --build-arg VERSION="$IMAGE_VERSION" \
     --progress plain \
     -f ${docker_root}/${docker_file} \

From aa5729fb413c226baaa908cb05ac7ab19dd69e52 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:08:11 +0000
Subject: [PATCH 011/127] PMM-12899 clean up server RPM specs

---
 build/local/build.sh                          |  6 +--
 build/packages/rpm/server/SPECS/grafana.spec  | 15 +++----
 .../rpm/server/SPECS/percona-dashboards.spec  | 44 ++++++++++--------
 .../rpm/server/SPECS/percona-qan-api2.spec    | 41 +++++++++--------
 build/packages/rpm/server/SPECS/pmm-dump.spec | 16 ++++---
 .../rpm/server/SPECS/pmm-managed.spec         | 45 ++++++++++---------
 .../packages/rpm/server/SPECS/pmm-update.spec | 32 ++++++-------
 .../rpm/server/SPECS/victoriametrics.spec     | 20 ++++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  | 44 +++++++++---------
 build/scripts/vars                            |  2 +-
 10 files changed, 139 insertions(+), 126 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 80caaa0d2d..bd174fb965 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -12,7 +12,7 @@ while test "$#" -gt 0; do
       NO_UPDATE=1
       ;;
     --no-client)
-      NO_CLIENT=1
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
       ;;
     --no-client-docker)
       NO_CLIENT_DOCKER=1
@@ -99,7 +99,7 @@ fi
 # Building client docker image takes 17s
 GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then
+if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-client-docker"
 fi
 
@@ -113,7 +113,6 @@ fi
 # total time: ??? - subsequent build, using cache from prior builds
 
 export RPM_EPOCH=1
-export RPMBUILD_DIST="el9"
 
 if [ "$NO_SERVER_RPM" -eq 0 ]; then
   "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
@@ -129,7 +128,6 @@ if [ "$NO_SERVER_RPM" -eq 0 ]; then
 fi
 
 export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export RPMBUILD_DIST=el9
 export DOCKERFILE=Dockerfile.el9
 ${PATH_TO_SCRIPTS}/build-server-docker
 
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index a535cf69b1..11e7b246a0 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -1,15 +1,13 @@
-%global debug_package   %{nil}
 %global commit          f283b87d773da4ad61ca65660e72c084ac8f8716
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         104
+%define release         105
 %define grafana_version 10.4.0
 %define full_pmm_version 2.0.0
 %define full_version    v%{grafana_version}-%{full_pmm_version}
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom|od -An -tx1|tr -d ' \\n')" -a -v -x %{?**};
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
 Name:           percona-grafana
@@ -24,8 +22,9 @@ ExclusiveArch:  %{ix86} x86_64 %{arm}
 BuildRequires: fontconfig
 
 %description
-Grafana is an open source, feature rich metrics dashboard and graph editor for
-Graphite, InfluxDB & OpenTSDB.
+Grafana is an open source observability and data visualization platform.
+Visualize metrics, logs, and traces from multiple sources like
+Prometheus, Loki, Elasticsearch, InfluxDB, Postgres and many more.
 
 %prep
 %setup -q -n grafana-%{commit}
@@ -77,7 +76,7 @@ getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it
 exit 0
 
 %changelog
-* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-2
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
 - PMM-12899 Use module and build cache
 
 * Tue Mar 12 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 10.4.0-1
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index 5a1a895ed3..e356038179 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -1,27 +1,30 @@
-%global debug_package   %{nil}
 %global __strip         /bin/true
 
-%global repo		grafana-dashboards
-%global provider	github.com/percona/%{repo}
-%global import_path	%{provider}
-%global commit		ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
-%global shortcommit	%(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         21
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
-
-Name:		percona-dashboards
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Grafana dashboards for monitoring
-
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+%global repo            grafana-dashboards
+%global provider        github.com/percona/%{repo}
+%global import_path     %{provider}
+%global commit          ad4af6808bcd361284e8eb8cd1f36b1e98e32bce
+%global shortcommit     %(c=%{commit}; echo ${c:0:7})
+%define release         22
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:     percona-dashboards
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Grafana dashboards for monitoring
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildRequires:	nodejs
 Requires:	percona-grafana
-Provides:	percona-grafana-dashboards = %{version}-%{release}
 
 %description
 This is a set of Grafana dashboards for database and system monitoring
@@ -57,6 +60,9 @@ echo %{version} > %{buildroot}%{_datadir}/%{name}/VERSION
 
 
 %changelog
+* Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-22
+- PMM-12899 Use module and build cache
+
 * Wed Nov 29 2023 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
 - PMM-12693 Run Grafana as non-root user
 
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 14e89677b5..ae33484d71 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -1,8 +1,3 @@
-# Go build id is not supported for now.
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://bugzilla.redhat.com/show_bug.cgi?id=1295951
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global import_path     %{provider}
@@ -10,17 +5,22 @@
 # see: https://github.com/percona/pmm/blob/main/build/scripts/build-server-rpm#L58
 %global commit          0000000000000000000000000000000000000000
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         17
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         18
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           percona-qan-api2
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
-Summary:        Query Analytics API v2 for PMM
+Summary:        Query Analytics API for PMM
 
 License:        AGPLv3
 URL:            https://%{provider}
@@ -32,10 +32,7 @@ See PMM docs for more information - https://docs.percona.com/percona-monitoring-
 
 
 %prep
-%setup -T -c -n %{repo}-%{version}
-%setup -q -c -a 0 -n %{repo}-%{version}
-mkdir -p src/github.com/percona
-mv %{repo}-%{commit} src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
@@ -43,29 +40,31 @@ export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/%{provider}/qan-api2
-make release
+make -C qan-api2 release
 
 
 %install
 
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 src/%{provider}/bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
+install -p -m 0755 ./bin/qan-api2 %{buildroot}%{_sbindir}/%{name}
 
 
 %files
 %attr(0755, root, root) %{_sbindir}/%{name}
-%license src/%{provider}/qan-api2/LICENSE
-%doc src/%{provider}/qan-api2/README.md
+%license qan-api2/LICENSE
+%doc qan-api2/README.md
 
 %changelog
-* Mon Nov  7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
+* Wed Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-18
+- PMM-12899 Use module and build cache
+
+* Mon Nov 7 2022 Alexander Tymchuk <alexander.tymchuk@percona.com> - 2.0.0-17
 - PMM-10117 migrate QAN API to monorepo
 
 * Mon May 16 2022 Nikita Beletskii <nikita.beletskii@percona.com> - 2.0.0-16
 - PMM-10027 remove useless packages
 
-* Thu Jul  2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
+* Thu Jul 2 2020 Mykyta Solomko <mykyta.solomko@percona.com> - 2.0.0-15
 - PMM-5645 built using Golang 1.14
 
 * Tue Mar 19 2019 Vadim Yalovets <vadim.yalovets@percona.com> - 2.0.0-4
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index c8882aa25b..e5ddbee367 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,12 +1,15 @@
-%undefine _missing_build_ids_terminate_build
-
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
 Name:		pmm-dump
 Version:	0.7.0
@@ -37,6 +40,9 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
+- PMM-12899 Use module and build cache
+
 * Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index b313be8ba8..d9cc53019c 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f3d007617941033867aea6a134c48b39142427f
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         20
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         21
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-managed
-Version:	%{version}
-Release:	%{rpm_release}
-Summary:	Percona Monitoring and Management management daemon
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		  https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     pmm-managed
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Percona Monitoring and Management management daemon
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 pmm-managed manages configuration of PMM server components (VictoriaMetrics,
@@ -28,19 +32,14 @@ See PMM docs for more information.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
-
+%setup -q -n %{repo}-%{commit}
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/managed
-make release
+make -C managed release
 
 
 %install
@@ -51,13 +50,12 @@ install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed
 install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init
 install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark
 
-cd src/github.com/percona/pmm
-cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
+cp -pa api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %files
-%license src/%{provider}/LICENSE
-%doc src/%{provider}/README.md
+%license managed/LICENSE
+%doc managed/README.md
 %{_sbindir}/pmm-managed
 %{_sbindir}/pmm-managed-init
 %{_sbindir}/pmm-managed-starlark
@@ -65,6 +63,9 @@ cp -pa ./api/swagger %{buildroot}%{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-21
+- PMM-12899 Use module and build cache
+
 * Thu Jul 28 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 2.30.0-1
 - PMM-10036 migrate to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index e68fc9f55d..9a9a5eac21 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,5 +1,3 @@
-%undefine _missing_build_ids_terminate_build
-
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -10,20 +8,25 @@
 %global import_path     %{provider}
 %global commit	        592eddf656bce32a11bd958af0a32c62bd5ea34c
 %global shortcommit	    %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         67
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         68
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		pmm-update
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
+Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
 Summary:	Tool for updating packages and OS configuration for PMM Server
 
 License:	AGPLv3
-URL:		https://%{provider}
+URL:		  https://%{provider}
 Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 BuildArch:	noarch
@@ -34,16 +37,13 @@ BuildArch:	noarch
 
 %prep
 %setup -q -n %{repo}-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
 
 %build
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/update
-make release
+make -C update release
 
 
 %install
@@ -51,18 +51,20 @@ install -d %{buildroot}%{_datadir}/%{name}
 cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
 
 install -d %{buildroot}%{_sbindir}
-cd src/github.com/percona/pmm/update
-install -p -m 0755 bin/pmm-update %{buildroot}%{_sbindir}/
+install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
 
 %files
-%license LICENSE
-%doc README.md
+%license update/LICENSE
+%doc update/README.md
 %{_sbindir}/pmm-update
 %{_datadir}/%{name}
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-68
+- PMM-12899 Use module and build cache
+
 * Thu Dec 8 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-67
 - PMM-11207 Migrate pmm-update to monorepo
 
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index ec55d239d1..585a35954a 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,19 +1,16 @@
-%undefine _missing_build_ids_terminate_build
-
-%define copying() \
-%if 0%{?fedora} >= 21 || 0%{?rhel} >= 7 \
-%license %{*} \
-%else \
-%doc %{*} \
-%endif
-
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
+
 Name:           percona-victoriametrics
 Version:        1.93.4
-Release:        1%{?dist}
+Release:        2%{?dist}
 Summary:        VictoriaMetrics monitoring solution and time series database
 License:        Apache-2.0
 URL:            https://%{provider}
@@ -50,6 +47,9 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 1.93.4-2
+- PMM-12899 Use module and build cache
+
 * Thu Sep 14 2023 Alex Tymchuk <alexander.tymchuk@percona.com> - 1.93.4-1
 - upgrade victoriametrics to 1.93.4 release
 
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index be5cdca4ca..fe4100e2d8 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,25 +1,29 @@
-%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global commit          8f74cea10d85e441ee88ef4b12bc47bc05165ba9
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define build_timestamp %(date -u +"%y%m%d%H%M")
-%define release         1
-%define rpm_release     %{release}.%{build_timestamp}.%{shortcommit}%{?dist}
+%define release         2
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:		vmproxy
-Version:	%{full_pmm_version}
-Release:	%{rpm_release}
-Summary:	Percona VMProxy stateless reverse proxy for VictoriaMetrics
+%if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
+%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
+%endif
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Name:     vmproxy
+Version:  %{full_pmm_version}
+Release:  %{rpm_release}
+Summary:  Percona VMProxy stateless reverse proxy for VictoriaMetrics
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
 VMProxy is a stateless reverse proxy which proxies requests to VictoriaMetrics and
@@ -27,33 +31,31 @@ optionally adds `extra_filters` query based on the provided configuration.
 
 
 %prep
-%setup -q -n pmm-%{commit}
-mkdir -p src/github.com/percona
-ln -s $(pwd) src/%{provider}
+%setup -q -n %{repo}-%{commit}
 
 
 %build
-
 export PMM_RELEASE_VERSION=%{full_pmm_version}
 export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
-cd src/github.com/percona/pmm/vmproxy
-make release
+make -C vmproxy release
 
 
 %install
-install -d -p %{buildroot}%{_bindir}
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files
-%license src/%{provider}/vmproxy/LICENSE
-%doc src/%{provider}/vmproxy/README.md
+%license vmproxy/LICENSE
+%doc vmproxy/README.md
 %{_sbindir}/vmproxy
 
 
 %changelog
+* Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-2
+- PMM-12899 Use module and build cache
+
 * Mon Dec 5 2022 Michal Kralik <michal.kralik@percona.com> - 2.34.0-1
 - Initial release of VMProxy
diff --git a/build/scripts/vars b/build/scripts/vars
index ea72294352..de57bfb8a4 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -41,7 +41,7 @@ echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versio
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
-rpmbuild_dist=${RPMBUILD_DIST:-"el7"}
+rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}
 client_properties=${root_dir}/results/pmm-client.properties

From aa5e8b82cd3cfa621be875d28df3ee6ab05d5df7 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 20:16:41 +0000
Subject: [PATCH 012/127] PMM-12899 fix directory permissions

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 031aa7707e..ff5ab06553 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -128,7 +128,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache /home/builder/go
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then

From ee45fd6aadb70d8bc59d4aa5b9eee695b4723572 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 21:49:29 +0000
Subject: [PATCH 013/127] PMM-12899 remove unnecessary macro

---
 build/packages/rpm/server/SPECS/grafana.spec  |  2 ++
 .../rpm/server/SPECS/percona-dashboards.spec  |  6 -----
 .../rpm/server/SPECS/percona-qan-api2.spec    |  6 -----
 build/packages/rpm/server/SPECS/pmm-dump.spec | 24 ++++++++-----------
 .../rpm/server/SPECS/pmm-managed.spec         |  7 +-----
 .../packages/rpm/server/SPECS/pmm-update.spec |  7 +-----
 .../rpm/server/SPECS/victoriametrics.spec     |  8 ++-----
 build/packages/rpm/server/SPECS/vmproxy.spec  |  9 ++-----
 8 files changed, 18 insertions(+), 51 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 11e7b246a0..e4c3e1558d 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -7,6 +7,8 @@
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 %if ! 0%{?gobuild:1}
+# https://github.com/rpm-software-management/rpm/issues/367
+# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
 %define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index e356038179..d52adfdddf 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -8,12 +8,6 @@
 %define release         22
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     percona-dashboards
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index ae33484d71..46995118b1 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           pmm-qan
 Version:        %{version}
 Release:        %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index e5ddbee367..b5b4edad32 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,3 +1,5 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
 %global commit          0d49b27729506dc62950f9fa59147d63df194db2
@@ -5,20 +7,14 @@
 %define release         2
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
-Name:		pmm-dump
-Version:	0.7.0
-Release:	%{rpm_release}
-Summary:	Percona PMM Dump allows to export and import monitoring metrics and query analytics.
+Name:     pmm-dump
+Version:  0.7.0
+Release:  %{rpm_release}
+Summary:  Percona PMM Dump allows to export and import monitoring metrics and query analytics.
 
-License:	AGPLv3
-URL:		https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}.tar.gz
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 %{summary}
@@ -43,7 +39,7 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 * Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
 - PMM-12899 Use module and build cache
 
-* Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
+* Thu Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
 * Tue Mar 29 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 0.6.0-1
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index d9cc53019c..02f1578043 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     pmm-managed
 Version:  %{version}
 Release:  %{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 9a9a5eac21..cb048070c3 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 # TODO: remove it as soon as we remove all noarch pmm-update rpms
 # from 'pmm3-components/yum/laboratory'
 %define _binaries_in_noarch_packages_terminate_build   0
@@ -14,12 +15,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:		  pmm-update
 Version:	%{full_pmm_version}
 Release:	%{rpm_release}
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index 585a35954a..97e2b39a78 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,13 +1,9 @@
+%undefine _missing_build_ids_terminate_build
+
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
 %global commit          pmm-6401-v1.93.4
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:           percona-victoriametrics
 Version:        1.93.4
 Release:        2%{?dist}
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index fe4100e2d8..11ca103785 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,3 +1,4 @@
+%undefine _missing_build_ids_terminate_build
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
@@ -10,12 +11,6 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-%if ! 0%{?gobuild:1}
-# https://github.com/rpm-software-management/rpm/issues/367
-# https://fedoraproject.org/wiki/PackagingDrafts/Go#Build_ID
-%define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
-%endif
-
 Name:     vmproxy
 Version:  %{full_pmm_version}
 Release:  %{rpm_release}
@@ -44,7 +39,7 @@ make -C vmproxy release
 
 %install
 install -d -p %{buildroot}%{_sbindir}
-install -p -m 0755 vmproxy/bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
+install -p -m 0755 ./bin/vmproxy %{buildroot}%{_sbindir}/vmproxy
 
 
 %files

From afde65ad8151a097eec42d8a80f4f1671c58686f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 1 Apr 2024 22:24:15 +0000
Subject: [PATCH 014/127] PMM-12899 revert QAN package name

---
 build/packages/rpm/server/SPECS/percona-qan-api2.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 46995118b1..3f851cb36f 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -11,7 +11,7 @@
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
 %define full_pmm_version 2.0.0
 
-Name:           pmm-qan
+Name:           percona-qan-api2
 Version:        %{version}
 Release:        %{rpm_release}
 Summary:        Query Analytics API for PMM

From f28cbccca37377a86d6abff3b6f08a9993955551 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:03:47 +0000
Subject: [PATCH 015/127] PMM-12899 merge update.sh to build.sh

---
 build/local/build.sh              | 316 ++++++++++++++++++++++--------
 build/local/update.sh             | 101 ----------
 build/scripts/build-client-binary |   4 +
 build/scripts/build-client-docker |   6 +-
 build/scripts/build-client-rpm    |   6 +-
 build/scripts/build-client-source |   4 +
 build/scripts/build-client-srpm   |   4 +
 build/scripts/build-server-docker |   9 +-
 build/scripts/build-server-rpm    |   6 +-
 9 files changed, 269 insertions(+), 187 deletions(-)
 delete mode 100755 build/local/update.sh

diff --git a/build/local/build.sh b/build/local/build.sh
index bd174fb965..6e9985e706 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -2,12 +2,116 @@
 
 # Define global variables
 NO_UPDATE=0
+UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+OUTPUT_LOG_FILE="/tmp/build.log"
+
+needs-to-pull() {
+  local UPSTREAM=${1:-'@{u}'}
+  local LOCAL=$(git rev-parse @)
+  local BASE=$(git merge-base @ "$UPSTREAM")
+  local REMOTE=$(git rev-parse "$UPSTREAM")
+
+  if [ "$LOCAL" = "$REMOTE" ]; then
+    return 1 # false, we are up-to-date
+  fi
+
+  if [ "$LOCAL" = "$BASE" ]; then
+    return 0 # true, we are behind upstream
+  fi
+}
+
+rewind() {
+  local DIR="$1"
+  local BRANCH="$2"
+
+  cd "$DIR"
+  CURRENT=$(git branch --show-current)
+  git fetch
+
+  if [ "$CURRENT" != "$BRANCH" ]; then
+    echo "Currently on $CURRENT, checking out $BRANCH"
+    git checkout "$BRANCH"
+  fi
+
+  if needs-to-pull; then
+    git pull origin
+    echo "Submodule has pulled from upstream"
+    git logs -n 2
+    cd - >/dev/null
+    git add "$DIR"
+  else
+    cd - >/dev/null
+    echo "Submodule is up-to-date with upstream"
+  fi
+}
+
+check-files() {
+  local DIR="$1"
+
+  test -z "DIR" && exit 1
+
+  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
+    return 0
+  fi
+
+  return 1
+}
+
+update() {
+  local DEPS=
+  local CURDIR="$PWD"
+  local DIR=pmm-submodules
+
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if check-files "."; then # pwd is pmm-submodules
+    DIR="."
+  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$DIR"; then
+      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      exit 1
+    fi
+  else
+    echo "FATAL: could not locate known files in $PWD"
+    exit 1
+  fi
+
+  cd "$DIR"
+
+  # Join the dependencies from ci-default.yml and ci.yml
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+  echo "$DEPS" | jq -c '.[]' | while read -r item; do
+    branch=$(echo "$item" | jq -r '.branch')
+    path=$(echo "$item" | jq -r '.path')
+    name=$(echo "$item" | jq -r '.name')
+    echo
+    echo "Rewinding submodule '$name' ..."
+    echo "path: ${path}, branch: ${branch}"
+
+    rewind "$path" "$branch"
+  done
+
+  echo
+  echo "Printing git status..."
+  git status --short
+  echo
+  echo "Printing git submodule status..."
+  git submodule status
+
+  cd "$CURDIR" > /dev/null
+}
+
 
 while test "$#" -gt 0; do
   case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
     --no-update)
       NO_UPDATE=1
       ;;
@@ -22,7 +126,7 @@ while test "$#" -gt 0; do
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
       exit 1
       ;;
   esac
@@ -30,8 +134,18 @@ while test "$#" -gt 0; do
 done
 
 if [ "$NO_UPDATE" -eq 0 ]; then
+  MD5SUM=$(md5sum $(dirname $0)/build.sh)
+  
   # Update submodules and PR branches
-  /bin/bash $(dirname $0)/update.sh
+  update
+
+  test "$UPDATE_ONLY" -eq 1 && return
+
+  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+    echo "The updated version of this script has been fetched from the repository, exiting..."
+    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    return
+  fi
 fi
 
 get_branch_name() {
@@ -46,97 +160,141 @@ get_branch_name() {
   echo $branch_name
 }
 
-# Define global variables
-pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-echo $pmm_commit > apiCommitSha
-pmm_branch=$(get_branch_name pmm)
-echo $pmm_branch > apiBranch
-pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-echo $pmm_url > apiURL
-pmm_qa_branch=$(get_branch_name pmm-qa)
-echo $pmm_qa_branch > pmmQABranch
-pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-echo $pmm_qa_commit > pmmQACommitSha
-pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-echo $pmm_ui_tests_branch > pmmUITestBranch
-pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-fb_commit_sha=$(git rev-parse HEAD)
-echo $fb_commit_sha > fbCommitSha
-
-# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-# Important: the docker container's user need to be able to write to these directories.
-# The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-# that the directories we create on the host are owned by a user with same uid and gid.
-
-# Create cache directories. 
-test -d "${root_dir}/go-path" || mkdir -p "go-path"
-test -d "${root_dir}/go-build" || mkdir -p "go-build"
-test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-
-PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+build_with_logs() {
+  local script="$1"
+  local start_time
+  local end_time
+
+  if [ ! -f "$script" ]; then
+    echo "Fatal: script $script does not exist"
+    exit 1
+  fi
+
+  start_time=$(date +%s)
+  if [ "$#" -gt 1 ]; then
+    shift
+    $script "$@" | tee -a $OUTPUT_LOG_FILE
+  else
+    $script | tee -a $OUTPUT_LOG_FILE
+  fi
+  end_time=$(date +%s)
+
+  echo "Execution time for $script: $((end_time - start_time)) seconds"
+}
+
+init() {
+  # Remove the temp directory
+  if [ -d tmp ]; then
+    echo "Removing tmp directory..."
+    rm -rf tmp
+  fi
+  if [ -f "$OUTPUT_LOG_FILE" ]; then
+    echo "Removing the log file..."
+    rm -f $OUTPUT_LOG_FILE
+  fi
+
+  # Define global variables
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  echo $pmm_commit > apiCommitSha
+  pmm_branch=$(get_branch_name pmm)
+  echo $pmm_branch > apiBranch
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  echo $pmm_url > apiURL
+  pmm_qa_branch=$(get_branch_name pmm-qa)
+  echo $pmm_qa_branch > pmmQABranch
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  echo $pmm_qa_commit > pmmQACommitSha
+  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+  fb_commit_sha=$(git rev-parse HEAD)
+  echo $fb_commit_sha > fbCommitSha
+
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+
+  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
+  # Important: the docker container's user needs to be able to write to these directories.
+  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
+  # that the directories we create on the host are owned by a user with the same uid and gid.
+
+  # Create cache directories.
+  test -d "${root_dir}/go-path" || mkdir -p "go-path"
+  test -d "${root_dir}/go-build" || mkdir -p "go-build"
+  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
+  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+
+}
+
+cleanup() {
+  # Clean up temporary files
+  rm -f apiBranch \
+    apiCommitSha \
+    apiURL \
+    fbCommitSha \
+    pmmQABranch \
+    pmmQACommitSha \
+    pmmUITestBranch \
+    pmmUITestsCommitSha
+}
 
 # Local reference test environment
 # CPU: 4 cores
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-if [ "$NO_CLIENT" -eq 0 ]; then
-  # Build client source: 4m39s from scratch, 0m27s using cache
-  "$PATH_TO_SCRIPTS/build-client-source"
+main() {
 
-  # Build client binary: ??? from scratch, 0m20s using cache
-  "$PATH_TO_SCRIPTS/build-client-binary"
+  if [ "$NO_CLIENT" -eq 0 ]; then
+    # Build client source: 4m39s from scratch, 0m27s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
 
-  # Building client source rpm takes 13s (caching does not apply)
-  "$PATH_TO_SCRIPTS/build-client-srpm"
+    # Build client binary: ??? from scratch, 0m20s using cache
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
 
-  # Building client rpm takes 1m40s
-  "$PATH_TO_SCRIPTS/build-client-rpm"
-fi
+    # Building client source rpm takes 13s (caching does not apply)
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
 
-# Building client docker image takes 17s
-GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
-export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
-if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-client-docker"
-fi
+    # Building client rpm takes 1m40s
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+  fi
 
-# Building PMM CLient locally (non-CI, i.e. non-Jenkins)
-# total time: 6m26s - build from scratch, no initial cache
-# total time: 2m49s - subsequent build, using cache from prior builds
+  # Building client docker image takes 17s
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+  export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+  if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+  fi
 
+  # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
+  # total time: 6m26s - build from scratch, no initial cache
+  # total time: 2m49s - subsequent build, using cache from prior builds
 
-# Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
-# total time: 8m45s - build from scratch, no initial cache
-# total time: ??? - subsequent build, using cache from prior builds
 
-export RPM_EPOCH=1
+  # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
+  # total time: 8m45s - build from scratch, no initial cache
+  # total time: ??? - subsequent build, using cache from prior builds
 
-if [ "$NO_SERVER_RPM" -eq 0 ]; then
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-  "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-  "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+  export RPM_EPOCH=1
+  if [ "$NO_SERVER_RPM" -eq 0 ]; then
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
 
-  # 3rd-party
-  "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-  "$PATH_TO_SCRIPTS/build-server-rpm" grafana
-fi
+    # 3rd-party
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
+    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+  fi
+
+  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+  export DOCKERFILE=Dockerfile.el9
+  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+}
 
-export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-export DOCKERFILE=Dockerfile.el9
-${PATH_TO_SCRIPTS}/build-server-docker
-
-# Clean up temporary files
-rm -f apiBranch \
-  apiCommitSha \
-	apiURL \
-	fbCommitSha \
-	pmmQABranch \
-	pmmQACommitSha \
-	pmmUITestBranch \
-	pmmUITestsCommitSha
+init
+main
+cleanup
diff --git a/build/local/update.sh b/build/local/update.sh
deleted file mode 100755
index 4922dd9bda..0000000000
--- a/build/local/update.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash -e
-
-needs-to-pull() {
-  local UPSTREAM=${1:-'@{u}'}
-  local LOCAL=$(git rev-parse @)
-  local BASE=$(git merge-base @ "$UPSTREAM")
-  local REMOTE=$(git rev-parse "$UPSTREAM")
-
-  if [ "$LOCAL" = "$REMOTE" ]; then
-    return 1 # false, we are up-to-date
-  fi
-
-  if [ "$LOCAL" = "$BASE" ]; then
-    return 0 # true, we are behind upstream
-  fi
-}
-
-rewind() {
-  local DIR="$1"
-  local BRANCH="$2"
-
-  cd "$DIR"
-  CURRENT=$(git branch --show-current)
-  git fetch
-
-  if [ "$CURRENT" != "$BRANCH" ]; then
-    echo "Currently on $CURRENT, checking out $BRANCH"
-    git checkout "$BRANCH"
-  fi
-
-  if needs-to-pull; then
-    git pull origin
-    echo "Submodule has pulled from upstream"
-    git logs -n 2
-    cd - >/dev/null
-    git add "$DIR"
-  else
-    cd - >/dev/null
-    echo "Submodule is up-to-date with upstream"
-  fi
-}
-
-check-files() {
-  local DIR="$1"
-
-  test -z "DIR" && exit 1
-
-  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
-    return 0
-  fi
-
-  return 1
-}
-
-main() {
-  local DEPS=
-  local CURDIR="$PWD"
-  local DIR=pmm-submodules
-
-  # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
-      exit 1
-    fi
-  else
-    echo "FATAL: could not locate known files in $PWD"
-    exit 1
-  fi
-
-  cd "$DIR"
-
-  # Join the dependencies from ci-default.yml and ci.yml
-  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
-
-  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
-
-  echo "$DEPS" | jq -c '.[]' | while read -r item; do
-    branch=$(echo "$item" | jq -r '.branch')
-    path=$(echo "$item" | jq -r '.path')
-    name=$(echo "$item" | jq -r '.name')
-    echo
-    echo "Rewinding submodule '$name' ..."
-    echo "path: ${path}, branch: ${branch}"
-
-    rewind "$path" "$branch"
-  done
-
-  echo
-  echo "Printing git status..."
-  git status --short
-  echo
-  echo "Printing git submodule status..."
-  git submodule status
-
-  cd "$CURDIR" > /dev/null
-}
-
-main
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index f580e3f5d8..6b9460baf4 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -108,6 +108,10 @@ copy_component() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client binary files..."
+    echo -----------------------------------------
+
     extract_source_tarball
 
     gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 119a47f7a3..af6cbae9b7 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -6,10 +6,14 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_client_tarball}" ]; then
-    echo skip docker build
+    echo Docker client tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Client..."
+echo -----------------------------------------
+
 DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index ae5780ae24..e11e691ae9 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
@@ -13,7 +17,7 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$(ls /usr/bin/sudo)
+        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 978f09a1b0..a1112413ad 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -59,6 +59,10 @@ prepare_vmagent_tarball() {
 }
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source files..."
+    echo -----------------------------------------
+
     rm -rf ${source_tarball}
     mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 20747d0779..8e4fe8e315 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -6,6 +6,10 @@ set -o errexit
 set -o xtrace
 
 main() {
+    echo -----------------------------------------
+    echo "Building PMM Client source RPM files..."
+    echo -----------------------------------------
+
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
         set -o errexit
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 16027c3beb..e6bffbaa20 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -6,15 +6,16 @@ set -o xtrace
 . $(dirname $0)/vars
 
 if [ -f "${docker_tarball}" ]; then
-    echo skip docker build
+    echo Docker tarball found, skipping docker build...
     exit 0
 fi
 
+echo -----------------------------------------
+echo "Building docker image for PMM Server..."
+echo -----------------------------------------
+
 docker_root=$(realpath ${rpms_dir}/..)
 cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-ls ${docker_root}/ansible
-ls ${docker_root}/ansible/roles/pmm-images
-ls ${docker_root}/ansible/roles/pmm-images/tasks
 cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
 cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index ff5ab06553..4ada6b52d1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -43,7 +43,7 @@ is_build_needed() {
             --region us-east-2 \
             --no-sign-request \
             s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version}
+            ${rpms_dir}/${spec_name}-${rpm_version} || :
     fi
 
     packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
@@ -109,6 +109,10 @@ build() {
         return
     fi
 
+    echo -------------------------------------------------------
+    echo "Building PMM Server RPM for ${spec_name} component..."
+    echo -------------------------------------------------------
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi

From 093179afc763d58df0a9c7445e6fab5ebbbbd6ec Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 11:37:36 +0000
Subject: [PATCH 016/127] PMM-12899 reorganize the script structure

---
 build/local/build.sh | 93 +++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 6e9985e706..e59c9d8174 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -8,6 +8,32 @@ NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
 OUTPUT_LOG_FILE="/tmp/build.log"
 
+while test "$#" -gt 0; do
+  case "$1" in
+    --update-only)
+      UPDATE_ONLY=1; NO_UPDATE=0
+      ;;
+    --no-update)
+      NO_UPDATE=1
+      ;;
+    --no-client)
+      NO_CLIENT=1; NO_CLIENT_DOCKER=1
+      ;;
+    --no-client-docker)
+      NO_CLIENT_DOCKER=1
+      ;;
+    --no-server-rpm)
+      NO_SERVER_RPM=1
+      ;;
+    *)
+      echo "Unknown argument: $1"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      exit 1
+      ;;
+  esac
+  shift
+done
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -106,48 +132,6 @@ update() {
   cd "$CURDIR" > /dev/null
 }
 
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      NO_SERVER_RPM=1
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-if [ "$NO_UPDATE" -eq 0 ]; then
-  MD5SUM=$(md5sum $(dirname $0)/build.sh)
-  
-  # Update submodules and PR branches
-  update
-
-  test "$UPDATE_ONLY" -eq 1 && return
-
-  if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-    echo "The updated version of this script has been fetched from the repository, exiting..."
-    echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
-    return
-  fi
-fi
-
 get_branch_name() {
   local module="${1:-}"
   local branch_name
@@ -224,7 +208,6 @@ init() {
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
   test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
-
 }
 
 cleanup() {
@@ -239,12 +222,21 @@ cleanup() {
     pmmUITestsCommitSha
 }
 
-# Local reference test environment
-# CPU: 4 cores
-# RAM: 16GB
-# OS: Ubuntu 22.04.1 LTS
-
 main() {
+  if [ "$NO_UPDATE" -eq 0 ]; then
+    MD5SUM=$(md5sum $(dirname $0)/build.sh)
+    
+    # Update submodules and PR branches
+    update
+
+    test "$UPDATE_ONLY" -eq 1 && return
+
+    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
+      echo "The updated version of this script has been fetched from the repository, exiting..."
+      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+      return
+    fi
+  fi
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
@@ -295,6 +287,11 @@ main() {
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
 }
 
+# Local reference test environment
+# CPU: 4 cores
+# RAM: 16GB
+# OS: Ubuntu 22.04.1 LTS
+
 init
 main
 cleanup

From 12fd453a88a2898ea97728209be8e7f06b11409c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:20:14 +0000
Subject: [PATCH 017/127] PMM-12899 get rid of `exit 0` in callee scripts

---
 build/local/build.sh                         | 23 +++--
 build/packages/rpm/server/SPECS/grafana.spec |  2 +-
 build/scripts/build-client-binary            |  1 -
 build/scripts/build-client-deb               |  1 -
 build/scripts/build-client-docker            | 76 ++++++++-------
 build/scripts/build-client-rpm               |  1 -
 build/scripts/build-client-sdeb              |  1 -
 build/scripts/build-client-source            |  1 -
 build/scripts/build-client-srpm              |  1 -
 build/scripts/build-server-docker            | 98 ++++++++++----------
 build/scripts/vars                           |  4 +-
 11 files changed, 110 insertions(+), 99 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index e59c9d8174..32c8caa52f 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,7 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
-OUTPUT_LOG_FILE="/tmp/build.log"
+LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -25,9 +25,18 @@ while test "$#" -gt 0; do
     --no-server-rpm)
       NO_SERVER_RPM=1
       ;;
+    --log-file)
+      shift
+      if [ -z "$1" ]; then
+        echo "Missing argument for --log-file"
+        exit 1
+      fi
+      LOG_FILE="$1"
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm]"
+      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo
       exit 1
       ;;
   esac
@@ -150,16 +159,16 @@ build_with_logs() {
   local end_time
 
   if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist"
+    echo "FATAL: script $script does not exist"
     exit 1
   fi
 
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
-    $script "$@" | tee -a $OUTPUT_LOG_FILE
+    $script "$@" | tee -a $LOG_FILE
   else
-    $script | tee -a $OUTPUT_LOG_FILE
+    $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
@@ -172,9 +181,9 @@ init() {
     echo "Removing tmp directory..."
     rm -rf tmp
   fi
-  if [ -f "$OUTPUT_LOG_FILE" ]; then
+  if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
-    rm -f $OUTPUT_LOG_FILE
+    rm -f $LOG_FILE
   fi
 
   # Define global variables
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index e4c3e1558d..94e993bd3c 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -75,7 +75,7 @@ install -d -p %{buildroot}%{_sharedstatedir}/grafana
 %pre
 getent group pmm >/dev/null || echo "Group pmm does not exist. Please create it manually."
 getent passwd pmm >/dev/null || echo "User pmm does not exist. Please create it manually."
-exit 0
+
 
 %changelog
 * Wed Mar 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 10.4.0-105
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 6b9460baf4..c26344bb6c 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -157,6 +157,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 2a94002557..ccedf8e497 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -35,6 +35,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index af6cbae9b7..9494d41f5d 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -5,41 +5,45 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_client_tarball}" ]; then
-    echo Docker client tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Client..."
-echo -----------------------------------------
-
-DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
-
-if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-    DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
-fi
-
-
-CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
-
-docker buildx build \
-    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-    --progress plain \
-    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-    -t ${DOCKER_CLIENT_TAG} \
-    ${DOCKER_FILE_LOCATION}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tag_file})
-    echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
-    docker push ${DOCKER_CLIENT_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_client_tarball})
-    docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
-fi
+main() {
+    echo -----------------------------------------
+    echo "Building docker image for PMM Client..."
+    echo -----------------------------------------
+
+    if [ -f "${docker_client_tarball}" ]; then
+        echo Docker client tarball found, skipping docker build...
+        return
+    fi
+
+    DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
+    cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+
+    if [ -z "${DOCKER_CLIENT_TAG}" ]; then
+        DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
+    fi
+
+
+    CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
+
+    docker buildx build \
+        --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+        --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+        --progress plain \
+        -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+        -t ${DOCKER_CLIENT_TAG} \
+        ${DOCKER_FILE_LOCATION}
+
+    if [ -n "${PUSH_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tag_file})
+        echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
+        docker push ${DOCKER_CLIENT_TAG}
+    fi
+    if [ -n "${SAVE_DOCKER}" ]; then
+        mkdir -p $(dirname ${docker_client_tarball})
+        docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
+    fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index e11e691ae9..6cf2291207 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -29,6 +29,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index f52cff6577..fb6bb50a43 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -32,6 +32,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index a1112413ad..7b0b62fb7a 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -84,6 +84,5 @@ main() {
 }
 
 main
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 8e4fe8e315..ff4671659b 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -28,6 +28,5 @@ main() {
 }
 
 main $*
-exit 0
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index e6bffbaa20..005fbee466 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -5,52 +5,56 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-if [ -f "${docker_tarball}" ]; then
-    echo Docker tarball found, skipping docker build...
-    exit 0
-fi
-
-echo -----------------------------------------
-echo "Building docker image for PMM Server..."
-echo -----------------------------------------
-
-docker_root=$(realpath ${rpms_dir}/..)
-cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
-cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
-
-git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
-ls -la ${rpms_dir}
-docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-    sudo chown -R builder /home/builder/rpm/RPMS
-    until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-        echo "waiting"
-        sleep 1
-    done
-"
-
-if [ -z "${DOCKER_TAG}" ]; then
-    DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
-fi
-
-IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
-
-docker buildx build \
-    --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
-    --build-arg VERSION="$IMAGE_VERSION" \
-    --progress plain \
-    -f ${docker_root}/${docker_file} \
-    -t ${DOCKER_TAG} \
-    ${docker_root}
-
-if [ -n "${PUSH_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tag_file})
-    echo ${DOCKER_TAG} > ${docker_tag_file}
-    docker push ${DOCKER_TAG}
-fi
-if [ -n "${SAVE_DOCKER}" ]; then
-    mkdir -p $(dirname ${docker_tarball})
-    docker save ${DOCKER_TAG} > ${docker_tarball}
-fi
+main() {
+  echo -----------------------------------------
+  echo "Building docker image for PMM Server..."
+  echo -----------------------------------------
+
+  if [ -f "${docker_tarball}" ]; then
+      echo Docker tarball found, skipping docker build...
+      return
+  fi
+
+  docker_root=$(realpath ${rpms_dir}/..)
+  cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
+  cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
+  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
+
+  git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
+  ls -la ${rpms_dir}
+  docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+      sudo chown -R builder /home/builder/rpm/RPMS
+      until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+          echo "waiting"
+          sleep 1
+      done
+  "
+
+  if [ -z "${DOCKER_TAG}" ]; then
+      DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
+  fi
+
+  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+
+  docker buildx build \
+      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg VERSION="$IMAGE_VERSION" \
+      --progress plain \
+      -f ${docker_root}/${docker_file} \
+      -t ${DOCKER_TAG} \
+      ${docker_root}
+
+  if [ -n "${PUSH_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tag_file})
+      echo ${DOCKER_TAG} > ${docker_tag_file}
+      docker push ${DOCKER_TAG}
+  fi
+  if [ -n "${SAVE_DOCKER}" ]; then
+      mkdir -p $(dirname ${docker_tarball})
+      docker save ${DOCKER_TAG} > ${docker_tarball}
+  fi
+}
+
+main
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/vars b/build/scripts/vars
index de57bfb8a4..338e6512e2 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -2,8 +2,8 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
 # NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines
 # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
-root_dir_tmp=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
-root_dir=${ROOT_DIR:-$root_dir_tmp}
+root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
+root_dir=${ROOT_DIR:-$root_dir}
 tmp_dir=${root_dir}/tmp
 
 # In VERSION file we can have numeric value like '2.0.0' as well as

From 9587f21bcf70678c278c53f85c6b3b352ab11506 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 13:39:18 +0000
Subject: [PATCH 018/127] PMM-12899 clean up pmm-update.spec

---
 build/packages/rpm/server/SPECS/pmm-update.spec | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index cb048070c3..5e93dea47a 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -42,9 +42,6 @@ make -C update release
 
 
 %install
-install -d %{buildroot}%{_datadir}/%{name}
-cp -pav ./update/ansible %{buildroot}%{_datadir}/%{name}
-
 install -d %{buildroot}%{_sbindir}
 install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 
@@ -53,7 +50,6 @@ install -p -m 0755 update/bin/pmm-update %{buildroot}%{_sbindir}/
 %license update/LICENSE
 %doc update/README.md
 %{_sbindir}/pmm-update
-%{_datadir}/%{name}
 
 
 %changelog

From 13b4f6517e670e24d1dd906922ec50f586808099 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 14:23:50 +0000
Subject: [PATCH 019/127] PMM-12899 send exec time to the log file

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 32c8caa52f..7f46190b8e 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -172,7 +172,7 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds"
+  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
 }
 
 init() {

From 6f9eac8ea978c432ad15f16a7f9a0525d3f0373b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:07:58 +0000
Subject: [PATCH 020/127] PMM-12899 remove a redundant directory

---
 build/local/build.sh    | 4 +++-
 update/ansible/.gitkeep | 0
 2 files changed, 3 insertions(+), 1 deletion(-)
 delete mode 100644 update/ansible/.gitkeep

diff --git a/build/local/build.sh b/build/local/build.sh
index 7f46190b8e..9dcef31aec 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -157,6 +157,7 @@ build_with_logs() {
   local script="$1"
   local start_time
   local end_time
+  local script_name=$(basename $script)
 
   if [ ! -f "$script" ]; then
     echo "FATAL: script $script does not exist"
@@ -166,13 +167,14 @@ build_with_logs() {
   start_time=$(date +%s)
   if [ "$#" -gt 1 ]; then
     shift
+    script_name="${script_name}:($1)"
     $script "$@" | tee -a $LOG_FILE
   else
     $script | tee -a $LOG_FILE
   fi
   end_time=$(date +%s)
 
-  echo "Execution time for $script: $((end_time - start_time)) seconds" | tee -a $LOG_FILE
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
 }
 
 init() {
diff --git a/update/ansible/.gitkeep b/update/ansible/.gitkeep
deleted file mode 100644
index e69de29bb2..0000000000

From 3eb50f8b04721030c5be7778da3dd595cff496d4 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 17:42:18 +0000
Subject: [PATCH 021/127] PMM-12899 add total exec time

---
 build/local/build.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9dcef31aec..3c156e9349 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
 while test "$#" -gt 0; do
@@ -296,6 +297,9 @@ main() {
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
   build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+
+  echo "Done building PMM artifacts."
+  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
 }
 
 # Local reference test environment

From 36869b7a500ec7190e5af805468d57e50ec12f5f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 2 Apr 2024 23:23:06 +0000
Subject: [PATCH 022/127] PMM-12899 reorg some variables

---
 build/local/build.sh                | 58 +++++++++++++++++------------
 build/scripts/build-client-packages | 24 ++++++------
 build/scripts/build-client-source   |  2 +-
 3 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 3c156e9349..ca1ac4a9dd 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -106,11 +106,11 @@ update() {
     DIR="."
   elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
     if ! check-files "$DIR"; then
-      echo "FATAL: could not locate known files in ${PWD}/${DIR}"
+      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
       exit 1
     fi
   else
-    echo "FATAL: could not locate known files in $PWD"
+    echo "Fatal: could not locate known files in $PWD"
     exit 1
   fi
 
@@ -155,13 +155,13 @@ get_branch_name() {
 }
 
 build_with_logs() {
-  local script="$1"
+  local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
-  local script_name=$(basename $script)
+  local script_name="$1"
 
   if [ ! -f "$script" ]; then
-    echo "FATAL: script $script does not exist"
+    echo "Fatal: script $script does not exist"
     exit 1
   fi
 
@@ -175,14 +175,23 @@ build_with_logs() {
   fi
   end_time=$(date +%s)
 
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee 
+  echo ---
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
+  echo ---
 }
 
 init() {
-  # Remove the temp directory
+  local tmp_files
+  # Remove stale files and directories
   if [ -d tmp ]; then
-    echo "Removing tmp directory..."
-    rm -rf tmp
+    echo "Removing stale files and directories..."
+    if [ -d "tmp/pmm-server" ]; then
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=($tmp_files)
+      for f in "${tmp_files[@]}"; do
+        rm -rf "$f"
+      done
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
@@ -252,23 +261,23 @@ main() {
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-source"
+    build_with_logs build-client-source
 
     # Build client binary: ??? from scratch, 0m20s using cache
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-binary"
+    build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-srpm"
+    build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-rpm"
+    build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-client-docker"
+    build_with_logs build-client-docker
   fi
 
   # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
@@ -282,24 +291,27 @@ main() {
 
   export RPM_EPOCH=1
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm
+    build_with_logs build-server-rpm percona-dashboards grafana-dashboards
+    build_with_logs build-server-rpm pmm-managed pmm
+    build_with_logs build-server-rpm percona-qan-api2 pmm
+    build_with_logs build-server-rpm pmm-update pmm
+    build_with_logs build-server-rpm pmm-dump
+    build_with_logs build-server-rpm vmproxy pmm
 
     # 3rd-party
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics
-    build_with_logs "$PATH_TO_SCRIPTS/build-server-rpm" grafana
+    build_with_logs build-server-rpm victoriametrics
+    build_with_logs build-server-rpm grafana
   fi
 
   export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
   export DOCKERFILE=Dockerfile.el9
-  build_with_logs "${PATH_TO_SCRIPTS}/build-server-docker"
+  build_with_logs build-server-docker
 
+  echo
   echo "Done building PMM artifacts."
+  echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
+  echo ---
 }
 
 # Local reference test environment
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index e7205ae7bb..6f80554bc0 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -174,26 +174,26 @@ get_system(){
 }
 
 install_deps() {
-    local RUN_CMD
+    local PACKAGES
     if [ $INSTALL = 0 ]; then
-        echo "Dependencies will not be installed"
+        echo "Dependencies will not be installed."
         return;
     fi
     if [ ! $( id -u ) -eq 0 ]; then
-        echo "It is not possible to instal dependencies. Please run as root"
+        echo "It is not possible to install dependencies. Please run as root."
         exit 1
     fi
     if [ "$OS" = "rpm" ]; then
         # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
-        command -v git || RUN_CMD+=" git"
-        command -v curl || RUN_CMD+=" curl"
-        command -v wget || RUN_CMD+=" wget"
-        command -v spectool || RUN_CMD+=" rpmdevtools"
-        command -v bison || RUN_CMD+=" bison"
-        command -v repoquery || RUN_CMD+=" yum-utils"
-        command -v rpmbuild || RUN_CMD+=" rpm-build"
-        if [ -n "$RUN_CMD" ]; then
-            yum -y install "$RUN_CMD"
+        command -v git || PACKAGES+=" git"
+        command -v curl || PACKAGES+=" curl"
+        command -v wget || PACKAGES+=" wget"
+        command -v spectool || PACKAGES+=" rpmdevtools"
+        command -v bison || PACKAGES+=" bison"
+        command -v repoquery || PACKAGES+=" yum-utils"
+        command -v rpmbuild || PACKAGES+=" rpm-build"
+        if [ -n "$PACKAGES" ]; then
+            yum -y install "$PACKAGES"
         fi
     else
         apt-get update
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 7b0b62fb7a..0786be9cd1 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -54,7 +54,7 @@ prepare_vmagent_tarball() {
     if [ -f "${tarball}" ]; then
         echo ${tarball} already exists, skipping download
     else
-        curl -o ${tarball} -fSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
+        curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
     fi
 }
 

From bf640caa27e5eb9fb0a0670b643704232d382511 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 00:02:13 +0000
Subject: [PATCH 023/127] PMM-12899 fix the regexp

---
 build/local/build.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index ca1ac4a9dd..999691aade 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -186,12 +186,15 @@ init() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"
       done
     fi
+    if [ -d "tmp/source/pmm" ]; then
+      rm -rf tmp/source/pmm
+    fi
   fi
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."

From f28202aad1f4df24d3cff9ed50f6dfd65d6433d5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 07:50:16 +0000
Subject: [PATCH 024/127] PMM-12899 structure the code

---
 build/local/build.sh | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 999691aade..0779cbf53c 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -180,7 +180,7 @@ build_with_logs() {
   echo ---
 }
 
-init() {
+purge_files() {
   local tmp_files
   # Remove stale files and directories
   if [ -d tmp ]; then
@@ -200,7 +200,9 @@ init() {
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+}
 
+init() {
   # Define global variables
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   echo $pmm_commit > apiCommitSha
@@ -262,6 +264,9 @@ main() {
     fi
   fi
 
+  init
+  purge_files
+
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
     build_with_logs build-client-source
@@ -315,6 +320,8 @@ main() {
   echo ---
   echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
   echo ---
+
+  cleanup
 }
 
 # Local reference test environment
@@ -322,6 +329,4 @@ main() {
 # RAM: 16GB
 # OS: Ubuntu 22.04.1 LTS
 
-init
 main
-cleanup

From a0f7b5011f3661b4597c960fee4ef57386940272 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 08:36:11 +0000
Subject: [PATCH 025/127] PMM-12899 add a flag to skip building server docker

---
 build/local/build.sh | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0779cbf53c..84c071d851 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -6,6 +6,7 @@ UPDATE_ONLY=0
 NO_CLIENT=0
 NO_CLIENT_DOCKER=0
 NO_SERVER_RPM=0
+NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
 
@@ -24,8 +25,19 @@ while test "$#" -gt 0; do
       NO_CLIENT_DOCKER=1
       ;;
     --no-server-rpm)
+      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
       NO_SERVER_RPM=1
       ;;
+    --no-server-docker)
+      if [ "$NO_SERVER_RPM" -eq 1 ]; then
+        echo "Cannot disable both server RPM and server Docker"
+        exit 1
+      fi
+      NO_SERVER_DOCKER=1
+      ;;
     --log-file)
       shift
       if [ -z "$1" ]; then
@@ -311,9 +323,11 @@ main() {
     build_with_logs build-server-rpm grafana
   fi
 
-  export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
-  export DOCKERFILE=Dockerfile.el9
-  build_with_logs build-server-docker
+  if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
+    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKERFILE=Dockerfile.el9
+    build_with_logs build-server-docker
+  fi
 
   echo
   echo "Done building PMM artifacts."

From 988cfb0c32ef52158bfc05898362567171578df8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 3 Apr 2024 09:16:40 +0000
Subject: [PATCH 026/127] PMM-12899 fix a failure of find when no files are
 found

---
 build/local/build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 84c071d851..81cd1cddba 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -198,7 +198,7 @@ purge_files() {
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
     if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$")
+      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
       tmp_files=($tmp_files)
       for f in "${tmp_files[@]}"; do
         rm -rf "$f"

From 0582998db6ff3244ba69fd1335a9ab324b107e68 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 10 Apr 2024 16:05:51 +0300
Subject: [PATCH 027/127] PMM-12899 update the documentation

---
 build/local/README.md | 51 +++++++++++++++++++++++++++++++++++++++----
 build/local/build.sh  | 17 +++++++++------
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 1517e9da3f..5f9051784c 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -12,6 +12,7 @@ The build process is mostly based on bash scripts, which control the build flow.
 
 The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
+
 ## Prerequisites
 
 Below is a list of prerequisites that are required to build PMM locally.
@@ -26,9 +27,51 @@ Below is a list of prerequisites that are required to build PMM locally.
 - yq: 4.42.0+ (tested on 4.42.1)
 - jq: 1.6+ (tested on 1.6)
 
-## Build Steps
+
+## How to build PMM
 
 1. Install the prerequisites
-2. Clone the PMM repository
-3. Change to the `build/local` directory
-4. Run the `build.sh` script
+2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
+3. Change to the `build/local` directory in the cloned repo.
+4. Run `build.sh --help` to print the help message and check the usage.
+5. Run `build.sh` with parameters of your choice to build PMM v3.
+
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build.sh` as often as those changes need to be factored in to the next build.
+
+Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
+
+
+## The `rpmbuild` image and docker cache
+
+We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. It comes provisioned with tools which are required to build PMM artifacts, for example RPM packages. As a build tool, it offers a number of benefits, two most obvious of which are:
+
+- it frees the user from installing dependencies on their host machine
+- it leverages a very powerful docker caching system, which results in reduced build times
+
+During the first run, `build.sh` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+
+
+## Avoiding unnecessary builds
+
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build.sh` script offers several parameters to help control what you want to build.
+
+* --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
+* --update-only: pull changes from the repo without building PMM
+* --no-client: do not build the client, use the cached PMM Client artifacts
+* --no-client-docker: skip building PMM Client docker container
+* --no-server-rpm: skip building PMM Server RPM artifacts
+* --log-file <path>: change the path of the build log file
+
+It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together.
+
+
+## Target environments
+
+Currently, local builds target the following environments:
+- PMM Client
+  - tarball - virtually any amd64 Linux environment
+  - RPM - RHEL9-compatible environments
+  - docker image - docker and Kubernetes environments (amd64)
+- PMM Server
+  - docker image - docker and Kubernetes environments (amd64)
+
diff --git a/build/local/build.sh b/build/local/build.sh
index 81cd1cddba..a821dad1e7 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -9,6 +9,8 @@ NO_SERVER_RPM=0
 NO_SERVER_DOCKER=0
 START_TIME=$(date +%s)
 LOG_FILE="/tmp/build.log"
+BASE_NAME=$(basename $0)
+USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
 
 while test "$#" -gt 0; do
   case "$1" in
@@ -46,9 +48,15 @@ while test "$#" -gt 0; do
       fi
       LOG_FILE="$1"
       ;;
+    --help | -h)
+      shift
+      echo "$USAGE"
+      echo
+      exit 0
+      ;;
     *)
       echo "Unknown argument: $1"
-      echo "Usage: $0 [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>]"
+      echo "$USAGE"
       echo
       exit 1
       ;;
@@ -236,12 +244,7 @@ init() {
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
-  # We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`.
-  # Important: the docker container's user needs to be able to write to these directories.
-  # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure
-  # that the directories we create on the host are owned by a user with the same uid and gid.
-
-  # Create cache directories.
+  # Create cache directories. Read more in the section about `rpmbuild`.
   test -d "${root_dir}/go-path" || mkdir -p "go-path"
   test -d "${root_dir}/go-build" || mkdir -p "go-build"
   test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"

From ea42fe7904ab925058749c62d568e1830e854b98 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 11:34:28 +0300
Subject: [PATCH 028/127] PMM-12899 update README.md

---
 build/local/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 5f9051784c..b775030f66 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -6,11 +6,11 @@ This directory contains a set of scripts aimed at providing a simple way to buil
 
 Historically, PMM used to be built using Jenkins. This worked well for the team, but not for the community. The learning curve was, and still is, rather steep, and it is hard for folks, even internally, to contribute to.
 
-Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to making it easier to build PMM locally.
+Therefore, we decided to make it possible to build PMM locally. This is a work in progress, but we are definitely committed to bring the developer experience to an acceptable level.
 
-The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on to make the build process easy to understand and contribute to. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
+The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
 
-The build process is designed to be run on a Linux host. We believe it can be run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC).
 
 
 ## Prerequisites

From fc248de6e988937af6e7085fe7bfa7b5146e18ef Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 12 Oct 2024 14:20:03 +0300
Subject: [PATCH 029/127] PMM-12899 fix duplicate labels in Dockerfile

---
 build/docker/server/Dockerfile.el9 | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 656ab8615c..9131565bb4 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -33,12 +33,6 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-
 LABEL org.opencontainers.image.created ${BUILD_DATE}
 LABEL org.opencontainers.image.licenses AGPL-3.0
 LABEL org.opencontainers.image.title Percona Monitoring and Management

From 4b469bda90ddc03f14b072020225b5b09157c94b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 15 Oct 2024 00:45:07 +0300
Subject: [PATCH 030/127] PMM-12899 add a config file for all PMM components

---
 build/local/ci-default.yml | 89 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 build/local/ci-default.yml

diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml
new file mode 100644
index 0000000000..c344eeac13
--- /dev/null
+++ b/build/local/ci-default.yml
@@ -0,0 +1,89 @@
+deps:
+  # COMMON
+  - name: pmm
+    branch: v3
+    path: sources/pmm/src/github.com/percona/pmm
+    url: https://github.com/percona/pmm
+    component: common
+
+  # CLIENT
+  - name: node_exporter
+    branch: main
+    path: sources/node_exporter/src/github.com/prometheus/node_exporter
+    url: https://github.com/percona/node_exporter
+    component: client
+
+  - name: mysqld_exporter
+    branch: main
+    path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter
+    url: https://github.com/percona/mysqld_exporter
+    component: client
+
+  - name: mongodb_exporter
+    branch: main
+    path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
+    url: https://github.com/percona/mongodb_exporter
+    component: client
+
+  - name: postgres_exporter
+    branch: main
+    path: sources/postgres_exporter/src/github.com/percona/postgres_exporter
+    url: https://github.com/percona/postgres_exporter
+    component: client
+
+  - name: proxysql_exporter
+    branch: main
+    path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter
+    url: https://github.com/percona/proxysql_exporter
+    component: client
+
+  - name: rds_exporter
+    branch: main
+    path: sources/rds_exporter/src/github.com/percona/rds_exporter
+    url: https://github.com/percona/rds_exporter
+    component: client
+
+  - name: azure_metrics_exporter
+    branch: main
+    path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter
+    url: https://github.com/percona/azure_metrics_exporter
+    component: client
+
+  - name: percona-toolkit
+    branch: release-v3.5.2
+    path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
+    url: https://github.com/percona/percona-toolkit
+    component: client
+
+  # SERVER
+
+  - name: grafana-dashboards
+    branch: v3
+    path: sources/grafana-dashboards
+    url: https://github.com/percona/grafana-dashboards
+    component: server
+
+  - name: grafana
+    branch: v3
+    path: sources/grafana/src/github.com/grafana/grafana
+    url: https://github.com/percona/grafana
+    component: server
+
+  - name: pmm-dump
+    branch: main
+    path: sources/pmm-dump
+    url: https://github.com/percona/pmm-dump
+    component: server
+
+  # QA
+  - name: pmm-qa
+    branch: v3
+    path: sources/pmm-qa/src/github.com/percona/pmm-qa
+    url: https://github.com/percona/pmm-qa
+    component: qa
+
+  - name: pmm-ui-tests
+    branch: v3
+    path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests
+    url: https://github.com/percona/pmm-ui-tests
+    component: qa
\ No newline at end of file

From f35a85f16afa21f964837605f5194f0094f0ed99 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 19 Oct 2024 18:52:16 +0300
Subject: [PATCH 031/127] PMM-12899 simplify init, add usage

---
 build/local/build.sh | 50 +++++++++++++++++++-------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index a821dad1e7..76b0327185 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -50,20 +50,31 @@ while test "$#" -gt 0; do
       ;;
     --help | -h)
       shift
-      echo "$USAGE"
-      echo
+      usage
       exit 0
       ;;
     *)
       echo "Unknown argument: $1"
-      echo "$USAGE"
-      echo
+      usage
       exit 1
       ;;
   esac
   shift
 done
 
+usage() {
+  cat <<-EOF
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+--no-update              Do not fetch the latest changes from the repo
+--update-only            Only fetch the latest changes from the repo
+--no-client              Do not (re)build PMM client
+--no-client-docker       Do not (re)build PMM Client docker image
+--no-server-rpm          Do not (re)build Server RPM packages
+--log-file <path>        Save build logs to a file located at <path>
+--help | -h              Display help
+EOF
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -83,7 +94,7 @@ rewind() {
   local DIR="$1"
   local BRANCH="$2"
 
-  cd "$DIR"
+  cd "$DIR" > /dev/null
   CURRENT=$(git branch --show-current)
   git fetch
 
@@ -96,10 +107,10 @@ rewind() {
     git pull origin
     echo "Submodule has pulled from upstream"
     git logs -n 2
-    cd - >/dev/null
+    cd - > /dev/null
     git add "$DIR"
   else
-    cd - >/dev/null
+    cd - > /dev/null
     echo "Submodule is up-to-date with upstream"
   fi
 }
@@ -223,25 +234,8 @@ purge_files() {
 }
 
 init() {
-  # Define global variables
-  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-  echo $pmm_commit > apiCommitSha
-  pmm_branch=$(get_branch_name pmm)
-  echo $pmm_branch > apiBranch
-  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-  echo $pmm_url > apiURL
-  pmm_qa_branch=$(get_branch_name pmm-qa)
-  echo $pmm_qa_branch > pmmQABranch
-  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-  echo $pmm_qa_commit > pmmQACommitSha
-  pmm_ui_tests_branch=$(get_branch_name pmm-ui-tests)
-  echo $pmm_ui_tests_branch > pmmUITestBranch
-  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-  fb_commit_sha=$(git rev-parse HEAD)
-  echo $fb_commit_sha > fbCommitSha
-
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+  "${PATH_TO_SCRIPTS}/build-submodules"
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
   # Create cache directories. Read more in the section about `rpmbuild`.
@@ -290,10 +284,10 @@ main() {
     build_with_logs build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    build_with_logs build-client-srpm
+    # build_with_logs build-client-srpm
 
     # Building client rpm takes 1m40s
-    build_with_logs build-client-rpm
+    # build_with_logs build-client-rpm
   fi
 
   # Building client docker image takes 17s
@@ -327,7 +321,7 @@ main() {
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
-    export DOCKER_TAG=local/pmm-server:${GIT_COMMIT}
+    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
     build_with_logs build-server-docker
   fi

From 1aad3f74dc3617a2ebecb8734627e2e6d5283f62 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 01:38:07 +0300
Subject: [PATCH 032/127] PMM-12899 fix architecture for the client

---
 build/scripts/build-client-binary | 4 ++++
 build/scripts/build-client-rpm    | 3 +--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 53cb44a8b0..8fdacdd3d8 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -27,6 +27,10 @@ gobuild_component() {
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
 
+    if [ "$(uname -o)" = "Darwin" ]; then
+      local GOARCH=$(uname -m)
+    fi
+
     if [ -d "${root_dir}/go-build" ]; then
         volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
     fi
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 6ec65cc016..517993a868 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -25,14 +25,13 @@ main() {
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        export sudo_path=\$$(command -v sudo)
         mkdir -p /tmp/pmm
         cd /home/builder/results
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_rpm=1
 
-        \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
+        sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
     "
 }
 

From 23aaaf6f3c63970753ed645c22ff0aef899bf10e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 02:44:57 +0300
Subject: [PATCH 033/127] PMM-12899 fix the build time for MacOS

---
 build/local/build.sh              | 234 ++++++++++++++++++------------
 build/scripts/build-server-docker |   5 +-
 2 files changed, 148 insertions(+), 91 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 76b0327185..0811217ca9 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -1,80 +1,86 @@
 #!/bin/bash -e
-
-# Define global variables
-NO_UPDATE=0
-UPDATE_ONLY=0
-NO_CLIENT=0
-NO_CLIENT_DOCKER=0
-NO_SERVER_RPM=0
-NO_SERVER_DOCKER=0
-START_TIME=$(date +%s)
-LOG_FILE="/tmp/build.log"
-BASE_NAME=$(basename $0)
-USAGE="Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]"
-
-while test "$#" -gt 0; do
-  case "$1" in
-    --update-only)
-      UPDATE_ONLY=1; NO_UPDATE=0
-      ;;
-    --no-update)
-      NO_UPDATE=1
-      ;;
-    --no-client)
-      NO_CLIENT=1; NO_CLIENT_DOCKER=1
-      ;;
-    --no-client-docker)
-      NO_CLIENT_DOCKER=1
-      ;;
-    --no-server-rpm)
-      if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_RPM=1
-      ;;
-    --no-server-docker)
-      if [ "$NO_SERVER_RPM" -eq 1 ]; then
-        echo "Cannot disable both server RPM and server Docker"
-        exit 1
-      fi
-      NO_SERVER_DOCKER=1
-      ;;
-    --log-file)
-      shift
-      if [ -z "$1" ]; then
-        echo "Missing argument for --log-file"
-        exit 1
-      fi
-      LOG_FILE="$1"
-      ;;
-    --help | -h)
-      shift
-      usage
-      exit 0
-      ;;
-    *)
-      echo "Unknown argument: $1"
-      usage
-      exit 1
-      ;;
-  esac
-  shift
-done
+set -o errexit
+set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
---no-client              Do not (re)build PMM client
---no-client-docker       Do not (re)build PMM Client docker image
---no-server-rpm          Do not (re)build Server RPM packages
+--no-client              Do not build PMM Client
+--client-docker          Build PMM Client docker image
+--no-server-rpm          Do not build Server RPM packages
+--no-server-docker       Do not build PMM Server docker image
 --log-file <path>        Save build logs to a file located at <path>
 --help | -h              Display help
 EOF
 }
 
+parse-params() {
+  # Define global variables
+  NO_UPDATE=0
+  UPDATE_ONLY=0
+  NO_CLIENT=0
+  NO_CLIENT_DOCKER=1
+  NO_SERVER_RPM=0
+  NO_SERVER_DOCKER=0
+  START_TIME=$(date +%s)
+  LOG_FILE="$(dirname $0)/build.log"
+  BASE_NAME=$(basename $0)
+  SUBMODULES=pmm-submodules
+  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+
+  while test "$#" -gt 0; do
+    case "$1" in
+      --update-only)
+        UPDATE_ONLY=1; NO_UPDATE=0
+        ;;
+      --no-update)
+        NO_UPDATE=1
+        ;;
+      --no-client)
+        NO_CLIENT=1; NO_CLIENT_DOCKER=1
+        ;;
+      --client-docker)
+        NO_CLIENT_DOCKER=0
+        ;;
+      --no-server-rpm)
+        if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_RPM=1
+        ;;
+      --no-server-docker)
+        if [ "$NO_SERVER_RPM" -eq 1 ]; then
+          echo "Error: cannot disable both server RPM and server Docker"
+          exit 1
+        fi
+        NO_SERVER_DOCKER=1
+        ;;
+      --log-file)
+        shift
+        if [ -z "$1" ]; then
+          echo "Missing argument for --log-file"
+          exit 1
+        fi
+        LOG_FILE="$1"
+        ;;
+      --help | -h)
+        shift
+        usage
+        exit 0
+        ;;
+      *)
+        echo "Unknown argument: $1"
+        usage
+        exit 1
+        ;;
+    esac
+    shift
+  done
+}
+
 needs-to-pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
@@ -95,7 +101,7 @@ rewind() {
   local BRANCH="$2"
 
   cd "$DIR" > /dev/null
-  CURRENT=$(git branch --show-current)
+  local CURRENT=$(git branch --show-current)
   git fetch
 
   if [ "$CURRENT" != "$BRANCH" ]; then
@@ -130,14 +136,17 @@ check-files() {
 update() {
   local DEPS=
   local CURDIR="$PWD"
-  local DIR=pmm-submodules
 
   # Thouroughly verify the presence of known files, otherwise bail out
-  if check-files "."; then # pwd is pmm-submodules
-    DIR="."
-  elif [ -d "$DIR" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$DIR"; then
-      echo "Fatal: could not locate known files in ${PWD}/${DIR}"
+  if [ ! -d "$SUBMODULES" ] ; then # pwd must outside of pmm-submodules
+    echo "Warn: the current working directory must be outside of pmm-submodules"
+    echo "cd .."
+    cd .. > /dev/null
+  fi
+
+  if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules
+    if ! check-files "$SUBMODULES"; then
+      echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}"
       exit 1
     fi
   else
@@ -145,7 +154,7 @@ update() {
     exit 1
   fi
 
-  cd "$DIR"
+  cd "$SUBMODULES"
 
   # Join the dependencies from ci-default.yml and ci.yml
   DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
@@ -186,13 +195,17 @@ get_branch_name() {
 }
 
 build_with_logs() {
+  local CURDIR="$PWD"
   local script="$PATH_TO_SCRIPTS/$1"
   local start_time
   local end_time
   local script_name="$1"
 
+  cd "$SUBMODULES" > /dev/null
+
   if [ ! -f "$script" ]; then
     echo "Fatal: script $script does not exist"
+    cd "$CURDIR" > /dev/null
     exit 1
   fi
 
@@ -209,43 +222,82 @@ build_with_logs() {
   echo ---
   echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
   echo ---
+
+  cd "$CURDIR" > /dev/null
 }
 
 purge_files() {
+  local CURDIR=$PWD
   local tmp_files
+
+  cd "$SUBMODULES" > /dev/null
   # Remove stale files and directories
   if [ -d tmp ]; then
     echo "Removing stale files and directories..."
+
     if [ -d "tmp/pmm-server" ]; then
       tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
-      tmp_files=($tmp_files)
-      for f in "${tmp_files[@]}"; do
-        rm -rf "$f"
-      done
+      if [ -n "$tmp_files" ]; then
+        tmp_files=( $tmp_files )
+        for f in "${tmp_files[@]}"; do
+          rm -rf "$f"
+        done
+      fi
     fi
+
     if [ -d "tmp/source/pmm" ]; then
       rm -rf tmp/source/pmm
     fi
   fi
+
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file..."
     rm -f $LOG_FILE
   fi
+
+  cd "$CURDIR"
 }
 
 init() {
-  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-  "${PATH_TO_SCRIPTS}/build-submodules"
+  local CURDIR="$PWD"
+
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
+  if [ -d "$SUBMODULES" ]; then
+    cd "$SUBMODULES" > /dev/null
+  fi
+
+  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
+  pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
+  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
+  pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
+  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
+  pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
+  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
+  fb_commit_sha=$(git rev-parse HEAD)
+
+  echo $fb_commit_sha > fbCommitSha
+  echo $pmm_commit > apiCommitSha
+  echo $pmm_branch > apiBranch
+  echo $pmm_url > apiURL
+  echo $pmm_qa_branch > pmmQABranch
+  echo $pmm_qa_commit > pmmQACommitSha
+  echo $pmm_ui_tests_branch > pmmUITestBranch
+  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
+
   # Create cache directories. Read more in the section about `rpmbuild`.
-  test -d "${root_dir}/go-path" || mkdir -p "go-path"
-  test -d "${root_dir}/go-build" || mkdir -p "go-build"
-  test -d "${root_dir}/yarn-cache" || mkdir -p "yarn-cache"
-  test -d "${root_dir}/yum-cache" || mkdir -p "yum-cache"
+  test -d "go-path" || mkdir -p "go-path"
+  test -d "go-build" || mkdir -p "go-build"
+  test -d "yarn-cache" || mkdir -p "yarn-cache"
+  test -d "yum-cache" || mkdir -p "yum-cache"
+
+  cd "$CURDIR" > /dev/null
 }
 
 cleanup() {
+  local CURDIR="$PWD"
+  cd "$SUBMODULES" > /dev/null
+
   # Clean up temporary files
   rm -f apiBranch \
     apiCommitSha \
@@ -254,13 +306,15 @@ cleanup() {
     pmmQABranch \
     pmmQACommitSha \
     pmmUITestBranch \
-    pmmUITestsCommitSha
+    pmmUITestsCommitSha || :
+
+  cd "$CURDIR" > /dev/null
 }
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
-    
+
     # Update submodules and PR branches
     update
 
@@ -274,6 +328,7 @@ main() {
   fi
 
   init
+
   purge_files
 
   if [ "$NO_CLIENT" -eq 0 ]; then
@@ -301,7 +356,6 @@ main() {
   # total time: 6m26s - build from scratch, no initial cache
   # total time: 2m49s - subsequent build, using cache from prior builds
 
-
   # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
   # total time: 8m45s - build from scratch, no initial cache
   # total time: ??? - subsequent build, using cache from prior builds
@@ -335,9 +389,11 @@ main() {
   cleanup
 }
 
-# Local reference test environment
+# Reference test environment
 # CPU: 4 cores
-# RAM: 16GB
+# RAM: 16 GB
 # OS: Ubuntu 22.04.1 LTS
 
+parse-params "$@"
+
 main
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 005fbee466..dd096f5c31 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -34,10 +34,11 @@ main() {
       DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
   fi
 
-  IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
+  local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@')
 
   docker buildx build \
-      --build-arg BUILD_DATE="$(date --rfc-3339=seconds)" \
+      --build-arg BUILD_DATE="$BUILD_DATE" \
       --build-arg VERSION="$IMAGE_VERSION" \
       --progress plain \
       -f ${docker_root}/${docker_file} \

From c2357c5d4674af6ae05cda4f5cbc831482cab688 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 03:06:33 +0300
Subject: [PATCH 034/127] PMM-12899 fix git commit variable

---
 build/local/build.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 0811217ca9..723945780d 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -267,6 +267,8 @@ init() {
     cd "$SUBMODULES" > /dev/null
   fi
 
+  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
   pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
   pmm_url=$(git config -f .gitmodules submodule.pmm.url)
@@ -313,6 +315,7 @@ cleanup() {
 
 main() {
   if [ "$NO_UPDATE" -eq 0 ]; then
+    local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
     MD5SUM=$(md5sum $(dirname $0)/build.sh)
 
     # Update submodules and PR branches
@@ -320,9 +323,10 @@ main() {
 
     test "$UPDATE_ONLY" -eq 1 && return
 
-    if [ "$MD5SUM" != "$(md5sum $(dirname $0)/build.sh)" ]; then
-      echo "The updated version of this script has been fetched from the repository, exiting..."
-      echo "Please run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+      echo "The local copy of this script differs from the one fetched from the repo." 
+      echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+      echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
       return
     fi
   fi
@@ -346,7 +350,6 @@ main() {
   fi
 
   # Building client docker image takes 17s
-  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     build_with_logs build-client-docker

From 79250792ebf331db5edfc8a36550d98300658736 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 20 Oct 2024 15:28:58 +0300
Subject: [PATCH 035/127] PMM-12899 don't use S3 cache for local builds

---
 build/local/build.sh | 1 +
 build/scripts/vars   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 723945780d..e52e60b3d5 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -364,6 +364,7 @@ main() {
   # total time: ??? - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
+  export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     build_with_logs build-server-rpm percona-dashboards grafana-dashboards
     build_with_logs build-server-rpm pmm-managed pmm
diff --git a/build/scripts/vars b/build/scripts/vars
index 338e6512e2..7a7d15fb3b 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -38,7 +38,7 @@ unset new_pmm_version
 unset new_pmm_release
 echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
 
-rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:2}
+rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}

From 1d8fe377ed61051b77fe33904d1be98a744758ac Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 14:34:27 +0300
Subject: [PATCH 036/127] PMM-12899 dont chown with an id external to the
 container

---
 build/docker/server/Dockerfile.el9 | 1 -
 build/scripts/build-client-binary  | 2 +-
 build/scripts/build-server-rpm     | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 9131565bb4..8f2009bc44 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -4,7 +4,6 @@ ARG VERSION
 ARG BUILD_DATE
 
 ENV LANG=en_US.utf8
-ENV LC_ALL=en_US.utf8
 ENV GF_PLUGIN_DIR=/srv/grafana/plugins
 ENV PS1="[\u@\h \W] # "
 
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8fdacdd3d8..96c7b9598b 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -90,7 +90,7 @@ gobuild_component() {
 
         ${build_command}
 
-        sudo chown -R $(id -u):$(id -g) /home/builder/tmp
+        # sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 9d587fe16d..ba56942b3d 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -56,7 +56,7 @@ prepare_specs() {
     local spec_name=$1
     local repo_name=$2
 
-    sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
+    # sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
     cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return

From 48d481fafc32c8ae49add743683626c823054e9c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 14:35:34 +0300
Subject: [PATCH 037/127] PMM-12899 refactor the log output

---
 build/local/build.sh | 51 ++++++++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index e52e60b3d5..24fdc89490 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -194,17 +194,17 @@ get_branch_name() {
   echo $branch_name
 }
 
-build_with_logs() {
+run_build_script() {
   local CURDIR="$PWD"
   local script="$PATH_TO_SCRIPTS/$1"
+  local script_name="$1"
   local start_time
   local end_time
-  local script_name="$1"
 
   cd "$SUBMODULES" > /dev/null
 
   if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist"
+    echo "Fatal: script $script does not exist."
     cd "$CURDIR" > /dev/null
     exit 1
   fi
@@ -213,14 +213,14 @@ build_with_logs() {
   if [ "$#" -gt 1 ]; then
     shift
     script_name="${script_name}:($1)"
-    $script "$@" | tee -a $LOG_FILE
+    $script "$@"
   else
-    $script | tee -a $LOG_FILE
+    $script
   fi
   end_time=$(date +%s)
 
   echo ---
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))" | tee -a $LOG_FILE
+  echo "Execution time (in sec) for $script_name: $((end_time - start_time))"
   echo ---
 
   cd "$CURDIR" > /dev/null
@@ -240,19 +240,21 @@ purge_files() {
       if [ -n "$tmp_files" ]; then
         tmp_files=( $tmp_files )
         for f in "${tmp_files[@]}"; do
+          echo "Removing file or directory $f ..."
           rm -rf "$f"
         done
       fi
     fi
 
     if [ -d "tmp/source/pmm" ]; then
+      echo "Removing tmp/source/pmm ..."
       rm -rf tmp/source/pmm
     fi
   fi
 
   if [ -f "$LOG_FILE" ]; then
-    echo "Removing the log file..."
-    rm -f $LOG_FILE
+    echo "Removing the log file $LOG_FILE ..."
+    rm -f "$LOG_FILE"
   fi
 
   cd "$CURDIR"
@@ -337,22 +339,22 @@ main() {
 
   if [ "$NO_CLIENT" -eq 0 ]; then
     # Build client source: 4m39s from scratch, 0m27s using cache
-    build_with_logs build-client-source
+    run_build_script build-client-source
 
     # Build client binary: ??? from scratch, 0m20s using cache
-    build_with_logs build-client-binary
+    run_build_script build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    # build_with_logs build-client-srpm
+    # run_build_script build-client-srpm
 
     # Building client rpm takes 1m40s
-    # build_with_logs build-client-rpm
+    # run_build_script build-client-rpm
   fi
 
   # Building client docker image takes 17s
   export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    build_with_logs build-client-docker
+    run_build_script build-client-docker
   fi
 
   # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
@@ -366,28 +368,28 @@ main() {
   export RPM_EPOCH=1
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    build_with_logs build-server-rpm percona-dashboards grafana-dashboards
-    build_with_logs build-server-rpm pmm-managed pmm
-    build_with_logs build-server-rpm percona-qan-api2 pmm
-    build_with_logs build-server-rpm pmm-update pmm
-    build_with_logs build-server-rpm pmm-dump
-    build_with_logs build-server-rpm vmproxy pmm
+    run_build_script build-server-rpm percona-dashboards grafana-dashboards
+    run_build_script build-server-rpm pmm-managed pmm
+    run_build_script build-server-rpm percona-qan-api2 pmm
+    run_build_script build-server-rpm pmm-update pmm
+    run_build_script build-server-rpm pmm-dump
+    run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    build_with_logs build-server-rpm victoriametrics
-    build_with_logs build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
     export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
-    build_with_logs build-server-docker
+    run_build_script build-server-docker
   fi
 
   echo
   echo "Done building PMM artifacts."
   echo ---
-  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))" | tee -a $LOG_FILE
+  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))"
   echo ---
 
   cleanup
@@ -400,4 +402,7 @@ main() {
 
 parse-params "$@"
 
+# Capture the output in the log file
+exec > >(tee -a "$LOG_FILE") 2>&1
+
 main

From e8e3f0f4c25d48503e3468b601b142f9cca66f6f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 15:45:28 +0300
Subject: [PATCH 038/127] PMM-12899 don't chown with builder id outside of
 container

---
 build/scripts/build-server-rpm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index ba56942b3d..a123f2bd0a 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -123,8 +123,6 @@ build() {
         volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
     fi
 
-    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
     echo "repo_name: ${repo_name}"

From 344b783238de2510efe53b30fdf976ee0d5d44d1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 16:26:25 +0300
Subject: [PATCH 039/127] PMM-12899 add aarch to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index f2de9a6970..7a1f0ea221 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -19,7 +19,7 @@ Summary:        Grafana is an open source, feature rich metrics dashboard and gr
 License:        AGPLv3
 URL:            https://github.com/percona/grafana
 Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
-ExclusiveArch:  %{ix86} x86_64 %{arm}
+ExclusiveArch:  %{ix86} x86_64 %{arm} %{aarch}
 
 BuildRequires: fontconfig
 

From 7aa9a307956b545359cfee8427bc4f62976009aa Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 16:39:09 +0300
Subject: [PATCH 040/127] PMM-12899 add aarch64 to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 7a1f0ea221..2c48a7f35e 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -19,7 +19,7 @@ Summary:        Grafana is an open source, feature rich metrics dashboard and gr
 License:        AGPLv3
 URL:            https://github.com/percona/grafana
 Source0:        https://github.com/percona/grafana/archive/%{commit}.tar.gz
-ExclusiveArch:  %{ix86} x86_64 %{arm} %{aarch}
+ExclusiveArch:  %{ix86} x86_64 %{arm} aarch64
 
 BuildRequires: fontconfig
 

From f28ecdeaa2d44a56ac966d99b6da9319693b4d70 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 19:15:06 +0300
Subject: [PATCH 041/127] PMM-12899 add the platform

---
 build/docker/rpmbuild/Dockerfile.el9 |  5 +++--
 build/local/build.sh                 | 10 ++++++++++
 build/scripts/build-client-binary    |  1 +
 build/scripts/build-server-docker    |  2 +-
 build/scripts/build-server-rpm       |  2 +-
 build/scripts/vars                   |  1 +
 6 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 239f73be9c..88373bb787 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -28,8 +28,9 @@ RUN yum install -y gcc gcc-c++ \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION 1.23.2
-ENV GO_RELEASER_VERSION 1.24.0
+ENV GO_VERSION=1.23.2
+ENV GO_RELEASER_VERSION=1.24.0
+ENV NODE_OPTIONS=--max_old_space_size=8000
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
diff --git a/build/local/build.sh b/build/local/build.sh
index 24fdc89490..9298938d3a 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -5,6 +5,7 @@ set -o nounset
 usage() {
   cat <<-EOF
 Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+--platform <platform>    Build for a specific platform (default - linux/amd64)
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
 --no-client              Do not build PMM Client
@@ -27,6 +28,7 @@ parse-params() {
   START_TIME=$(date +%s)
   LOG_FILE="$(dirname $0)/build.log"
   BASE_NAME=$(basename $0)
+  PLATFORM=linux/amd64
   SUBMODULES=pmm-submodules
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 
@@ -58,6 +60,14 @@ parse-params() {
         fi
         NO_SERVER_DOCKER=1
         ;;
+      --platform)
+        shift
+        if [ -z "$1" ]; then
+          echo "Missing argument for --platform"
+          exit 1
+        fi
+        PLATFORM="$1"
+        ;;
       --log-file)
         shift
         if [ -z "$1" ]; then
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 96c7b9598b..a811da16fc 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -59,6 +59,7 @@ gobuild_component() {
     fi
 
     docker run --rm \
+      --platform=${platform}
       ${volume_mounts} \
       ${rpmbuild_docker_image} sh -c "
         set -o errexit
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index dd096f5c31..e3a3953f01 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -22,7 +22,7 @@ main() {
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
-  docker run --rm -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+  docker run --rm --patform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
       sudo chown -R builder /home/builder/rpm/RPMS
       until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
           echo "waiting"
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a123f2bd0a..e9543af810 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -128,7 +128,7 @@ build() {
     echo "repo_name: ${repo_name}"
     echo "rpm_verison: ${rpm_version}"
 
-    docker run --rm ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+    docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
 
diff --git a/build/scripts/vars b/build/scripts/vars
index 7a7d15fb3b..fac887e0f4 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -38,6 +38,7 @@ unset new_pmm_version
 unset new_pmm_release
 echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
 
+platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server

From 87d75a7faccd86e30dedf178c62174e5909cf4ef Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 20:39:36 +0300
Subject: [PATCH 042/127] PMM-12899 use docker volumes for caching

---
 build/local/build.sh                  | 39 ++++++++++++++++-----------
 build/scripts/build-client-binary     |  9 +++----
 build/scripts/build-server-rpm        | 15 ++++-------
 build/scripts/build-server-rpm-script |  2 +-
 4 files changed, 33 insertions(+), 32 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9298938d3a..606c07e2ff 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -261,13 +261,13 @@ purge_files() {
       rm -rf tmp/source/pmm
     fi
   fi
+  
+  cd "$CURDIR"
 
   if [ -f "$LOG_FILE" ]; then
     echo "Removing the log file $LOG_FILE ..."
     rm -f "$LOG_FILE"
   fi
-
-  cd "$CURDIR"
 }
 
 init() {
@@ -299,11 +299,20 @@ init() {
   echo $pmm_ui_tests_branch > pmmUITestBranch
   echo $pmm_ui_tests_commit > pmmUITestsCommitSha
 
-  # Create cache directories. Read more in the section about `rpmbuild`.
-  test -d "go-path" || mkdir -p "go-path"
-  test -d "go-build" || mkdir -p "go-build"
-  test -d "yarn-cache" || mkdir -p "yarn-cache"
-  test -d "yum-cache" || mkdir -p "yum-cache"
+  # Create docker volume to persist package and build cache
+  # Read more in the section about `rpmbuild`.
+  if ! docker volume ls | grep pmm-gobuild; then
+    docker volume create pmm-gobuild
+  fi
+  if ! docker volume ls | grep pmm-gomod; then
+    docker volume create pmm-gomod
+  fi
+  if ! docker volume ls | grep pmm-yarn; then
+    docker volume create pmm-yarn
+  fi
+  if ! docker volume ls | grep pmm-dnf; then
+    docker volume create pmm-dnf
+  fi
 
   cd "$CURDIR" > /dev/null
 }
@@ -373,21 +382,21 @@ main() {
 
   # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
   # total time: 8m45s - build from scratch, no initial cache
-  # total time: ??? - subsequent build, using cache from prior builds
+  # total time: N/A - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
-    run_build_script build-server-rpm pmm-managed pmm
-    run_build_script build-server-rpm percona-qan-api2 pmm
-    run_build_script build-server-rpm pmm-update pmm
-    run_build_script build-server-rpm pmm-dump
-    run_build_script build-server-rpm vmproxy pmm
+    # run_build_script build-server-rpm pmm-managed pmm
+    # run_build_script build-server-rpm percona-qan-api2 pmm
+    # run_build_script build-server-rpm pmm-update pmm
+    # run_build_script build-server-rpm pmm-dump
+    # run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    run_build_script build-server-rpm victoriametrics
-    run_build_script build-server-rpm grafana
+    # run_build_script build-server-rpm victoriametrics
+    # run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index a811da16fc..673b8af401 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -31,12 +31,9 @@ gobuild_component() {
       local GOARCH=$(uname -m)
     fi
 
-    if [ -d "${root_dir}/go-build" ]; then
-        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
-    fi
-    if [ -d "${root_dir}/go-path" ]; then
-        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
-    fi
+    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
         result_file=${result_file/_metrics/}
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index e9543af810..fa46adbf4f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -113,15 +113,10 @@ build() {
     echo "Building PMM Server RPM for ${spec_name} component..."
     echo -------------------------------------------------------
 
-    if [ -d "${root_dir}/go-build" ]; then
-        volume_mounts+=" -v ${root_dir}/go-build:/home/builder/.cache/go-build"
-    fi
-    if [ -d "${root_dir}/yarn-cache" ]; then
-        volume_mounts+=" -v ${root_dir}/yarn-cache:/home/builder/.yarn"
-    fi
-    if [ -d "${root_dir}/go-path" ]; then
-        volume_mounts+=" -v ${root_dir}/go-path:/home/builder/go/pkg/mod"
-    fi
+    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+    volume_mounts+=" -v pmm-yarn:/home/builder/.yarn"
+    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"
@@ -132,7 +127,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache /home/builder/go
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/go
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
index cece634cdc..02a899e8a8 100755
--- a/build/scripts/build-server-rpm-script
+++ b/build/scripts/build-server-rpm-script
@@ -5,7 +5,7 @@ set -o xtrace
 
 . $(dirname $0)/vars
 
-sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache
+sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
 # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
 if [ ${RPM_EPOCH} = 1 ]; then

From 15d9344371684ea37440bd3f9167e4f675da596d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 21:04:59 +0300
Subject: [PATCH 043/127] PMM-12899 exit if the spec cannot be found

---
 build/scripts/build-client-binary | 4 ----
 build/scripts/build-server-rpm    | 7 +++++--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 673b8af401..035e1c15a0 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -65,8 +65,6 @@ gobuild_component() {
         export GOARCH=${GOARCH:-amd64}
         export CGO_ENABLED=0
         export GO111MODULE=auto
-        export GOPATH=/tmp/go
-        export GOMODCACHE=/home/builder/go/pkg/mod
 
         # Setting variable for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
@@ -87,8 +85,6 @@ gobuild_component() {
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
         ${build_command}
-
-        # sudo chown -R $(id -u):$(id -g) /home/builder/tmp
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index fa46adbf4f..28b526f1cb 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -61,6 +61,9 @@ prepare_specs() {
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return
     fi
+    if [ ! -f "${rpmbuild_dir}/SOURCES/${spec_name}.spec" ]; then
+      echo "The spec file could not be found."
+    fi
     
     local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
@@ -127,7 +130,7 @@ build() {
         set -o errexit
         set -o xtrace
 
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/go
+        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then
@@ -146,7 +149,7 @@ build() {
 
         # Only these two specs have build dependencies
         if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sleep 5s
+            sleep 2s
             sudo yum-builddep -y SOURCES/${spec_name}.spec
         fi
 

From 5a8f6e4b533f3c468e4533eb6a6a48bd68658c69 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 22:26:37 +0300
Subject: [PATCH 044/127] PMM-12899 fix yarn cache

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 28b526f1cb..6ce1f8878d 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -118,7 +118,7 @@ build() {
 
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
-    volume_mounts+=" -v pmm-yarn:/home/builder/.yarn"
+    volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     echo "Start building Server RPMs..."

From e326d9491795b1b4db9829b930418ff41d11867b Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 22:33:46 +0300
Subject: [PATCH 045/127] PMM-12899 fix the platform

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 035e1c15a0..b064f6622f 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -56,7 +56,7 @@ gobuild_component() {
     fi
 
     docker run --rm \
-      --platform=${platform}
+      --platform=${platform} \
       ${volume_mounts} \
       ${rpmbuild_docker_image} sh -c "
         set -o errexit

From 9010f426fe778bc8c0a7b1f4c6c3d1f131060144 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:32:21 +0300
Subject: [PATCH 046/127] PMM-12899 chown the cache dirs before starting

---
 build/local/build.sh              |  5 -----
 build/scripts/build-client-binary | 12 +++++++-----
 build/scripts/build-server-docker |  1 +
 build/scripts/build-server-rpm    |  9 +++++++++
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 606c07e2ff..acd9b9e3fc 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -263,11 +263,6 @@ purge_files() {
   fi
   
   cd "$CURDIR"
-
-  if [ -f "$LOG_FILE" ]; then
-    echo "Removing the log file $LOG_FILE ..."
-    rm -f "$LOG_FILE"
-  fi
 }
 
 init() {
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index b064f6622f..56ce6e582f 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -27,10 +27,6 @@ gobuild_component() {
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
 
-    if [ "$(uname -o)" = "Darwin" ]; then
-      local GOARCH=$(uname -m)
-    fi
-
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
@@ -62,7 +58,7 @@ gobuild_component() {
         set -o errexit
         set -o xtrace
         export GOOS=${GOOS:-linux}
-        export GOARCH=${GOARCH:-amd64}
+        export GOARCH=${GOARCH:-$(uname -m)}
         export CGO_ENABLED=0
         export GO111MODULE=auto
 
@@ -71,6 +67,12 @@ gobuild_component() {
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
         export PMM_RELEASE_BRANCH=""
         sudo chown -R builder:builder /home/builder/tmp
+        if [ ! -w /home/builder/.cache/go-build ]; then
+            sudo chown builder /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/go/pkg/mod ]; then
+            sudo chown builder /home/builder/go/pkg/mod
+        fi
 
         export PMM_RELEASE_PATH=${bin_dir}/bin
         if [ "${component}" = vmagent ]; then
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index e3a3953f01..0e4b2556a2 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -40,6 +40,7 @@ main() {
   docker buildx build \
       --build-arg BUILD_DATE="$BUILD_DATE" \
       --build-arg VERSION="$IMAGE_VERSION" \
+      --platform=${platform} \
       --progress plain \
       -f ${docker_root}/${docker_file} \
       -t ${DOCKER_TAG} \
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6ce1f8878d..2c8ff8af42 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,6 +131,15 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+        if [ ! -w /home/builder/.cache/go-build ]; then
+            sudo chown builder /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/go/pkg/mod ]; then
+            sudo chown builder /home/builder/go/pkg/mod
+        fi
+        if [ ! -w pmm-yarn:/home/builder/.cache/yarn ]; then
+            sudo chown builder /home/builder/.cache/yarn
+        fi
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
         if [ ${RPM_EPOCH} = 1 ]; then

From 0ff180fde917b4643dc8e0ce48f74a3c05ef55c3 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:39:35 +0300
Subject: [PATCH 047/127] PMM-12899 remove GOARCH setting

---
 build/scripts/build-client-binary | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 56ce6e582f..9ec917fc08 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -57,8 +57,6 @@ gobuild_component() {
       ${rpmbuild_docker_image} sh -c "
         set -o errexit
         set -o xtrace
-        export GOOS=${GOOS:-linux}
-        export GOARCH=${GOARCH:-$(uname -m)}
         export CGO_ENABLED=0
         export GO111MODULE=auto
 

From c9d8e3522c4d950cbb5bb34e5b3aabd1d281bd55 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 1 Nov 2024 23:43:45 +0300
Subject: [PATCH 048/127] PMM-12899 add --platform to --help

---
 build/local/build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index acd9b9e3fc..f588e1fc75 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,7 +4,7 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
 --platform <platform>    Build for a specific platform (default - linux/amd64)
 --no-update              Do not fetch the latest changes from the repo
 --update-only            Only fetch the latest changes from the repo
@@ -417,6 +417,6 @@ main() {
 parse-params "$@"
 
 # Capture the output in the log file
-exec > >(tee -a "$LOG_FILE") 2>&1
+exec > >(tee "$LOG_FILE") 2>&1
 
 main

From 46c893443ff0e28355e8e00a0db852a451478355 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 2 Nov 2024 00:27:28 +0300
Subject: [PATCH 049/127] PMM-12899 do not upload to S3 when building locally

---
 build/local/build.sh           |  3 ++-
 build/scripts/build-server-rpm | 11 ++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index f588e1fc75..7b8bf328f2 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -19,6 +19,7 @@ EOF
 
 parse-params() {
   # Define global variables
+  LOCAL_BUILD=1
   NO_UPDATE=0
   UPDATE_ONLY=0
   NO_CLIENT=0
@@ -148,7 +149,7 @@ update() {
   local CURDIR="$PWD"
 
   # Thouroughly verify the presence of known files, otherwise bail out
-  if [ ! -d "$SUBMODULES" ] ; then # pwd must outside of pmm-submodules
+  if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules
     echo "Warn: the current working directory must be outside of pmm-submodules"
     echo "cd .."
     cd .. > /dev/null
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 2c8ff8af42..a5a8cbb2d8 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -27,9 +27,7 @@ is_build_needed() {
     local s3_cache_dir=RELEASE
 
     # Structure of S3 build cache
-    # s3://pmm-build-cache/PR-BUILDS/7 - el7
     # s3://pmm-build-cache/PR-BUILDS/9 - el9
-    # s3://pmm-build-cache/RELEASE/7 - el7
     # s3://pmm-build-cache/RELEASE/9 - el9
 
     [ "${FORCE_REBUILD}" = 1 ] && return 0
@@ -38,7 +36,7 @@ is_build_needed() {
     if [ "$RPM_EPOCH" = 1 ]; then
         s3_cache_dir=PR-BUILDS
     fi
-    if command -v aws &> /dev/null; then
+    if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then
         aws s3 sync \
             --region us-east-2 \
             --no-sign-request \
@@ -175,8 +173,15 @@ build() {
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
+    if [ -n "$LOCAL_BUILD" ]; then
+        return
+        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    fi
+
     if ! command -v aws &> /dev/null; then
         echo "AWS CLI is not installed. Upload to S3 will be skipped."
+        echo
+        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
         return
     fi
 

From 10ea625bf6a2365129aec3b7ea291acc092519f8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 2 Nov 2024 00:33:02 +0300
Subject: [PATCH 050/127] PMM-12899 fix a typo

---
 build/scripts/build-server-docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 0e4b2556a2..6cb0c7ed70 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -22,7 +22,7 @@ main() {
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
-  docker run --rm --patform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
+  docker run --rm --platform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
       sudo chown -R builder /home/builder/rpm/RPMS
       until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
           echo "waiting"

From 891a2f0d249aa64bb8e64d5358e4f9e1c533725e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 3 Nov 2024 14:50:16 +0300
Subject: [PATCH 051/127] PMM-12899 clean up scripts

---
 build/local/build.sh                | 24 ++++++++++++------------
 build/scripts/build-client-packages | 14 +++-----------
 build/scripts/build-client-rpm      | 17 +++++------------
 build/scripts/build-client-srpm     | 12 +-----------
 build/scripts/build-server-rpm      | 17 +++++++++--------
 5 files changed, 30 insertions(+), 54 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 7b8bf328f2..9530bf96b6 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -297,16 +297,16 @@ init() {
 
   # Create docker volume to persist package and build cache
   # Read more in the section about `rpmbuild`.
-  if ! docker volume ls | grep pmm-gobuild; then
+  if ! docker volume ls | grep pmm-gobuild >/dev/null; then
     docker volume create pmm-gobuild
   fi
-  if ! docker volume ls | grep pmm-gomod; then
+  if ! docker volume ls | grep pmm-gomod >/dev/null; then
     docker volume create pmm-gomod
   fi
-  if ! docker volume ls | grep pmm-yarn; then
+  if ! docker volume ls | grep pmm-yarn >/dev/null; then
     docker volume create pmm-yarn
   fi
-  if ! docker volume ls | grep pmm-dnf; then
+  if ! docker volume ls | grep pmm-dnf >/dev/null; then
     docker volume create pmm-dnf
   fi
 
@@ -367,7 +367,7 @@ main() {
   fi
 
   # Building client docker image takes 17s
-  export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT}
+  export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker
   fi
@@ -384,15 +384,15 @@ main() {
   export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
-    # run_build_script build-server-rpm pmm-managed pmm
-    # run_build_script build-server-rpm percona-qan-api2 pmm
-    # run_build_script build-server-rpm pmm-update pmm
-    # run_build_script build-server-rpm pmm-dump
-    # run_build_script build-server-rpm vmproxy pmm
+    run_build_script build-server-rpm pmm-managed pmm
+    run_build_script build-server-rpm percona-qan-api2 pmm
+    run_build_script build-server-rpm pmm-update pmm
+    run_build_script build-server-rpm pmm-dump
+    run_build_script build-server-rpm vmproxy pmm
 
     # 3rd-party
-    # run_build_script build-server-rpm victoriametrics
-    # run_build_script build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index bc6e1a0244..f578a422d0 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -30,12 +30,6 @@ append_arg_to_args () {
 }
 
 parse_arguments() {
-    pick_args=
-    if test "$1" = PICK-ARGS-FROM-ARGV; then
-        pick_args=1
-        shift
-    fi
-
     for arg do
         val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
         optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
@@ -51,9 +45,7 @@ parse_arguments() {
             --install_deps=*) INSTALL="$val" ;;
             --help) usage ;;
             *)
-              if test -n "$pick_args"; then
-                  append_arg_to_args "$arg"
-              fi
+                append_arg_to_args "$arg"
               ;;
         esac
     done
@@ -87,7 +79,7 @@ get_repos() {
 }
 
 check_workdir(){
-    if [ "x$WORKDIR" = "x$CURDIR" ]; then
+    if [ "$WORKDIR" = "$CURDIR" ]; then
         echo >&2 "Current directory cannot be used for building!"
         exit 1
     fi
@@ -461,7 +453,7 @@ if [ -n "$pmm_release" ]; then
     DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
-parse_arguments PICK-ARGS-FROM-ARGV "$@"
+parse_arguments "$@"
 get_system
 install_deps
 
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 517993a868..374b79c5ac 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -11,18 +11,13 @@ main() {
     echo -----------------------------------------
 
     local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
+    docker run --rm \
+      -v ${bin_dir}:/home/builder/bin \
+      -v ${root_dir}/results:/home/builder/results \
+      ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
-        if grep -q 'CentOS Linux 7' /etc/os-release; then
-            # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL
-            sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf
-
-            sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo
-            sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo
-        fi
-
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
         mkdir -p /tmp/pmm
@@ -30,9 +25,7 @@ main() {
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_rpm=1
-
-        sudo_path chown -R $(id -u):$(id -g) /home/builder/results/rpm
-    "
+      "
 }
 
 main $*
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index f23a6e24f6..c68b949a33 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -15,23 +15,13 @@ main() {
         set -o errexit
         set -o xtrace
 
-        if grep -q 'CentOS Linux 7' /etc/os-release; then
-            # disable fastestmirror plugin, which mostly fails due to CentOS 7 being EOL
-            sed -i 's/enabled=1/enabled=0/g' /etc/yum/pluginconf.d/fastestmirror.conf
-
-            sed -i -e 's/^\(mirrorlist\)/#\1/g' /etc/yum.repos.d/CentOS-Base.repo
-            sed -i -e 's|^#baseurl.*|baseurl=http://vault.centos.org/centos/\$releasever/os/\$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo
-        fi
-
         export pmm_release=$pmm_release
-        export sudo_path=\$(command -v sudo)
-        [[ ${IMAGE} = ${rpmbuild_docker_image} ]] || \$sudo_path yum -y install git rpm-build
         mkdir -p /tmp/pmm
         cd /home/builder/results
+
         /home/builder/bin/build-client-packages \
             --builddir=/tmp/pmm \
             --build_src_rpm=1
-        \$sudo_path chown -R $(id -u):$(id -g) /home/builder/results/srpm
     "
 }
 
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a5a8cbb2d8..bb94358b23 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -9,7 +9,9 @@ get_rpm_version() {
     local spec_name=$1
 
     local rpm_version=$(
-        docker run --rm -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+        docker run --rm \
+            --platform=${platform} \
+            -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
             rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
                 | sed -re 's/\.[0-9]{10}\././; s/.${rpmbuild_dist}//' \
                 | head -1
@@ -130,13 +132,13 @@ build() {
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
         if [ ! -w /home/builder/.cache/go-build ]; then
-            sudo chown builder /home/builder/.cache/go-build
+            sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then
-            sudo chown builder /home/builder/go/pkg/mod
+            sudo chown builder:builder /home/builder/go/pkg/mod
         fi
-        if [ ! -w pmm-yarn:/home/builder/.cache/yarn ]; then
-            sudo chown builder /home/builder/.cache/yarn
+        if [ ! -w /home/builder/.cache/yarn ]; then
+            sudo chown builder:builder /home/builder/.cache/yarn
         fi
 
         # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
@@ -167,15 +169,14 @@ build() {
                   -ba SOURCES/${spec_name}.spec
 
         rm -f SOURCES/${spec_name}.spec*
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
     "
 
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
-    if [ -n "$LOCAL_BUILD" ]; then
-        return
+    if [ "$LOCAL_BUILD" -ne 1 ]; then
         echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+        return
     fi
 
     if ! command -v aws &> /dev/null; then

From b1933557a050b44b0f57dd3a884c3c7a0cc553b9 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 08:51:24 +0300
Subject: [PATCH 052/127] PMM-12899 revert changes to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 2c48a7f35e..4e3eba1aec 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,8 +35,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-# mkdir -p _build/src
-# export GOPATH="$(pwd)/_build"
+mkdir -p _build/src
+export GOPATH="$(pwd)/_build"
 
 make build-go
 

From 53de2842182760d81a6c4ef7a70a12ca1cf50f85 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 09:12:21 +0300
Subject: [PATCH 053/127] PMM-12899 revert changes to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 4e3eba1aec..781716494d 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,8 +35,6 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
-mkdir -p _build/src
-export GOPATH="$(pwd)/_build"
 
 make build-go
 

From a7ebc00d6822e44817882fe92f13263d09d543e1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 13:53:05 +0300
Subject: [PATCH 054/127] PMM-12899 update goreleaser to latest

---
 build/docker/rpmbuild/Dockerfile.el9 | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 88373bb787..e958731d93 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -28,15 +28,16 @@ RUN yum install -y gcc gcc-c++ \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION=1.23.2
-ENV GO_RELEASER_VERSION=1.24.0
+ARG GO_VERSION
+ENV GO_VERSION=${GO_VERSION:-1.23.2}
+ENV GO_RELEASER_VERSION=2.4.1
 ENV NODE_OPTIONS=--max_old_space_size=8000
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
     curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
     tar -C /usr/local -xzf /tmp/golang.tar.gz && \
-    yum install -y /tmp/goreleaser.rpm && \
+    yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \
     rm /tmp/golang.tar.gz /tmp/goreleaser.rpm
 
 RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0

From 6ea712549f164e230471a636f41d04ddc26563aa Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 16:02:58 +0300
Subject: [PATCH 055/127] PMM-12899 use a different Go version for grafana

---
 build/local/build.sh           | 10 ++++++----
 build/scripts/build-server-rpm |  2 --
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 9530bf96b6..85785cd814 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -269,12 +269,12 @@ purge_files() {
 init() {
   local CURDIR="$PWD"
 
-  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
-
   if [ -d "$SUBMODULES" ]; then
     cd "$SUBMODULES" > /dev/null
   fi
 
+  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 
   pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
@@ -381,8 +381,11 @@ main() {
   # total time: N/A - subsequent build, using cache from prior builds
 
   export RPM_EPOCH=1
-  export FORCE_REBUILD=1 # Don't use S3 cache
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
+    # Grafana build fails to build with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
+    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:go1.22.4
+    run_build_script build-server-rpm grafana
+    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
     run_build_script build-server-rpm pmm-managed pmm
     run_build_script build-server-rpm percona-qan-api2 pmm
@@ -392,7 +395,6 @@ main() {
 
     # 3rd-party
     run_build_script build-server-rpm victoriametrics
-    run_build_script build-server-rpm grafana
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index bb94358b23..eedd6cc34f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -32,8 +32,6 @@ is_build_needed() {
     # s3://pmm-build-cache/PR-BUILDS/9 - el9
     # s3://pmm-build-cache/RELEASE/9 - el9
 
-    [ "${FORCE_REBUILD}" = 1 ] && return 0
-
     # RPM_EPOCH is set to 1 for Feature Builds
     if [ "$RPM_EPOCH" = 1 ]; then
         s3_cache_dir=PR-BUILDS

From f3b7eceed844286aa32058fa18ecd7b9518454b3 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 17:23:47 +0300
Subject: [PATCH 056/127] PMM-12899 fix copying ansible files

---
 build/local/build.sh              | 28 +---------------------------
 build/scripts/build-client-binary |  9 ++++-----
 build/scripts/build-server-docker | 15 ++++-----------
 build/scripts/vars                |  1 -
 4 files changed, 9 insertions(+), 44 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 85785cd814..a89cbbcbcd 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -277,24 +277,6 @@ init() {
 
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 
-  pmm_commit=$(git submodule status | grep 'sources/pmm/src' | awk -F ' ' '{print $1}')
-  pmm_branch=$(git config -f .gitmodules submodule.pmm.branch)
-  pmm_url=$(git config -f .gitmodules submodule.pmm.url)
-  pmm_qa_branch=$(git config -f .gitmodules submodule.pmm-qa.branch)
-  pmm_qa_commit=$(git submodule status | grep 'pmm-qa' | awk -F ' ' '{print $1}')
-  pmm_ui_tests_branch=$(git config -f .gitmodules submodule.pmm-ui-tests.branch)
-  pmm_ui_tests_commit=$(git submodule status | grep 'pmm-ui-tests' | awk -F ' ' '{print $1}')
-  fb_commit_sha=$(git rev-parse HEAD)
-
-  echo $fb_commit_sha > fbCommitSha
-  echo $pmm_commit > apiCommitSha
-  echo $pmm_branch > apiBranch
-  echo $pmm_url > apiURL
-  echo $pmm_qa_branch > pmmQABranch
-  echo $pmm_qa_commit > pmmQACommitSha
-  echo $pmm_ui_tests_branch > pmmUITestBranch
-  echo $pmm_ui_tests_commit > pmmUITestsCommitSha
-
   # Create docker volume to persist package and build cache
   # Read more in the section about `rpmbuild`.
   if ! docker volume ls | grep pmm-gobuild >/dev/null; then
@@ -317,15 +299,7 @@ cleanup() {
   local CURDIR="$PWD"
   cd "$SUBMODULES" > /dev/null
 
-  # Clean up temporary files
-  rm -f apiBranch \
-    apiCommitSha \
-    apiURL \
-    fbCommitSha \
-    pmmQABranch \
-    pmmQACommitSha \
-    pmmUITestBranch \
-    pmmUITestsCommitSha || :
+  # Implement cleanup logic here
 
   cd "$CURDIR" > /dev/null
 }
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 9ec917fc08..d2dbaac338 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -21,12 +21,11 @@ gobuild_component() {
     local extract_path=${3:-"github.com/percona/$parent_component"}
     local component_path=$4
     local result_file=${5:-bin/$(basename ${component_path:-$extract_path})}
-    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
     local src_dir=/home/builder/tmp/source/pmm-client-${pmm_version}
     local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
-
+    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
@@ -60,10 +59,11 @@ gobuild_component() {
         export CGO_ENABLED=0
         export GO111MODULE=auto
 
-        # Setting variable for 'make release'
+        # Setting variables for 'make release'
         export PMM_RELEASE_VERSION=${full_pmm_version}
         export PMM_RELEASE_TIMESTAMP=$(date '+%s')
-        export PMM_RELEASE_BRANCH=""
+        export PMM_RELEASE_PATH=${bin_dir}/bin
+
         sudo chown -R builder:builder /home/builder/tmp
         if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder /home/builder/.cache/go-build
@@ -72,7 +72,6 @@ gobuild_component() {
             sudo chown builder /home/builder/go/pkg/mod
         fi
 
-        export PMM_RELEASE_PATH=${bin_dir}/bin
         if [ "${component}" = vmagent ]; then
             export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
         else
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 6cb0c7ed70..ecf27b081b 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -10,14 +10,11 @@ main() {
   echo "Building docker image for PMM Server..."
   echo -----------------------------------------
 
-  if [ -f "${docker_tarball}" ]; then
-      echo Docker tarball found, skipping docker build...
-      return
-  fi
+  local docker_root=$(realpath ${rpms_dir}/..)
+  local pmm_source_dir=$(git config -f .gitmodules submodule.pmm.path)
 
-  docker_root=$(realpath ${rpms_dir}/..)
-  cp -r ${root_dir}/tmp/source/pmm/build/ansible ${docker_root}/ansible
-  cp ${root_dir}/tmp/source/pmm/build/docker/server/* ${docker_root}/
+  cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
+  cp "${root_dir}/${pmm_source_dir}/build/docker/server/*" "${docker_root}"
   cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
@@ -51,10 +48,6 @@ main() {
       echo ${DOCKER_TAG} > ${docker_tag_file}
       docker push ${DOCKER_TAG}
   fi
-  if [ -n "${SAVE_DOCKER}" ]; then
-      mkdir -p $(dirname ${docker_tarball})
-      docker save ${DOCKER_TAG} > ${docker_tarball}
-  fi
 }
 
 main
diff --git a/build/scripts/vars b/build/scripts/vars
index fac887e0f4..039e75b215 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -49,7 +49,6 @@ client_properties=${root_dir}/results/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/results/docker/TAG
 docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG
-docker_tarball=${root_dir}/results/docker/pmm-server-${pmm_version}.docker
 docker_client_tarball=${root_dir}/results/docker/pmm-client-${pmm_version}.docker
 source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz
 binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz

From 3293ab58eaef93f6d62c74599c72a564cce454fe Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 17:47:44 +0300
Subject: [PATCH 057/127] PMM-12899 fix copying Dockerfile

---
 build/scripts/build-server-docker | 4 ++--
 build/scripts/build-server-rpm    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index ecf27b081b..f1df62ac72 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -14,8 +14,8 @@ main() {
   local pmm_source_dir=$(git config -f .gitmodules submodule.pmm.path)
 
   cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
-  cp "${root_dir}/${pmm_source_dir}/build/docker/server/*" "${docker_root}"
-  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${docker_root}/pmm-client.tar.gz
+  cp "${root_dir}/${pmm_source_dir}/build/docker/server/"* "${docker_root}"
+  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index eedd6cc34f..5e69e9eba1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -172,7 +172,7 @@ build() {
     # TODO: see if this is even needed
     # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
-    if [ "$LOCAL_BUILD" -ne 1 ]; then
+    if [ "$LOCAL_BUILD" = 1 ]; then
         echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
         return
     fi

From d501abc35119fd80740b3e48c55942ed8f0ea642 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 4 Nov 2024 18:01:26 +0300
Subject: [PATCH 058/127] PMM-12899 fix legacy Docker label format

---
 build/docker/server/Dockerfile.el9 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 8f2009bc44..4beaaae520 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -32,13 +32,13 @@ RUN install -T -p -m 644 /opt/ansible/ansible.cfg /etc/ansible/ansible.cfg && \
     ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm/post-build-actions.yml && \
     sed -i '/^assumeyes/d' /etc/dnf/dnf.conf
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses AGPL-3.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management
-LABEL org.opencontainers.image.vendor Percona LLC
-LABEL org.opencontainers.image.version ${VERSION}
-LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
-LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
+LABEL org.opencontainers.image.created=${BUILD_DATE}
+LABEL org.opencontainers.image.licenses=AGPL-3.0
+LABEL org.opencontainers.image.title="Percona Monitoring and Management"
+LABEL org.opencontainers.image.vendor="Percona LLC"
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL org.opencontainers.image.sourc=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
 
 USER pmm
 

From 15be81593df33e273f933ba467d959e31c6f07a2 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 00:20:30 +0300
Subject: [PATCH 059/127] PMM-12899 do not use the repo subpaths for mutations

---
 build/docker/server/Dockerfile.el9 | 2 +-
 build/scripts/vars                 | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9
index 4beaaae520..5fc8efc221 100644
--- a/build/docker/server/Dockerfile.el9
+++ b/build/docker/server/Dockerfile.el9
@@ -37,7 +37,7 @@ LABEL org.opencontainers.image.licenses=AGPL-3.0
 LABEL org.opencontainers.image.title="Percona Monitoring and Management"
 LABEL org.opencontainers.image.vendor="Percona LLC"
 LABEL org.opencontainers.image.version=${VERSION}
-LABEL org.opencontainers.image.sourc=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
+LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/server/Dockerfile.el9
 LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/server/docker.html
 
 USER pmm
diff --git a/build/scripts/vars b/build/scripts/vars
index 039e75b215..3689f05476 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -36,12 +36,12 @@ pmm_version=${new_pmm_version}
 pmm_release=${new_pmm_release}
 unset new_pmm_version
 unset new_pmm_release
-echo -e "\n\n\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release}\n\n\n"
+echo -e "\n\---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n\---\n"
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
-rpmbuild_dir=${root_dir}/sources/pmm/src/github.com/percona/pmm/build/packages/rpm/server
+rpmbuild_dir=${root_dir}/tmp/pmm-server/SOURCES
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From eeccd8af9fb877e2c193c6821796618904e7b8f1 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 01:41:13 +0300
Subject: [PATCH 060/127] PMM-12899 use a dedicated spec directory

---
 build/scripts/build-server     |  2 +-
 build/scripts/build-server-rpm | 35 +++++++++++++++++-----------------
 build/scripts/vars             |  5 +++--
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/build/scripts/build-server b/build/scripts/build-server
index 7a2b053be9..89e2262622 100755
--- a/build/scripts/build-server
+++ b/build/scripts/build-server
@@ -10,7 +10,7 @@ set -o xtrace
 pmm_version=${pmm_version} ${bin_dir}/build-client
 pmm_version=${pmm_version} ${bin_dir}/build-client-srpm
 pmm_version=${pmm_version} ${bin_dir}/build-client-rpm
-mkdir -p ${rpms_dir}/
+mkdir -p ${rpms_dir} ${rpmbuild_dir}
 cp ${root_dir}/results/rpm/*.rpm ${rpms_dir}/
 
 # install all rpms
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 5e69e9eba1..6c62dff34b 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -11,10 +11,9 @@ get_rpm_version() {
     local rpm_version=$(
         docker run --rm \
             --platform=${platform} \
-            -v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+            -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
             rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
-                | sed -re 's/\.[0-9]{10}\././; s/.${rpmbuild_dist}//' \
-                | head -1
+                | sed -re 's/\.[0-9]{10}\././' | head -1
         "
     )
 
@@ -46,52 +45,52 @@ is_build_needed() {
 
     packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
 
-    # return result as true or false
+    # return result as true (0) or false (>0)
     [[ ${packages// /} == 0 ]]
 }
 
 prepare_specs() {
     local spec_name=$1
     local repo_name=$2
+    local spec_file=${rpmbuild_dir}/${spec_name}.spec
 
-    # sudo chown -R $(id -u):$(id -g) ${rpmbuild_dir}/SPECS ${rpmbuild_dir}/SOURCES
-    cp ${rpmbuild_dir}/SPECS/${spec_name}.spec ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return
     fi
-    if [ ! -f "${rpmbuild_dir}/SOURCES/${spec_name}.spec" ]; then
-      echo "The spec file could not be found."
+    if [ ! -f "${spec_file}" ]; then
+      echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type -d | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 
-    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
     if [ "${spec_name}" != "grafana" ]; then
-        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${spec_file}
     fi
 
     if [ -z "${full_pmm_version}" ]; then
         echo 'The full_pmm_version is not specified.'
         exit 1
     fi
-    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${spec_file}
 
     if [ -n "$pmm_release" ]; then
-        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${rpmbuild_dir}/SOURCES/${spec_name}.spec
-        grep -r 'define release' ${rpmbuild_dir}/SOURCES/${spec_name}.spec
+        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${spec_file}
+        grep -r 'define release' ${spec_file}
     fi
 
-    if [ -f "${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz" ]; then
-        echo SOURCES/${repo_name}-${short_commit}.tar.gz already exists, skip archiving...
+    if [ -f "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" ]; then
+        echo "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" already exists, skip archiving...
         return
     fi
 
     git -C "${git_dir}" archive \
         --format=tar.gz \
         --prefix=${repo_name}-${full_commit}/ \
-        -o ${rpmbuild_dir}/SOURCES/${repo_name}-${short_commit}.tar.gz \
+        -o ${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz \
         "${full_commit}"
 }
 
@@ -104,7 +103,6 @@ build() {
     local CH_VERSION=${rpm_version%-*}
     local CH_TAG=${rpm_version#*-}
     local s3_cache_dir=RELEASE
-    local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
 
     if ! is_build_needed "${spec_name}" "${rpm_version}"; then
         return
@@ -114,6 +112,7 @@ build() {
     echo "Building PMM Server RPM for ${spec_name} component..."
     echo -------------------------------------------------------
 
+    local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
diff --git a/build/scripts/vars b/build/scripts/vars
index 3689f05476..e271e36763 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -36,12 +36,13 @@ pmm_version=${new_pmm_version}
 pmm_release=${new_pmm_release}
 unset new_pmm_version
 unset new_pmm_release
-echo -e "\n\---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n\---\n"
+echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_version} pmm_release=${pmm_release} <<<\n---\n"
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
-rpmbuild_dir=${root_dir}/tmp/pmm-server/SOURCES
+rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD
+rpmspec_dir=${root_dir}/$(git config -f .gitmodules submodule.${module}.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From 8c8bf03db56adeec5dadf59d135bbbe86914c100 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 01:56:39 +0300
Subject: [PATCH 061/127] PMM-12899 fix the spec directory evaluation

---
 build/scripts/build-client-deb  | 1 -
 build/scripts/build-client-sdeb | 1 -
 build/scripts/vars              | 2 +-
 3 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index f307f4971a..72deb8a62e 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -6,7 +6,6 @@ set -o errexit
 set -o xtrace
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
 
 main() {
     local IMAGE=${1:-ubuntu:focal}
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index fb6bb50a43..2429d5de8d 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -6,7 +6,6 @@ set -o errexit
 set -o xtrace
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-tmp_dir=${root_dir}/tmp
 
 main() {
     local IMAGE=${1:-ubuntu:focal}
diff --git a/build/scripts/vars b/build/scripts/vars
index e271e36763..d1c0ffe67a 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -42,7 +42,7 @@ platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 rpms_dir=${root_dir}/tmp/pmm-server/RPMS
 rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD
-rpmspec_dir=${root_dir}/$(git config -f .gitmodules submodule.${module}.path)/build/packages/rpm/server/SPECS
+rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}

From 4aa5012fd8d831a4e904834d4ee94618a334bc3f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:12:53 +0300
Subject: [PATCH 062/127] PMM-12899 create rpms directories first

---
 build/local/build.sh           | 16 ++++++++--------
 build/scripts/build-server-rpm |  1 +
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index a89cbbcbcd..09d9079cc7 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -195,14 +195,14 @@ update() {
 
 get_branch_name() {
   local module="${1:-}"
-  local branch_name
-  local path
-
-  path=$(git config -f .gitmodules submodule.${module}.path)
-  cd "$path" || exit 1
-  branch_name=$(git branch --show-current)
-  cd - > /dev/null
-  echo $branch_name
+  local path=$(git config -f .gitmodules submodule.${module}.path)
+
+  if [ ! -d "$path" ]; then
+    echo "Error: could not resolve the path to submodule ${module}"
+    exit 1
+  fi
+
+  echo $(git -C "$path" branch --show-current)
 }
 
 run_build_script() {
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6c62dff34b..dce2d7f068 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -54,6 +54,7 @@ prepare_specs() {
     local repo_name=$2
     local spec_file=${rpmbuild_dir}/${spec_name}.spec
 
+    mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
     cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
     if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
         return

From eaaeaf23e1e827ba291e0e476677cbd5c4e416fb Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:30:11 +0300
Subject: [PATCH 063/127] PMM-12899 fix find syntax

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index dce2d7f068..a5abdae5e5 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -63,7 +63,7 @@ prepare_specs() {
       echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type -d | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type d | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 

From 2325c5312e4ecefd75dbbd3966a6e4b71cade123 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 02:38:31 +0300
Subject: [PATCH 064/127] PMM-12899 fix find syntax to search for both files
 and dirs

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index a5abdae5e5..6fb85d34c4 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -63,7 +63,7 @@ prepare_specs() {
       echo "Warning: the spec file could not be found."
     fi
     
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git -type d | head -1))
+    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
     local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
     local short_commit=${full_commit:0:7}
 

From dc6caddb76d6e1af619a2fef039e4f17e4cf6690 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 03:12:24 +0300
Subject: [PATCH 065/127] PMM-12899 revert changes made to grafana.spec

---
 build/packages/rpm/server/SPECS/grafana.spec | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 781716494d..4e3eba1aec 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -35,6 +35,8 @@ sed -i "s/unknown-dev/%{grafana_version}/" pkg/build/git.go
 sudo npm install -g grunt-cli
 
 %build
+mkdir -p _build/src
+export GOPATH="$(pwd)/_build"
 
 make build-go
 

From 2819fc9a8634eb184f2b7c13949be2c0064d5d7f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 09:58:51 +0300
Subject: [PATCH 066/127] PMM-12899 try not to use cache for grafana

---
 build/scripts/build-server-rpm | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 6fb85d34c4..f717359a1e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -114,10 +114,12 @@ build() {
     echo -------------------------------------------------------
 
     local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
-    volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+    if [ ${spec_name} != "grafana" ]; then
+        volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+        volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+    fi
 
     echo "Start building Server RPMs..."
     echo "spec_name: ${spec_name}"

From cd0df5c1486006f6a77c4cb63802cb87e708ee39 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 10:03:34 +0300
Subject: [PATCH 067/127] PMM-12899 try not to use cache for grafana

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index f717359a1e..1dc65beaad 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,7 +131,7 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        if [ ! -w /home/builder/.cache/go-build ]; then
+        if [ -d /home/builder/.cache/go-build ] && [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then

From f5dc7453fe813ea297be7f72b90a8466c2c6349d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:10:04 +0300
Subject: [PATCH 068/127] PMM-12899 fix the cache directory not being writable

---
 build/scripts/build-server-rpm | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 1dc65beaad..c429e57550 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -131,7 +131,11 @@ build() {
         set -o xtrace
 
         sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        if [ -d /home/builder/.cache/go-build ] && [ ! -w /home/builder/.cache/go-build ]; then
+        sudo chown builder:builder /home/builder/.cache
+        if [ ! -d /home/builder/.cache/go-build ]; then
+            mkdir -p /home/builder/.cache/go-build
+        fi
+        if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then

From a28edba5b7068c84f06486f54e5a2f6cdd0d7303 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:24:31 +0300
Subject: [PATCH 069/127] PMM-12899 use no Go cache for grafana

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index c429e57550..fb0ddf27e4 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -114,9 +114,9 @@ build() {
     echo -------------------------------------------------------
 
     local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
     if [ ${spec_name} != "grafana" ]; then
+        volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
         volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
         volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
     fi

From 44fc04b8817b883b74033b40d1ce595be8b29a1d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 11:31:26 +0300
Subject: [PATCH 070/127] PMM-12899 use no Go cache for grafana

---
 build/scripts/build-server-rpm | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index fb0ddf27e4..2cb55dde8c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -135,6 +135,9 @@ build() {
         if [ ! -d /home/builder/.cache/go-build ]; then
             mkdir -p /home/builder/.cache/go-build
         fi
+        if [ ! -d /home/builder/go ]; then
+            mkdir -p /home/builder/go/pkg/mod
+        fi        
         if [ ! -w /home/builder/.cache/go-build ]; then
             sudo chown builder:builder /home/builder/.cache/go-build
         fi

From e1fbecab5c80a19340bc56b79ae55f3a13cd19ab Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 20:26:29 +0300
Subject: [PATCH 071/127] PMM-12899 remove docker tarballs

---
 build/local/build.sh              | 49 ++++++++++++-------------------
 build/scripts/build-client-docker | 10 -------
 build/scripts/vars                |  1 -
 3 files changed, 19 insertions(+), 41 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 09d9079cc7..4c57936d9b 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,16 +4,17 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
---platform <platform>    Build for a specific platform (default - linux/amd64)
---no-update              Do not fetch the latest changes from the repo
---update-only            Only fetch the latest changes from the repo
---no-client              Do not build PMM Client
---client-docker          Build PMM Client docker image
---no-server-rpm          Do not build Server RPM packages
---no-server-docker       Do not build PMM Server docker image
---log-file <path>        Save build logs to a file located at <path>
---help | -h              Display help
+Usage: $BASE_NAME [OPTIONS] [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Options:
+      --platform <platform>    Build for a specific platform (defaults to linux/amd64)
+      --no-update              Do not fetch the latest changes from the repo
+      --update-only            Only fetch the latest changes from the repo
+      --no-client              Do not build PMM Client
+      --client-docker          Build PMM Client docker image
+      --no-server-rpm          Do not build Server RPM packages
+      --no-server-docker       Do not build PMM Server docker image
+      --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
+      --help | -h              Display help
 EOF
 }
 
@@ -346,29 +347,22 @@ main() {
     run_build_script build-client-docker
   fi
 
-  # Building PMM CLient locally (non-CI, i.e. non-Jenkins)
-  # total time: 6m26s - build from scratch, no initial cache
-  # total time: 2m49s - subsequent build, using cache from prior builds
-
-  # Building PMM CLient in a CI environment, i.e. Jenkins running on AWS
-  # total time: 8m45s - build from scratch, no initial cache
-  # total time: N/A - subsequent build, using cache from prior builds
-
   export RPM_EPOCH=1
   if [ "$NO_SERVER_RPM" -eq 0 ]; then
-    # Grafana build fails to build with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
-    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:go1.22.4
-    run_build_script build-server-rpm grafana
+    # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
+    # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it
     export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+    # 3rd-party components
+    run_build_script build-server-rpm grafana
+    run_build_script build-server-rpm victoriametrics
+
+    # 1st-party components
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
     run_build_script build-server-rpm pmm-managed pmm
     run_build_script build-server-rpm percona-qan-api2 pmm
     run_build_script build-server-rpm pmm-update pmm
     run_build_script build-server-rpm pmm-dump
     run_build_script build-server-rpm vmproxy pmm
-
-    # 3rd-party
-    run_build_script build-server-rpm victoriametrics
   fi
 
   if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
@@ -386,14 +380,9 @@ main() {
   cleanup
 }
 
-# Reference test environment
-# CPU: 4 cores
-# RAM: 16 GB
-# OS: Ubuntu 22.04.1 LTS
-
 parse-params "$@"
 
-# Capture the output in the log file
+# Capture the build logs in the log file
 exec > >(tee "$LOG_FILE") 2>&1
 
 main
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 9494d41f5d..a564c5c4cf 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -10,11 +10,6 @@ main() {
     echo "Building docker image for PMM Client..."
     echo -----------------------------------------
 
-    if [ -f "${docker_client_tarball}" ]; then
-        echo Docker client tarball found, skipping docker build...
-        return
-    fi
-
     DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
     cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
@@ -22,7 +17,6 @@ main() {
         DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
     fi
 
-
     CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
     docker buildx build \
@@ -38,10 +32,6 @@ main() {
         echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
         docker push ${DOCKER_CLIENT_TAG}
     fi
-    if [ -n "${SAVE_DOCKER}" ]; then
-        mkdir -p $(dirname ${docker_client_tarball})
-        docker save ${DOCKER_CLIENT_TAG} | xz > ${docker_client_tarball}
-    fi
 }
 
 main
diff --git a/build/scripts/vars b/build/scripts/vars
index d1c0ffe67a..4c6a25f273 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -50,7 +50,6 @@ client_properties=${root_dir}/results/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/results/docker/TAG
 docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG
-docker_client_tarball=${root_dir}/results/docker/pmm-client-${pmm_version}.docker
 source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz
 binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz
 

From c94b549e4298dc8ea352dc640f5e89d628fa1f46 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 21:45:50 +0300
Subject: [PATCH 072/127] PMM-12899 add --client-only parameter

---
 build/local/build.sh | 83 +++++++++++++++++++++++---------------------
 1 file changed, 44 insertions(+), 39 deletions(-)

diff --git a/build/local/build.sh b/build/local/build.sh
index 4c57936d9b..dd96c41418 100755
--- a/build/local/build.sh
+++ b/build/local/build.sh
@@ -4,15 +4,15 @@ set -o nounset
 
 usage() {
   cat <<-EOF
-Usage: $BASE_NAME [OPTIONS] [--platform] [--no-update | --update-only] [--no-client] [--no-client-docker] [--no-server-rpm] [--no-server-docker] [--log-file <path>] [--help | -h]
+Usage: $BASE_NAME [OPTIONS]
 Options:
       --platform <platform>    Build for a specific platform (defaults to linux/amd64)
       --no-update              Do not fetch the latest changes from the repo
       --update-only            Only fetch the latest changes from the repo
       --no-client              Do not build PMM Client
-      --client-docker          Build PMM Client docker image
-      --no-server-rpm          Do not build Server RPM packages
-      --no-server-docker       Do not build PMM Server docker image
+      --no-client-docker       Do not build PMM Client docker image (default)
+      --client-only            Build only PMM Client (client binaries + docker)
+      --no-server              Do not build PMM Server (docker image)
       --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
       --help | -h              Display help
 EOF
@@ -25,8 +25,7 @@ parse-params() {
   UPDATE_ONLY=0
   NO_CLIENT=0
   NO_CLIENT_DOCKER=1
-  NO_SERVER_RPM=0
-  NO_SERVER_DOCKER=0
+  NO_SERVER=0
   START_TIME=$(date +%s)
   LOG_FILE="$(dirname $0)/build.log"
   BASE_NAME=$(basename $0)
@@ -40,27 +39,34 @@ parse-params() {
         UPDATE_ONLY=1; NO_UPDATE=0
         ;;
       --no-update)
+        if [ "$UPDATE_ONLY" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --update-only and --no-update"
+          exit 1
+        fi      
         NO_UPDATE=1
         ;;
+      --client-only)
+        NO_CLIENT=0; NO_CLIENT_DOCKER=0; NO_SERVER=1
+        ;;
       --no-client)
         NO_CLIENT=1; NO_CLIENT_DOCKER=1
         ;;
       --client-docker)
         NO_CLIENT_DOCKER=0
         ;;
-      --no-server-rpm)
-        if [ "$NO_SERVER_DOCKER" -eq 1 ]; then
-          echo "Error: cannot disable both server RPM and server Docker"
+      --no-client-docker)
+        if [ "$NO_CLIENT" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --client-docker and --no-client"
           exit 1
         fi
-        NO_SERVER_RPM=1
-        ;;
-      --no-server-docker)
-        if [ "$NO_SERVER_RPM" -eq 1 ]; then
-          echo "Error: cannot disable both server RPM and server Docker"
+        if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
+          echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
           exit 1
         fi
-        NO_SERVER_DOCKER=1
+        NO_CLIENT_DOCKER=1
+        ;;
+      --no-server)
+        NO_SERVER=1
         ;;
       --platform)
         shift
@@ -145,9 +151,17 @@ check-files() {
   return 1
 }
 
+# Update submodules and PR branches
 update() {
   local DEPS=
   local CURDIR="$PWD"
+  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
+  local MD5SUM=$(md5sum $(dirname $0)/build.sh)
+
+   if [ "$NO_UPDATE" -eq 1 ]; then
+    echo "Running without refreshing the source code from repositories..."
+    return
+   fi
 
   # Thouroughly verify the presence of known files, otherwise bail out
   if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules
@@ -192,6 +206,13 @@ update() {
   git submodule status
 
   cd "$CURDIR" > /dev/null
+
+  if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+    echo "The local copy of this script differs from the one fetched from the repo." 
+    echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    exit 0
+  fi
 }
 
 get_branch_name() {
@@ -306,22 +327,7 @@ cleanup() {
 }
 
 main() {
-  if [ "$NO_UPDATE" -eq 0 ]; then
-    local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
-    MD5SUM=$(md5sum $(dirname $0)/build.sh)
-
-    # Update submodules and PR branches
-    update
-
-    test "$UPDATE_ONLY" -eq 1 && return
-
-    if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
-      echo "The local copy of this script differs from the one fetched from the repo." 
-      echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
-      echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
-      return
-    fi
-  fi
+  update
 
   init
 
@@ -335,10 +341,10 @@ main() {
     run_build_script build-client-binary
 
     # Building client source rpm takes 13s (caching does not apply)
-    # run_build_script build-client-srpm
+    run_build_script build-client-srpm
 
     # Building client rpm takes 1m40s
-    # run_build_script build-client-rpm
+    run_build_script build-client-rpm
   fi
 
   # Building client docker image takes 17s
@@ -347,11 +353,14 @@ main() {
     run_build_script build-client-docker
   fi
 
-  export RPM_EPOCH=1
-  if [ "$NO_SERVER_RPM" -eq 0 ]; then
+  if [ "$NO_SERVER" -eq 0 ]; then
     # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
     # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it
     export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
+    export DOCKERFILE=Dockerfile.el9
+    export RPM_EPOCH=1
+
     # 3rd-party components
     run_build_script build-server-rpm grafana
     run_build_script build-server-rpm victoriametrics
@@ -363,11 +372,7 @@ main() {
     run_build_script build-server-rpm pmm-update pmm
     run_build_script build-server-rpm pmm-dump
     run_build_script build-server-rpm vmproxy pmm
-  fi
 
-  if [ "$NO_SERVER_DOCKER" -eq 0 ]; then
-    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
-    export DOCKERFILE=Dockerfile.el9
     run_build_script build-server-docker
   fi
 

From 8944ef5af26daffeb8cb7ebcf43253106aa350fb Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 22:56:23 +0300
Subject: [PATCH 073/127] PMM-12899 rename build.sh to build

---
 build/local/README.md               | 10 +++++-----
 build/local/{build.sh => build}     | 17 +++++++++++------
 build/scripts/build-client-packages | 15 +++++++++------
 3 files changed, 25 insertions(+), 17 deletions(-)
 rename build/local/{build.sh => build} (98%)

diff --git a/build/local/README.md b/build/local/README.md
index b775030f66..7c1fac211a 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -33,10 +33,10 @@ Below is a list of prerequisites that are required to build PMM locally.
 1. Install the prerequisites
 2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
 3. Change to the `build/local` directory in the cloned repo.
-4. Run `build.sh --help` to print the help message and check the usage.
-5. Run `build.sh` with parameters of your choice to build PMM v3.
+4. Run `./build --help` to print the help message and check the usage.
+5. Run `./build` with parameters of your choice to build PMM v3.
 
-Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build.sh` as often as those changes need to be factored in to the next build.
+Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build` as often as those changes need to be factored in to the next build.
 
 Once the build is finished, you can proceed with launching a new instance of PMM Server, or installing a freshly built PMM Client, and testing the changes.
 
@@ -48,12 +48,12 @@ We use a special docker image to build various PMM artifacts - `perconalab/rpmbu
 - it frees the user from installing dependencies on their host machine
 - it leverages a very powerful docker caching system, which results in reduced build times
 
-During the first run, `build.sh` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
+During the first run, `build` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
 
 
 ## Avoiding unnecessary builds
 
-Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build.sh` script offers several parameters to help control what you want to build.
+Sometimes, the changes you make affect only PMM Client. Other times, they affect only PMM Server. Therefore, you may want to skip building parts of PMM. The `build` script offers several parameters to help control what you want to build.
 
 * --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
 * --update-only: pull changes from the repo without building PMM
diff --git a/build/local/build.sh b/build/local/build
similarity index 98%
rename from build/local/build.sh
rename to build/local/build
index dd96c41418..9a92834af7 100755
--- a/build/local/build.sh
+++ b/build/local/build
@@ -52,13 +52,13 @@ parse-params() {
         NO_CLIENT=1; NO_CLIENT_DOCKER=1
         ;;
       --client-docker)
-        NO_CLIENT_DOCKER=0
-        ;;
-      --no-client-docker)
         if [ "$NO_CLIENT" -eq 1 ]; then
           echo "Error. Mutually exclusive options: --client-docker and --no-client"
           exit 1
         fi
+        NO_CLIENT_DOCKER=0
+        ;;
+      --no-client-docker)
         if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
           echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
           exit 1
@@ -155,8 +155,8 @@ check-files() {
 update() {
   local DEPS=
   local CURDIR="$PWD"
-  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build.sh"
-  local MD5SUM=$(md5sum $(dirname $0)/build.sh)
+  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build"
+  local MD5SUM=$(md5sum $(dirname $0)/build)
 
    if [ "$NO_UPDATE" -eq 1 ]; then
     echo "Running without refreshing the source code from repositories..."
@@ -210,7 +210,7 @@ update() {
   if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
     echo "The local copy of this script differs from the one fetched from the repo." 
     echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
-    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build.sh --no-update'"
+    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build --no-update'"
     exit 0
   fi
 }
@@ -284,6 +284,11 @@ purge_files() {
       rm -rf tmp/source/pmm
     fi
   fi
+
+  if [ -d results/rpm ]; then
+    echo "Removing results/rpm/*.rpm ..."
+    rm results/rpm/*.rpm
+  fi
   
   cd "$CURDIR"
 }
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index f578a422d0..ed4836ec63 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -34,7 +34,6 @@ parse_arguments() {
         val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
         optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
         case "$arg" in
-            # these get passed explicitly to mysqld
             --builddir=*) WORKDIR="$val" ;;
             --build_src_rpm=*) SRPM="$val" ;;
             --build_source_deb=*) SDEB="$val" ;;
@@ -44,9 +43,7 @@ parse_arguments() {
             --branch=*) SUBMODULE_BRANCH="$val" ;;
             --install_deps=*) INSTALL="$val" ;;
             --help) usage ;;
-            *)
-                append_arg_to_args "$arg"
-              ;;
+            *) append_arg_to_args "$arg" ;;
         esac
     done
 }
@@ -59,7 +56,7 @@ get_branches() {
     cd ${WORKDIR}/pmm-submodules
       git reset --hard > /dev/null 2>&1
       git clean -xdf > /dev/null 2>&1
-      git checkout $SUBMODULE_BRANCH > /dev/null 2>&1
+      git checkout $SUBMODULE_BRANCH
       # this is the return value of this function
       git submodule status | grep $COMPONENT | awk '{print $1}' | awk -F'-' '{print $2}'
     cd - >/dev/null
@@ -257,7 +254,13 @@ build_srpm(){
     cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
     cp ${WORKDIR}/${TARFILE} ../SOURCES/
     cd ${WORKDIR}
-    rpmbuild -bs --define "_topdir ${WORKDIR}/rpmbuild" --define "version $VERSION" --define "release $RPM_RELEASE" --define "dist %{nil}" "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
+    rpmbuild -bs \
+      --define "_topdir ${WORKDIR}/rpmbuild" \
+      --define "version $VERSION" \
+      --define "release $RPM_RELEASE" \
+      --define "dist %{nil}" \
+      "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
+
     mkdir -p ${WORKDIR}/srpm
     mkdir -p ${CURDIR}/srpm
     cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm

From 69899db8fc0f7979da0b148e2ddc8c53556bf551 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:04:05 +0300
Subject: [PATCH 074/127] PMM-12899 purge more files before starting

---
 build/local/build | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/build/local/build b/build/local/build
index 9a92834af7..4e51fcc4c4 100755
--- a/build/local/build
+++ b/build/local/build
@@ -285,9 +285,9 @@ purge_files() {
     fi
   fi
 
-  if [ -d results/rpm ]; then
-    echo "Removing results/rpm/*.rpm ..."
-    rm results/rpm/*.rpm
+  if [ -d results ]; then
+    echo "Removing results/* ..."
+    rm -rf results/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
   fi
   
   cd "$CURDIR"

From a8756abcce9d6ebc486e14b4fa454e910e357d26 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:49:53 +0300
Subject: [PATCH 075/127] PMM-12899 pass NO_VCS parameter to go build

---
 admin/Makefile                    | 8 ++++++--
 agent/Makefile                    | 8 ++++++--
 build/local/build                 | 2 +-
 build/scripts/build-client-binary | 2 +-
 4 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/admin/Makefile b/admin/Makefile
index f6c12416e1..0d29552f70 100644
--- a/admin/Makefile
+++ b/admin/Makefile
@@ -11,6 +11,10 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
+
+ifdef NO_VCS
+	NO_VCS := -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -31,8 +35,8 @@ clean:                          ## Remove generated files
 	find . -name *_reform.go -delete
 
 release:                        ## Build pmm-admin release binary
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
 
 install:                        ## Install pmm & pmm-admin binary
 	go build -v $(LD_FLAGS) -o $(GOBIN)/pmm-admin ./cmd/pmm-admin/
diff --git a/agent/Makefile b/agent/Makefile
index 2507f82fcb..6c1e863b6a 100644
--- a/agent/Makefile
+++ b/agent/Makefile
@@ -12,6 +12,10 @@ PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 PMM_DEV_SERVER_PORT ?= 8443
+
+ifdef NO_VCS
+	NO_VCS := -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -25,8 +29,8 @@ VERSION_FLAGS = -X 'github.com/percona/pmm/version.ProjectName=pmm-agent' \
 				-X 'github.com/percona/pmm/version.Branch=$(PMM_RELEASE_BRANCH)'
 
 release:                        ## Build static pmm-agent release binary (Linux only)
-	env CGO_ENABLED=1 go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
-	go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
+	env CGO_ENABLED=1 go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
+	go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
 	$(PMM_RELEASE_PATH)/pmm-agent --version
 	ldd $(PMM_RELEASE_PATH)/pmm-agent 2>&1 | grep -Fq 'not a dynamic executable'
 
diff --git a/build/local/build b/build/local/build
index 4e51fcc4c4..2c5b549206 100755
--- a/build/local/build
+++ b/build/local/build
@@ -352,7 +352,7 @@ main() {
     run_build_script build-client-rpm
   fi
 
-  # Building client docker image takes 17s
+  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache). 
   export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index d2dbaac338..8706136537 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -39,7 +39,7 @@ gobuild_component() {
         return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && make release"
+        build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
         build_command="cd /tmp/go/src/${extract_path} && make build"
     elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then

From be7405eff8fa09596e95c86f62dabbee47d7ee46 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 5 Nov 2024 23:53:07 +0300
Subject: [PATCH 076/127] PMM-12899 add dist directory to .gitignore

---
 .gitignore        | 3 +++
 build/local/build | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index da7596fcd5..e862c50236 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,6 @@ api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
 
 packer.log
+
+# Goreleaser
+dist/
diff --git a/build/local/build b/build/local/build
index 2c5b549206..3c65ca4472 100755
--- a/build/local/build
+++ b/build/local/build
@@ -352,7 +352,7 @@ main() {
     run_build_script build-client-rpm
   fi
 
-  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache). 
+  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache).
   export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker

From 93d5d7cbc39f7e8c4b777281480d8937b78b05d5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 00:05:33 +0300
Subject: [PATCH 077/127] PMM-12899 create cache directories in rpmbuild

---
 build/docker/rpmbuild/Dockerfile.el9 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index e958731d93..6f3c445b23 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -49,7 +49,7 @@ RUN useradd builder -u 1000 -m -G users,wheel && \
     echo "builder ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers && \
     echo "# macros"                      >  /home/builder/.rpmmacros && \
     echo "%_topdir    /home/builder/rpm" >> /home/builder/.rpmmacros && \
-    mkdir /home/builder/rpm && \
+    mkdir -p /home/builder/rpm /home/builder/.cache/go-build /home/builder/go/pkg/mod && \
     chmod 755 /home/builder && \
     chown -R builder:builder /home/builder
 

From 45fd300b04871b61bc7746808403ea93335c39a5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 00:17:59 +0300
Subject: [PATCH 078/127] PMM-12899 fix "build_command" extrapolation

---
 admin/Makefile                    | 2 +-
 agent/Makefile                    | 2 +-
 build/scripts/build-client-binary | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/admin/Makefile b/admin/Makefile
index 0d29552f70..b3f4249f03 100644
--- a/admin/Makefile
+++ b/admin/Makefile
@@ -13,7 +13,7 @@ PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 
 ifdef NO_VCS
-	NO_VCS := -buildvcs=false
+	NO_VCS = -buildvcs=false
 endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
diff --git a/agent/Makefile b/agent/Makefile
index 6c1e863b6a..fb346bd539 100644
--- a/agent/Makefile
+++ b/agent/Makefile
@@ -14,7 +14,7 @@ PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
 PMM_DEV_SERVER_PORT ?= 8443
 
 ifdef NO_VCS
-	NO_VCS := -buildvcs=false
+	NO_VCS = -buildvcs=false
 endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8706136537..f20b905ddc 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -83,7 +83,7 @@ gobuild_component() {
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
-        ${build_command}
+        "${build_command}"
     "
 }
 

From ce911e17e27d2b4f185c99759db0b8e706a21777 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 18:00:00 +0300
Subject: [PATCH 079/127] PMM-12899 major code cleanup, remove redundancies

---
 build/scripts/build-client-deb      |  12 +--
 build/scripts/build-client-docker   |   2 +-
 build/scripts/build-client-packages | 151 ++++++----------------------
 build/scripts/build-client-rpm      |  10 +-
 build/scripts/build-client-sdeb     |  12 +--
 build/scripts/build-client-srpm     |   9 +-
 build/scripts/build-server          |   2 +-
 build/scripts/build-server-docker   |   2 +-
 build/scripts/vars                  |  10 +-
 9 files changed, 52 insertions(+), 158 deletions(-)

diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 72deb8a62e..0343b1d912 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -11,7 +11,7 @@ main() {
     local IMAGE=${1:-ubuntu:focal}
     docker run --rm -e DEBIAN_FRONTEND=noninteractive \
                     -v ${bin_dir}:/home/builder/bin \
-                    -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
+                    -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
@@ -23,13 +23,9 @@ main() {
             11 | 12 | 22.04 | 24.04) apt-get -y install git lsb-release devscripts dh-make;;
             *) apt-get -y install git lsb-release devscripts dh-make dh-systemd;;
         esac
-        mkdir -p /tmp/pmm
-        cd /home/builder/results
-            bash /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_deb=1
-        cd -
-        chown -R $(id -u):$(id -g) /home/builder/results/deb
+
+        cd /home/builder/build
+        bash /home/builder/bin/build-client-packages --build_deb=1
     "
 }
 
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index a564c5c4cf..2b88622242 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -11,7 +11,7 @@ main() {
     echo -----------------------------------------
 
     DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
-    cp ${root_dir}/results/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+    cp ${root_dir}/build/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
 
     if [ -z "${DOCKER_CLIENT_TAG}" ]; then
         DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index ed4836ec63..98fa44d31d 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -11,8 +11,6 @@ usage () {
     cat <<EOF
 Usage: $0 [OPTIONS]
     The following options may be given :
-        --builddir=DIR      Absolute path to the dir where all actions will be performed
-        --get_sources       Source will be downloaded from github
         --build_src_rpm     If it is 1 src rpm will be built
         --build_source_deb  If it is 1 source deb package will be built
         --build_rpm         If it is 1 rpm will be built
@@ -20,7 +18,7 @@ Usage: $0 [OPTIONS]
         --install_deps      Install build dependencies(root privileges are required)
         --branch            Branch from which submodules should be taken(default v3)
         --help) usage ;;
-Example $0 --builddir=/tmp/PMM_CLIENT --get_sources=1 --build_src_rpm=1 --build_rpm=1
+Example $0 --build_src_rpm=1 --build_rpm=1
 EOF
     exit 1
 }
@@ -30,16 +28,15 @@ append_arg_to_args () {
 }
 
 parse_arguments() {
+    WORKDIR=/tmp/pmm
     for arg do
         val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
         optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
         case "$arg" in
-            --builddir=*) WORKDIR="$val" ;;
             --build_src_rpm=*) SRPM="$val" ;;
-            --build_source_deb=*) SDEB="$val" ;;
             --build_rpm=*) RPM="$val" ;;
+            --build_source_deb=*) SDEB="$val" ;;
             --build_deb=*) DEB="$val" ;;
-            --get_sources=*) SOURCE="$val" ;;
             --branch=*) SUBMODULE_BRANCH="$val" ;;
             --install_deps=*) INSTALL="$val" ;;
             --help) usage ;;
@@ -82,70 +79,8 @@ check_workdir(){
     fi
     
     if ! test -d "$WORKDIR"; then
-        echo >&2 "$WORKDIR is not a directory."
-        exit 1
-    fi
-}
-
-get_sources(){
-    cd $WORKDIR
-    if [ $SOURCE = 0 ]; then
-        echo "Sources will not be downloaded"
-        return 0
-    fi
-    git clone $REPO
-    retval=$?
-    if [ $retval != 0 ]; then
-        echo "There were some issues during repo cloning from github. Please retry."
-        exit 1
+        mkdir -p "$WORKDIR"
     fi
-    cd pmm-client
-    if [ -n $BRANCH ]; then
-        git reset --hard
-        git clean -xdf
-        git checkout $BRANCH
-    fi
-    REVISION=$(git rev-parse --short HEAD)
-    git reset --hard
-    #
-    VERSION=$(pmm_version)
-    mv Makefile build/
-    echo "VERSION=${VERSION}" > $VERSION_FILE
-    echo "REVISION=${REVISION}" >> $VERSION_FILE
-    echo "RPM_RELEASE=${RPM_RELEASE}" >> $VERSION_FILE
-    echo "DEB_RELEASE=${DEB_RELEASE}" >> $VERSION_FILE
-    echo "GIT_REPO=${REPO}" >> $VERSION_FILE
-    echo "BRANCH_NAME=${BRANCH}" >> $VERSION_FILE
-    echo "NodeExp_BRANCH_NAME=${NodeExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "MongoExp_BRANCH_NAME=${MongoExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "MysqlExp_BRANCH_NAME=${MysqlExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "ProxysqlExp_BRANCH_NAME=${ProxysqlExp_BRANCH_NAME}" >> $VERSION_FILE
-    echo "TOOLKIT_REPO=${TOOLKIT_REPO}" >> $VERSION_FILE
-    echo "TOOLKIT_BRANCH_NAME=${TOOLKIT_BRANCH_NAME}" >> $VERSION_FILE
-    PRODUCT=pmm-client
-    PRODUCT_NAME=pmm
-    echo "PRODUCT=${PRODUCT}" >> $VERSION_FILE
-    echo "PRODUCT_NAME=${PRODUCT_NAME}" >> $VERSION_FILE
-    PRODUCT_FULL=${PRODUCT}-${VERSION}
-    echo "PRODUCT_FULL=${PRODUCT_FULL}" >> $VERSION_FILE
-    echo "BUILD_NUMBER=${BUILD_NUMBER}" >> $VERSION_FILE
-    echo "BUILD_ID=${BUILD_ID}" >> $VERSION_FILE
-    echo "UPLOAD=UPLOAD/experimental/BUILDS/${PRODUCT_NAME}/${VERSION}/${BRANCH_NAME}/${REVISION}/${BUILD_ID}" >> $VERSION_FILE
-    echo "MongoExp_REPO=${MongoExp_REPO}" >> $VERSION_FILE
-    echo "MysqlExp_REPO=${MysqlExp_REPO}" >> $VERSION_FILE
-    echo "ProxysqlExp_REPO=${ProxysqlExp_REPO}" >> $VERSION_FILE
-    echo "NodeExp_REPO=${NodeExp_REPO}" >> $VERSION_FILE
-    cd ../
-    mv ${PRODUCT} ${PRODUCT}-${VERSION}
-
-    tar -zcvf ${PRODUCT}-${VERSION}.tar.gz ${PRODUCT}-${VERSION} --exclude=.bzr*
-    mkdir $WORKDIR/source_tarball
-    mkdir $CURDIR/source_tarball
-    cp ${PRODUCT}-${VERSION}.tar.gz $WORKDIR/source_tarball
-    cp ${PRODUCT}-${VERSION}.tar.gz $CURDIR/source_tarball
-    cd $CURDIR
-    rm -rf pmm-client
-    return
 }
 
 get_system(){
@@ -168,7 +103,7 @@ install_deps() {
         echo "Dependencies will not be installed."
         return;
     fi
-    if [ ! $( id -u ) -eq 0 ]; then
+    if [ $(id -u) -ne 0 ]; then
         echo "It is not possible to install dependencies. Please run as root."
         exit 1
     fi
@@ -192,8 +127,8 @@ install_deps() {
 }
 
 get_tar(){
-    TARBALL=$1
-    TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+    local TARBALL=$1
+    local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
     if [ -z $TARFILE ]; then
         TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
         if [ -z $TARFILE ]; then
@@ -205,13 +140,12 @@ get_tar(){
     else
         cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
     fi
-    return
 }
 
 get_deb_sources(){
-    param=$1
+    local param=$1
     echo $param
-    FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
+    local FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
     if [ -z $FILE ]; then
         FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
         if [ -z $FILE ]; then
@@ -227,10 +161,6 @@ get_deb_sources(){
 }
 
 build_srpm(){
-    if [ "$SRPM" = 0 ]; then
-        echo "SRC RPM will not be created"
-        return;
-    fi
     if [ "$OS" = "deb" ]; then
         echo "It is not possible to build src rpm here"
         exit 1
@@ -268,10 +198,6 @@ build_srpm(){
 }
 
 build_rpm(){
-    if [ "$RPM" = 0 ]; then
-        echo "RPM will not be created"
-        return;
-    fi
     if [ "$OS" = "deb" ]; then
         echo "It is not possible to build rpm here"
         exit 1
@@ -316,10 +242,6 @@ build_rpm(){
 }
 
 build_source_deb(){
-    if [ "$SDEB" = 0 ]; then
-        echo "source deb package will not be created"
-        return;
-    fi
     if [ "$OS" = "rpm" ]; then
         echo "It is not possible to build source deb here"
         exit 1
@@ -346,10 +268,8 @@ build_source_deb(){
             git clean -xdf
             git checkout ${BRANCH}
         fi
-    popd
-    pushd ${NAME}-${VERSION}_all
-    REVISION=$(git rev-parse --short HEAD)
-    mkdir distro
+        REVISION=$(git rev-parse --short HEAD)
+        mkdir distro
     popd
 
     tar xzf ${NEWTAR}
@@ -396,10 +316,6 @@ build_source_deb(){
 }
 
 build_deb(){
-    if [ "$DEB" = 0 ]; then
-        echo "deb package will not be created"
-        return;
-    fi
     if [ "$OS" = "rpm" ]; then
         echo "It is not possible to build deb here"
         exit 1
@@ -440,8 +356,6 @@ SRPM=0
 SDEB=0
 RPM=0
 DEB=0
-SOURCE=0
-TARBALL=0
 OS_NAME=
 ARCH=
 OS=
@@ -450,6 +364,8 @@ INSTALL=0
 RPM_RELEASE=1
 DEB_RELEASE=1
 REVISION=0
+BRANCH="v3"
+REPO="https://github.com/Percona-Lab/pmm-submodules.git"
 
 if [ -n "$pmm_release" ]; then
     RPM_RELEASE="$RPM_RELEASE.$pmm_release"
@@ -460,31 +376,22 @@ parse_arguments "$@"
 get_system
 install_deps
 
-BRANCH="v3"
-REPO="https://github.com/Percona-Lab/pmm-submodules.git"
-
-if [ "$TARBALL" = '1' -o "$SOURCE" = '1' ]; then
-    MongoExp_BRANCH_NAME=$(get_branches "mongodb_exporter")
-    MongoExp_REPO=$(get_repos "mongodb_exporter")
-    TOOLKIT_BRANCH_NAME=$(get_branches "percona-toolkit")
-    TOOLKIT_REPO=$(get_repos "percona-toolkit")
-    MysqlExp_BRANCH_NAME=$(get_branches "mysqld_exporter")
-    MysqlExp_REPO=$(get_repos "mysqld_exporter")
-    PostgreSQLExp_BRANCH_NAME=$(get_branches "postgres_exporter")
-    PostgreSQLExp_REPO=$(get_repos "postgres_exporter")
-    ProxysqlExp_BRANCH_NAME=$(get_branches "proxysql_exporter")
-    ProxysqlExp_REPO=$(get_repos "proxysql_exporter")
-    NodeExp_BRANCH_NAME=$(get_branches "node_exporter")
-    NodeExp_REPO=$(get_repos "node_exporter")
-    AzureExp_BRANCH_NAME=$(get_branches "azure_metrics_exporter")
-    AzureExp_REPO=$(get_repos "azure_metrics_exporter")
-fi
-
 check_workdir
-get_sources
-build_srpm
-build_source_deb
-build_rpm
-build_deb
+if [ "$SRPM" != 0 ]; then
+  echo "Building srpm packages..."
+  build_srpm
+fi
+if [ "$SDEB" = 0 ]; then
+  echo "Building source deb packages..."
+  build_source_deb
+fi
+if [ "$RPM" = 0 ]; then
+  echo "Building rpm packages..."
+  build_rpm
+fi
+if [ "$DEB" = 0 ]; then
+  echo "Building deb packages..."
+  build_deb
+fi
 
 # vim: expandtab shiftwidth=4 tabstop=4
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 374b79c5ac..acbb84074d 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -13,18 +13,16 @@ main() {
     local IMAGE=${1:-${rpmbuild_docker_image}}
     docker run --rm \
       -v ${bin_dir}:/home/builder/bin \
-      -v ${root_dir}/results:/home/builder/results \
+      -v ${root_dir}/build:/home/builder/build \
       ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
         export pmm_version=$pmm_version
         export pmm_release=$pmm_release
-        mkdir -p /tmp/pmm
-        cd /home/builder/results
-        /home/builder/bin/build-client-packages \
-            --builddir=/tmp/pmm \
-            --build_rpm=1
+        cd /home/builder/build
+
+        /home/builder/bin/build-client-packages --build_rpm=1
       "
 }
 
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index 2429d5de8d..ebd78ba270 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -11,7 +11,7 @@ main() {
     local IMAGE=${1:-ubuntu:focal}
     docker run --rm -e DEBIAN_FRONTEND=noninteractive \
                     -v ${bin_dir}:/home/builder/bin \
-                    -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
+                    -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
@@ -20,13 +20,9 @@ main() {
 
         apt-get update
         DEBIAN_FRONTEND=noninteractive apt-get -y install git lsb-release devscripts dh-make dh-systemd
-        mkdir -p /tmp/pmm
-        cd /home/builder/results
-            bash /home/builder/bin/build-client-packages \
-                --builddir=/tmp/pmm \
-                --build_source_deb=1
-        cd -
-        chown -R $(id -u):$(id -g) /home/builder/results/source_deb
+        cd /home/builder/build
+
+        bash /home/builder/bin/build-client-packages --build_source_deb=1
     "
 }
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index c68b949a33..0dcc25e854 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -11,17 +11,14 @@ main() {
     echo -----------------------------------------
 
     local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/results:/home/builder/results ${IMAGE} sh -c "
+    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
         export pmm_release=$pmm_release
-        mkdir -p /tmp/pmm
-        cd /home/builder/results
+        cd /home/builder/build
 
-        /home/builder/bin/build-client-packages \
-            --builddir=/tmp/pmm \
-            --build_src_rpm=1
+        /home/builder/bin/build-client-packages --build_src_rpm=1
     "
 }
 
diff --git a/build/scripts/build-server b/build/scripts/build-server
index 89e2262622..67dd9fcaa5 100755
--- a/build/scripts/build-server
+++ b/build/scripts/build-server
@@ -11,7 +11,7 @@ pmm_version=${pmm_version} ${bin_dir}/build-client
 pmm_version=${pmm_version} ${bin_dir}/build-client-srpm
 pmm_version=${pmm_version} ${bin_dir}/build-client-rpm
 mkdir -p ${rpms_dir} ${rpmbuild_dir}
-cp ${root_dir}/results/rpm/*.rpm ${rpms_dir}/
+cp ${root_dir}/build/rpm/*.rpm ${rpms_dir}/
 
 # install all rpms
 ${bin_dir}/build-server-rpm-all
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index f1df62ac72..682f7cae9f 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -15,7 +15,7 @@ main() {
 
   cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
   cp "${root_dir}/${pmm_source_dir}/build/docker/server/"* "${docker_root}"
-  cp ${root_dir}/results/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
+  cp ${root_dir}/build/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
diff --git a/build/scripts/vars b/build/scripts/vars
index 4c6a25f273..3c9818106e 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -46,12 +46,12 @@ rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
 binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}
-client_properties=${root_dir}/results/pmm-client.properties
+client_properties=${root_dir}/build/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
-docker_tag_file=${root_dir}/results/docker/TAG
-docker_client_tag_file=${root_dir}/results/docker/CLIENT_TAG
-source_tarball=${root_dir}/results/source_tarball/pmm-client-${pmm_version}.tar.gz
-binary_tarball=${root_dir}/results/tarball/pmm-client-${pmm_version}.tar.gz
+docker_tag_file=${root_dir}/build/docker/TAG
+docker_client_tag_file=${root_dir}/build/docker/CLIENT_TAG
+source_tarball=${root_dir}/build/source_tarball/pmm-client-${pmm_version}.tar.gz
+binary_tarball=${root_dir}/build/tarball/pmm-client-${pmm_version}.tar.gz
 
 # https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/pmm-6401-v1.93.4
 vmagent_commit_hash=58ecb9066574f38f1d1c91ace467316e7f175b09

From 8eb26005e87a59e736ffdbe9c348b7347ba32c75 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 18:15:11 +0300
Subject: [PATCH 080/127] PMM-12899 fix the build logic

---
 build/scripts/build-client-packages | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 98fa44d31d..220872e272 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -17,7 +17,7 @@ Usage: $0 [OPTIONS]
         --build_deb         If it is 1 deb will be built
         --install_deps      Install build dependencies(root privileges are required)
         --branch            Branch from which submodules should be taken(default v3)
-        --help) usage ;;
+        --help)             Display this help
 Example $0 --build_src_rpm=1 --build_rpm=1
 EOF
     exit 1
@@ -381,15 +381,15 @@ if [ "$SRPM" != 0 ]; then
   echo "Building srpm packages..."
   build_srpm
 fi
-if [ "$SDEB" = 0 ]; then
+if [ "$SDEB" != 0 ]; then
   echo "Building source deb packages..."
   build_source_deb
 fi
-if [ "$RPM" = 0 ]; then
+if [ "$RPM" != 0 ]; then
   echo "Building rpm packages..."
   build_rpm
 fi
-if [ "$DEB" = 0 ]; then
+if [ "$DEB" != 0 ]; then
   echo "Building deb packages..."
   build_deb
 fi

From b9e05a583645c37d5d8a9bf71f0dbef569ec5168 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 20:51:27 +0300
Subject: [PATCH 081/127] PMM-12899 refactor the directory structure

---
 build/local/build                     |  50 +++--
 build/local/ci-default.yml            |  14 --
 build/local/ci.yml                    |   3 +
 build/scripts/build                   |   2 +-
 build/scripts/build-client            |   2 +-
 build/scripts/build-client-binary     | 160 ++++++-------
 build/scripts/build-client-deb        |   2 +-
 build/scripts/build-client-docker     |  54 ++---
 build/scripts/build-client-packages   | 312 ++++++++++++--------------
 build/scripts/build-client-rpm        |   2 +-
 build/scripts/build-client-sdeb       |   2 +-
 build/scripts/build-client-source     |   8 +-
 build/scripts/build-client-srpm       |   2 +-
 build/scripts/build-server            |   2 +-
 build/scripts/build-server-docker     |   2 +-
 build/scripts/build-server-rpm        |   2 +-
 build/scripts/build-server-rpm-all    |   2 +-
 build/scripts/build-server-rpm-script |   2 +-
 build/scripts/vars                    |  10 +-
 19 files changed, 303 insertions(+), 330 deletions(-)
 create mode 100644 build/local/ci.yml

diff --git a/build/local/build b/build/local/build
index 3c65ca4472..aaa9ebe38d 100755
--- a/build/local/build
+++ b/build/local/build
@@ -14,7 +14,8 @@ Options:
       --client-only            Build only PMM Client (client binaries + docker)
       --no-server              Do not build PMM Server (docker image)
       --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
-      --help | -h              Display help
+  -d  --debug                  Log output in debug mode, which also prints the commands
+  -h  --help                   Display help
 EOF
 }
 
@@ -26,6 +27,7 @@ parse-params() {
   NO_CLIENT=0
   NO_CLIENT_DOCKER=1
   NO_SERVER=0
+  DEBUG_MODE=0
   START_TIME=$(date +%s)
   LOG_FILE="$(dirname $0)/build.log"
   BASE_NAME=$(basename $0)
@@ -84,6 +86,10 @@ parse-params() {
         fi
         LOG_FILE="$1"
         ;;
+      --debug | -d)
+        shift
+        export DEBUG_MODE=1
+        ;;
       --help | -h)
         shift
         usage
@@ -128,7 +134,10 @@ rewind() {
   fi
 
   if needs-to-pull; then
-    git pull origin
+    if ! git pull origin; then
+      git reset --hard HEAD~20
+      git pull origin
+    fi
     echo "Submodule has pulled from upstream"
     git logs -n 2
     cd - > /dev/null
@@ -265,11 +274,11 @@ purge_files() {
 
   cd "$SUBMODULES" > /dev/null
   # Remove stale files and directories
-  if [ -d tmp ]; then
+  if [ -d build ]; then
     echo "Removing stale files and directories..."
 
-    if [ -d "tmp/pmm-server" ]; then
-      tmp_files=$(find tmp/pmm-server | grep -v "RPMS" | grep -Ev "^tmp/pmm-server$" || :)
+    if [ -d "build/pmm-server" ]; then
+      tmp_files=$(find build/pmm-server | grep -v "RPMS" | grep -Ev "^build/pmm-server$" || :)
       if [ -n "$tmp_files" ]; then
         tmp_files=( $tmp_files )
         for f in "${tmp_files[@]}"; do
@@ -279,15 +288,15 @@ purge_files() {
       fi
     fi
 
-    if [ -d "tmp/source/pmm" ]; then
-      echo "Removing tmp/source/pmm ..."
-      rm -rf tmp/source/pmm
+    if [ -d "build/source/pmm" ]; then
+      echo "Removing build/source/pmm ..."
+      rm -rf build/source/pmm
     fi
   fi
 
-  if [ -d results ]; then
+  if [ -d build ]; then
     echo "Removing results/* ..."
-    rm -rf results/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
+    rm -rf build/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
   fi
   
   cd "$CURDIR"
@@ -304,20 +313,15 @@ init() {
 
   GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
 
-  # Create docker volume to persist package and build cache
+  # Create docker volumes to persist package and build cache
   # Read more in the section about `rpmbuild`.
-  if ! docker volume ls | grep pmm-gobuild >/dev/null; then
-    docker volume create pmm-gobuild
-  fi
-  if ! docker volume ls | grep pmm-gomod >/dev/null; then
-    docker volume create pmm-gomod
-  fi
-  if ! docker volume ls | grep pmm-yarn >/dev/null; then
-    docker volume create pmm-yarn
-  fi
-  if ! docker volume ls | grep pmm-dnf >/dev/null; then
-    docker volume create pmm-dnf
-  fi
+  for volume in pmm-gobuild pmm-gomod pmm-yarn pmm-dnf; do
+    if ! docker volume ls | grep "$volume" >/dev/null; then
+      docker volume create "$volume"
+    else
+      echo "Info: docker volume $volume checked."
+    fi
+  done
 
   cd "$CURDIR" > /dev/null
 }
diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml
index c344eeac13..ce413e6723 100644
--- a/build/local/ci-default.yml
+++ b/build/local/ci-default.yml
@@ -4,56 +4,47 @@ deps:
     branch: v3
     path: sources/pmm/src/github.com/percona/pmm
     url: https://github.com/percona/pmm
-    component: common
 
   # CLIENT
   - name: node_exporter
     branch: main
     path: sources/node_exporter/src/github.com/prometheus/node_exporter
     url: https://github.com/percona/node_exporter
-    component: client
 
   - name: mysqld_exporter
     branch: main
     path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter
     url: https://github.com/percona/mysqld_exporter
-    component: client
 
   - name: mongodb_exporter
     branch: main
     path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
     url: https://github.com/percona/mongodb_exporter
-    component: client
 
   - name: postgres_exporter
     branch: main
     path: sources/postgres_exporter/src/github.com/percona/postgres_exporter
     url: https://github.com/percona/postgres_exporter
-    component: client
 
   - name: proxysql_exporter
     branch: main
     path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter
     url: https://github.com/percona/proxysql_exporter
-    component: client
 
   - name: rds_exporter
     branch: main
     path: sources/rds_exporter/src/github.com/percona/rds_exporter
     url: https://github.com/percona/rds_exporter
-    component: client
 
   - name: azure_metrics_exporter
     branch: main
     path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter
     url: https://github.com/percona/azure_metrics_exporter
-    component: client
 
   - name: percona-toolkit
     branch: release-v3.5.2
     path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
     url: https://github.com/percona/percona-toolkit
-    component: client
 
   # SERVER
 
@@ -61,29 +52,24 @@ deps:
     branch: v3
     path: sources/grafana-dashboards
     url: https://github.com/percona/grafana-dashboards
-    component: server
 
   - name: grafana
     branch: v3
     path: sources/grafana/src/github.com/grafana/grafana
     url: https://github.com/percona/grafana
-    component: server
 
   - name: pmm-dump
     branch: main
     path: sources/pmm-dump
     url: https://github.com/percona/pmm-dump
-    component: server
 
   # QA
   - name: pmm-qa
     branch: v3
     path: sources/pmm-qa/src/github.com/percona/pmm-qa
     url: https://github.com/percona/pmm-qa
-    component: qa
 
   - name: pmm-ui-tests
     branch: v3
     path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests
     url: https://github.com/percona/pmm-ui-tests
-    component: qa
\ No newline at end of file
diff --git a/build/local/ci.yml b/build/local/ci.yml
new file mode 100644
index 0000000000..5ab572bc08
--- /dev/null
+++ b/build/local/ci.yml
@@ -0,0 +1,3 @@
+deps:
+  - name: pmm
+    branch: PMM-13487-build-pmm-locally
diff --git a/build/scripts/build b/build/scripts/build
index 1c39d9c6da..881f99ca5f 100755
--- a/build/scripts/build
+++ b/build/scripts/build
@@ -8,4 +8,4 @@ bin_dir=$(dirname $0)
 ${bin_dir}/build-client
 ${bin_dir}/build-server
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client b/build/scripts/build-client
index 02aec7d45c..9c6d576f88 100755
--- a/build/scripts/build-client
+++ b/build/scripts/build-client
@@ -9,4 +9,4 @@ ${bin_dir}/build-client-source
 ${bin_dir}/build-client-binary
 ${bin_dir}/build-client-docker
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index f20b905ddc..56b31ef307 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -9,10 +9,10 @@ set -o xtrace
 # inside the container, while `binary_dir` defines the directory for binary files outside the container.
 
 extract_source_tarball() {
-    if [ ! -d "${source_dir}" ]; then
-        mkdir -p $(dirname ${source_dir})
-        tar -C $(dirname ${source_dir}) -zxpf ${source_tarball}
-    fi
+  if [ ! -d "${source_dir}" ]; then
+    mkdir -p $(dirname ${source_dir})
+    tar -C $(dirname ${source_dir}) -zxpf ${source_tarball}
+  fi
 }
 
 gobuild_component() {
@@ -31,23 +31,23 @@ gobuild_component() {
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
     if [ ${component} = 'azure_metrics_exporter' ]; then
-        result_file=${result_file/_metrics/}
+      result_file=${result_file/_metrics/}
     fi
 
     if [ -x "${binary_dir}/${result_file}" ]; then
-        echo Skipping the build...
-        return
+      echo "Skipping the build for ${result_file} ..."
+      return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release"
+      build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
-        build_command="cd /tmp/go/src/${extract_path} && make build"
+      build_command="cd /tmp/go/src/${extract_path} && make build"
     elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then
-        build_command="cd /tmp/go/src/${extract_path} && make release"
+      build_command="cd /tmp/go/src/${extract_path} && make release"
     elif [ ${component} = 'percona-toolkit' ]; then
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
+      build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     else
-        build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
+      build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
 
     docker run --rm \
@@ -66,16 +66,16 @@ gobuild_component() {
 
         sudo chown -R builder:builder /home/builder/tmp
         if [ ! -w /home/builder/.cache/go-build ]; then
-            sudo chown builder /home/builder/.cache/go-build
+          sudo chown builder /home/builder/.cache/go-build
         fi
         if [ ! -w /home/builder/go/pkg/mod ]; then
-            sudo chown builder /home/builder/go/pkg/mod
+          sudo chown builder /home/builder/go/pkg/mod
         fi
 
-        if [ "${component}" = vmagent ]; then
-            export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
+        if [ ${component} = vmagent ]; then
+          export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
         else
-            export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
+          export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
         fi
 
         mkdir -p ${bin_dir}/bin
@@ -83,75 +83,79 @@ gobuild_component() {
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
-        "${build_command}"
+        \"${build_command}\"
     "
 }
 
 copy_component() {
-    local component=$1
-    local component_path=$2
-    local component_dest=${3:-$component_path}
-    local component_dir=${tmp_dir}/source/${component}
-    local archive=$(ls ${source_dir}/${component}-*.tar.gz | head -n1) # temporary hack for pmm repo
-
-    if [ ! -d "${component_dir}" ]; then
-        mkdir -p ${component_dir}
-        tar -C ${component_dir} \
-            --strip-components=1 \
-            -zxpf "$archive"
-    fi
-
-    cp -r ${component_dir}/${component_path} ${binary_dir}/${component_dest}
+  local component=$1
+  local component_path=$2
+  local component_dest=${3:-$component_path}
+  local component_dir=${tmp_dir}/source/${component}
+  local archive=$(ls ${source_dir}/${component}-*.tar.gz | head -n1) # temporary hack for pmm repo
+
+  if [ ! -d "${component_dir}" ]; then
+      mkdir -p ${component_dir}
+      tar -C ${component_dir} \
+          --strip-components=1 \
+          -zxpf "$archive"
+  fi
+
+  cp -r ${component_dir}/${component_path} ${binary_dir}/${component_dest}
 }
 
 main() {
-    echo -----------------------------------------
-    echo "Building PMM Client binary files..."
-    echo -----------------------------------------
-
-    extract_source_tarball
-
-    gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
-    gobuild_component "node_exporter" "" "github.com/prometheus/node_exporter"
-    gobuild_component "mysqld_exporter"
-    gobuild_component "postgres_exporter" "" "" "cmd/postgres_exporter"
-    gobuild_component "mongodb_exporter"
-    gobuild_component "proxysql_exporter"
-    gobuild_component "rds_exporter"
-    gobuild_component "azure_metrics_exporter"
-    gobuild_component "pmm-admin" "pmm" "" "admin"
-    gobuild_component "pmm-agent" "pmm" "" "agent"
-
-    echo ${pmm_version} > ${binary_dir}/VERSION
-
-    copy_component "pmm" "build/packages/rpm/client" "rpm"
-    copy_component "pmm" "build/packages/config" "config"
-    copy_component "pmm" "build/packages/deb" "debian"
-    copy_component "pmm" "build/scripts/install_tarball" "install_tarball"
-    copy_component "node_exporter" "example.prom"
-    copy_component "mysqld_exporter" "queries-mysqld.yml"
-    copy_component "mysqld_exporter" "queries-mysqld-group-replication.yml"
-    copy_component "postgres_exporter" "example-queries-postgres.yml"
-    copy_component "postgres_exporter" "queries-postgres-uptime.yml"
-    copy_component "postgres_exporter" "queries-mr.yaml"
-    copy_component "postgres_exporter" "queries-lr.yaml"
-    copy_component "percona-toolkit" "bin/pt-summary"
-    copy_component "percona-toolkit" "bin/pt-mysql-summary"
-
-    gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
-    gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
-
-    rm -rf ${binary_tarball}
-    mkdir -p $(dirname ${binary_tarball}) || :
-    
-    # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
-    if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
-        rm -f "${binary_dir}/bin/pmm"
-    fi
-
-    tar -C $(dirname ${binary_dir}) -zcpf ${binary_tarball} $(basename ${binary_dir})
+  echo -----------------------------------------
+  echo "Building PMM Client binary files..."
+  echo -----------------------------------------
+
+  extract_source_tarball
+
+  gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
+  gobuild_component "node_exporter" "" "github.com/prometheus/node_exporter"
+  gobuild_component "mysqld_exporter"
+  gobuild_component "postgres_exporter" "" "" "cmd/postgres_exporter"
+  gobuild_component "mongodb_exporter"
+  gobuild_component "proxysql_exporter"
+  gobuild_component "rds_exporter"
+  gobuild_component "azure_metrics_exporter"
+  gobuild_component "pmm-admin" "pmm" "" "admin"
+  gobuild_component "pmm-agent" "pmm" "" "agent"
+
+  echo ${pmm_version} > ${binary_dir}/VERSION
+
+  copy_component "pmm" "build/packages/rpm/client" "rpm"
+  copy_component "pmm" "build/packages/config" "config"
+  copy_component "pmm" "build/packages/deb" "debian"
+  copy_component "pmm" "build/scripts/install_tarball" "install_tarball"
+  copy_component "node_exporter" "example.prom"
+  copy_component "mysqld_exporter" "queries-mysqld.yml"
+  copy_component "mysqld_exporter" "queries-mysqld-group-replication.yml"
+  copy_component "postgres_exporter" "example-queries-postgres.yml"
+  copy_component "postgres_exporter" "queries-postgres-uptime.yml"
+  copy_component "postgres_exporter" "queries-mr.yaml"
+  copy_component "postgres_exporter" "queries-lr.yaml"
+  copy_component "percona-toolkit" "bin/pt-summary"
+  copy_component "percona-toolkit" "bin/pt-mysql-summary"
+
+  gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
+  gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
+
+  rm -rf ${binary_tarball}
+  mkdir -p $(dirname ${binary_tarball}) || :
+  
+  # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
+  if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
+      rm -f "${binary_dir}/bin/pmm"
+  fi
+  
+  local noxattrs=
+  if [[ $(uname -o) =~ [Dd]arwin ]]; then
+    noxattrs="--no-xattrs"
+  fi
+  tar -C $(dirname ${binary_dir}) ${noxattrs} -zcpf ${binary_tarball} $(basename ${binary_dir})
 }
 
 main
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 0343b1d912..38112cbabb 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -31,4 +31,4 @@ main() {
 
 main $*
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 2b88622242..ee66fb4521 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -6,34 +6,34 @@ set -o xtrace
 . $(dirname $0)/vars
 
 main() {
-    echo -----------------------------------------
-    echo "Building docker image for PMM Client..."
-    echo -----------------------------------------
-
-    DOCKER_FILE_LOCATION=tmp/source/pmm/build/docker/client # relative to `root_dir`
-    cp ${root_dir}/build/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
-
-    if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-        DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
-    fi
-
-    CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
-
-    docker buildx build \
-        --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
-        --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
-        --progress plain \
-        -f ${DOCKER_FILE_LOCATION}/${docker_file} \
-        -t ${DOCKER_CLIENT_TAG} \
-        ${DOCKER_FILE_LOCATION}
-
-    if [ -n "${PUSH_DOCKER}" ]; then
-        mkdir -p $(dirname ${docker_client_tag_file})
-        echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
-        docker push ${DOCKER_CLIENT_TAG}
-    fi
+  echo -----------------------------------------
+  echo "Building docker image for PMM Client..."
+  echo -----------------------------------------
+
+  local DOCKER_FILE_LOCATION=build/source/pmm/build/docker/client # relative to `root_dir`
+  cp ${root_dir}/build/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+
+  if [ -z "${DOCKER_CLIENT_TAG}" ]; then
+    DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
+  fi
+
+  CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
+
+  docker buildx build \
+    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+    --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
+    --progress plain \
+    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+    -t ${DOCKER_CLIENT_TAG} \
+    ${DOCKER_FILE_LOCATION}
+
+  if [ -n "${PUSH_DOCKER}" ]; then
+    mkdir -p $(dirname ${docker_client_tag_file})
+    echo ${DOCKER_CLIENT_TAG} > ${docker_client_tag_file}
+    docker push ${DOCKER_CLIENT_TAG}
+  fi
 }
 
 main
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 220872e272..c6499fa956 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -8,7 +8,7 @@ shell_quote_string() {
 }
 
 usage () {
-    cat <<EOF
+  cat <<EOF
 Usage: $0 [OPTIONS]
     The following options may be given :
         --build_src_rpm     If it is 1 src rpm will be built
@@ -20,7 +20,8 @@ Usage: $0 [OPTIONS]
         --help)             Display this help
 Example $0 --build_src_rpm=1 --build_rpm=1
 EOF
-    exit 1
+
+  exit 1
 }
 
 append_arg_to_args () {
@@ -28,118 +29,89 @@ append_arg_to_args () {
 }
 
 parse_arguments() {
-    WORKDIR=/tmp/pmm
-    for arg do
-        val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
-        optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
-        case "$arg" in
-            --build_src_rpm=*) SRPM="$val" ;;
-            --build_rpm=*) RPM="$val" ;;
-            --build_source_deb=*) SDEB="$val" ;;
-            --build_deb=*) DEB="$val" ;;
-            --branch=*) SUBMODULE_BRANCH="$val" ;;
-            --install_deps=*) INSTALL="$val" ;;
-            --help) usage ;;
-            *) append_arg_to_args "$arg" ;;
-        esac
-    done
-}
-
-get_branches() {
-    COMPONENT=$1
-    if [ ! -e ${WORKDIR}/pmm-submodules ]; then
-        git clone https://github.com/Percona-Lab/pmm-submodules.git ${WORKDIR}/pmm-submodules 1>&2
-    fi
-    cd ${WORKDIR}/pmm-submodules
-      git reset --hard > /dev/null 2>&1
-      git clean -xdf > /dev/null 2>&1
-      git checkout $SUBMODULE_BRANCH
-      # this is the return value of this function
-      git submodule status | grep $COMPONENT | awk '{print $1}' | awk -F'-' '{print $2}'
-    cd - >/dev/null
-}
-
-get_repos() {
-    COMPONENT=$1
-    if [ ! -e ${WORKDIR}/pmm-submodules ]; then
-        git clone https://github.com/Percona-Lab/pmm-submodules.git ${WORKDIR}/pmm-submodules
-    fi
-    cd ${WORKDIR}/pmm-submodules
-      git reset --hard > /dev/null 2>&1
-      git clean -xdf > /dev/null 2>&1
-      git checkout $SUBMODULE_BRANCH > /dev/null 2>&1
-      grep -A 3 "\[submodule \"${COMPONENT}\"\]" .gitmodules | grep "url" | awk '{print $3}'
-    cd - >/dev/null
+  WORKDIR=/tmp/pmm
+  for arg do
+    val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
+    optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
+    case "$arg" in
+      --build_src_rpm=*) SRPM="$val" ;;
+      --build_rpm=*) RPM="$val" ;;
+      --build_source_deb=*) SDEB="$val" ;;
+      --build_deb=*) DEB="$val" ;;
+      --branch=*) SUBMODULE_BRANCH="$val" ;;
+      --install_deps=*) INSTALL="$val" ;;
+      --help) usage ;;
+      *) append_arg_to_args "$arg" ;;
+    esac
+  done
 }
 
 check_workdir(){
-    if [ "$WORKDIR" = "$CURDIR" ]; then
-        echo >&2 "Current directory cannot be used for building!"
-        exit 1
-    fi
-    
-    if ! test -d "$WORKDIR"; then
-        mkdir -p "$WORKDIR"
-    fi
+  if [ "$WORKDIR" = "$CURDIR" ]; then
+    echo >&2 "Current directory cannot be used for building!"
+    exit 1
+  fi
+  
+  if ! test -d "$WORKDIR"; then
+    mkdir -p "$WORKDIR"
+  fi
 }
 
 get_system(){
-    if [ -f /etc/redhat-release ]; then
-        RHEL=$(rpm --eval %rhel)
-        ARCH=$(echo $(uname -m) | sed -e 's:i686:i386:g')
-        OS_NAME="el$RHEL"
-        OS="rpm"
-    else
-        ARCH=$(uname -m)
-        OS_NAME="$(lsb_release -sc)"
-        OS="deb"
-    fi
-    return
+  if [ -f /etc/redhat-release ]; then
+    RHEL=$(rpm --eval %rhel)
+    ARCH=$(echo $(uname -m) | sed -e 's:i686:i386:g')
+    OS_NAME="el$RHEL"
+    OS="rpm"
+else
+    ARCH=$(uname -m)
+    OS_NAME="$(lsb_release -sc)"
+    OS="deb"
+  fi
 }
 
 install_deps() {
-    local PACKAGES
-    if [ $INSTALL = 0 ]; then
-        echo "Dependencies will not be installed."
-        return;
-    fi
-    if [ $(id -u) -ne 0 ]; then
-        echo "It is not possible to install dependencies. Please run as root."
-        exit 1
-    fi
-    if [ "$OS" = "rpm" ]; then
-        # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
-        command -v git || PACKAGES+=" git"
-        command -v curl || PACKAGES+=" curl"
-        command -v wget || PACKAGES+=" wget"
-        command -v spectool || PACKAGES+=" rpmdevtools"
-        command -v bison || PACKAGES+=" bison"
-        command -v repoquery || PACKAGES+=" yum-utils"
-        command -v rpmbuild || PACKAGES+=" rpm-build"
-        if [ -n "$PACKAGES" ]; then
-            yum -y install "$PACKAGES"
-        fi
-    else
-        apt-get update
-        apt-get -y install devscripts equivs debhelper
-    fi
+  local PACKAGES
+  if [ $INSTALL = 0 ]; then
+    echo "Dependencies will not be installed."
     return;
+  fi
+  if [ $(id -u) -ne 0 ]; then
+    echo "It is not possible to install dependencies. Please run as root."
+    exit 1
+  fi
+  if [ "$OS" = "rpm" ]; then
+    # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
+    command -v git || PACKAGES+=" git"
+    command -v curl || PACKAGES+=" curl"
+    command -v wget || PACKAGES+=" wget"
+    command -v spectool || PACKAGES+=" rpmdevtools"
+    command -v bison || PACKAGES+=" bison"
+    command -v repoquery || PACKAGES+=" yum-utils"
+    command -v rpmbuild || PACKAGES+=" rpm-build"
+    if [ -n "$PACKAGES" ]; then
+      yum -y install "$PACKAGES"
+    fi
+  else
+    apt-get update
+    apt-get -y install devscripts equivs debhelper
+  fi
 }
 
 get_tar(){
-    local TARBALL=$1
-    local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+  local TARBALL=$1
+  local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+  if [ -z $TARFILE ]; then
+    TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
     if [ -z $TARFILE ]; then
-        TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-        if [ -z $TARFILE ]; then
-            echo "There is no $TARBALL for build"
-            exit 1
-        else
-            cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
-        fi
+      echo "There is no $TARBALL for build"
+      exit 1
     else
-        cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
+      cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
     fi
+  else
+    cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
+  fi
 }
 
 get_deb_sources(){
@@ -161,84 +133,84 @@ get_deb_sources(){
 }
 
 build_srpm(){
-    if [ "$OS" = "deb" ]; then
-        echo "It is not possible to build src rpm here"
-        exit 1
-    fi
-    cd $WORKDIR
-    get_tar "tarball"
-
-    rm -fr rpmbuild
-    ls | grep -v tar.gz | xargs rm -rf
-
-    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
-    NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
-    VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
-    VERSION=${VERSION_TMP%.tar.gz}
-
-    mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-
-    tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
-
-    cd ${WORKDIR}/rpmbuild/SPECS
-    cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
-    cp ${WORKDIR}/${TARFILE} ../SOURCES/
-    cd ${WORKDIR}
-    rpmbuild -bs \
-      --define "_topdir ${WORKDIR}/rpmbuild" \
-      --define "version $VERSION" \
-      --define "release $RPM_RELEASE" \
-      --define "dist %{nil}" \
-      "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
-
-    mkdir -p ${WORKDIR}/srpm
-    mkdir -p ${CURDIR}/srpm
-    cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
-    cp rpmbuild/SRPMS/*.src.rpm ${WORKDIR}/srpm
+  if [ "$OS" = "deb" ]; then
+    echo "It is not possible to build src rpm here"
+    exit 1
+  fi
+  cd $WORKDIR
+  get_tar "tarball"
+
+  rm -fr rpmbuild
+  ls | grep -v tar.gz | xargs rm -rf
+
+  TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
+  NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
+  VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
+  VERSION=${VERSION_TMP%.tar.gz}
+
+  mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
+
+  tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
+
+  cd ${WORKDIR}/rpmbuild/SPECS
+  cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
+  cp ${WORKDIR}/${TARFILE} ../SOURCES/
+  cd ${WORKDIR}
+  rpmbuild -bs \
+    --define "_topdir ${WORKDIR}/rpmbuild" \
+    --define "version $VERSION" \
+    --define "release $RPM_RELEASE" \
+    --define "dist %{nil}" \
+    "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
+
+  mkdir -p ${WORKDIR}/srpm
+  mkdir -p ${CURDIR}/srpm
+  cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
+  cp rpmbuild/SRPMS/*.src.rpm ${WORKDIR}/srpm
 }
 
 build_rpm(){
-    if [ "$OS" = "deb" ]; then
-        echo "It is not possible to build rpm here"
-        exit 1
-    fi
+  if [ "$OS" = "deb" ]; then
+    echo "It is not possible to build rpm here"
+    exit 1
+  fi
 
-    SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
+  SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
+  if [ -z $SRC_RPM ]; then
+    SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
     if [ -z $SRC_RPM ]; then
-        SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
-        if [ -z $SRC_RPM ]; then
-            echo "There is no src rpm for build"
-            echo "You can create it using key --build_src_rpm=1"
-            exit 1
-        else
-            cp $CURDIR/srpm/$SRC_RPM $WORKDIR
-        fi
+      echo "There is no src rpm for build"
+      echo "You can create it using key --build_src_rpm=1"
+      exit 1
     else
-        cp $WORKDIR/srpm/$SRC_RPM $WORKDIR
-    fi
-
-    cd $WORKDIR
-    rm -fr rpmbuild
-    mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-    cp $SRC_RPM rpmbuild/SRPMS/
-    if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
-        # Remove older client rpm packages before building a new one
-        rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
-    fi
-    rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
-             --define "version $pmm_version" \
-             --define "release $RPM_RELEASE" \
-             --define "dist .$OS_NAME" \
-             --rebuild rpmbuild/SRPMS/$SRC_RPM
-
-    return_code=$?
-    if [ "$return_code" -ne 0 ]; then
-        echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n"
-        exit $return_code
+      cp $CURDIR/srpm/$SRC_RPM $WORKDIR
     fi
-
-    mkdir -p ${CURDIR}/rpm
-    cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
+  else
+    cp $WORKDIR/srpm/$SRC_RPM $WORKDIR
+  fi
+
+  cd $WORKDIR
+  rm -fr rpmbuild
+  mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
+  cp $SRC_RPM rpmbuild/SRPMS/
+  if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
+    # Remove older client rpm packages before building a new one
+    rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
+  fi
+  rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
+            --define "version $pmm_version" \
+            --define "release $RPM_RELEASE" \
+            --define "dist .$OS_NAME" \
+            --rebuild rpmbuild/SRPMS/$SRC_RPM
+
+  return_code=$?
+  if [ "$return_code" -ne 0 ]; then
+    echo "Error: rpmbuild exited with a non-zero return code '$return_code'\n"
+    exit $return_code
+  fi
+
+  mkdir -p ${CURDIR}/rpm
+  cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
 }
 
 build_source_deb(){
@@ -394,4 +366,4 @@ if [ "$DEB" != 0 ]; then
   build_deb
 fi
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index acbb84074d..5069ced42e 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -28,4 +28,4 @@ main() {
 
 main $*
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index ebd78ba270..845663de91 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -28,4 +28,4 @@ main() {
 
 main $*
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 5f82ccc77b..fbcfc00d4a 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -78,10 +78,14 @@ main() {
     prepare_tarball azure_metrics_exporter
 
     prepare_tarball percona-toolkit
+    local noxattrs=
+    if [[ $(uname -o) =~ [Dd]arwin ]]; then
+      noxattrs="--no-xattrs"
+    fi
 
-    tar -C $(dirname ${source_dir}) -zcpf ${source_tarball} $(basename ${source_dir})
+    tar -C $(dirname ${source_dir}) ${noxattrs} -zcpf ${source_tarball} $(basename ${source_dir})
 }
 
 main
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index 0dcc25e854..b669415486 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -24,4 +24,4 @@ main() {
 
 main $*
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-server b/build/scripts/build-server
index 67dd9fcaa5..0b53f5731f 100755
--- a/build/scripts/build-server
+++ b/build/scripts/build-server
@@ -19,4 +19,4 @@ ${bin_dir}/build-server-rpm-all
 # build pmm-server
 ${bin_dir}/build-server-docker
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 682f7cae9f..165b534cd7 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -52,4 +52,4 @@ main() {
 
 main
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 2cb55dde8c..25c8c71388 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -207,4 +207,4 @@ build() {
 
 build "$@"
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-server-rpm-all b/build/scripts/build-server-rpm-all
index e6accd289a..ba93725001 100755
--- a/build/scripts/build-server-rpm-all
+++ b/build/scripts/build-server-rpm-all
@@ -16,4 +16,4 @@ ${bin_dir}/build-server-rpm vmproxy pmm
 ${bin_dir}/build-server-rpm victoriametrics
 ${bin_dir}/build-server-rpm grafana
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
index 02a899e8a8..677f95f6a6 100755
--- a/build/scripts/build-server-rpm-script
+++ b/build/scripts/build-server-rpm-script
@@ -37,4 +37,4 @@ rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
 rm -f SOURCES/${spec_name}.spec*
 sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
 
-# vim: expandtab shiftwidth=4 tabstop=4
+# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/vars b/build/scripts/vars
index 3c9818106e..4a10cecf40 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -4,7 +4,7 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
 root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
 root_dir=${ROOT_DIR:-$root_dir}
-tmp_dir=${root_dir}/tmp
+tmp_dir=${root_dir}/build
 
 # In VERSION file we can have numeric value like '2.0.0' as well as
 # alphanumeric value like '2.0.0-alpha3' which can not be used entirely
@@ -40,12 +40,12 @@ echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versi
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
-rpms_dir=${root_dir}/tmp/pmm-server/RPMS
-rpmbuild_dir=${root_dir}/tmp/pmm-server/BUILD
+rpms_dir=${tmp_dir}/pmm-server/RPMS
+rpmbuild_dir=${tmp_dir}/pmm-server/BUILD
 rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
-source_dir=${root_dir}/tmp/source/pmm-client-${pmm_version}
-binary_dir=${root_dir}/tmp/binary/pmm-client-${pmm_version}
+source_dir=${tmp_dir}/source/pmm-client-${pmm_version}
+binary_dir=${tmp_dir}/binary/pmm-client-${pmm_version}
 client_properties=${root_dir}/build/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/build/docker/TAG

From 170d53c90721345b465acf5d9047df8e4b836025 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:42:04 +0300
Subject: [PATCH 082/127] PMM-12899 fix the cd syntax

---
 build/Makefile                    | 3 +--
 build/scripts/build-client-binary | 4 ++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/build/Makefile b/build/Makefile
index c7111bfe9f..b4883b25f5 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -6,8 +6,7 @@ export PMM_SERVER_IMAGE ?= docker.io/perconalab/pmm-server:3-dev-latest
 fetch:
 	mkdir -p ${PACKER_CACHE_DIR}/box || :
 	test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
-	    || curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
-		-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
+		|| curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant -o ${PACKER_CACHE_DIR}/id_rsa_vagrant
 	chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
 	test -f ${PACKER_CACHE_DIR}/box/oracle9.ova \
 		|| curl -fL https://pmm-build-cache.s3.us-east-2.amazonaws.com/VBOXES/oracle9-202401.31.0.box -o ${PACKER_CACHE_DIR}/box/oracle9.ova
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 56b31ef307..b239e7f9fb 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -82,6 +82,9 @@ gobuild_component() {
         mkdir -p /tmp/go/src/${extract_path}
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
+        if [ -d /tmp/go/src/${extract_path}/admin ]; then
+          ls -lah /tmp/go/src/${extract_path}
+        fi
 
         \"${build_command}\"
     "
@@ -111,6 +114,7 @@ main() {
 
   extract_source_tarball
 
+  # gobuild_component(component, parent_component, extract_path, component_path)
   gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
   gobuild_component "node_exporter" "" "github.com/prometheus/node_exporter"
   gobuild_component "mysqld_exporter"

From 8f95477381ea9eb964f38c873a9c3fd46510bbd0 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:46:46 +0300
Subject: [PATCH 083/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index b239e7f9fb..03e5d1aae0 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -86,7 +86,7 @@ gobuild_component() {
           ls -lah /tmp/go/src/${extract_path}
         fi
 
-        \"${build_command}\"
+        '${build_command}'
     "
 }
 

From 0f37c7217de0d2934ba5acae01383f2f6be1edde Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:50:19 +0300
Subject: [PATCH 084/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 03e5d1aae0..448e4935f4 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -54,6 +54,7 @@ gobuild_component() {
       --platform=${platform} \
       ${volume_mounts} \
       ${rpmbuild_docker_image} sh -c "
+        # Note: use no double quotes inside this script.
         set -o errexit
         set -o xtrace
         export CGO_ENABLED=0
@@ -86,7 +87,7 @@ gobuild_component() {
           ls -lah /tmp/go/src/${extract_path}
         fi
 
-        '${build_command}'
+        bash ${build_command}
     "
 }
 

From 95f74ca72f2b83dcdbbf82958158e5d21bc44075 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:51:42 +0300
Subject: [PATCH 085/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 448e4935f4..8523455894 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -87,7 +87,7 @@ gobuild_component() {
           ls -lah /tmp/go/src/${extract_path}
         fi
 
-        bash ${build_command}
+        bash -E ${build_command}
     "
 }
 

From 418e2fa8a08c5f737223fbf44f5483e849302bfa Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:52:58 +0300
Subject: [PATCH 086/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8523455894..22f3334dac 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -83,9 +83,6 @@ gobuild_component() {
         mkdir -p /tmp/go/src/${extract_path}
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
-        if [ -d /tmp/go/src/${extract_path}/admin ]; then
-          ls -lah /tmp/go/src/${extract_path}
-        fi
 
         bash -E ${build_command}
     "

From 7d28db4bed323bce1a8b986b373a72782b895984 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 21:59:40 +0300
Subject: [PATCH 087/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 22f3334dac..f985b6a1f9 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -39,7 +39,7 @@ gobuild_component() {
       return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
-      build_command="cd /tmp/go/src/${extract_path}/${component_path} && NO_VCS=1 make release"
+      build_command="cd '/tmp/go/src/${extract_path}/${component_path}' && 'NO_VCS=1 make release'"
     elif [ ${component} = 'mongodb_exporter' ]; then
       build_command="cd /tmp/go/src/${extract_path} && make build"
     elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then
@@ -84,7 +84,7 @@ gobuild_component() {
 
         tar -C /tmp/go/src/${extract_path} --strip-components=1 -zxpf ${src_dir}/${archive}
 
-        bash -E ${build_command}
+        ${build_command}
     "
 }
 

From c8b30758108a671e21281075280ec8fc4c9606d0 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Wed, 6 Nov 2024 22:03:50 +0300
Subject: [PATCH 088/127] PMM-12899 fix the build_command syntax

---
 build/scripts/build-client-binary | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index f985b6a1f9..1d3167ab6c 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -39,7 +39,7 @@ gobuild_component() {
       return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
-      build_command="cd '/tmp/go/src/${extract_path}/${component_path}' && 'NO_VCS=1 make release'"
+      build_command="cd '/tmp/go/src/${extract_path}/${component_path}' && NO_VCS=1 make release"
     elif [ ${component} = 'mongodb_exporter' ]; then
       build_command="cd /tmp/go/src/${extract_path} && make build"
     elif [[ ${component} =~ node_exporter|mysqld_exporter|proxysql_exporter|rds_exporter|postgres_exporter|azure_metrics_exporter ]]; then

From 13a2dab9198a5c67cb86ce148a1841418edff094 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 7 Nov 2024 12:45:41 +0300
Subject: [PATCH 089/127] PMM-12899 rename tmp_dir to build_dir

---
 build/scripts/build-client-binary |   4 +-
 build/scripts/build-server-rpm    | 356 +++++++++++++++---------------
 build/scripts/vars                |  10 +-
 3 files changed, 191 insertions(+), 179 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 1d3167ab6c..8e52ba5c37 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -25,7 +25,7 @@ gobuild_component() {
     local bin_dir=/home/builder/tmp/binary/pmm-client-${pmm_version}
     local archive=$(ls ${source_dir} | grep -e "${parent_component}-\w*\.tar\.gz" | head -n1)
     local build_command
-    local volume_mounts="-v ${tmp_dir}:/home/builder/tmp"
+    local volume_mounts="-v ${build_dir}:/home/builder/tmp"
     volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
@@ -92,7 +92,7 @@ copy_component() {
   local component=$1
   local component_path=$2
   local component_dest=${3:-$component_path}
-  local component_dir=${tmp_dir}/source/${component}
+  local component_dir=${build_dir}/source/${component}
   local archive=$(ls ${source_dir}/${component}-*.tar.gz | head -n1) # temporary hack for pmm repo
 
   if [ ! -d "${component_dir}" ]; then
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 25c8c71388..fe7ff71c10 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -6,203 +6,215 @@ set -o xtrace
 . $(dirname $0)/vars
 
 get_rpm_version() {
-    local spec_name=$1
-
-    local rpm_version=$(
-        docker run --rm \
-            --platform=${platform} \
-            -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
-            rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
-                | sed -re 's/\.[0-9]{10}\././' | head -1
-        "
-    )
-
-    # return version
-    echo ${rpm_version}
+  local spec_name=$1
+
+  local rpm_version=$(
+    docker run --rm \
+        --platform=${platform} \
+        -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+        rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
+            | sed -re 's/\.[0-9]{10}\././' | head -1
+    "
+  )
+
+  # return version
+  echo ${rpm_version}
 }
 
 is_build_needed() {
-    local spec_name=$1
-    local rpm_version=$2
-    local packages=
-    local s3_cache_dir=RELEASE
-
-    # Structure of S3 build cache
-    # s3://pmm-build-cache/PR-BUILDS/9 - el9
-    # s3://pmm-build-cache/RELEASE/9 - el9
-
-    # RPM_EPOCH is set to 1 for Feature Builds
-    if [ "$RPM_EPOCH" = 1 ]; then
-        s3_cache_dir=PR-BUILDS
-    fi
-    if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then
-        aws s3 sync \
-            --region us-east-2 \
-            --no-sign-request \
-            s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-            ${rpms_dir}/${spec_name}-${rpm_version} || :
-    fi
+  local spec_name=$1
+  local rpm_version=$2
+  local packages=
+  local s3_cache_dir=RELEASE
+
+  # Structure of S3 build cache
+  # s3://pmm-build-cache/PR-BUILDS/9 - el9
+  # s3://pmm-build-cache/RELEASE/9 - el9
+
+  # RPM_EPOCH is set to 1 for Feature Builds
+  if [ "$RPM_EPOCH" = 1 ]; then
+    s3_cache_dir=PR-BUILDS
+  fi
+  if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then
+    aws s3 sync \
+        --region us-east-2 \
+        --no-sign-request \
+        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+        ${rpms_dir}/${spec_name}-${rpm_version} || :
+  fi
 
-    packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
+  packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
 
-    # return result as true (0) or false (>0)
-    [[ ${packages// /} == 0 ]]
+  # return result as true (0) or false (>0)
+  [[ ${packages// /} == 0 ]]
 }
 
-prepare_specs() {
-    local spec_name=$1
-    local repo_name=$2
-    local spec_file=${rpmbuild_dir}/${spec_name}.spec
+prepare_spec() {
+  local spec_name=$1
+  local repo_name=$2
+  local spec_file=${rpmbuild_dir}/${spec_name}.spec
+
+  mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
+  cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
+  if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
+    # TODO: see if we should not exit with error here
+    return
+  fi
+  if [ ! -f "${spec_file}" ]; then
+    echo "Error: the spec file '${spec_file}' could not be found."
+    exit 1
+  fi
+  
+  local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+  local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+  local short_commit=${full_commit:0:7}
+  local tar_archive=${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz
+
+  if [ -z "$git_dir" ]; then
+    echo "Fatal: unable to find the .git file/directory for ${spec_file}"
+    exit 1
+  fi
+
+  sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
+  if [ "${spec_name}" != "grafana" ]; then
+    sed -i -e "s/Version:.*/Version: ${pmm_version}/" ${spec_file}
+  fi
+
+  if [ -z "${full_pmm_version}" ]; then
+    echo 'The full_pmm_version is not specified.'
+    exit 1
+  fi
+  sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${spec_file}
+
+  if [ -n "$pmm_release" ]; then
+    sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${spec_file}
+    grep -r 'define release' ${spec_file}
+  fi
+
+  if [ "${spec_name}" = "grafana" ]; then
+    sed -i -e "s;\(Source0:[[:space:]]*\)http.*;\1${tar_archive};" ${spec_file}
+  fi
+
+  if [ -f "${tar_archive}" ]; then
+    echo "${tar_archive}" already exists, skip archiving...
+    return
+  fi
+
+  git -C "${git_dir}" archive \
+      --format=tar.gz \
+      --prefix="${repo_name}-${full_commit}/" \
+      -o "${tar_archive}" \
+      "${full_commit}"
+}
 
-    mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
-    cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
-    if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
-        return
-    fi
-    if [ ! -f "${spec_file}" ]; then
-      echo "Warning: the spec file could not be found."
+build() {
+  local spec_name=$1
+  local repo_name=${2:-$1}
+  prepare_spec "${spec_name}" "${repo_name}"
+
+  local rpm_version=$(get_rpm_version "${spec_name}")
+  local CH_VERSION=${rpm_version%-*}
+  local CH_TAG=${rpm_version#*-}
+  local s3_cache_dir=RELEASE
+
+  if ! is_build_needed "${spec_name}" "${rpm_version}"; then
+      return
+  fi
+
+  echo -------------------------------------------------------
+  echo "Building PMM Server RPM for ${spec_name} component..."
+  echo -------------------------------------------------------
+
+  local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
+  volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
+  volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+  if [ ${spec_name} != "grafana" ]; then
+      volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+      volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
+  fi
+
+  echo "Start building Server RPMs..."
+  echo "spec_name: ${spec_name}"
+  echo "repo_name: ${repo_name}"
+  echo "rpm_verison: ${rpm_version}"
+
+  docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c "
+    set -o errexit
+    set -o xtrace
+
+    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    sudo chown builder:builder /home/builder/.cache
+    if [ ! -d /home/builder/.cache/go-build ]; then
+      mkdir -p /home/builder/.cache/go-build
     fi
-    
-    local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-    local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
-    local short_commit=${full_commit:0:7}
-
-    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
-    if [ "${spec_name}" != "grafana" ]; then
-        sed -i -e "s/Version:.*/Version: ${pmm_version}/"       ${spec_file}
+    if [ ! -d /home/builder/go ]; then
+      mkdir -p /home/builder/go/pkg/mod
+    fi        
+    if [ ! -w /home/builder/.cache/go-build ]; then
+      sudo chown builder:builder /home/builder/.cache/go-build
     fi
-
-    if [ -z "${full_pmm_version}" ]; then
-        echo 'The full_pmm_version is not specified.'
-        exit 1
+    if [ ! -w /home/builder/go/pkg/mod ]; then
+      sudo chown builder:builder /home/builder/go/pkg/mod
     fi
-    sed -i -e "s/%define full_pmm_version.*/%define full_pmm_version ${full_pmm_version}/" ${spec_file}
-
-    if [ -n "$pmm_release" ]; then
-        sed -i -e "s/\(%define release.*\)/\1.$pmm_release/" ${spec_file}
-        grep -r 'define release' ${spec_file}
+    if [ ! -w /home/builder/.cache/yarn ]; then
+      sudo chown builder:builder /home/builder/.cache/yarn
     fi
 
-    if [ -f "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" ]; then
-        echo "${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz" already exists, skip archiving...
-        return
+    # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
+    if [ ${RPM_EPOCH} = 1 ]; then
+      sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
     fi
 
-    git -C "${git_dir}" archive \
-        --format=tar.gz \
-        --prefix=${repo_name}-${full_commit}/ \
-        -o ${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz \
-        "${full_commit}"
-}
-
-build() {
-    local spec_name=$1
-    local repo_name=${2:-$1}
-    prepare_specs "${spec_name}" "${repo_name}"
-
-    local rpm_version=$(get_rpm_version "${spec_name}")
-    local CH_VERSION=${rpm_version%-*}
-    local CH_TAG=${rpm_version#*-}
-    local s3_cache_dir=RELEASE
+    rm -rf /home/builder/rpm/RPMS/${spec_name}-*
 
-    if ! is_build_needed "${spec_name}" "${rpm_version}"; then
-        return
-    fi
+    printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
+      | sudo tee /etc/yum.repos.d/local.repo
 
-    echo -------------------------------------------------------
-    echo "Building PMM Server RPM for ${spec_name} component..."
-    echo -------------------------------------------------------
+    until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+      echo Waiting for createrepo_c to finish...
+      sleep 1
+    done
 
-    local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-    volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
-    if [ ${spec_name} != "grafana" ]; then
-        volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
-        volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
-        volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
+    # Only these two specs have build dependencies
+    if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+      sleep 2s
+      sudo yum-builddep -y SOURCES/${spec_name}.spec
     fi
 
-    echo "Start building Server RPMs..."
-    echo "spec_name: ${spec_name}"
-    echo "repo_name: ${repo_name}"
-    echo "rpm_verison: ${rpm_version}"
-
-    docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c "
-        set -o errexit
-        set -o xtrace
-
-        sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-        sudo chown builder:builder /home/builder/.cache
-        if [ ! -d /home/builder/.cache/go-build ]; then
-            mkdir -p /home/builder/.cache/go-build
-        fi
-        if [ ! -d /home/builder/go ]; then
-            mkdir -p /home/builder/go/pkg/mod
-        fi        
-        if [ ! -w /home/builder/.cache/go-build ]; then
-            sudo chown builder:builder /home/builder/.cache/go-build
-        fi
-        if [ ! -w /home/builder/go/pkg/mod ]; then
-            sudo chown builder:builder /home/builder/go/pkg/mod
-        fi
-        if [ ! -w /home/builder/.cache/yarn ]; then
-            sudo chown builder:builder /home/builder/.cache/yarn
-        fi
-
-        # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-        if [ ${RPM_EPOCH} = 1 ]; then
-            sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-        fi
-
-        rm -rf /home/builder/rpm/RPMS/${spec_name}-*
-
-        printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
-            | sudo tee /etc/yum.repos.d/local.repo
-
-        until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-            echo waiting
-            sleep 1
-        done
-
-        # Only these two specs have build dependencies
-        if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-            sleep 2s
-            sudo yum-builddep -y SOURCES/${spec_name}.spec
-        fi
-
-        spectool -C SOURCES -g SOURCES/${spec_name}.spec
-        rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
-                  --define 'dist .${rpmbuild_dist}' \
-                  --define 'debug_package %{nil}' \
-                  -ba SOURCES/${spec_name}.spec
-
-        rm -f SOURCES/${spec_name}.spec*
-    "
+    spectool -C SOURCES -g SOURCES/${spec_name}.spec
+    rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
+              --define 'dist .${rpmbuild_dist}' \
+              --define 'debug_package %{nil}' \
+              -ba SOURCES/${spec_name}.spec
 
-    # TODO: see if this is even needed
-    # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
+    rm -f SOURCES/${spec_name}.spec*
+  "
 
-    if [ "$LOCAL_BUILD" = 1 ]; then
-        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
-        return
-    fi
+  # TODO: see if this is even needed
+  # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
 
-    if ! command -v aws &> /dev/null; then
-        echo "AWS CLI is not installed. Upload to S3 will be skipped."
-        echo
-        echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
-        return
-    fi
-
-    if [ "$RPM_EPOCH" = 1 ]; then
-      s3_cache_dir=PR-BUILDS
-    fi
-    aws s3 sync \
-        --region us-east-2 \
-        ${rpms_dir}/${spec_name}-${rpm_version} \
-        s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
-        || :
+  if [ "$LOCAL_BUILD" = 1 ]; then
+    echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    return
+  fi
 
+  if ! command -v aws &> /dev/null; then
+    echo "AWS CLI is not installed. Upload to S3 will be skipped."
+    echo
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    return
+  fi
+
+  if [ "$RPM_EPOCH" = 1 ]; then
+    s3_cache_dir=PR-BUILDS
+  fi
+  aws s3 sync \
+      --region us-east-2 \
+      ${rpms_dir}/${spec_name}-${rpm_version} \
+      s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
+      || :
+
+  echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
 }
 
 build "$@"
diff --git a/build/scripts/vars b/build/scripts/vars
index 4a10cecf40..ae8a18547d 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -4,7 +4,7 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 # Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
 root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
 root_dir=${ROOT_DIR:-$root_dir}
-tmp_dir=${root_dir}/build
+build_dir=${root_dir}/build
 
 # In VERSION file we can have numeric value like '2.0.0' as well as
 # alphanumeric value like '2.0.0-alpha3' which can not be used entirely
@@ -40,12 +40,12 @@ echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versi
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
-rpms_dir=${tmp_dir}/pmm-server/RPMS
-rpmbuild_dir=${tmp_dir}/pmm-server/BUILD
+rpms_dir=${build_dir}/pmm-server/RPMS
+rpmbuild_dir=${build_dir}/pmm-server/BUILD
 rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
-source_dir=${tmp_dir}/source/pmm-client-${pmm_version}
-binary_dir=${tmp_dir}/binary/pmm-client-${pmm_version}
+source_dir=${build_dir}/source/pmm-client-${pmm_version}
+binary_dir=${build_dir}/binary/pmm-client-${pmm_version}
 client_properties=${root_dir}/build/pmm-client.properties
 docker_file=${DOCKERFILE:-Dockerfile}
 docker_tag_file=${root_dir}/build/docker/TAG

From 96296c8d339c331e38d52de9c598dfcf76abc09a Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 8 Nov 2024 00:12:04 +0300
Subject: [PATCH 090/127] PMM-12899 refactor server rpm caching

---
 build/docker/client/Dockerfile            | 16 ++++-----
 build/local/README.md                     |  8 +++++
 build/local/build                         | 12 ++++---
 build/packages/rpm/client/pmm-client.spec |  2 +-
 build/scripts/build-server-rpm            | 32 +++++++-----------
 build/scripts/build-server-rpm-script     | 40 -----------------------
 build/scripts/vars                        |  4 +++
 7 files changed, 40 insertions(+), 74 deletions(-)
 delete mode 100755 build/scripts/build-server-rpm-script

diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile
index a93c134b3b..2e729c4ece 100644
--- a/build/docker/client/Dockerfile
+++ b/build/docker/client/Dockerfile
@@ -1,4 +1,4 @@
-FROM redhat/ubi9-minimal as users
+FROM redhat/ubi9-minimal AS users
 
 RUN microdnf install shadow-utils jq tar -y
 
@@ -47,13 +47,13 @@ RUN cd /tmp/pmm-client-* \
     && cd /tmp \
     && rm -rf /tmp/pmm-client*
 
-LABEL org.opencontainers.image.created ${BUILD_DATE}
-LABEL org.opencontainers.image.licenses Apache-2.0
-LABEL org.opencontainers.image.title Percona Monitoring and Management Client
-LABEL org.opencontainers.image.vendor Percona
-LABEL org.opencontainers.image.version ${VERSION}
-LABEL org.opencontainers.image.source https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
-LABEL org.opencontainers.image.docs https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+LABEL org.opencontainers.image.created=${BUILD_DATE}
+LABEL org.opencontainers.image.licenses=Apache-2.0
+LABEL org.opencontainers.image.title="Percona Monitoring and Management Client"
+LABEL org.opencontainers.image.vendor=Percona
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
 
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
diff --git a/build/local/README.md b/build/local/README.md
index 7c1fac211a..cd46216213 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -50,6 +50,14 @@ We use a special docker image to build various PMM artifacts - `perconalab/rpmbu
 
 During the first run, `build` will create a few directories on the host machine, which are necessary to make use of docker cache. Please be aware, that the docker container's user needs to be able to write to these directories. The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure that the directories we create on the host are owned by a user with the same uid and gid. If the build fails, this is the first thing to check.
 
+## Using S3 to cache packages
+
+In order to save time and to avoid building the same package versions repeatedly, we use a dedicated AWS S3 bucket for caching in the following manner:
+
+- before proceeding to building a package, we check if this package version can be found in S3 and we download the package instead of building it;
+- if the package can not be found, we build and upload it to S3 for future reuse.
+
+There is special variable `LOCAL_BUILD`, which needs to be set to '1' in case you don't have AWS CLI installed or you don't want to use the cache. Please be aware, that interacting with Percona's AWS S3 account, i.e. upload and download artefacts, requires authentication and is therefore reserved for Percona's own purposes. This is why, when building packages locally, you are requested to set this variable to '1', which happens to be the default value. Please note, that an attempt to interact with the S3 bucket without proper authorization will lead to a build failure.
 
 ## Avoiding unnecessary builds
 
diff --git a/build/local/build b/build/local/build
index aaa9ebe38d..7db0625018 100755
--- a/build/local/build
+++ b/build/local/build
@@ -270,6 +270,7 @@ run_build_script() {
 
 purge_files() {
   local CURDIR=$PWD
+  local PMM_DIR="build/source/pmm"
   local tmp_files
 
   cd "$SUBMODULES" > /dev/null
@@ -288,14 +289,14 @@ purge_files() {
       fi
     fi
 
-    if [ -d "build/source/pmm" ]; then
-      echo "Removing build/source/pmm ..."
-      rm -rf build/source/pmm
+    if [ -d "$PMM_DIR" ]; then
+      echo "Removing $PMM_DIR ..."
+      rm -rf "$PMM_DIR"
     fi
   fi
 
   if [ -d build ]; then
-    echo "Removing results/* ..."
+    echo "Removing build/* ..."
     rm -rf build/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
   fi
   
@@ -368,7 +369,8 @@ main() {
     export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
     export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export DOCKERFILE=Dockerfile.el9
-    export RPM_EPOCH=1
+    export RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
+    export LOCAL_BUILD
 
     # 3rd-party components
     run_build_script build-server-rpm grafana
diff --git a/build/packages/rpm/client/pmm-client.spec b/build/packages/rpm/client/pmm-client.spec
index 98e78c8eb7..44d87c0f04 100644
--- a/build/packages/rpm/client/pmm-client.spec
+++ b/build/packages/rpm/client/pmm-client.spec
@@ -22,7 +22,7 @@ Requires(postun): systemd
 
 AutoReq:        no
 Conflicts:      pmm-client
-Obsoletes:	pmm2-client < 3.0.0
+Obsoletes:      pmm2-client < 3.0.0
 
 %description
 Percona Monitoring and Management (PMM) is an open-source platform for managing and monitoring MySQL and MongoDB
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index fe7ff71c10..4152df2d1f 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -31,7 +31,7 @@ is_build_needed() {
   # s3://pmm-build-cache/PR-BUILDS/9 - el9
   # s3://pmm-build-cache/RELEASE/9 - el9
 
-  # RPM_EPOCH is set to 1 for Feature Builds
+  # Check if this is a feature build
   if [ "$RPM_EPOCH" = 1 ]; then
     s3_cache_dir=PR-BUILDS
   fi
@@ -43,7 +43,7 @@ is_build_needed() {
         ${rpms_dir}/${spec_name}-${rpm_version} || :
   fi
 
-  packages=$(find ${rpms_dir}/${spec_name}-${rpm_version} -name "*.${rpmbuild_dist}.*.rpm" | wc -l)
+  packages=$(find ${rpms_dir}/${spec_name}-${rpm_version}/${arch} -type f -name "*-${rpm_version}.${arch}.rpm" | wc -l)
 
   # return result as true (0) or false (>0)
   [[ ${packages// /} == 0 ]]
@@ -113,11 +113,10 @@ build() {
   prepare_spec "${spec_name}" "${repo_name}"
 
   local rpm_version=$(get_rpm_version "${spec_name}")
-  local CH_VERSION=${rpm_version%-*}
-  local CH_TAG=${rpm_version#*-}
   local s3_cache_dir=RELEASE
 
   if ! is_build_needed "${spec_name}" "${rpm_version}"; then
+      echo "Using cached rpm artefacts for ${spec_name}-${rpm_version}."
       return
   fi
 
@@ -126,16 +125,14 @@ build() {
   echo -------------------------------------------------------
 
   local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
-  volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
   volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
-  if [ ${spec_name} != "grafana" ]; then
-      volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
-      volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
-  fi
+  volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
+  volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
+  volume_mounts+=" -v pmm-gobuild:/home/builder/.cache/go-build"
 
   echo "Start building Server RPMs..."
-  echo "spec_name: ${spec_name}"
   echo "repo_name: ${repo_name}"
+  echo "spec_name: ${spec_name}"
   echo "rpm_verison: ${rpm_version}"
 
   docker run --rm --platform=${platform} ${volume_mounts} ${rpmbuild_docker_image} sh -c "
@@ -160,11 +157,6 @@ build() {
       sudo chown builder:builder /home/builder/.cache/yarn
     fi
 
-    # Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-    if [ ${RPM_EPOCH} = 1 ]; then
-      sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-    fi
-
     rm -rf /home/builder/rpm/RPMS/${spec_name}-*
 
     printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
@@ -190,18 +182,17 @@ build() {
     rm -f SOURCES/${spec_name}.spec*
   "
 
-  # TODO: see if this is even needed
-  # cp ${rpms_dir}/${spec_name}-${rpm_version}/*/*.rpm ${rpms_dir}
-
   if [ "$LOCAL_BUILD" = 1 ]; then
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    echo
     return
   fi
 
   if ! command -v aws &> /dev/null; then
-    echo "AWS CLI is not installed. Upload to S3 will be skipped."
+    echo "AWS CLI is not installed, skipping upload to S3..."
     echo
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+    echo
     return
   fi
 
@@ -210,11 +201,12 @@ build() {
   fi
   aws s3 sync \
       --region us-east-2 \
-      ${rpms_dir}/${spec_name}-${rpm_version} \
+      ${rpms_dir}/${spec_name}-${rpm_version}/${arch} \
       s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \
       || :
 
   echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
+  echo
 }
 
 build "$@"
diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script
deleted file mode 100755
index 677f95f6a6..0000000000
--- a/build/scripts/build-server-rpm-script
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-. $(dirname $0)/vars
-
-sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-
-# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build
-if [ ${RPM_EPOCH} = 1 ]; then
-    sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec
-fi
-
-rm -rf /home/builder/rpm/RPMS/${spec_name}-*
-
-printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \
-    | sudo tee /etc/yum.repos.d/local.repo
-
-until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-    echo waiting
-    sleep 1
-done
-
-# Only these two specs have build dependencies
-if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
-    sleep 5s
-    sudo yum-builddep -y SOURCES/${spec_name}.spec
-fi
-
-spectool -C SOURCES -g SOURCES/${spec_name}.spec
-rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
-          --define 'dist .${rpmbuild_dist}' \
-          --define 'debug_package %{nil}' \
-          -ba SOURCES/${spec_name}.spec
-
-rm -f SOURCES/${spec_name}.spec*
-sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
-
-# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/vars b/build/scripts/vars
index ae8a18547d..98405b9fce 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -5,6 +5,10 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
 root_dir=${ROOT_DIR:-$root_dir}
 build_dir=${root_dir}/build
+arch=$(uname -m)
+if [ "${arch}" = "arm64" ]; then
+  arch=aarch64
+fi
 
 # In VERSION file we can have numeric value like '2.0.0' as well as
 # alphanumeric value like '2.0.0-alpha3' which can not be used entirely

From b5f6a58d4fb2770df870d585390807ccaec9d5b5 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 8 Nov 2024 02:03:49 +0300
Subject: [PATCH 091/127] PMM-12899 fix the architecture detection

---
 .../client/{Dockerfile => Dockerfile.el9}     |  0
 build/local/README.md                         |  5 ++++
 build/local/build                             |  4 ---
 build/packages/rpm/server/SPECS/pmm-dump.spec |  2 +-
 build/scripts/build                           | 11 -------
 build/scripts/build-client-docker             | 10 +++----
 build/scripts/build-server-rpm                |  1 -
 build/scripts/create-tags                     | 10 +++----
 build/scripts/vars                            | 30 +++++++++++--------
 9 files changed, 33 insertions(+), 40 deletions(-)
 rename build/docker/client/{Dockerfile => Dockerfile.el9} (100%)
 delete mode 100755 build/scripts/build

diff --git a/build/docker/client/Dockerfile b/build/docker/client/Dockerfile.el9
similarity index 100%
rename from build/docker/client/Dockerfile
rename to build/docker/client/Dockerfile.el9
diff --git a/build/local/README.md b/build/local/README.md
index cd46216213..617eb93d61 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -83,3 +83,8 @@ Currently, local builds target the following environments:
 - PMM Server
   - docker image - docker and Kubernetes environments (amd64)
 
+
+## Todo List
+
+* have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
+* have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
diff --git a/build/local/build b/build/local/build
index 7db0625018..408952e252 100755
--- a/build/local/build
+++ b/build/local/build
@@ -364,11 +364,7 @@ main() {
   fi
 
   if [ "$NO_SERVER" -eq 0 ]; then
-    # Grafana build fails to compile with Go 1.23.x, see https://github.com/grafana/grafana/issues/89796
-    # We need to apply this [patch](https://github.com/grafana/grafana/pull/94742) to fix it
-    export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
     export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
-    export DOCKERFILE=Dockerfile.el9
     export RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
     export LOCAL_BUILD
 
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index 434c034621..d97cb8a68c 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -42,7 +42,7 @@ install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump
 * Mon Apr 1 2024 Alex Demidoff <alexander.demidoff@percona.com> - 0.7.0-2
 - PMM-12899 Use module and build cache
 
-* Tue Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
+* Thu Nov 23 2023 Artem Gavrilov <artem.gavrilov@percona.com> - 0.7.0-ga
 - PMM-12460 Update pmm-dump to v0.7.0-ga version
 
 * Tue Mar 29 2022 Alex Tymchuk <alexander.tymchuk@percona.com> - 0.6.0-1
diff --git a/build/scripts/build b/build/scripts/build
deleted file mode 100755
index 881f99ca5f..0000000000
--- a/build/scripts/build
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o xtrace
-
-bin_dir=$(dirname $0)
-
-${bin_dir}/build-client
-${bin_dir}/build-server
-
-# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index ee66fb4521..d6eb3ee04b 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -10,11 +10,11 @@ main() {
   echo "Building docker image for PMM Client..."
   echo -----------------------------------------
 
-  local DOCKER_FILE_LOCATION=build/source/pmm/build/docker/client # relative to `root_dir`
-  cp ${root_dir}/build/tarball/pmm-client-*.tar.gz ${root_dir}/${DOCKER_FILE_LOCATION}/pmm-client.tar.gz
+  local DOCKERFILE_LOCATION="${build_dir}/source/pmm/build/docker/client"
+  cp ${build_dir}/tarball/pmm-client-*.tar.gz ${DOCKERFILE_LOCATION}/pmm-client.tar.gz
 
   if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-    DOCKER_CLIENT_TAG=perconalab/pmm-client-fb:${full_pmm_version}
+    DOCKER_CLIENT_TAG="perconalab/pmm-client-fb:${full_pmm_version}"
   fi
 
   CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
@@ -23,9 +23,9 @@ main() {
     --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
     --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
     --progress plain \
-    -f ${DOCKER_FILE_LOCATION}/${docker_file} \
+    -f ${DOCKERFILE_LOCATION}/${docker_file} \
     -t ${DOCKER_CLIENT_TAG} \
-    ${DOCKER_FILE_LOCATION}
+    ${DOCKERFILE_LOCATION}
 
   if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_client_tag_file})
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 4152df2d1f..f15c486339 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 set -o errexit
 set -o xtrace
 
diff --git a/build/scripts/create-tags b/build/scripts/create-tags
index 0f12d9f27c..382c30be18 100755
--- a/build/scripts/create-tags
+++ b/build/scripts/create-tags
@@ -52,8 +52,8 @@ for REPO in "${repos[@]}"; do
         git remote set-url origin git@github.com:${REPO_URL}.git
 
         if git tag -l | grep "$TAG"; then
-            echo "Fatal: tag $TAG already exists in $REPO, exiting..."
-            break
+          echo "Fatal: tag $TAG already exists in $REPO, exiting..."
+          break
         fi
 
         echo "Tagging SHA: $(git rev-parse HEAD)"
@@ -65,9 +65,9 @@ for REPO in "${repos[@]}"; do
         fi
 
         if ! git push origin "$TAG"; then
-            echo "Fatal: failed to tag the repository $REPO with $TAG, exiting..."
-            popd >/dev/null
-            break
+          echo "Fatal: failed to tag the repository $REPO with $TAG, exiting..."
+          popd >/dev/null
+          break
         fi
     popd >/dev/null
 done
diff --git a/build/scripts/vars b/build/scripts/vars
index 98405b9fce..c6a0141b3b 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -1,3 +1,6 @@
+# The variables in this file are global. Every script located in this
+# directory needs to have them available in its scope to function properly.
+
 bin_dir=$(cd $(dirname $0); pwd -P)
 # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
 # NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines
@@ -5,10 +8,6 @@ bin_dir=$(cd $(dirname $0); pwd -P)
 root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
 root_dir=${ROOT_DIR:-$root_dir}
 build_dir=${root_dir}/build
-arch=$(uname -m)
-if [ "${arch}" = "arm64" ]; then
-  arch=aarch64
-fi
 
 # In VERSION file we can have numeric value like '2.0.0' as well as
 # alphanumeric value like '2.0.0-alpha3' which can not be used entirely
@@ -24,7 +23,7 @@ full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD)
 full_pmm_version=${full_pmm_version//\//-}
 
 if [[ "${pmm_branch}" =~ ^pmm-3.|^v3$|^main$ ]]; then
-    full_pmm_version=${pmm_base_version}
+  full_pmm_version=${pmm_base_version}
 fi
 
 new_pmm_version=$(cat ${root_dir}/VERSION | awk -F'-' '{print $1}')
@@ -32,8 +31,8 @@ new_pmm_release=$(cat ${root_dir}/VERSION | awk -F'-' '{print $2}')
 
 # Failsafe during refactoring - check that we did not redefine pmm_version in some other script
 if [ -n "${pmm_version}" ] && [ "${new_pmm_version}" != "${pmm_version}" ]; then
-    echo "pmm_version is already defined: ${pmm_version}"
-    exit 1
+  echo "pmm_version is already defined: ${pmm_version}"
+  exit 1
 fi
 
 pmm_version=${new_pmm_version}
@@ -44,18 +43,23 @@ echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versi
 
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
+arch=$(docker run --rm --platform=${platform} ${rpmbuild_docker_image} uname -m 2>/dev/null)
+if [ "${arch}" = "arm64" ]; then
+  arch=aarch64
+fi
+
 rpms_dir=${build_dir}/pmm-server/RPMS
 rpmbuild_dir=${build_dir}/pmm-server/BUILD
 rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
 rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
 source_dir=${build_dir}/source/pmm-client-${pmm_version}
 binary_dir=${build_dir}/binary/pmm-client-${pmm_version}
-client_properties=${root_dir}/build/pmm-client.properties
-docker_file=${DOCKERFILE:-Dockerfile}
-docker_tag_file=${root_dir}/build/docker/TAG
-docker_client_tag_file=${root_dir}/build/docker/CLIENT_TAG
-source_tarball=${root_dir}/build/source_tarball/pmm-client-${pmm_version}.tar.gz
-binary_tarball=${root_dir}/build/tarball/pmm-client-${pmm_version}.tar.gz
+client_properties=${build_dir}/pmm-client.properties
+docker_file=${DOCKERFILE:-Dockerfile.el9}
+docker_tag_file=${build_dir}/docker/TAG
+docker_client_tag_file=${build_dir}/docker/CLIENT_TAG
+source_tarball=${build_dir}/source_tarball/pmm-client-${pmm_version}.tar.gz
+binary_tarball=${build_dir}/tarball/pmm-client-${pmm_version}.tar.gz
 
 # https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/pmm-6401-v1.93.4
 vmagent_commit_hash=58ecb9066574f38f1d1c91ace467316e7f175b09

From 9232350a5b1fd85c36704c2be348866b151fd41c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 8 Nov 2024 02:35:00 +0300
Subject: [PATCH 092/127] PMM-12899 use short commit for FB docker tags

---
 build/local/README.md             |  1 +
 build/local/build                 | 16 ++++++--------
 build/scripts/build-client-docker |  2 +-
 build/scripts/build-server-docker | 36 +++++++++++++++++--------------
 build/scripts/vars                | 15 +++++++------
 5 files changed, 37 insertions(+), 33 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 617eb93d61..d6f06774cb 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -88,3 +88,4 @@ Currently, local builds target the following environments:
 
 * have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
 * have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
+* copy all build scripts to `build/local` to have a separation between old an new build flows
diff --git a/build/local/build b/build/local/build
index 408952e252..3688a04026 100755
--- a/build/local/build
+++ b/build/local/build
@@ -312,7 +312,7 @@ init() {
 
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
 
-  GIT_COMMIT=$(git rev-parse HEAD | head -c 8)
+  GIT_COMMIT=$(git rev-parse --short HEAD)
 
   # Create docker volumes to persist package and build cache
   # Read more in the section about `rpmbuild`.
@@ -337,10 +337,10 @@ cleanup() {
 }
 
 main() {
-  update
-
   init
 
+  update
+
   purge_files
 
   if [ "$NO_CLIENT" -eq 0 ]; then
@@ -358,20 +358,14 @@ main() {
   fi
 
   # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache).
-  export DOCKER_CLIENT_TAG=percona/pmm-client:${GIT_COMMIT}
   if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
     run_build_script build-client-docker
   fi
 
   if [ "$NO_SERVER" -eq 0 ]; then
-    export DOCKER_TAG=percona/pmm-server:${GIT_COMMIT}
     export RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
     export LOCAL_BUILD
 
-    # 3rd-party components
-    run_build_script build-server-rpm grafana
-    run_build_script build-server-rpm victoriametrics
-
     # 1st-party components
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
     run_build_script build-server-rpm pmm-managed pmm
@@ -380,6 +374,10 @@ main() {
     run_build_script build-server-rpm pmm-dump
     run_build_script build-server-rpm vmproxy pmm
 
+    # 3rd-party components
+    run_build_script build-server-rpm victoriametrics
+    run_build_script build-server-rpm grafana
+
     run_build_script build-server-docker
   fi
 
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index d6eb3ee04b..bf446b0eed 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -14,7 +14,7 @@ main() {
   cp ${build_dir}/tarball/pmm-client-*.tar.gz ${DOCKERFILE_LOCATION}/pmm-client.tar.gz
 
   if [ -z "${DOCKER_CLIENT_TAG}" ]; then
-    DOCKER_CLIENT_TAG="perconalab/pmm-client-fb:${full_pmm_version}"
+    DOCKER_CLIENT_TAG="perconalab/pmm-client-fb:${shortcommit}"
   fi
 
   CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 165b534cd7..9cbe95cd64 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -20,33 +20,37 @@ main() {
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
   docker run --rm --platform=${platform} -v ${rpms_dir}:/home/builder/rpm/RPMS ${rpmbuild_docker_image} sh -c "
-      sudo chown -R builder /home/builder/rpm/RPMS
-      until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
-          echo "waiting"
-          sleep 1
-      done
+    sudo chown -R builder /home/builder/rpm/RPMS
+    until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do
+        echo "waiting"
+        sleep 1
+    done
   "
 
   if [ -z "${DOCKER_TAG}" ]; then
-      DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version}
+    DOCKER_TAG=perconalab/pmm-server-fb:${shortcommit}
   fi
 
   local IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2)
   local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@')
+  if [ -z "$IMAGE_VERSION" ]; then
+    echo "Error: could not detect the docker image version"
+    exit 1
+  fi
 
   docker buildx build \
-      --build-arg BUILD_DATE="$BUILD_DATE" \
-      --build-arg VERSION="$IMAGE_VERSION" \
-      --platform=${platform} \
-      --progress plain \
-      -f ${docker_root}/${docker_file} \
-      -t ${DOCKER_TAG} \
-      ${docker_root}
+    --build-arg BUILD_DATE="$BUILD_DATE" \
+    --build-arg VERSION="$IMAGE_VERSION" \
+    --platform=${platform} \
+    --progress plain \
+    -f ${docker_root}/${docker_file} \
+    -t ${DOCKER_TAG} \
+    ${docker_root}
 
   if [ -n "${PUSH_DOCKER}" ]; then
-      mkdir -p $(dirname ${docker_tag_file})
-      echo ${DOCKER_TAG} > ${docker_tag_file}
-      docker push ${DOCKER_TAG}
+    mkdir -p $(dirname ${docker_tag_file})
+    echo ${DOCKER_TAG} > ${docker_tag_file}
+    docker push ${DOCKER_TAG}
   fi
 }
 
diff --git a/build/scripts/vars b/build/scripts/vars
index c6a0141b3b..98601381ed 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -10,15 +10,16 @@ root_dir=${ROOT_DIR:-$root_dir}
 build_dir=${root_dir}/build
 
 # In VERSION file we can have numeric value like '2.0.0' as well as
-# alphanumeric value like '2.0.0-alpha3' which can not be used entirely
-# e.g. in Version directive in spec files. So we define:
-# * full_pmm_version to contain full version and build metadata: '2.0.0-alpha3-PMM-1234-fb-branch-123abc';
-# * pmm_version to contain only MAJOR.MINOR.PATCH: '2.0.0';
-# * pmm_release to contain only pre-release part (can be empty): 'alpha3'.
+# alphanumeric value like '2.0.0-alpha3' which can not be used entirely,
+# e.g. as in Version directive of RPM spec files. So we define the following:
+# 'full_pmm_version' - must contain full version and build metadata: '2.0.0-alpha3-PMM-1234-fb-branch-abc123de'
+# 'pmm_version'      - must contain only MAJOR.MINOR.PATCH: '2.0.0'
+# 'pmm_release'      - can contain only pre-release part (but can also be empty): 'alpha3'
 pmm_branch=$(git rev-parse --abbrev-ref HEAD)
-
 pmm_base_version=$(cat ${root_dir}/VERSION)
-full_pmm_version=${pmm_base_version}-${pmm_branch}-$(git rev-parse --short HEAD)
+
+shortcommit=$(git rev-parse --short HEAD)
+full_pmm_version=${pmm_base_version}-${pmm_branch}-${shortcommit}
 # Replace '/' with '-' to prevent sed from failing on dependabot-authored PRs
 full_pmm_version=${full_pmm_version//\//-}
 

From b170e7199da62838cf8e417db696017488f13397 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 8 Nov 2024 21:51:43 +0300
Subject: [PATCH 093/127] PMM-13487 clean up rpm specs

---
 build/local/README.md                         |  10 +-
 build/local/build                             |  19 +-
 build/packages/rpm/server/SPECS/grafana.spec  |   8 +-
 .../rpm/server/SPECS/percona-qan-api2.spec    |   9 +-
 .../rpm/server/SPECS/pmm-managed.spec         |   2 +-
 .../packages/rpm/server/SPECS/pmm-update.spec |   4 -
 build/scripts/build-client-packages           | 198 +++++++++---------
 build/scripts/build-server-rpm                |   2 +-
 build/scripts/vars                            |   2 +-
 9 files changed, 124 insertions(+), 130 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index d6f06774cb..533a557ba7 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -77,15 +77,15 @@ It's important to note, however, that once all changes are made and tested, you
 
 Currently, local builds target the following environments:
 - PMM Client
-  - tarball - virtually any amd64 Linux environment
-  - RPM - RHEL9-compatible environments
-  - docker image - docker and Kubernetes environments (amd64)
+  - tarball - most Linux environments, noarch
+  - RPM - RHEL9-compatible environments, amd64/aarch64
+  - docker image - docker and Kubernetes environments amd64/aarch64
 - PMM Server
-  - docker image - docker and Kubernetes environments (amd64)
+  - docker image - docker and Kubernetes environments amd64/aarch64
 
 
 ## Todo List
 
 * have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
 * have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
-* copy all build scripts to `build/local` to have a separation between old an new build flows
+* copy all build scripts to `build/local` to isolate the new build flows from the old ones
diff --git a/build/local/build b/build/local/build
index 3688a04026..b8658559ee 100755
--- a/build/local/build
+++ b/build/local/build
@@ -9,17 +9,16 @@ Options:
       --platform <platform>    Build for a specific platform (defaults to linux/amd64)
       --no-update              Do not fetch the latest changes from the repo
       --update-only            Only fetch the latest changes from the repo
+      --client-only            Build only PMM Client (client binaries + docker)
       --no-client              Do not build PMM Client
       --no-client-docker       Do not build PMM Client docker image (default)
-      --client-only            Build only PMM Client (client binaries + docker)
-      --no-server              Do not build PMM Server (docker image)
       --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
   -d  --debug                  Log output in debug mode, which also prints the commands
   -h  --help                   Display help
 EOF
 }
 
-parse-params() {
+parse_params() {
   # Define global variables
   LOCAL_BUILD=1
   NO_UPDATE=0
@@ -67,9 +66,6 @@ parse-params() {
         fi
         NO_CLIENT_DOCKER=1
         ;;
-      --no-server)
-        NO_SERVER=1
-        ;;
       --platform)
         shift
         if [ -z "$1" ]; then
@@ -97,6 +93,7 @@ parse-params() {
         ;;
       *)
         echo "Unknown argument: $1"
+        echo
         usage
         exit 1
         ;;
@@ -105,7 +102,7 @@ parse-params() {
   done
 }
 
-needs-to-pull() {
+needs_to_pull() {
   local UPSTREAM=${1:-'@{u}'}
   local LOCAL=$(git rev-parse @)
   local BASE=$(git merge-base @ "$UPSTREAM")
@@ -133,7 +130,7 @@ rewind() {
     git checkout "$BRANCH"
   fi
 
-  if needs-to-pull; then
+  if needs_to_pull; then
     if ! git pull origin; then
       git reset --hard HEAD~20
       git pull origin
@@ -148,7 +145,7 @@ rewind() {
   fi
 }
 
-check-files() {
+check_files() {
   local DIR="$1"
 
   test -z "DIR" && exit 1
@@ -180,7 +177,7 @@ update() {
   fi
 
   if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules
-    if ! check-files "$SUBMODULES"; then
+    if ! check_files "$SUBMODULES"; then
       echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}"
       exit 1
     fi
@@ -390,7 +387,7 @@ main() {
   cleanup
 }
 
-parse-params "$@"
+parse_params "$@"
 
 # Capture the build logs in the log file
 exec > >(tee "$LOG_FILE") 2>&1
diff --git a/build/packages/rpm/server/SPECS/grafana.spec b/build/packages/rpm/server/SPECS/grafana.spec
index 4e3eba1aec..39d48ff337 100644
--- a/build/packages/rpm/server/SPECS/grafana.spec
+++ b/build/packages/rpm/server/SPECS/grafana.spec
@@ -12,6 +12,8 @@
 %define gobuild(o:) go build -ldflags "${LDFLAGS:-} -B 0x$(head -c20 /dev/urandom | od -An -tx1 | tr -d ' \\n')" -a -v -x %{?**};
 %endif
 
+# Source0: is modified by build-server-rpm's sed
+
 Name:           percona-grafana
 Version:        %{grafana_version}
 Release:        %{rpm_release}
@@ -65,9 +67,9 @@ install -d -p %{buildroot}%{_sharedstatedir}/grafana
 %{_datadir}/grafana
 %doc CHANGELOG.md README.md
 %license LICENSE
-%attr(0755, pmm, pmm) %{_sbindir}/grafana
-%attr(0755, pmm, pmm) %{_sbindir}/grafana-server
-%attr(0755, pmm, pmm) %{_bindir}/grafana-cli
+%attr(0755, root, root) %{_sbindir}/grafana
+%attr(0755, root, root) %{_sbindir}/grafana-server
+%attr(0755, root, root) %{_bindir}/grafana-cli
 %{_sysconfdir}/grafana/grafana.ini
 %{_sysconfdir}/grafana/ldap.toml
 %dir %{_sharedstatedir}/grafana
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 3f851cb36f..4230bdfb51 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -1,14 +1,13 @@
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
 %global import_path     %{provider}
-# The commit hash gets sed'ed by build-server-rpm script to set a correct version
-# see: https://github.com/percona/pmm/blob/main/build/scripts/build-server-rpm#L58
+# The commit hash gets sed'ed by build-server-rpm script
 %global commit          0000000000000000000000000000000000000000
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
 %define release         18
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
-# the line below is sed'ed by build/bin/build-server-rpm to set a correct version
+# The line below is sed'ed by build-server-rpm
 %define full_pmm_version 2.0.0
 
 Name:           percona-qan-api2
@@ -21,8 +20,8 @@ URL:            https://%{provider}
 Source0:        https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
 
 %description
-Percona Query Analytics (QAN) API v2 is part of Percona Monitoring and Management (PMM).
-See PMM docs for more information - https://docs.percona.com/percona-monitoring-and-management/using/query-analytics.html.
+Percona Query Analytics (QAN) API is part of Percona Monitoring and Management (PMM).
+Refer to PMM docs for more information - https://docs.percona.com/percona-monitoring-and-management/get-started/query-analytics.html.
 
 
 %prep
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index a7ad40049e..81f7ed58e6 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -9,7 +9,7 @@
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
-%define full_pmm_version 2.0.0
+%define full_pmm_version 3.0.0
 
 Name:     pmm-managed
 Version:  %{version}
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 5e93dea47a..8e577bad02 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,12 +1,8 @@
 %undefine _missing_build_ids_terminate_build
-# TODO: remove it as soon as we remove all noarch pmm-update rpms
-# from 'pmm3-components/yum/laboratory'
-%define _binaries_in_noarch_packages_terminate_build   0
 %define _unpackaged_files_terminate_build   0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
-%global import_path     %{provider}
 %global commit	        592eddf656bce32a11bd958af0a32c62bd5ea34c
 %global shortcommit	    %(c=%{commit}; echo ${c:0:7})
 %define release         68
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index c6499fa956..9596666fc1 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -214,107 +214,107 @@ build_rpm(){
 }
 
 build_source_deb(){
-    if [ "$OS" = "rpm" ]; then
-        echo "It is not possible to build source deb here"
-        exit 1
+  if [ "$OS" = "rpm" ]; then
+    echo "It is not possible to build source deb here"
+    exit 1
+  fi
+  rm -rf pmm-client*
+  get_tar "tarball"
+  rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
+
+  TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
+  NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
+  VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
+  VERSION=${VERSION_TMP%.tar.gz}
+
+  rm -fr ${NAME}-${VERSION}
+
+  NEWTAR=${NAME}_${VERSION}.orig.tar.gz
+  mv ${TARFILE} ${NEWTAR}
+
+  git clone $REPO ${NAME}-${VERSION}_all
+  pushd ${NAME}-${VERSION}_all
+    git fetch origin
+    if [ ! -z ${BRANCH} ]; then
+      git reset --hard
+      git clean -xdf
+      git checkout ${BRANCH}
     fi
-    rm -rf pmm-client*
-    get_tar "tarball"
-    rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
-
-    TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
-    NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
-    VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
-    VERSION=${VERSION_TMP%.tar.gz}
-
-    rm -fr ${NAME}-${VERSION}
-
-    NEWTAR=${NAME}_${VERSION}.orig.tar.gz
-    mv ${TARFILE} ${NEWTAR}
-
-    git clone $REPO ${NAME}-${VERSION}_all
-    pushd ${NAME}-${VERSION}_all
-        git fetch origin
-        if [ ! -z ${BRANCH} ]; then
-            git reset --hard
-            git clean -xdf
-            git checkout ${BRANCH}
-        fi
-        REVISION=$(git rev-parse --short HEAD)
-        mkdir distro
-    popd
-
-    tar xzf ${NEWTAR}
-    cd ${NAME}-${VERSION}
-    mv bin/* ../${NAME}-${VERSION}_all/distro/
-    mv example.prom ../${NAME}-${VERSION}_all/distro/
-    mv queries-mysqld.yml ../${NAME}-${VERSION}_all/distro/
-    mv queries-mysqld-group-replication.yml ../${NAME}-${VERSION}_all/distro/
-    mv example-queries-postgres.yml ../${NAME}-${VERSION}_all/distro/
-    mv queries-postgres-uptime.yml ../${NAME}-${VERSION}_all/distro/
-    mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/
-    mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/
-    mv debian ../${NAME}-${VERSION}_all/
-    mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service
-    cd ../
-
-    rm -rf ${NAME}-${VERSION}
-    mv ${NAME}-${VERSION}_all ${NAME}-${VERSION}
-    rm -rf *.tar.gz
-    tar -zcvf ${NEWTAR} ${NAME}-${VERSION}
-    cd ${NAME}-${VERSION}
-    sed -i "s/%{version}/$VERSION-$DEB_RELEASE/" debian/control
-
-    cd debian
-    echo "${NAME} (${VERSION}) unstable; urgency=low" >> changelog
-    echo "  * Initial Release." >> changelog
-    echo " -- EvgeniyPatlan <evgeniy.patlan@percona.com> $(date -R)" >> changelog
-
-    cd ../
-
-    dch -D unstable --force-distribution -v "${VERSION}-${DEB_RELEASE}" "Update to new upstream release PMM-client ${VERSION}-${DEB_RELEASE}"
-    dpkg-buildpackage -S
-    cd ../
-    mkdir -p $WORKDIR/source_deb
-    mkdir -p $CURDIR/source_deb
-    cp *.diff.gz $WORKDIR/source_deb
-    cp *_source.changes $WORKDIR/source_deb
-    cp *.dsc $WORKDIR/source_deb
-    cp *.orig.tar.gz $WORKDIR/source_deb
-    cp *.diff.gz $CURDIR/source_deb
-    cp *_source.changes $CURDIR/source_deb
-    cp *.dsc $CURDIR/source_deb
-    cp *.orig.tar.gz $CURDIR/source_deb
+    REVISION=$(git rev-parse --short HEAD)
+    mkdir distro
+  popd
+
+  tar xzf ${NEWTAR}
+  cd ${NAME}-${VERSION}
+  mv bin/* ../${NAME}-${VERSION}_all/distro/
+  mv example.prom ../${NAME}-${VERSION}_all/distro/
+  mv queries-mysqld.yml ../${NAME}-${VERSION}_all/distro/
+  mv queries-mysqld-group-replication.yml ../${NAME}-${VERSION}_all/distro/
+  mv example-queries-postgres.yml ../${NAME}-${VERSION}_all/distro/
+  mv queries-postgres-uptime.yml ../${NAME}-${VERSION}_all/distro/
+  mv queries-mr.yaml ../${NAME}-${VERSION}_all/distro/
+  mv queries-lr.yaml ../${NAME}-${VERSION}_all/distro/
+  mv debian ../${NAME}-${VERSION}_all/
+  mv config/pmm-agent.service ../${NAME}-${VERSION}_all/debian/${PACKAGE_NAME}.pmm-agent.service
+  cd ../
+
+  rm -rf ${NAME}-${VERSION}
+  mv ${NAME}-${VERSION}_all ${NAME}-${VERSION}
+  rm -rf *.tar.gz
+  tar -zcvf ${NEWTAR} ${NAME}-${VERSION}
+  cd ${NAME}-${VERSION}
+  sed -i "s/%{version}/$VERSION-$DEB_RELEASE/" debian/control
+
+  cd debian
+  echo "${NAME} (${VERSION}) unstable; urgency=low" >> changelog
+  echo "  * Initial Release." >> changelog
+  echo " -- EvgeniyPatlan <evgeniy.patlan@percona.com> $(date -R)" >> changelog
+
+  cd ../
+
+  dch -D unstable --force-distribution -v "${VERSION}-${DEB_RELEASE}" "Update to new upstream release PMM-client ${VERSION}-${DEB_RELEASE}"
+  dpkg-buildpackage -S
+  cd ../
+  mkdir -p $WORKDIR/source_deb
+  mkdir -p $CURDIR/source_deb
+  cp *.diff.gz $WORKDIR/source_deb
+  cp *_source.changes $WORKDIR/source_deb
+  cp *.dsc $WORKDIR/source_deb
+  cp *.orig.tar.gz $WORKDIR/source_deb
+  cp *.diff.gz $CURDIR/source_deb
+  cp *_source.changes $CURDIR/source_deb
+  cp *.dsc $CURDIR/source_deb
+  cp *.orig.tar.gz $CURDIR/source_deb
 }
 
 build_deb(){
-    if [ "$OS" = "rpm" ]; then
-        echo "It is not possible to build deb here"
-        exit 1
-    fi
-    for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
-        get_deb_sources $file
-    done
-
-    cd $WORKDIR
-    rm -fv *.deb
-    export DEBIAN_VERSION="$(lsb_release -sc)"
-    DSC=$(basename $(find . -name '*.dsc' | sort | tail -n 1))
-    DIRNAME=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $1"-"$2}')
-    VERSION=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $3}')
-    ARCH=$(uname -m)
-    echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $VERSION_FILE
-    echo "ARCH=${ARCH}" >> $VERSION_FILE
-    #
-    dpkg-source -x ${DSC}
-    cd ${DIRNAME}-${VERSION}
-    dch -b -m -D "$DEBIAN_VERSION" --force-distribution -v "${VERSION}-${DEB_RELEASE}.${DEBIAN_VERSION}" 'Update distribution'
-
-    dpkg-buildpackage -rfakeroot -uc -us -b
-    mkdir -p $CURDIR/deb
-    mkdir -p $WORKDIR/deb
-    cp $WORKDIR/*.deb $WORKDIR/deb
-    cp $WORKDIR/*.deb $CURDIR/deb
+  if [ "$OS" = "rpm" ]; then
+    echo "It is not possible to build deb here"
+    exit 1
+  fi
+  for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
+    get_deb_sources $file
+  done
+
+  cd $WORKDIR
+  rm -fv *.deb
+  export DEBIAN_VERSION="$(lsb_release -sc)"
+  DSC=$(basename $(find . -name '*.dsc' | sort | tail -n 1))
+  DIRNAME=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $1"-"$2}')
+  VERSION=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $3}')
+  ARCH=$(uname -m)
+  echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $VERSION_FILE
+  echo "ARCH=${ARCH}" >> $VERSION_FILE
+  #
+  dpkg-source -x ${DSC}
+  cd ${DIRNAME}-${VERSION}
+  dch -b -m -D "$DEBIAN_VERSION" --force-distribution -v "${VERSION}-${DEB_RELEASE}.${DEBIAN_VERSION}" 'Update distribution'
+
+  dpkg-buildpackage -rfakeroot -uc -us -b
+  mkdir -p $CURDIR/deb
+  mkdir -p $WORKDIR/deb
+  cp $WORKDIR/*.deb $WORKDIR/deb
+  cp $WORKDIR/*.deb $CURDIR/deb
 }
 
 #main
@@ -340,8 +340,8 @@ BRANCH="v3"
 REPO="https://github.com/Percona-Lab/pmm-submodules.git"
 
 if [ -n "$pmm_release" ]; then
-    RPM_RELEASE="$RPM_RELEASE.$pmm_release"
-    DEB_RELEASE="$DEB_RELEASE.$pmm_release"
+  RPM_RELEASE="$RPM_RELEASE.$pmm_release"
+  DEB_RELEASE="$DEB_RELEASE.$pmm_release"
 fi
 
 parse_arguments "$@"
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index f15c486339..f462406bb1 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -56,7 +56,7 @@ prepare_spec() {
   mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
   cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
   if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
-    # TODO: see if we should not exit with error here
+    # Exit if the repo can't be found in pmm-submodules (e.g. VictoriaMetrics)
     return
   fi
   if [ ! -f "${spec_file}" ]; then
diff --git a/build/scripts/vars b/build/scripts/vars
index 98601381ed..30b0a082e2 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -30,7 +30,7 @@ fi
 new_pmm_version=$(cat ${root_dir}/VERSION | awk -F'-' '{print $1}')
 new_pmm_release=$(cat ${root_dir}/VERSION | awk -F'-' '{print $2}')
 
-# Failsafe during refactoring - check that we did not redefine pmm_version in some other script
+# Check that we did not redefine pmm_version in some other script
 if [ -n "${pmm_version}" ] && [ "${new_pmm_version}" != "${pmm_version}" ]; then
   echo "pmm_version is already defined: ${pmm_version}"
   exit 1

From 958b8187a187fd5f5b4a064262097d29bd79b666 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 9 Nov 2024 01:27:46 +0300
Subject: [PATCH 094/127] PMM-13487 promote the use of build_dir

---
 build/local/README.md             |  4 ++-
 build/local/build                 |  7 +++--
 build/scripts/build-client-deb    | 44 +++++++++++++++----------------
 build/scripts/build-client-docker |  8 +++---
 build/scripts/build-client-rpm    | 36 ++++++++++++-------------
 build/scripts/build-client-sdeb   |  2 +-
 build/scripts/build-client-srpm   |  2 +-
 build/scripts/build-server        |  2 +-
 build/scripts/build-server-docker |  2 +-
 build/scripts/build-server-rpm    |  2 +-
 build/scripts/vars                |  2 --
 11 files changed, 56 insertions(+), 55 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index 533a557ba7..c2d163796a 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -65,9 +65,9 @@ Sometimes, the changes you make affect only PMM Client. Other times, they affect
 
 * --no-update: run the build tasks without pulling the changes from `pmm-submodules` repository
 * --update-only: pull changes from the repo without building PMM
+* --client-only: build PMM Client only, skip building the Server
 * --no-client: do not build the client, use the cached PMM Client artifacts
 * --no-client-docker: skip building PMM Client docker container
-* --no-server-rpm: skip building PMM Server RPM artifacts
 * --log-file <path>: change the path of the build log file
 
 It's important to note, however, that once all changes are made and tested, you most probably want to re-build both PMM Client and Server to test them together.
@@ -86,6 +86,8 @@ Currently, local builds target the following environments:
 
 ## Todo List
 
+* download the sources to a local directory `.modules` w/o using pmm-submodules
 * have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
 * have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
 * copy all build scripts to `build/local` to isolate the new build flows from the old ones
+* use the `--debug` parameter to control the verbosity of the logs
diff --git a/build/local/build b/build/local/build
index b8658559ee..496a7d93ef 100755
--- a/build/local/build
+++ b/build/local/build
@@ -32,6 +32,7 @@ parse_params() {
   BASE_NAME=$(basename $0)
   PLATFORM=linux/amd64
   SUBMODULES=pmm-submodules
+  ROOT_DIR="$(realpath ./${SUBMODULES})"
   PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
 
   while test "$#" -gt 0; do
@@ -83,8 +84,8 @@ parse_params() {
         LOG_FILE="$1"
         ;;
       --debug | -d)
+        DEBUG_MODE=1
         shift
-        export DEBUG_MODE=1
         ;;
       --help | -h)
         shift
@@ -308,6 +309,9 @@ init() {
   fi
 
   export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+  export DEBUG_MODE
+  export LOCAL_BUILD
+  export ROOT_DIR
 
   GIT_COMMIT=$(git rev-parse --short HEAD)
 
@@ -361,7 +365,6 @@ main() {
 
   if [ "$NO_SERVER" -eq 0 ]; then
     export RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
-    export LOCAL_BUILD
 
     # 1st-party components
     run_build_script build-server-rpm percona-dashboards grafana-dashboards
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index 38112cbabb..bac55a0892 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -1,32 +1,30 @@
 #!/bin/bash
-
-. $(dirname $0)/vars
-
 set -o errexit
 set -o xtrace
 
-bin_dir=$(cd $(dirname $0); pwd -P)
+. $(dirname $0)/vars
 
 main() {
-    local IMAGE=${1:-ubuntu:focal}
-    docker run --rm -e DEBIAN_FRONTEND=noninteractive \
-                    -v ${bin_dir}:/home/builder/bin \
-                    -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
-        set -o errexit
-        set -o xtrace
-
-        export pmm_release=$pmm_release
-        OS_VERSION=\$(cat /etc/os-release | grep VERSION_ID | awk -F'\"' '{print \$2}')
-
-        apt-get update
-        case "\$OS_VERSION" in
-            11 | 12 | 22.04 | 24.04) apt-get -y install git lsb-release devscripts dh-make;;
-            *) apt-get -y install git lsb-release devscripts dh-make dh-systemd;;
-        esac
-
-        cd /home/builder/build
-        bash /home/builder/bin/build-client-packages --build_deb=1
-    "
+  local IMAGE=${1:-ubuntu:focal}
+  docker run --rm \
+    -e DEBIAN_FRONTEND=noninteractive \
+    -v ${bin_dir}:/home/builder/bin \
+    -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
+      set -o errexit
+      set -o xtrace
+
+      export pmm_release=$pmm_release
+      OS_VERSION=\$(cat /etc/os-release | grep VERSION_ID | awk -F'\"' '{print \$2}')
+
+      apt-get update
+      case "\$OS_VERSION" in
+          11 | 12 | 22.04 | 24.04) apt-get -y install git lsb-release devscripts dh-make;;
+          *) apt-get -y install git lsb-release devscripts dh-make dh-systemd;;
+      esac
+
+      cd /home/builder/build
+      bash /home/builder/bin/build-client-packages --build_deb=1
+  "
 }
 
 main $*
diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index bf446b0eed..38531f7151 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -10,8 +10,8 @@ main() {
   echo "Building docker image for PMM Client..."
   echo -----------------------------------------
 
-  local DOCKERFILE_LOCATION="${build_dir}/source/pmm/build/docker/client"
-  cp ${build_dir}/tarball/pmm-client-*.tar.gz ${DOCKERFILE_LOCATION}/pmm-client.tar.gz
+  local DOCKER_CONTEXT_DIR="${bin_dir}/../docker/client"
+  cp ${build_dir}/tarball/pmm-client-*.tar.gz ${DOCKER_CONTEXT_DIR}/pmm-client.tar.gz
 
   if [ -z "${DOCKER_CLIENT_TAG}" ]; then
     DOCKER_CLIENT_TAG="perconalab/pmm-client-fb:${shortcommit}"
@@ -23,9 +23,9 @@ main() {
     --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
     --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
     --progress plain \
-    -f ${DOCKERFILE_LOCATION}/${docker_file} \
+    -f ${DOCKER_CONTEXT_DIR}/${docker_file} \
     -t ${DOCKER_CLIENT_TAG} \
-    ${DOCKERFILE_LOCATION}
+    ${DOCKER_CONTEXT_DIR}
 
   if [ -n "${PUSH_DOCKER}" ]; then
     mkdir -p $(dirname ${docker_client_tag_file})
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 5069ced42e..5069943c01 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -6,24 +6,24 @@ set -o errexit
 set -o xtrace
 
 main() {
-    echo -----------------------------------------
-    echo "Building PMM Client RPM files..."
-    echo -----------------------------------------
-
-    local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm \
-      -v ${bin_dir}:/home/builder/bin \
-      -v ${root_dir}/build:/home/builder/build \
-      ${IMAGE} sh -c "
-        set -o errexit
-        set -o xtrace
-
-        export pmm_version=$pmm_version
-        export pmm_release=$pmm_release
-        cd /home/builder/build
-
-        /home/builder/bin/build-client-packages --build_rpm=1
-      "
+  echo -----------------------------------------
+  echo "Building PMM Client RPM files..."
+  echo -----------------------------------------
+
+  local IMAGE=${1:-${rpmbuild_docker_image}}
+  docker run --rm \
+    -v ${bin_dir}:/home/builder/bin \
+    -v ${build_dir}:/home/builder/build \
+    ${IMAGE} sh -c "
+      set -o errexit
+      set -o xtrace
+
+      export pmm_version=$pmm_version
+      export pmm_release=$pmm_release
+      cd /home/builder/build
+
+      /home/builder/bin/build-client-packages --build_rpm=1
+    "
 }
 
 main $*
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index 845663de91..6466bc7f9a 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -11,7 +11,7 @@ main() {
     local IMAGE=${1:-ubuntu:focal}
     docker run --rm -e DEBIAN_FRONTEND=noninteractive \
                     -v ${bin_dir}:/home/builder/bin \
-                    -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
+                    -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index b669415486..b6bef5729f 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -11,7 +11,7 @@ main() {
     echo -----------------------------------------
 
     local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${root_dir}/build:/home/builder/build ${IMAGE} sh -c "
+    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
         set -o errexit
         set -o xtrace
 
diff --git a/build/scripts/build-server b/build/scripts/build-server
index 0b53f5731f..51692b6495 100755
--- a/build/scripts/build-server
+++ b/build/scripts/build-server
@@ -11,7 +11,7 @@ pmm_version=${pmm_version} ${bin_dir}/build-client
 pmm_version=${pmm_version} ${bin_dir}/build-client-srpm
 pmm_version=${pmm_version} ${bin_dir}/build-client-rpm
 mkdir -p ${rpms_dir} ${rpmbuild_dir}
-cp ${root_dir}/build/rpm/*.rpm ${rpms_dir}/
+cp ${build_dir}/rpm/*.rpm ${rpms_dir}/
 
 # install all rpms
 ${bin_dir}/build-server-rpm-all
diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker
index 9cbe95cd64..78cd009abd 100755
--- a/build/scripts/build-server-docker
+++ b/build/scripts/build-server-docker
@@ -15,7 +15,7 @@ main() {
 
   cp -r "${root_dir}/${pmm_source_dir}/build/ansible" "${docker_root}/ansible"
   cp "${root_dir}/${pmm_source_dir}/build/docker/server/"* "${docker_root}"
-  cp ${root_dir}/build/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
+  cp ${build_dir}/tarball/pmm-client-*.tar.gz "${docker_root}/pmm-client.tar.gz"
 
   git -C ${root_dir} rev-parse HEAD > ${docker_root}/gitCommit
   ls -la ${rpms_dir}
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index f462406bb1..b2acadaba9 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -90,7 +90,7 @@ prepare_spec() {
     grep -r 'define release' ${spec_file}
   fi
 
-  if [ "${spec_name}" = "grafana" ]; then
+  if [ "${spec_name}" != "victorimetrics" ]; then
     sed -i -e "s;\(Source0:[[:space:]]*\)http.*;\1${tar_archive};" ${spec_file}
   fi
 
diff --git a/build/scripts/vars b/build/scripts/vars
index 30b0a082e2..fa98134428 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -3,8 +3,6 @@
 
 bin_dir=$(cd $(dirname $0); pwd -P)
 # TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
-# NOTE: in most cases, this evaluates to Jenkins ${WORKSPACE} directory when called from pipelines
-# Example: /home/ec2-user/workspace/ol9-build-server/, see how it's used in ol9-build-server.groovy
 root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
 root_dir=${ROOT_DIR:-$root_dir}
 build_dir=${root_dir}/build

From 65ab5f5a199c05ce1106335f6337378b4534e49c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 9 Nov 2024 02:37:26 +0300
Subject: [PATCH 095/127] PMM-13487 rename image from users to base

---
 build/docker/client/Dockerfile.el9 | 42 +++++++++++++++---------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/build/docker/client/Dockerfile.el9 b/build/docker/client/Dockerfile.el9
index 2e729c4ece..0958a03f22 100644
--- a/build/docker/client/Dockerfile.el9
+++ b/build/docker/client/Dockerfile.el9
@@ -1,4 +1,4 @@
-FROM redhat/ubi9-minimal AS users
+FROM redhat/ubi9-minimal AS base
 
 RUN microdnf install shadow-utils jq tar -y
 
@@ -13,30 +13,30 @@ ARG VERSION
 ARG BUILD_DATE
 
 COPY LICENSE /licenses/
-COPY --from=users --chown=0:0 /etc/passwd /etc/passwd
-COPY --from=users --chown=0:0 /etc/group /etc/group
+COPY --from=base --chown=0:0 /etc/passwd /etc/passwd
+COPY --from=base --chown=0:0 /etc/group /etc/group
 
 # Install jq along with its dependencies
-COPY --from=users --chown=0:0 /usr/bin/jq /usr/bin/
-COPY --from=users --chown=0:0 /usr/bin/tar /usr/bin/
-COPY --from=users --chown=0:0 /usr/bin/curl /usr/bin/
+COPY --from=base --chown=0:0 /usr/bin/jq /usr/bin/
+COPY --from=base --chown=0:0 /usr/bin/tar /usr/bin/
+COPY --from=base --chown=0:0 /usr/bin/curl /usr/bin/
 
-COPY --from=users --chown=0:0 /usr/lib64/libjq.so.1.0.4 /lib64/libjq.so.1
-COPY --from=users --chown=0:0 /usr/lib64/libonig.so.5.1.0 /lib64/libonig.so.5
+COPY --from=base --chown=0:0 /usr/lib64/libjq.so.1.0.4 /lib64/libjq.so.1
+COPY --from=base --chown=0:0 /usr/lib64/libonig.so.5.1.0 /lib64/libonig.so.5
 
 # Dependencies of curl
-COPY --from=users --chown=0:0 /usr/lib64/libkeyutils.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libz.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libkrb5support.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libkrb5.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libk5crypto.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libgssapi_krb5.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libcom_err.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libssl.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libcrypto.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libcurl.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libnghttp2.so* /lib64/
-COPY --from=users --chown=0:0 /usr/lib64/libresolv.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libkeyutils.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libz.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libkrb5support.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libkrb5.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libk5crypto.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libgssapi_krb5.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcom_err.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libssl.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcrypto.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcurl.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libnghttp2.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libresolv.so* /lib64/
 
 RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
 
@@ -57,6 +57,6 @@ LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-
 
 USER pmm-agent
 WORKDIR /usr/local/percona/pmm/
-ENV PATH /usr/local/percona/pmm/bin/:$PATH
+ENV PATH=/usr/local/percona/pmm/bin/:$PATH
 
 ENTRYPOINT ["/usr/local/percona/pmm/bin/pmm-agent-entrypoint"]

From 56173826e6f558a5815eaf4a4cad345ea63b4272 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 9 Nov 2024 03:40:51 +0300
Subject: [PATCH 096/127] PMM-13487 refactor check_files

---
 build/local/README.md             |   2 +-
 build/local/build                 |  43 +++++-----
 build/scripts/build-client-source | 128 +++++++++++++++---------------
 build/scripts/vars                |   5 +-
 4 files changed, 85 insertions(+), 93 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index c2d163796a..7055338fd6 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -84,7 +84,7 @@ Currently, local builds target the following environments:
   - docker image - docker and Kubernetes environments amd64/aarch64
 
 
-## Todo List
+## Draft Todo List
 
 * download the sources to a local directory `.modules` w/o using pmm-submodules
 * have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
diff --git a/build/local/build b/build/local/build
index 496a7d93ef..33817696ab 100755
--- a/build/local/build
+++ b/build/local/build
@@ -149,13 +149,22 @@ rewind() {
 check_files() {
   local DIR="$1"
 
-  test -z "DIR" && exit 1
+  # Thouroughly verify the presence of known files, otherwise bail out
+  if [ ! -d "$DIR" ] ; then
+    echo "Error: could not locate the 'pmm-submodules' directory."
+    exit 1
+  fi
 
-  if [ -d "$DIR/sources" ] && [ -f "$DIR/ci-default.yml" ] && [ -f "$DIR/ci.yml" ]; then
-    return 0
+  if [ ! -d "$DIR/sources" ] || [ ! -d "$DIR/.git" ] || [ ! -f "$DIR/.gitmodules" ] || [ ! -f "$DIR/ci-default.yml" ]; then
+    echo "Error: directory $DIR does not look like a clone of https://github.com/percona-lab/pmm-submodules repository, exiting..."
+    exit 1
   fi
 
-  return 1
+  if [ ! -f "ci.yml" ] || [ ! -s "ci.yml" ]; then
+    echo "Error: the working directory '$PWD' must contain a non-empty ci.yml file."
+    echo "Please refer to this [README](https://github.com/Percona-Lab/pmm-submodules/blob/v3/README.md#how-to-create-a-feature-build) for more information."
+    exit 1
+  fi
 }
 
 # Update submodules and PR branches
@@ -170,29 +179,13 @@ update() {
     return
    fi
 
-  # Thouroughly verify the presence of known files, otherwise bail out
-  if [ ! -d "$SUBMODULES" ] ; then # pwd must be outside of pmm-submodules
-    echo "Warn: the current working directory must be outside of pmm-submodules"
-    echo "cd .."
-    cd .. > /dev/null
-  fi
-
-  if [ -d "$SUBMODULES" ]; then # pwd is outside pmm-submodules
-    if ! check_files "$SUBMODULES"; then
-      echo "Fatal: could not locate known files in ${PWD}/${SUBMODULES}"
-      exit 1
-    fi
-  else
-    echo "Fatal: could not locate known files in $PWD"
-    exit 1
-  fi
-
   cd "$SUBMODULES"
 
   # Join the dependencies from ci-default.yml and ci.yml
-  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ci.yml | jq '.deps')
+  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ../ci.yml | jq '.deps')
 
-  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+  echo
+  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and '../ci.yml'"
 
   echo "$DEPS" | jq -c '.[]' | while read -r item; do
     branch=$(echo "$item" | jq -r '.branch')
@@ -313,8 +306,6 @@ init() {
   export LOCAL_BUILD
   export ROOT_DIR
 
-  GIT_COMMIT=$(git rev-parse --short HEAD)
-
   # Create docker volumes to persist package and build cache
   # Read more in the section about `rpmbuild`.
   for volume in pmm-gobuild pmm-gomod pmm-yarn pmm-dnf; do
@@ -338,6 +329,8 @@ cleanup() {
 }
 
 main() {
+  check_files "$SUBMODULES"
+
   init
 
   update
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index fbcfc00d4a..358447e8d2 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -6,84 +6,84 @@ set -o xtrace
 . $(dirname $0)/vars
 
 prepare_tarball() {
-    local repo_name=$1
-    local git_dir
-    if [ $repo_name = 'pmm-submodules' ]; then
-        git_dir="${root_dir}/sources/${repo_name}"
-        mkdir -p "${git_dir}"
-    else
-        git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-    fi
-    local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
-    local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
-    local repo_var=$(
-        echo "${repo_name}_commit" \
-            | sed -e 's:-:_:g' \
-            | tr '[[:lower:]]' '[[:upper:]]'
-    )
-    echo "${repo_var}=$commit" >> ${client_properties}
+  local repo_name=$1
+  local git_dir
+  if [ $repo_name = 'pmm-submodules' ]; then
+    git_dir="${root_dir}/sources/${repo_name}"
+    mkdir -p "${git_dir}"
+  else
+    git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+  fi
+  local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
+  local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
+  local repo_var=$(
+    echo "${repo_name}_commit" \
+        | sed -e 's:-:_:g' \
+        | tr '[[:lower:]]' '[[:upper:]]'
+  )
+  echo "${repo_var}=$commit" >> ${client_properties}
 
-    if [ -f "${tarball}" ]; then
-        echo ${tarball} already exists, skipping build
-        return
-    fi
+  if [ -f "${tarball}" ]; then
+    echo ${tarball} already exists, skipping build
+    return
+  fi
 
-    if [ ${repo_name} = 'pmm-submodules' ]; then
-        tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
-        mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
-        return
-    fi
+  if [ ${repo_name} = 'pmm-submodules' ]; then
+    tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
+    mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
+    return
+  fi
 
-    git -C "${git_dir}" archive \
-        --format=tar.gz \
-        --prefix=${repo_name}-${commit}/ \
-        -o ${tarball} \
-        "${commit}"
+  git -C "${git_dir}" archive \
+      --format=tar.gz \
+      --prefix=${repo_name}-${commit}/ \
+      -o ${tarball} \
+      "${commit}"
 }
 
 prepare_vmagent_tarball() {
-    local repo_name=vmagent
-    local commit_hash=${vmagent_commit_hash}
-    local commit_short=${commit_hash::7}
-    local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
-    local repo_var=$(
-        echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
-    )
-    echo "${repo_var}=$commit_short" >> ${client_properties}
+  local repo_name=vmagent
+  local commit_hash=${vmagent_commit_hash}
+  local commit_short=${commit_hash::7}
+  local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
+  local repo_var=$(
+    echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
+  )
+  echo "${repo_var}=$commit_short" >> ${client_properties}
 
-    if [ -f "${tarball}" ]; then
-        echo ${tarball} already exists, skipping download
-    else
-        curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
-    fi
+  if [ -f "${tarball}" ]; then
+    echo ${tarball} already exists, skipping download
+  else
+    curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
+  fi
 }
 
 main() {
-    echo -----------------------------------------
-    echo "Building PMM Client source files..."
-    echo -----------------------------------------
+  echo -----------------------------------------
+  echo "Building PMM Client source files..."
+  echo -----------------------------------------
 
-    rm -rf ${source_tarball}
-    mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
+  rm -rf ${source_tarball}
+  mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
-    prepare_vmagent_tarball
-    prepare_tarball pmm-submodules
-    prepare_tarball pmm
-    prepare_tarball mongodb_exporter
-    prepare_tarball mysqld_exporter
-    prepare_tarball postgres_exporter
-    prepare_tarball proxysql_exporter
-    prepare_tarball node_exporter
-    prepare_tarball rds_exporter
-    prepare_tarball azure_metrics_exporter
+  prepare_vmagent_tarball
+  prepare_tarball pmm-submodules
+  prepare_tarball pmm
+  prepare_tarball mongodb_exporter
+  prepare_tarball mysqld_exporter
+  prepare_tarball postgres_exporter
+  prepare_tarball proxysql_exporter
+  prepare_tarball node_exporter
+  prepare_tarball rds_exporter
+  prepare_tarball azure_metrics_exporter
 
-    prepare_tarball percona-toolkit
-    local noxattrs=
-    if [[ $(uname -o) =~ [Dd]arwin ]]; then
-      noxattrs="--no-xattrs"
-    fi
+  prepare_tarball percona-toolkit
+  local noxattrs=
+  if [[ $(uname -o) =~ [Dd]arwin ]]; then
+    noxattrs="--no-xattrs"
+  fi
 
-    tar -C $(dirname ${source_dir}) ${noxattrs} -zcpf ${source_tarball} $(basename ${source_dir})
+  tar -C $(dirname ${source_dir}) ${noxattrs} -zcpf ${source_tarball} $(basename ${source_dir})
 }
 
 main
diff --git a/build/scripts/vars b/build/scripts/vars
index fa98134428..8215441dff 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -2,9 +2,8 @@
 # directory needs to have them available in its scope to function properly.
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-# TODO: refactor to pass ${WORKSPACE} as ROOT_DIR from either Jenkins or, if relevant, from GH actions
-root_dir=$(cd $(dirname $0)/../../../../../../../..; pwd -P)
-root_dir=${ROOT_DIR:-$root_dir}
+# Note: `ROOT_DIR` can be passed as ${WORKSPACE} from Jenkins or ${PWD} from GH actions
+root_dir=${ROOT_DIR:-$(cd $(dirname $0)/../../../../../../../..; pwd -P)}
 build_dir=${root_dir}/build
 
 # In VERSION file we can have numeric value like '2.0.0' as well as

From 88638541255ae2a79e2551b28a2f9543c797a87c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 08:55:09 +0300
Subject: [PATCH 097/127] PMM-13487 simplify build_client_binary

---
 build/local/README.md                         | 32 ++++----
 build/local/build                             |  2 +-
 build/local/ci-default.yml                    | 75 -------------------
 .../rpm/server/SPECS/percona-dashboards.spec  |  2 +-
 .../rpm/server/SPECS/percona-qan-api2.spec    |  2 +-
 .../rpm/server/SPECS/pmm-managed.spec         |  2 +-
 .../packages/rpm/server/SPECS/pmm-update.spec |  2 +-
 build/packages/rpm/server/SPECS/vmproxy.spec  |  2 +-
 build/scripts/build-client-binary             | 12 +--
 build/scripts/build-client-source             | 55 +++++---------
 build/scripts/vars                            |  2 +-
 11 files changed, 48 insertions(+), 140 deletions(-)
 delete mode 100644 build/local/ci-default.yml

diff --git a/build/local/README.md b/build/local/README.md
index 7055338fd6..36a38e2057 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -10,31 +10,33 @@ Therefore, we decided to make it possible to build PMM locally. This is a work i
 
 The build process is mostly based on bash scripts, which control the build flow. This was an intentional decision early on, since every developer should have at least a basic command of bash. Apart from bash and a few other well-known utilitites like `curl` or `make`, it also uses Docker for environment isolation and caching.
 
-The build process is designed to run on a Linux host. We believe it can run on other flavors of Linux, including MacOS, with little to no modification (TBC).
+The build process is designed to run on Linux or MacOS. We believe it could be adapated to run on other types of operating systems with little to no modification.
 
 
 ## Prerequisites
 
 Below is a list of prerequisites that are required to build PMM locally.
 
-- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS)
-- Docker: 25.0.2+ (tested on 25.0.2)
-- Docker Compose Plugin: 2.24.7+ (tested on 2.24.7)
-- make: 4.3+ (tested on 4.3)
-- bash (GNU): 5.1+ (tested on 5.1)
-- git: 2.34.1+ (tested on 2.34.1)
-- curl: 7.81.0+ (tested on 7.81.0)
-- yq: 4.42.0+ (tested on 4.42.1)
-- jq: 1.6+ (tested on 1.6)
+- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS), MacOS (tested on Sequoia 15.1)
+- Docker: 25.0.2+
+- Docker buildx plugin: 0.16.0+, https://github.com/docker/buildx
+- make
+- bash (GNU)
+- tar
+- git
+- curl
+- yq: 4.42.0+
+- jq: 1.6+
 
 
-## How to build PMM
+## How to use this script to build PMM
 
 1. Install the prerequisites
-2. Clone the PMM repository to the user's home directory, e.g.: `git clone https://github.com/percona/pmm /home/user/pmm`. 
-3. Change to the `build/local` directory in the cloned repo.
-4. Run `./build --help` to print the help message and check the usage.
-5. Run `./build` with parameters of your choice to build PMM v3.
+2. Download the `build` script to the user's home directory, e.g.: `curl -o $HOME/build https://raw.githubusercontent.com/percona/pmm/refs/heads/PMM-13487-build-pmm-locally/build/local/build`. 
+3. Change to the user `cd $HOME` directory.
+4. Run `./build --provision` to download the source code using our `https://github.com/percona-lab/pmm-submodules` repository.
+5. Run `./build --help` to display the script usage.
+6. Run `./build` with parameters of your choice to build PMM v3.
 
 Usually, you will want to rebuild PMM whenever there are changes in at least one of its components. All components of PMM are gathered together in one repository - `github.com/percona-lab/pmm-submodules` (or `pmm-submodules`). Therefore, you can run `build` as often as those changes need to be factored in to the next build.
 
diff --git a/build/local/build b/build/local/build
index 33817696ab..7c21839921 100755
--- a/build/local/build
+++ b/build/local/build
@@ -175,7 +175,7 @@ update() {
   local MD5SUM=$(md5sum $(dirname $0)/build)
 
    if [ "$NO_UPDATE" -eq 1 ]; then
-    echo "Running without refreshing the source code from repositories..."
+    echo "Info: skip refreshing the source code from upstream repositories..."
     return
    fi
 
diff --git a/build/local/ci-default.yml b/build/local/ci-default.yml
deleted file mode 100644
index ce413e6723..0000000000
--- a/build/local/ci-default.yml
+++ /dev/null
@@ -1,75 +0,0 @@
-deps:
-  # COMMON
-  - name: pmm
-    branch: v3
-    path: sources/pmm/src/github.com/percona/pmm
-    url: https://github.com/percona/pmm
-
-  # CLIENT
-  - name: node_exporter
-    branch: main
-    path: sources/node_exporter/src/github.com/prometheus/node_exporter
-    url: https://github.com/percona/node_exporter
-
-  - name: mysqld_exporter
-    branch: main
-    path: sources/mysqld_exporter/src/github.com/percona/mysqld_exporter
-    url: https://github.com/percona/mysqld_exporter
-
-  - name: mongodb_exporter
-    branch: main
-    path: sources/mongodb_exporter/src/github.com/percona/mongodb_exporter
-    url: https://github.com/percona/mongodb_exporter
-
-  - name: postgres_exporter
-    branch: main
-    path: sources/postgres_exporter/src/github.com/percona/postgres_exporter
-    url: https://github.com/percona/postgres_exporter
-
-  - name: proxysql_exporter
-    branch: main
-    path: sources/proxysql_exporter/src/github.com/percona/proxysql_exporter
-    url: https://github.com/percona/proxysql_exporter
-
-  - name: rds_exporter
-    branch: main
-    path: sources/rds_exporter/src/github.com/percona/rds_exporter
-    url: https://github.com/percona/rds_exporter
-
-  - name: azure_metrics_exporter
-    branch: main
-    path: sources/azure_metrics_exporter/src/github.com/percona/azure_metrics_exporter
-    url: https://github.com/percona/azure_metrics_exporter
-
-  - name: percona-toolkit
-    branch: release-v3.5.2
-    path: sources/percona-toolkit/src/github.com/percona/percona-toolkit
-    url: https://github.com/percona/percona-toolkit
-
-  # SERVER
-
-  - name: grafana-dashboards
-    branch: v3
-    path: sources/grafana-dashboards
-    url: https://github.com/percona/grafana-dashboards
-
-  - name: grafana
-    branch: v3
-    path: sources/grafana/src/github.com/grafana/grafana
-    url: https://github.com/percona/grafana
-
-  - name: pmm-dump
-    branch: main
-    path: sources/pmm-dump
-    url: https://github.com/percona/pmm-dump
-
-  # QA
-  - name: pmm-qa
-    branch: v3
-    path: sources/pmm-qa/src/github.com/percona/pmm-qa
-    url: https://github.com/percona/pmm-qa
-
-  - name: pmm-ui-tests
-    branch: v3
-    path: sources/pmm-ui-tests/src/github.com/percona/pmm-ui-tests
-    url: https://github.com/percona/pmm-ui-tests
diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index 7b85359520..19a7a93071 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -15,7 +15,7 @@ Summary:  Grafana dashboards for monitoring
 
 License:  AGPLv3
 URL:      https://%{provider}
-Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 BuildRequires:	nodejs
 Requires:	percona-grafana
diff --git a/build/packages/rpm/server/SPECS/percona-qan-api2.spec b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
index 4230bdfb51..ef97785166 100644
--- a/build/packages/rpm/server/SPECS/percona-qan-api2.spec
+++ b/build/packages/rpm/server/SPECS/percona-qan-api2.spec
@@ -17,7 +17,7 @@ Summary:        Query Analytics API for PMM
 
 License:        AGPLv3
 URL:            https://%{provider}
-Source0:        https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Source0:        https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 Percona Query Analytics (QAN) API is part of Percona Monitoring and Management (PMM).
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index 81f7ed58e6..d84f751136 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -18,7 +18,7 @@ Summary:  Percona Monitoring and Management management daemon
 
 License:  AGPLv3
 URL:      https://%{provider}
-Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 pmm-managed manages configuration of PMM server components (VictoriaMetrics,
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 8e577bad02..2b6aefb344 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -18,7 +18,7 @@ Summary:	Tool for updating packages and OS configuration for PMM Server
 
 License:	AGPLv3
 URL:		  https://%{provider}
-Source0:	https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Source0:	https://%{provider}/archive/%{commit}.tar.gz
 
 BuildArch:	noarch
 
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index 11ca103785..b32e19b1fe 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -18,7 +18,7 @@ Summary:  Percona VMProxy stateless reverse proxy for VictoriaMetrics
 
 License:  AGPLv3
 URL:      https://%{provider}
-Source0:  https://%{provider}/archive/%{commit}/%{repo}-%{shortcommit}.tar.gz
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
 %description
 VMProxy is a stateless reverse proxy which proxies requests to VictoriaMetrics and
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 8e52ba5c37..a074ae5059 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -35,7 +35,7 @@ gobuild_component() {
     fi
 
     if [ -x "${binary_dir}/${result_file}" ]; then
-      echo "Skipping the build for ${result_file} ..."
+      echo "The binary ${result_file} was found, skipping the build ..."
       return
     fi
     if [[ ${component} =~ pmm-admin|pmm-agent ]]; then
@@ -96,10 +96,10 @@ copy_component() {
   local archive=$(ls ${source_dir}/${component}-*.tar.gz | head -n1) # temporary hack for pmm repo
 
   if [ ! -d "${component_dir}" ]; then
-      mkdir -p ${component_dir}
-      tar -C ${component_dir} \
-          --strip-components=1 \
-          -zxpf "$archive"
+    mkdir -p ${component_dir}
+    tar -C ${component_dir} \
+        --strip-components=1 \
+        -zxpf "$archive"
   fi
 
   cp -r ${component_dir}/${component_path} ${binary_dir}/${component_dest}
@@ -148,7 +148,7 @@ main() {
   
   # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
   if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
-      rm -f "${binary_dir}/bin/pmm"
+    rm -f "${binary_dir}/bin/pmm"
   fi
   
   local noxattrs=
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 358447e8d2..c7b9e7ee4f 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -7,30 +7,29 @@ set -o xtrace
 
 prepare_tarball() {
   local repo_name=$1
-  local git_dir
-  if [ $repo_name = 'pmm-submodules' ]; then
-    git_dir="${root_dir}/sources/${repo_name}"
-    mkdir -p "${git_dir}"
-  else
-    git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-  fi
-  local commit=$(git -C "${git_dir}" rev-parse --short HEAD)
-  local tarball=${source_dir}/${repo_name}-${commit}.tar.gz
+  local commit=$2
+  local repo_url=$3
+  local git_dir=
   local repo_var=$(
-    echo "${repo_name}_commit" \
-        | sed -e 's:-:_:g' \
-        | tr '[[:lower:]]' '[[:upper:]]'
+    echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
   )
-  echo "${repo_var}=$commit" >> ${client_properties}
+  local tarball=${source_dir}/${repo_name}-${commit::7}.tar.gz
+
+  if [ -z "$commit" ]; then
+    git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
+    commit=$(git -C "${git_dir}" rev-parse --short HEAD)
+    tarball=${source_dir}/${repo_name}-${commit}.tar.gz
+  fi
+
+  echo "${repo_var}=${commit::7}" >> ${client_properties}
 
   if [ -f "${tarball}" ]; then
-    echo ${tarball} already exists, skipping build
+    echo "Info: ${tarball} already exists, skip creating the tarball..."
     return
   fi
 
-  if [ ${repo_name} = 'pmm-submodules' ]; then
-    tar -C sources -zcf ${repo_name}-${commit}.tar.gz  ${repo_name}
-    mv ${repo_name}-${commit}.tar.gz  ${source_dir}/
+  if [ -n "$repo_url" ]; then
+    curl -o ${tarball} -fsSL ${repo_url}/archive/${commit}.tar.gz
     return
   fi
 
@@ -41,23 +40,6 @@ prepare_tarball() {
       "${commit}"
 }
 
-prepare_vmagent_tarball() {
-  local repo_name=vmagent
-  local commit_hash=${vmagent_commit_hash}
-  local commit_short=${commit_hash::7}
-  local tarball=${source_dir}/${repo_name}-${commit_short}.tar.gz
-  local repo_var=$(
-    echo "${repo_name}_commit" | sed -e 's:-:_:g' | tr '[[:lower:]]' '[[:upper:]]'
-  )
-  echo "${repo_var}=$commit_short" >> ${client_properties}
-
-  if [ -f "${tarball}" ]; then
-    echo ${tarball} already exists, skipping download
-  else
-    curl -o ${tarball} -fsSL https://github.com/VictoriaMetrics/VictoriaMetrics/archive/${commit_hash}.tar.gz
-  fi
-}
-
 main() {
   echo -----------------------------------------
   echo "Building PMM Client source files..."
@@ -66,8 +48,7 @@ main() {
   rm -rf ${source_tarball}
   mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
-  prepare_vmagent_tarball
-  prepare_tarball pmm-submodules
+  prepare_tarball vmagent ${vmagent_commit_hash} https://github.com/VictoriaMetrics/VictoriaMetrics
   prepare_tarball pmm
   prepare_tarball mongodb_exporter
   prepare_tarball mysqld_exporter
@@ -76,8 +57,8 @@ main() {
   prepare_tarball node_exporter
   prepare_tarball rds_exporter
   prepare_tarball azure_metrics_exporter
-
   prepare_tarball percona-toolkit
+
   local noxattrs=
   if [[ $(uname -o) =~ [Dd]arwin ]]; then
     noxattrs="--no-xattrs"
diff --git a/build/scripts/vars b/build/scripts/vars
index 8215441dff..e0edc1daad 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -42,7 +42,7 @@ echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versi
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 arch=$(docker run --rm --platform=${platform} ${rpmbuild_docker_image} uname -m 2>/dev/null)
-if [ "${arch}" = "arm64" ]; then
+if [ "${arch}" = "arm64" ] && [[ "$(uname -o)" =~ [Dd]arwin ]]; then
   arch=aarch64
 fi
 

From 85c58a326956a4c015fbc742228874d7303baa69 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 09:43:41 +0300
Subject: [PATCH 098/127] PMM-13487 avoid creating duplicate client archives

---
 build/local/build                 |  2 +-
 build/scripts/build-client-binary | 12 ++++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/build/local/build b/build/local/build
index 7c21839921..33817696ab 100755
--- a/build/local/build
+++ b/build/local/build
@@ -175,7 +175,7 @@ update() {
   local MD5SUM=$(md5sum $(dirname $0)/build)
 
    if [ "$NO_UPDATE" -eq 1 ]; then
-    echo "Info: skip refreshing the source code from upstream repositories..."
+    echo "Running without refreshing the source code from repositories..."
     return
    fi
 
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index a074ae5059..c225c8eaa5 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -92,14 +92,10 @@ copy_component() {
   local component=$1
   local component_path=$2
   local component_dest=${3:-$component_path}
-  local component_dir=${build_dir}/source/${component}
-  local archive=$(ls ${source_dir}/${component}-*.tar.gz | head -n1) # temporary hack for pmm repo
-
-  if [ ! -d "${component_dir}" ]; then
-    mkdir -p ${component_dir}
-    tar -C ${component_dir} \
-        --strip-components=1 \
-        -zxpf "$archive"
+  local component_dir=$(git config -f .gitmodules submodule.${component}.path)
+  if [ ! -d "$component_dir" ]; then
+    echo "Error: could not locate the component directory for $component."
+    exit 1
   fi
 
   cp -r ${component_dir}/${component_path} ${binary_dir}/${component_dest}

From 47bd1768118efcbbc21a5a24c55a108964a697c8 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 12:08:55 +0300
Subject: [PATCH 099/127] PMM-13487 simplify build_client_packages

---
 build/local/build                             |  4 +-
 .../packages/rpm/server/SPECS/pmm-update.spec |  3 +-
 build/scripts/build-client-deb                |  2 +-
 build/scripts/build-client-packages           | 80 ++++++-------------
 build/scripts/build-client-rpm                |  2 +-
 build/scripts/build-client-sdeb               |  2 +-
 build/scripts/build-client-srpm               |  2 +-
 7 files changed, 31 insertions(+), 64 deletions(-)

diff --git a/build/local/build b/build/local/build
index 33817696ab..130299b3b7 100755
--- a/build/local/build
+++ b/build/local/build
@@ -175,7 +175,7 @@ update() {
   local MD5SUM=$(md5sum $(dirname $0)/build)
 
    if [ "$NO_UPDATE" -eq 1 ]; then
-    echo "Running without refreshing the source code from repositories..."
+    echo "Info: skip refreshing the source code from upstream repositories..."
     return
    fi
 
@@ -253,7 +253,7 @@ run_build_script() {
   end_time=$(date +%s)
 
   echo ---
-  echo "Execution time (in sec) for $script_name: $((end_time - start_time))"
+  echo "Execution time for $script_name, sec: $((end_time - start_time))"
   echo ---
 
   cd "$CURDIR" > /dev/null
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index 2b6aefb344..e4fda14f4e 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,4 +1,5 @@
-%undefine _missing_build_ids_terminate_build
+%define _missing_build_ids_terminate_build   0
+%define _binaries_in_noarch_packages_terminate_build   0
 %define _unpackaged_files_terminate_build   0
 
 %global repo            pmm
diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index bac55a0892..bd5781d155 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -23,7 +23,7 @@ main() {
       esac
 
       cd /home/builder/build
-      bash /home/builder/bin/build-client-packages --build_deb=1
+      bash /home/builder/bin/build-client-packages --build_deb
   "
 }
 
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 9596666fc1..7da616e469 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -11,14 +11,13 @@ usage () {
   cat <<EOF
 Usage: $0 [OPTIONS]
     The following options may be given :
-        --build_src_rpm     If it is 1 src rpm will be built
-        --build_source_deb  If it is 1 source deb package will be built
-        --build_rpm         If it is 1 rpm will be built
-        --build_deb         If it is 1 deb will be built
-        --install_deps      Install build dependencies(root privileges are required)
-        --branch            Branch from which submodules should be taken(default v3)
+        --build_src_rpm     Build RPM packages
+        --build_source_deb  Build debian source packages
+        --build_rpm         Build RPM packages
+        --build_deb         Build debian packages
+        --branch            Branch from which submodules should be taken (default v3)
         --help)             Display this help
-Example $0 --build_src_rpm=1 --build_rpm=1
+Example $0 --build_src_rpm --build_rpm
 EOF
 
   exit 1
@@ -31,15 +30,11 @@ append_arg_to_args () {
 parse_arguments() {
   WORKDIR=/tmp/pmm
   for arg do
-    val=`echo "$arg" | sed -e 's;^--[^=]*=;;'`
-    optname=`echo "$arg" | sed -e 's/^\(--[^=]*\)=.*$/\1/'`
     case "$arg" in
-      --build_src_rpm=*) SRPM="$val" ;;
-      --build_rpm=*) RPM="$val" ;;
-      --build_source_deb=*) SDEB="$val" ;;
-      --build_deb=*) DEB="$val" ;;
-      --branch=*) SUBMODULE_BRANCH="$val" ;;
-      --install_deps=*) INSTALL="$val" ;;
+      --build_src_rpm) SRPM=1 ;;
+      --build_rpm) RPM=1 ;;
+      --build_source_deb=*) SDEB=1 ;;
+      --build_deb) DEB=1 ;;
       --help) usage ;;
       *) append_arg_to_args "$arg" ;;
     esac
@@ -70,48 +65,22 @@ else
   fi
 }
 
-install_deps() {
-  local PACKAGES
-  if [ $INSTALL = 0 ]; then
-    echo "Dependencies will not be installed."
-    return;
-  fi
-  if [ $(id -u) -ne 0 ]; then
-    echo "It is not possible to install dependencies. Please run as root."
-    exit 1
-  fi
-  if [ "$OS" = "rpm" ]; then
-    # yum -y install git curl wget rpmdevtools bison yum-utils rpm-build
-    command -v git || PACKAGES+=" git"
-    command -v curl || PACKAGES+=" curl"
-    command -v wget || PACKAGES+=" wget"
-    command -v spectool || PACKAGES+=" rpmdevtools"
-    command -v bison || PACKAGES+=" bison"
-    command -v repoquery || PACKAGES+=" yum-utils"
-    command -v rpmbuild || PACKAGES+=" rpm-build"
-    if [ -n "$PACKAGES" ]; then
-      yum -y install "$PACKAGES"
-    fi
-  else
-    apt-get update
-    apt-get -y install devscripts equivs debhelper
-  fi
-}
-
 get_tar(){
   local TARBALL=$1
   local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-  if [ -z $TARFILE ]; then
-    TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-    if [ -z $TARFILE ]; then
-      echo "There is no $TARBALL for build"
-      exit 1
-    else
-      cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
-    fi
-  else
+  if [ -n $TARFILE ]; then
     cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
+    return
   fi
+
+  TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+  if [ -n $TARFILE ]; then
+    cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
+    return
+  fi
+
+  echo "Error: $TARBALL could not be found, exiting..."
+  exit 1
 }
 
 get_deb_sources(){
@@ -134,7 +103,7 @@ get_deb_sources(){
 
 build_srpm(){
   if [ "$OS" = "deb" ]; then
-    echo "It is not possible to build src rpm here"
+    echo "Error: it is not possible to build source RPM on a debian based OS, exiting..."
     exit 1
   fi
   cd $WORKDIR
@@ -268,7 +237,7 @@ build_source_deb(){
   cd debian
   echo "${NAME} (${VERSION}) unstable; urgency=low" >> changelog
   echo "  * Initial Release." >> changelog
-  echo " -- EvgeniyPatlan <evgeniy.patlan@percona.com> $(date -R)" >> changelog
+  echo " -- Percona LLC <no-reply@percona.com> $(date -R)" >> changelog
 
   cd ../
 
@@ -331,8 +300,6 @@ DEB=0
 OS_NAME=
 ARCH=
 OS=
-SUBMODULE_BRANCH="v3"
-INSTALL=0
 RPM_RELEASE=1
 DEB_RELEASE=1
 REVISION=0
@@ -346,7 +313,6 @@ fi
 
 parse_arguments "$@"
 get_system
-install_deps
 
 check_workdir
 if [ "$SRPM" != 0 ]; then
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 5069943c01..b71ad6516e 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -22,7 +22,7 @@ main() {
       export pmm_release=$pmm_release
       cd /home/builder/build
 
-      /home/builder/bin/build-client-packages --build_rpm=1
+      /home/builder/bin/build-client-packages --build_rpm
     "
 }
 
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
index 6466bc7f9a..386eb6e2ba 100755
--- a/build/scripts/build-client-sdeb
+++ b/build/scripts/build-client-sdeb
@@ -22,7 +22,7 @@ main() {
         DEBIAN_FRONTEND=noninteractive apt-get -y install git lsb-release devscripts dh-make dh-systemd
         cd /home/builder/build
 
-        bash /home/builder/bin/build-client-packages --build_source_deb=1
+        bash /home/builder/bin/build-client-packages --build_source_deb
     "
 }
 
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
index b6bef5729f..ed36793240 100755
--- a/build/scripts/build-client-srpm
+++ b/build/scripts/build-client-srpm
@@ -18,7 +18,7 @@ main() {
         export pmm_release=$pmm_release
         cd /home/builder/build
 
-        /home/builder/bin/build-client-packages --build_src_rpm=1
+        /home/builder/bin/build-client-packages --build_src_rpm
     "
 }
 

From 6f17fadd3df9666a54941c7c00da7b0a2351947e Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 12:21:41 +0300
Subject: [PATCH 100/127] PMM-13487 fix finding tarball in get_tar

---
 build/scripts/build-client-packages | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 7da616e469..f054ddd2fa 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -15,7 +15,6 @@ Usage: $0 [OPTIONS]
         --build_source_deb  Build debian source packages
         --build_rpm         Build RPM packages
         --build_deb         Build debian packages
-        --branch            Branch from which submodules should be taken (default v3)
         --help)             Display this help
 Example $0 --build_src_rpm --build_rpm
 EOF
@@ -23,12 +22,7 @@ EOF
   exit 1
 }
 
-append_arg_to_args () {
-  args="$args "`shell_quote_string "$1"`
-}
-
 parse_arguments() {
-  WORKDIR=/tmp/pmm
   for arg do
     case "$arg" in
       --build_src_rpm) SRPM=1 ;;
@@ -36,7 +30,7 @@ parse_arguments() {
       --build_source_deb=*) SDEB=1 ;;
       --build_deb) DEB=1 ;;
       --help) usage ;;
-      *) append_arg_to_args "$arg" ;;
+      *) usage ;;
     esac
   done
 }
@@ -67,13 +61,13 @@ else
 
 get_tar(){
   local TARBALL=$1
-  local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-  if [ -n $TARFILE ]; then
-    cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
-    return
-  fi
+  # local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+  # if [ -n $TARFILE ]; then
+  #   cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
+  #   return
+  # fi
 
-  TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
+  local TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
   if [ -n $TARFILE ]; then
     cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
     return
@@ -291,8 +285,7 @@ build_deb(){
 CURDIR=$(pwd)
 PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties
-args=
-WORKDIR=
+WORKDIR=/tmp/pmm
 SRPM=0
 SDEB=0
 RPM=0

From c39179c3e9f9289f02f499df99071189db0d6f6c Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 14:04:23 +0300
Subject: [PATCH 101/127] PMM-13487 simplify build-client-packages

---
 build/scripts/build-client-packages | 58 ++++++++++++-----------------
 build/scripts/build-server-rpm      | 16 ++++----
 2 files changed, 31 insertions(+), 43 deletions(-)

diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index f054ddd2fa..cc50b58abe 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -1,12 +1,9 @@
 #!/bin/sh
+# NOTE: this script is always executed in a container environment.
 
 set -o errexit
 set -o xtrace
 
-shell_quote_string() {
-  echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g'
-}
-
 usage () {
   cat <<EOF
 Usage: $0 [OPTIONS]
@@ -61,12 +58,6 @@ else
 
 get_tar(){
   local TARBALL=$1
-  # local TARFILE=$(basename $(find $WORKDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-  # if [ -n $TARFILE ]; then
-  #   cp $WORKDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
-  #   return
-  # fi
-
   local TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
   if [ -n $TARFILE ]; then
     cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
@@ -106,10 +97,10 @@ build_srpm(){
   rm -fr rpmbuild
   ls | grep -v tar.gz | xargs rm -rf
 
-  TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
-  NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
-  VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
-  VERSION=${VERSION_TMP%.tar.gz}
+  local TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
+  local NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
+  local VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
+  local VERSION=${VERSION_TMP%.tar.gz}
 
   mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
 
@@ -126,10 +117,10 @@ build_srpm(){
     --define "dist %{nil}" \
     "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
 
-  mkdir -p ${WORKDIR}/srpm
+  # mkdir -p ${WORKDIR}/srpm
   mkdir -p ${CURDIR}/srpm
   cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
-  cp rpmbuild/SRPMS/*.src.rpm ${WORKDIR}/srpm
+  # cp rpmbuild/SRPMS/*.src.rpm ${WORKDIR}/srpm
 }
 
 build_rpm(){
@@ -138,24 +129,21 @@ build_rpm(){
     exit 1
   fi
 
-  SRC_RPM=$(basename $(find $WORKDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
-  if [ -z $SRC_RPM ]; then
-    SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
-    if [ -z $SRC_RPM ]; then
-      echo "There is no src rpm for build"
-      echo "You can create it using key --build_src_rpm=1"
-      exit 1
-    else
-      cp $CURDIR/srpm/$SRC_RPM $WORKDIR
-    fi
-  else
-    cp $WORKDIR/srpm/$SRC_RPM $WORKDIR
+  local SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
+  if [ -z "$SRC_RPM" ]; then
+    echo "Error: no source RPM could be found, exiting..."
+    echo "You can create SRPM using the parameter --build_src_rpm"
+    exit 1
+  # else
+    # rm -f "$CURDIR/srpm/$SRC_RPM" || :
+    # cp "$CURDIR/srpm/$SRC_RPM" "$WORKDIR"
   fi
 
   cd $WORKDIR
   rm -fr rpmbuild
   mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
-  cp $SRC_RPM rpmbuild/SRPMS/
+  cp "$CURDIR/srpm/$SRC_RPM" rpmbuild/SRPMS/
+
   if find "${CURDIR}/rpm" -name "${PACKAGE_NAME}-${pmm_version}-*.rpm"; then
     # Remove older client rpm packages before building a new one
     rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
@@ -262,10 +250,10 @@ build_deb(){
   cd $WORKDIR
   rm -fv *.deb
   export DEBIAN_VERSION="$(lsb_release -sc)"
-  DSC=$(basename $(find . -name '*.dsc' | sort | tail -n 1))
-  DIRNAME=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $1"-"$2}')
-  VERSION=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $3}')
-  ARCH=$(uname -m)
+  local DSC=$(basename $(find . -name '*.dsc' | sort | tail -n 1))
+  local DIRNAME=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $1"-"$2}')
+  local VERSION=$(echo ${DSC} | sed -e 's:_:-:g' | awk -F'-' '{print $3}')
+  local ARCH=$(uname -m)
   echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $VERSION_FILE
   echo "ARCH=${ARCH}" >> $VERSION_FILE
   #
@@ -282,10 +270,10 @@ build_deb(){
 
 #main
 
-CURDIR=$(pwd)
+CURDIR=$(pwd) # /home/builder/build
+WORKDIR=/tmp/pmm
 PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties
-WORKDIR=/tmp/pmm
 SRPM=0
 SDEB=0
 RPM=0
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index b2acadaba9..2d839a047a 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -10,8 +10,8 @@ get_rpm_version() {
   local rpm_version=$(
     docker run --rm \
         --platform=${platform} \
-        -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
-        rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
+        -v ${rpmbuild_dir}:/home/builder/rpm/BUILD ${rpmbuild_docker_image} sh -c "
+        rpmspec -q --qf '%{version}-%{release}\n' BUILD/${spec_name}.spec \
             | sed -re 's/\.[0-9]{10}\././' | head -1
     "
   )
@@ -123,7 +123,7 @@ build() {
   echo "Building PMM Server RPM for ${spec_name} component..."
   echo -------------------------------------------------------
 
-  local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
+  local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/BUILD -v ${rpms_dir}:/home/builder/rpm/RPMS"
   volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
   volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
   volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
@@ -138,7 +138,7 @@ build() {
     set -o errexit
     set -o xtrace
 
-    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
+    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/BUILD
     sudo chown builder:builder /home/builder/.cache
     if [ ! -d /home/builder/.cache/go-build ]; then
       mkdir -p /home/builder/.cache/go-build
@@ -169,16 +169,16 @@ build() {
     # Only these two specs have build dependencies
     if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
       sleep 2s
-      sudo yum-builddep -y SOURCES/${spec_name}.spec
+      sudo yum-builddep -y BUILD/${spec_name}.spec
     fi
 
-    spectool -C SOURCES -g SOURCES/${spec_name}.spec
+    spectool -C BUILD -g BUILD/${spec_name}.spec
     rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
               --define 'dist .${rpmbuild_dist}' \
               --define 'debug_package %{nil}' \
-              -ba SOURCES/${spec_name}.spec
+              -ba BUILD/${spec_name}.spec
 
-    rm -f SOURCES/${spec_name}.spec*
+    rm -f BUILD/${spec_name}.spec*
   "
 
   if [ "$LOCAL_BUILD" = 1 ]; then

From 61b9bc4efb0e5b461a551e2eb7669b4fb6286130 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 10 Nov 2024 17:19:32 +0300
Subject: [PATCH 102/127] PMM-13487 revert back to sources dir

---
 build/scripts/build-server-rpm | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 2d839a047a..b2acadaba9 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -10,8 +10,8 @@ get_rpm_version() {
   local rpm_version=$(
     docker run --rm \
         --platform=${platform} \
-        -v ${rpmbuild_dir}:/home/builder/rpm/BUILD ${rpmbuild_docker_image} sh -c "
-        rpmspec -q --qf '%{version}-%{release}\n' BUILD/${spec_name}.spec \
+        -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+        rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
             | sed -re 's/\.[0-9]{10}\././' | head -1
     "
   )
@@ -123,7 +123,7 @@ build() {
   echo "Building PMM Server RPM for ${spec_name} component..."
   echo -------------------------------------------------------
 
-  local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/BUILD -v ${rpms_dir}:/home/builder/rpm/RPMS"
+  local volume_mounts="-v ${rpmbuild_dir}:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS"
   volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
   volume_mounts+=" -v pmm-yarn:/home/builder/.cache/yarn"
   volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
@@ -138,7 +138,7 @@ build() {
     set -o errexit
     set -o xtrace
 
-    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/BUILD
+    sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES
     sudo chown builder:builder /home/builder/.cache
     if [ ! -d /home/builder/.cache/go-build ]; then
       mkdir -p /home/builder/.cache/go-build
@@ -169,16 +169,16 @@ build() {
     # Only these two specs have build dependencies
     if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
       sleep 2s
-      sudo yum-builddep -y BUILD/${spec_name}.spec
+      sudo yum-builddep -y SOURCES/${spec_name}.spec
     fi
 
-    spectool -C BUILD -g BUILD/${spec_name}.spec
+    spectool -C SOURCES -g SOURCES/${spec_name}.spec
     rpmbuild  --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \
               --define 'dist .${rpmbuild_dist}' \
               --define 'debug_package %{nil}' \
-              -ba BUILD/${spec_name}.spec
+              -ba SOURCES/${spec_name}.spec
 
-    rm -f BUILD/${spec_name}.spec*
+    rm -f SOURCES/${spec_name}.spec*
   "
 
   if [ "$LOCAL_BUILD" = 1 ]; then

From 651092ae2d033de9d23450580211f1b7e7687822 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 11 Nov 2024 10:18:46 +0300
Subject: [PATCH 103/127] PMM-13487 update README

---
 build/local/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build/local/README.md b/build/local/README.md
index 36a38e2057..b5974abe39 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -86,10 +86,11 @@ Currently, local builds target the following environments:
   - docker image - docker and Kubernetes environments amd64/aarch64
 
 
-## Draft Todo List
+## Ideas to evaluate
 
 * download the sources to a local directory `.modules` w/o using pmm-submodules
 * have a VERSION file, similar to the one in https://github.com/percona-lab/pmm-submodules/blob/v3/VERSION
 * have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
 * copy all build scripts to `build/local` to isolate the new build flows from the old ones
 * use the `--debug` parameter to control the verbosity of the logs
+* better caching for pmm-qan, pmm-agent, vmproxy and pmm-admin, which all reside in one monorepo

From 3016c0daef2a3f86594a22d1518c39852bce01b6 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 12 Nov 2024 01:24:17 +0300
Subject: [PATCH 104/127] PMM-13487 consistency fixes in install_tarball

---
 build/local/README.md         |  2 +-
 build/scripts/install_tarball | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/build/local/README.md b/build/local/README.md
index b5974abe39..d7d2996e35 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -93,4 +93,4 @@ Currently, local builds target the following environments:
 * have a package.yml file containing hashes of all build artefacts (PMM components, i.e. grafana, pmm-managed, etc.)
 * copy all build scripts to `build/local` to isolate the new build flows from the old ones
 * use the `--debug` parameter to control the verbosity of the logs
-* better caching for pmm-qan, pmm-agent, vmproxy and pmm-admin, which all reside in one monorepo
+* provide better caching for pmm-qan, pmm-agent, vmproxy and pmm-admin, which all reside in one monorepo, by calculating a sha256sum on their directories
diff --git a/build/scripts/install_tarball b/build/scripts/install_tarball
index a0e613952d..f014e803dd 100755
--- a/build/scripts/install_tarball
+++ b/build/scripts/install_tarball
@@ -77,7 +77,7 @@ ${INSTALL_COMMAND} -m 0755 queries-lr.yaml "${PMM_DIR}"/collectors/custom-querie
 ${INSTALL_COMMAND} -m 0755 queries-mysqld-group-replication.yml "${PMM_DIR}"/collectors/custom-queries/mysql/high-resolution
 
 for FILE in $( ls ${PWD}/bin ); do
-    if [ "x${FILE}" = "xpmm-admin" ] || [ "x${FILE}" = "xpmm-agent" ] || [ "x${FILE}" = "xpmm-agent-entrypoint" ]; then
+    if [[ "${FILE}" =~ pmm-admin|^pmm-agent$|pmm-agent-entrypoint ]]; then
         ${INSTALL_COMMAND} -m 0755 ${PWD}/bin/${FILE} "${PMM_DIR}"/bin
     elif [[ "${FILE}" =~ pt-summary|pt-mysql-summary|pt-pg-summary|pt-mongodb-summary|nomad ]]; then
         ${INSTALL_COMMAND} -m 0755 ${PWD}/bin/"${FILE}" "${PMM_DIR}"/tools
@@ -88,12 +88,12 @@ done
 
 for FILE in example.prom queries-mysqld.yml example-queries-postgres.yml; do
     for RESOLUTION in low medium high; do
-        if [ "x${FILE}" = "xexample.prom" ]; then
-            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}"/collectors/textfile-collector/${RESOLUTION}-resolution
-        elif [ "x${FILE}" = "xqueries-mysqld.yml" ]; then
-            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}"/collectors/custom-queries/mysql/${RESOLUTION}-resolution
-        elif [ "x${FILE}" = "xexample-queries-postgres.yml" ]; then
-            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}"/collectors/custom-queries/postgresql/${RESOLUTION}-resolution
+        if [ "${FILE}" = "example.prom" ]; then
+            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}/collectors/textfile-collector/${RESOLUTION}-resolution"
+        elif [ "${FILE}" = "queries-mysqld.yml" ]; then
+            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}/collectors/custom-queries/mysql/${RESOLUTION}-resolution"
+        elif [ "${FILE}" = "example-queries-postgres.yml" ]; then
+            ${INSTALL_COMMAND} -m 0755 ${FILE} "${PMM_DIR}/collectors/custom-queries/postgresql/${RESOLUTION}-resolution"
         fi
     done
 done

From 52008593a0bc94ea27a4798ae7fdcf27f9eb1518 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 12 Nov 2024 15:10:01 +0300
Subject: [PATCH 105/127] PMM-13487 fix nomad builds

---
 Makefile.include                  | 13 +++++++++----
 build/scripts/build-client-binary |  5 +++++
 build/scripts/build-client-source |  1 +
 build/scripts/vars                |  4 ++--
 4 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/Makefile.include b/Makefile.include
index 7a09469422..a3e092955f 100644
--- a/Makefile.include
+++ b/Makefile.include
@@ -30,16 +30,16 @@ gen: clean            ## Generate files
 	make -C admin gen
 	make -C managed gen
 
-	make gen-mocks      ## Generate mocks
+	make gen-mocks
 
 	make format
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
-clean:               ## Remove generated files
+clean:                ## Remove generated files
 	make -C api clean
 
-gen-mocks:
+gen-mocks:            ## Generate mocks
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
@@ -55,7 +55,7 @@ check:                ## Run required checkers and linters
 	bin/go-sumtype ./...
 	bin/go-consistent -pedantic ./...
 
-check-license:          ## Run license header checks against source files
+check-license:        ## Run license header checks against source files
 	bin/license-eye -c .licenserc.yaml header check
 
 check-all: check-license check    ## Run golangci linter to check for changes against main
@@ -69,5 +69,10 @@ format:               ## Format source code
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
 
+tidy:                 ## Tidy up the go mod files
+	@echo "==> Tidy up submodules"
+	@go mod tidy
+	@cd tools && go mod tidy
+
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf
diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index c225c8eaa5..9478717645 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -46,6 +46,8 @@ gobuild_component() {
       build_command="cd /tmp/go/src/${extract_path} && make release"
     elif [ ${component} = 'percona-toolkit' ]; then
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
+    elif [ ${component} = 'nomad' ]; then
+      build_command="cd /tmp/go/src/${extract_path} && TARGETS='linux_amd64 linux_arm64' make deps release"      
     else
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
@@ -110,6 +112,7 @@ main() {
 
   # gobuild_component(component, parent_component, extract_path, component_path)
   gobuild_component "vmagent" "" "github.com/VictoriaMetrics/VictoriaMetrics" "app/vmagent"
+  gobuild_component "nomad" "" "github.com/hashicorp/nomad"
   gobuild_component "node_exporter" "" "github.com/prometheus/node_exporter"
   gobuild_component "mysqld_exporter"
   gobuild_component "postgres_exporter" "" "" "cmd/postgres_exporter"
@@ -139,6 +142,8 @@ main() {
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
+  cp -r ${tmp_dir}/go/src/github.com/hashicorp/nomad/pkg/linux_amd64/nomad ${binary_dir}/bin/nomad
+
   rm -rf ${binary_tarball}
   mkdir -p $(dirname ${binary_tarball}) || :
   
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index c7b9e7ee4f..0e96567058 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -49,6 +49,7 @@ main() {
   mkdir -p ${source_dir}/ $(dirname ${source_tarball}) || :
 
   prepare_tarball vmagent ${vmagent_commit_hash} https://github.com/VictoriaMetrics/VictoriaMetrics
+  prepare_tarball nomad ${nomad_commit_hash} https://github.com/hashicorp/nomad
   prepare_tarball pmm
   prepare_tarball mongodb_exporter
   prepare_tarball mysqld_exporter
diff --git a/build/scripts/vars b/build/scripts/vars
index 174bfda10a..92124459ec 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -61,5 +61,5 @@ binary_tarball=${build_dir}/tarball/pmm-client-${pmm_version}.tar.gz
 
 # https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/pmm-6401-v1.93.4
 vmagent_commit_hash=58ecb9066574f38f1d1c91ace467316e7f175b09
-# https://github.com/hashicorp/nomad/releases/tag/v1.9.0
-nomad_commit_hash=7ad36851ec02f875e0814775ecf1df0229f0a615
+# https://github.com/hashicorp/nomad/releases/tag/v1.9.3
+nomad_commit_hash=d92bf1014886c0ff9f882f4a2691d5ae8ad8131c

From 4cdea4144bf10507d34b6ef45471bb115f679dfb Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Tue, 12 Nov 2024 18:39:31 +0300
Subject: [PATCH 106/127] PMM-13487 Install jq and yq into rpmbuild

---
 build/docker/rpmbuild/Dockerfile.el9 | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 6f3c445b23..f87844fb93 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -22,6 +22,7 @@ RUN yum install -y gcc gcc-c++ \
     automake autoconf \
     rpmdevtools createrepo_c epel-release \
     bison yum-utils rpm-build \
+    jq \
     rsync \
     wget && \
     yum install -y --enablerepo=ol9_codeready_builder glibc-static && \
@@ -38,7 +39,10 @@ RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
     tar -C /usr/local -xzf /tmp/golang.tar.gz && \
     yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \
-    rm /tmp/golang.tar.gz /tmp/goreleaser.rpm
+    rm /tmp/golang.tar.gz /tmp/goreleaser.rpm && \
+    curl -o /usr/bin/yq -fSsL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${ARCH} && \
+    chmod +x /usr/bin/yq
+
 
 RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0
 RUN update-alternatives --set go /usr/local/go/bin/go

From b24820cfddb02c2543e33692062a0558122e5a7f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 00:17:51 +0300
Subject: [PATCH 107/127] PMM-13487 Bump up gimme

---
 .devcontainer/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.devcontainer/setup.py b/.devcontainer/setup.py
index 6e4995689c..701118a205 100755
--- a/.devcontainer/setup.py
+++ b/.devcontainer/setup.py
@@ -41,7 +41,7 @@ def install_go():
     """Installs Go toolchain."""
 
     run_commands([
-        "curl -sS https://raw.githubusercontent.com/travis-ci/gimme/v1.5.5/gimme -o /usr/local/bin/gimme",
+        "curl -sS https://raw.githubusercontent.com/travis-ci/gimme/v1.5.6/gimme -o /usr/local/bin/gimme",
         "chmod +x /usr/local/bin/gimme"
     ])
 

From 56c7d110db2b0747ddff20de6b4a8fb312c6fe78 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 00:18:25 +0300
Subject: [PATCH 108/127] PMM-13487 Do not require yq

---
 Makefile.devcontainer |   3 -
 build/local/README.md |   1 -
 build/local/build     | 714 +++++++++++++++++++++++-------------------
 3 files changed, 385 insertions(+), 333 deletions(-)

diff --git a/Makefile.devcontainer b/Makefile.devcontainer
index ff73f16726..1e2b143154 100644
--- a/Makefile.devcontainer
+++ b/Makefile.devcontainer
@@ -49,9 +49,6 @@ run-qan: release-qan
 
 run-all: run-agent run-managed       ## Run pmm-managed and pmm-agent
 
-run:                                 ## Deprecated
-	echo "Deprecated: please use run-all"
-
 # TODO https://jira.percona.com/browse/PMM-3484, see maincover_test.go
 # run-race-cover: install-race    ## Run pmm-managed with race detector and collect coverage information.
 # 	go test -coverpkg="github.com/percona/pmm/managed/..." \
diff --git a/build/local/README.md b/build/local/README.md
index d7d2996e35..f418d6aebc 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -25,7 +25,6 @@ Below is a list of prerequisites that are required to build PMM locally.
 - tar
 - git
 - curl
-- yq: 4.42.0+
 - jq: 1.6+
 
 
diff --git a/build/local/build b/build/local/build
index 130299b3b7..22e6a259c0 100755
--- a/build/local/build
+++ b/build/local/build
@@ -3,389 +3,445 @@ set -o errexit
 set -o nounset
 
 usage() {
-  cat <<-EOF
+	cat <<-EOF
 Usage: $BASE_NAME [OPTIONS]
 Options:
-      --platform <platform>    Build for a specific platform (defaults to linux/amd64)
-      --no-update              Do not fetch the latest changes from the repo
-      --update-only            Only fetch the latest changes from the repo
-      --client-only            Build only PMM Client (client binaries + docker)
-      --no-client              Do not build PMM Client
-      --no-client-docker       Do not build PMM Client docker image (default)
-      --log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
-  -d  --debug                  Log output in debug mode, which also prints the commands
-  -h  --help                   Display help
+			--init                   Clone the source, initialize directories and check for pre-requisites
+			--platform <platform>    Build for a specific platform (defaults to linux/amd64)
+			--no-update              Do not fetch the latest changes from the repo
+			--update-only            Only fetch the latest changes from the repo
+			--client-only            Build only PMM Client (client binaries + docker)
+			--no-client              Do not build PMM Client
+			--no-client-docker       Do not build PMM Client docker image (default)
+			--log-file <path>        Save build logs to a file located at <path> (defaults to PWD).
+															 Note: the log file will get reset on every subsequent run.
+	-d  --debug                  Log output in debug mode, which also prints the commands
+	-h  --help                   Display help
 EOF
 }
 
 parse_params() {
-  # Define global variables
-  LOCAL_BUILD=1
-  NO_UPDATE=0
-  UPDATE_ONLY=0
-  NO_CLIENT=0
-  NO_CLIENT_DOCKER=1
-  NO_SERVER=0
-  DEBUG_MODE=0
-  START_TIME=$(date +%s)
-  LOG_FILE="$(dirname $0)/build.log"
-  BASE_NAME=$(basename $0)
-  PLATFORM=linux/amd64
-  SUBMODULES=pmm-submodules
-  ROOT_DIR="$(realpath ./${SUBMODULES})"
-  PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
-
-  while test "$#" -gt 0; do
-    case "$1" in
-      --update-only)
-        UPDATE_ONLY=1; NO_UPDATE=0
-        ;;
-      --no-update)
-        if [ "$UPDATE_ONLY" -eq 1 ]; then
-          echo "Error. Mutually exclusive options: --update-only and --no-update"
-          exit 1
-        fi      
-        NO_UPDATE=1
-        ;;
-      --client-only)
-        NO_CLIENT=0; NO_CLIENT_DOCKER=0; NO_SERVER=1
-        ;;
-      --no-client)
-        NO_CLIENT=1; NO_CLIENT_DOCKER=1
-        ;;
-      --client-docker)
-        if [ "$NO_CLIENT" -eq 1 ]; then
-          echo "Error. Mutually exclusive options: --client-docker and --no-client"
-          exit 1
-        fi
-        NO_CLIENT_DOCKER=0
-        ;;
-      --no-client-docker)
-        if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
-          echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
-          exit 1
-        fi
-        NO_CLIENT_DOCKER=1
-        ;;
-      --platform)
-        shift
-        if [ -z "$1" ]; then
-          echo "Missing argument for --platform"
-          exit 1
-        fi
-        PLATFORM="$1"
-        ;;
-      --log-file)
-        shift
-        if [ -z "$1" ]; then
-          echo "Missing argument for --log-file"
-          exit 1
-        fi
-        LOG_FILE="$1"
-        ;;
-      --debug | -d)
-        DEBUG_MODE=1
-        shift
-        ;;
-      --help | -h)
-        shift
-        usage
-        exit 0
-        ;;
-      *)
-        echo "Unknown argument: $1"
-        echo
-        usage
-        exit 1
-        ;;
-    esac
-    shift
-  done
+	# All global variables must be defined here
+	NO_UPDATE=0
+	UPDATE_ONLY=0
+	NO_CLIENT=0
+	NO_CLIENT_DOCKER=1
+	NO_SERVER=0
+	START_TIME=$(date +%s)
+	LOG_FILE="$(dirname $0)/build.log"
+	BASE_NAME=$(basename $0)
+	PLATFORM=linux/amd64
+	SUBMODULES=pmm-submodules
+	PATH_TO_SCRIPTS="sources/pmm/src/github.com/percona/pmm/build/scripts"
+	VAR_PREFIX="__PMM"
+
+	# Exported variables
+	export LOCAL_BUILD=1
+	export DEBUG_MODE=0
+	export ROOT_DIR="$(realpath ./${SUBMODULES})"
+	export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
+	export RPM_EPOCH=0
+
+	while test "$#" -gt 0; do
+		case "$1" in
+			--update-only)
+				UPDATE_ONLY=1; NO_UPDATE=0
+				;;
+			--no-update)
+				if [ "$UPDATE_ONLY" -eq 1 ]; then
+					echo "Error. Mutually exclusive options: --update-only and --no-update"
+					exit 1
+				fi      
+				NO_UPDATE=1
+				;;
+			--client-only)
+				NO_CLIENT=0; NO_CLIENT_DOCKER=0; NO_SERVER=1
+				;;
+			--no-client)
+				NO_CLIENT=1; NO_CLIENT_DOCKER=1
+				;;
+			--client-docker)
+				if [ "$NO_CLIENT" -eq 1 ]; then
+					echo "Error. Mutually exclusive options: --client-docker and --no-client"
+					exit 1
+				fi
+				NO_CLIENT_DOCKER=0
+				;;
+			--no-client-docker)
+				if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
+					echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
+					exit 1
+				fi
+				NO_CLIENT_DOCKER=1
+				;;
+			--platform)
+				shift
+				if [ -z "$1" ]; then
+					echo "Missing argument for --platform"
+					exit 1
+				fi
+				PLATFORM="$1"
+				;;
+			--log-file)
+				shift
+				if [ -z "$1" ]; then
+					echo "Missing argument for --log-file"
+					exit 1
+				fi
+				LOG_FILE="$1"
+				;;
+			--debug | -d)
+				DEBUG_MODE=1
+				shift
+				;;
+			--help | -h)
+				shift
+				usage
+				exit 0
+				;;
+			*)
+				echo "Unknown argument: $1"
+				echo
+				usage
+				exit 1
+				;;
+		esac
+		shift
+	done
+}
+
+# Function to set a key/value pair
+set_value() {
+	local key="$1"
+	local value="$2"
+	validate_key "$key"
+	eval "${VAR_PREFIX}_${key}=\"${value}\""
+}
+
+# Function to retrieve a value by key
+get_value() {
+	local key="$1"
+	validate_key "$key"
+	local value=$(eval "echo -n ${VAR_PREFIX}_${key}")
+	if [ -z "$value" ]; then
+		echo "Error: variable '${VAR_PREFIX}_${key}' is undefined" >&2
+	fi
+	echo -n "$value"
+}
+
+# Function to unset a key
+unset_value() {
+	local key="$1"
+	validate_key "$key"
+	eval "unset ${VAR_PREFIX}_${key}"
+}
+
+# Function to validate a key (allow only alphanumeric and underscore characters)
+validate_key() {
+	local key="$1"
+	if ! [[ "$key" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
+		echo "The key ${key} is invalid, exiting..." >&2
+		exit 1
+	fi
 }
 
 needs_to_pull() {
-  local UPSTREAM=${1:-'@{u}'}
-  local LOCAL=$(git rev-parse @)
-  local BASE=$(git merge-base @ "$UPSTREAM")
-  local REMOTE=$(git rev-parse "$UPSTREAM")
-
-  if [ "$LOCAL" = "$REMOTE" ]; then
-    return 1 # false, we are up-to-date
-  fi
-
-  if [ "$LOCAL" = "$BASE" ]; then
-    return 0 # true, we are behind upstream
-  fi
+	local UPSTREAM=${1:-'@{u}'}
+	local LOCAL=$(git rev-parse @)
+	local BASE=$(git merge-base @ "$UPSTREAM")
+	local REMOTE=$(git rev-parse "$UPSTREAM")
+
+	if [ "$LOCAL" = "$REMOTE" ]; then
+		return 1 # false, we are up-to-date
+	fi
+
+	if [ "$LOCAL" = "$BASE" ]; then
+		return 0 # true, we are behind upstream
+	fi
 }
 
 rewind() {
-  local DIR="$1"
-  local BRANCH="$2"
-
-  cd "$DIR" > /dev/null
-  local CURRENT=$(git branch --show-current)
-  git fetch
-
-  if [ "$CURRENT" != "$BRANCH" ]; then
-    echo "Currently on $CURRENT, checking out $BRANCH"
-    git checkout "$BRANCH"
-  fi
-
-  if needs_to_pull; then
-    if ! git pull origin; then
-      git reset --hard HEAD~20
-      git pull origin
-    fi
-    echo "Submodule has pulled from upstream"
-    git logs -n 2
-    cd - > /dev/null
-    git add "$DIR"
-  else
-    cd - > /dev/null
-    echo "Submodule is up-to-date with upstream"
-  fi
+	local DIR="$1"
+	local BRANCH="$2"
+
+	cd "$DIR" > /dev/null
+	local CURRENT=$(git branch --show-current)
+	git fetch
+
+	if [ "$CURRENT" != "$BRANCH" ]; then
+		echo "Currently on $CURRENT, checking out $BRANCH"
+		git checkout "$BRANCH"
+	fi
+
+	if needs_to_pull; then
+		if ! git pull origin; then
+			git reset --hard HEAD~20
+			git pull origin
+		fi
+		echo "Submodule has pulled from upstream"
+		git logs -n 2
+		cd - > /dev/null
+		git add "$DIR"
+	else
+		cd - > /dev/null
+		echo "Submodule is up-to-date with upstream"
+	fi
 }
 
 check_files() {
-  local DIR="$1"
-
-  # Thouroughly verify the presence of known files, otherwise bail out
-  if [ ! -d "$DIR" ] ; then
-    echo "Error: could not locate the 'pmm-submodules' directory."
-    exit 1
-  fi
-
-  if [ ! -d "$DIR/sources" ] || [ ! -d "$DIR/.git" ] || [ ! -f "$DIR/.gitmodules" ] || [ ! -f "$DIR/ci-default.yml" ]; then
-    echo "Error: directory $DIR does not look like a clone of https://github.com/percona-lab/pmm-submodules repository, exiting..."
-    exit 1
-  fi
-
-  if [ ! -f "ci.yml" ] || [ ! -s "ci.yml" ]; then
-    echo "Error: the working directory '$PWD' must contain a non-empty ci.yml file."
-    echo "Please refer to this [README](https://github.com/Percona-Lab/pmm-submodules/blob/v3/README.md#how-to-create-a-feature-build) for more information."
-    exit 1
-  fi
+	local DIR="$1"
+
+	# Thouroughly verify the presence of known files, otherwise bail out
+	if [ ! -d "$DIR" ] ; then
+		echo "Error: could not locate the 'pmm-submodules' directory."
+		exit 1
+	fi
+
+	if [ ! -d "$DIR/sources" ] || [ ! -d "$DIR/.git" ] || [ ! -f "$DIR/.gitmodules" ] || [ ! -f "$DIR/ci-default.yml" ]; then
+		echo "Error: directory $DIR does not look like a clone of https://github.com/percona-lab/pmm-submodules repository, exiting..."
+		exit 1
+	fi
+
+	if [ ! -f "ci.yml" ] || [ ! -s "ci.yml" ]; then
+		echo "Error: the current directory '$PWD' must contain a non-empty ci.yml file."
+		echo "Please refer to this [README](https://github.com/Percona-Lab/pmm-submodules/blob/v3/README.md#how-to-create-a-feature-build) for more information."
+		exit 1
+	fi
 }
 
 # Update submodules and PR branches
 update() {
-  local DEPS=
-  local CURDIR="$PWD"
-  local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build"
-  local MD5SUM=$(md5sum $(dirname $0)/build)
-
-   if [ "$NO_UPDATE" -eq 1 ]; then
-    echo "Info: skip refreshing the source code from upstream repositories..."
-    return
-   fi
-
-  cd "$SUBMODULES"
-
-  # Join the dependencies from ci-default.yml and ci.yml
-  DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' ci-default.yml ../ci.yml | jq '.deps')
-
-  echo
-  echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and '../ci.yml'"
-
-  echo "$DEPS" | jq -c '.[]' | while read -r item; do
-    branch=$(echo "$item" | jq -r '.branch')
-    path=$(echo "$item" | jq -r '.path')
-    name=$(echo "$item" | jq -r '.name')
-    echo
-    echo "Rewinding submodule '$name' ..."
-    echo "path: ${path}, branch: ${branch}"
-
-    rewind "$path" "$branch"
-  done
-
-  echo
-  echo "Printing git status..."
-  git status --short
-  echo
-  echo "Printing git submodule status..."
-  git submodule status
-
-  cd "$CURDIR" > /dev/null
-
-  if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
-    echo "The local copy of this script differs from the one fetched from the repo." 
-    echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
-    echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build --no-update'"
-    exit 0
-  fi
+	local DEPS=
+	local CURDIR="$PWD"
+	local UPDATED_SCRIPT="$SUBMODULES/$PATH_TO_SCRIPTS/build/local/build"
+	local MD5SUM=$(md5sum $(dirname $0)/build)
+
+	 if [ "$NO_UPDATE" -eq 1 ]; then
+		echo "Info: skip refreshing the source code from upstream repositories..."
+		return
+	 fi
+
+	cat <<-'EOF' > entrypoint.sh
+		#!/bin/bash -e
+		DEPS=$(yq -o=json eval-all '. as $item ireduce ({}; . *d $item )' /ci-default.yml /ci.yml | jq '.deps')
+		echo $DEPS | jq -r '[.[] | {name: .name, branch: .branch, path: .path, url: .url}]'
+	EOF
+
+	chmod +x "$CURDIR/entrypoint.sh"
+	# Join the dependencies from ci-default.yml and ci.yml
+	DEPS=$(
+		docker run --rm --platform="$PLATFORM" \
+			-v $ROOT_DIR/ci-default.yml:/ci-default.yml \
+			-v $CURDIR/ci.yml:/ci.yml \
+			-v $CURDIR/entrypoint.sh:/entrypoint.sh \
+			--entrypoint=/entrypoint.sh \
+			"$RPMBUILD_DOCKER_IMAGE"
+	)
+
+	echo "$DEPS" > sbom.json
+
+	rm -f "$CURDIR/entrypoint.sh"
+
+	echo
+	echo "This script rewinds submodule branches as per the joint config of 'ci-default.yml' and 'ci.yml'"
+
+	cd "$SUBMODULES"
+
+	echo "$DEPS" | jq -c '.[]' | while read -r item; do
+		branch=$(echo "$item" | jq -r '.branch')
+		path=$(echo "$item" | jq -r '.path')
+		name=$(echo "$item" | jq -r '.name')
+		echo
+		echo "Rewinding submodule '$name' ..."
+		echo "path: ${path}, branch: ${branch}"
+
+		rewind "$path" "$branch"
+	done
+
+	echo
+	echo "Printing git status..."
+	git status --short
+	echo
+	echo "Printing git submodule status..."
+	git submodule status
+
+	cd "$CURDIR" > /dev/null
+
+	if [ -f "$UPDATED_SCRIPT" ] && [ "$MD5SUM" != "$(md5sum $UPDATED_SCRIPT)" ]; then
+		echo "The local copy of this script differs from the one fetched from the repo." 
+		echo "Apparently, that version is newer. We will halt to give you the change to run a fresh version."
+		echo "You can copy it over and run it again, i.e. '/bin/bash $(dirname $0)/build --no-update'"
+		exit 0
+	fi
 }
 
 get_branch_name() {
-  local module="${1:-}"
-  local path=$(git config -f .gitmodules submodule.${module}.path)
+	local module="${1:-}"
+	local path=$(git config -f .gitmodules submodule.${module}.path)
 
-  if [ ! -d "$path" ]; then
-    echo "Error: could not resolve the path to submodule ${module}"
-    exit 1
-  fi
+	if [ ! -d "$path" ]; then
+		echo "Error: could not resolve the path to submodule ${module}"
+		exit 1
+	fi
 
-  echo $(git -C "$path" branch --show-current)
+	echo $(git -C "$path" branch --show-current)
 }
 
 run_build_script() {
-  local CURDIR="$PWD"
-  local script="$PATH_TO_SCRIPTS/$1"
-  local script_name="$1"
-  local start_time
-  local end_time
-
-  cd "$SUBMODULES" > /dev/null
-
-  if [ ! -f "$script" ]; then
-    echo "Fatal: script $script does not exist."
-    cd "$CURDIR" > /dev/null
-    exit 1
-  fi
-
-  start_time=$(date +%s)
-  if [ "$#" -gt 1 ]; then
-    shift
-    script_name="${script_name}:($1)"
-    $script "$@"
-  else
-    $script
-  fi
-  end_time=$(date +%s)
-
-  echo ---
-  echo "Execution time for $script_name, sec: $((end_time - start_time))"
-  echo ---
-
-  cd "$CURDIR" > /dev/null
+	local CURDIR="$PWD"
+	local script="$PATH_TO_SCRIPTS/$1"
+	local script_name="$1"
+	local start_time
+	local end_time
+
+	cd "$SUBMODULES" > /dev/null
+
+	if [ ! -f "$script" ]; then
+		echo "Fatal: script $script does not exist."
+		cd "$CURDIR" > /dev/null
+		exit 1
+	fi
+
+	start_time=$(date +%s)
+	if [ "$#" -gt 1 ]; then
+		shift
+		script_name="${script_name}:($1)"
+		$script "$@"
+	else
+		$script
+	fi
+	end_time=$(date +%s)
+
+	echo ---
+	echo "Execution time for $script_name, sec: $((end_time - start_time))"
+	echo ---
+
+	cd "$CURDIR" > /dev/null
 }
 
 purge_files() {
-  local CURDIR=$PWD
-  local PMM_DIR="build/source/pmm"
-  local tmp_files
-
-  cd "$SUBMODULES" > /dev/null
-  # Remove stale files and directories
-  if [ -d build ]; then
-    echo "Removing stale files and directories..."
-
-    if [ -d "build/pmm-server" ]; then
-      tmp_files=$(find build/pmm-server | grep -v "RPMS" | grep -Ev "^build/pmm-server$" || :)
-      if [ -n "$tmp_files" ]; then
-        tmp_files=( $tmp_files )
-        for f in "${tmp_files[@]}"; do
-          echo "Removing file or directory $f ..."
-          rm -rf "$f"
-        done
-      fi
-    fi
-
-    if [ -d "$PMM_DIR" ]; then
-      echo "Removing $PMM_DIR ..."
-      rm -rf "$PMM_DIR"
-    fi
-  fi
-
-  if [ -d build ]; then
-    echo "Removing build/* ..."
-    rm -rf build/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
-  fi
-  
-  cd "$CURDIR"
+	local CURDIR=$PWD
+	local PMM_DIR="build/source/pmm"
+	local tmp_files
+
+	cd "$SUBMODULES" > /dev/null
+	# Remove stale files and directories
+	if [ -d build ]; then
+		echo "Removing stale files and directories..."
+
+		if [ -d "build/pmm-server" ]; then
+			tmp_files=$(find build/pmm-server | grep -v "RPMS" | grep -Ev "^build/pmm-server$" || :)
+			if [ -n "$tmp_files" ]; then
+				tmp_files=( $tmp_files )
+				for f in "${tmp_files[@]}"; do
+					echo "Removing file or directory $f ..."
+					rm -rf "$f"
+				done
+			fi
+		fi
+
+		if [ -d "$PMM_DIR" ]; then
+			echo "Removing $PMM_DIR ..."
+			rm -rf "$PMM_DIR"
+		fi
+	fi
+
+	if [ -d build ]; then
+		echo "Removing build/* ..."
+		rm -rf build/{rpm,srpm,tarball,source_tarball,pmm-client.properties}
+	fi
+	
+	cd "$CURDIR"
 }
 
-init() {
-  local CURDIR="$PWD"
-
-  if [ -d "$SUBMODULES" ]; then
-    cd "$SUBMODULES" > /dev/null
-  fi
-
-  export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
-  export DEBUG_MODE
-  export LOCAL_BUILD
-  export ROOT_DIR
-
-  # Create docker volumes to persist package and build cache
-  # Read more in the section about `rpmbuild`.
-  for volume in pmm-gobuild pmm-gomod pmm-yarn pmm-dnf; do
-    if ! docker volume ls | grep "$volume" >/dev/null; then
-      docker volume create "$volume"
-    else
-      echo "Info: docker volume $volume checked."
-    fi
-  done
-
-  cd "$CURDIR" > /dev/null
+check_volumes() {
+	local CURDIR="$PWD"
+
+	if [ -d "$SUBMODULES" ]; then
+		cd "$SUBMODULES" > /dev/null
+	fi
+
+	# Create docker volumes to persist package and build cache
+	# Read more in the section about `rpmbuild`.
+	for volume in pmm-gobuild pmm-gomod pmm-yarn pmm-dnf; do
+		if ! docker volume ls | grep "$volume" >/dev/null; then
+			docker volume create "$volume"
+		else
+			echo "Info: docker volume $volume checked."
+		fi
+	done
+
+	cd "$CURDIR" > /dev/null
 }
 
 cleanup() {
-  local CURDIR="$PWD"
-  cd "$SUBMODULES" > /dev/null
+	local CURDIR="$PWD"
+	cd "$SUBMODULES" > /dev/null
 
-  # Implement cleanup logic here
+	# Implement cleanup logic here as/if necessary
 
-  cd "$CURDIR" > /dev/null
+	cd "$CURDIR" > /dev/null
 }
 
 main() {
-  check_files "$SUBMODULES"
+	# All global variables are declared in `parse_params`
+	parse_params "$@"
 
-  init
+	# Capture the build logs in the log file
+	exec > >(tee "$LOG_FILE") 2>&1
 
-  update
+	check_files "$SUBMODULES"
 
-  purge_files
+	check_volumes
 
-  if [ "$NO_CLIENT" -eq 0 ]; then
-    # Build client source: 4m39s from scratch, 0m27s using cache
-    run_build_script build-client-source
+	update
 
-    # Build client binary: ??? from scratch, 0m20s using cache
-    run_build_script build-client-binary
+	purge_files
 
-    # Building client source rpm takes 13s (caching does not apply)
-    run_build_script build-client-srpm
+	if [ "$NO_CLIENT" -eq 0 ]; then
+		# Build client source: 4m39s from scratch, 0m27s using cache
+		run_build_script build-client-source
 
-    # Building client rpm takes 1m40s
-    run_build_script build-client-rpm
-  fi
+		# Build client binary: ??? from scratch, 0m20s using cache
+		run_build_script build-client-binary
 
-  # Building client docker image takes from 17s (using docker cache) to 43s (no docker cache).
-  if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
-    run_build_script build-client-docker
-  fi
+		# Building client source rpm takes 13s (caching does not apply)
+		run_build_script build-client-srpm
 
-  if [ "$NO_SERVER" -eq 0 ]; then
-    export RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
+		# Building client rpm takes 1m40s
+		run_build_script build-client-rpm
+	fi
 
-    # 1st-party components
-    run_build_script build-server-rpm percona-dashboards grafana-dashboards
-    run_build_script build-server-rpm pmm-managed pmm
-    run_build_script build-server-rpm percona-qan-api2 pmm
-    run_build_script build-server-rpm pmm-update pmm
-    run_build_script build-server-rpm pmm-dump
-    run_build_script build-server-rpm vmproxy pmm
+	# Building client docker image takes from 17s (using docker cache) to 43s (no docker cache).
+	if [ "$NO_CLIENT_DOCKER" -eq 0 ]; then
+		run_build_script build-client-docker
+	fi
 
-    # 3rd-party components
-    run_build_script build-server-rpm victoriametrics
-    run_build_script build-server-rpm grafana
+	if [ "$NO_SERVER" -eq 0 ]; then
+		RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
 
-    run_build_script build-server-docker
-  fi
+		# 1st-party components
+		run_build_script build-server-rpm percona-dashboards grafana-dashboards
+		run_build_script build-server-rpm pmm-managed pmm
+		run_build_script build-server-rpm percona-qan-api2 pmm
+		run_build_script build-server-rpm pmm-update pmm
+		run_build_script build-server-rpm pmm-dump
+		run_build_script build-server-rpm vmproxy pmm
 
-  echo
-  echo "Done building PMM artifacts."
-  echo ---
-  echo "Total execution time, sec: $(($(date +%s) - $START_TIME))"
-  echo ---
+		# 3rd-party components
+		run_build_script build-server-rpm victoriametrics
+		run_build_script build-server-rpm grafana
 
-  cleanup
-}
+		run_build_script build-server-docker
+	fi
 
-parse_params "$@"
+	echo
+	echo "Done building PMM artifacts."
+	echo ---
+	echo "Total execution time, sec: $(($(date +%s) - $START_TIME))"
+	echo ---
 
-# Capture the build logs in the log file
-exec > >(tee "$LOG_FILE") 2>&1
+	cleanup
+}
 
-main
+main "$@"

From 829f652cd11119201a07aad9670fab887a94e71f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 01:45:52 +0300
Subject: [PATCH 109/127] PMM-13487 clean yum once all packages are installed

---
 build/docker/rpmbuild/Dockerfile.el9 | 6 +++---
 build/scripts/build-client-packages  | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index f87844fb93..6c937990b5 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -25,8 +25,7 @@ RUN yum install -y gcc gcc-c++ \
     jq \
     rsync \
     wget && \
-    yum install -y --enablerepo=ol9_codeready_builder glibc-static && \
-    yum clean all && rm -rf /var/cache/yum
+    yum install -y --enablerepo=ol9_codeready_builder glibc-static
 
 # keep that format for easier search
 ARG GO_VERSION
@@ -41,7 +40,8 @@ RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
     yum install -y --disablerepo="*" /tmp/goreleaser.rpm && \
     rm /tmp/golang.tar.gz /tmp/goreleaser.rpm && \
     curl -o /usr/bin/yq -fSsL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${ARCH} && \
-    chmod +x /usr/bin/yq
+    chmod +x /usr/bin/yq && \
+    yum clean all && rm -rf /var/cache/yum
 
 
 RUN update-alternatives --install "/usr/bin/go" "go" "/usr/local/go/bin/go" 0
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index cc50b58abe..14da8e39df 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -270,7 +270,8 @@ build_deb(){
 
 #main
 
-CURDIR=$(pwd) # /home/builder/build
+# CURDIR is set to `/home/builder/build` inside the container, which is mapped to `pmm-submodules/build`
+CURDIR=$(pwd) 
 WORKDIR=/tmp/pmm
 PACKAGE_NAME=pmm-client
 VERSION_FILE=$CURDIR/pmm-client.properties

From ed87c6753b7dbc8fa0331a3932472604664cb0f4 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 02:43:57 +0300
Subject: [PATCH 110/127] PMM-13487 fix the platform for nomad

---
 build/scripts/build-client-binary | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 25f0880008..89893e7433 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -30,19 +30,18 @@ gobuild_component() {
     volume_mounts+=" -v pmm-gomod:/home/builder/go/pkg/mod"
     volume_mounts+=" -v pmm-dnf:/var/cache/dnf"
 
-    # Set target for Nomad accordingly
-    local target
-    case "$(uname -m)" in
-        x86_64)
-            target="linux_amd64"
-            ;;
-        aarch64)
-            target="linux_arm64"
-            ;;
-        *)
-            echo "Unsupported architecture: $(uname -m)"
-            exit 1
-            ;;
+    # Set target for Nomad
+    case "$platform" in
+      linux/amd64)
+        target="linux_amd64"
+        ;;
+      linux/arm64)
+        target="linux_arm64"
+        ;;
+      *)
+        echo "Unsupported platform: $platform"
+        exit 1
+        ;;
     esac
 
     if [ ${component} = 'azure_metrics_exporter' ]; then

From 031d5dd44a80143dd6ed36b0d0eb9e1aa91231a9 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 02:52:48 +0300
Subject: [PATCH 111/127] PMM-13487 install tree package required by nomad

---
 build/docker/rpmbuild/Dockerfile.el9 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
index 6c937990b5..7cfae1f18e 100644
--- a/build/docker/rpmbuild/Dockerfile.el9
+++ b/build/docker/rpmbuild/Dockerfile.el9
@@ -22,6 +22,7 @@ RUN yum install -y gcc gcc-c++ \
     automake autoconf \
     rpmdevtools createrepo_c epel-release \
     bison yum-utils rpm-build \
+    tree \
     jq \
     rsync \
     wget && \

From c2d3af99c2be2c395a39bc57430bc4d76a856e02 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 03:45:08 +0300
Subject: [PATCH 112/127] PMM-13487 fix the bin_dir path for nomad

---
 build/scripts/build-client-binary | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 89893e7433..5c679cfd15 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -61,7 +61,8 @@ gobuild_component() {
     elif [ ${component} = 'percona-toolkit' ]; then
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     elif [ ${component} = 'nomad' ]; then
-      build_command="cd /tmp/go/src/${extract_path} && TARGETS=${target} make deps release"      
+      build_command="cd /tmp/go/src/${extract_path} && TARGETS=${target} make deps release"
+      build_command+="; cp /tmp/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${bin_dir}/bin/nomad"
     else
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
@@ -91,6 +92,8 @@ gobuild_component() {
 
         if [ ${component} = vmagent ]; then
           export PMM_RELEASE_FULLCOMMIT=${vmagent_commit_hash}
+        elif [ ${component} = nomad ]; then
+          echo 'Skipping PMM_RELEASE_FULLCOMMIT for nomad...'
         else
           export PMM_RELEASE_FULLCOMMIT=$(git -C sources/${parent_component}/src/${extract_path} rev-parse HEAD)
         fi
@@ -156,7 +159,7 @@ main() {
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
-  cp -r ${tmp_dir}/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${binary_dir}/bin/nomad
+  # cp -r ${tmp_dir}/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${binary_dir}/bin/nomad
 
   rm -rf ${binary_tarball}
   mkdir -p $(dirname ${binary_tarball}) || :

From 52e8c94de79b31a087bc90d25dbe53875adccc21 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 14 Nov 2024 04:46:53 +0300
Subject: [PATCH 113/127] PMM-13487 combine srpm with rpm pkgs

---
 build/scripts/build-client-deb      | 19 +++++++++++++++++-
 build/scripts/build-client-packages |  5 -----
 build/scripts/build-client-rpm      | 16 ++++++++++++++-
 build/scripts/build-client-sdeb     | 31 -----------------------------
 build/scripts/build-client-srpm     | 27 -------------------------
 5 files changed, 33 insertions(+), 65 deletions(-)
 delete mode 100755 build/scripts/build-client-sdeb
 delete mode 100755 build/scripts/build-client-srpm

diff --git a/build/scripts/build-client-deb b/build/scripts/build-client-deb
index bd5781d155..19af43064c 100755
--- a/build/scripts/build-client-deb
+++ b/build/scripts/build-client-deb
@@ -6,6 +6,23 @@ set -o xtrace
 
 main() {
   local IMAGE=${1:-ubuntu:focal}
+  docker run --rm \
+    -e DEBIAN_FRONTEND=noninteractive \
+    -v ${bin_dir}:/home/builder/bin \
+    -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
+      set -o errexit
+      set -o xtrace
+
+      export pmm_release=$pmm_release
+      OS_VERSION=\$(cat /etc/os-release | grep VERSION_ID | awk -F'\"' '{print \$2}')
+
+      apt-get update
+      DEBIAN_FRONTEND=noninteractive apt-get -y install git lsb-release devscripts dh-make dh-systemd
+      cd /home/builder/build
+
+      bash /home/builder/bin/build-client-packages --build_source_deb
+    "
+
   docker run --rm \
     -e DEBIAN_FRONTEND=noninteractive \
     -v ${bin_dir}:/home/builder/bin \
@@ -24,7 +41,7 @@ main() {
 
       cd /home/builder/build
       bash /home/builder/bin/build-client-packages --build_deb
-  "
+    "
 }
 
 main $*
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 14da8e39df..cb22cd0cf4 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -117,10 +117,8 @@ build_srpm(){
     --define "dist %{nil}" \
     "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
 
-  # mkdir -p ${WORKDIR}/srpm
   mkdir -p ${CURDIR}/srpm
   cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
-  # cp rpmbuild/SRPMS/*.src.rpm ${WORKDIR}/srpm
 }
 
 build_rpm(){
@@ -134,9 +132,6 @@ build_rpm(){
     echo "Error: no source RPM could be found, exiting..."
     echo "You can create SRPM using the parameter --build_src_rpm"
     exit 1
-  # else
-    # rm -f "$CURDIR/srpm/$SRC_RPM" || :
-    # cp "$CURDIR/srpm/$SRC_RPM" "$WORKDIR"
   fi
 
   cd $WORKDIR
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index b71ad6516e..b8ba2fcc10 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -7,10 +7,24 @@ set -o xtrace
 
 main() {
   echo -----------------------------------------
-  echo "Building PMM Client RPM files..."
+  echo "Building PMM Client source RPM files..."
   echo -----------------------------------------
 
   local IMAGE=${1:-${rpmbuild_docker_image}}
+  docker run --rm -v ${bin_dir}:/home/builder/bin -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
+      set -o errexit
+      set -o xtrace
+
+      export pmm_release=$pmm_release
+      cd /home/builder/build
+
+      /home/builder/bin/build-client-packages --build_src_rpm
+  "
+
+  echo -----------------------------------------
+  echo "Building PMM Client RPM files..."
+  echo -----------------------------------------
+
   docker run --rm \
     -v ${bin_dir}:/home/builder/bin \
     -v ${build_dir}:/home/builder/build \
diff --git a/build/scripts/build-client-sdeb b/build/scripts/build-client-sdeb
deleted file mode 100755
index 386eb6e2ba..0000000000
--- a/build/scripts/build-client-sdeb
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/vars
-
-set -o errexit
-set -o xtrace
-
-bin_dir=$(cd $(dirname $0); pwd -P)
-
-main() {
-    local IMAGE=${1:-ubuntu:focal}
-    docker run --rm -e DEBIAN_FRONTEND=noninteractive \
-                    -v ${bin_dir}:/home/builder/bin \
-                    -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
-        set -o errexit
-        set -o xtrace
-
-        export pmm_release=$pmm_release
-        OS_VERSION=\$(cat /etc/os-release | grep VERSION_ID | awk -F'\"' '{print \$2}')
-
-        apt-get update
-        DEBIAN_FRONTEND=noninteractive apt-get -y install git lsb-release devscripts dh-make dh-systemd
-        cd /home/builder/build
-
-        bash /home/builder/bin/build-client-packages --build_source_deb
-    "
-}
-
-main $*
-
-# vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
deleted file mode 100755
index ed36793240..0000000000
--- a/build/scripts/build-client-srpm
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-. $(dirname $0)/vars
-
-set -o errexit
-set -o xtrace
-
-main() {
-    echo -----------------------------------------
-    echo "Building PMM Client source RPM files..."
-    echo -----------------------------------------
-
-    local IMAGE=${1:-${rpmbuild_docker_image}}
-    docker run --rm -v ${bin_dir}:/home/builder/bin -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
-        set -o errexit
-        set -o xtrace
-
-        export pmm_release=$pmm_release
-        cd /home/builder/build
-
-        /home/builder/bin/build-client-packages --build_src_rpm
-    "
-}
-
-main $*
-
-# vim: expandtab shiftwidth=2 tabstop=2

From e82ec43bb0d74c6c1f87b2ee47470467632e84d0 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 15 Nov 2024 17:30:25 +0300
Subject: [PATCH 114/127] PMM-13487 Deprecate RPM_EPOCH for server RPMs

---
 build/local/build                         | 14 +++++++++-----
 build/packages/rpm/client/pmm-client.spec |  4 ++--
 build/scripts/build-client-packages       |  2 +-
 build/scripts/build-client-rpm            | 13 +++++++++----
 build/scripts/build-server-rpm            | 14 +++++++-------
 5 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/build/local/build b/build/local/build
index 22e6a259c0..9e71ab1bcf 100755
--- a/build/local/build
+++ b/build/local/build
@@ -13,8 +13,9 @@ Options:
 			--client-only            Build only PMM Client (client binaries + docker)
 			--no-client              Do not build PMM Client
 			--no-client-docker       Do not build PMM Client docker image (default)
-			--log-file <path>        Save build logs to a file located at <path> (defaults to PWD).
-															 Note: the log file will get reset on every subsequent run.
+			--log-file <path>        Save build logs to a file located at <path> (defaults to PWD)
+															 Note: the log file will get reset on every subsequent run
+      --release                Mark it as a release build (otherwise it's a feature build)
 	-d  --debug                  Log output in debug mode, which also prints the commands
 	-h  --help                   Display help
 EOF
@@ -40,7 +41,8 @@ parse_params() {
 	export DEBUG_MODE=0
 	export ROOT_DIR="$(realpath ./${SUBMODULES})"
 	export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3
-	export RPM_EPOCH=0
+  # This replaces the old `RPM_EPOCH=1`, which was used for feature builds
+	export RELEASE_BUILD=0
 
 	while test "$#" -gt 0; do
 		case "$1" in
@@ -90,6 +92,10 @@ parse_params() {
 				fi
 				LOG_FILE="$1"
 				;;
+      --release)
+        RELEASE_BUILD=1
+        shift
+        ;;
 			--debug | -d)
 				DEBUG_MODE=1
 				shift
@@ -418,8 +424,6 @@ main() {
 	fi
 
 	if [ "$NO_SERVER" -eq 0 ]; then
-		RPM_EPOCH=1 # TODO: replace with RELEASE_BUILD=1
-
 		# 1st-party components
 		run_build_script build-server-rpm percona-dashboards grafana-dashboards
 		run_build_script build-server-rpm pmm-managed pmm
diff --git a/build/packages/rpm/client/pmm-client.spec b/build/packages/rpm/client/pmm-client.spec
index dea85fd042..a924b487ad 100644
--- a/build/packages/rpm/client/pmm-client.spec
+++ b/build/packages/rpm/client/pmm-client.spec
@@ -114,8 +114,8 @@ install -m 0660 example-queries-postgres.yml $RPM_BUILD_ROOT/usr/local/percona/p
 install -m 0660 queries-postgres-uptime.yml $RPM_BUILD_ROOT/usr/local/percona/pmm/collectors/custom-queries/postgresql/high-resolution/
 install -m 0660 queries-mr.yaml $RPM_BUILD_ROOT/usr/local/percona/pmm/collectors/custom-queries/postgresql/medium-resolution/
 install -m 0660 queries-lr.yaml $RPM_BUILD_ROOT/usr/local/percona/pmm/collectors/custom-queries/postgresql/low-resolution/
-install -m 0755 -d $RPM_BUILD_ROOT/%{_unitdir}
-install -m 0644 config/pmm-agent.service %{buildroot}/%{_unitdir}/pmm-agent.service
+install -m 0755 -d $RPM_BUILD_ROOT%{_unitdir}
+install -m 0644 config/pmm-agent.service %{buildroot}%{_unitdir}/pmm-agent.service
 
 
 %clean
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index cb22cd0cf4..604cce780a 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -143,7 +143,7 @@ build_rpm(){
     # Remove older client rpm packages before building a new one
     rm -f "${CURDIR}/rpm/${PACKAGE_NAME}-${pmm_version}-*.rpm"
   fi
-  rpmbuild --define "_topdir ${WORKDIR}/rpmbuild" \
+  rpmbuild  --define "_topdir ${WORKDIR}/rpmbuild" \
             --define "version $pmm_version" \
             --define "release $RPM_RELEASE" \
             --define "dist .$OS_NAME" \
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index b8ba2fcc10..4ee0182913 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -7,11 +7,15 @@ set -o xtrace
 
 main() {
   echo -----------------------------------------
-  echo "Building PMM Client source RPM files..."
+  echo "Building PMM Client source RPM packages..."
   echo -----------------------------------------
 
   local IMAGE=${1:-${rpmbuild_docker_image}}
-  docker run --rm -v ${bin_dir}:/home/builder/bin -v ${build_dir}:/home/builder/build ${IMAGE} sh -c "
+  docker run --rm \
+    --platform=${platform} \
+    -v ${bin_dir}:/home/builder/bin \
+    -v ${build_dir}:/home/builder/build \
+    ${IMAGE} sh -c "
       set -o errexit
       set -o xtrace
 
@@ -19,13 +23,14 @@ main() {
       cd /home/builder/build
 
       /home/builder/bin/build-client-packages --build_src_rpm
-  "
+    "
 
   echo -----------------------------------------
-  echo "Building PMM Client RPM files..."
+  echo "Building PMM Client RPM packages..."
   echo -----------------------------------------
 
   docker run --rm \
+    --platform=${platform} \
     -v ${bin_dir}:/home/builder/bin \
     -v ${build_dir}:/home/builder/build \
     ${IMAGE} sh -c "
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index b2acadaba9..43438c1f3d 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -24,15 +24,15 @@ is_build_needed() {
   local spec_name=$1
   local rpm_version=$2
   local packages=
-  local s3_cache_dir=RELEASE
+  local s3_cache_dir=PR-BUILDS
 
   # Structure of S3 build cache
   # s3://pmm-build-cache/PR-BUILDS/9 - el9
   # s3://pmm-build-cache/RELEASE/9 - el9
 
-  # Check if this is a feature build
-  if [ "$RPM_EPOCH" = 1 ]; then
-    s3_cache_dir=PR-BUILDS
+  # Check if this is a release build
+  if [ -z "$RPM_EPOCH"] && [ "$RELEASE_BUILD" = 1 ]; then
+    s3_cache_dir=RELEASE
   fi
   if command -v aws &> /dev/null && [ -z "$LOCAL_BUILD" ]; then
     aws s3 sync \
@@ -112,7 +112,7 @@ build() {
   prepare_spec "${spec_name}" "${repo_name}"
 
   local rpm_version=$(get_rpm_version "${spec_name}")
-  local s3_cache_dir=RELEASE
+  local s3_cache_dir=PR-BUILDS
 
   if ! is_build_needed "${spec_name}" "${rpm_version}"; then
       echo "Using cached rpm artefacts for ${spec_name}-${rpm_version}."
@@ -195,8 +195,8 @@ build() {
     return
   fi
 
-  if [ "$RPM_EPOCH" = 1 ]; then
-    s3_cache_dir=PR-BUILDS
+  if [ -z "$RPM_EPOCH" ] && [ "$RELEASE_BUILD" = 1 ]; then
+    s3_cache_dir=RELEASE
   fi
   aws s3 sync \
       --region us-east-2 \

From ecf02167a60079e65347cd04cd46ee98aa1ca7dd Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sat, 16 Nov 2024 16:42:36 +0300
Subject: [PATCH 115/127] PMM-13487 Refactor and clean up build-client-packages

---
 build/packages/deb/control          |   2 +-
 build/scripts/build-client-packages | 209 +++++++++++-----------------
 build/scripts/build-client-rpm      |  20 +--
 build/scripts/build-client-srpm     |  31 +++++
 4 files changed, 111 insertions(+), 151 deletions(-)
 create mode 100755 build/scripts/build-client-srpm

diff --git a/build/packages/deb/control b/build/packages/deb/control
index ce9884286c..8e9b5411c2 100644
--- a/build/packages/deb/control
+++ b/build/packages/deb/control
@@ -12,7 +12,7 @@ Breaks: pmm-client
 Conflicts: pmm2-client
 Replaces: pmm2-client
 Description: Percona Monitoring and Management Client
- Percona Monitoring and Management (PMM) is an open-source platform for managing and monitoring MySQL and MongoDB
+ Percona Monitoring and Management (PMM) is an open-source platform for managing and monitoring MySQL, PostgreSQL and MongoDB
  performance. It is developed by Percona in collaboration with experts in the field of managed database services,
  support and consulting.
  PMM is a free and open-source solution that you can run in your own environment for maximum security and reliability.
diff --git a/build/scripts/build-client-packages b/build/scripts/build-client-packages
index 604cce780a..d4919cf4cb 100755
--- a/build/scripts/build-client-packages
+++ b/build/scripts/build-client-packages
@@ -1,5 +1,5 @@
 #!/bin/sh
-# NOTE: this script is always executed in a container environment.
+# NOTE: this script must be executed in a container environment.
 
 set -o errexit
 set -o xtrace
@@ -8,9 +8,9 @@ usage () {
   cat <<EOF
 Usage: $0 [OPTIONS]
     The following options may be given :
-        --build_src_rpm     Build RPM packages
-        --build_source_deb  Build debian source packages
+        --build_src_rpm     Build source RPM packages
         --build_rpm         Build RPM packages
+        --build_source_deb  Build debian source packages
         --build_deb         Build debian packages
         --help)             Display this help
 Example $0 --build_src_rpm --build_rpm
@@ -19,13 +19,25 @@ EOF
   exit 1
 }
 
-parse_arguments() {
+run() {
   for arg do
     case "$arg" in
-      --build_src_rpm) SRPM=1 ;;
-      --build_rpm) RPM=1 ;;
-      --build_source_deb=*) SDEB=1 ;;
-      --build_deb) DEB=1 ;;
+      --build_src_rpm)
+        echo "Building srpm packages..."
+        build_srpm
+        ;;
+      --build_rpm)
+        echo "Building rpm packages..."
+        build_rpm
+        ;;
+      --build_source_deb=*)
+        echo "Building source deb packages..."
+        build_source_deb
+        ;;
+      --build_deb)
+        echo "Building deb packages..."
+        build_deb
+        ;;
       --help) usage ;;
       *) usage ;;
     esac
@@ -43,73 +55,44 @@ check_workdir(){
   fi
 }
 
-get_system(){
-  if [ -f /etc/redhat-release ]; then
-    RHEL=$(rpm --eval %rhel)
-    ARCH=$(echo $(uname -m) | sed -e 's:i686:i386:g')
-    OS_NAME="el$RHEL"
-    OS="rpm"
-else
-    ARCH=$(uname -m)
-    OS_NAME="$(lsb_release -sc)"
-    OS="deb"
-  fi
-}
-
+# Find the tarball and copy it to WORKDIR
 get_tar(){
-  local TARBALL=$1
-  local TARFILE=$(basename $(find $CURDIR/$TARBALL -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1) || :)
-  if [ -n $TARFILE ]; then
-    cp $CURDIR/$TARBALL/$TARFILE $WORKDIR/$TARFILE
-    return
+  local TARBALL_DIR=$1
+  local TARFILE=$(basename $(find "$CURDIR/$TARBALL_DIR" -name "${PACKAGE_NAME}*.tar.gz" | sort | tail -n1))
+  if [ ! -f "$CURDIR/$TARBALL_DIR/$TARFILE" ]; then
+    echo "Error: $TARBALL_DIR could not be found, exiting..."
+    exit 1
   fi
 
-  echo "Error: $TARBALL could not be found, exiting..."
-  exit 1
+  cp "$CURDIR/$TARBALL_DIR/$TARFILE" "$WORKDIR/$TARFILE"
+  echo "$TARFILE"
 }
 
 get_deb_sources(){
-    local param=$1
-    echo $param
-    local FILE=$(basename $(find $WORKDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
-    if [ -z $FILE ]; then
-        FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1) || :)
-        if [ -z $FILE ]; then
-            echo "There is no sources for build"
-            exit 1
-        else
-            cp $CURDIR/source_deb/$FILE $WORKDIR/
-        fi
-    else
-        cp $WORKDIR/source_deb/$FILE $WORKDIR/
-    fi
-    return
-}
-
-build_srpm(){
-  if [ "$OS" = "deb" ]; then
-    echo "Error: it is not possible to build source RPM on a debian based OS, exiting..."
+  local param=$1
+  local FILE=$(basename $(find $CURDIR/source_deb -name "${PACKAGE_NAME}*.$param" | sort | tail -n1))
+  if [ -z $FILE ]; then
+    echo "There is no source to build from"
     exit 1
+  else
+    cp $CURDIR/source_deb/$FILE $WORKDIR/
   fi
-  cd $WORKDIR
-  get_tar "tarball"
-
-  rm -fr rpmbuild
-  ls | grep -v tar.gz | xargs rm -rf
+}
 
-  local TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
+build_srpm(){
+  local TARFILE=$(get_tar "tarball")
   local NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
   local VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
   local VERSION=${VERSION_TMP%.tar.gz}
 
+  cd $WORKDIR
   mkdir -vp rpmbuild/{SOURCES,SPECS,BUILD,SRPMS,RPMS}
 
   tar -C ${WORKDIR} -zxpf ${TARFILE} ${NAME}-${VERSION}/rpm
 
-  cd ${WORKDIR}/rpmbuild/SPECS
-  cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec .
-  cp ${WORKDIR}/${TARFILE} ../SOURCES/
-  cd ${WORKDIR}
+  cp ${WORKDIR}/${NAME}-${VERSION}/rpm/*.spec ${WORKDIR}/rpmbuild/SPECS/
+  cp ${WORKDIR}/${TARFILE} ${WORKDIR}/rpmbuild/SOURCES/
+
   rpmbuild -bs \
     --define "_topdir ${WORKDIR}/rpmbuild" \
     --define "version $VERSION" \
@@ -117,20 +100,16 @@ build_srpm(){
     --define "dist %{nil}" \
     "rpmbuild/SPECS/${PACKAGE_NAME}.spec"
 
-  mkdir -p ${CURDIR}/srpm
-  cp rpmbuild/SRPMS/*.src.rpm ${CURDIR}/srpm
+  mkdir -p "${CURDIR}/srpm"
+  cp rpmbuild/SRPMS/*.src.rpm "${CURDIR}/srpm"
 }
 
 build_rpm(){
-  if [ "$OS" = "deb" ]; then
-    echo "It is not possible to build rpm here"
-    exit 1
-  fi
-
   local SRC_RPM=$(basename $(find $CURDIR/srpm -name "${PACKAGE_NAME}*.src.rpm" | sort | tail -n1) || :)
+  local RHEL=$(rpm --eval %rhel)
   if [ -z "$SRC_RPM" ]; then
     echo "Error: no source RPM could be found, exiting..."
-    echo "You can create SRPM using the parameter --build_src_rpm"
+    echo "You can build source RPM using the parameter --build_src_rpm"
     exit 1
   fi
 
@@ -146,7 +125,7 @@ build_rpm(){
   rpmbuild  --define "_topdir ${WORKDIR}/rpmbuild" \
             --define "version $pmm_version" \
             --define "release $RPM_RELEASE" \
-            --define "dist .$OS_NAME" \
+            --define "dist .el${RHEL}" \
             --rebuild rpmbuild/SRPMS/$SRC_RPM
 
   return_code=$?
@@ -155,23 +134,19 @@ build_rpm(){
     exit $return_code
   fi
 
-  mkdir -p ${CURDIR}/rpm
-  cp rpmbuild/RPMS/*/*.rpm ${CURDIR}/rpm
+  mkdir -p "${CURDIR}/rpm"
+  cp rpmbuild/RPMS/*/*.rpm "${CURDIR}/rpm"
 }
 
 build_source_deb(){
-  if [ "$OS" = "rpm" ]; then
-    echo "It is not possible to build source deb here"
-    exit 1
-  fi
   rm -rf pmm-client*
-  get_tar "tarball"
   rm -f *.dsc *.orig.tar.gz *.debian.tar.gz *.changes
 
-  TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
-  NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
-  VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
-  VERSION=${VERSION_TMP%.tar.gz}
+  local TARFILE=$(get_tar "tarball")
+  # TARFILE=$(basename $(find . -name "${PACKAGE_NAME}-*.tar.gz" | sort | tail -n1))
+  local NAME=$(echo ${TARFILE}| awk -F '-' '{print $1"-"$2}')
+  local VERSION_TMP=$(echo ${TARFILE}| awk -F '-' '{print $3}')
+  local VERSION=${VERSION_TMP%.tar.gz}
 
   rm -fr ${NAME}-${VERSION}
 
@@ -181,7 +156,7 @@ build_source_deb(){
   git clone $REPO ${NAME}-${VERSION}_all
   pushd ${NAME}-${VERSION}_all
     git fetch origin
-    if [ ! -z ${BRANCH} ]; then
+    if [ -n ${BRANCH} ]; then
       git reset --hard
       git clean -xdf
       git checkout ${BRANCH}
@@ -218,7 +193,7 @@ build_source_deb(){
 
   cd ../
 
-  dch -D unstable --force-distribution -v "${VERSION}-${DEB_RELEASE}" "Update to new upstream release PMM-client ${VERSION}-${DEB_RELEASE}"
+  dch -D unstable --force-distribution -v "${VERSION}-${DEB_RELEASE}" "Update to new upstream release of PMM Client ${VERSION}-${DEB_RELEASE}"
   dpkg-buildpackage -S
   cd ../
   mkdir -p $WORKDIR/source_deb
@@ -234,10 +209,6 @@ build_source_deb(){
 }
 
 build_deb(){
-  if [ "$OS" = "rpm" ]; then
-    echo "It is not possible to build deb here"
-    exit 1
-  fi
   for file in 'dsc' 'orig.tar.gz' 'changes' 'diff.gz'; do
     get_deb_sources $file
   done
@@ -258,55 +229,31 @@ build_deb(){
 
   dpkg-buildpackage -rfakeroot -uc -us -b
   mkdir -p $CURDIR/deb
-  mkdir -p $WORKDIR/deb
-  cp $WORKDIR/*.deb $WORKDIR/deb
   cp $WORKDIR/*.deb $CURDIR/deb
 }
 
-#main
-
-# CURDIR is set to `/home/builder/build` inside the container, which is mapped to `pmm-submodules/build`
-CURDIR=$(pwd) 
-WORKDIR=/tmp/pmm
-PACKAGE_NAME=pmm-client
-VERSION_FILE=$CURDIR/pmm-client.properties
-SRPM=0
-SDEB=0
-RPM=0
-DEB=0
-OS_NAME=
-ARCH=
-OS=
-RPM_RELEASE=1
-DEB_RELEASE=1
-REVISION=0
-BRANCH="v3"
-REPO="https://github.com/Percona-Lab/pmm-submodules.git"
-
-if [ -n "$pmm_release" ]; then
-  RPM_RELEASE="$RPM_RELEASE.$pmm_release"
-  DEB_RELEASE="$DEB_RELEASE.$pmm_release"
-fi
-
-parse_arguments "$@"
-get_system
-
-check_workdir
-if [ "$SRPM" != 0 ]; then
-  echo "Building srpm packages..."
-  build_srpm
-fi
-if [ "$SDEB" != 0 ]; then
-  echo "Building source deb packages..."
-  build_source_deb
-fi
-if [ "$RPM" != 0 ]; then
-  echo "Building rpm packages..."
-  build_rpm
-fi
-if [ "$DEB" != 0 ]; then
-  echo "Building deb packages..."
-  build_deb
-fi
+main() {
+  # CURDIR points to `/home/builder/build` inside the container, which is mapped to `pmm-submodules/build`
+  CURDIR=$(pwd) 
+  WORKDIR=/tmp/pmm
+  PACKAGE_NAME=pmm-client
+  VERSION_FILE=${CURDIR}/${PACKAGE_NAME}.properties
+  RPM_RELEASE=1
+  DEB_RELEASE=1
+  REVISION=0
+  BRANCH="v3"
+  REPO="https://github.com/Percona-Lab/pmm-submodules.git"
+
+  if [ -n "$pmm_release" ]; then
+    RPM_RELEASE="$RPM_RELEASE.$pmm_release"
+    DEB_RELEASE="$DEB_RELEASE.$pmm_release"
+  fi
+
+  check_workdir
+
+  run "$@"
+}
+
+main "$@"
 
 # vim: expandtab shiftwidth=2 tabstop=2
diff --git a/build/scripts/build-client-rpm b/build/scripts/build-client-rpm
index 4ee0182913..048950bd0f 100755
--- a/build/scripts/build-client-rpm
+++ b/build/scripts/build-client-rpm
@@ -6,29 +6,11 @@ set -o errexit
 set -o xtrace
 
 main() {
-  echo -----------------------------------------
-  echo "Building PMM Client source RPM packages..."
-  echo -----------------------------------------
-
-  local IMAGE=${1:-${rpmbuild_docker_image}}
-  docker run --rm \
-    --platform=${platform} \
-    -v ${bin_dir}:/home/builder/bin \
-    -v ${build_dir}:/home/builder/build \
-    ${IMAGE} sh -c "
-      set -o errexit
-      set -o xtrace
-
-      export pmm_release=$pmm_release
-      cd /home/builder/build
-
-      /home/builder/bin/build-client-packages --build_src_rpm
-    "
-
   echo -----------------------------------------
   echo "Building PMM Client RPM packages..."
   echo -----------------------------------------
 
+  local IMAGE=${1:-${rpmbuild_docker_image}}
   docker run --rm \
     --platform=${platform} \
     -v ${bin_dir}:/home/builder/bin \
diff --git a/build/scripts/build-client-srpm b/build/scripts/build-client-srpm
new file mode 100755
index 0000000000..a378a1719b
--- /dev/null
+++ b/build/scripts/build-client-srpm
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+. $(dirname $0)/vars
+
+set -o errexit
+set -o xtrace
+
+main() {
+  echo -----------------------------------------
+  echo "Building PMM Client source RPM packages..."
+  echo -----------------------------------------
+
+  local IMAGE=${1:-${rpmbuild_docker_image}}
+  docker run --rm \
+    --platform=${platform} \
+    -v ${bin_dir}:/home/builder/bin \
+    -v ${build_dir}:/home/builder/build \
+    ${IMAGE} sh -c "
+      set -o errexit
+      set -o xtrace
+
+      export pmm_release=$pmm_release
+      cd /home/builder/build
+
+      /home/builder/bin/build-client-packages --build_src_rpm
+    "
+}
+
+main $*
+
+# vim: expandtab shiftwidth=2 tabstop=2

From eaee27a13c8f76daec71ae53a17c3240497988e6 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 17 Nov 2024 14:48:19 +0300
Subject: [PATCH 116/127] PMM-13487 Remove debian:buster docker image

---
 build/packer/ansible/agent-aws.yml |  1 -
 build/scripts/build-server-rpm     | 12 +++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/build/packer/ansible/agent-aws.yml b/build/packer/ansible/agent-aws.yml
index 7bf8539408..519b4f37d7 100644
--- a/build/packer/ansible/agent-aws.yml
+++ b/build/packer/ansible/agent-aws.yml
@@ -195,7 +195,6 @@
         name: "{{ item }}"
         source: pull
       loop:
-        - debian:buster
         - debian:bullseye
         - debian:bookworm
         - ubuntu:focal
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 43438c1f3d..606f68fb55 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -9,15 +9,16 @@ get_rpm_version() {
 
   local rpm_version=$(
     docker run --rm \
-        --platform=${platform} \
-        -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES ${rpmbuild_docker_image} sh -c "
+      --platform=${platform} \
+      -v ${rpmbuild_dir}:/home/builder/rpm/SOURCES \
+      ${rpmbuild_docker_image} sh -c "
         rpmspec -q --qf '%{version}-%{release}\n' SOURCES/${spec_name}.spec \
             | sed -re 's/\.[0-9]{10}\././' | head -1
-    "
+      "
   )
 
   # return version
-  echo ${rpm_version}
+  echo "${rpm_version}"
 }
 
 is_build_needed() {
@@ -56,7 +57,7 @@ prepare_spec() {
   mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
   cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
   if [ ! -d "${root_dir}/sources/${repo_name}" ]; then
-    # Exit if the repo can't be found in pmm-submodules (e.g. VictoriaMetrics)
+    # Exit if the repo can't be found in pmm-submodules (e.g. VictoriaMetrics and Nomad)
     return
   fi
   if [ ! -f "${spec_file}" ]; then
@@ -91,6 +92,7 @@ prepare_spec() {
   fi
 
   if [ "${spec_name}" != "victorimetrics" ]; then
+    # Substitute the https source with the one on the filesystem
     sed -i -e "s;\(Source0:[[:space:]]*\)http.*;\1${tar_archive};" ${spec_file}
   fi
 

From 510f1c3360a0e12abf4a51fdaa0fd25c676c0e1d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Sun, 17 Nov 2024 14:51:24 +0300
Subject: [PATCH 117/127] PMM-13487 Bump up pip dependencies

---
 build/packer/ansible/agent-aws.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/packer/ansible/agent-aws.yml b/build/packer/ansible/agent-aws.yml
index 519b4f37d7..f57aded39d 100644
--- a/build/packer/ansible/agent-aws.yml
+++ b/build/packer/ansible/agent-aws.yml
@@ -148,9 +148,9 @@
         name: "{{ item }}"
         virtualenv: /home/ec2-user/venv
       loop: # Lookup versions at: https://pypi.org/search/
-        - pyaml==24.7.0
+        - pyaml==24.9.0
         - GitPython==3.1.43
-        - PyGithub==2.3.0
+        - PyGithub==2.5.0
         - python-digitalocean==1.17.0
         - docker==7.1.0
         - linode-cli==5.50.0

From 107999e2807e9247f59a5d9bb147db3fdc2546e9 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Mon, 18 Nov 2024 01:03:41 +0300
Subject: [PATCH 118/127] PMM-13487 Remove the commented code

---
 build/scripts/build-client-binary | 13 ++++++-------
 build/scripts/build-client-source |  2 +-
 build/scripts/vars                |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/build/scripts/build-client-binary b/build/scripts/build-client-binary
index 5c679cfd15..a9a3ed684f 100755
--- a/build/scripts/build-client-binary
+++ b/build/scripts/build-client-binary
@@ -61,8 +61,8 @@ gobuild_component() {
     elif [ ${component} = 'percona-toolkit' ]; then
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     elif [ ${component} = 'nomad' ]; then
-      build_command="cd /tmp/go/src/${extract_path} && TARGETS=${target} make deps release"
-      build_command+="; cp /tmp/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${bin_dir}/bin/nomad"
+      build_command="cd /tmp/go/src/${extract_path} && TARGETS=${target} make deps release;"
+      build_command+=" cp /tmp/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${bin_dir}/bin/nomad"
     else
       build_command="cd /tmp/go/src/${extract_path}/${component_path} && go build -o ${bin_dir}/${result_file} ."
     fi
@@ -142,6 +142,7 @@ main() {
 
   echo ${pmm_version} > ${binary_dir}/VERSION
 
+  # copy_component(component, path_in_submodules, path_in_binary_dir)
   copy_component "pmm" "build/packages/rpm/client" "rpm"
   copy_component "pmm" "build/packages/config" "config"
   copy_component "pmm" "build/packages/deb" "debian"
@@ -159,18 +160,16 @@ main() {
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-mongodb-summary"
   gobuild_component "percona-toolkit" "" "github.com/percona/percona-toolkit" "src/go/pt-pg-summary"
 
-  # cp -r ${tmp_dir}/go/src/github.com/hashicorp/nomad/pkg/linux_*/nomad ${binary_dir}/bin/nomad
-
   rm -rf ${binary_tarball}
-  mkdir -p $(dirname ${binary_tarball}) || :
+  mkdir -p $(dirname ${binary_tarball})
   
-  # Removes pmm binary (aka CLI) in case this is not a PR build - such as RC/GA build
+  # Removes pmm binary (aka CLI) in case this is not a feature build - such as RC/GA build
   if [[ "$(basename ${root_dir})" != pmm-submodules_PR-* ]]; then
     rm -f "${binary_dir}/bin/pmm"
   fi
   
   local noxattrs=
-  if [[ $(uname -o) =~ [Dd]arwin ]]; then
+  if [ $(uname -s) = "Darwin" ]; then
     noxattrs="--no-xattrs"
   fi
   tar -C $(dirname ${binary_dir}) ${noxattrs} -zcpf ${binary_tarball} $(basename ${binary_dir})
diff --git a/build/scripts/build-client-source b/build/scripts/build-client-source
index 0e96567058..02d789348c 100755
--- a/build/scripts/build-client-source
+++ b/build/scripts/build-client-source
@@ -61,7 +61,7 @@ main() {
   prepare_tarball percona-toolkit
 
   local noxattrs=
-  if [[ $(uname -o) =~ [Dd]arwin ]]; then
+  if [ $(uname -s) = "Darwin" ]; then
     noxattrs="--no-xattrs"
   fi
 
diff --git a/build/scripts/vars b/build/scripts/vars
index 92124459ec..1129b73b4d 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -42,7 +42,7 @@ echo -e "\n---\n>>> full_pmm_version=${full_pmm_version} pmm_version=${pmm_versi
 platform=${PLATFORM:-linux/amd64}
 rpmbuild_docker_image=${RPMBUILD_DOCKER_IMAGE:-public.ecr.aws/e7j3v3n0/rpmbuild:3}
 arch=$(docker run --rm --platform=${platform} ${rpmbuild_docker_image} uname -m 2>/dev/null)
-if [ "${arch}" = "arm64" ] && [[ "$(uname -o)" =~ [Dd]arwin ]]; then
+if [ "${arch}" = "arm64" ] && [ $(uname -s) = "Darwin" ]; then
   arch=aarch64
 fi
 

From 91844739fff9287701c90e873b94aff47d5bc575 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 12:43:03 +0300
Subject: [PATCH 119/127] PMM-13487 Add pmm-ui.spec

---
 build/local/build                             |   1 +
 .../rpm/server/SPECS/pmm-managed.spec         |  11 +-
 build/packages/rpm/server/SPECS/pmm-ui.spec   |  48 ++
 ui/LICENSE                                    | 661 ++++++++++++++++++
 4 files changed, 715 insertions(+), 6 deletions(-)
 create mode 100644 build/packages/rpm/server/SPECS/pmm-ui.spec
 create mode 100644 ui/LICENSE

diff --git a/build/local/build b/build/local/build
index 9e71ab1bcf..951ed01abc 100755
--- a/build/local/build
+++ b/build/local/build
@@ -427,6 +427,7 @@ main() {
 		# 1st-party components
 		run_build_script build-server-rpm percona-dashboards grafana-dashboards
 		run_build_script build-server-rpm pmm-managed pmm
+		run_build_script build-server-rpm pmm-ui pmm
 		run_build_script build-server-rpm percona-qan-api2 pmm
 		run_build_script build-server-rpm pmm-update pmm
 		run_build_script build-server-rpm pmm-dump
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index f5a14d6e62..6738ccee7f 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -5,7 +5,7 @@
 %global provider        github.com/percona/%{repo}
 %global commit          8f3d007617941033867aea6a134c48b39142427f
 %global shortcommit     %(c=%{commit}; echo ${c:0:7})
-%define release         21
+%define release         22
 %define rpm_release     %{release}.%{shortcommit}%{?dist}
 
 # the line below is sed'ed by build/bin/build-server-rpm to set a correct version
@@ -14,7 +14,7 @@
 Name:     pmm-managed
 Version:  %{version}
 Release:  %{rpm_release}
-Summary:  Percona Monitoring and Management management daemon
+Summary:  Percona Monitoring and Management daemon
 
 License:  AGPLv3
 URL:      https://%{provider}
@@ -35,20 +35,17 @@ export PMM_RELEASE_FULLCOMMIT=%{commit}
 export PMM_RELEASE_BRANCH=""
 
 make -C managed release
-make -C ui release
 
 %install
 install -d -p %{buildroot}%{_bindir}
 install -d -p %{buildroot}%{_sbindir}
 install -d -p %{buildroot}%{_datadir}/%{name}
-install -d -p %{buildroot}%{_datadir}/pmm-ui
 install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed
 install -p -m 0755 bin/pmm-encryption-rotation %{buildroot}%{_sbindir}/pmm-encryption-rotation
 install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init
 install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark
 
 cp -pa api/swagger %{buildroot}%{_datadir}/%{name}
-cp -pa ui/dist/. %{buildroot}%{_datadir}/pmm-ui
 
 %files
 %license managed/LICENSE
@@ -58,9 +55,11 @@ cp -pa ui/dist/. %{buildroot}%{_datadir}/pmm-ui
 %{_sbindir}/pmm-managed-init
 %{_sbindir}/pmm-managed-starlark
 %{_datadir}/%{name}
-%{_datadir}/pmm-ui
 
 %changelog
+* Wed Nov 20 2024 Alex Demidoff <alexander.demidoff@percona.com> - 3.0.0-22
+- PMM-13487 Split pmm-managed and pmm-ui rpm specifications
+
 * Mon Sep 23 2024 Jiri Ctvrtka <jiri.ctvrtka@ext.percona.com> - 3.0.0-1
 - PMM-13132 add PMM encryption rotation tool
 
diff --git a/build/packages/rpm/server/SPECS/pmm-ui.spec b/build/packages/rpm/server/SPECS/pmm-ui.spec
new file mode 100644
index 0000000000..ea9991190f
--- /dev/null
+++ b/build/packages/rpm/server/SPECS/pmm-ui.spec
@@ -0,0 +1,48 @@
+%global _missing_build_ids_terminate_build 0
+%global _dwz_low_mem_die_limit 0
+
+%global repo            pmm
+%global provider        github.com/percona/%{repo}
+%global commit          8f3d007617941033867aea6a134c48b39142427f
+%global shortcommit     %(c=%{commit}; echo ${c:0:7})
+%define release         1
+%define rpm_release     %{release}.%{shortcommit}%{?dist}
+
+# the line below is sed'ed by build/bin/build-server-rpm to set a correct version
+%define full_pmm_version 3.0.0
+
+Name:     pmm-ui
+Version:  %{version}
+Release:  %{rpm_release}
+Summary:  Percona Monitoring and Management UI
+
+License:  AGPLv3
+URL:      https://%{provider}
+Source0:  https://%{provider}/archive/%{commit}.tar.gz
+
+%description
+pmm-ui is the frontend application for pmm-managed.
+
+
+%prep
+%setup -q -n %{repo}-%{commit}
+
+%build
+export PMM_RELEASE_VERSION=%{full_pmm_version}
+export PMM_RELEASE_FULLCOMMIT=%{commit}
+export PMM_RELEASE_BRANCH=""
+
+make -C ui release
+
+%install
+install -d -p %{buildroot}%{_datadir}/%{name}
+cp -pa ui/dist/. %{buildroot}%{_datadir}/%{name}
+
+%files
+%license ui/LICENSE
+%doc ui/README.md
+%{_datadir}/%{name}
+
+%changelog
+* Fri Mar 22 2024 Matej Kubinec <matej.kubinec@ext.percona.com> - 3.0.0-1
+- PMM-11231 add pmm ui
diff --git a/ui/LICENSE b/ui/LICENSE
new file mode 100644
index 0000000000..be3f7b28e5
--- /dev/null
+++ b/ui/LICENSE
@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

From 5174e3f98cfa822d3ea484ed2a553854d54b2750 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 12:52:22 +0300
Subject: [PATCH 120/127] PMM-13487 FIx the BUILD_DATE for Mac

---
 build/scripts/build-client-docker  | 3 ++-
 build/scripts/build-server-rpm-all | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/build/scripts/build-client-docker b/build/scripts/build-client-docker
index 38531f7151..846b86b7ba 100755
--- a/build/scripts/build-client-docker
+++ b/build/scripts/build-client-docker
@@ -11,6 +11,7 @@ main() {
   echo -----------------------------------------
 
   local DOCKER_CONTEXT_DIR="${bin_dir}/../docker/client"
+  local BUILD_DATE=$(date -u +'%F %T%z' | sed 's@^.\{22\}@&:@')
   cp ${build_dir}/tarball/pmm-client-*.tar.gz ${DOCKER_CONTEXT_DIR}/pmm-client.tar.gz
 
   if [ -z "${DOCKER_CLIENT_TAG}" ]; then
@@ -20,7 +21,7 @@ main() {
   CLIENT_IMAGE_VERSION=`echo $DOCKER_CLIENT_TAG | cut -d ':' -f2`
 
   docker buildx build \
-    --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \
+    --build-arg BUILD_DATE="$BUILD_DATE" \
     --build-arg VERSION="$CLIENT_IMAGE_VERSION" \
     --progress plain \
     -f ${DOCKER_CONTEXT_DIR}/${docker_file} \
diff --git a/build/scripts/build-server-rpm-all b/build/scripts/build-server-rpm-all
index ba93725001..5f1e642115 100755
--- a/build/scripts/build-server-rpm-all
+++ b/build/scripts/build-server-rpm-all
@@ -7,6 +7,7 @@ set -o xtrace
 # 1st-party
 ${bin_dir}/build-server-rpm percona-dashboards grafana-dashboards
 ${bin_dir}/build-server-rpm pmm-managed pmm
+${bin_dir}/build-server-rpm pmm-ui pmm
 ${bin_dir}/build-server-rpm percona-qan-api2 pmm
 ${bin_dir}/build-server-rpm pmm-update pmm
 ${bin_dir}/build-server-rpm pmm-dump

From c71328c9012d7b6c183fc8c0b5c0bf3ad102f31d Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 13:41:34 +0300
Subject: [PATCH 121/127] PMM-13487 Fix the boolean logic

---
 build/local/build | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build/local/build b/build/local/build
index 951ed01abc..859cab280f 100755
--- a/build/local/build
+++ b/build/local/build
@@ -70,8 +70,8 @@ parse_params() {
 				NO_CLIENT_DOCKER=0
 				;;
 			--no-client-docker)
-				if [ "$NO_CLIENT_DOCKER" -eq 1 ]; then
-					echo "Error. Mutually exclusive options: --client-docker and --no-client-docker"
+				if [ "$NO_CLIENT" -eq 1 ]; then
+					echo "Error. Mutually exclusive options: --client-docker and --no-client"
 					exit 1
 				fi
 				NO_CLIENT_DOCKER=1

From 12dfdf1f905e8dc9889d2937ed685734b1ae081f Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 14:56:32 +0300
Subject: [PATCH 122/127] PMM-13487 Add pmm-ui to the list of installed rpm
 packages

---
 build/ansible/roles/pmm-images/tasks/main.yml | 1 +
 build/local/README.md                         | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml
index 615e468d68..0e9d53bb0d 100644
--- a/build/ansible/roles/pmm-images/tasks/main.yml
+++ b/build/ansible/roles/pmm-images/tasks/main.yml
@@ -92,6 +92,7 @@
       - percona-qan-api2
       - percona-dashboards
       - pmm-managed
+      - pmm-ui
       - pmm-update
       - pmm-dump
       - vmproxy
diff --git a/build/local/README.md b/build/local/README.md
index f418d6aebc..ab6779a188 100644
--- a/build/local/README.md
+++ b/build/local/README.md
@@ -17,7 +17,7 @@ The build process is designed to run on Linux or MacOS. We believe it could be a
 
 Below is a list of prerequisites that are required to build PMM locally.
 
-- OS: Linux (tested on Oracle Linux 9.3, CentOS 7, Ubuntu 22.04.3 LTS), MacOS (tested on Sequoia 15.1)
+- OS: Linux (tested on Oracle Linux 9.3, Ubuntu 22.04.3 LTS), MacOS (tested on Sequoia 15.1)
 - Docker: 25.0.2+
 - Docker buildx plugin: 0.16.0+, https://github.com/docker/buildx
 - make
@@ -27,6 +27,7 @@ Below is a list of prerequisites that are required to build PMM locally.
 - curl
 - jq: 1.6+
 
+Please note, that building some of the PMM internals, such as Grafana, requires at least 8GB of memory available to docker. The number of CPUs, however, does not matter that much.
 
 ## How to use this script to build PMM
 

From 1454a6da34f9ad21ab2ab4dd1f36df981e9069d0 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 15:37:03 +0300
Subject: [PATCH 123/127] PMM-13487 Do not run builddep on dashboars

---
 build/packages/rpm/server/SPECS/percona-dashboards.spec | 3 ---
 build/scripts/build-server-rpm                          | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/percona-dashboards.spec b/build/packages/rpm/server/SPECS/percona-dashboards.spec
index 19a7a93071..cacd2ebbf3 100644
--- a/build/packages/rpm/server/SPECS/percona-dashboards.spec
+++ b/build/packages/rpm/server/SPECS/percona-dashboards.spec
@@ -17,9 +17,6 @@ License:  AGPLv3
 URL:      https://%{provider}
 Source0:  https://%{provider}/archive/%{commit}.tar.gz
 
-BuildRequires:	nodejs
-Requires:	percona-grafana
-
 %description
 This is a set of Grafana dashboards for database and system monitoring
 using VictoriaMetrics datasource.
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 606f68fb55..5ca29d215c 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -169,7 +169,7 @@ build() {
     done
 
     # Only these two specs have build dependencies
-    if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then
+    if [[ ${spec_name} =~ ^grafana$ ]]; then
       sleep 2s
       sudo yum-builddep -y SOURCES/${spec_name}.spec
     fi

From 907309961219c4ec732b8248dd663aac574b44cf Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 23:04:12 +0300
Subject: [PATCH 124/127] PMM-13487 Cache pmm-ui as well

---
 build/scripts/build-server-rpm | 14 ++++++++++----
 build/scripts/vars             | 15 +++++++++++++--
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 5ca29d215c..3963789e7e 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -75,7 +75,13 @@ prepare_spec() {
     exit 1
   fi
 
-  sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
+  if [ "${spec_name}" = "pmm-ui" ]; then
+    local hash=$(get_shasum256 "ui")
+    sed -i -e "s/global commit.*/global commit ${hash:0:7}/" ${spec_file}
+  else
+    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
+  fi
+
   if [ "${spec_name}" != "grafana" ]; then
     sed -i -e "s/Version:.*/Version: ${pmm_version}/" ${spec_file}
   fi
@@ -92,12 +98,12 @@ prepare_spec() {
   fi
 
   if [ "${spec_name}" != "victorimetrics" ]; then
-    # Substitute the https source with the one on the filesystem
+    # Substitute the https source by pointing to the repo on the filesystem
     sed -i -e "s;\(Source0:[[:space:]]*\)http.*;\1${tar_archive};" ${spec_file}
   fi
 
   if [ -f "${tar_archive}" ]; then
-    echo "${tar_archive}" already exists, skip archiving...
+    echo "Info: ${tar_archive} already exists, skip archiving..."
     return
   fi
 
@@ -190,7 +196,7 @@ build() {
   fi
 
   if ! command -v aws &> /dev/null; then
-    echo "AWS CLI is not installed, skipping upload to S3..."
+    echo "Skipping upload to S3..."
     echo
     echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}"
     echo
diff --git a/build/scripts/vars b/build/scripts/vars
index 1129b73b4d..2234c7805b 100644
--- a/build/scripts/vars
+++ b/build/scripts/vars
@@ -2,7 +2,8 @@
 # directory needs to have them available in its scope to function properly.
 
 bin_dir=$(cd $(dirname $0); pwd -P)
-# Note: `ROOT_DIR` can be passed as ${WORKSPACE} from Jenkins or ${PWD} from GH actions
+# Note: `ROOT_DIR` can be passed as ${WORKSPACE} from Jenkins or ${PWD} from GH actions.
+# It points to the root of [pmm-submodules](https://github.com/percona-lab/pmm-submodules).
 root_dir=${ROOT_DIR:-$(cd $(dirname $0)/../../../../../../../..; pwd -P)}
 build_dir=${root_dir}/build
 
@@ -49,7 +50,7 @@ fi
 rpms_dir=${build_dir}/pmm-server/RPMS
 rpmbuild_dir=${build_dir}/pmm-server/BUILD
 rpmspec_dir=${root_dir}/$(git -C ${root_dir} config -f .gitmodules submodule.pmm.path)/build/packages/rpm/server/SPECS
-rpmbuild_dist=${RPMBUILD_DIST:-"el9"}
+rpmbuild_dist=${RPMBUILD_DIST:-el9}
 source_dir=${build_dir}/source/pmm-client-${pmm_version}
 binary_dir=${build_dir}/binary/pmm-client-${pmm_version}
 client_properties=${build_dir}/pmm-client.properties
@@ -63,3 +64,13 @@ binary_tarball=${build_dir}/tarball/pmm-client-${pmm_version}.tar.gz
 vmagent_commit_hash=58ecb9066574f38f1d1c91ace467316e7f175b09
 # https://github.com/hashicorp/nomad/releases/tag/v1.9.3
 nomad_commit_hash=d92bf1014886c0ff9f882f4a2691d5ae8ad8131c
+
+get_shasum256() {
+  local DIR=${1:-}
+  if [ ! -d "$DIR" ]; then
+    echo -e "Error: directory '$1' does not exist, exiting..."
+    exit 1
+  fi
+
+  find "$DIR" -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -d " " -f1 | cut -c 1-8
+}

From db2904d5ba76a1a083229d0161b303aab3140321 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 23:12:43 +0300
Subject: [PATCH 125/127] PMM-13487 Do not warn about missing build IDs

---
 build/packages/rpm/client/pmm-client.spec            | 1 +
 build/packages/rpm/server/SPECS/pmm-dump.spec        | 2 +-
 build/packages/rpm/server/SPECS/pmm-managed.spec     | 2 +-
 build/packages/rpm/server/SPECS/pmm-update.spec      | 6 +++---
 build/packages/rpm/server/SPECS/victoriametrics.spec | 2 +-
 build/packages/rpm/server/SPECS/vmproxy.spec         | 2 +-
 6 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/build/packages/rpm/client/pmm-client.spec b/build/packages/rpm/client/pmm-client.spec
index a924b487ad..840e86bb97 100644
--- a/build/packages/rpm/client/pmm-client.spec
+++ b/build/packages/rpm/client/pmm-client.spec
@@ -1,3 +1,4 @@
+%global _missing_build_ids_terminate_build 0
 %define debug_package %{nil}
 
 Name:           pmm-client
diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec
index d97cb8a68c..e4f576ec4f 100644
--- a/build/packages/rpm/server/SPECS/pmm-dump.spec
+++ b/build/packages/rpm/server/SPECS/pmm-dump.spec
@@ -1,4 +1,4 @@
-%undefine _missing_build_ids_terminate_build
+%global _missing_build_ids_terminate_build 0
 
 %global repo            pmm-dump
 %global provider        github.com/percona/%{repo}
diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec
index 6738ccee7f..e03f10da72 100644
--- a/build/packages/rpm/server/SPECS/pmm-managed.spec
+++ b/build/packages/rpm/server/SPECS/pmm-managed.spec
@@ -1,4 +1,4 @@
-%undefine _missing_build_ids_terminate_build
+%global _missing_build_ids_terminate_build 0
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm
diff --git a/build/packages/rpm/server/SPECS/pmm-update.spec b/build/packages/rpm/server/SPECS/pmm-update.spec
index e4fda14f4e..f23ac3c60b 100644
--- a/build/packages/rpm/server/SPECS/pmm-update.spec
+++ b/build/packages/rpm/server/SPECS/pmm-update.spec
@@ -1,6 +1,6 @@
-%define _missing_build_ids_terminate_build   0
-%define _binaries_in_noarch_packages_terminate_build   0
-%define _unpackaged_files_terminate_build   0
+%global _missing_build_ids_terminate_build  0
+%define _binaries_in_noarch_packages_terminate_build  0
+%define _unpackaged_files_terminate_build  0
 
 %global repo            pmm
 %global provider        github.com/percona/%{repo}
diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec
index 97e2b39a78..1fcb9a3eb3 100644
--- a/build/packages/rpm/server/SPECS/victoriametrics.spec
+++ b/build/packages/rpm/server/SPECS/victoriametrics.spec
@@ -1,4 +1,4 @@
-%undefine _missing_build_ids_terminate_build
+%global _missing_build_ids_terminate_build 0
 
 %global repo            VictoriaMetrics
 %global provider        github.com/VictoriaMetrics/%{repo}
diff --git a/build/packages/rpm/server/SPECS/vmproxy.spec b/build/packages/rpm/server/SPECS/vmproxy.spec
index b32e19b1fe..771aeae83f 100644
--- a/build/packages/rpm/server/SPECS/vmproxy.spec
+++ b/build/packages/rpm/server/SPECS/vmproxy.spec
@@ -1,4 +1,4 @@
-%undefine _missing_build_ids_terminate_build
+%global _missing_build_ids_terminate_build 0
 %global _dwz_low_mem_die_limit 0
 
 %global repo            pmm

From 0ba4b938868b246536370e5acc221b0c43ab1c23 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Thu, 21 Nov 2024 23:14:05 +0300
Subject: [PATCH 126/127] PMM-13487 Fix path to pmm-ui

---
 build/scripts/build-server-rpm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 3963789e7e..3c66d0a690 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -76,7 +76,7 @@ prepare_spec() {
   fi
 
   if [ "${spec_name}" = "pmm-ui" ]; then
-    local hash=$(get_shasum256 "ui")
+    local hash=$(get_shasum256 "${git_dir}/ui")
     sed -i -e "s/global commit.*/global commit ${hash:0:7}/" ${spec_file}
   else
     sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}

From 78235423f573986ca900412af6e7905442917655 Mon Sep 17 00:00:00 2001
From: Alex Demidoff <alexander.demidoff@percona.com>
Date: Fri, 22 Nov 2024 16:59:58 +0300
Subject: [PATCH 127/127] PMM-13487 Use full_commit for the hash as well

---
 build/packages/rpm/server/SPECS/pmm-ui.spec |  4 ----
 build/scripts/build-server-rpm              | 17 +++++++++--------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/build/packages/rpm/server/SPECS/pmm-ui.spec b/build/packages/rpm/server/SPECS/pmm-ui.spec
index ea9991190f..c8fd25eecf 100644
--- a/build/packages/rpm/server/SPECS/pmm-ui.spec
+++ b/build/packages/rpm/server/SPECS/pmm-ui.spec
@@ -28,10 +28,6 @@ pmm-ui is the frontend application for pmm-managed.
 %setup -q -n %{repo}-%{commit}
 
 %build
-export PMM_RELEASE_VERSION=%{full_pmm_version}
-export PMM_RELEASE_FULLCOMMIT=%{commit}
-export PMM_RELEASE_BRANCH=""
-
 make -C ui release
 
 %install
diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm
index 3c66d0a690..fa2869f636 100755
--- a/build/scripts/build-server-rpm
+++ b/build/scripts/build-server-rpm
@@ -53,6 +53,7 @@ prepare_spec() {
   local spec_name=$1
   local repo_name=$2
   local spec_file=${rpmbuild_dir}/${spec_name}.spec
+  local full_commit=
 
   mkdir -p ${rpms_dir} ${rpmbuild_dir} 2>/dev/null || :
   cp ${rpmspec_dir}/${spec_name}.spec ${spec_file}
@@ -66,7 +67,12 @@ prepare_spec() {
   fi
   
   local git_dir=$(dirname $(find "${root_dir}/sources/${repo_name}" -name .git | head -1))
-  local full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+  if [ "${spec_name}" = "pmm-ui" ]; then
+    full_commit=$(get_shasum256 "${git_dir}/ui")
+  else
+    full_commit=$(git -C "${git_dir}" rev-parse HEAD)
+  fi
+
   local short_commit=${full_commit:0:7}
   local tar_archive=${rpmbuild_dir}/${repo_name}-${short_commit}.tar.gz
 
@@ -75,12 +81,7 @@ prepare_spec() {
     exit 1
   fi
 
-  if [ "${spec_name}" = "pmm-ui" ]; then
-    local hash=$(get_shasum256 "${git_dir}/ui")
-    sed -i -e "s/global commit.*/global commit ${hash:0:7}/" ${spec_file}
-  else
-    sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
-  fi
+  sed -i -e "s/global commit.*/global commit ${full_commit}/" ${spec_file}
 
   if [ "${spec_name}" != "grafana" ]; then
     sed -i -e "s/Version:.*/Version: ${pmm_version}/" ${spec_file}
@@ -111,7 +112,7 @@ prepare_spec() {
       --format=tar.gz \
       --prefix="${repo_name}-${full_commit}/" \
       -o "${tar_archive}" \
-      "${full_commit}"
+      "$(git -C "${git_dir}" rev-parse HEAD)"
 }
 
 build() {