Support arm packages generation (#376)

wazuh · Oct 28, 2024 · 888d29c · 888d29c
1 parent 37af645
commit 888d29c
Show file tree

Hide file tree

Showing 17 changed files with 578 additions and 886 deletions.
diff --git a/.github/workflows/build_base.yml b/.github/workflows/build_base.yml
@@ -10,25 +10,34 @@ on:
         required: true
         default: ''
         type: string
+      ARCHITECTURE:
+        description: 'The architecture to build the package for'
+        required: true
+        default: 'amd64'
+        type: string
 
   workflow_dispatch:
     inputs:
       CHECKOUT_TO: # This is the branch to checkout to. Defaults to 'master'
         description: 'The branch/tag/commit to checkout to'
         required: true
         default: ''
+      ARCHITECTURE:
+        description: 'The architecture to build the package for'
+        required: true
+        default: 'amd64'
+        type: string
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'ubuntu-latest' || 'wz-linux-arm64' }}
     name: Build
     defaults:
       run:
         working-directory: ./artifacts
     strategy:
       matrix:
         DISTRIBUTION: [tar.gz]
-        ARCHITECTURE: [x64]
 
     steps:
       - name: Checkout code
@@ -69,21 +78,16 @@ jobs:
 
       - name: Get artifact build name
         run: |
-          echo "ARTIFACT_BUILD_NAME=wazuh-dashboard_${{ env.WZD_VERSION }}-${{ env.WZD_REVISION }}_${{ matrix.ARCHITECTURE }}.${{ matrix.DISTRIBUTION }}" >> $GITHUB_ENV
+          echo "ARTIFACT_BUILD_NAME=wazuh-dashboard_${{ env.WZD_VERSION }}-${{ env.WZD_REVISION }}_${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'x64' || 'arm64' }}.${{ matrix.DISTRIBUTION }}" >> $GITHUB_ENV
 
       - name: Run bootstrap
         run: yarn osd bootstrap
 
-      - name: Build linux-x64
-        if: matrix.ARCHITECTURE == 'x64'
-        run: yarn build-platform --linux --skip-os-packages --release
-
-      - name: Build linux-arm64
-        if: matrix.ARCHITECTURE == 'arm64'
-        run: yarn build-platform --linux-arm --skip-os-packages --release
+      - name: Build
+        run: yarn build-platform --${{(inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'linux' || 'linux-arm'}} --skip-os-packages --release
 
       - name: Rename artifact
-        run: mv /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/opensearch-dashboards-${{ env.VERSION }}-linux-${{ matrix.ARCHITECTURE }}.${{ matrix.DISTRIBUTION }} /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
+        run: mv /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/opensearch-dashboards-${{ env.VERSION }}-linux-${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'x64' || 'arm64' }}.${{ matrix.DISTRIBUTION }} /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
 
       - uses: actions/upload-artifact@v3
         if: success()

diff --git a/.github/workflows/build_wazuh_dashboard_with_plugins.yml b/.github/workflows/build_wazuh_dashboard_with_plugins.yml
@@ -19,6 +19,8 @@ on:
         options:
           - amd64
           - x86_64
+          - aarch64
+          - arm64
         default: amd64
       revision:
         type: string
@@ -91,11 +93,11 @@ jobs:
     steps:
       - name: Validate inputs
         run: |
-          if [ "${{ inputs.architecture }}" = "amd64" ] && [ "${{ inputs.system }}" = "rpm" ]; then
+          if [[ "${{ inputs.architecture }}" == "amd64" || "${{ inputs.architecture }}" == "arm64" ]] && [[ "${{ inputs.system }}" == "rpm" ]]; then
             echo "Invalid combination of architecture and system"
             exit 1
           fi
-          if [ "${{ inputs.architecture }}" = "x86_64" ] && [ "${{ inputs.system }}" = "deb" ]; then
+          if [[ "${{ inputs.architecture }}" == "x86_64" || "${{ inputs.architecture }}" == "aarch64" ]] && [[ "${{ inputs.system }}" == "deb" ]]; then
             echo "Invalid combination of architecture and system"
             exit 1
           fi
@@ -106,6 +108,7 @@ jobs:
     uses: wazuh/wazuh-dashboard/.github/workflows/[email protected]
     with:
       CHECKOUT_TO: ${{ github.head_ref || github.ref_name }}
+      ARCHITECTURE: ${{ inputs.architecture }}
 
   build-main-plugins:
     needs: [validate-inputs]
@@ -123,7 +126,7 @@ jobs:
 
   build-and-test-package:
     needs: [build-main-plugins, build-base, build-security-plugin]
-    runs-on: ubuntu-latest
+    runs-on: ${{ (inputs.architecture == 'arm64' || inputs.architecture == 'aarch64') && 'wz-linux-arm64' || 'ubuntu-22.04' }}
     name: Generate packages
     steps:
       - name: Checkout code
@@ -147,9 +150,14 @@ jobs:
 
       - name: Setup packages names
         run: |
-          echo "WAZUH_DASHBOARD_SLIM=wazuh-dashboard_${{ env.VERSION }}-${{ env.REVISION }}_x64.tar.gz" >> $GITHUB_ENV
+          echo "WAZUH_DASHBOARD_SLIM=wazuh-dashboard_${{ env.VERSION }}-${{ env.REVISION }}_${{(inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'x64' || 'arm64'}}.tar.gz" >> $GITHUB_ENV
           echo "WAZUH_SECURITY_PLUGIN=wazuh-security-dashboards-plugin_${{ env.VERSION }}-${{ env.REVISION }}_${{ inputs.reference_security_plugins }}.zip" >> $GITHUB_ENV
           echo "WAZUH_PLUGINS=wazuh-dashboard-plugins_${{ env.VERSION }}-${{ env.REVISION }}_${{ inputs.reference_wazuh_plugins }}.zip" >> $GITHUB_ENV
+          if [[ "${{ inputs.architecture }}" == "x86_64" || "${{ inputs.architecture }}" == "amd64" ]]; then
+            echo "ARCHITECTURE_FLAG=" >> $GITHUB_ENV
+          else
+            echo "ARCHITECTURE_FLAG=--arm" >> $GITHUB_ENV
+          fi
           if [ "${{ inputs.system }}" = "deb" ]; then
             if [ "${{ inputs.is_stage }}" = "true" ]; then
               echo "PACKAGE_NAME=wazuh-dashboard_${{ env.VERSION }}-${{ inputs.revision }}_${{ inputs.architecture }}.deb" >> $GITHUB_ENV
@@ -193,17 +201,16 @@ jobs:
           cd ${{ env.CURRENT_DIR }}/dev-tools/build-packages
           bash ./build-packages.sh \
             -v ${{ env.VERSION }} \
-            -r ${{ inputs.revision }} \
+            -r ${{ inputs.revision }} ${{env.ARCHITECTURE_FLAG}}\
             -a file://${{env.CURRENT_DIR}}/artifacts/wazuh-package.zip \
             -s file://${{env.CURRENT_DIR}}/artifacts/security-package.zip \
             -b file://${{env.CURRENT_DIR}}/artifacts/dashboard-package.zip \
-            --${{ inputs.system }} ${{ env.PRODUCTION }}
+            --${{ inputs.system }} --debug ${{ env.PRODUCTION }}
 
       - name: Test package
         run: |
           cd ${{ env.CURRENT_DIR }}/dev-tools/test-packages
-          ls -la ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{ inputs.system }}
-          cp ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{ inputs.system }}/${{env.PACKAGE_NAME}}  ${{ env.CURRENT_DIR }}/dev-tools/test-packages/${{ inputs.system }}
+          cp ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{env.PACKAGE_NAME}}  ${{ env.CURRENT_DIR }}/dev-tools/test-packages/${{ inputs.system }}
           bash ./test-packages.sh \
             -p ${{env.PACKAGE_NAME}}
 
@@ -217,14 +224,14 @@ jobs:
       - name: Upload package
         run: |
           echo "Uploading package"
-          aws s3 cp  ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{ inputs.system }}/${{env.PACKAGE_NAME}} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
+          aws s3 cp  ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{env.PACKAGE_NAME}} s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
           s3uri="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/${{env.PACKAGE_NAME}}"
           echo "S3 URI: ${s3uri}"
 
       - name: Upload SHA512
         if: ${{ inputs.checksum }}
         run: |
           echo "Uploading checksum"
-          aws s3 cp ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{ inputs.system }}/${{env.PACKAGE_NAME}}.sha512 s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
+          aws s3 cp ${{ env.CURRENT_DIR }}/dev-tools/build-packages/output/${{env.PACKAGE_NAME}}.sha512 s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/
           s3uri="s3://packages-dev.internal.wazuh.com/development/wazuh/4.x/main/packages/${{env.PACKAGE_NAME}}.sha512"
           echo "S3 sha512 URI: ${s3uri}"
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,8 @@
 .aws-config.json
 .signing-config.json
 .ackrc
-/dev-tools/build-packages/*/output
+/dev-tools/build-packages/output
+/dev-tools/build-packages/tmp
 /.opensearch
 /.chromium
 /package.json.bak
@@ -71,4 +72,4 @@ snapshots.js
 .yarn-local-mirror
 
 # Ignore the generated antlr files
-/src/plugins/data/public/antlr/opensearch_sql/grammar/.antlr
+/src/plugins/data/public/antlr/opensearch_sql/grammar/.antlr
diff --git a/dev-tools/build-packages/README.md b/dev-tools/build-packages/README.md
@@ -1,70 +1,56 @@
 # Package building
-This folder contains tools used to create `rpm` and `deb` packages. 
+
+This folder contains tools used to create `tar`, `rpm` and `deb` packages.
 
 ## Requirements
- - A system with Docker.
- - Internet connection (to download the docker images the first time).
 
-## Builders
+- A system with Docker.
+- Internet connection (to download the docker images the first time).
 
-### Tarball
+## How to build packages
 
-To system packages (deb and rpm), a tarball of Wazuh dashboard `.tar.gz` is required.
-This tarball contains the [Wazuh plugin][wazuh-plugin], the [Wazuh Security plugin][wazuh-security-plugin], 
-a set of OpenSearch plugins and the default configuration for the app. 
+The script `build-packages.sh` is in charge of coordinating the different steps to build each type of packages.
 
-The `generate_base.sh` script generates a `.tar.gz` file using the following inputs:
-- `-a` | `--app`: URL to the zipped Wazuh plugin.*
-- `-b` | `--base`: URL to the Wazuh dashboard `.tar.gz`, as generated with `yarn build --skip-os-packages --release`.*
-- `-s` | `--security`: URL to the zipped Wazuh Security plugin, as generated with `yarn build`.*
-- `-v` | `--version`: the Wazuh version of the package.
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script.
+### Pre-requisites
 
-*Note:* use `file://<absolute_path>` to indicate a local file. Otherwise, the script will try to download the file from the given URL.
+The script needs 3 different zip files, containing the following respectively:
 
-Example:
-```bash
-bash generate_base.sh \
-    --app https://packages-dev.wazuh.com/pre-release/ui/dashboard/wazuh-4.6.0-1.zip \
-    --base file:///home/user/wazuh-dashboard/target/opensearch-dashboards-2.4.1-linux-x64.tar.gz \
-    --security file:///home/user/wazuh-security-dashboards-plugin/build/security-dashboards-2.4.1.0.zip \
-    --version 4.6.0
-```
+- The base of Wazuh Dashboard, generated by running `yarn build --linux --skip-os-packages --release` or `yarn build --linux-arm --skip-os-packages --release` for arm packages
+- The build of each plugin in `wazuh-dashboard-plugins` repo
+- The build of the `wazuh-security-dashboards-plugin` repo
 
-### DEB
+### Building packages
 
-The `launcher.sh` script generates a `.deb` package based on the previously generated `.tar.gz`. 
-A Docker container is used to generate the package. It takes the following inputs:
-- `-v` | `--version`: the Wazuh version of the package.
-- `-p` | `--package`: the location of the `.tar.gz` file. It can be a URL or a PATH, with the format `file://<absolute_path>`
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script. 
-- `--dont-build-docker`: [Optional] Locally built Docker image will be used instead of generating a new one.
+The script can build a `.tar.gz` (former base), and `rpm` and `deb` packages. This can be for x64 and arm architectures (it is not cross-architecture building. You need to run the script in a machine of the same architecture that you are building).
 
-Example:
-```bash
-bash launcher.sh \
-    --version 4.6.0 \
-    --package file:///home/user/wazuh-dashboard/dev_tools/build_packages/base/output/wazuh-dashboard-4.6.0-1-linux-x64.tar.gz
-```
+The inputs are the following:
+
+- `-a`, `--app`: URL or path to the zip that contains the `wazuh-dashboard-plugins` plugins build.
+- `-b`, `--base`: URL or path to the zip that contains the `wazuh-dashboard build`.
+- `-s`, `--security`: URL or path to the zip that contains the `wazuh-security-dashboards-plugin` build.
+- `-v`, `--version`: Set the version of this build.
+- `-r`, `--revision`: [Optional] Set the revision of this build. By default, it is set to 1.
+- `--all-platforms`: Build all platforms.
+- `--deb`: Build deb.
+- `--rpm`: Build rpm.
+- `--tar`: Build tar.gz.
+- `--production`:[Optional] The naming of the package will be ready for production. Otherwise, it will include the hash of the current commit.
+- `--arm`: [Optional] Build for arm64 instead of x64.
+- `--debug`: [Optional] Enables debug mode, which will show detailed information during the script run.
+- `--silent`: [Optional] Enables silent mode, which will show the minimum possible information during the script run. `--debug` has priority over this.
 
-### RPM
+> [!IMPORTANT]
+> In the inputs where a local path is available, use `file://<absolute_path>` to indicate it.
 
-The `launcher.sh` script generates a `.rpm` package based on the previously generated `.tar.gz`. 
-A Docker container is used to generate the package. It takes the following inputs:
-- `-v` | `--version`: the Wazuh version of the package.
-- `-p` | `--package`: the location of the `.tar.gz` file. It can be a URL or a PATH, with the format `file://<absolute_path>`
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script. 
-- `--dont-build-docker`: [Optional] Locally built Docker image will be used instead of generating a new one.
+> [!WARNING]
+> To build `arm` packages, you need to run the script in an arm machine, and use an arm build of the wazuh-dashboard base with `-b`
 
 Example:
+
 ```bash
-bash launcher.sh \
-    --version 4.6.0 \
-    --package file:///home/user/wazuh-dashboard/dev_tools/build_packages/base/output/wazuh-dashboard-4.6.0-1-linux-x64.tar.gz
+bash build-packages.sh \
+    --app file:///home/user/packages/wazuh-package.zip \
+    --base file:///home/user/packages/dashboard-package.zip \
+    --security file:///home/user/packages/security-package.zip \
+    --version 4.10.0 --revision 2 --deb --silent
 ```
-
-[wazuh-plugin]: https://github.com/wazuh/wazuh-kibana-app
-[wazuh-security-plugin]: https://github.com/wazuh/wazuh-security-dashboards-plugin
diff --git a/dev-tools/build-packages/base/Docker/Dockerfile b/dev-tools/build-packages/base/Docker/Dockerfile
@@ -0,0 +1,35 @@
+FROM rockylinux:8.5
+
+# Install all the necessary tools to build the packages
+RUN yum clean all && yum update -y
+RUN yum install -y \
+    curl \
+    tar \
+    findutils \
+    git \
+    xz  \
+    gcc \
+    make \
+    bc \
+    sed \
+    gzip \
+    autoconf \
+    automake \
+    libtool \
+    python3-devel \
+    python3-pip \
+    jq \
+    unzip
+
+RUN git clone https://github.com/google/brotli.git -b v1.0.9
+
+RUN cd brotli && chmod +x ./bootstrap && ./bootstrap && ./configure --prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --libexecdir=/usr/lib64/brotli --libdir=/usr/lib64/brotli --datarootdir=/usr/share --mandir=/usr/share/man/man1 --docdir=/usr/share/doc \
+    && make && make install
+
+# Add the scripts to build the RPM package
+ADD base-builder.sh /usr/local/bin/builder
+ADD plugins /usr/local/bin/plugins
+RUN chmod +x /usr/local/bin/builder
+
+# Set the entrypoint
+ENTRYPOINT ["/usr/local/bin/builder"]