Merge 4.10.2 into master (#398)

Co-authored-by: Ian Yenien Serrano <[email protected]>

.github/workflows/build_base.yml

@@ -10,25 +10,34 @@ on:
         required: true
         default: ''
         type: string
+      ARCHITECTURE:
+        description: 'The architecture to build the package for'
+        required: true
+        default: 'amd64'
+        type: string
 
   workflow_dispatch:
     inputs:
       CHECKOUT_TO: # This is the branch to checkout to. Defaults to 'master'
         description: 'The branch/tag/commit to checkout to'
         required: true
         default: ''
+      ARCHITECTURE:
+        description: 'The architecture to build the package for'
+        required: true
+        default: 'amd64'
+        type: string
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'ubuntu-latest' || 'wz-linux-arm64' }}
     name: Build
     defaults:
       run:
         working-directory: ./artifacts
     strategy:
       matrix:
         DISTRIBUTION: [tar.gz]
-        ARCHITECTURE: [x64]
 
     steps:
       - name: Checkout code
@@ -69,21 +78,16 @@ jobs:
 
       - name: Get artifact build name
         run: |
-          echo "ARTIFACT_BUILD_NAME=wazuh-dashboard_${{ env.WZD_VERSION }}-${{ env.WZD_REVISION }}_${{ matrix.ARCHITECTURE }}.${{ matrix.DISTRIBUTION }}" >> $GITHUB_ENV
+          echo "ARTIFACT_BUILD_NAME=wazuh-dashboard_${{ env.WZD_VERSION }}-${{ env.WZD_REVISION }}_${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'x64' || 'arm64' }}.${{ matrix.DISTRIBUTION }}" >> $GITHUB_ENV
 
       - name: Run bootstrap
         run: yarn osd bootstrap
 
-      - name: Build linux-x64
-        if: matrix.ARCHITECTURE == 'x64'
-        run: yarn build-platform --linux --skip-os-packages --release
-
-      - name: Build linux-arm64
-        if: matrix.ARCHITECTURE == 'arm64'
-        run: yarn build-platform --linux-arm --skip-os-packages --release
+      - name: Build
+        run: yarn build-platform --${{(inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'linux' || 'linux-arm'}} --skip-os-packages --release
 
       - name: Rename artifact
-        run: mv /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/opensearch-dashboards-${{ env.VERSION }}-linux-${{ matrix.ARCHITECTURE }}.${{ matrix.DISTRIBUTION }} /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
+        run: mv /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/opensearch-dashboards-${{ env.VERSION }}-linux-${{ (inputs.ARCHITECTURE == 'x86_64' || inputs.ARCHITECTURE == 'amd64') && 'x64' || 'arm64' }}.${{ matrix.DISTRIBUTION }} /home/runner/work/wazuh-dashboard/wazuh-dashboard/artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
 
       - uses: actions/upload-artifact@v3
         if: success()

.gitignore

@@ -1,7 +1,8 @@
 .aws-config.json
 .signing-config.json
 .ackrc
-/dev-tools/build-packages/*/output
+/dev-tools/build-packages/output
+/dev-tools/build-packages/tmp
 /.opensearch
 /.chromium
 /package.json.bak
@@ -71,4 +72,7 @@ snapshots.js
 .yarn-local-mirror
 
 # Ignore the generated antlr files
-/src/plugins/data/public/antlr/**/grammar/.antlr/
+/src/plugins/data/public/antlr/opensearch_sql/grammar/.antlr
+
+# Ignore the generated packages
+/dev-tools/build-packages/base-packages-to-base/packages

dev-tools/build-packages/README.md

@@ -1,70 +1,59 @@
 # Package building
-This folder contains tools used to create `rpm` and `deb` packages. 
+
+This folder contains tools used to create `tar`, `rpm` and `deb` packages.
 
 ## Requirements
- - A system with Docker.
- - Internet connection (to download the docker images the first time).
 
-## Builders
+- A system with Docker.
+- Internet connection (to download the docker images the first time).
 
-### Tarball
+## How to build packages
 
-To system packages (deb and rpm), a tarball of Wazuh dashboard `.tar.gz` is required.
-This tarball contains the [Wazuh plugin][wazuh-plugin], the [Wazuh Security plugin][wazuh-security-plugin], 
-a set of OpenSearch plugins and the default configuration for the app. 
+The script `build-packages.sh` is in charge of coordinating the different steps to build each type of packages.
 
-The `generate_base.sh` script generates a `.tar.gz` file using the following inputs:
-- `-a` | `--app`: URL to the zipped Wazuh plugin.*
-- `-b` | `--base`: URL to the Wazuh dashboard `.tar.gz`, as generated with `yarn build --skip-os-packages --release`.*
-- `-s` | `--security`: URL to the zipped Wazuh Security plugin, as generated with `yarn build`.*
-- `-v` | `--version`: the Wazuh version of the package.
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script.
+### Pre-requisites
 
-*Note:* use `file://<absolute_path>` to indicate a local file. Otherwise, the script will try to download the file from the given URL.
+The script needs 4 different zip files, containing the following respectively:
 
-Example:
-```bash
-bash generate_base.sh \
-    --app https://packages-dev.wazuh.com/pre-release/ui/dashboard/wazuh-4.6.0-1.zip \
-    --base file:///home/user/wazuh-dashboard/target/opensearch-dashboards-2.4.1-linux-x64.tar.gz \
-    --security file:///home/user/wazuh-security-dashboards-plugin/build/security-dashboards-2.4.1.0.zip \
-    --version 4.6.0
-```
+- The base of Wazuh Dashboard, generated by running `yarn build --linux --skip-os-packages --release` or `yarn build --linux-arm --skip-os-packages --release` for arm packages
+- The build of each plugin in `wazuh-dashboard-plugins` repo
+- The build of the `wazuh-security-dashboards-plugin` repo
+- The build of the `wazuh-dashboards-reporting ` repo
 
-### DEB
+### Building packages
 
-The `launcher.sh` script generates a `.deb` package based on the previously generated `.tar.gz`. 
-A Docker container is used to generate the package. It takes the following inputs:
-- `-v` | `--version`: the Wazuh version of the package.
-- `-p` | `--package`: the location of the `.tar.gz` file. It can be a URL or a PATH, with the format `file://<absolute_path>`
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script. 
-- `--dont-build-docker`: [Optional] Locally built Docker image will be used instead of generating a new one.
+The script can build a `.tar.gz` (former base), and `rpm` and `deb` packages. This can be for x64 and arm architectures (it is not cross-architecture building. You need to run the script in a machine of the same architecture that you are building).
 
-Example:
-```bash
-bash launcher.sh \
-    --version 4.6.0 \
-    --package file:///home/user/wazuh-dashboard/dev_tools/build_packages/base/output/wazuh-dashboard-4.6.0-1-linux-x64.tar.gz
-```
+The inputs are the following:
+
+- `-a`, `--app`: URL or path to the zip that contains the `wazuh-dashboard-plugins` plugins build.
+- `-b`, `--base`: URL or path to the zip that contains the `wazuh-dashboard build`.
+- `-s`, `--security`: URL or path to the zip that contains the `wazuh-security-dashboards-plugin` build.
+- `-rp`, `--reportPlugin`: URL or path to the zip that contains the `wazuh-dashboards-reporting` build.
+- `-v`, `--version`: Set the version of this build.
+- `-r`, `--revision`: [Optional] Set the revision of this build. By default, it is set to 1.
+- `--all-platforms`: Build all platforms.
+- `--deb`: Build deb.
+- `--rpm`: Build rpm.
+- `--tar`: Build tar.gz.
+- `--production`:[Optional] The naming of the package will be ready for production. Otherwise, it will include the hash of the current commit.
+- `--arm`: [Optional] Build for arm64 instead of x64.
+- `--debug`: [Optional] Enables debug mode, which will show detailed information during the script run.
+- `--silent`: [Optional] Enables silent mode, which will show the minimum possible information during the script run. `--debug` has priority over this.
 
-### RPM
+> [!IMPORTANT]
+> In the inputs where a local path is available, use `file://<absolute_path>` to indicate it.
 
-The `launcher.sh` script generates a `.rpm` package based on the previously generated `.tar.gz`. 
-A Docker container is used to generate the package. It takes the following inputs:
-- `-v` | `--version`: the Wazuh version of the package.
-- `-p` | `--package`: the location of the `.tar.gz` file. It can be a URL or a PATH, with the format `file://<absolute_path>`
-- `-r` | `--revision`: [Optional] Set the revision of the build. By default, it is set to 1.
-- `-o` | `--output` [Optional] Set the destination path of package. By default, an output folder will be created in the same directory as the script. 
-- `--dont-build-docker`: [Optional] Locally built Docker image will be used instead of generating a new one.
+> [!WARNING]
+> To build `arm` packages, you need to run the script in an arm machine, and use an arm build of the wazuh-dashboard base with `-b`
 
 Example:
+
 ```bash
-bash launcher.sh \
-    --version 4.6.0 \
-    --package file:///home/user/wazuh-dashboard/dev_tools/build_packages/base/output/wazuh-dashboard-4.6.0-1-linux-x64.tar.gz
+bash build-packages.sh \
+    --app file:///home/user/packages/wazuh-package.zip \
+    --base file:///home/user/packages/dashboard-package.zip \
+    --security file:///home/user/packages/security-package.zip \
+    --reportPlugin file:///home/user/packages/report-package.zip
+    --version 5.0.0 --revision 2 --deb --silent
 ```
-
-[wazuh-plugin]: https://github.com/wazuh/wazuh-kibana-app
-[wazuh-security-plugin]: https://github.com/wazuh/wazuh-security-dashboards-plugin

dev-tools/build-packages/base-packages-to-base/README.md

@@ -0,0 +1,39 @@
+# Package building
+
+This folder contains the tools used to create packages from the different repositories necessary for the creation of deb and rpm packages.
+
+## Requirements
+
+- A system with Docker.
+- Internet connection (to download the docker images the first time).
+
+## How to build packages
+
+The script `run-docker-compose.sh` is in charge of coordinating the different steps to build each package.
+
+### Building packages
+
+The script can build a `.tar.gz` (former base), and `rpm` and `deb` packages. This can be for x64 and arm architectures (it is not cross-architecture building. You need to run the script in a machine of the same architecture that you are building).
+
+The inputs are the following:
+
+- `-a`, `--app`: Set the `wazuh-dashboard-plugins` branch.
+- `-b`, `--base`: Set the `wazuh-dashboard` branch.
+- `-s`, `--security`: Set the `wazuh-security-dashboards-plugin` branch.
+- `-r`, `--reporting`: Set the `wazuh-dashboards-reporting` branch.
+- `--node-version`: [Optional] Set the node version.
+- `--arm`: [Optional] Build for arm64 instead of x64.
+
+Example:
+
+```bash
+bash run-docker-compose.sh \
+    --app master \
+    --base master \
+    --security master \
+    --reporting master \
+    --arm \
+    --node-version 18.19.0
+```
+
+This example will create a packages folder that inside will have the packages divided by repository of the master branch of each one.

dev-tools/build-packages/base-packages-to-base/base-packages.Dockerfile

@@ -0,0 +1,33 @@
+# Usage:
+# docker build \
+#         --build-arg NODE_VERSION=18.19.0 \
+#         --build-arg WAZUH_DASHBOARD_BRANCH=4.10.2 \
+#         --build-arg WAZUH_DASHBOARD_SECURITY_BRANCH=4.10.2 \
+#         --build-arg WAZUH_DASHBOARD_PLUGINS_BRANCH=4.10.2 \
+#         --build-arg ARCHITECTURE=arm \
+#         -t wazuh-packages-to-base:4.10.2 \
+#         -f base-packages.Dockerfile .
+
+ARG NODE_VERSION=18.19.0
+FROM node:${NODE_VERSION} AS base
+ARG ARCHITECTURE='amd'
+ARG WAZUH_DASHBOARD_BRANCH
+ARG WAZUH_DASHBOARD_SECURITY_BRANCH
+ARG WAZUH_DASHBOARD_PLUGINS_BRANCH
+ARG WAZUH_DASHBOARD_REPORTING_BRANCH
+ENV OPENSEARCH_DASHBOARDS_VERSION=2.16.0
+ENV ENV_ARCHITECTURE=${ARCHITECTURE}
+USER root
+RUN apt-get update && apt-get install -y jq
+USER node
+ADD ./clone-plugins.sh /home/node/clone-plugins.sh
+ADD ./repositories/wazuh-dashboard.sh /home/node/repositories/wazuh-dashboard.sh
+ADD ./repositories/plugins/wazuh-security-dashboards-plugin.sh /home/node/repositories/plugins/wazuh-security-dashboards-plugin.sh
+ADD ./repositories/plugins/wazuh-dashboard-reporting.sh /home/node/repositories/plugins/wazuh-dashboard-reporting.sh
+ADD ./repositories/plugins/wazuh-dashboard-plugins.sh /home/node/repositories/plugins/wazuh-dashboard-plugins.sh
+RUN bash /home/node/clone-plugins.sh
+
+FROM node:${NODE_VERSION}
+USER node
+COPY --chown=node:node --from=base /home/node/packages /home/node/packages
+WORKDIR /home/node/packages

dev-tools/build-packages/base-packages-to-base/clone-plugins.sh

@@ -0,0 +1,16 @@
+base_path_plugins="/home/node/app/plugins"
+base_path_repositories_scripts="/home/node/repositories"
+base_path_repositories_plugins_scripts="/home/node/repositories/plugins"
+plugins=$(ls $base_path_repositories_plugins_scripts)
+
+mkdir /home/node/packages
+echo "Cloning Wazuh dashboards"
+source $base_path_repositories_scripts/wazuh-dashboard.sh
+
+for plugin in $plugins; do
+  echo "Cloning $plugin"
+  source $base_path_repositories_plugins_scripts/$plugin
+done
+
+
+

dev-tools/build-packages/base-packages-to-base/docker-compose.yml

@@ -0,0 +1,23 @@
+
+services:
+  base:
+    image: base-${WAZUH_DASHBOARD_BRANCH}-${WAZUH_SECURITY_PLUGIN_BRANCH}-${WAZUH_DASHBOARD_PLUGINS_BRANCH}-${ARCHITECTURE}
+    build:
+      context: .
+      dockerfile: base-packages.Dockerfile
+      args:
+        NODE_VERSION: ${NODE_VERSION}
+        WAZUH_DASHBOARD_BRANCH: ${WAZUH_DASHBOARD_BRANCH}
+        WAZUH_DASHBOARD_SECURITY_BRANCH: ${WAZUH_SECURITY_PLUGIN_BRANCH}
+        WAZUH_DASHBOARD_PLUGINS_BRANCH: ${WAZUH_DASHBOARD_PLUGINS_BRANCH}
+        WAZUH_DASHBOARD_REPORTING_BRANCH: ${WAZUH_DASHBOARD_REPORTING_BRANCH}
+        ARCHITECTURE: ${ARCHITECTURE}
+    container_name: base-${WAZUH_DASHBOARD_BRANCH}-${WAZUH_SECURITY_PLUGIN_BRANCH}-${WAZUH_DASHBOARD_PLUGINS_BRANCH}-${WAZUH_DASHBOARD_REPORTING_BRANCH}-${ARCHITECTURE}
+    environment:
+      OPENSEARCH_DASHBOARDS_VERSION: 2.16.0
+    volumes:
+      - ./:/home/node/output
+    command: >
+      bash -c '
+        cp -r /home/node/packages /home/node/output
+      '

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-dashboard-plugins.sh

@@ -0,0 +1,17 @@
+base_path_plugins="/home/node/app/plugins"
+cd $base_path_plugins
+git clone --depth 1 --branch ${WAZUH_DASHBOARD_PLUGINS_BRANCH} https://github.com/wazuh/wazuh-dashboard-plugins.git
+wazuh_dashboard_plugins=$(ls $base_path_plugins/wazuh-dashboard-plugins/plugins)
+mv wazuh-dashboard-plugins/plugins/* ./
+mkdir /home/node/packages/wazuh-dashboard-plugins
+for wazuh_dashboard_plugin in $wazuh_dashboard_plugins; do
+  cd $base_path_plugins/$wazuh_dashboard_plugin
+  yarn install
+  echo "Building $wazuh_dashboard_plugin"
+  yarn build
+  echo "Copying $wazuh_dashboard_plugin"
+  package_name=$(jq -r '.id' ./opensearch_dashboards.json)
+  cp $base_path_plugins/$wazuh_dashboard_plugin/build/$package_name-$OPENSEARCH_DASHBOARDS_VERSION.zip /home/node/packages/wazuh-dashboard-plugins/$package_name-$OPENSEARCH_DASHBOARDS_VERSION.zip
+done
+cd $base_path_plugins
+rm -rf wazuh-dashboard-plugins

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-dashboard-reporting.sh

@@ -0,0 +1,10 @@
+# Clone the Wazuh security plugin
+cd /home/node/app/plugins
+git clone --depth 1 --branch ${WAZUH_DASHBOARD_REPORTING_BRANCH} https://github.com/wazuh/wazuh-dashboards-reporting.git
+cd wazuh-dashboards-reporting
+yarn install
+echo "Building Wazuh reporting plugin"
+yarn build
+echo "Copying Wazuh reporting plugin"
+mkdir /home/node/packages/wazuh-dashboards-reporting
+cp -r build/* /home/node/packages/wazuh-dashboards-reporting

dev-tools/build-packages/base-packages-to-base/repositories/plugins/wazuh-security-dashboards-plugin.sh

@@ -0,0 +1,10 @@
+# Clone the Wazuh security plugin
+cd /home/node/app/plugins
+git clone --depth 1 --branch ${WAZUH_DASHBOARD_SECURITY_BRANCH} https://github.com/wazuh/wazuh-security-dashboards-plugin.git
+cd wazuh-security-dashboards-plugin
+yarn install
+echo "Building Wazuh security plugin"
+yarn build
+echo "Copying Wazuh security plugin"
+mkdir /home/node/packages/wazuh-security-dashboards-plugin
+cp -r build/* /home/node/packages/wazuh-security-dashboards-plugin

dev-tools/build-packages/base-packages-to-base/repositories/wazuh-dashboard.sh

@@ -0,0 +1,13 @@
+git clone --depth 1 --branch ${WAZUH_DASHBOARD_BRANCH} https://github.com/wazuh/wazuh-dashboard.git /home/node/app
+cd /home/node/app
+yarn osd bootstrap --production
+echo "Building Wazuh dashboards"
+if [ $ENV_ARCHITECTURE == "arm" ]; then
+  yarn build-platform --linux-arm --skip-os-packages --release
+else
+  yarn build-platform --linux --skip-os-packages --release
+fi
+mkdir /home/node/packages/wazuh-dashboard
+echo "Copying Wazuh dashboards"
+ls -la /home/node/app/target
+cp -r /home/node/app/target/*.tar.gz /home/node/packages/wazuh-dashboard