diff --git a/.github/playbooks/distributed.yml b/.github/playbooks/distributed.yml new file mode 100644 index 000000000..c5911580e --- /dev/null +++ b/.github/playbooks/distributed.yml @@ -0,0 +1,72 @@ +- name: Generate certificates prior to converging + hosts: aio + become: true + become_user: root + roles: + # 1. Check packages + - role: ../../roles/wazuh/check-packages + become: no + delegate_to: aio + run_once: true + # 2. Generate certificates + - role: ../../roles/wazuh/wazuh-indexer + vars: + generate_certs: true + perform_installation: false + macos_localhost: false + vars: + # wazuh_endpoint_list: "{{ groups['aio'] }}" + instances: + node1: + name: wazuh-es01 # Important: must be equal to indexer_node_name. + ip: "127.0.0.1" # When unzipping, the node will search for its node name folder to get the cert. + role: indexer + node3: + name: wazuh-mgr01 + ip: "127.0.0.1" + role: wazuh + node5: + name: wazuh-dash01 + ip: "127.0.0.1" + role: dashboard + pre_tasks: + - name: overview of cert configuration + debug: + # var: wazuh_endpoint_list + var: instances + +- name: Converge + hosts: aio + become: true + become_user: root + roles: + # 1. Wazuh indexer + - role: ../../roles/wazuh/wazuh-indexer + vars: + indexer_node_name: "wazuh-es01" + single_node: true + # 2. Managers + - role: ../../roles/wazuh/ansible-wazuh-manager + - role: ../../roles/wazuh/ansible-filebeat-oss + vars: + filebeat_node_name: "wazuh-mgr01" + filebeat_output_indexer_hosts: + - "localhost:9200" + # 3. Wazuh dashboard + - role: ../../roles/wazuh/wazuh-dashboard + vars: + dashboard_node_name: "wazuh-dash01" + vars: + instances: + node1: + name: wazuh-es01 # Important: must be equal to indexer_node_name. + ip: "127.0.0.1" # When unzipping, the node will search for its node name folder to get the cert. + role: indexer + node3: + name: wazuh-mgr01 + ip: "127.0.0.1" + role: wazuh + node5: + name: wazuh-dash01 + ip: "127.0.0.1" + role: dashboard \ No newline at end of file diff --git a/.github/workflows/aio.yml b/.github/workflows/aio.yml new file mode 100644 index 000000000..dd64f0e6c --- /dev/null +++ b/.github/workflows/aio.yml @@ -0,0 +1,234 @@ +--- +name: AIO-Single-Instance-Test +on: + workflow_dispatch: + inputs: + AUTOMATION_REFERENCE: + description: 'Branch or tag of the wazuh-automation repository' + required: true + default: 'master' + SYSTEMS: + description: 'Operating Systems (list of comma-separated quoted strings enclosed in square brackets)' + required: true + default: '["CentOS_8", "AmazonLinux_2", "Ubuntu_22", "RHEL8"]' + type: string + VERBOSITY: + description: 'Verbosity level on playbooks execution' + required: true + default: '-v' + type: choice + options: + - -v + - -vv + - -vvv + - -vvvv + pull_request: + release: +env: + COMPOSITE_NAME: "linux-SUBNAME-amd64" + ALLOCATOR_PATH: "/tmp/allocator_instance" +permissions: + id-token: write # JWT + contents: read # actions/checkout +jobs: + setup-runner: + name: Setup runner + runs-on: ubuntu-latest + steps: + - name: Display workflow inputs + run: echo "${{ toJson(inputs) }}" + - name: Compute outputs + id: compute-outputs + run: | + # Set SYSTEMS_LIST + if [ "${{ inputs.SYSTEMS }}" != "null" && "${{ inputs.SYSTEMS }}" != "" ]; then + SYSTEMS_LIST=${{ inputs.SYSTEMS }} + else + SYSTEMS_LIST='["CentOS_8", "AmazonLinux_2", "Ubuntu_22", "RHEL8"]' + fi + echo "SYSTEMS_JSON=$(echo $SYSTEMS_LIST | jq -c '.')" >> $GITHUB_OUTPUT + echo "Systems JSON: $SYSTEMS_JSON" + + # Set VERBOSITY + if [ "${{ inputs.VERBOSITY }}" != "null" && "${{ inputs.VERBOSITY }}" != "" ]; then + VERBOSITY=${{ inputs.VERBOSITY }} + else + VERBOSITY='-v' + fi + echo "VERBOSITY=$VERBOSITY" >> $GITHUB_OUTPUT + echo "Verbosity level: $VERBOSITY" + outputs: + SYSTEMS_JSON: ${{ steps.compute-outputs.outputs.SYSTEMS_JSON }} + VERBOSITY: ${{ steps.compute-outputs.outputs.VERBOSITY }} + start-runner: + name: Start EC2 instance using allocator and run ansible playbook + runs-on: ubuntu-latest + # outputs: + # label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + needs: setup-runner + strategy: + fail-fast: false # all jobs will run even if one fails + matrix: + system: ${{ fromJson(needs.setup-runner.outputs.SYSTEMS_JSON) }} + steps: + - name: Configure AWS credentials (assume role) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + role-session-name: + aws-region: us-east-1 + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.AUTOMATION_REFERENCE }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation/ + - name: Install python and create virtual environment + run: | + sudo apt-get update + sudo apt-get install -y python3 python3-venv + python3 -m pip install --upgrade pip + python3 -m venv venv + source venv/bin/activate + - name: Install and set allocator requirements + run: | + pip install -r wazuh-automation/deployability/deps/requirements.txt + pip install ansible-core==2.16 + ansible-galaxy collection install community.general + ansible-galaxy collection install community.docker + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: machulav/ec2-github-runner@v2 + # with: + # mode: start + # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + # ec2-image-id: ${{ secrets.AL2_AMI_ID }} + # ec2-instance-type: t3a.large + # subnet-id: ${{ secrets.SUBNET_ID }} + # security-group-id: ${{ secrets.SG_ID }} + # aws-resource-tags: > # optional, requires additional permissions + # [ + # {"Key": "Name", "Value": "wazuh-ansible-gh-runner-aio"}, + # {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}, + # {"Key": "team", "Value": "CICD"}, + # {"Key": "termination_date", "Value": "2022-12-31 21:00:00"} + # ] + - name: Set COMPOSITE_NAME variable + run: | + case "${{ matrix.system }}" in + "CentOS_7") + SUBNAME="centos-7" + ;; + "CentOS_8") + SUBNAME="centos-8" + ;; + "AmazonLinux_2") + SUBNAME="amazon-2" + ;; + "Ubuntu_16") + SUBNAME="ubuntu-16.04" + ;; + "Ubuntu_18") + SUBNAME="ubuntu-18.04" + ;; + "Ubuntu_20") + SUBNAME="ubuntu-20.04" + ;; + "Ubuntu_22") + SUBNAME="ubuntu-22.04" + ;; + "RHEL7") + SUBNAME="redhat-7" + ;; + "RHEL8") + SUBNAME="redhat-8" + ;; + *) + echo "Invalid SYSTEM selection" >&2 + exit 1 + ;; + esac + COMPOSITE_NAME="${COMPOSITE_NAME/SUBNAME/$SUBNAME}" + echo "COMPOSITE_NAME=$COMPOSITE_NAME" >> $GITHUB_ENV + - name: Allocate EC2 instance + id: allocator_instance + run: | + python3 wazuh-automation/deployability/modules/allocation/main.py --action create --provider aws --size large --composite-name ${{ env.COMPOSITE_NAME }} --working-dir $ALLOCATOR_PATH \ + --track-output $ALLOCATOR_PATH/track.yml --inventory-output $ALLOCATOR_PATH/inventory.yml --instance-name gha_ansible_${{ matrix.system }}_${{ github.run_id }} \ + --label-team devops --label-termination-date 1d + sed 's/: */=/g' $ALLOCATOR_PATH/inventory.yml > $ALLOCATOR_PATH/inventory_mod.yml + sed -i 's/-o StrictHostKeyChecking=no/\"-o StrictHostKeyChecking=no\"/g' $ALLOCATOR_PATH/inventory_mod.yml + source $ALLOCATOR_PATH/inventory_mod.yml + echo "[aio]" > $ALLOCATOR_PATH/inventory + echo "$ansible_host ansible_port=$ansible_port ansible_user=$ansible_user ansible_ssh_private_key_file=$ansible_ssh_private_key_file ansible_ssh_common_args='$ansible_ssh_common_args'" >> $ALLOCATOR_PATH/inventory + - name: Save ALLOCATOR_PATH directory as artifact + uses: actions/upload-artifact@v4 + with: + name: allocator-instance-${{ matrix.system }}-${{ github.run_id }} + path: ${{ env.ALLOCATOR_PATH }} + - name: Check out the codebase. + uses: actions/checkout@v4 + - name: Ansible Playbook run Wazuh AIO Single instance + run: | + ansible-playbook ./.github/playbooks/aio-wazuh.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l all \ + ${{ needs.setup-runner.outputs.VERBOSITY }} + stop-runner: + name: Stop allocated EC2 instance + needs: + # - start-runner # required to get output from the start-runner job + # - install-aio-single-instance # required to wait when the main job is done + - setup-runner + - start-runner + runs-on: ubuntu-latest + strategy: + fail-fast: false # all jobs will run even if one fails + matrix: + system: ${{ fromJson(needs.setup-runner.outputs.SYSTEMS_JSON) }} + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ secrets.AWS_REGION }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + aws-region: us-east-1 + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.AUTOMATION_REFERENCE }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation/ + - name: Install python and create virtual environment + run: | + sudo apt-get update + sudo apt-get install -y python3 python3-venv + python3 -m pip install --upgrade pip + python3 -m venv venv + source venv/bin/activate + - name: Install and set allocator requirements + run: | + pip install -r wazuh-automation/deployability/deps/requirements.txt + - name: Download the allocator artifact + uses: actions/download-artifact@v4 + with: + name: allocator-instance-${{ matrix.system }}-${{ github.run_id }} + path: ${{ env.ALLOCATOR_PATH }} + # - name: Stop EC2 runner + # uses: machulav/ec2-github-runner@v2 + # with: + # mode: stop + # github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + # label: ${{ needs.start-runner.outputs.label }} + # ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }} + - name: Delete allocated EC2 instance + run: python3 wazuh-automation/deployability/modules/allocation/main.py --action delete --track-output $ALLOCATOR_PATH/track.yml diff --git a/.github/workflows/distributed.yml b/.github/workflows/distributed.yml new file mode 100644 index 000000000..62a35cc28 --- /dev/null +++ b/.github/workflows/distributed.yml @@ -0,0 +1,267 @@ +--- +name: Distributed-Wazuh-Test +on: + workflow_dispatch: + inputs: + AUTOMATION_REFERENCE: + description: 'Branch or tag of the wazuh-automation repository' + required: true + default: 'master' + SYSTEMS: + description: 'Operating Systems (list of comma-separated quoted strings enclosed in square brackets)' + required: true + default: '["Ubuntu_22"]' + type: string + VERBOSITY: + description: 'Verbosity level on playbooks execution' + required: true + default: '-v' + type: choice + options: + - -v + - -vv + - -vvv + - -vvvv + pull_request: + release: +env: + COMPOSITE_NAME: "linux-SUBNAME-amd64" + ALLOCATOR_PATH: "/tmp/allocator_instance" +permissions: + id-token: write # JWT + contents: read # actions/checkout +jobs: + setup-runner: + name: Setup runner + runs-on: ubuntu-latest + steps: + - name: Display workflow inputs + run: echo "${{ toJson(inputs) }}" + - name: Compute outputs + id: compute-outputs + run: | + # Set SYSTEMS_LIST + if [ "${{ inputs.SYSTEMS }}" != "null" && "${{ inputs.SYSTEMS }}" != "" ]; then + SYSTEMS_LIST=${{ inputs.SYSTEMS }} + else + SYSTEMS_LIST='["CentOS_8", "AmazonLinux_2", "Ubuntu_22", "RHEL8"]' + fi + echo "SYSTEMS_JSON=$(echo $SYSTEMS_LIST | jq -c '.')" >> $GITHUB_OUTPUT + echo "Systems JSON: $SYSTEMS_JSON" + + # Set VERBOSITY + if [ "${{ inputs.VERBOSITY }}" != "null" && "${{ inputs.VERBOSITY }}" != "" ]; then + VERBOSITY=${{ inputs.VERBOSITY }} + else + VERBOSITY='-v' + fi + echo "VERBOSITY=$VERBOSITY" >> $GITHUB_OUTPUT + echo "Verbosity level: $VERBOSITY" + outputs: + SYSTEMS_JSON: ${{ steps.compute-outputs.outputs.SYSTEMS_JSON }} + VERBOSITY: ${{ steps.compute-outputs.outputs.VERBOSITY }} + start-runner: + name: Start EC2 instance using allocator and run ansible playbook + runs-on: ubuntu-latest + needs: setup-runner + strategy: + fail-fast: false # all jobs will run even if one fails + matrix: + system: ${{ fromJson(needs.setup-runner.outputs.SYSTEMS_JSON) }} + steps: + - name: Configure AWS credentials (assume role) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + role-session-name: + aws-region: us-east-1 + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.AUTOMATION_REFERENCE }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation/ + - name: Install python and create virtual environment + run: | + sudo apt-get update + sudo apt-get install -y python3 python3-venv + python3 -m pip install --upgrade pip + python3 -m venv venv + source venv/bin/activate + - name: Install and set allocator requirements + run: | + pip install -r wazuh-automation/deployability/deps/requirements.txt + pip install ansible-core==2.16 + ansible-galaxy collection install community.general + ansible-galaxy collection install community.docker + - name: Set COMPOSITE_NAME variable + run: | + case "${{ matrix.system }}" in + "CentOS_7") + SUBNAME="centos-7" + ;; + "CentOS_8") + SUBNAME="centos-8" + ;; + "AmazonLinux_2") + SUBNAME="amazon-2" + ;; + "Ubuntu_16") + SUBNAME="ubuntu-16.04" + ;; + "Ubuntu_18") + SUBNAME="ubuntu-18.04" + ;; + "Ubuntu_20") + SUBNAME="ubuntu-20.04" + ;; + "Ubuntu_22") + SUBNAME="ubuntu-22.04" + ;; + "RHEL7") + SUBNAME="redhat-7" + ;; + "RHEL8") + SUBNAME="redhat-8" + ;; + *) + echo "Invalid SYSTEM selection" >&2 + exit 1 + ;; + esac + COMPOSITE_NAME="${COMPOSITE_NAME/SUBNAME/$SUBNAME}" + echo "COMPOSITE_NAME=$COMPOSITE_NAME" >> $GITHUB_ENV + - name: Allocate cluster of EC2 instances + id: allocator_instance + run: | + instance_names=("wi1" "wi2" "wi3", "dashboard", "manager", "worker") + inventory_file="$ALLOCATOR_PATH/inventory_all" + + mkdir -p $ALLOCATOR_PATH + + for i in ${!instance_names[@]}; do + instance_name=${instance_names[$i]} + # Provision instance in parallel + ( + python3 wazuh-automation/deployability/modules/allocation/main.py \ + --action create --provider aws --size large \ + --composite-name ${{ env.COMPOSITE_NAME }} \ + --working-dir $ALLOCATOR_PATH --track-output $ALLOCATOR_PATH/track_${instance_name}.yml \ + --inventory-output $ALLOCATOR_PATH/inventory_${instance_name}.yml \ + --instance-name gha_ansible_${{ matrix.system }}_${instance_name}_${{ github.run_id }} --label-team devops --label-termination-date 1d + + instance_id=$(grep '^identifier' $ALLOCATOR_PATH/track_${instance_name}.yml | awk '{print $2}') + private_ip=$(aws ec2 describe-instances \ + --instance-ids $instance_id \ + --query 'Reservations[*].Instances[*].PrivateIpAddress' \ + --output text) + + sed 's/: */=/g' $ALLOCATOR_PATH/inventory_${instance_name}.yml > $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + sed -i 's/-o StrictHostKeyChecking=no/\"-o StrictHostKeyChecking=no\"/g' $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + source $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + + # Add instance to corresponding group + if [[ $i -eq 0 ]]; then + echo "wi1 ansible_host=$ansible_host private_ip=$private_ip indexer_node_name=node-1" >> $inventory_file + elif [[ $i -eq 1 ]]; then + echo "wi2 ansible_host=$ansible_host private_ip=$private_ip indexer_node_name=node-2" >> $inventory_file + elif [[ $i -eq 2 ]]; then + echo "wi3 ansible_host=$ansible_host private_ip=$private_ip indexer_node_name=node-3" >> $inventory_file + elif [[ $i -eq 3 ]]; then + echo "dashboard ansible_host=$ansible_host private_ip=$private_ip" >> $inventory_file + elif [[ $i -eq 4 ]]; then + echo "manager ansible_host=$ansible_host private_ip=$private_ip" >> $inventory_file + elif [[ $i -eq 5 ]]; then + echo "worker ansible_host=$ansible_host private_ip=$private_ip" >> $inventory_file + fi + + if [[ $i -eq ${#instance_names[@]}-1 ]]; then + echo "[wi_cluster]" >> $inventory_file + echo "wi1" >> $inventory_file + echo "wi2" >> $inventory_file + echo "wi3" >> $inventory_file + echo "[all:vars]" >> $inventory_file + echo "ansible_ssh_user=$ansible_user" >> $inventory_file + echo "ansible_port=$ansible_port" >> $inventory_file + echo "ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_file + echo "ansible_ssh_common_args='$ansible_ssh_common_args'" >> $inventory_file + echo "ansible_ssh_extra_args='-o StrictHostKeyChecking=no'" >> $inventory_file + fi + ) & + done + + # Wait for all provisioning tasks to complete + wait + - name: Save ALLOCATOR_PATH directory as artifact + uses: actions/upload-artifact@v4 + with: + name: allocator-instance-${{ matrix.system }}-${{ github.run_id }} + path: ${{ env.ALLOCATOR_PATH }} + - name: Check out the codebase. + uses: actions/checkout@v4 + - name: Ansible Playbook run Wazuh Distributed Production Ready + run: | + # ansible-playbook ./.github/playbooks/aio-wazuh.yml \ + ansible-playbook playbooks/wazuh-production-ready.yml -b -K \ + -i $ALLOCATOR_PATH/inventory_all \ + -l all \ + ${{ needs.setup-runner.outputs.VERBOSITY }} + stop-runner: + name: Stop allocated EC2 instance + needs: + # - start-runner # required to get output from the start-runner job + # - install-aio-single-instance # required to wait when the main job is done + - setup-runner + - start-runner + runs-on: ubuntu-latest + strategy: + fail-fast: false # all jobs will run even if one fails + matrix: + system: ${{ fromJson(needs.setup-runner.outputs.SYSTEMS_JSON) }} + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + aws-region: us-east-1 + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.AUTOMATION_REFERENCE }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation/ + - name: Install python and create virtual environment + run: | + sudo apt-get update + sudo apt-get install -y python3 python3-venv + python3 -m pip install --upgrade pip + python3 -m venv venv + source venv/bin/activate + - name: Install and set allocator requirements + run: | + pip install -r wazuh-automation/deployability/deps/requirements.txt + - name: Download the allocator artifact + uses: actions/download-artifact@v4 + with: + name: allocator-instance-${{ matrix.system }}-${{ github.run_id }} + path: ${{ env.ALLOCATOR_PATH }} + - name: Delete allocated EC2 instance + run: | + # python3 wazuh-automation/deployability/modules/allocation/main.py \ + # --action delete + # --track-output $ALLOCATOR_PATH/track_${instance_name}.yml + for i in ${!instance_names[@]}; do + instance_name=${instance_names[$i]} + # Delete instance in parallel + ( + python3 wazuh-automation/deployability/modules/allocation/main.py \ + --action delete --provider aws --size large \ + --composite-name ${{ env.COMPOSITE_NAME }} \ + --working-dir $ALLOCATOR_PATH --track-output $ALLOCATOR_PATH/track_${instance_name}.yml \ + --inventory-output $ALLOCATOR_PATH/inventory_${instance_name}.yml \ + --instance-name gha_ansible_${{ matrix.system }}_${instance_name} _${{ github.run_id }} --label-team devops --label-termination-date 1d + ) & + done