Skip to content

Commit

Permalink
Merge branch 'master' into devo-enh-3
Browse files Browse the repository at this point in the history
  • Loading branch information
manas-metron authored Apr 3, 2024
2 parents f950882 + bb827e8 commit 0facaaa
Show file tree
Hide file tree
Showing 3,066 changed files with 114,677 additions and 129,950 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ references:
neo4j_conf_file="/etc/neo4j/neo4j.conf"
sudo echo "dbms.security.procedures.unrestricted=apoc.*" >> $neo4j_conf_file
sudo echo "dbms.security.procedures.allowlist=apoc.*" >> $neo4j_conf_file
sudo echo "dbms.memory.transaction.total.max=600m" >> $neo4j_conf_file
sudo echo "dbms.memory.transaction.total.max=2000m" >> $neo4j_conf_file
apoc_conf_file="/etc/neo4j/apoc.conf"
sudo echo "apoc.export.file.enabled=true" > $apoc_conf_file
Expand Down
13 changes: 8 additions & 5 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
FROM python:3.10-slim-bullseye
FROM python:3.10-slim-bookworm

ENV USERNAME demisto
ENV HOME /home/$USERNAME
ENV NODE_EXTRA_CA_CERTS /usr/local/share/ca-certificates/certs.crt
ENV PATH $PATH:$HOME/.local/bin:/root/.local/bin:/usr/local/share/nvm/current/bin
ENV FEATURES_COMMIT_HASH fc62e9abf47c5ea52e02de997c91c5d52a5edc3a


ADD createCerts.sh .
RUN apt-get update && apt-get install dos2unix git python2 curl -y \
RUN apt-get update && apt-get install dos2unix git curl -y \
&& dos2unix /createCerts.sh \
&& chmod +x /createCerts.sh \
&& /createCerts.sh $NODE_EXTRA_CA_CERTS \
Expand All @@ -15,13 +17,14 @@ RUN apt-get update && apt-get install dos2unix git python2 curl -y \
&& git clone https://github.com/devcontainers/features.git /features \
&& cd /features \
# locking to the latest master commit in this repo (https://github.com/devcontainers/features.git) to prevent breaking changes
# We should update this commit hash from time to time to
&& git checkout 96bff0097028001e6e4126c5528d37cb8c13e785
# We should update this commit hash from time to time to time
&& git checkout $FEATURES_COMMIT_HASH

# This is a workaround for VSCode devcontainer features in self signed certificate
RUN UID="1000" GID="1000" bash /features/src/common-utils/install.sh
RUN VERSION="os-provided" bash /features/src/git/install.sh
RUN VERSION="latest" bash /features/src/docker-in-docker/install.sh
# Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; 0.12.0 is the last known good version)
RUN VERSION="lts" MOBYBUILDXVERSION="0.12.0" bash /features/src/docker-in-docker/install.sh
RUN VERSION="lts" bash /features/src/node/install.sh
RUN bash /features/src/sshd/install.sh
RUN bash /features/src/github-cli/install.sh
Expand Down
25 changes: 13 additions & 12 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -82,18 +82,19 @@
.gitlab/ci/* @yucohen
.gitlab/* @yucohen
.gitlab-ci.yml @yucohen
/Tests/scripts/wait_in_line_for_cloud_env.sh @daryakoval
/Tests/scripts/wait_in_line_for_cloud_env.sh @yucohen
.gitlab/ci/.gitlab-ci.staging.yml @ilaner
/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @daryakoval
/Tests/Marketplace/search_and_uninstall_pack.py @daryakoval
/Tests/scripts/install_content_and_test_integrations.sh @daryakoval
/Tests/configure_and_test_integration_instances.py @daryakoval
/Tests/scripts/print_cloud_machine_details.sh @daryakoval
/Tests/scripts/run_tests.sh @daryakoval
/Tests/scripts/download_demisto_conf.sh @daryakoval
Tests/scripts/test_modeling_rules.sh @daryakoval
Tests/scripts/lock_cloud_machines.py @daryakoval
/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @yucohen
/Tests/Marketplace/search_and_uninstall_pack.py @yucohen
/Tests/scripts/install_content_and_test_integrations.sh @yucohen
/Tests/configure_and_test_integration_instances.py @yucohen
/Tests/scripts/print_cloud_machine_details.sh @yucohen
/Tests/scripts/run_tests.sh @yucohen
/Tests/scripts/download_demisto_conf.sh @yucohen
Tests/scripts/test_modeling_rules.sh @AradCarmi
Tests/scripts/lock_cloud_machines.py @yucohen
Tests/Marketplace/server_content_items.json @dantavori
validation_config.toml @YuvHayun @JudahSchwartz @GuyAfik @anara123

# SDK Related
.gitlab/ci/.gitlab-ci.sdk-nightly.yml @dorschw
Expand All @@ -113,7 +114,7 @@ poetry.lock @ilaner @dorschw
.devcontainer/* @ilaner

# Demisto Class
Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @daryakoval
Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @dantavori

# TIM Related
/Packs/TAXIIServer/Integrations/* @MLainer1
Expand All @@ -139,4 +140,4 @@ Packs/ApiModules/Scripts/DemistoClassApiModule/DemistoClassApiModule.py @daryako
/Packs/Base/Scripts/DBotFindSimilarIncidentsByIndicators/ @jlevypaloalto
/Packs/Base/Scripts/DBotSuggestClassifierMapping/ @jlevypaloalto
/Packs/Base/Scripts/GetMLModelEvaluation/ @jlevypaloalto
/Packs/Base/Scripts/DBotMLFetchData/ @jlevypaloalto
/Packs/Base/Scripts/DBotMLFetchData/ @jlevypaloalto
12 changes: 6 additions & 6 deletions .github/content_roles.json
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{
"CONTRIBUTION_REVIEWERS": [
"YuvHayun",
"yucohen",
"shmuel44"
"anas-yousef",
"mmhw",
"maimorag"
],
"CONTRIBUTION_TL": "AradCarmi",
"CONTRIBUTION_TL": "JasBeilin",
"CONTRIBUTION_SECURITY_REVIEWER": "melamedbn",
"ON_CALL_DEVS": [
"dfried",
"meichler"
"adaud",
"ilappe"
],
"DOC_REVIEWER": "ShirleyDenkberg",
"TIM_REVIEWER": "MLainer1"
Expand Down
51 changes: 51 additions & 0 deletions .github/workflows/check-nightly-ok-label.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Check nightly-ok label

on:
pull_request:
types: [opened, synchronize, labeled, unlabeled]

jobs:
check_label:
runs-on: ubuntu-latest
if: github.repository == 'demisto/content' && github.event.pull_request.head.repo.fork == false

steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Check if files under .gitlab directory are changed
id: check-changes
run: |
CHANGED_FILES=$(git diff --name-only origin/master...origin/${{ github.head_ref || github.ref_name }})
echo "All changed files:"
echo "${CHANGED_FILES}"
GITLAB_CHANGED_FILES=$( [[ $CHANGED_FILES == *".gitlab/ci"* ]] && echo true || echo false)
echo "Files in the .gitlab folder have changed: ${GITLAB_CHANGED_FILES}"
echo "gitlab_changed_files=$GITLAB_CHANGED_FILES" >> $GITHUB_OUTPUT
if [[ $GITLAB_CHANGED_FILES == true ]]; then
echo 'Files under .gitlab folder has changed, Will check if the PR has the `nightly-ok` label.'
else
echo 'Files in the .gitlab folder have not been changed.'
fi
- name: Check if PR has the nightly-ok label
uses: actions/github-script@v7
id: check-label
with:
script: |
const gitlabChangedFiles = ${{ steps.check-changes.outputs.gitlab_changed_files }};
if(gitlabChangedFiles) {
console.log('Files under .gitlab folder has changed, Will check if the PR has the `nightly-ok` label.');
const labels = context.payload.pull_request.labels.map(label => label.name);
const hasLabel = labels.includes('nightly-ok');
if (hasLabel) {
console.log('All good, the PR has the `nightly-ok` label.');
} else {
console.log('PR does not have the `nightly-ok` label. It is required when changing files under the `.gitlab` directory. Please run nightly using the Utils/gitlab_triggers/trigger_content_nightly_build.sh script, check that succeeded, and add the `nightly-ok` label');
process.exit(1); // Exit with failure status if label is missing
}
} else {
console.log('Files in the .gitlab folder have not been changed.');
}
15 changes: 13 additions & 2 deletions .github/workflows/create-internal-pr-from-external.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ on:
branches:
- contrib/**

permissions:
pull-requests: write

jobs:
create_internal_pr:
runs-on: ubuntu-latest
Expand All @@ -16,11 +19,11 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v3
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Setup Poetry
uses: Gr1N/setup-poetry@v8
uses: Gr1N/setup-poetry@v9
- name: Print Context
run: |
echo "$GITHUB_CONTEXT"
Expand All @@ -39,3 +42,11 @@ jobs:
cd Utils/github_workflow_scripts
poetry run ./create_internal_pr.py
echo "Finished Creating Internal PR"
- name: Notify External PR Merge
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
PR_URL: ${{ github.event.pull_request.html_url }}
run: |
echo "Add a comment"
gh pr comment "$PR_URL" --body "Thank you for your contribution. Your external PR has been merged and the changes are now included in an internal PR for further review. The internal PR will be merged to the master branch within 3 business days."
4 changes: 2 additions & 2 deletions .github/workflows/handle-new-external-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ jobs:
fetch-depth: 2

- name: Setup Python
uses: actions/setup-python@v3
uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Setup Poetry
uses: Gr1N/setup-poetry@v8
uses: Gr1N/setup-poetry@v9
- name: Print Context
run: |
echo "$GITHUB_CONTEXT"
Expand Down
38 changes: 36 additions & 2 deletions .github/workflows/pre-commit-reuse.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ jobs:
with:
fetch-depth: 0

- name: Set PYTHONPATH
run: echo "PYTHONPATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV

- name: Setup python
uses: actions/setup-python@v4
with:
Expand All @@ -45,13 +48,44 @@ jobs:
- name: "Check coverage.xml exists"
if: always()
id: check_files
id: check-coverage-xml-exists
uses: andstor/file-existence-action@v2
with:
files: "coverage_report/coverage.xml"

- name: "Check pytest report exists"
if: always()
id: check-pytest-junit-exists
uses: andstor/file-existence-action@v2
with:
files: ".report_pytest.xml"

- name: Create pack-wise pytest report
run: poetry run python Utils/github_workflow_scripts/parse_junit_per_pack.py
if: |
always() &&
steps.check-pytest-junit-exists.outputs.files_exists == 'true' &&
github.event.pull_request.head.repo.fork == false
- name: Upload junit & pack-wise pytest report
uses: actions/upload-artifact@v4
if: |
always() &&
steps.check-pytest-junit-exists.outputs.files_exists == 'true' &&
github.event.pull_request.head.repo.fork == false
with:
name: pytest
path: |
packwise_pytest_time.csv
.report_pytest.xml
if-no-files-found: error

- name: Pytest coverage comment
if: always() && steps.check_files.outputs.files_exists == 'true' && github.event.pull_request.head.repo.fork == false
if: |
always() &&
steps.check-coverage-xml-exists.outputs.files_exists == 'true' &&
steps.check-pytest-junit-exists.outputs.files_exists == false &&
github.event.pull_request.head.repo.fork == false
uses: MishaKav/pytest-coverage-comment@main
with:
pytest-xml-coverage-path: coverage_report/coverage.xml
Expand Down
33 changes: 0 additions & 33 deletions .github/workflows/trigger-contribution-build.yml

This file was deleted.

18 changes: 15 additions & 3 deletions .github/workflows/update-demisto-sdk-version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ on:
description: The SDK release changes
required: true
type: string
is_draft:
description: Is draft pull request
required: false
type: boolean
default: false

permissions:
contents: write
Expand All @@ -38,7 +43,7 @@ jobs:
python-version: '3.10'
cache: 'poetry'

- name: Create brnach
- name: Create branch
run: |
git config --global user.email "[email protected]"
git config --global user.name "Content Bot"
Expand All @@ -49,14 +54,21 @@ jobs:
run: |
poetry add --group dev demisto-sdk@${{inputs.release_version}}
poetry lock --no-update
git add .
source .venv/bin/activate
demisto-sdk pre-commit --no-validate --no-secrets
demisto-sdk pre-commit --mode=ci
git add .
- name: Create pull request
run: |
git commit -m "poetry files" -a
git push origin ${{ inputs.release_version }}
gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "${{ inputs.release_changes }}" --reviewer ${{ inputs.reviewer }} -l "docs-approved"
if ${{ inputs.is_draft == true }}; then
echo "creating draft release pull request"
gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "${{ inputs.release_changes }}" --reviewer ${{ inputs.reviewer }} -l "docs-approved" --draft
else
echo "creating release pull request"
gh pr create -B master -H ${{ inputs.release_version }} --title "demisto-sdk-release ${{ inputs.release_version }}" --body "${{ inputs.release_changes }}" --reviewer ${{ inputs.reviewer }} -l "docs-approved"
fi
env:
GH_TOKEN: ${{ secrets.CONTENTBOT_GH_ADMIN_TOKEN }}
7 changes: 6 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -99,4 +99,9 @@ demisto_sdk_debug.log.*
*.log

# Ignore Modeling Rules test conf
Packs/**/ModelingRules/**/**/*_testdata.json
Packs/**/ModelingRules/**/**/*_testdata.json

# Build files
.cache/
.npm/
pipeline_jobs_folder/
14 changes: 14 additions & 0 deletions .gitlab/ci/.gitlab-ci.bucket-upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,19 @@
fi
run-validations-upload-flow:
variables:
DEMISTO_SDK_GRAPH_FORCE_CREATE: "true"
extends:
- .run-validations
- .bucket-upload-rule

run-validations-upload-flow-new-validate-flow:
variables:
DEMISTO_SDK_GRAPH_FORCE_CREATE: "true"
extends:
- .run-validations-new-validate-flow
- .bucket-upload-rule
allow_failure: true

run-unittests-and-lint-upload-flow:
cache:
Expand Down Expand Up @@ -181,6 +190,11 @@ xpanse-prepare-testing-bucket-upload-flow:
- section_start "Install Packs"
- ./Tests/Marketplace/install_packs.sh || EXIT_CODE=$?
- section_end "Install Packs"
- section_start "Get instance ssh-command"
- echo "INSTANCE_ROLE -> ${INSTANCE_ROLE}"
- INSTANCE_NAME=$(jq -r --arg role "$INSTANCE_ROLE" '.[] | select(.Role == $role) | .InstanceName' $ENV_RESULTS_PATH)
- echo -e "\e[1m gcloud compute ssh --zone \"us-central1-a\" \"${INSTANCE_NAME}\" --tunnel-through-iap --project "xsoar-content-build" \e[0m"
- section_end "Get instance ssh-command"
- job-done
- exit "${EXIT_CODE}"
after_script:
Expand Down
Loading

0 comments on commit 0facaaa

Please sign in to comment.