Merge branch 'develop' into nterl0k-o365-azure-workload-1

.github/CODEOWNERS

@@ -1 +1 @@
-* @patel-bhavin @P4T12ICK
+* @patel-bhavin @ljstella

.github/dependabot.yml

@@ -1,7 +1,6 @@
 version: 2
 updates:
-- package-ecosystem: pip
+- package-ecosystem: "github-actions"
   directory: "/"
   schedule:
-    interval: monthly
-  open-pull-requests-limit: 6
+    interval: "weekly"

.github/labeler.yml

@@ -17,4 +17,8 @@ Macros:
 
 Lookups:
 - changed-files:
-  - any-glob-to-any-file: lookups/*
+  - any-glob-to-any-file: lookups/*
+
+Datasource:
+- changed-files:
+  - any-glob-to-any-file: data_sources/*

.github/pull_request_template.md

@@ -9,3 +9,10 @@ _What does this PR have in it? Screenshots are worth 1000 words 😄_
 - [ ] Validated SPL logic.
 - [ ] Validated tags, description, and how to implement.
 - [ ] Verified references match analytic.
+- [ ] Confirm updates to lookups are handled properly.
+
+### Notes For Submitters and Reviewers
+
+- If you're submitting a PR from a fork, ensuring the box to allow updates from maintainers is checked will help speed up the process of getting it merged.
+- Checking the output of the `build` CI job when it fails will likely show an error about what is failing. You may have a very descriptive error of the specific field(s) in the specific file(s) that is causing an issue. In some cases, its also possible there is an issue with the YAML. Many of these can be caught with the pre-commit hooks if you set them up. These errors will be less descriptive as to what exactly is wrong, but will give you a column and row position in a specific file where the YAML processing breaks. If you're having trouble with this, feel free to add a comment to your PR tagging one of the maintainers and we'll be happy to help troubleshoot it.
+- Updates to existing lookup files can be tricky, because of how Splunk handles application updates and the differences between existing lookup files being updated vs new lookups. You can read more [here](https://docs.splunk.com/Documentation/SplunkCloud/8.2.2203/Admin/PrivateApps#Manage_lookups_in_Splunk_Cloud_Platform) but the short version is that any changes to lookup files need to bump the datestamp in the lookup CSV filename, and the reference to it in the YAML needs to be updated.

.github/workflows/appinspect.yml

@@ -0,0 +1,47 @@
+name: appinspect
+on:
+  pull_request_target:
+  push:
+    branches:
+      - develop
+
+jobs:
+  appinspect:
+    #Note that the CircleCI job used a Container.  The way to do this with Github Actions
+    #is to first start up a Virtual Machine, then we can by following:
+    # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository code
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
+          architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
+
+
+      - name: Install Python Dependencies and ContentCTL and Atomic Red Team
+        run: |
+          pip install contentctl>=4.0.0
+          git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git
+      
+      - name: Running build with enrichments
+        env:
+          APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}"
+          APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}"
+        run: |
+          echo $APPINSPECTUSERNAME
+          contentctl inspect --splunk-api-username "$APPINSPECTUSERNAME" --splunk-api-password "$APPINSPECTPASSWORD" --stack_type victoria --enrichments
+          echo "done appinspect"
+          mkdir -p artifacts/app_inspect_report
+          cp -r dist/*.html artifacts/app_inspect_report
+          cp -r dist/*.tar.gz artifacts/
+
+      - name: store_artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: content-latest
+          path: |
+            artifacts/DA-ESS-ContentUpdate-latest.tar.gz
+            artifacts/app_inspect_report

.github/workflows/build.yml

@@ -19,11 +19,10 @@ jobs:
         with:
           python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
           architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
-
 
       - name: Install Python Dependencies and ContentCTL and Atomic Red Team
         run: |
-          pip install contentctl==4.1.5
+          pip install contentctl>=4.0.0
           git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git
       
       - name: Running build with enrichments

.github/workflows/format_test_results.py

@@ -36,16 +36,14 @@ def main():
     for detection in data['tested_detections']:
         for test in detection['tests']:
             if test['test_type'].strip() == "unit":  # Check if the test type is "unit"
-                name = detection['name'].strip()
-                status = 'PASS' if test['success'] else 'FAIL'
-                test_type = test['test_type'].strip()
-                exception = test.get('exception', 'N/A')  # Get exception if exists, else 'N/A'
+                name = (detection.get('name') or 'N/A').strip()
+                status = 'PASS' if test.get('success') else 'FAIL'
+                test_type = (test.get('test_type') or 'N/A').strip()
+                exception = (test.get('exception') or 'N/A')  # Get exception if exists, else 'N/A'
                 if status == 'FAIL':
                     print(f"{name:<80} | 🔴 {status:<6}  | {test_type:<10} | {exception:<50}")
                 else:
                     print(f"{name:<80} | 🟢  {status:<6} | {test_type:<10} | {'-':<50}")
-    # Print table footer
-    # print(f"{'----':<80} | {'------':<6} | {'---------':<10} | {'---------':<50}")
 
     # Check if total_fail is a valid integer and greater than or equal to one
     print("\n")  # Print a newline for separation

.github/workflows/unit-testing.yml

@@ -6,7 +6,6 @@ jobs:
   unit-testing:
     runs-on: ubuntu-latest
     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
-    # needs: [validate-tag-if-present, quit-for-dependabot]
     steps:
         #For fork PRs, always check out security_content and the PR target in security content!
         - name: Check out the repository code
@@ -24,7 +23,7 @@ jobs:
         - name: Install Python Dependencies and ContentCTL
           run: |
             python -m pip install --upgrade pip
-            pip install contentctl==4.1.5
+            pip install contentctl>=4.0.0
            
         # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop
         # Make sure we check out the PR, even if it actually lives in a fork
@@ -36,11 +35,10 @@ jobs:
             echo "Current Branch (Head Ref): ${{ github.head_ref }}"
             echo "Target Branch (Base Ref): ${{ github.base_ref }}"
             git pull > /dev/null 2>&1
-            git fetch origin pull/${{ github.event.pull_request.number }}/head:${{ github.head_ref }}
-            #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo
-            git switch ${{ github.head_ref }}
-            #git checkout ${{ github.head_ref }}
-            #echo "The target branch for this PR is ${{ github.base_ref }}"
+            #We checkout into a new branch - new_branch_for_testing to avoid name collisions with develop incase the forked PR is from develop
+            git fetch origin pull/${{ github.event.pull_request.number }}/head:new_branch_for_testing
+            #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo           
+            git switch new_branch_for_testing
             contentctl test --disable-tqdm --no-enable-integration-testing --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }}
             echo "contentctl test - COMPLETED"
           continue-on-error: true
@@ -65,4 +63,4 @@ jobs:
           run: |       
             echo "This job will fail if there are failures in unit-testing"  
             python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY
-            echo "The Unit testing is completed. See details in the unit-testing job summary UI "
+            echo "The Unit testing is completed. See details in the unit-testing job summary UI "

contentctl.yml

@@ -3,7 +3,7 @@ app:
   uid: 3449
   title: ES Content Updates
   appid: DA-ESS-ContentUpdate
-  version: 4.35.0
+  version: 4.37.0
   description: Explore the Analytic Stories included with ES Content Updates.
   prefix: ESCU
   label: ESCU
@@ -182,4 +182,11 @@ apps:
   version: 1.9.2
   description: description of app
   hardcoded_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/url-toolbox_192.tgz
+- uid: 5082
+  title: CrowdStrike Falcon Event Streams Technical Add-On
+  appid: TA-crowdstrike-falcon-event-streams
+  version: 3.2.1
+  description: description of app
+  hardcoded_path: https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/crowdstrike-falcon-event-streams-technical-add-on_321.tgz
+
 githash: d6fac80e6d50ae06b40f91519a98489d4ce3a3fd