Merge branch 'add-degrading-config' of github.com:VihasMakwana/beats …

…into add-degrading-config

.buildkite/packaging.pipeline.yml

@@ -24,7 +24,9 @@ steps:
     key: start-gate-snapshot
 
   - name: Start of concurrency group for DRA Staging
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # exceptionally allow building staging from main when VERSION_QUALIFIER is set, to allow prerelease testing
+    # TODO remove OR clause below and above comment, and only allow matching /^[0-9]+\.[0-9x]+\$/ for build.branch
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     command: echo "--> Start of concurrency gate dra-staging"
     concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
     concurrency: 1
@@ -58,7 +60,8 @@ steps:
           - build/distributions/**/*
 
       - label: Staging dashboards
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         depends_on: start-gate-staging
         key: dashboards-staging
         # TODO: container with go and make
@@ -170,8 +173,8 @@ steps:
   - group: Packaging Staging
     key: packaging-staging
     depends_on: start-gate-staging
-    ## Only for release
-    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+    # TODO remove OR clause below (see earlier comment)
+    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
     steps:
       - label: "STAGING: {{matrix}}"
         env:
@@ -276,8 +279,8 @@ steps:
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
 
       - label: DRA Staging
-        ## Only for release branches
-        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/
+        # TODO remove OR clause below (see earlier comment)
+        if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
         key: dra-staging
         env:
           DRA_WORKFLOW: staging

.buildkite/scripts/dra.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 
 BEAT_VERSION=$(make get-version)
-
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 
 function release_manager_login {
@@ -49,7 +49,9 @@ docker run --rm \
         --commit "${BUILDKITE_COMMIT}" \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
-        --artifact-set "main"
+        --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}"
+
 
 echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]"
 
@@ -68,6 +70,7 @@ docker run --rm \
         --workflow "${DRA_WORKFLOW}" \
         --version "${BEAT_VERSION}" \
         --artifact-set "main" \
+        --qualifier "${VERSION_QUALIFIER}" \
         ${DRY_RUN} | tee rm-output.txt
 
 

.buildkite/scripts/packaging/prepare-release-manager.sh

@@ -8,13 +8,21 @@
 set -ueo pipefail
 
 readonly TYPE=${1:-snapshot}
+readonly VERSION_QUALIFIER="${VERSION_QUALIFIER:-""}"
 
 # rename dependencies.csv to the name expected by release-manager.
 VERSION=$(make get-version)
 FINAL_VERSION=$VERSION-SNAPSHOT
 if [ "$TYPE" != "snapshot" ] ; then
   FINAL_VERSION=$VERSION
 fi
+
+set +e
+if [[ -n "$VERSION_QUALIFIER" ]]; then
+  FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}"
+fi
+set -e
+
 echo "Rename dependencies to $FINAL_VERSION"
 mv build/distributions/dependencies.csv \
    build/distributions/dependencies-"$FINAL_VERSION".csv

.github/workflows/bump-elastic-stack-snapshot.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli --experimental apply --config .github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml --values .github/workflows/updatecli.d/values.d/scm.yml
@@ -41,7 +41,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

.github/workflows/bump-golang.yml

@@ -23,15 +23,15 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli --experimental apply --config .github/workflows/updatecli.d/${{ matrix.file }} --values .github/workflows/updatecli.d/values.d/scm.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

.github/workflows/check-default.yml

@@ -20,6 +20,11 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version-file: .go-version
+    #  when using ubuntu-latest, python 3.10 is not the default version.
+    - name: Fix Code is not compatible with Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
     - name: Run check-default
       run: |
         go install github.com/magefile/mage

.github/workflows/check-docs.yml

@@ -28,6 +28,11 @@ jobs:
       run: sudo apt-get install -y libsystemd-dev
     - name: Install librpm-dev
       run: sudo apt-get install -y librpm-dev
+    #  when using ubuntu-latest, python 3.10 is not the default version.
+    - name: Fix Code is not compatible with Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
     - name: Run check
       run: |
         make check

.github/workflows/mergify-labels-copier.yml

@@ -0,0 +1,23 @@
+name: mergify backport labels copier
+
+on:
+  pull_request:
+    types:
+      - opened
+
+permissions:
+  contents: read
+
+jobs:
+  mergify-backport-labels-copier:
+    runs-on: ubuntu-latest
+    if: startsWith(github.head_ref, 'mergify/bp/')
+    permissions:
+      # Add GH labels
+      pull-requests: write
+      # See https://github.com/cli/cli/issues/6274
+      repository-projects: read
+    steps:
+      - uses: elastic/oblt-actions/mergify/labels-copier@v1
+        with:
+          excluded-labels-regex: "^backport-*"

.github/workflows/notify-stalled-snapshots.yml

@@ -55,7 +55,7 @@ jobs:
 
       - if: ${{ contains(steps.search.outputs.found, 'true') }}
         name: Report obsoleted branches (slack)
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload:  ":red_circle: Elastic Stack version for the `${{ matrix.branch }}` branch has not been updated for a while (`> 7 days`). Review the (<${{ env.URL_QUERY }}|open PRs>)"
@@ -64,7 +64,7 @@ jobs:
 
       - if: ${{ contains(steps.search.outputs.found, 'true') }}
         name: Report obsoleted branches (email)
-        uses: dawidd6/action-send-mail@2cea9617b09d79a095af21254fbcb7ae95903dde
+        uses: dawidd6/action-send-mail@611879133a9569642c41be66f4a323286e9b8a3b
         with:
           server_address: ${{ secrets.MAIL_SERVER }}
           username: ${{ secrets.MAIL_USERNAME }}

.github/workflows/updatecli-compose.yml

@@ -38,7 +38,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

CHANGELOG-developer.next.asciidoc

@@ -109,6 +109,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Fix flaky test in cel and httpjson inputs of filebeat. {issue}40503[40503] {pull}41358[41358]
 - Fix documentation and implementation of raw message handling in Filebeat http_endpoint by removing it. {pull}41498[41498]
 - Fix flaky test in filebeat Okta entity analytics provider. {issue}42059[42059] {pull}42123[42123]
+- Fix IIS module logging errors in case application pool PDH counter is not found. {pull}42274[42274]
 
 ==== Added
 

CHANGELOG.next.asciidoc

@@ -17,7 +17,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Drop support for Debian 10 and upgrade statically linked glibc from 2.28 to 2.31 {pull}41402[41402]
 - Fix metrics not being ingested, due to "Limit of total fields [10000] has been exceeded while adding new fields [...]". The total fields limit has been increased to 12500. No significant performance impact on Elasticsearch is anticipated. {pull}41640[41640]
 - Set default kafka version to 2.1.0 in kafka output and filebeat. {pull}41662[41662]
+- Replace default Ubuntu-based images with UBI-minimal-based ones {pull}42150[42150]
 - Fix templates and docs to use correct `--` version of command line arguments. {issue}42038[42038] {pull}42060[42060]
+- removed support for a single `-` to precede multi-letter command line arguments.  Use `--` instead. {issue}42117[42117] {pull}42209[42209] 
 
 *Auditbeat*
 
@@ -203,6 +205,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix awss3 document ID construction when using the CSV decoder. {pull}42019[42019]
 - The `_id` generation process for S3 events has been updated to incorporate the LastModified field. This enhancement ensures that the `_id` is unique. {pull}42078[42078]
 - Fix Netflow Template Sharing configuration handling. {pull}42080[42080]
+- Updated websocket retry error code list to allow more scenarios to be retried which could have been missed previously. {pull}42218[42218] 
 
 *Heartbeat*
 
@@ -236,6 +239,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Do not report non-existant 0 values for RSS metrics in docker/memory {pull}41449[41449]
 - Log Cisco Meraki `getDevicePerformanceScores` errors without stopping metrics collection. {pull}41622[41622]
 - Don't skip first bucket value in GCP metrics metricset for distribution type metrics {pull}41822[41822]
+- [K8s Integration] Enhance HTTP authentication in case of token updates for Apiserver, Controllermanager and Scheduler metricsets  {issue}41910[41910] {pull}42016[42016]
 - Fixed `creation_date` scientific notation output in the `elasticsearch.index` metricset. {pull}42053[42053]
 - Fix bug where metricbeat unintentionally triggers Windows ASR. {pull}42177[42177]
 
@@ -244,6 +248,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
+- Properly marshal nested structs in ECS fields, fixing issues with mixed cases in field names {pull}42116[42116]
+
 
 *Winlogbeat*
 
@@ -381,6 +387,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 - Added default values in the streaming input for websocket retries and put a cap on retry wait time to be lesser than equal to the maximum defined wait time. {pull}42012[42012]
 - Rate limiting fault tolerance improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42094[42094]
+- Added OAuth2 support with auto token refresh for websocket streaming input. {issue}41989[41989] {pull}42212[42212]
+- Added infinite & blanket retry options to websockets and improved logging and retry logic. {pull}42225[42225]
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
 
 *Auditbeat*
 
@@ -436,6 +445,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Collect .NET CLR (IIS) Memory, Exceptions and LocksAndThreads metrics {pull}41929[41929]
 - Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
 - Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
+- Preserve queries for debugging when `merge_results: true` in SQL module {pull}42271[42271]
 
 *Metricbeat*
 - Add benchmark module {pull}41801[41801]

NOTICE.txt

@@ -12488,11 +12488,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.18.0
+Version: v0.18.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -13121,11 +13121,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v8
-Version: v8.14.0
+Version: v8.17.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.14.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.17.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/auditbeat.reference.yml

@@ -1573,7 +1573,7 @@ logging.files:
   #path: /var/log/auditbeat
 
   # The name of the files where the logs are written to.
-  #name: auditbeat-event-data
+  #name: auditbeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

auditbeat/main_test.go

@@ -24,7 +24,6 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/v7/auditbeat/cmd"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/tests/system/template"
 )
 
@@ -35,14 +34,11 @@ func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
 
 	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("systemTest")
 	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("test.coverprofile")
 }
 
 // Test started when the test binary is started. Only calls main.
 func TestSystem(*testing.T) {
-	cfgfile.ConvertFlagsForBackwardsCompatibility()
 	if *systemTest {
 		main()
 	}

auditbeat/tests/system/test_show_command.py

@@ -57,7 +57,7 @@ def test_show_auditd_rules(self):
                 }
             }]
         )
-        proc = self.start_beat(extra_args=['-strict.perms=false'])
+        proc = self.start_beat(extra_args=['--strict.perms=false'])
         # auditbeat adds an extra rule to ignore itself
         self.wait_log_contains('Successfully added {0} of {0} audit rules.'.format(len(rules) + 1),
                                max_timeout=30)

dev-tools/cmd/dashboards/export_dashboards.go

@@ -46,7 +46,7 @@ func main() {
 	insecure := flag.Bool("insecure", false, "Disable TLS verification.")
 	spaceID := flag.String("space-id", "", "Space ID")
 	dashboard := flag.String("dashboard", "", "Dashboard ID")
-	fileOutput := flag.String("output", "", "Output NDJSON file, when exporting dashboards for Beats, please use -folder instead")
+	fileOutput := flag.String("output", "", "Output NDJSON file, when exporting dashboards for Beats, please use --folder instead")
 	folderOutput := flag.String("folder", "", "Output folder to save all assets to more human friendly JSON format")
 	ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards")
 	flag.BoolVar(&indexPattern, "indexPattern", false, "include index-pattern in output")
@@ -56,7 +56,7 @@ func main() {
 	log.SetFlags(0)
 
 	if len(*fileOutput) > 0 {
-		log.Fatalf("-output is configured, please use -folder flag instead to get the expected formatting of assets")
+		log.Fatalf("--output is configured, please use --folder flag instead to get the expected formatting of assets")
 	}
 
 	u, err := url.Parse(*kibanaURL)
@@ -92,10 +92,10 @@ func main() {
 
 	if len(*ymlFile) == 0 && len(*dashboard) == 0 {
 		flag.Usage()
-		log.Fatalf("Please specify a dashboard ID (-dashboard) or a manifest file (-yml)")
+		log.Fatalf("Please specify a dashboard ID (--dashboard) or a manifest file (--yml)")
 	}
 	if len(*folderOutput) == 0 {
-		log.Fatalf("Please specify a target folder using -folder flag")
+		log.Fatalf("Please specify a target folder using --folder flag")
 	}
 
 	if len(*ymlFile) > 0 {

dev-tools/mage/dashboard.go

@@ -49,8 +49,8 @@ func ExportDashboard() error {
 	folder := CWD("module", module)
 
 	args := []string{
-		"-folder", folder,
-		"-dashboard", id,
+		"--folder", folder,
+		"--dashboard", id,
 	}
 	if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
 		args = append(args, "-kibana", kibanaURL)

dev-tools/mage/gotool/go.go

@@ -163,7 +163,7 @@ func HasTests(pkg string) (bool, error) {
 }
 
 func (goTest) WithCoverage(to string) ArgOpt {
-	return combine(flagArg("-cover", ""), flagArgIf("-test.coverprofile", to))
+	return combine(flagArg("-cover", ""), flagArgIf("--test.coverprofile", to))
 }
 func (goTest) Short(b bool) ArgOpt        { return flagBoolIf("-test.short", b) }
 func (goTest) Use(bin string) ArgOpt      { return extraArgIf("use", bin) }
@@ -224,10 +224,6 @@ func runVGo(cmd string, args *Args) error {
 	}, cmd, args)
 }
 
-func runGo(cmd string, args *Args) error {
-	return execGoWith(sh.RunWith, cmd, args)
-}
-
 func execGoWith(
 	fn func(map[string]string, string, ...string) error,
 	cmd string, args *Args,