From 3628998360457c9896b8a83080284fe1a382aff9 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Sun, 22 Sep 2024 16:17:22 +0300 Subject: [PATCH 01/17] go.d/sensors update meta (#18595) --- src/go/plugin/go.d/modules/sensors/collect.go | 2 -- .../integrations/linux_sensors_lm-sensors.md | 0 .../plugin/go.d/modules/sensors/metadata.yaml | 26 +++++++++---------- .../go.d/modules/sensors/sensors_test.go | 2 ++ 4 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 src/go/plugin/go.d/modules/sensors/integrations/linux_sensors_lm-sensors.md diff --git a/src/go/plugin/go.d/modules/sensors/collect.go b/src/go/plugin/go.d/modules/sensors/collect.go index 915b1f4ef7aa32..8d77fa1a2a90d2 100644 --- a/src/go/plugin/go.d/modules/sensors/collect.go +++ b/src/go/plugin/go.d/modules/sensors/collect.go @@ -11,8 +11,6 @@ import ( const precision = 1000 -func ptr[T any](v T) *T { return &v } - func (s *Sensors) collect() (map[string]int64, error) { if s.sc == nil { return nil, errors.New("sysfs scanner is not initialized") diff --git a/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors_lm-sensors.md b/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors_lm-sensors.md deleted file mode 100644 index e69de29bb2d1d6..00000000000000 diff --git a/src/go/plugin/go.d/modules/sensors/metadata.yaml b/src/go/plugin/go.d/modules/sensors/metadata.yaml index c9484ecb236ad2..5fcb839888eb73 100644 --- a/src/go/plugin/go.d/modules/sensors/metadata.yaml +++ b/src/go/plugin/go.d/modules/sensors/metadata.yaml @@ -29,9 +29,18 @@ modules: overview: data_collection: metrics_description: > - This collector gathers real-time system sensor statistics, - including temperature, voltage, current, power, fan speed, energy consumption, and humidity, - utilizing the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. + This collector gathers real-time system sensor statistics using the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. + + Supported sensors: + + - Temperature + - Voltage + - Fan + - Current + - Power + - Energy + - Humidity + - Intrusion method_description: "" supported_platforms: include: [] @@ -42,16 +51,7 @@ modules: default_behavior: auto_detection: description: | - The following type of sensors are auto-detected: - - - temperature - - voltage - - fan - - current - - power - - energy - - humidity - - intrusion + Automatically discovers and exposes all available sensors on the system through the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. limits: description: "" performance_impact: diff --git a/src/go/plugin/go.d/modules/sensors/sensors_test.go b/src/go/plugin/go.d/modules/sensors/sensors_test.go index 507e6aa7aba608..87863b9171fa5d 100644 --- a/src/go/plugin/go.d/modules/sensors/sensors_test.go +++ b/src/go/plugin/go.d/modules/sensors/sensors_test.go @@ -306,6 +306,8 @@ func (m *mockScanner) Scan() ([]*lmsensors.Chip, error) { return m.scanData, nil } +func ptr[T any](v T) *T { return &v } + func mockChips() []*lmsensors.Chip { return []*lmsensors.Chip{ { From 1e8358383527f3b811da07b08bf2718a040aa6a1 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Sun, 22 Sep 2024 16:35:56 +0300 Subject: [PATCH 02/17] go.d/sensors fix meta (#18597) --- src/go/plugin/go.d/modules/sensors/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/go/plugin/go.d/modules/sensors/metadata.yaml b/src/go/plugin/go.d/modules/sensors/metadata.yaml index 5fcb839888eb73..810917adeec97e 100644 --- a/src/go/plugin/go.d/modules/sensors/metadata.yaml +++ b/src/go/plugin/go.d/modules/sensors/metadata.yaml @@ -28,7 +28,7 @@ modules: most_popular: false overview: data_collection: - metrics_description: > + metrics_description: | This collector gathers real-time system sensor statistics using the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. Supported sensors: From d7625356030233af42eff6ca020c9ee27a9c03bb Mon Sep 17 00:00:00 2001 From: Netdata bot <43409846+netdatabot@users.noreply.github.com> Date: Sun, 22 Sep 2024 09:57:04 -0400 Subject: [PATCH 03/17] Regenerate integrations.js (#18598) Co-authored-by: ilyam8 <22274335+ilyam8@users.noreply.github.com> --- integrations/integrations.js | 2 +- integrations/integrations.json | 2 +- .../sensors/integrations/linux_sensors.md | 24 ++++++++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/integrations/integrations.js b/integrations/integrations.js index 20226f4ad78336..a978fadaa6d720 100644 --- a/integrations/integrations.js +++ b/integrations/integrations.js @@ -16518,7 +16518,7 @@ export const integrations = [ }, "most_popular": false }, - "overview": "# Linux Sensors\n\nPlugin: go.d.plugin\nModule: sensors\n\n## Overview\n\nThis collector gathers real-time system sensor statistics, including temperature, voltage, current, power, fan speed, energy consumption, and humidity, utilizing the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\n\n\n\nThis collector is supported on all platforms.\n\nThis collector only supports collecting metrics from a single instance of this integration.\n\n\n### Default Behavior\n\n#### Auto-Detection\n\nThe following type of sensors are auto-detected:\n\n- temperature\n- voltage\n- fan\n- current\n- power\n- energy\n- humidity\n- intrusion\n\n\n#### Limits\n\nThe default configuration for this integration does not impose any limits on data collection.\n\n#### Performance Impact\n\nThe default configuration for this integration is not expected to impose a significant performance impact on the system.\n", + "overview": "# Linux Sensors\n\nPlugin: go.d.plugin\nModule: sensors\n\n## Overview\n\nThis collector gathers real-time system sensor statistics using the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\nSupported sensors:\n\n- Temperature\n- Voltage\n- Fan\n- Current\n- Power\n- Energy\n- Humidity\n- Intrusion\n\n\n\n\nThis collector is supported on all platforms.\n\nThis collector only supports collecting metrics from a single instance of this integration.\n\n\n### Default Behavior\n\n#### Auto-Detection\n\nAutomatically discovers and exposes all available sensors on the system through the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\n\n#### Limits\n\nThe default configuration for this integration does not impose any limits on data collection.\n\n#### Performance Impact\n\nThe default configuration for this integration is not expected to impose a significant performance impact on the system.\n", "setup": "## Setup\n\n### Prerequisites\n\nNo action required.\n\n### Configuration\n\n#### File\n\nThe configuration file name for this integration is `go.d/sensors.conf`.\n\n\nYou can edit the configuration file using the `edit-config` script from the\nNetdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory).\n\n```bash\ncd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata\nsudo ./edit-config go.d/sensors.conf\n```\n#### Options\n\nThe following options can be defined globally: update_every.\n\n\n{% details open=true summary=\"Config options\" %}\n| Name | Description | Default | Required |\n|:----|:-----------|:-------|:--------:|\n| update_every | Data collection frequency. | 10 | no |\n\n{% /details %}\n#### Examples\n\n##### Custom update_every\n\nAllows you to override the default data collection interval.\n\n{% details open=true summary=\"Config\" %}\n```yaml\njobs:\n - name: sensors\n update_every: 5 # Collect sensors statistics every 5 seconds\n\n```\n{% /details %}\n", "troubleshooting": "## Troubleshooting\n\n### Debug Mode\n\n**Important**: Debug mode is not supported for data collection jobs created via the UI using the Dyncfg feature.\n\nTo troubleshoot issues with the `sensors` collector, run the `go.d.plugin` with the debug option enabled. The output\nshould give you clues as to why the collector isn't working.\n\n- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on\n your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`.\n\n ```bash\n cd /usr/libexec/netdata/plugins.d/\n ```\n\n- Switch to the `netdata` user.\n\n ```bash\n sudo -u netdata -s\n ```\n\n- Run the `go.d.plugin` to debug the collector:\n\n ```bash\n ./go.d.plugin -d -m sensors\n ```\n\n### Getting Logs\n\nIf you're encountering problems with the `sensors` collector, follow these steps to retrieve logs and identify potential issues:\n\n- **Run the command** specific to your system (systemd, non-systemd, or Docker container).\n- **Examine the output** for any warnings or error messages that might indicate issues. These messages should provide clues about the root cause of the problem.\n\n#### System with systemd\n\nUse the following command to view logs generated since the last Netdata service restart:\n\n```bash\njournalctl _SYSTEMD_INVOCATION_ID=\"$(systemctl show --value --property=InvocationID netdata)\" --namespace=netdata --grep sensors\n```\n\n#### System without systemd\n\nLocate the collector log file, typically at `/var/log/netdata/collector.log`, and use `grep` to filter for collector's name:\n\n```bash\ngrep sensors /var/log/netdata/collector.log\n```\n\n**Note**: This method shows logs from all restarts. Focus on the **latest entries** for troubleshooting current issues.\n\n#### Docker Container\n\nIf your Netdata runs in a Docker container named \"netdata\" (replace if different), use this command:\n\n```bash\ndocker logs netdata 2>&1 | grep sensors\n```\n\n", "alerts": "## Alerts\n\nThere are no alerts configured by default for this integration.\n", diff --git a/integrations/integrations.json b/integrations/integrations.json index a28d1ee08c6e52..585b9505daa099 100644 --- a/integrations/integrations.json +++ b/integrations/integrations.json @@ -16516,7 +16516,7 @@ }, "most_popular": false }, - "overview": "# Linux Sensors\n\nPlugin: go.d.plugin\nModule: sensors\n\n## Overview\n\nThis collector gathers real-time system sensor statistics, including temperature, voltage, current, power, fan speed, energy consumption, and humidity, utilizing the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\n\n\n\nThis collector is supported on all platforms.\n\nThis collector only supports collecting metrics from a single instance of this integration.\n\n\n### Default Behavior\n\n#### Auto-Detection\n\nThe following type of sensors are auto-detected:\n\n- temperature\n- voltage\n- fan\n- current\n- power\n- energy\n- humidity\n- intrusion\n\n\n#### Limits\n\nThe default configuration for this integration does not impose any limits on data collection.\n\n#### Performance Impact\n\nThe default configuration for this integration is not expected to impose a significant performance impact on the system.\n", + "overview": "# Linux Sensors\n\nPlugin: go.d.plugin\nModule: sensors\n\n## Overview\n\nThis collector gathers real-time system sensor statistics using the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\nSupported sensors:\n\n- Temperature\n- Voltage\n- Fan\n- Current\n- Power\n- Energy\n- Humidity\n- Intrusion\n\n\n\n\nThis collector is supported on all platforms.\n\nThis collector only supports collecting metrics from a single instance of this integration.\n\n\n### Default Behavior\n\n#### Auto-Detection\n\nAutomatically discovers and exposes all available sensors on the system through the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface.\n\n\n#### Limits\n\nThe default configuration for this integration does not impose any limits on data collection.\n\n#### Performance Impact\n\nThe default configuration for this integration is not expected to impose a significant performance impact on the system.\n", "setup": "## Setup\n\n### Prerequisites\n\nNo action required.\n\n### Configuration\n\n#### File\n\nThe configuration file name for this integration is `go.d/sensors.conf`.\n\n\nYou can edit the configuration file using the `edit-config` script from the\nNetdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory).\n\n```bash\ncd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata\nsudo ./edit-config go.d/sensors.conf\n```\n#### Options\n\nThe following options can be defined globally: update_every.\n\n\n| Name | Description | Default | Required |\n|:----|:-----------|:-------|:--------:|\n| update_every | Data collection frequency. | 10 | no |\n\n#### Examples\n\n##### Custom update_every\n\nAllows you to override the default data collection interval.\n\n```yaml\njobs:\n - name: sensors\n update_every: 5 # Collect sensors statistics every 5 seconds\n\n```\n", "troubleshooting": "## Troubleshooting\n\n### Debug Mode\n\n**Important**: Debug mode is not supported for data collection jobs created via the UI using the Dyncfg feature.\n\nTo troubleshoot issues with the `sensors` collector, run the `go.d.plugin` with the debug option enabled. The output\nshould give you clues as to why the collector isn't working.\n\n- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on\n your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`.\n\n ```bash\n cd /usr/libexec/netdata/plugins.d/\n ```\n\n- Switch to the `netdata` user.\n\n ```bash\n sudo -u netdata -s\n ```\n\n- Run the `go.d.plugin` to debug the collector:\n\n ```bash\n ./go.d.plugin -d -m sensors\n ```\n\n### Getting Logs\n\nIf you're encountering problems with the `sensors` collector, follow these steps to retrieve logs and identify potential issues:\n\n- **Run the command** specific to your system (systemd, non-systemd, or Docker container).\n- **Examine the output** for any warnings or error messages that might indicate issues. These messages should provide clues about the root cause of the problem.\n\n#### System with systemd\n\nUse the following command to view logs generated since the last Netdata service restart:\n\n```bash\njournalctl _SYSTEMD_INVOCATION_ID=\"$(systemctl show --value --property=InvocationID netdata)\" --namespace=netdata --grep sensors\n```\n\n#### System without systemd\n\nLocate the collector log file, typically at `/var/log/netdata/collector.log`, and use `grep` to filter for collector's name:\n\n```bash\ngrep sensors /var/log/netdata/collector.log\n```\n\n**Note**: This method shows logs from all restarts. Focus on the **latest entries** for troubleshooting current issues.\n\n#### Docker Container\n\nIf your Netdata runs in a Docker container named \"netdata\" (replace if different), use this command:\n\n```bash\ndocker logs netdata 2>&1 | grep sensors\n```\n\n", "alerts": "## Alerts\n\nThere are no alerts configured by default for this integration.\n", diff --git a/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors.md b/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors.md index 703790d06693bf..e70ad137f0e28e 100644 --- a/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors.md +++ b/src/go/plugin/go.d/modules/sensors/integrations/linux_sensors.md @@ -21,7 +21,18 @@ Module: sensors ## Overview -This collector gathers real-time system sensor statistics, including temperature, voltage, current, power, fan speed, energy consumption, and humidity, utilizing the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. +This collector gathers real-time system sensor statistics using the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. + +Supported sensors: + +- Temperature +- Voltage +- Fan +- Current +- Power +- Energy +- Humidity +- Intrusion @@ -35,16 +46,7 @@ This collector only supports collecting metrics from a single instance of this i #### Auto-Detection -The following type of sensors are auto-detected: - -- temperature -- voltage -- fan -- current -- power -- energy -- humidity -- intrusion +Automatically discovers and exposes all available sensors on the system through the [sysfs](https://www.kernel.org/doc/Documentation/hwmon/sysfs-interface) interface. #### Limits From d09f3e65ff1d92963f6167a80b7f332f65875969 Mon Sep 17 00:00:00 2001 From: netdatabot Date: Mon, 23 Sep 2024 00:19:58 +0000 Subject: [PATCH 04/17] [ci skip] Update changelog and version for nightly build: v1.99.0-182-nightly. --- CHANGELOG.md | 6 +++--- packaging/version | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc016a1c539edc..b89d0c4814d9bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ **Merged pull requests:** +- Regenerate integrations.js [\#18598](https://github.com/netdata/netdata/pull/18598) ([netdatabot](https://github.com/netdatabot)) +- go.d/sensors fix meta [\#18597](https://github.com/netdata/netdata/pull/18597) ([ilyam8](https://github.com/ilyam8)) +- go.d/sensors update meta [\#18595](https://github.com/netdata/netdata/pull/18595) ([ilyam8](https://github.com/ilyam8)) - Regenerate integrations.js [\#18592](https://github.com/netdata/netdata/pull/18592) ([netdatabot](https://github.com/netdatabot)) - Bump go.mongodb.org/mongo-driver from 1.16.1 to 1.17.0 in /src/go [\#18589](https://github.com/netdata/netdata/pull/18589) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/docker/docker from 27.2.1+incompatible to 27.3.0+incompatible in /src/go [\#18588](https://github.com/netdata/netdata/pull/18588) ([dependabot[bot]](https://github.com/apps/dependabot)) @@ -416,9 +419,6 @@ - fix installing netdata-updater svc/timer for native packages [\#18032](https://github.com/netdata/netdata/pull/18032) ([ilyam8](https://github.com/ilyam8)) - Troubleshooter must be assigned to rooms docs [\#18031](https://github.com/netdata/netdata/pull/18031) ([Ancairon](https://github.com/Ancairon)) - Regenerate integrations.js [\#18030](https://github.com/netdata/netdata/pull/18030) ([netdatabot](https://github.com/netdatabot)) -- go.d/postfix: simplify and fix tests [\#18029](https://github.com/netdata/netdata/pull/18029) ([ilyam8](https://github.com/ilyam8)) -- go.d k8state: skip jobs/cronjobs Pods [\#18028](https://github.com/netdata/netdata/pull/18028) ([ilyam8](https://github.com/ilyam8)) -- Port postfix collector from python to go [\#18026](https://github.com/netdata/netdata/pull/18026) ([Ancairon](https://github.com/Ancairon)) ## [v1.46.3](https://github.com/netdata/netdata/tree/v1.46.3) (2024-07-23) diff --git a/packaging/version b/packaging/version index 14155a819f0cbe..3ec991da57b7a4 100644 --- a/packaging/version +++ b/packaging/version @@ -1 +1 @@ -v1.99.0-178-nightly +v1.99.0-182-nightly From c51e92b754616ef38939cce217a8d0113bb3e207 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:33:12 +0300 Subject: [PATCH 05/17] Bump github.com/docker/docker from 27.3.0+incompatible to 27.3.1+incompatible in /src/go (#18600) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- src/go/go.mod | 2 +- src/go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/go/go.mod b/src/go/go.mod index ebf0d03e006b13..0e7b207b5e4946 100644 --- a/src/go/go.mod +++ b/src/go/go.mod @@ -16,7 +16,7 @@ require ( github.com/clbanning/rfile/v2 v2.0.0-20231024120205-ac3fca974b0e github.com/cloudflare/cfssl v1.6.5 github.com/coreos/go-systemd/v22 v22.5.0 - github.com/docker/docker v27.3.0+incompatible + github.com/docker/docker v27.3.1+incompatible github.com/facebook/time v0.0.0-20240626113945-18207c5d8ddc github.com/fsnotify/fsnotify v1.7.0 github.com/go-sql-driver/mysql v1.8.1 diff --git a/src/go/go.sum b/src/go/go.sum index f14b65e6f6ddb8..c4220579a46958 100644 --- a/src/go/go.sum +++ b/src/go/go.sum @@ -64,8 +64,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.3.0+incompatible h1:BNb1QY6o4JdKpqwi9IB+HUYcRRrVN4aGFUTvDmWYK1A= -github.com/docker/docker v27.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= +github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= From 0c9373f4d6847dd1b1d394239195a89e7a531902 Mon Sep 17 00:00:00 2001 From: thiagoftsm Date: Mon, 23 Sep 2024 12:07:33 +0000 Subject: [PATCH 06/17] Add DLLs to CmakeLists.txt (#18590) --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56213bcfb56f70..6a63c24d328691 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3083,6 +3083,10 @@ if(NOT OS_WINDOWS) endif() if(OS_WINDOWS) + install(FILES /usr/bin/msys-protobuf-32.dll + /usr/bin/msys-uv-1.dll + DESTINATION "${BINDIR}") + # Make bash & netdata happy install(DIRECTORY DESTINATION tmp) From 27fcf0af5704aa396d7dda6d6c19cd82a295763c Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Mon, 23 Sep 2024 14:04:03 -0400 Subject: [PATCH 07/17] Publish Windows installers on nightly builds. (#18603) Also do some general cleanup of the build workflow, including a bit of reorganization. --- .github/workflows/build.yml | 288 +++++++++++++++--------------------- 1 file changed, 122 insertions(+), 166 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e80efffcff237..f17a214351cee5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -275,12 +275,118 @@ jobs: && needs.file-check.outputs.run == 'true' }} + windows-build: # Test building on Windows + name: Test building on Windows + runs-on: windows-latest + needs: + - file-check + steps: + - name: Skip Check + id: skip + if: needs.file-check.outputs.run != 'true' + run: Write-Output "SKIPPED" + - name: Checkout + uses: actions/checkout@v4 + id: checkout + if: needs.file-check.outputs.run == 'true' + with: + submodules: recursive + lfs: true + - name: Set Up Go + id: golang + if: needs.file-check.outputs.run == 'true' + uses: actions/setup-go@v5 + with: + go-version: "^1.22" + - name: Set Up Dependencies + id: deps + if: needs.file-check.outputs.run == 'true' + run: ./packaging/windows/install-dependencies.ps1 + - name: Build Netdata + id: build + if: needs.file-check.outputs.run == 'true' + env: + BUILD_DIR: ${{ github.workspace }}\build + run: ./packaging/windows/build.ps1 + - name: Sign Agent Code + id: sign-agent + if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' + uses: azure/trusted-signing-action@v0.4.0 + with: + azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} + azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} + azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} + endpoint: "https://eus.codesigning.azure.net/" + trusted-signing-account-name: Netdata + certificate-profile-name: Netdata + files-folder: ${{ github.workspace }}\build + files-folder-filter: exe,dll + files-folder-recurse: true + file-digest: SHA256 + timestamp-rfc3161: "http://timestamp.acs.microsoft.com" + timestamp-digest: SHA256 + - name: Package Netdata + id: package + if: needs.file-check.outputs.run == 'true' + env: + BUILD_DIR: ${{ github.workspace }}\build + run: ./packaging/windows/package.ps1 + - name: Sign Installer + id: sign-installer + if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' + uses: azure/trusted-signing-action@v0.4.0 + with: + azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} + azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} + azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} + endpoint: "https://eus.codesigning.azure.net/" + trusted-signing-account-name: Netdata + certificate-profile-name: Netdata + files: ${{ github.workspace }}\packaging\windows\netdata-installer.exe + file-digest: SHA256 + timestamp-rfc3161: "http://timestamp.acs.microsoft.com" + timestamp-digest: SHA256 + - name: Upload Installer + id: upload + uses: actions/upload-artifact@v4 + with: + name: windows-x86_64-installer + path: packaging\windows\netdata-installer.exe + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Windows build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed. + Checkout: ${{ steps.checkout.outcome }} + Set Up Dependencies: ${{ steps.deps.outcome }} + Build Netdata: ${{ steps.build.outcome }} + Sign Agent Code: ${{ steps.sign-agent.outcome }} + Package Netdata: ${{ steps.package.outcome }} + Sign Installer: ${{ steps.sign-installer.outcome }} + Upload Installer: ${{ steps.upload.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + && github.repository == 'netdata/netdata' + && needs.file-check.outputs.run == 'true' + }} + prepare-upload: # Consolidate the artifacts for uploading or releasing. name: Prepare Artifacts runs-on: ubuntu-latest needs: - build-dist - build-static + - windows-build - file-check steps: - name: Skip Check @@ -307,6 +413,18 @@ jobs: merge-multiple: true attempt_limit: 3 attempt_delay: 2000 + - name: Retrieve Windows Artifacts + id: fetch-windows + if: needs.file-check.outputs.run == 'true' && github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' + uses: Wandalen/wretry.action@v3 + with: + action: actions/download-artifact@v4 + with: | + pattern: windows-*-installer + path: dist-artifacts + merge-multiple: true + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare Artifacts id: consolidate if: needs.file-check.outputs.run == 'true' @@ -338,7 +456,8 @@ jobs: ${{ github.repository }}: Failed to prepare release artifacts for upload. Checkout: ${{ steps.checkout.outcome }} Prepare environment: ${{ steps.prepare.outcome }} - Fetch dist tarball: ${{ steps.fetch-dist.outcome }} + Fetch dist artifacts: ${{ steps.fetch-dist.outcome }} + Fetch Windows installers: ${{ steps.fetch-windows.outcome }} Consolidate artifacts: ${{ steps.consolidate.outcome }} Store: ${{ steps.store.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -489,68 +608,12 @@ jobs: && needs.file-check.outputs.run == 'true' }} - upload-nightly: # Upload the nightly build artifacts to GCS. - name: Upload Nightly Artifacts - runs-on: ubuntu-latest - if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata' - needs: - - artifact-verification-dist - - artifact-verification-static - steps: - - name: Retrieve Artifacts - id: fetch - uses: Wandalen/wretry.action@v3 - with: - action: actions/download-artifact@v4 - with: | - name: final-artifacts - path: final-artifacts - attempt_limit: 3 - attempt_delay: 2000 - - name: Authenticate to GCS - id: gcs-auth - uses: google-github-actions/auth@v2 - with: - project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} - credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }} - - name: Setup GCS - id: gcs-setup - uses: google-github-actions/setup-gcloud@v2.1.1 - - name: Upload Artifacts - id: upload - uses: google-github-actions/upload-cloud-storage@v2.2.0 - with: - destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} - gzip: false - path: ./final-artifacts/latest-version.txt - parent: false - - name: Failure Notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_COLOR: 'danger' - SLACK_FOOTER: '' - SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Failed to upload nightly release artifacts:' - SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: |- - ${{ github.repository }}: Failed to upload nightly release artifacts. - Fetch artifacts: ${{ steps.fetch.outcome }} - Authenticatie GCS: ${{ steps.gcs-auth.outcome }} - Setup GCS: ${{ steps.gcs-setup.outcome }} - Upload artifacts: ${{ steps.upload.outcome }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} - if: >- - ${{ - failure() - && startsWith(github.ref, 'refs/heads/master') - && github.event_name != 'pull_request' - }} - create-nightly: # Create a nightly build release in netdata/netdata-nightlies name: Create Nightly Release runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata' needs: + - prepare-upload - artifact-verification-dist - artifact-verification-static steps: @@ -955,8 +1018,7 @@ jobs: Checkout: ${{ steps.checkout.outcome }} Fetch test environment: ${{ steps.fetch.outcome }} Load test environment: ${{ steps.load.outcome }} - netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }} - netdata-installer, require cloud: ${{ steps.build-cloud.outcome }} + netdata-installer: ${{ steps.build-cloud.outcome }} netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- @@ -1040,112 +1102,6 @@ jobs: && github.repository == 'netdata/netdata' }} - windows-build: # Test building on Windows - name: Test building on Windows - runs-on: windows-latest - if: github.event_name != 'workflow_dispatch' - needs: - - file-check - steps: - - name: Skip Check - id: skip - if: needs.file-check.outputs.run != 'true' - run: Write-Output "SKIPPED" - - name: Checkout - uses: actions/checkout@v4 - id: checkout - if: needs.file-check.outputs.run == 'true' - with: - submodules: recursive - lfs: true - - name: Set Up Go - id: golang - if: needs.file-check.outputs.run == 'true' - uses: actions/setup-go@v5 - with: - go-version: "^1.22" - - name: Set Up Dependencies - id: deps - if: needs.file-check.outputs.run == 'true' - run: ./packaging/windows/install-dependencies.ps1 - - name: Build Netdata - id: build - if: needs.file-check.outputs.run == 'true' - env: - BUILD_DIR: ${{ github.workspace }}\build - run: ./packaging/windows/build.ps1 - - name: Sign Agent Code - id: sign-agent - if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' - uses: azure/trusted-signing-action@v0.4.0 - with: - azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} - azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} - azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} - endpoint: "https://eus.codesigning.azure.net/" - trusted-signing-account-name: Netdata - certificate-profile-name: Netdata - files-folder: ${{ github.workspace }}\build - files-folder-filter: exe,dll - files-folder-recurse: true - file-digest: SHA256 - timestamp-rfc3161: "http://timestamp.acs.microsoft.com" - timestamp-digest: SHA256 - - name: Package Netdata - id: package - if: needs.file-check.outputs.run == 'true' - env: - BUILD_DIR: ${{ github.workspace }}\build - run: ./packaging/windows/package.ps1 - - name: Sign Installer - id: sign-installer - if: needs.file-check.outputs.run == 'true' && github.event_name != 'pull_request' - uses: azure/trusted-signing-action@v0.4.0 - with: - azure-tenant-id: ${{ secrets.CODE_SIGNING_TENNANT_ID }} - azure-client-id: ${{ secrets.CODE_SIGNING_CLIENT_ID }} - azure-client-secret: ${{ secrets.CODE_SIGNING_CLIENT_SECRET }} - endpoint: "https://eus.codesigning.azure.net/" - trusted-signing-account-name: Netdata - certificate-profile-name: Netdata - files: ${{ github.workspace }}\packaging\windows\netdata-installer.exe - file-digest: SHA256 - timestamp-rfc3161: "http://timestamp.acs.microsoft.com" - timestamp-digest: SHA256 - - name: Upload Installer - id: upload - uses: actions/upload-artifact@v4 - with: - name: windows-x86_64-installer - path: packaging\windows\netdata-installer.exe - retention-days: 30 - - name: Failure Notification - uses: rtCamp/action-slack-notify@v2 - env: - SLACK_COLOR: 'danger' - SLACK_FOOTER: '' - SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Windows build failed:' - SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: |- - ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed. - Checkout: ${{ steps.checkout.outcome }} - Set Up Dependencies: ${{ steps.deps.outcome }} - Build Netdata: ${{ steps.build.outcome }} - Sign Agent Code: ${{ steps.sign-agent.outcome }} - Package Netdata: ${{ steps.package.outcome }} - Sign Installer: ${{ steps.sign-installer.outcome }} - Upload Installer: ${{ steps.upload.outcome }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} - if: >- - ${{ - failure() - && startsWith(github.ref, 'refs/heads/master') - && github.event_name != 'pull_request' - && github.repository == 'netdata/netdata' - && needs.file-check.outputs.run == 'true' - }} - updater-check: # Test the generated dist archive using the updater code. name: Test Generated Distfile and Updater Code runs-on: ubuntu-latest From 3f452446ac8c82d66a697556f7d43baf4de022e6 Mon Sep 17 00:00:00 2001 From: netdatabot Date: Tue, 24 Sep 2024 00:19:29 +0000 Subject: [PATCH 08/17] [ci skip] Update changelog and version for nightly build: v1.99.0-186-nightly. --- CHANGELOG.md | 7 +++---- packaging/version | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b89d0c4814d9bd..fe138117bf3b2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,13 @@ **Merged pull requests:** +- Publish Windows installers on nightly builds. [\#18603](https://github.com/netdata/netdata/pull/18603) ([Ferroin](https://github.com/Ferroin)) +- Bump github.com/docker/docker from 27.3.0+incompatible to 27.3.1+incompatible in /src/go [\#18600](https://github.com/netdata/netdata/pull/18600) ([dependabot[bot]](https://github.com/apps/dependabot)) - Regenerate integrations.js [\#18598](https://github.com/netdata/netdata/pull/18598) ([netdatabot](https://github.com/netdatabot)) - go.d/sensors fix meta [\#18597](https://github.com/netdata/netdata/pull/18597) ([ilyam8](https://github.com/ilyam8)) - go.d/sensors update meta [\#18595](https://github.com/netdata/netdata/pull/18595) ([ilyam8](https://github.com/ilyam8)) - Regenerate integrations.js [\#18592](https://github.com/netdata/netdata/pull/18592) ([netdatabot](https://github.com/netdatabot)) +- Add DLLs to CmakeLists.txt [\#18590](https://github.com/netdata/netdata/pull/18590) ([thiagoftsm](https://github.com/thiagoftsm)) - Bump go.mongodb.org/mongo-driver from 1.16.1 to 1.17.0 in /src/go [\#18589](https://github.com/netdata/netdata/pull/18589) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/docker/docker from 27.2.1+incompatible to 27.3.0+incompatible in /src/go [\#18588](https://github.com/netdata/netdata/pull/18588) ([dependabot[bot]](https://github.com/apps/dependabot)) - Update kickstart.sh [\#18587](https://github.com/netdata/netdata/pull/18587) ([eya46](https://github.com/eya46)) @@ -415,10 +418,6 @@ - Regenerate integrations.js [\#18038](https://github.com/netdata/netdata/pull/18038) ([netdatabot](https://github.com/netdatabot)) - docs: fix go.d/weblog parser config [\#18037](https://github.com/netdata/netdata/pull/18037) ([ilyam8](https://github.com/ilyam8)) - fix diskspace plugin in Docker [\#18035](https://github.com/netdata/netdata/pull/18035) ([ilyam8](https://github.com/ilyam8)) -- Bump repository config fetched by kickstart to latest version. [\#18034](https://github.com/netdata/netdata/pull/18034) ([Ferroin](https://github.com/Ferroin)) -- fix installing netdata-updater svc/timer for native packages [\#18032](https://github.com/netdata/netdata/pull/18032) ([ilyam8](https://github.com/ilyam8)) -- Troubleshooter must be assigned to rooms docs [\#18031](https://github.com/netdata/netdata/pull/18031) ([Ancairon](https://github.com/Ancairon)) -- Regenerate integrations.js [\#18030](https://github.com/netdata/netdata/pull/18030) ([netdatabot](https://github.com/netdatabot)) ## [v1.46.3](https://github.com/netdata/netdata/tree/v1.46.3) (2024-07-23) diff --git a/packaging/version b/packaging/version index 3ec991da57b7a4..e877a4595ba13d 100644 --- a/packaging/version +++ b/packaging/version @@ -1 +1 @@ -v1.99.0-182-nightly +v1.99.0-186-nightly From afefbe7d5982218e659597e2854043c03ec24477 Mon Sep 17 00:00:00 2001 From: Fotis Voutsas Date: Tue, 24 Sep 2024 13:34:54 +0300 Subject: [PATCH 09/17] Add ceph commands to ndsudo (#18583) --- src/collectors/plugins.d/ndsudo.c | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/collectors/plugins.d/ndsudo.c b/src/collectors/plugins.d/ndsudo.c index dd3f6d72dca378..e47942a47f1b89 100644 --- a/src/collectors/plugins.d/ndsudo.c +++ b/src/collectors/plugins.d/ndsudo.c @@ -13,6 +13,42 @@ struct command { const char *params; const char *search[MAX_SEARCH]; } allowed_commands[] = { + { + .name = "ceph-osd-perf", + .params = "osd perf --format json ", + .search = + { + [0] = "ceph", + [1] = NULL, + }, + }, + { + .name = "ceph-osd-df", + .params = "osd df --format json ", + .search = + { + [0] = "ceph", + [1] = NULL, + }, + }, + { + .name = "ceph-osd-pool-stats", + .params = "osd pool stats --format json ", + .search = + { + [0] = "ceph", + [1] = NULL, + }, + }, + { + .name = "ceph-df", + .params = "df --format json", + .search = + { + [0] = "ceph", + [1] = NULL, + }, + }, { .name = "varnishadm-backend-list", .params = "backend.list", From 9fee3df924ac3cf38def297614568531e2ff5828 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 24 Sep 2024 13:57:24 +0300 Subject: [PATCH 10/17] WEB SERVER: retry sending data when errno is EAGAIN (#18607) retry sending data when errno is EAGAIN --- src/web/server/web_client.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/web/server/web_client.c b/src/web/server/web_client.c index d65f068b433da4..1cf0b880b2487a 100644 --- a/src/web/server/web_client.c +++ b/src/web/server/web_client.c @@ -818,21 +818,28 @@ HTTP_VALIDATION http_request_validate(struct web_client *w) { static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,size_t len, int flags) { - ssize_t bytes; - if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) { - if (SSL_connection(&w->ssl)) { - bytes = netdata_ssl_write(&w->ssl, buf, len) ; - web_client_enable_wait_from_ssl(w); - } + do { + errno_clear(); + + ssize_t bytes; + if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) { + if (SSL_connection(&w->ssl)) { + bytes = netdata_ssl_write(&w->ssl, buf, len); + web_client_enable_wait_from_ssl(w); + } else + bytes = send(w->ofd, buf, len, flags); + } else if (web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) + bytes = send(w->ofd, buf, len, flags); else - bytes = send(w->ofd,buf, len , flags); - } - else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) - bytes = send(w->ofd,buf, len , flags); - else - bytes = -999; + bytes = -999; - return bytes; + if(bytes < 0 && errno == EAGAIN) { + tinysleep(); + continue; + } + + return bytes; + } while(true); } void web_client_build_http_header(struct web_client *w) { From c2f8863550461b1abbcf568cb64410cc955c0934 Mon Sep 17 00:00:00 2001 From: "Austin S. Hemmelgarn" Date: Tue, 24 Sep 2024 09:26:28 -0400 Subject: [PATCH 11/17] Correctly include Windows installer in release creation. (#18609) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f17a214351cee5..835cc3b7e9bab3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -652,7 +652,7 @@ jobs: with: allowUpdates: false artifactErrorsFailBuild: true - artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js' + artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.exe,final-artifacts/integrations.js' owner: netdata repo: netdata-nightlies body: Netdata nightly build for ${{ steps.version.outputs.date }}. @@ -770,7 +770,7 @@ jobs: with: allowUpdates: false artifactErrorsFailBuild: true - artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js' + artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/netdata-*.exe,final-artifacts/integrations.js' draft: true tag: ${{ needs.normalize-tag.outputs.tag }} token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} From cc6ff35685ad66ea6f8ed46446a16c1a6e606198 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Tue, 24 Sep 2024 17:05:30 +0300 Subject: [PATCH 12/17] POST Functions (#18611) * fix functions payload parsing * remove raw facets field --- .../systemd-journal.plugin/systemd-journal.c | 2 +- .../windows-events.plugin/windows-events.c | 2 +- src/libnetdata/facets/facets.c | 47 ++++++++++++++----- src/libnetdata/facets/facets.h | 5 +- src/libnetdata/facets/logs_query_status.h | 17 +++++-- .../functions_evloop/functions_evloop.c | 10 +++- src/web/api/v1/api_v1_function.c | 2 +- 7 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/collectors/systemd-journal.plugin/systemd-journal.c b/src/collectors/systemd-journal.plugin/systemd-journal.c index e26d6829901d32..56d37b807fa293 100644 --- a/src/collectors/systemd-journal.plugin/systemd-journal.c +++ b/src/collectors/systemd-journal.plugin/systemd-journal.c @@ -1166,7 +1166,7 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s LOGS_QUERY_STATUS tmp_fqs = { .facets = lqs_facets_create( LQS_DEFAULT_ITEMS_PER_QUERY, - FACETS_OPTION_ALL_KEYS_FTS, + FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS, SYSTEMD_ALWAYS_VISIBLE_KEYS, SYSTEMD_KEYS_INCLUDED_IN_FACETS, SYSTEMD_KEYS_EXCLUDED_FROM_FACETS, diff --git a/src/collectors/windows-events.plugin/windows-events.c b/src/collectors/windows-events.plugin/windows-events.c index 099e83cc94b159..29881c039f7739 100644 --- a/src/collectors/windows-events.plugin/windows-events.c +++ b/src/collectors/windows-events.plugin/windows-events.c @@ -1081,7 +1081,7 @@ void function_windows_events(const char *transaction, char *function, usec_t *st LOGS_QUERY_STATUS tmp_fqs = { .facets = lqs_facets_create( LQS_DEFAULT_ITEMS_PER_QUERY, - FACETS_OPTION_ALL_KEYS_FTS, + FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS, WEVT_ALWAYS_VISIBLE_KEYS, WEVT_KEYS_INCLUDED_IN_FACETS, WEVT_KEYS_EXCLUDED_FROM_FACETS, diff --git a/src/libnetdata/facets/facets.c b/src/libnetdata/facets/facets.c index dd6e99b3560127..632d1655a6f247 100644 --- a/src/libnetdata/facets/facets.c +++ b/src/libnetdata/facets/facets.c @@ -359,6 +359,28 @@ uint32_t facets_rows(FACETS *facets) { return facets->items_to_return; } +static const char *facets_key_id(FACET_KEY *k) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(k->hash); + else + return k->name ? k->name : hash_to_static_string(k->hash); +} + +static const char *facets_key_value_id(FACET_KEY *k, FACET_VALUE *v) { + if(k->facets->options & FACETS_OPTION_HASH_IDS) + return hash_to_static_string(v->hash); + else + return v->name ? v->name : hash_to_static_string(v->hash); +} + +void facets_use_hashes_for_ids(FACETS *facets, bool set) { + netdata_log_info("FACETS_OPTION_HASH_IDS = %s", set ? "true" : "false"); + if(set) + facets->options |= FACETS_OPTION_HASH_IDS; + else + facets->options &= ~(FACETS_OPTION_HASH_IDS); +} + // ---------------------------------------------------------------------------- static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row); @@ -2346,7 +2368,7 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w if (!k->values.enabled) continue; - buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash)); + buffer_json_add_array_item_string(wb, facets_key_id(k)); } foreach_key_in_facets_done(k); } @@ -2526,8 +2548,8 @@ static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) { return ret; } -void facets_table_config(BUFFER *wb) { - buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user +void facets_table_config(FACETS *facets, BUFFER *wb) { + buffer_json_member_add_boolean(wb, "show_ids", (facets->options & FACETS_OPTION_HASH_IDS) ? false : true); buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before buffer_json_member_add_object(wb, "pagination"); @@ -2543,8 +2565,9 @@ void facets_table_config(BUFFER *wb) { void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) { facets->report.used_hashes_registry = used_hashes_registry; + facets_table_config(facets, wb); + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) { - facets_table_config(wb); facets_accepted_parameters_to_json_array(facets, wb, true); } @@ -2577,13 +2600,13 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) buffer_json_add_array_item_object(wb); // key { buffer_json_member_add_string( - wb, "id", hash_to_static_string(k->hash)); + wb, "id", facets_key_id(k)); buffer_json_member_add_string( wb, "name", facets_key_name_cached(k, facets->report.used_hashes_registry)); - buffer_json_member_add_string(wb, "raw", k->name); + // buffer_json_member_add_string(wb, "raw", k->name); if(!k->order) k->order = facets->order++; buffer_json_member_add_uint64(wb, "order", k->order); @@ -2600,11 +2623,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) buffer_json_add_array_item_object(wb); { - buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash)); + buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v)); facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET); buffer_json_member_add_string(wb, "name", buffer_tostring(tb)); - buffer_json_member_add_string(wb, "raw", v->name); + // buffer_json_member_add_string(wb, "raw", v->name); buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter); buffer_json_member_add_uint64(wb, "order", v->order); } @@ -2673,11 +2696,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) if (k->options & FACET_KEY_OPTION_PRETTY_XML) transform = RRDF_FIELD_TRANSFORM_XML; - const char *hash_str = hash_to_static_string(k->hash); + const char *key_id = facets_key_id(k); buffer_rrdf_table_add_field( wb, field_id++, - hash_str, k->name ? k->name : hash_str, + key_id, k->name ? k->name : key_id, RRDF_FIELD_TYPE_STRING, visual, transform, 0, NULL, NAN, RRDF_FIELD_SORT_FIXED, @@ -2771,7 +2794,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) first_histogram_hash = k->hash; buffer_json_add_array_item_object(wb); - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash)); + buffer_json_member_add_string(wb, "id", facets_key_id(k)); buffer_json_member_add_string(wb, "name", k->name); buffer_json_member_add_uint64(wb, "order", k->order); buffer_json_object_close(wb); @@ -2799,7 +2822,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) } if(show_histogram) { - buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : ""); + buffer_json_member_add_string(wb, "id", k ? facets_key_id(k) : ""); buffer_json_member_add_string(wb, "name", k ? k->name : ""); buffer_json_member_add_object(wb, "chart"); { diff --git a/src/libnetdata/facets/facets.h b/src/libnetdata/facets/facets.h index 1d417ee2ff1800..5f14c4e9386856 100644 --- a/src/libnetdata/facets/facets.h +++ b/src/libnetdata/facets/facets.h @@ -88,6 +88,7 @@ typedef enum __attribute__((packed)) { FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6), FACETS_OPTION_SHOW_DELTAS = (1 << 7), + FACETS_OPTION_HASH_IDS = (1 << 8), // when set, the id of the facets, keys and values will be their hash } FACETS_OPTIONS; FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys); @@ -138,7 +139,7 @@ usec_t facets_row_oldest_ut(FACETS *facets); usec_t facets_row_newest_ut(FACETS *facets); uint32_t facets_rows(FACETS *facets); -void facets_table_config(BUFFER *wb); +void facets_table_config(FACETS *facets, BUFFER *wb); const char *facets_severity_to_string(FACET_ROW_SEVERITY severity); @@ -148,4 +149,6 @@ bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_ void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data); void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row); +void facets_use_hashes_for_ids(FACETS *facets, bool set); + #endif diff --git a/src/libnetdata/facets/logs_query_status.h b/src/libnetdata/facets/logs_query_status.h index 88ccfd890adbaa..4be902ea72d1ce 100644 --- a/src/libnetdata/facets/logs_query_status.h +++ b/src/libnetdata/facets/logs_query_status.h @@ -354,6 +354,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_object_get_ex(jobj, LQS_PARAMETER_SOURCE, &sources)) { if (json_object_get_type(sources) != json_type_array) { buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_SOURCE); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an array", LQS_PARAMETER_SOURCE); return false; } @@ -369,6 +370,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_get_type(src) != json_type_string) { buffer_sprintf(error, "sources array item %zu is not a string", i); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: sources array item %zu is not a string", i); return false; } @@ -401,6 +403,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_object_get_ex(jobj, LQS_PARAMETER_FACETS, &fcts)) { if (json_object_get_type(sources) != json_type_array) { buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_FACETS); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an array", LQS_PARAMETER_FACETS); return false; } @@ -415,6 +418,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_get_type(fct) != json_type_string) { buffer_sprintf(error, "facets array item %zu is not a string", i); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: facets array item %zu is not a string", i); return false; } @@ -430,6 +434,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_object_get_ex(jobj, "selections", &selections)) { if (json_object_get_type(selections) != json_type_object) { buffer_sprintf(error, "member 'selections' is not an object"); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an object", "selections"); return false; } @@ -438,6 +443,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char json_object_object_foreach(selections, key, val) { if (json_object_get_type(val) != json_type_array) { buffer_sprintf(error, "selection '%s' is not an array", key); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' is not an array", key); return false; } @@ -449,6 +455,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char if (json_object_get_type(value_obj) != json_type_string) { buffer_sprintf(error, "selection '%s' array item %zu is not a string", key, i); + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' array item %zu is not a string", key, i); return false; } @@ -468,6 +475,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char buffer_json_object_close(wb); // selections } + facets_use_hashes_for_ids(facets, false); rq->fields_are_ids = false; return true; } @@ -476,6 +484,8 @@ static inline bool lqs_request_parse_POST(LOGS_QUERY_STATUS *lqs, BUFFER *wb, BU FACETS *facets = lqs->facets; LOGS_QUERY_REQUEST *rq = &lqs->rq; + buffer_json_member_add_object(wb, "_request"); + struct logs_query_data qd = { .transaction = transaction, .facets = facets, @@ -498,7 +508,7 @@ static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, cha buffer_json_member_add_object(wb, "_request"); char func_copy[strlen(function) + 1]; - strcpy(func_copy, function); + memcpy(func_copy, function, sizeof(func_copy)); char *words[LQS_MAX_PARAMS] = { NULL }; size_t num_words = quoted_strings_splitter_pluginsd(func_copy, words, LQS_MAX_PARAMS); @@ -661,6 +671,7 @@ static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, cha } } + facets_use_hashes_for_ids(facets, true); rq->fields_are_ids = true; return true; } @@ -689,7 +700,7 @@ static inline void lqs_info_response(BUFFER *wb, FACETS *facets) { } buffer_json_array_close(wb); // required_params array - facets_table_config(wb); + facets_table_config(facets, wb); buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK); buffer_json_member_add_string(wb, "type", "table"); @@ -702,7 +713,7 @@ static inline void lqs_info_response(BUFFER *wb, FACETS *facets) { static inline BUFFER *lqs_create_output_buffer(void) { BUFFER *wb = buffer_create(0, NULL); - buffer_flush(wb); + buffer_reset(wb); buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY); return wb; } diff --git a/src/libnetdata/functions_evloop/functions_evloop.c b/src/libnetdata/functions_evloop/functions_evloop.c index 5000d038f9c213..e8e61c4c1a8c3a 100644 --- a/src/libnetdata/functions_evloop/functions_evloop.c +++ b/src/libnetdata/functions_evloop/functions_evloop.c @@ -137,6 +137,8 @@ static void worker_add_job(struct functions_evloop_globals *wg, const char *keyw function?function:"(unset)"); } else { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "WORKER JOB WITH PAYLOAD '%s'", payload ? buffer_tostring(payload) : "NONE"); + int timeout = str2i(timeout_s); const char *msg = "No function with this name found"; @@ -222,6 +224,8 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { char *s = (char *)buffer_tostring(buffer); if(strstr(&s[deferred.last_len], PLUGINSD_CALL_FUNCTION_PAYLOAD_END "\n") != NULL) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD END"); + if(deferred.last_len > 0) // remove the trailing newline from the buffer deferred.last_len--; @@ -254,6 +258,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { const char *keyword = get_word(words, num_words, 0); if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION) == 0)) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CALL"); char *transaction = get_word(words, num_words, 1); char *timeout_s = get_word(words, num_words, 2); char *function = get_word(words, num_words, 3); @@ -262,6 +267,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { worker_add_job(wg, keyword, transaction, function, timeout_s, NULL, access, source); } else if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION_PAYLOAD_BEGIN) == 0)) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD CALL"); char *transaction = get_word(words, num_words, 1); char *timeout_s = get_word(words, num_words, 2); char *function = get_word(words, num_words, 3); @@ -279,6 +285,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { deferred.enabled = true; } else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_CANCEL) == 0) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CANCEL"); char *transaction = get_word(words, num_words, 1); const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); if(acquired) { @@ -292,6 +299,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received CANCEL for transaction '%s', but it not available here", transaction); } else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_PROGRESS) == 0) { + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PROGRESS"); char *transaction = get_word(words, num_words, 1); const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction); if(acquired) { @@ -305,7 +313,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) { nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received PROGRESS for transaction '%s', but it not available here", transaction); } else - nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)"); + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword ? keyword : "(unset)"); buffer_flush(buffer); } diff --git a/src/web/api/v1/api_v1_function.c b/src/web/api/v1/api_v1_function.c index 495f7494b839e4..761164fd06746c 100644 --- a/src/web/api/v1/api_v1_function.c +++ b/src/web/api/v1/api_v1_function.c @@ -39,6 +39,6 @@ int api_v1_function(RRDHOST *host, struct web_client *w, char *url) { return rrd_function_run(host, wb, timeout, w->access, function, true, transaction, NULL, NULL, web_client_progress_functions_update, w, - web_client_interrupt_callback, w, NULL, + web_client_interrupt_callback, w, w->payload, buffer_tostring(source), false); } From 028ee8c7169a94639e1efd7c04b7379dc098aaa2 Mon Sep 17 00:00:00 2001 From: netdatabot Date: Wed, 25 Sep 2024 00:20:15 +0000 Subject: [PATCH 13/17] [ci skip] Update changelog and version for nightly build: v1.99.0-191-nightly. --- CHANGELOG.md | 15 +++++++++------ packaging/version | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe138117bf3b2d..cd6d2658679a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,13 @@ ## [**Next release**](https://github.com/netdata/netdata/tree/HEAD) -[Full Changelog](https://github.com/netdata/netdata/compare/v1.47.1...HEAD) +[Full Changelog](https://github.com/netdata/netdata/compare/v1.47.2...HEAD) **Merged pull requests:** +- POST Functions [\#18611](https://github.com/netdata/netdata/pull/18611) ([ktsaou](https://github.com/ktsaou)) +- Correctly include Windows installer in release creation. [\#18609](https://github.com/netdata/netdata/pull/18609) ([Ferroin](https://github.com/Ferroin)) +- WEB SERVER: retry sending data when errno is EAGAIN [\#18607](https://github.com/netdata/netdata/pull/18607) ([ktsaou](https://github.com/ktsaou)) - Publish Windows installers on nightly builds. [\#18603](https://github.com/netdata/netdata/pull/18603) ([Ferroin](https://github.com/Ferroin)) - Bump github.com/docker/docker from 27.3.0+incompatible to 27.3.1+incompatible in /src/go [\#18600](https://github.com/netdata/netdata/pull/18600) ([dependabot[bot]](https://github.com/apps/dependabot)) - Regenerate integrations.js [\#18598](https://github.com/netdata/netdata/pull/18598) ([netdatabot](https://github.com/netdatabot)) @@ -16,6 +19,7 @@ - Bump go.mongodb.org/mongo-driver from 1.16.1 to 1.17.0 in /src/go [\#18589](https://github.com/netdata/netdata/pull/18589) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/docker/docker from 27.2.1+incompatible to 27.3.0+incompatible in /src/go [\#18588](https://github.com/netdata/netdata/pull/18588) ([dependabot[bot]](https://github.com/apps/dependabot)) - Update kickstart.sh [\#18587](https://github.com/netdata/netdata/pull/18587) ([eya46](https://github.com/eya46)) +- Add ceph commands to ndsudo [\#18583](https://github.com/netdata/netdata/pull/18583) ([Ancairon](https://github.com/Ancairon)) - go.d/sensors refactor [\#18581](https://github.com/netdata/netdata/pull/18581) ([ilyam8](https://github.com/ilyam8)) - go.d move packages [\#18580](https://github.com/netdata/netdata/pull/18580) ([ilyam8](https://github.com/ilyam8)) - WEIGHTS: use node\_id when available, otherwise host\_id [\#18579](https://github.com/netdata/netdata/pull/18579) ([ktsaou](https://github.com/ktsaou)) @@ -158,6 +162,10 @@ - fix warnings in Dockerfile [\#18395](https://github.com/netdata/netdata/pull/18395) ([NicolasCARPi](https://github.com/NicolasCARPi)) - Use existing ACLK event loop for cloud queries [\#18218](https://github.com/netdata/netdata/pull/18218) ([stelfrag](https://github.com/stelfrag)) +## [v1.47.2](https://github.com/netdata/netdata/tree/v1.47.2) (2024-09-24) + +[Full Changelog](https://github.com/netdata/netdata/compare/v1.47.1...v1.47.2) + ## [v1.47.1](https://github.com/netdata/netdata/tree/v1.47.1) (2024-09-10) [Full Changelog](https://github.com/netdata/netdata/compare/v1.99.0...v1.47.1) @@ -413,11 +421,6 @@ - Bump github.com/likexian/whois-parser from 1.24.16 to 1.24.18 in /src/go/collectors/go.d.plugin [\#18046](https://github.com/netdata/netdata/pull/18046) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/gofrs/flock from 0.8.1 to 0.11.0 in /src/go/collectors/go.d.plugin [\#18045](https://github.com/netdata/netdata/pull/18045) ([dependabot[bot]](https://github.com/apps/dependabot)) - Semaphore \(common context\) [\#18041](https://github.com/netdata/netdata/pull/18041) ([thiagoftsm](https://github.com/thiagoftsm)) -- proc/diskstats: Increase accuracy of average IO operation time [\#18040](https://github.com/netdata/netdata/pull/18040) ([ilyam8](https://github.com/ilyam8)) -- diskspace: update exclude paths/filesystems [\#18039](https://github.com/netdata/netdata/pull/18039) ([ilyam8](https://github.com/ilyam8)) -- Regenerate integrations.js [\#18038](https://github.com/netdata/netdata/pull/18038) ([netdatabot](https://github.com/netdatabot)) -- docs: fix go.d/weblog parser config [\#18037](https://github.com/netdata/netdata/pull/18037) ([ilyam8](https://github.com/ilyam8)) -- fix diskspace plugin in Docker [\#18035](https://github.com/netdata/netdata/pull/18035) ([ilyam8](https://github.com/ilyam8)) ## [v1.46.3](https://github.com/netdata/netdata/tree/v1.46.3) (2024-07-23) diff --git a/packaging/version b/packaging/version index e877a4595ba13d..8347849e122cc0 100644 --- a/packaging/version +++ b/packaging/version @@ -1 +1 @@ -v1.99.0-186-nightly +v1.99.0-191-nightly From 99eed4690a35890a746b4f986e4d7de95a5c1d97 Mon Sep 17 00:00:00 2001 From: Mateusz Bularz <60339703+M4itee@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:49:48 +0200 Subject: [PATCH 14/17] feat: HW req for onprem installation. (#18608) * feat: HW req for onprem installation. * Apply suggestions from code review * Update docs/netdata-cloud/netdata-cloud-on-prem/installation.md Co-authored-by: Fotis Voutsas * Apply suggestions from code review * small changes in wording --------- Co-authored-by: Fotis Voutsas --- .../netdata-cloud-on-prem/installation.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/netdata-cloud/netdata-cloud-on-prem/installation.md b/docs/netdata-cloud/netdata-cloud-on-prem/installation.md index 259ddb5ced1efe..28a0b12ee3cc75 100644 --- a/docs/netdata-cloud/netdata-cloud-on-prem/installation.md +++ b/docs/netdata-cloud/netdata-cloud-on-prem/installation.md @@ -10,6 +10,18 @@ The following components are required to install Netdata Cloud On-Prem: - **Helm** version 3.12+ with OCI Configuration (explained in the installation section) - **Kubectl** +The minimum requirements for Netdata-Cloud are: +- 4 CPU cores +- 15GiB of memory +- Cloud services are ephemeral + +The requirements for the non-production Dependencies helm chart: +- 8 CPU cores +- 14GiB of memory +- 160GiB for PVCs (SSD) + +> **_NOTE:_** Values for each component may vary depending on the type of load. The most compute-intensive task that the On-Prem needs to perform is the initial sync of directly connected Agents. The testing for these requirements was conducted with 1,000 nodes directly connected to the On-Prem. If you plan on spawning hundreds of new nodes within a few minutes, Postgres will be the first bottleneck. For example, a 2 vCPU / 8 GiB memory / 1k IOPS database can handle 1,000 nodes without any problems if your environment is fairly steady, adding nodes in batches of 10-30 (directly connected). + ## Preparations for Installation ### Configure AWS CLI From a89facce8142fed8221b0e52fdccdc3a2fb488ac Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Wed, 25 Sep 2024 16:48:20 +0300 Subject: [PATCH 15/17] go.d/uwsgi: don't write just connect and read (#18618) --- src/go/plugin/go.d/modules/uwsgi/client.go | 33 +++++++------- src/go/plugin/go.d/modules/uwsgi/collect.go | 19 +------- src/go/plugin/go.d/modules/uwsgi/uwsgi.go | 5 ++- .../plugin/go.d/modules/uwsgi/uwsgi_test.go | 45 +++---------------- src/go/plugin/go.d/pkg/socket/client.go | 9 ++++ 5 files changed, 34 insertions(+), 77 deletions(-) diff --git a/src/go/plugin/go.d/modules/uwsgi/client.go b/src/go/plugin/go.d/modules/uwsgi/client.go index 4036807434c9e5..487aeb930784b8 100644 --- a/src/go/plugin/go.d/modules/uwsgi/client.go +++ b/src/go/plugin/go.d/modules/uwsgi/client.go @@ -5,35 +5,25 @@ package uwsgi import ( "bytes" "fmt" + "time" "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/socket" ) type uwsgiConn interface { - connect() error - disconnect() queryStats() ([]byte, error) } func newUwsgiConn(conf Config) uwsgiConn { - return &uwsgiClient{conn: socket.New(socket.Config{ - Address: conf.Address, - ConnectTimeout: conf.Timeout.Duration(), - ReadTimeout: conf.Timeout.Duration(), - WriteTimeout: conf.Timeout.Duration(), - })} + return &uwsgiClient{ + address: conf.Address, + timeout: conf.Timeout.Duration(), + } } type uwsgiClient struct { - conn socket.Client -} - -func (c *uwsgiClient) connect() error { - return c.conn.Connect() -} - -func (c *uwsgiClient) disconnect() { - _ = c.conn.Disconnect() + address string + timeout time.Duration } func (c *uwsgiClient) queryStats() ([]byte, error) { @@ -42,7 +32,14 @@ func (c *uwsgiClient) queryStats() ([]byte, error) { var err error const readLineLimit = 1000 * 10 - clientErr := c.conn.Command("", func(bs []byte) bool { + cfg := socket.Config{ + Address: c.address, + ConnectTimeout: c.timeout, + ReadTimeout: c.timeout, + WriteTimeout: c.timeout, + } + + clientErr := socket.ConnectAndRead(cfg, func(bs []byte) bool { b.Write(bs) b.WriteByte('\n') diff --git a/src/go/plugin/go.d/modules/uwsgi/collect.go b/src/go/plugin/go.d/modules/uwsgi/collect.go index 3f440535435e96..a89704c81988a1 100644 --- a/src/go/plugin/go.d/modules/uwsgi/collect.go +++ b/src/go/plugin/go.d/modules/uwsgi/collect.go @@ -27,14 +27,7 @@ type workerStats struct { } func (u *Uwsgi) collect() (map[string]int64, error) { - conn, err := u.establishConn() - if err != nil { - return nil, fmt.Errorf("failed to connect: %v", err) - } - - defer conn.disconnect() - - stats, err := conn.queryStats() + stats, err := u.conn.queryStats() if err != nil { return nil, fmt.Errorf("failed to query stats: %v", err) } @@ -110,16 +103,6 @@ func (u *Uwsgi) collectStats(mx map[string]int64, stats []byte) error { return nil } -func (u *Uwsgi) establishConn() (uwsgiConn, error) { - conn := u.newConn(u.Config) - - if err := conn.connect(); err != nil { - return nil, err - } - - return conn, nil -} - func boolToInt(b bool) int64 { if b { return 1 diff --git a/src/go/plugin/go.d/modules/uwsgi/uwsgi.go b/src/go/plugin/go.d/modules/uwsgi/uwsgi.go index 527d8fd13ff6e8..ff247244f2264d 100644 --- a/src/go/plugin/go.d/modules/uwsgi/uwsgi.go +++ b/src/go/plugin/go.d/modules/uwsgi/uwsgi.go @@ -28,7 +28,6 @@ func New() *Uwsgi { Address: "127.0.0.1:1717", Timeout: confopt.Duration(time.Second * 1), }, - newConn: newUwsgiConn, charts: charts.Copy(), seenWorkers: make(map[int]bool), } @@ -46,7 +45,7 @@ type Uwsgi struct { charts *module.Charts - newConn func(Config) uwsgiConn + conn uwsgiConn seenWorkers map[int]bool } @@ -61,6 +60,8 @@ func (u *Uwsgi) Init() error { return errors.New("address not set") } + u.conn = newUwsgiConn(u.Config) + return nil } diff --git a/src/go/plugin/go.d/modules/uwsgi/uwsgi_test.go b/src/go/plugin/go.d/modules/uwsgi/uwsgi_test.go index 900c48538d4f2d..ce3f2125550374 100644 --- a/src/go/plugin/go.d/modules/uwsgi/uwsgi_test.go +++ b/src/go/plugin/go.d/modules/uwsgi/uwsgi_test.go @@ -81,7 +81,7 @@ func TestUwsgi_Cleanup(t *testing.T) { "after check": { prepare: func() *Uwsgi { uw := New() - uw.newConn = func(config Config) uwsgiConn { return prepareMockOk() } + uw.conn = prepareMockOk() _ = uw.Check() return uw }, @@ -89,7 +89,7 @@ func TestUwsgi_Cleanup(t *testing.T) { "after collect": { prepare: func() *Uwsgi { uw := New() - uw.newConn = func(config Config) uwsgiConn { return prepareMockOk() } + uw.conn = prepareMockOk() _ = uw.Collect() return uw }, @@ -122,10 +122,6 @@ func TestUwsgi_Check(t *testing.T) { wantFail: false, prepareMock: prepareMockOkNoWorkers, }, - "err on connect": { - wantFail: true, - prepareMock: prepareMockErrOnConnect, - }, "unexpected response": { wantFail: true, prepareMock: prepareMockUnexpectedResponse, @@ -140,7 +136,7 @@ func TestUwsgi_Check(t *testing.T) { t.Run(name, func(t *testing.T) { uw := New() mock := test.prepareMock() - uw.newConn = func(config Config) uwsgiConn { return mock } + uw.conn = mock if test.wantFail { assert.Error(t, uw.Check()) @@ -229,12 +225,6 @@ func TestUwsgi_Collect(t *testing.T) { disconnectBeforeCleanup: true, disconnectAfterCleanup: true, }, - "err on connect": { - prepareMock: prepareMockErrOnConnect, - wantCharts: len(charts), - disconnectBeforeCleanup: false, - disconnectAfterCleanup: false, - }, "err on query stats": { prepareMock: prepareMockErrOnQueryStats, wantCharts: len(charts), @@ -247,7 +237,7 @@ func TestUwsgi_Collect(t *testing.T) { t.Run(name, func(t *testing.T) { uw := New() mock := test.prepareMock() - uw.newConn = func(config Config) uwsgiConn { return mock } + uw.conn = mock mx := uw.Collect() @@ -257,10 +247,6 @@ func TestUwsgi_Collect(t *testing.T) { module.TestMetricsHasAllChartsDims(t, uw.Charts(), mx) } assert.Equal(t, test.wantCharts, len(*uw.Charts()), "want charts") - - assert.Equal(t, test.disconnectBeforeCleanup, mock.disconnectCalled, "disconnect before cleanup") - uw.Cleanup() - assert.Equal(t, test.disconnectAfterCleanup, mock.disconnectCalled, "disconnect after cleanup") }) } } @@ -277,12 +263,6 @@ func prepareMockOkNoWorkers() *mockUwsgiConn { } } -func prepareMockErrOnConnect() *mockUwsgiConn { - return &mockUwsgiConn{ - errOnConnect: true, - } -} - func prepareMockErrOnQueryStats() *mockUwsgiConn { return &mockUwsgiConn{ errOnQueryStats: true, @@ -300,21 +280,8 @@ func prepareMockEmptyResponse() *mockUwsgiConn { } type mockUwsgiConn struct { - errOnConnect bool - errOnQueryStats bool - statsResponse []byte - disconnectCalled bool -} - -func (m *mockUwsgiConn) connect() error { - if m.errOnConnect { - return errors.New("mock.connect() error") - } - return nil -} - -func (m *mockUwsgiConn) disconnect() { - m.disconnectCalled = true + errOnQueryStats bool + statsResponse []byte } func (m *mockUwsgiConn) queryStats() ([]byte, error) { diff --git a/src/go/plugin/go.d/pkg/socket/client.go b/src/go/plugin/go.d/pkg/socket/client.go index 26ae1dfa65d9d6..c2bcbd9e9c76de 100644 --- a/src/go/plugin/go.d/pkg/socket/client.go +++ b/src/go/plugin/go.d/pkg/socket/client.go @@ -21,6 +21,15 @@ func New(config Config) *Socket { } } +func ConnectAndRead(config Config, process Processor) error { + s := New(config) + if err := s.Connect(); err != nil { + return err + } + defer func() { _ = s.Disconnect() }() + return read(s.conn, process, s.ReadTimeout) +} + // Socket is the implementation of a socket client. type Socket struct { Config From 732acca7187d8aee3942b2a3114207a95b237374 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Wed, 25 Sep 2024 17:12:11 +0300 Subject: [PATCH 16/17] go.d/hddtemp: connect and read (#18619) --- src/go/plugin/go.d/modules/hddtemp/client.go | 41 +++++++++++-------- src/go/plugin/go.d/modules/hddtemp/collect.go | 10 +---- src/go/plugin/go.d/modules/hddtemp/hddtemp.go | 33 ++++++--------- .../go.d/modules/hddtemp/hddtemp_test.go | 39 ++---------------- 4 files changed, 42 insertions(+), 81 deletions(-) diff --git a/src/go/plugin/go.d/modules/hddtemp/client.go b/src/go/plugin/go.d/modules/hddtemp/client.go index b89be10a269912..f3cd200caa373a 100644 --- a/src/go/plugin/go.d/modules/hddtemp/client.go +++ b/src/go/plugin/go.d/modules/hddtemp/client.go @@ -3,42 +3,49 @@ package hddtemp import ( + "time" + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/socket" ) -func newHddTempConn(conf Config) hddtempConn { - return &hddtempClient{conn: socket.New(socket.Config{ - Address: conf.Address, - ConnectTimeout: conf.Timeout.Duration(), - ReadTimeout: conf.Timeout.Duration(), - WriteTimeout: conf.Timeout.Duration(), - })} +type hddtempConn interface { + queryHddTemp() (string, error) } -type hddtempClient struct { - conn socket.Client -} - -func (c *hddtempClient) connect() error { - return c.conn.Connect() +func newHddTempConn(conf Config) hddtempConn { + return &hddtempClient{ + address: conf.Address, + timeout: conf.Timeout.Duration(), + } } -func (c *hddtempClient) disconnect() { - _ = c.conn.Disconnect() +type hddtempClient struct { + address string + timeout time.Duration } func (c *hddtempClient) queryHddTemp() (string, error) { var i int var s string - err := c.conn.Command("", func(bytes []byte) bool { + + cfg := socket.Config{ + Address: c.address, + ConnectTimeout: c.timeout, + ReadTimeout: c.timeout, + WriteTimeout: c.timeout, + } + + err := socket.ConnectAndRead(cfg, func(bs []byte) bool { if i++; i > 1 { return false } - s = string(bytes) + s = string(bs) return true + }) if err != nil { return "", err } + return s, nil } diff --git a/src/go/plugin/go.d/modules/hddtemp/collect.go b/src/go/plugin/go.d/modules/hddtemp/collect.go index f5c75db041296d..8e0ebf486ef9e5 100644 --- a/src/go/plugin/go.d/modules/hddtemp/collect.go +++ b/src/go/plugin/go.d/modules/hddtemp/collect.go @@ -17,15 +17,7 @@ type diskStats struct { } func (h *HddTemp) collect() (map[string]int64, error) { - conn := h.newHddTempConn(h.Config) - - if err := conn.connect(); err != nil { - return nil, err - } - - defer conn.disconnect() - - msg, err := conn.queryHddTemp() + msg, err := h.conn.queryHddTemp() if err != nil { return nil, err } diff --git a/src/go/plugin/go.d/modules/hddtemp/hddtemp.go b/src/go/plugin/go.d/modules/hddtemp/hddtemp.go index 6b9a522e3da0c2..1c838b273d3695 100644 --- a/src/go/plugin/go.d/modules/hddtemp/hddtemp.go +++ b/src/go/plugin/go.d/modules/hddtemp/hddtemp.go @@ -28,10 +28,9 @@ func New() *HddTemp { Address: "127.0.0.1:7634", Timeout: confopt.Duration(time.Second * 1), }, - newHddTempConn: newHddTempConn, - charts: &module.Charts{}, - disks: make(map[string]bool), - disksTemp: make(map[string]bool), + charts: &module.Charts{}, + disks: make(map[string]bool), + disksTemp: make(map[string]bool), } } @@ -41,25 +40,17 @@ type Config struct { Timeout confopt.Duration `yaml:"timeout" json:"timeout"` } -type ( - HddTemp struct { - module.Base - Config `yaml:",inline" json:""` +type HddTemp struct { + module.Base + Config `yaml:",inline" json:""` - charts *module.Charts + charts *module.Charts - newHddTempConn func(Config) hddtempConn + conn hddtempConn - disks map[string]bool - disksTemp map[string]bool - } - - hddtempConn interface { - connect() error - disconnect() - queryHddTemp() (string, error) - } -) + disks map[string]bool + disksTemp map[string]bool +} func (h *HddTemp) Configuration() any { return h.Config @@ -71,6 +62,8 @@ func (h *HddTemp) Init() error { return errors.New("address not set") } + h.conn = newHddTempConn(h.Config) + return nil } diff --git a/src/go/plugin/go.d/modules/hddtemp/hddtemp_test.go b/src/go/plugin/go.d/modules/hddtemp/hddtemp_test.go index 97f32305773a2f..0194bb0c925d7a 100644 --- a/src/go/plugin/go.d/modules/hddtemp/hddtemp_test.go +++ b/src/go/plugin/go.d/modules/hddtemp/hddtemp_test.go @@ -82,7 +82,7 @@ func TestHddTemp_Cleanup(t *testing.T) { "after check": { prepare: func() *HddTemp { hdd := New() - hdd.newHddTempConn = func(config Config) hddtempConn { return prepareMockAllDisksOk() } + hdd.conn = prepareMockAllDisksOk() _ = hdd.Check() return hdd }, @@ -90,7 +90,7 @@ func TestHddTemp_Cleanup(t *testing.T) { "after collect": { prepare: func() *HddTemp { hdd := New() - hdd.newHddTempConn = func(config Config) hddtempConn { return prepareMockAllDisksOk() } + hdd.conn = prepareMockAllDisksOk() _ = hdd.Collect() return hdd }, @@ -123,10 +123,6 @@ func TestHddTemp_Check(t *testing.T) { wantFail: false, prepareMock: prepareMockAllDisksSleep, }, - "err on connect": { - wantFail: true, - prepareMock: prepareMockErrOnConnect, - }, "unexpected response": { wantFail: true, prepareMock: prepareMockUnexpectedResponse, @@ -140,8 +136,7 @@ func TestHddTemp_Check(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { hdd := New() - mock := test.prepareMock() - hdd.newHddTempConn = func(config Config) hddtempConn { return mock } + hdd.conn = test.prepareMock() if test.wantFail { assert.Error(t, hdd.Check()) @@ -219,10 +214,6 @@ func TestHddTemp_Collect(t *testing.T) { "disk_ata-WDC_WD10EARS-00Y5B1_WD-WCAV5R693922_temp_sensor_status_unk": 0, }, }, - "err on connect": { - prepareMock: prepareMockErrOnConnect, - wantDisconnect: false, - }, "unexpected response": { prepareMock: prepareMockUnexpectedResponse, wantDisconnect: true, @@ -236,8 +227,7 @@ func TestHddTemp_Collect(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { hdd := New() - mock := test.prepareMock() - hdd.newHddTempConn = func(config Config) hddtempConn { return mock } + hdd.conn = test.prepareMock() mx := hdd.Collect() @@ -245,8 +235,6 @@ func TestHddTemp_Collect(t *testing.T) { assert.Len(t, *hdd.Charts(), test.wantCharts, "wantCharts") - assert.Equal(t, test.wantDisconnect, mock.disconnectCalled, "disconnectCalled") - module.TestMetricsHasAllChartsDims(t, hdd.Charts(), mx) }) } @@ -264,12 +252,6 @@ func prepareMockAllDisksSleep() *mockHddTempConn { } } -func prepareMockErrOnConnect() *mockHddTempConn { - return &mockHddTempConn{ - errOnConnect: true, - } -} - func prepareMockUnexpectedResponse() *mockHddTempConn { return &mockHddTempConn{ hddTempLine: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", @@ -283,21 +265,8 @@ func prepareMockEmptyResponse() *mockHddTempConn { } type mockHddTempConn struct { - errOnConnect bool errOnQueryHddTemp bool hddTempLine string - disconnectCalled bool -} - -func (m *mockHddTempConn) connect() error { - if m.errOnConnect { - return errors.New("mock.connect() error") - } - return nil -} - -func (m *mockHddTempConn) disconnect() { - m.disconnectCalled = true } func (m *mockHddTempConn) queryHddTemp() (string, error) { From 3a7401773d7ed4136fc6dbc7f082a62bfa5be6fd Mon Sep 17 00:00:00 2001 From: netdatabot Date: Thu, 26 Sep 2024 00:19:51 +0000 Subject: [PATCH 17/17] [ci skip] Update changelog and version for nightly build: v1.99.0-195-nightly. --- CHANGELOG.md | 5 +++-- packaging/version | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd6d2658679a47..f7e5d4d6b09135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,11 @@ **Merged pull requests:** +- go.d/hddtemp: connect and read [\#18619](https://github.com/netdata/netdata/pull/18619) ([ilyam8](https://github.com/ilyam8)) +- go.d/uwsgi: don't write just connect and read [\#18618](https://github.com/netdata/netdata/pull/18618) ([ilyam8](https://github.com/ilyam8)) - POST Functions [\#18611](https://github.com/netdata/netdata/pull/18611) ([ktsaou](https://github.com/ktsaou)) - Correctly include Windows installer in release creation. [\#18609](https://github.com/netdata/netdata/pull/18609) ([Ferroin](https://github.com/Ferroin)) +- feat: HW req for onprem installation. [\#18608](https://github.com/netdata/netdata/pull/18608) ([M4itee](https://github.com/M4itee)) - WEB SERVER: retry sending data when errno is EAGAIN [\#18607](https://github.com/netdata/netdata/pull/18607) ([ktsaou](https://github.com/ktsaou)) - Publish Windows installers on nightly builds. [\#18603](https://github.com/netdata/netdata/pull/18603) ([Ferroin](https://github.com/Ferroin)) - Bump github.com/docker/docker from 27.3.0+incompatible to 27.3.1+incompatible in /src/go [\#18600](https://github.com/netdata/netdata/pull/18600) ([dependabot[bot]](https://github.com/apps/dependabot)) @@ -419,8 +422,6 @@ - Bump go.mongodb.org/mongo-driver from 1.15.1 to 1.16.0 in /src/go/collectors/go.d.plugin [\#18048](https://github.com/netdata/netdata/pull/18048) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/docker/docker from 27.0.0+incompatible to 27.0.2+incompatible in /src/go/collectors/go.d.plugin [\#18047](https://github.com/netdata/netdata/pull/18047) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump github.com/likexian/whois-parser from 1.24.16 to 1.24.18 in /src/go/collectors/go.d.plugin [\#18046](https://github.com/netdata/netdata/pull/18046) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump github.com/gofrs/flock from 0.8.1 to 0.11.0 in /src/go/collectors/go.d.plugin [\#18045](https://github.com/netdata/netdata/pull/18045) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Semaphore \(common context\) [\#18041](https://github.com/netdata/netdata/pull/18041) ([thiagoftsm](https://github.com/thiagoftsm)) ## [v1.46.3](https://github.com/netdata/netdata/tree/v1.46.3) (2024-07-23) diff --git a/packaging/version b/packaging/version index 8347849e122cc0..9227296c26ecec 100644 --- a/packaging/version +++ b/packaging/version @@ -1 +1 @@ -v1.99.0-191-nightly +v1.99.0-195-nightly