chore: merge upstream development

CrunchyData · Jul 19, 2024 · 6e5112a · 6e5112a
2 parents 635bc1d + 93ed6c0
commit 6e5112a
Show file tree

Hide file tree

Showing 75 changed files with 665 additions and 541 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 *.swp
 exporter/postgres/queries.yml
+hugo/.hugo_build.lock
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@ default_install_hook_types:
   - pre-commit
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.3.0
+  rev: v4.6.0
   hooks:
     - id: check-merge-conflict
     - id: check-symlinks
@@ -15,25 +15,11 @@ repos:
     - id: trailing-whitespace
       args: ['--markdown-linebreak-ext=md']
 - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
-  rev: v9.0.0
+  rev: v9.16.0
   hooks:
     - id: commitlint
       stages: [commit-msg]
       additional_dependencies: ['@commitlint/config-conventional']
-- repo: https://github.com/ansible-community/ansible-lint.git
-  rev: v5.4.0
-    # rev: v5.4.0 will not change until we drop ansible 2.9
-  hooks:
-    - id: ansible-lint
-      always_run: true
-      pass_filenames: false
-      # do not add file filters here as ansible-lint does not give reliable
-      # results when called with individual files.
-      # https://github.com/ansible/ansible-lint/issues/611
-      verbose: true
-      entry: ansible-lint --force-color -p -v .
-      additional_dependencies:
-        - 'ansible<2.10'
 - repo: https://github.com/codespell-project/codespell
   rev: v2.1.0
   hooks:

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -5,6 +5,41 @@ pgMonitor Release Notes
 .. contents:: Topics
 
 
+v5.0.0
+======
+
+Release Summary
+---------------
+
+Crunchy Data is pleased to announce the availability of pgMonitor 5.0.0. This release brings support for a new Prometheus exporter for PostgreSQL - sql_exporter. It also supports a new monitoring extension to make metric collection easier and more performant. This changelog contains all changes that have been added since the 4.11.0 release.
+
+Major Changes
+-------------
+
+- grafana - Add new dashboards for sql_exporter support. New PostgreSQL Overview and PgBouncer direct metrics dashboards
+- grafana - New Grafana minimum version is now 10.4. All dashboards have been updated to fix AngularJS deprecation warnings and re-exported from 10.4.
+- grafana - Organize packages to allow better choice of available Grafana dashboards
+- grafana - Remove top level general Overview dashboard
+- pgmonitor-extension - Add more extensive support for materialized views and refreshed tables for expensive or custom metric queries
+- pgmonitor-extension - Add support for using the PostgreSQL pgmonitor-extension to aid in metrics collection with sql_exporter
+- postgres_exporter - Note that postgres_exporter is still supported but will be deprecated in a future version
+- sql_exporter - Add support for directly connecting to PgBouncer to collect metrics
+- sql_exporter - Add support for new PostgreSQL metrics collecting exporter (sql_exporter)
+
+Minor Changes
+-------------
+
+- prometheus - Added OOMKiller Alert using node_exporter metrics
+
+Bugfixes
+--------
+
+- docs - add reference links to upstream configuration docs
+- exporter - fix the pgbackrest-info.sh to force the necessary console output level that it expects
+- grafana - fix some queries that were searching on the wrong label (datname vs. dbname)
+- sql_exporter - add new metric for n_tup_newpage_upd
+- sql_exporter - use the new views from pgmonitor-extension instead of full queries
+
 v4.11.0
 =======
 

diff --git a/blackbox_exporter/common/blackbox_exporter.sysconfig b/blackbox_exporter/common/blackbox_exporter.sysconfig
@@ -7,4 +7,3 @@
 # This file must be in a pathname that matches the EnvironmentFile entry in the service file (Default: /etc/sysconfig/blackbox_exporter)
 #
 OPT="--config.file=/etc/blackbox_exporter/crunchy-blackbox.yml"
-
diff --git a/build/packages.yml b/build/packages.yml
@@ -108,7 +108,7 @@ packages:
 
     - sql-exporter:
         files:
-          - { source: "https://github.com/CrunchyData/pgmonitor/sql_exporter/linux/[email protected]" target: "/usr/lib/systemd/system/[email protected]", mode: "0645", type: "file", owner: "root", group: "root" }
+          - { source: "https://github.com/CrunchyData/pgmonitor/sql_exporter/linux/[email protected]", target: "/usr/lib/systemd/system/[email protected]", mode: "0645", type: "file", owner: "root", group: "root" }
           - { target: "/usr/bin/sql_exporter", mode: "0755", type: "file", owner: "root", group: "root" }
           - { target: "/etc/sql_exporter", type: "folder", mode: "0755", owner: "root", group: "root" }
         upstream_repo: "https://github.com/burningalchemist/sql_exporter"
@@ -133,7 +133,7 @@ packages:
         pkg_conflict:
           - { pkg_name: "pgmonitor-pg-common" }
         pkg_dependency:
-          - { pkg_name: "sql-exporter", gte: 0.13.0, lt: 0.15.0 }
+          - { pkg_name: "sql-exporter", gte: 0.13.0, lt: 0.16.0 }
 
 
     - grafana:
@@ -160,9 +160,9 @@ packages:
         files:
           - { source: "grafana/linux/Filesystem_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
           - { source: "grafana/linux/Network_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-           - { source: "grafana/linux/OS_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-           - { source: "grafana/linux/OS_Overview.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-         pkg_dependency:
+          - { source: "grafana/linux/OS_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
+          - { source: "grafana/linux/OS_Overview.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
+        pkg_dependency:
           - { pkg_name: "pgmonitor-grafana-extras-common" }
         upstream_repo: "https://github.com/CrunchyData/pgmonitor"
 

diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml
@@ -109,3 +109,49 @@ releases:
     - 355.yml
     - release_summary.yml
     release_date: '2023-07-25'
+  5.0.0:
+    changes:
+      bugfixes:
+      - docs - add reference links to upstream configuration docs
+      - exporter - fix the pgbackrest-info.sh to force the necessary console output
+        level that it expects
+      - grafana - fix some queries that were searching on the wrong label (datname
+        vs. dbname)
+      - sql_exporter - add new metric for n_tup_newpage_upd
+      - sql_exporter - use the new views from pgmonitor-extension instead of full
+        queries
+      major_changes:
+      - grafana - Add new dashboards for sql_exporter support. New PostgreSQL Overview
+        and PgBouncer direct metrics dashboards
+      - grafana - New Grafana minimum version is now 10.4. All dashboards have been
+        updated to fix AngularJS deprecation warnings and re-exported from 10.4.
+      - grafana - Organize packages to allow better choice of available Grafana dashboards
+      - grafana - Remove top level general Overview dashboard
+      - pgmonitor-extension - Add more extensive support for materialized views and
+        refreshed tables for expensive or custom metric queries
+      - pgmonitor-extension - Add support for using the PostgreSQL pgmonitor-extension
+        to aid in metrics collection with sql_exporter
+      - postgres_exporter - Note that postgres_exporter is still supported but will
+        be deprecated in a future version
+      - sql_exporter - Add support for directly connecting to PgBouncer to collect
+        metrics
+      - sql_exporter - Add support for new PostgreSQL metrics collecting exporter
+        (sql_exporter)
+      minor_changes:
+      - prometheus - Added OOMKiller Alert using node_exporter metrics
+      release_summary: Crunchy Data is pleased to announce the availability of pgMonitor
+        5.0.0. This release brings support for a new Prometheus exporter for PostgreSQL
+        - sql_exporter. It also supports a new monitoring extension to make metric
+        collection easier and more performant. This changelog contains all changes
+        that have been added since the 4.11.0 release.
+    fragments:
+    - 361.yml
+    - 387.yml
+    - 397.yml
+    - 399.yml
+    - 401.yml
+    - 407.yml
+    - 411.yml
+    - 412.yml
+    - release_summary.yml
+    release_date: '2024-07-09'
diff --git a/changelogs/fragments/361.yml b/changelogs/fragments/361.yml
diff --git a/changelogs/fragments/387.yml b/changelogs/fragments/387.yml
diff --git a/changelogs/fragments/397.yml b/changelogs/fragments/397.yml
diff --git a/changelogs/fragments/399.yml b/changelogs/fragments/399.yml
diff --git a/changelogs/fragments/401.yml b/changelogs/fragments/401.yml
diff --git a/grafana/common/crunchy_grafana_datasource.yml b/grafana/common/crunchy_grafana_datasource.yml
@@ -17,4 +17,3 @@ datasources:
     editable: False
     orgId: 1
     version: 1
-
diff --git a/grafana/containers/postgresql_details.json b/grafana/containers/postgresql_details.json
@@ -701,7 +701,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m])) + sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m])) + sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -712,7 +712,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1172,7 +1172,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(irate(ccp_stat_database_tup_fetched{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_tup_fetched{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1183,7 +1183,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_tup_inserted{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_tup_inserted{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1193,7 +1193,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_tup_updated{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_tup_updated{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1203,7 +1203,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_tup_deleted{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_tup_deleted{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "Deleted",
@@ -1212,7 +1212,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_tup_returned{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_tup_returned{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
@@ -1435,7 +1435,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(ccp_stat_database_deadlocks{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(rate(ccp_stat_database_deadlocks{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1445,7 +1445,7 @@
           "step": 2
         },
         {
-          "expr": "sum(rate(ccp_stat_database_conflicts{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(rate(ccp_stat_database_conflicts{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "intervalFactor": 2,
           "legendFormat": "DeadLocks",
@@ -1454,7 +1454,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_xact_commit{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1465,7 +1465,7 @@
           "step": 2
         },
         {
-          "expr": "sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\"}[5m]))",
+          "expr": "sum(irate(ccp_stat_database_xact_rollback{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\"}[5m]))",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1831,7 +1831,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"accessexclusivelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"accessexclusivelock\"})",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -1841,7 +1841,7 @@
           "step": 2
         },
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"exclusivelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"exclusivelock\"})",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1850,7 +1850,7 @@
           "step": 2
         },
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"rowexclusivelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"rowexclusivelock\"})",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1859,7 +1859,7 @@
           "step": 2
         },
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"sharerowexclusivelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"sharerowexclusivelock\"})",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1868,7 +1868,7 @@
           "step": 2
         },
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"shareupdateexclusivelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"shareupdateexclusivelock\"})",
           "format": "time_series",
           "hide": false,
           "intervalFactor": 2,
@@ -1877,7 +1877,7 @@
           "step": 2
         },
         {
-          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",datname=~\"[[datname]]\",mode=\"accesssharelock\"})",
+          "expr": "sum by (mode) (ccp_locks_count{pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\",dbname=~\"[[datname]]\",mode=\"accesssharelock\"})",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{mode}}",

diff --git a/hugo/archetypes/default.md b/hugo/archetypes/default.md
@@ -3,4 +3,3 @@ title: "{{ replace .Name "-" " " | title }}"
 date: {{ .Date }}
 draft: true
 ---
-