Fix precommit linting issues (#413)

* fix: use the proper views for pgmonitor-extension queries. doc update. new table stat metric

* chore: add changelog fragment

* chore: correct precommit linting

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_install_hook_types:
   - pre-commit
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.3.0
+  rev: v4.6.0
   hooks:
     - id: check-merge-conflict
     - id: check-symlinks
@@ -15,25 +15,11 @@ repos:
     - id: trailing-whitespace
       args: ['--markdown-linebreak-ext=md']
 - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
-  rev: v9.0.0
+  rev: v9.16.0
   hooks:
     - id: commitlint
       stages: [commit-msg]
       additional_dependencies: ['@commitlint/config-conventional']
-- repo: https://github.com/ansible-community/ansible-lint.git
-  rev: v5.4.0
-    # rev: v5.4.0 will not change until we drop ansible 2.9
-  hooks:
-    - id: ansible-lint
-      always_run: true
-      pass_filenames: false
-      # do not add file filters here as ansible-lint does not give reliable
-      # results when called with individual files.
-      # https://github.com/ansible/ansible-lint/issues/611
-      verbose: true
-      entry: ansible-lint --force-color -p -v .
-      additional_dependencies:
-        - 'ansible<2.10'
 - repo: https://github.com/codespell-project/codespell
   rev: v2.1.0
   hooks:

blackbox_exporter/common/blackbox_exporter.sysconfig

@@ -7,4 +7,3 @@
 # This file must be in a pathname that matches the EnvironmentFile entry in the service file (Default: /etc/sysconfig/blackbox_exporter)
 #
 OPT="--config.file=/etc/blackbox_exporter/crunchy-blackbox.yml"
-

build/packages.yml

@@ -108,7 +108,7 @@ packages:
 
     - sql-exporter:
         files:
-          - { source: "https://github.com/CrunchyData/pgmonitor/sql_exporter/linux/[email protected]" target: "/usr/lib/systemd/system/[email protected]", mode: "0645", type: "file", owner: "root", group: "root" }
+          - { source: "https://github.com/CrunchyData/pgmonitor/sql_exporter/linux/[email protected]", target: "/usr/lib/systemd/system/[email protected]", mode: "0645", type: "file", owner: "root", group: "root" }
           - { target: "/usr/bin/sql_exporter", mode: "0755", type: "file", owner: "root", group: "root" }
           - { target: "/etc/sql_exporter", type: "folder", mode: "0755", owner: "root", group: "root" }
         upstream_repo: "https://github.com/burningalchemist/sql_exporter"
@@ -160,9 +160,9 @@ packages:
         files:
           - { source: "grafana/linux/Filesystem_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
           - { source: "grafana/linux/Network_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-           - { source: "grafana/linux/OS_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-           - { source: "grafana/linux/OS_Overview.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
-         pkg_dependency:
+          - { source: "grafana/linux/OS_Details.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
+          - { source: "grafana/linux/OS_Overview.json", target: "/etc/grafana/crunchy_dashboards/", mode: "0640", type: "file", rpm_new: false, owner: "grafana", group: "grafana" }
+        pkg_dependency:
           - { pkg_name: "pgmonitor-grafana-extras-common" }
         upstream_repo: "https://github.com/CrunchyData/pgmonitor"
 

changelogs/fragments/361.yml

@@ -6,5 +6,5 @@ major_changes:
   - postgres_exporter - Note that postgres_exporter is still supported but will be deprecated in a future version
   - grafana - New Grafana minimum version is now 10.4. All dashboards have been updated to fix AngularJS deprecation warnings and re-exported from 10.4.
   - grafana - Add new dashboards for sql_exporter support. New PostgreSQL Overview and PgBouncer direct metrics dashboards
-  - grafana - Organize packages to allow better choice of available Grafana dashboards 
+  - grafana - Organize packages to allow better choice of available Grafana dashboards
   - grafana - Remove top level general Overview dashboard

changelogs/fragments/399.yml

@@ -3,4 +3,3 @@ trivial:
   - packaging - Update the build file to set blackbox-exporter-extras group to ccp_monitoring for consistency with other extras packages
   - packaging - Take out remove_files lines since the files referenced have been gone for a long time now and packages aren't doing that anymore
   - packaging - Take out package conflicts with containers packages. There are no containers packages being done at this time
-

changelogs/fragments/412.yml

@@ -1,3 +1,4 @@
 bugfixes:
   - sql_exporter - use the new views from pgmonitor-extension instead of full queries
+  - sql_exporter - add new metric for n_tup_newpage_upd
   - docs - add reference links to upstream configuration docs

grafana/common/crunchy_grafana_datasource.yml

@@ -17,4 +17,3 @@ datasources:
     editable: False
     orgId: 1
     version: 1
-

hugo/archetypes/default.md

@@ -3,4 +3,3 @@ title: "{{ replace .Name "-" " " | title }}"
 date: {{ .Date }}
 draft: true
 ---
-

hugo/content/_index.md

@@ -93,4 +93,3 @@ Copyright © 2017-2024 Crunchy Data Solutions, Inc. All Rights Reserved.
 CRUNCHY DATA SOLUTIONS, INC. PROVIDES THIS GUIDE "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF NON INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
 
 Crunchy, Crunchy Data Solutions, Inc. and the Crunchy Hippo Logo are trademarks of Crunchy Data Solutions, Inc. All Rights Reserved.
-

hugo/content/changelog/v5_upgrade/_index.md

@@ -23,7 +23,7 @@ sudo systemctl disable crunchy-postgres-exporter@postgres_exporter_pg##_per_db
 
 Note the values after the @ symbol may be different depending on the sysconfig files that have been created for your exporters. There may also be exporters running for multiple clusters and we would recommend replacing all of the existing postgres_exporters with the new sql_exporter.
 
-If you've installed pgMonitor with the packages provided by Crunchy Data, those packages can now be uninstalled as well. 
+If you've installed pgMonitor with the packages provided by Crunchy Data, those packages can now be uninstalled as well.
 
 | Package Name                   | Description                                                               |
 |--------------------------------|---------------------------------------------------------------------------|
@@ -50,7 +50,7 @@ If postgres_exporter was not set up with packages, you can now manually remove a
 
 
 ### Prometheus
-All postgres_exporter Prometheus targets can now be removed. The default location for Prometheus targets is `/etc/prometheus/auto.d/`, but please check your Prometheus installation for possible additional target locations. In the identified location(s), remove any targets for the postgres_exporter. The default ports for postgres_exporter were 9187 and 9188, so any targets with these ports should be examined for removal. Note that if alerting had previously been enabled, the previous step likely caused multiple alerts to fire; once this step is done, you can simply reload Prometheus to clear these targets and any related alerts should resolve themselves. 
+All postgres_exporter Prometheus targets can now be removed. The default location for Prometheus targets is `/etc/prometheus/auto.d/`, but please check your Prometheus installation for possible additional target locations. In the identified location(s), remove any targets for the postgres_exporter. The default ports for postgres_exporter were 9187 and 9188, so any targets with these ports should be examined for removal. Note that if alerting had previously been enabled, the previous step likely caused multiple alerts to fire; once this step is done, you can simply reload Prometheus to clear these targets and any related alerts should resolve themselves.
 
 ```bash
 sudo systemctl reload prometheus

hugo/content/exporter/_index.md

@@ -18,7 +18,7 @@ The Linux instructions below use RHEL, but any Linux-based system should work. [
    - [System](#system)
 - [Legacy postgres_exporter Setup](#postgres-exporter)
 
-  
+
 IMPORTANT NOTE: As of pgMonitor version 5.0.0, postgres_exporter has been deprecated in favor of sql_exporter. Support for postgres_exporter is still possible with 5.0, but only for bug fixes while custom queries are still supported. No new features will be added using postgres_exporter and it will be fully obsoleted in a future version of pgMonitor. We recommend migrating to sql_exporter as soon as possible.
 
 ## Installation {#installation}
@@ -87,9 +87,9 @@ The following pgMonitor configuration files should be placed according to the fo
 
 ##### sql_exporter
 
-sql_exporter takes advantage of the Crunchy Data pgmonitor-extension (https://github.com/CrunchyData/pgmonitor-extension) to provide a much easier configuration and setup. The extension takes care of creating all the necessary objects inside the database. 
+sql_exporter takes advantage of the Crunchy Data pgmonitor-extension (https://github.com/CrunchyData/pgmonitor-extension) to provide a much easier configuration and setup. The extension takes care of creating all the necessary objects inside the database.
 
-The mimimum required version of pgmonitor-extension is currently 1.0.0.
+The minimum required version of pgmonitor-extension is currently 1.0.0.
 
 The following pgMonitor configuration files should be placed according to the following mapping:
 
@@ -154,7 +154,7 @@ You will need to restart your PostgreSQL instance for the change to take effect.
 
 The pgmonitor-extension uses its own background worker to refresh metric data.
 
-The following statement only needs to be run on the "global" database, typically the "postgres" database. If you want the pg_stat_statements view to be visible in other databases, this statement must be run there as well. 
+The following statement only needs to be run on the "global" database, typically the "postgres" database. If you want the pg_stat_statements view to be visible in other databases, this statement must be run there as well.
 
 ```sql
 CREATE EXTENSION pg_stat_statements;
@@ -197,7 +197,7 @@ This should generate one or more statements similar to the following:
 ```sql
 GRANT CONNECT ON DATABASE "postgres" TO ccp_monitoring;
 ```
-Run these grant statements to then allow monitoring to connect. 
+Run these grant statements to then allow monitoring to connect.
 
 
 ##### Bloat setup
@@ -255,7 +255,7 @@ sudo systemctl status crunchy-sql-exporter@sql_exporter_cluster2
 
 ```
 
-### Monitoring multiple databases 
+### Monitoring multiple databases
 
 sql_exporter can connect to as many databases as you need. Simply add another connection configuration to the `job_name` in the sql_exporter configuration file for the other databases you wish to monitor. If making use of pgMonitor's metrics, ensure that the pgmonitor-extension is also installed on those target databases.
 
@@ -281,11 +281,11 @@ jobs:
 
 ## Metrics Collected {#metrics-collected}
 
-The metrics collected by our exporters are outlined below. 
+The metrics collected by our exporters are outlined below.
 
 ### PostgreSQL {#postgresql}
 
-PostgreSQL metrics are collected by [sql_exporter](https://github.com/burningalchemist/sql_exporter). pgMonitor uses custom queries for its PG metrics. 
+PostgreSQL metrics are collected by [sql_exporter](https://github.com/burningalchemist/sql_exporter). pgMonitor uses custom queries for its PG metrics.
 
 
 #### Common Metrics
@@ -308,10 +308,10 @@ Metrics contained in the `queries_global.yml` file. These metrics are common to
 
  * *ccp_connection_stats_max_connections* - Current value of max_connections for reference
 
- * *ccp_connection_stats_max_idle_in_txn_time* - Runtime of longest idle in transaction (IIT) session. 
+ * *ccp_connection_stats_max_idle_in_txn_time* - Runtime of longest idle in transaction (IIT) session.
+
+ * *ccp_connection_stats_max_query_time* - Runtime of longest general query (inclusive of IIT).
 
- * *ccp_connection_stats_max_query_time* - Runtime of longest general query (inclusive of IIT). 
-
  * *ccp_connection_stats_max_blocked_query_time* - Runtime of the longest running query that has been blocked by a heavyweight lock
 
  * *ccp_locks_count* - Count of active lock types per database
@@ -342,7 +342,7 @@ Metrics contained in the `queries_global.yml` file. These metrics are common to
 
  * *ccp_wal_activity_last_5_min_size_bytes* - Current size in bytes of the last 5 minutes of WAL generation. Includes recycled WALs.
 
-The meaning of the following `ccp_transaction_wraparound` metrics, and how to manage when they are triggered, is covered more extensively in this blog post: https://info.crunchydata.com/blog/managing-transaction-id-wraparound-in-postgresql 
+The meaning of the following `ccp_transaction_wraparound` metrics, and how to manage when they are triggered, is covered more extensively in this blog post: https://info.crunchydata.com/blog/managing-transaction-id-wraparound-in-postgresql
 
  * *ccp_transaction_wraparound_percent_towards_emergency_autovac* - Recommended thresholds set to 75%/95% when first evaluating vacuum settings on new systems. Once those have been reviewed and at least one instance-wide vacuum has been run, recommend thresholds of 110%/125%. Reaching 100% is not a cause for immediate concern, but alerting above 100% for extended periods of time means that autovacuum is not able to keep up with current transaction rate and needs further tuning.
 
@@ -443,7 +443,7 @@ Bloat metrics are only available if the `pg_bloat_check` script has been setup t
 
 #### pgBouncer Metrics
 
-The following metric prefixes correspond to the SHOW command views found in the [pgBouncer documentation](https://www.pgbouncer.org/usage.html). Each column found in the SHOW view is a separate metric under the respective prefix. Ex: `ccp_pgbouncer_pools_client_active` corresponds to the `SHOW POOLS` view's `client_active` column. 
+The following metric prefixes correspond to the SHOW command views found in the [pgBouncer documentation](https://www.pgbouncer.org/usage.html). Each column found in the SHOW view is a separate metric under the respective prefix. Ex: `ccp_pgbouncer_pools_client_active` corresponds to the `SHOW POOLS` view's `client_active` column.
 
 sql_exporter can connect directly to pgBouncer with some specific configuration options set. See the example `sql_exporter.yml` and the `crunchy_pgbouncer_collector_###.yml` file.
 
@@ -536,15 +536,15 @@ Note that {{< shell >}}/etc/sysconfig/postgres_exporter_pg##{{< /shell >}} & {{<
 | queries_global.yml    | postgres_exporter query file with minimal recommended queries that are common across all PG versions and only need to be run once per database instance.    |
 | queries_global_dbsize.yml | postgres_exporter query file that contains metrics for monitoring database size. This is a separate file to allow the option to use a materialized view for very large databases |
 | queries_global_matview.yml | postgres_exporter query file that contains alternative metrics that use materialized views of common metrics across all PG versions |
-| queries_per_db.yml    | postgres_exporter query file with queries that gather per databse stats. WARNING: If your database has many tables this can greatly increase the storage requirements for your prometheus database. If necessary, edit the query to only gather tables you are interested in statistics for. The "PostgreSQL Details" and the "CRUD Details" Dashboards use these statistics.                                                   |
+| queries_per_db.yml    | postgres_exporter query file with queries that gather per database stats. WARNING: If your database has many tables this can greatly increase the storage requirements for your prometheus database. If necessary, edit the query to only gather tables you are interested in statistics for. The "PostgreSQL Details" and the "CRUD Details" Dashboards use these statistics.                                                   |
 | queries_per_db_matview.yml | postgres_exporter query files that contains alternative metrics that use materialized views of per database stats |
 | queries_general.yml      | postgres_exporter query file for queries that are specific to the version of PostgreSQL that is being monitored.   |
 | queries_backrest.yml | postgres_exporter query file for monitoring pgBackRest backup status. By default, new backrest data is only collected every 10 minutes to avoid excessive load when there are large backup lists. See sysconfig file for exporter service to adjust this throttling. |
 | queries_pgbouncer.yml | postgres_exporter query file for monitoring pgbouncer. |
 | queries_pg_stat_statements.yml | postgres_exporter query file for specific pg_stat_statements metrics that are most useful for monitoring and trending. |
 
 
-By default, there are two postgres_exporter services expected to be running. One connects to the default {{< shell >}}postgres{{< /shell >}} database that most PostgreSQL instances come with and is meant for collecting global metrics that are the same on all databases in the instance (connection/replication statistics, etc). This service uses the sysconfig file {{< shell >}}postgres_exporter_pg##{{< /shell >}}. Connect to this database and run the setup.sql script to install the required database objects for pgMonitor. 
+By default, there are two postgres_exporter services expected to be running. One connects to the default {{< shell >}}postgres{{< /shell >}} database that most PostgreSQL instances come with and is meant for collecting global metrics that are the same on all databases in the instance (connection/replication statistics, etc). This service uses the sysconfig file {{< shell >}}postgres_exporter_pg##{{< /shell >}}. Connect to this database and run the setup.sql script to install the required database objects for pgMonitor.
 
 The second postgres_exporter service is used to collect per-database metrics and uses the sysconfig file {{< shell >}}postgres_exporter_pg##_per_db{{< /shell >}}. By default it is set to also connect to the {{< shell >}}postgres{{< /shell >}} database, but you can add as many additional connection strings to this service for each individual database that you want metrics for. Per-db metrics include things like table/index statistics and bloat. See the section below for monitorig multiple databases for how to do this.
 
@@ -570,7 +570,7 @@ For replica servers, the setup is the same except that the setup.sql file does n
 
 With large databases/tables and some other conditions, certain metrics can cause excessive load. For those cases, materialized views and alternative metric queries have been made available. The materialized views are refreshed on their own schedule independent of the Prometheus data scrape, so any load that may be associated with gathering the underlying data is mitigated. A configuration table, seen below, contains options for how often these materialized views should be refreshed. And a single procedure can be called to refresh all materialized views relevant to monitoring.
 
-For every database that will be collecting materialized view metrics, you will have to run the {{< shell >}}setup_metric_views.sql{{< /shell >}} file against that database. This will likely need to be run as a superuser and must be run after running the base setup file mentioned above to create the necessary monitoring user first. 
+For every database that will be collecting materialized view metrics, you will have to run the {{< shell >}}setup_metric_views.sql{{< /shell >}} file against that database. This will likely need to be run as a superuser and must be run after running the base setup file mentioned above to create the necessary monitoring user first.
 ```
 psql -U postgres -d alphadb -f setup_metric_views.sql
 psql -U postgres -d betadb -f setup_metric_views.sql
@@ -640,4 +640,3 @@ Lastly, update the Prometheus auto.d target file to include the new exporter in
 #### General Metrics
 
 *pg_up* -  Database is up and connectable by metric collector. This metric is only available with postgres_exporter
-