From 981f70701978e92bdbe9fff01e56be6d765e8e12 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 28 Jun 2023 21:26:21 +0200 Subject: [PATCH 1/6] PG-592: Treat queries with different parent queries as separate entries 1. Previously pg_stat_monitor had a `topquery` and `topqueryid` field, but it was only a sample: it showed one of the top queries executing the specific query. With this change, the same entry executed by two different functions will result in two entries in the statistics table. 2. This also fixes a bug where the content of these field disappeared for every second query executed: previously the update function changed topqueryid to `0` if it was non zero, and changed it to the proper id when it was 0. This resulted in an alternating behavior, where for every second executed query the top query disappeared. After these changes, the top query is always shown. 3. The previous implementation also leaked dsa memory used to store the parent queries. This is now also fixed. --- .gitignore | 1 + pg_stat_monitor.c | 36 +++++++++++++++++++++++++++--------- pg_stat_monitor.h | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 3ba025a8..299caa8d 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ *.mod* *.cmd .tmp_versions/ +.deps/ modules.order Module.symvers Mkfile.old diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 3f1c6933..f1875817 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1438,14 +1438,13 @@ pgsm_update_entry(pgsmEntry * entry, e->counters.info.num_relations = num_relations; _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); - if (exec_nested_level > 0 && e->counters.info.parentid == 0 && pgsm_track == PGSM_TRACK_ALL) + if (exec_nested_level > 0 && e->key.parentid != 0 && pgsm_track == PGSM_TRACK_ALL) { - if (exec_nested_level >= 0 && exec_nested_level < max_stack_depth) + if (e->counters.info.parent_query == InvalidDsaPointer && exec_nested_level >= 0 && exec_nested_level < max_stack_depth) { int parent_query_len = nested_query_txts[exec_nested_level - 1] ? strlen(nested_query_txts[exec_nested_level - 1]) : 0; - e->counters.info.parentid = nested_queryids[exec_nested_level - 1]; e->counters.info.parent_query = InvalidDsaPointer; /* If we have a parent query, store it in the raw dsa area */ if (parent_query_len > 0) @@ -1473,8 +1472,11 @@ pgsm_update_entry(pgsmEntry * entry, } else { - e->counters.info.parentid = UINT64CONST(0); - e->counters.info.parent_query = InvalidDsaPointer; + if(e->counters.info.parent_query != InvalidDsaPointer) { + dsa_area *query_dsa_area = get_dsa_area_for_query_text(); + dsa_free(query_dsa_area, e->counters.info.parent_query); + e->counters.info.parent_query = InvalidDsaPointer; + } } } @@ -1693,6 +1695,7 @@ pgsm_create_hash_entry(uint64 bucket_id, uint64 queryid, PlanInfo * plan_info) entry->key.dbid = MyDatabaseId; entry->key.queryid = queryid; entry->key.bucket_id = bucket_id; + entry->key.parentid = 0; #if PG_VERSION_NUM < 140000 entry->key.toplevel = 1; @@ -1807,9 +1810,20 @@ pgsm_store(pgsmEntry * entry) memcpy(&jitusage.optimization_counter, &entry->counters.jitinfo.instr_optimization_counter, sizeof(instr_time)); memcpy(&jitusage.emission_counter, &entry->counters.jitinfo.instr_emission_counter, sizeof(instr_time)); + + // Update parent id if needed + if(pgsm_track == PGSM_TRACK_ALL && exec_nested_level > 0 && exec_nested_level < max_stack_depth) + { + entry->key.parentid = nested_queryids[exec_nested_level - 1]; + } + else + { + entry->key.parentid = UINT64CONST(0); + } + /* * Acquire a share lock to start with. We'd have to acquire exclusive if - * we need ot create the entry. + * we need to create the entry. */ LWLockAcquire(pgsm->lock, LW_SHARED); shared_hash_entry = (pgsmEntry *) pgsm_hash_find(get_pgsmHash(), &entry->key, &found); @@ -1900,6 +1914,8 @@ pgsm_store(pgsmEntry * entry) shared_hash_entry->pgsm_query_id = entry->pgsm_query_id; shared_hash_entry->encoding = entry->encoding; shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; + shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; + shared_hash_entry->counters.info.parent_query = InvalidDsaPointer; snprintf(shared_hash_entry->datname, sizeof(shared_hash_entry->datname), "%s", entry->datname); snprintf(shared_hash_entry->username, sizeof(shared_hash_entry->username), "%s", entry->username); @@ -2061,6 +2077,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, bool nulls[PG_STAT_MONITOR_COLS] = {0}; int i = 0; Counters tmp; + pgsmHashKey tmpkey; double stddev; uint64 queryid = entry->key.queryid; int64 bucketid = entry->key.bucket_id; @@ -2097,6 +2114,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, SpinLockAcquire(&e->mutex); tmp = e->counters; + tmpkey = e->key; SpinLockRelease(&e->mutex); } @@ -2113,7 +2131,7 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, } /* read the parent query text if any */ - if (tmp.info.parentid != UINT64CONST(0)) + if (tmpkey.parentid != UINT64CONST(0)) { if (DsaPointerIsValid(tmp.info.parent_query)) { @@ -2198,9 +2216,9 @@ pg_stat_monitor_internal(FunctionCallInfo fcinfo, nulls[i++] = true; /* parentid at column number 9 */ - if (tmp.info.parentid != UINT64CONST(0)) + if (tmpkey.parentid != UINT64CONST(0)) { - values[i++] = UInt64GetDatum(tmp.info.parentid); + values[i++] = UInt64GetDatum(tmpkey.parentid); values[i++] = CStringGetTextDatum(parent_query_txt); } else diff --git a/pg_stat_monitor.h b/pg_stat_monitor.h index 339f6474..7c493aa4 100644 --- a/pg_stat_monitor.h +++ b/pg_stat_monitor.h @@ -240,11 +240,11 @@ typedef struct pgsmHashKey Oid dbid; /* database OID */ uint32 ip; /* client ip address */ bool toplevel; /* query executed at top level */ + uint64 parentid; /* parent queryid of current query */ } pgsmHashKey; typedef struct QueryInfo { - uint64 parentid; /* parent queryid of current query */ dsa_pointer parent_query; int64 type; /* type of query, options are query, info, * warning, error, fatal */ From 8caa3eac43fc2e0ae0c177d7e2178c68af250712 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Tue, 4 Jul 2023 10:08:40 +0200 Subject: [PATCH 2/6] PG-502: Fixing review comments * dsa_free changed to assert as it can never happen * restructured the ifs to be cleaner Note: kept the two-level ifs, as that makes more sense with the assert Note: didn't convert nested_level checks to macro, as it is used differently at different parts of the code --- pg_stat_monitor.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index f1875817..2275f5b9 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -20,6 +20,7 @@ #include "nodes/pg_list.h" #include "utils/guc.h" #include +#include #include "pgstat.h" #include "commands/dbcommands.h" #include "commands/explain.h" @@ -1438,9 +1439,9 @@ pgsm_update_entry(pgsmEntry * entry, e->counters.info.num_relations = num_relations; _snprintf2(e->counters.info.relations, relations, num_relations, REL_LEN); - if (exec_nested_level > 0 && e->key.parentid != 0 && pgsm_track == PGSM_TRACK_ALL) + if (exec_nested_level > 0 && exec_nested_level < max_stack_depth && e->key.parentid != 0 && pgsm_track == PGSM_TRACK_ALL) { - if (e->counters.info.parent_query == InvalidDsaPointer && exec_nested_level >= 0 && exec_nested_level < max_stack_depth) + if (e->counters.info.parent_query == InvalidDsaPointer) { int parent_query_len = nested_query_txts[exec_nested_level - 1] ? strlen(nested_query_txts[exec_nested_level - 1]) : 0; @@ -1472,11 +1473,7 @@ pgsm_update_entry(pgsmEntry * entry, } else { - if(e->counters.info.parent_query != InvalidDsaPointer) { - dsa_area *query_dsa_area = get_dsa_area_for_query_text(); - dsa_free(query_dsa_area, e->counters.info.parent_query); - e->counters.info.parent_query = InvalidDsaPointer; - } + assert(e->counters.info.parent_query == InvalidDsaPointer); } } From 3c8549ed2a1008047290e7226720c3e562dc3c6b Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 5 Jul 2023 06:38:56 +0200 Subject: [PATCH 3/6] PG-502: Fixing review comments --- pg_stat_monitor.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 2275f5b9..25891e51 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -20,7 +20,6 @@ #include "nodes/pg_list.h" #include "utils/guc.h" #include -#include #include "pgstat.h" #include "commands/dbcommands.h" #include "commands/explain.h" @@ -1441,7 +1440,7 @@ pgsm_update_entry(pgsmEntry * entry, if (exec_nested_level > 0 && exec_nested_level < max_stack_depth && e->key.parentid != 0 && pgsm_track == PGSM_TRACK_ALL) { - if (e->counters.info.parent_query == InvalidDsaPointer) + if (!DsaPointerIsValid(e->counters.info.parent_query)) { int parent_query_len = nested_query_txts[exec_nested_level - 1] ? strlen(nested_query_txts[exec_nested_level - 1]) : 0; @@ -1473,7 +1472,7 @@ pgsm_update_entry(pgsmEntry * entry, } else { - assert(e->counters.info.parent_query == InvalidDsaPointer); + Assert(!DsaPointerIsValid(e->counters.info.parent_query)); } } @@ -1911,7 +1910,6 @@ pgsm_store(pgsmEntry * entry) shared_hash_entry->pgsm_query_id = entry->pgsm_query_id; shared_hash_entry->encoding = entry->encoding; shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; - shared_hash_entry->counters.info.cmd_type = entry->counters.info.cmd_type; shared_hash_entry->counters.info.parent_query = InvalidDsaPointer; snprintf(shared_hash_entry->datname, sizeof(shared_hash_entry->datname), "%s", entry->datname); From 621701956a2f38f5040cf13bc072f0a9cf4ca510 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Wed, 17 Jul 2024 12:54:20 +0200 Subject: [PATCH 4/6] PG-592 Add regression test --- Makefile | 2 +- .../expected/different_parent_queries.out | 60 +++++++++++++++++++ regression/sql/different_parent_queries.sql | 26 ++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 regression/expected/different_parent_queries.out create mode 100644 regression/sql/different_parent_queries.sql diff --git a/Makefile b/Makefile index 51fb11d5..29ddae25 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ LDFLAGS_SL += $(filter -lm, $(LIBS)) TAP_TESTS = 1 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_stat_monitor/pg_stat_monitor.conf --inputdir=regression -REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags user +REGRESS = basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query different_parent_queries cmd_type error rows tags user # Disabled because these tests require "shared_preload_libraries=pg_stat_statements", # which typical installcheck users do not have (e.g. buildfarm clients). diff --git a/regression/expected/different_parent_queries.out b/regression/expected/different_parent_queries.out new file mode 100644 index 00000000..e597e32e --- /dev/null +++ b/regression/expected/different_parent_queries.out @@ -0,0 +1,60 @@ +Create EXTENSION pg_stat_monitor; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +ALTER SYSTEM SET pg_stat_monitor.pgsm_track TO 'all'; +SELECT pg_reload_conf(); + pg_reload_conf +---------------- + t +(1 row) + +CREATE OR REPLACE FUNCTION test() RETURNS VOID AS +$$ +BEGIN + PERFORM 1 + 2; +END; $$ language plpgsql; +CREATE OR REPLACE FUNCTION test2() RETURNS VOID AS +$$ +BEGIN + PERFORM 1 + 2; +END; $$ language plpgsql; +SELECT pg_stat_monitor_reset(); + pg_stat_monitor_reset +----------------------- + +(1 row) + +SELECT test(); + test +------ + +(1 row) + +SELECT test2(); + test2 +------- + +(1 row) + +SELECT 1 + 2; + ?column? +---------- + 3 +(1 row) + +SELECT left(query, 15) as query, calls, top_query, toplevel, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; + query | calls | top_query | toplevel | pgsm_query_id +-----------------+-------+-----------------+----------+---------------------- + SELECT 1 + 2 | 1 | SELECT test(); | f | 5193804135051352284 + SELECT 1 + 2 | 1 | SELECT test2(); | f | 5193804135051352284 + SELECT 1 + 2 | 1 | | t | 5193804135051352284 + SELECT pg_stat_ | 1 | | t | 689150021118383254 + SELECT test() | 1 | | t | -6801876889834540522 + SELECT test2() | 1 | | t | 369102705908374543 +(6 rows) + +DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/different_parent_queries.sql b/regression/sql/different_parent_queries.sql new file mode 100644 index 00000000..d40ca9aa --- /dev/null +++ b/regression/sql/different_parent_queries.sql @@ -0,0 +1,26 @@ +Create EXTENSION pg_stat_monitor; +SELECT pg_stat_monitor_reset(); + +ALTER SYSTEM SET pg_stat_monitor.pgsm_track TO 'all'; +SELECT pg_reload_conf(); + +CREATE OR REPLACE FUNCTION test() RETURNS VOID AS +$$ +BEGIN + PERFORM 1 + 2; +END; $$ language plpgsql; + +CREATE OR REPLACE FUNCTION test2() RETURNS VOID AS +$$ +BEGIN + PERFORM 1 + 2; +END; $$ language plpgsql; + +SELECT pg_stat_monitor_reset(); +SELECT test(); +SELECT test2(); + +SELECT 1 + 2; +SELECT left(query, 15) as query, calls, top_query, toplevel, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; + +DROP EXTENSION pg_stat_monitor; \ No newline at end of file From 381137b16e5c05a86a012e9d052d0b1ccf7eac3b Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Wed, 17 Jul 2024 14:40:22 +0200 Subject: [PATCH 5/6] Make test compatible with PG12 --- .../expected/different_parent_queries.out | 18 +++++++++--------- regression/sql/different_parent_queries.sql | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/regression/expected/different_parent_queries.out b/regression/expected/different_parent_queries.out index e597e32e..842e5e25 100644 --- a/regression/expected/different_parent_queries.out +++ b/regression/expected/different_parent_queries.out @@ -46,15 +46,15 @@ SELECT 1 + 2; 3 (1 row) -SELECT left(query, 15) as query, calls, top_query, toplevel, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; - query | calls | top_query | toplevel | pgsm_query_id ------------------+-------+-----------------+----------+---------------------- - SELECT 1 + 2 | 1 | SELECT test(); | f | 5193804135051352284 - SELECT 1 + 2 | 1 | SELECT test2(); | f | 5193804135051352284 - SELECT 1 + 2 | 1 | | t | 5193804135051352284 - SELECT pg_stat_ | 1 | | t | 689150021118383254 - SELECT test() | 1 | | t | -6801876889834540522 - SELECT test2() | 1 | | t | 369102705908374543 +SELECT left(query, 15) as query, calls, top_query, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; + query | calls | top_query | pgsm_query_id +-----------------+-------+-----------------+---------------------- + SELECT 1 + 2 | 1 | SELECT test(); | 5193804135051352284 + SELECT 1 + 2 | 1 | SELECT test2(); | 5193804135051352284 + SELECT 1 + 2 | 1 | | 5193804135051352284 + SELECT pg_stat_ | 1 | | 689150021118383254 + SELECT test() | 1 | | -6801876889834540522 + SELECT test2() | 1 | | 369102705908374543 (6 rows) DROP EXTENSION pg_stat_monitor; diff --git a/regression/sql/different_parent_queries.sql b/regression/sql/different_parent_queries.sql index d40ca9aa..912c3507 100644 --- a/regression/sql/different_parent_queries.sql +++ b/regression/sql/different_parent_queries.sql @@ -21,6 +21,6 @@ SELECT test(); SELECT test2(); SELECT 1 + 2; -SELECT left(query, 15) as query, calls, top_query, toplevel, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; +SELECT left(query, 15) as query, calls, top_query, pgsm_query_id FROM pg_stat_monitor ORDER BY query, top_query COLLATE "C"; DROP EXTENSION pg_stat_monitor; \ No newline at end of file From ada635282da796e783b49ce899b80ef70a4a5c31 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Wed, 17 Jul 2024 15:09:58 +0200 Subject: [PATCH 6/6] Remove redundant line --- pg_stat_monitor.c | 1 - 1 file changed, 1 deletion(-) diff --git a/pg_stat_monitor.c b/pg_stat_monitor.c index 7679b911..b3ece6cc 100644 --- a/pg_stat_monitor.c +++ b/pg_stat_monitor.c @@ -1464,7 +1464,6 @@ pgsm_update_entry(pgsmEntry * entry, int parent_query_len = nested_query_txts[exec_nested_level - 1] ? strlen(nested_query_txts[exec_nested_level - 1]) : 0; - e->counters.info.parent_query = InvalidDsaPointer; /* If we have a parent query, store it in the raw dsa area */ if (parent_query_len > 0) {