From a4feb4149e2be816bad19a45e19c183f65814327 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 11:05:30 +0200 Subject: [PATCH 01/39] Create role instead of user + replaces as11 with as16 --- .../considerations/pem_pgbouncer/configuring_pgBouncer.mdx | 6 +++--- .../pem_pgbouncer/preparing_the_pem_database_server.mdx | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index 39c05d55cc9..e730e1b7351 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -58,9 +58,9 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro ;; These are the root.crt, server.key, server.crt files present ;; in the present under the data directory of the PEM database ;; server, used by the PEM Agents for connections. - client_tls_ca_file = /var/lib/edb/as11/data/root.crt - client_tls_key_file = /var/lib/edb/as11/data/server.key - client_tls_cert_file = /var/lib/edb/as11/data/server.crt + client_tls_ca_file = /var/lib/edb/as16/data/root.crt + client_tls_key_file = /var/lib/edb/as16/data/server.key + client_tls_cert_file = /var/lib/edb/as16/data/server.crt ;; Use hba file for client connections auth_type = hba ;; Authentication file, Reference: diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index eac027e19e2..0b6d82c9da6 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -13,7 +13,7 @@ You must configure the PEM database server to work with PgBouncer. This example 1. Create a dedicated user named pgbouncer on the PEM database server: ```sql - CREATE USER pgbouncer PASSWORD 'ANY_PASSWORD' LOGIN; + CREATE ROLE pgbouncer PASSWORD 'ANY_PASSWORD' LOGIN; __OUTPUT__ CREATE ROLE ``` @@ -21,7 +21,7 @@ You must configure the PEM database server to work with PgBouncer. This example 2. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership on the PEM database server: ```sql - CREATE USER pem_admin1 PASSWORD 'ANY_PASSWORD' LOGIN CREATEROLE; + CREATE ROLE pem_admin1 PASSWORD 'ANY_PASSWORD' LOGIN CREATEROLE; __OUTPUT__ CREATE ROLE ``` From 3c48158d1d6517a5e07edd61b20deb423c008891 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 11:29:11 +0200 Subject: [PATCH 02/39] Prepare PEM: changed granted roles in steps 1 and 2 --- .../preparing_the_pem_database_server.mdx | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 0b6d82c9da6..d539b1ffa01 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -10,7 +10,7 @@ redirects: You must configure the PEM database server to work with PgBouncer. This example shows how to configure the PEM database server. -1. Create a dedicated user named pgbouncer on the PEM database server: +1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership on the PEM database server: ```sql CREATE ROLE pgbouncer PASSWORD 'ANY_PASSWORD' LOGIN; @@ -18,6 +18,12 @@ You must configure the PEM database server to work with PgBouncer. This example CREATE ROLE ``` + ```sql + GRANT pem_agent_pool TO pgbouncer; + __OUTPUT__ + GRANT ROLE + ``` + 2. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership on the PEM database server: ```sql @@ -25,8 +31,15 @@ You must configure the PEM database server to work with PgBouncer. This example __OUTPUT__ CREATE ROLE ``` + + ```sql + GRANT pem_agent_pool TO pem_admin1; + __OUTPUT__ + GRANT ROLE + ``` + ```sql - GRANT pem_admin, pem_agent_pool TO pem_admin1; + GRANT pem_admin TO pem_admin1 WITH ADMIN OPTION; __OUTPUT__ GRANT ROLE ``` From 3e07928d109d6571ae93f6e5d0b1c7525777657d Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 12:39:48 +0200 Subject: [PATCH 03/39] Prepare PEM: altered entries to add in pg_hba file in step 7 and beyond --- .../preparing_the_pem_database_server.mdx | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index d539b1ffa01..8300a81fd55 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -80,12 +80,29 @@ You must configure the PEM database server to work with PgBouncer. This example The function creates a user with the same name and a random password and grants pem_agent and pem_agent_pool roles to the user. This approach allows pgBouncer to use a proxy user on behalf of the agent. -7. Add the following entries to the start of the `pg_hba.conf` file of the PEM database server. These entries allow the pgBouncer user to connect to the `pem` database using the md5 authentication method. +7. Allow the pgBouncer user to connect to the `pem` database using the SSL authentication method by adding the following entries to the `pg_hba.conf` file of the PEM database server: + + !!!note + Add the following entry before any `+pem_agent` entries + !!! ```shell - # Allow the PEM agent proxy user (used by - # pgbouncer) to connect the to PEM server using - # md5 + # Allow the PEM agent proxy user (used by pgbouncer) + # to connect the to PEM server using SSL + + hostssl pem +pem_agent_pool 127.0.0.1/32 cert map=pem_agent_pool + ``` + +8. Allow user mapping by adding these lines to the `$PGDATA/pg_ident.conf` file: - local pem pgbouncer,pem_admin1 md5 + ```shell + pem_agent_pool pem_agent_pool pem_agent_user1 + pem_agent_pool pem_agent_pool pem_admin1 + pem_agent_pool pem_agent_pool pgbouncer ``` + +9. Reload the PEM server's configuration: + + ```shell + pg_ctl reload -D $PGDATA + ``` From 76f607d819b1143fcf10b772eedf4b1dac7ccfb4 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 14:22:29 +0200 Subject: [PATCH 04/39] Prepare PEM: added new section on creating SLL key and cert --- .../preparing_the_pem_database_server.mdx | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 8300a81fd55..9a2d922f308 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -10,6 +10,8 @@ redirects: You must configure the PEM database server to work with PgBouncer. This example shows how to configure the PEM database server. +## Creating users and roles for the pgBouncer-PEM connection + 1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership on the PEM database server: ```sql @@ -106,3 +108,37 @@ You must configure the PEM database server to work with PgBouncer. This example ```shell pg_ctl reload -D $PGDATA ``` + +## Creating the SSL key and certificate for pgBouncer-PEM authentication + +Create a key and certificate for the `pem_agent_pool` group role. Copy them to the pgBouncer instance to allow users with `pem_agent_pool` permissions to connect to PEM. + +1. Create the signing key: + + ```shell + openssl genrsa -out pem_agent_pool.key 4096 + ``` + +1. Create a certificate-signing request (CSR). Replace the DN attributes in `<...>` as you require, and ensure the Common Name (CN) is set to the `pem_agent_pool` group role name: + + ```shell + openssl req -new -key pem_agent_pool.key -out pem_agent_pool.csr -subj '/C=/ST=/L=/O=/CN=pem_agent_pool' + ``` + +1. Use the PEM certificate authority (CA) and key to sign the CSR. + + Adapt the paths to the key and certificate according to your environment: + + ``` + openssl x509 -req -days 365 -in pem_agent_pool.csr -CA /var/lib/edb/as12/data/ca_certificate.crt -CAkey /var/lib/edb/as16/data/ca_key.key -CAcreateserial -out pem_agent_pool.crt + ``` + +1. Copy and paste the key and certificate in the pgbouncer user's `~/.postgresql` directory and ensure the `enterprisedb` user has permissions: + + ``` + mkdir -p /var/lib/edb/.postgresql + mv pem_agent_pool.key pem_agent_pool.crt /var/lib/edb/.postgresql + chmod 0600 /var/lib/edb/.postgresql/pem_agent_pool.key + chmod 0644 /var/lib/edb/.postgresql/pem_agent_pool.crt + chown enterprisedb:enterprisedb /var/lib/edb/.postgresql/pem_agent_pool.* + ``` From ffb1c8e11877bb5d0d1507ad93e9af2fad2f590b Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 14:41:46 +0200 Subject: [PATCH 05/39] Configure pgBouncer: edited values in ini file --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 45 ++++++------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index e730e1b7351..c01a18c7cfc 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -41,10 +41,9 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro ;; 'auth_user' will be used for authenticate the db user (proxy ;; agent user in our case) - pem = port=5444 host=localhost dbname=pem auth_user=pgbouncer + pem = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=80 pool_mode=transaction - * = port=5444 host=localhost dbname=pem auth_user=pgbouncer - pool_size=10 + * = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=10 [pgbouncer] logfile = /var/log/edb/pgbouncer<1.x>/edb-pgbouncer-<1.x>.log @@ -52,7 +51,7 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro listen_addr = * ;; Agent needs to use this port to connect the pem database now listen_port = 6432 - ;; Require to support for the SSL Certificate authentications + ;; Set to require to support SSL Certificate authentication ;; for PEM Agents client_tls_sslmode = require ;; These are the root.crt, server.key, server.crt files present @@ -61,6 +60,10 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro client_tls_ca_file = /var/lib/edb/as16/data/root.crt client_tls_key_file = /var/lib/edb/as16/data/server.key client_tls_cert_file = /var/lib/edb/as16/data/server.crt + ;; Allow pgBouncer to use pem_agent_pool certificate + ;; and key for connections to the server. + server_tls_key_file = /var/lib/edb/.postgresql/pem_agent_pool.key + server_tls_cert_file = /var/lib/edb/.postgresql/pem_agent_pool.crt ;; Use hba file for client connections auth_type = hba ;; Authentication file, Reference: @@ -73,6 +76,10 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro auth_query = SELECT * FROM pem.get_agent_pool_auth($1) ;; DB User for administration of the pgbouncer admin_users = pem_admin1 + ;; auth_dbname and auth_user allow + ;; admin console login by admin_users and stats_users + auth_dbname = pem + auth_user = pgbouncer ;; DB User for collecting the statistics of pgbouncer stats_users = pem_admin1 server_reset_query = DISCARD ALL @@ -86,31 +93,7 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro !!! Note For more information on `auth_user` see [Authentication settings](https://www.pgbouncer.org/config.html#authentication-settings). -4. Create and update the `/etc/edb/pgbouncer<1.x>/userlist.txt` authentication file for PgBouncer: - - ```sql - ## Connect to pem database as a superuser, - ## create the userslist.txt file and add - ## username and their password list in CSV format - $ psql -p 5444 -d pem -U enterprisedb - - pem=# - COPY ( - SELECT 'pgbouncer'::TEXT, 'pgbouncer_password' - UNION ALL - SELECT 'pem_admin1'::TEXT, 'pem_admin1_password') - TO '/etc/edb/pgbouncer<1.x>/userlist.txt' - WITH (FORMAT CSV, DELIMITER ' ', FORCE_QUOTE *); - __OUTPUT__ - COPY 2 - ``` - - This creates `/etc/edb/pgbouncer<1.x>/userlist.txt` file and adds the username and password list. - - !!! Note - If the pem_admin user is a superuser, you must add the password to the authentication file (`enterprisedb` in the example). This allows the pem_admin user to invoke the PEM authentication query function `pem.get_proxy_auth(text)`. - -5. Create an HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` for PgBouncer that contains the following content: +4. Create an HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` for PgBouncer that contains the following content: ```ini # Use authentication method md5 for the local connections to @@ -130,14 +113,14 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro hostssl pem pem_agent_user1 0.0.0.0/0 cert ``` -6. Change the owner of the HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` to `enterprisedb`, and change the directory permissions to `0600`: +5. Change the owner of the HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` to `enterprisedb`, and change the directory permissions to `0600`: ```shell $ chown enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x>/hba_file $ chmod 0600 /etc/edb/pgbouncer<1.x>/hba_file ``` -7. Enable the PgBouncer service, and start the service: +6. Enable the PgBouncer service, and start the service: ```shell $ systemctl enable edb-pgbouncer-<1.x> From f0af0957e0e0f72ab75b6ab0ccd13abc4c5bda4a Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 15:22:17 +0200 Subject: [PATCH 06/39] Edited connection pooling index --- .../docs/pem/9/considerations/pem_pgbouncer/index.mdx | 9 +++++---- .../pem_pgbouncer/preparing_the_pem_database_server.mdx | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx index d6a460e5bd7..96417a1288a 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx @@ -17,10 +17,11 @@ navigation: - configuring_the_pem_agent --- -You can use pgBouncer as a connection pooler for limiting the number of connections from the PEM agent to the Postgres Enterprise Manager (PEM) server on non-Windows machine: +You can use pgBouncer as a connection pooler for limiting the number of connections from the PEM agent to the Postgres Enterprise Manager (PEM) server on non-Windows machines: -- Preparing the PEM Database Server provides information about preparing the PEM database server to be used with pgBouncer. -- Configuring pgBouncer provides detailed information about configuring pgBouncer to make it work with the PEM database server. -- Configuring the PEM agent provides detailed information about configuring a PEM agent to connect to pgBouncer. +- [PEM server and agent connection management mechanism](pem_server_pem_agent_connection_management_mechanism) provides an introduction to the pgBouncer-PEM infrastructe. +- [Preparing the PEM Database Server](preparing_the_pem_database_server) provides information about preparing the PEM database server to be used with pgBouncer. +- [Configuring pgBouncer](configuring_pgBouncer) provides detailed information about configuring pgBouncer to allow it to work with the PEM database server. +- [Configuring the PEM agent](configuring_the_pem_agent) provides detailed information about configuring a PEM agent to connect to pgBouncer. For detailed information about using the PEM web interface, see the [Accessing the web interface ](../../pem_web_interface). \ No newline at end of file diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 9a2d922f308..c87558cb95e 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -82,7 +82,7 @@ You must configure the PEM database server to work with PgBouncer. This example The function creates a user with the same name and a random password and grants pem_agent and pem_agent_pool roles to the user. This approach allows pgBouncer to use a proxy user on behalf of the agent. -7. Allow the pgBouncer user to connect to the `pem` database using the SSL authentication method by adding the following entries to the `pg_hba.conf` file of the PEM database server: +7. Allow the pgBouncer user to connect to the `pem` database using the SSL authentication method by adding the following entry to the `pg_hba.conf` file of the PEM database server: !!!note Add the following entry before any `+pem_agent` entries @@ -95,7 +95,7 @@ You must configure the PEM database server to work with PgBouncer. This example hostssl pem +pem_agent_pool 127.0.0.1/32 cert map=pem_agent_pool ``` -8. Allow user mapping by adding these lines to the `$PGDATA/pg_ident.conf` file: +8. Allow the PEM server to identify all users involved in the PEM-pgBouncer connection by adding these lines to the `$PGDATA/pg_ident.conf` user mapping file: ```shell pem_agent_pool pem_agent_pool pem_agent_user1 From 5017f6aaaec47528decfb44c55c728231588436d Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 16 Aug 2024 17:02:49 +0200 Subject: [PATCH 07/39] Style edits and added clarificating content --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 1 - .../configuring_the_pem_agent.mdx | 4 +- .../9/considerations/pem_pgbouncer/index.mdx | 12 +++--- ..._agent_connection_management_mechanism.mdx | 4 +- .../preparing_the_pem_database_server.mdx | 42 ++++++++++--------- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index c01a18c7cfc..4f91ba6342b 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -40,7 +40,6 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro ;; to the PEM database server as required. ;; 'auth_user' will be used for authenticate the db user (proxy ;; agent user in our case) - pem = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=80 pool_mode=transaction * = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=10 diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index ca24ea8aec0..33902865928 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -10,7 +10,7 @@ redirects: You can use an RPM package to install a PEM agent. For detailed installation information, see [Installating the PEM agent](../../installing_pem_agent/). -Don't configure the PEM agent responsible for sending SNMP notifications with pgBouncer. For example, if the default PEM agent installed with PEM server is used for SNMP notifications, don't configure it with pgBouncer. +Don't configure the PEM agent responsible for sending SNMP notifications with PgBouncer. For example, if the default PEM agent installed with PEM server is used for SNMP notifications, don't configure it with PgBouncer. ## Configuring a new PEM agent (installed via RPM) @@ -74,7 +74,7 @@ Add a line to use agent_user as the agent: agent_user=pem_agent_user1 ``` -Update the port to specify the pgBouncer port: +Update the port to specify the PgBouncer port: ```ini pem_port=6432 diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx index 96417a1288a..687eb5eb2a1 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx @@ -1,5 +1,5 @@ --- -title: "Connection pooling using pgBouncer" +title: "Connection pooling using PgBouncer" navTitle: "Deploying connection pooling" legacyRedirectsGenerated: # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. @@ -17,11 +17,11 @@ navigation: - configuring_the_pem_agent --- -You can use pgBouncer as a connection pooler for limiting the number of connections from the PEM agent to the Postgres Enterprise Manager (PEM) server on non-Windows machines: +You can use PgBouncer as a connection pooler for limiting the number of connections from the PEM agent to the Postgres Enterprise Manager (PEM) server on non-Windows machines: -- [PEM server and agent connection management mechanism](pem_server_pem_agent_connection_management_mechanism) provides an introduction to the pgBouncer-PEM infrastructe. -- [Preparing the PEM Database Server](preparing_the_pem_database_server) provides information about preparing the PEM database server to be used with pgBouncer. -- [Configuring pgBouncer](configuring_pgBouncer) provides detailed information about configuring pgBouncer to allow it to work with the PEM database server. -- [Configuring the PEM agent](configuring_the_pem_agent) provides detailed information about configuring a PEM agent to connect to pgBouncer. +- [PEM server and agent connection management mechanism](pem_server_pem_agent_connection_management_mechanism) provides an introduction of the PgBouncer-PEM infrastructe. +- [Preparing the PEM Database Server](preparing_the_pem_database_server) provides information about preparing the PEM database server to be used with PgBouncer. +- [Configuring PgBouncer](configuring_pgBouncer) provides detailed information about configuring PgBouncer to allow it to work with the PEM database server. +- [Configuring the PEM agent](configuring_the_pem_agent) provides detailed information about configuring a PEM agent to connect to PgBouncer. For detailed information about using the PEM web interface, see the [Accessing the web interface ](../../pem_web_interface). \ No newline at end of file diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index ff5a0a68569..017cd7614ce 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -10,7 +10,7 @@ redirects: Each PEM agent connects to the PEM database server using the SSL certificates for each user. For example, an agent with `ID#1` connects to the PEM database server using the agent1 user. -![Connecting to the PEM database without pgBouncer](../../images/pem_database_without_pgbouncer.png) +![Connecting to the PEM database without PgBouncer](../../images/pem_database_without_pgbouncer.png) Prior to PEM version 7.5, the following limitations disallowed the use of the connection pooler between the PEM server and PEM agent: @@ -21,4 +21,4 @@ EDB modified the PEM agent to allow the agent to use a common database user (ins ![Connecting to pgBouncer.](../../images/pem_database_with_pgbouncer.png) -We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. PEM agents can connect to pgBouncer using SSL certificates. +We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. PEM agents can connect to PgBouncer using SSL certificates. diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index c87558cb95e..818f7163bc6 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -8,9 +8,11 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/02_pem_pgbouncer_preparing_dbserver/ --- -You must configure the PEM database server to work with PgBouncer. This example shows how to configure the PEM database server. +You must configure dedicated users and create an SSL key and certificate on the PEM database server to enable connection pooling for PEM with PgBouncer. -## Creating users and roles for the pgBouncer-PEM connection +This example shows how to prepare the PEM database server. + +## Creating users and roles for PgBouncer-PEM connections 1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership on the PEM database server: @@ -26,7 +28,7 @@ You must configure the PEM database server to work with PgBouncer. This example GRANT ROLE ``` -2. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership on the PEM database server: +1. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership on the PEM database server: ```sql CREATE ROLE pem_admin1 PASSWORD 'ANY_PASSWORD' LOGIN CREATEROLE; @@ -46,7 +48,7 @@ You must configure the PEM database server to work with PgBouncer. This example GRANT ROLE ``` -3. Grant CONNECT privileges to the pgbouncer user on the `pem` database: +1. Grant CONNECT privileges to the pgbouncer user on the `pem` database: ```sql GRANT CONNECT ON DATABASE pem TO pgbouncer; @@ -54,7 +56,7 @@ You must configure the PEM database server to work with PgBouncer. This example GRANT ``` -4. Grant USAGE privileges to the pgbouncer user for the `pem` schema on the `pem` database: +1. Grant USAGE privileges to the pgbouncer user for the `pem` schema on the `pem` database: ```sql GRANT USAGE ON SCHEMA pem TO pgbouncer; @@ -62,7 +64,7 @@ You must configure the PEM database server to work with PgBouncer. This example GRANT ``` -5. Grant EXECUTE privileges to the pgbouncer user on the `pem.get_agent_pool_auth(text)` function in the `pem` database. For example: +1. Grant EXECUTE privileges to the pgbouncer user on the `pem.get_agent_pool_auth(text)` function in the `pem` database. For example: ```sql GRANT EXECUTE ON FUNCTION pem.get_agent_pool_auth(text) TO pgbouncer; @@ -70,7 +72,7 @@ You must configure the PEM database server to work with PgBouncer. This example GRANT ``` -6. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1 on the PEM database server: +1. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1 on the PEM database server: ```sql SELECT pem.create_proxy_agent_user('pem_agent_user1'); @@ -80,9 +82,11 @@ You must configure the PEM database server to work with PgBouncer. This example (1 row) ``` - The function creates a user with the same name and a random password and grants pem_agent and pem_agent_pool roles to the user. This approach allows pgBouncer to use a proxy user on behalf of the agent. + The function creates a user with the same name and a random password and grants pem_agent and pem_agent_pool roles to the user. This approach allows PgBouncer to use a proxy user on behalf of the agent. + +## Updating the configuration files to allow PgBouncer-PEM connections -7. Allow the pgBouncer user to connect to the `pem` database using the SSL authentication method by adding the following entry to the `pg_hba.conf` file of the PEM database server: +1. Allow the pgbouncer user to connect to the `pem` database using the SSL authentication method by adding the following entry to the `pg_hba.conf` file of the PEM database server: !!!note Add the following entry before any `+pem_agent` entries @@ -95,7 +99,7 @@ You must configure the PEM database server to work with PgBouncer. This example hostssl pem +pem_agent_pool 127.0.0.1/32 cert map=pem_agent_pool ``` -8. Allow the PEM server to identify all users involved in the PEM-pgBouncer connection by adding these lines to the `$PGDATA/pg_ident.conf` user mapping file: +1. Allow the PEM server to map all users involved in PgBouncer-PEM connections by adding these lines to the `$PGDATA/pg_ident.conf` user mapping file: ```shell pem_agent_pool pem_agent_pool pem_agent_user1 @@ -103,37 +107,35 @@ You must configure the PEM database server to work with PgBouncer. This example pem_agent_pool pem_agent_pool pgbouncer ``` -9. Reload the PEM server's configuration: +1. Reload the PEM server's configuration: ```shell pg_ctl reload -D $PGDATA ``` -## Creating the SSL key and certificate for pgBouncer-PEM authentication +## Creating the SSL key and certificate for PgBouncer-PEM authentication -Create a key and certificate for the `pem_agent_pool` group role. Copy them to the pgBouncer instance to allow users with `pem_agent_pool` permissions to connect to PEM. +Create a key and certificate for the `pem_agent_pool` group role. Then, move the files to the PgBouncer instance to allow bilateral authentication between the PEM instance and PgBouncer. -1. Create the signing key: +1. Create the signing key with openssl: ```shell openssl genrsa -out pem_agent_pool.key 4096 ``` -1. Create a certificate-signing request (CSR). Replace the DN attributes in `<...>` as you require, and ensure the Common Name (CN) is set to the `pem_agent_pool` group role name: +1. Create a certificate-signing request (CSR). Replace the `-subj` attributes in `<...>` as required. Ensure the Common Name (CN) is set to the `pem_agent_pool` group role name: ```shell openssl req -new -key pem_agent_pool.key -out pem_agent_pool.csr -subj '/C=/ST=/L=/O=/CN=pem_agent_pool' ``` -1. Use the PEM certificate authority (CA) and key to sign the CSR. - - Adapt the paths to the key and certificate according to your environment: +1. Use the PEM CA and key to sign the CSR: ``` - openssl x509 -req -days 365 -in pem_agent_pool.csr -CA /var/lib/edb/as12/data/ca_certificate.crt -CAkey /var/lib/edb/as16/data/ca_key.key -CAcreateserial -out pem_agent_pool.crt + openssl x509 -req -days 365 -in pem_agent_pool.csr -CA /var/lib/edb/as16/data/ca_certificate.crt -CAkey /var/lib/edb/as16/data/ca_key.key -CAcreateserial -out pem_agent_pool.crt ``` -1. Copy and paste the key and certificate in the pgbouncer user's `~/.postgresql` directory and ensure the `enterprisedb` user has permissions: +1. Move the created key and certificate to the pgbouncer user's `~/.postgresql` directory and ensure the `enterprisedb` user has permissions: ``` mkdir -p /var/lib/edb/.postgresql From eea4611d9aac50576588171e79ff2f6deb2fa096 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 09:19:59 +0200 Subject: [PATCH 08/39] Implemented feedback by Josh E. --- .../docs/pem/9/considerations/pem_pgbouncer/index.mdx | 2 +- .../pem_pgbouncer/preparing_the_pem_database_server.mdx | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx index 687eb5eb2a1..32c71a2edda 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx @@ -19,7 +19,7 @@ navigation: You can use PgBouncer as a connection pooler for limiting the number of connections from the PEM agent to the Postgres Enterprise Manager (PEM) server on non-Windows machines: -- [PEM server and agent connection management mechanism](pem_server_pem_agent_connection_management_mechanism) provides an introduction of the PgBouncer-PEM infrastructe. +- [PEM server and agent connection management mechanism](pem_server_pem_agent_connection_management_mechanism) provides an introduction of the PgBouncer-PEM infrastructure. - [Preparing the PEM Database Server](preparing_the_pem_database_server) provides information about preparing the PEM database server to be used with PgBouncer. - [Configuring PgBouncer](configuring_pgBouncer) provides detailed information about configuring PgBouncer to allow it to work with the PEM database server. - [Configuring the PEM agent](configuring_the_pem_agent) provides detailed information about configuring a PEM agent to connect to PgBouncer. diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 818f7163bc6..90f5d4fe001 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -86,11 +86,9 @@ This example shows how to prepare the PEM database server. ## Updating the configuration files to allow PgBouncer-PEM connections -1. Allow the pgbouncer user to connect to the `pem` database using the SSL authentication method by adding the following entry to the `pg_hba.conf` file of the PEM database server: +1. Allow the pgbouncer user to connect to the `pem` database using the SSL authentication method by adding the `hostssl pem` entry in the `pg_hba.conf` file of the PEM database server. - !!!note - Add the following entry before any `+pem_agent` entries - !!! + In the list of records, ensure you place the `hostssl pem` entry before any other records assigned to the `+pem_agent` user. ```shell # Allow the PEM agent proxy user (used by pgbouncer) From 40cd2174be2b02b57aad8e9d9801041a648362f1 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 09:22:50 +0200 Subject: [PATCH 09/39] Implemented feedback from Jun and Shubham --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index 4f91ba6342b..ca4bdef20e6 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -13,13 +13,12 @@ You must configure PgBouncer to work with the PEM database server. The name and location of the directories and files in the configuration steps depend on whether you installed the community version of PgBouncer or EDB PgBouncer. The instructions assume you are using EDB PgBouncer. If you have installed PgBouncer from the community repo, replace the names of the files and directories in the example with the PgBouncer values. -| Name | PgBouncer | EDB PgBouncer | -|---------------------|------------------------------------|----------------------------------------| -| PgBouncer directory | `/etc/pgbouncer<1.x>` | `/etc/edb/pgbouncer<1.x>` | -| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | -| userlist file | `/etc/pgbouncer<1.x>/userlist.txt` | `/etc/edb/pgbouncer<1.x>/userlist.txt` | -| HBA file | `(/etc/pgbouncer<1.x>/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | -| Service file | `pgbouncer-<1.x>` | `edb-pgbouncer-<1.x>` | +| Name | PgBouncer | EDB PgBouncer | +|---------------------|----------------------------------|--------------------------------------| +| PgBouncer directory | `/etc/pgbouncer<1.x>` | `/etc/edb/pgbouncer<1.x>` | +| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | +| HBA file | `(/etc/pgbouncer<1.x>/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | +| Service file | `pgbouncer-<1.x>` | `edb-pgbouncer-<1.x>` | This example runs PgBouncer as the enterprisedb system user and outlines the process of configuring PgBouncer. @@ -40,8 +39,7 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro ;; to the PEM database server as required. ;; 'auth_user' will be used for authenticate the db user (proxy ;; agent user in our case) - pem = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer - pool_size=80 pool_mode=transaction + pem = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=80 pool_mode=transaction * = port=5444 host=127.0.0.1 dbname=pem auth_user=pgbouncer pool_size=10 [pgbouncer] From d6b8e402ea4c8681e8c58dc8540de973709e3241 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 10:11:47 +0200 Subject: [PATCH 10/39] Exchanged md5 with scram-sha-256 acc. to Jun's feedback --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index ca4bdef20e6..cbb87c49cee 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -93,19 +93,17 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro 4. Create an HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` for PgBouncer that contains the following content: ```ini - # Use authentication method md5 for the local connections to - # connect pem database & pgbouncer (virtual) database. - local pgbouncer all md5 - # Use authentication method md5 for the remote connections to - # connect to pgbouncer (virtual database) using enterprisedb - # user. - - host pgbouncer,pem pem_admin1 0.0.0.0/0 md5 + # Use the authentication method scram-sha-256 for local connections + # between the pem database & the pgbouncer (virtual) database. + local pgbouncer all scram-sha-256 + # Use the authentication method scram-sha-256 for remote connections + # to pgbouncer (virtual database) using the enterprisedb user. + host pgbouncer,pem pem_admin1 0.0.0.0/0 scram-sha-256 ``` ```ini - # Use authentication method cert for the TCP/IP connections to - # connect the pem database using pem_agent_user1 + # Use the authentication method cert for TCP/IP connections + # to the pem database using pem_agent_user1 hostssl pem pem_agent_user1 0.0.0.0/0 cert ``` From 95bfd426f3d73e0abe87d6f9df7f2e7e3796ace7 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 11:50:20 +0200 Subject: [PATCH 11/39] Add diagram --- .../pem_server_pem_agent_connection_management_mechanism.mdx | 2 +- product_docs/docs/pem/9/images/pem_database_with_pgbouncer.png | 3 --- product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100755 product_docs/docs/pem/9/images/pem_database_with_pgbouncer.png create mode 100644 product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 017cd7614ce..1a5735efd91 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -19,6 +19,6 @@ Prior to PEM version 7.5, the following limitations disallowed the use of the co EDB modified the PEM agent to allow the agent to use a common database user (instead of the dedicated agent users) to connect to the PEM database server. -![Connecting to pgBouncer.](../../images/pem_database_with_pgbouncer.png) +![Connecting to pgBouncer.](../../images/pem_db_with_pgbouncer.png) We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. PEM agents can connect to PgBouncer using SSL certificates. diff --git a/product_docs/docs/pem/9/images/pem_database_with_pgbouncer.png b/product_docs/docs/pem/9/images/pem_database_with_pgbouncer.png deleted file mode 100755 index ea025f63969..00000000000 --- a/product_docs/docs/pem/9/images/pem_database_with_pgbouncer.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:226d52b5c8e60a4d22ca95b4951a5e70e0814f723050bde1542a2b4fc51de929 -size 197095 diff --git a/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png b/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png new file mode 100644 index 00000000000..90c673d2235 --- /dev/null +++ b/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a84aca5d03e2e30abd75ce4f9c14b1b6c45587a3dc81aaecada7dbe18eb0704 +size 189178 From fb34851aeda8c998e3a9c01af6d44a947d1c10bf Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 12:04:46 +0200 Subject: [PATCH 12/39] Streamlined change to other diagram --- .../pem_server_pem_agent_connection_management_mechanism.mdx | 2 +- .../docs/pem/9/images/pem_database_without_pgbouncer.png | 3 --- product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100755 product_docs/docs/pem/9/images/pem_database_without_pgbouncer.png create mode 100644 product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 1a5735efd91..0777c5e200f 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -10,7 +10,7 @@ redirects: Each PEM agent connects to the PEM database server using the SSL certificates for each user. For example, an agent with `ID#1` connects to the PEM database server using the agent1 user. -![Connecting to the PEM database without PgBouncer](../../images/pem_database_without_pgbouncer.png) +![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) Prior to PEM version 7.5, the following limitations disallowed the use of the connection pooler between the PEM server and PEM agent: diff --git a/product_docs/docs/pem/9/images/pem_database_without_pgbouncer.png b/product_docs/docs/pem/9/images/pem_database_without_pgbouncer.png deleted file mode 100755 index 2eca5f6c393..00000000000 --- a/product_docs/docs/pem/9/images/pem_database_without_pgbouncer.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9c376a383bbc5a5999589d5932fa27004f6453dc3f14bf99686143743f8147fc -size 180375 diff --git a/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png b/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png new file mode 100644 index 00000000000..86530287faa --- /dev/null +++ b/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e8a00e26842de54ecfca3b0cf50769ccba035a967aeb0a8604a594c0744c561 +size 216058 From e23f9bc1eab4aec32e9107f2c37cbe28477767b0 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 19 Aug 2024 12:13:44 +0200 Subject: [PATCH 13/39] Remove userlist text since it is no longer recommended --- .../9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx | 3 --- 1 file changed, 3 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index cbb87c49cee..ec1c0b73c83 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -63,9 +63,6 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro server_tls_cert_file = /var/lib/edb/.postgresql/pem_agent_pool.crt ;; Use hba file for client connections auth_type = hba - ;; Authentication file, Reference: - ;; https://pgbouncer.github.io/config.html#auth_file - auth_file = /etc/edb/pgbouncer<1.x>/userlist.txt ;; HBA file auth_hba_file = /etc/edb/pgbouncer<1.x>/hba_file ;; Use pem.get_agent_pool_auth(TEXT) function to authenticate From 7ba5f83050b4cdb6ba0b0a933c076aabb9412513 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 13:44:17 +0200 Subject: [PATCH 14/39] Initial changes for Preparing the PEM database server --- .../preparing_the_pem_database_server.mdx | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 90f5d4fe001..8a04b0f6aa6 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -10,11 +10,23 @@ redirects: You must configure dedicated users and create an SSL key and certificate on the PEM database server to enable connection pooling for PEM with PgBouncer. -This example shows how to prepare the PEM database server. +This example shows how to prepare the PEM database server on a RHEL-based operating system with EDB Postgres Advanced Server version 16. The location of your data, configuration and key files might differ depending on your OS and Postgres distribution. + +## Prerequisites + +- You are connected to the `pem` database of the PEM database server. + +- You are connected as `enterprisedb` or `postgres` user. The user depends on your Postgres distribution. + + | Postgres distribution | User | + |------------------------------|--------------| + | EDB Postgres Advanced Server | enterprisedb | + | EDB Postgres Extended Server | postgres | + | PostgreSQL | postgres | ## Creating users and roles for PgBouncer-PEM connections -1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership on the PEM database server: +1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership: ```sql CREATE ROLE pgbouncer PASSWORD 'ANY_PASSWORD' LOGIN; @@ -28,7 +40,7 @@ This example shows how to prepare the PEM database server. GRANT ROLE ``` -1. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership on the PEM database server: +1. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership: ```sql CREATE ROLE pem_admin1 PASSWORD 'ANY_PASSWORD' LOGIN CREATEROLE; @@ -43,12 +55,12 @@ This example shows how to prepare the PEM database server. ``` ```sql - GRANT pem_admin TO pem_admin1 WITH ADMIN OPTION; + GRANT pem_agent TO pem_admin1 WITH ADMIN OPTION; __OUTPUT__ GRANT ROLE ``` -1. Grant CONNECT privileges to the pgbouncer user on the `pem` database: +1. Grant CONNECT privileges to the pgbouncer user: ```sql GRANT CONNECT ON DATABASE pem TO pgbouncer; @@ -56,7 +68,7 @@ This example shows how to prepare the PEM database server. GRANT ``` -1. Grant USAGE privileges to the pgbouncer user for the `pem` schema on the `pem` database: +1. Grant USAGE privileges to the pgbouncer user for the `pem` schema: ```sql GRANT USAGE ON SCHEMA pem TO pgbouncer; @@ -64,7 +76,7 @@ This example shows how to prepare the PEM database server. GRANT ``` -1. Grant EXECUTE privileges to the pgbouncer user on the `pem.get_agent_pool_auth(text)` function in the `pem` database. For example: +1. Grant EXECUTE privileges to the pgbouncer user on the `pem.get_agent_pool_auth(text)` function. For example: ```sql GRANT EXECUTE ON FUNCTION pem.get_agent_pool_auth(text) TO pgbouncer; @@ -72,7 +84,7 @@ This example shows how to prepare the PEM database server. GRANT ``` -1. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1 on the PEM database server: +1. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1: ```sql SELECT pem.create_proxy_agent_user('pem_agent_user1'); @@ -105,10 +117,10 @@ This example shows how to prepare the PEM database server. pem_agent_pool pem_agent_pool pgbouncer ``` -1. Reload the PEM server's configuration: +1. Restart the Postgres service. Replace the `` placeholder with the name of the Postgres instance systemd service name: ```shell - pg_ctl reload -D $PGDATA + systemctl restart ``` ## Creating the SSL key and certificate for PgBouncer-PEM authentication @@ -133,7 +145,9 @@ Create a key and certificate for the `pem_agent_pool` group role. Then, move the openssl x509 -req -days 365 -in pem_agent_pool.csr -CA /var/lib/edb/as16/data/ca_certificate.crt -CAkey /var/lib/edb/as16/data/ca_key.key -CAcreateserial -out pem_agent_pool.crt ``` -1. Move the created key and certificate to the pgbouncer user's `~/.postgresql` directory and ensure the `enterprisedb` user has permissions: +1. Move the created key and certificate to a path the `enterprisedb` user can access. + + In this example, create a folder called `~/.postgresql` in the home directory of the `enterprisedb` user and ensure it has permissions: ``` mkdir -p /var/lib/edb/.postgresql From 419d67efbb9391f9fced84a5985dbc84da7a6ed0 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 13:48:20 +0200 Subject: [PATCH 15/39] Added note on user usage --- .../pem_pgbouncer/preparing_the_pem_database_server.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 8a04b0f6aa6..39fa6e44193 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -10,7 +10,7 @@ redirects: You must configure dedicated users and create an SSL key and certificate on the PEM database server to enable connection pooling for PEM with PgBouncer. -This example shows how to prepare the PEM database server on a RHEL-based operating system with EDB Postgres Advanced Server version 16. The location of your data, configuration and key files might differ depending on your OS and Postgres distribution. +This example shows how to prepare the PEM database serverwith the `enterprisedb` user on a RHEL-based operating system with EDB Postgres Advanced Server version 16. The location of your data, the configuration and key files, and the user you employ to perform the configuration are different depending on your OS and Postgres distribution. ## Prerequisites From 19488a0cf525c3c069ad0ec1d2cab92cdc43424f Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 14:24:27 +0200 Subject: [PATCH 16/39] first edtis to Configuring PgBouncer --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index ec1c0b73c83..513d9d82d0a 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -3,35 +3,47 @@ title: "Configuring PgBouncer" legacyRedirectsGenerated: # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. - "/edb-docs/d/edb-postgres-enterprise-manager/installation-getting-started/pgbouncer-configuration-guide/8.0/configuring_pgBouncer.html" +deepToC: true redirects: - /pem/latest/pem_pgbouncer/03_configuring_pgBouncer/ - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/03_pem_pgbouncer_configuring_pgbouncer/ --- -You must configure PgBouncer to work with the PEM database server. +You must configure PgBouncer to work with the PEM database server. -The name and location of the directories and files in the configuration steps depend on whether you installed the community version of PgBouncer or EDB PgBouncer. The instructions assume you are using EDB PgBouncer. If you have installed PgBouncer from the community repo, replace the names of the files and directories in the example with the PgBouncer values. +## Prerequisite +- You have installed [EDB PgBouncer](/pgbouncer/latest/installing) if you are running EDB Postgres Advanced Server. -| Name | PgBouncer | EDB PgBouncer | -|---------------------|----------------------------------|--------------------------------------| -| PgBouncer directory | `/etc/pgbouncer<1.x>` | `/etc/edb/pgbouncer<1.x>` | -| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | -| HBA file | `(/etc/pgbouncer<1.x>/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | -| Service file | `pgbouncer-<1.x>` | `edb-pgbouncer-<1.x>` | + Or -This example runs PgBouncer as the enterprisedb system user and outlines the process of configuring PgBouncer. +- You have installed open-source [PgBouncer](https://www.pgbouncer.org/install.html) if you are running EDB Postgres Extended Server or PostgreSQL. + +## Location of PgBouncer directories + +The name and location of the directories and files in the configuration steps depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed PgBouncer from the community repo, replace the names of the files and directories in the example with the PgBouncer values. + +| Name | PgBouncer | EDB PgBouncer | +|---------------------|-----------------------------|--------------------------------------| +| PgBouncer directory | `/etc/pgbouncer` | `/etc/edb/pgbouncer<1.x>` | +| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | +| HBA file | `(/etc/pgbouncer/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | +| Service file | `pgbouncer` | `edb-pgbouncer-<1.x>` | + +## Configuring PgBouncer + +This example runs EDB PgBouncer as the `enterprisedb` system user and outlines the process of configuring PgBouncer. 1. Open a terminal window and navigate to the PgBouncer directory. -2. Change the owner of the `etc` directory for PgBouncer (where `pgbouncer.ini` resides) to `enterprisedb`, and change the directory permissions to `0700`: +1. Change the owner of the `etc` directory for PgBouncer (where `pgbouncer.ini` resides) to `enterprisedb`, and change the directory permissions to `0700`: ```shell $ chown -R enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x> $ chmod 0700 /etc/edb/pgbouncer<1.x> ``` -3. Change the contents of the `pgbouncer.ini` or `edb-pgbouncer.ini` file: +1. Change the contents of the `pgbouncer.ini` or `edb-pgbouncer.ini` file: ```ini [databases] @@ -84,10 +96,11 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro server_idle_timeout = 60 ``` -!!! Note - For more information on `auth_user` see [Authentication settings](https://www.pgbouncer.org/config.html#authentication-settings). + !!!note + For more information on `auth_user` see [Authentication settings](https://www.pgbouncer.org/config.html#authentication-settings). + !!! -4. Create an HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` for PgBouncer that contains the following content: +1. Create an HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` for PgBouncer that contains the following content: ```ini # Use the authentication method scram-sha-256 for local connections @@ -96,23 +109,19 @@ This example runs PgBouncer as the enterprisedb system user and outlines the pro # Use the authentication method scram-sha-256 for remote connections # to pgbouncer (virtual database) using the enterprisedb user. host pgbouncer,pem pem_admin1 0.0.0.0/0 scram-sha-256 - ``` - - ```ini # Use the authentication method cert for TCP/IP connections # to the pem database using pem_agent_user1 - hostssl pem pem_agent_user1 0.0.0.0/0 cert ``` -5. Change the owner of the HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` to `enterprisedb`, and change the directory permissions to `0600`: +1. Change the owner of the HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` to `enterprisedb`, and change the directory permissions to `0600`: ```shell $ chown enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x>/hba_file $ chmod 0600 /etc/edb/pgbouncer<1.x>/hba_file ``` -6. Enable the PgBouncer service, and start the service: +1. Enable the PgBouncer service, and start the service: ```shell $ systemctl enable edb-pgbouncer-<1.x> From fece50427adbe61d33486130f397ec591574493e Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 14:59:08 +0200 Subject: [PATCH 17/39] Initial changes to Configuring the PEM agent --- .../configuring_the_pem_agent.mdx | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index 33902865928..6391bc91e1b 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -8,16 +8,30 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/04_pem_pgbouncer_configuring_pem_agent/ --- -You can use an RPM package to install a PEM agent. For detailed installation information, see [Installating the PEM agent](../../installing_pem_agent/). +## Prerequisite -Don't configure the PEM agent responsible for sending SNMP notifications with PgBouncer. For example, if the default PEM agent installed with PEM server is used for SNMP notifications, don't configure it with PgBouncer. +You have [installed the PEM agent](../../installing_pem_agent/). -## Configuring a new PEM agent (installed via RPM) +!!!note + Don't configure the PEM agent responsible for sending SNMP notifications with PgBouncer. For example, if the default PEM agent installed with PEM server is used for SNMP notifications, don't configure it with PgBouncer. +!!! -After using an RPM package to install the PEM agent, you must configure it to work against a particular PEM database server. Use the following command: +Now you can choose to [configure a new PEM agent](#configuring-a-new-pem-agent) or [use an existing PEM agent](#configuring-an-existing-pem-agent) for PgBouncer. + +## Configuring a new PEM agent + +After installing the PEM agent, configure it to work with a particular PEM database server. Use the following command: ```shell -$ PGSSLMODE=require PEM_SERVER_PASSWORD=pem_admin1_password /usr/edb/pem/agent/bin/pemworker --register-agent --pem-server 172.16.254.22 --pem-port 6432 --pem-user pem_admin1 --pem-agent-user pem_agent_user1 --display-name *Agent_Name* +$ PGSSLMODE=require PEM_SERVER_PASSWORD=pem_admin1_password \ + /usr/edb/pem/agent/bin/pemworker \ + --register-agent \ + --pem-server 172.16.254.22 \ + --pem-port 6432 \ + --pem-user pem_admin1 \ + --pem-agent-user pem_agent_user1 \ + --display-name *Agent_Name* \ +__OUTPUT__ Postgres Enterprise Manager Agent registered successfully! ``` @@ -64,7 +78,7 @@ allow_batch_probes=false heartbeat_connection=false ``` -## Configuring an existing PEM agent (installed via RPM) +## Configuring an existing PEM agent If you're using an existing PEM agent, you can copy the SSL certificate and key files to the target machine and reuse the files. You must modify the files, adding a new parameter and replacing some parameters in the existing `agent.cfg` file. From 7f2e1d8630df20c962cbed6142b8da6880430a1a Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 15:24:19 +0200 Subject: [PATCH 18/39] Initial changes to PEM server and agent connection management mechanism --- ...rver_pem_agent_connection_management_mechanism.mdx | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 0777c5e200f..1bc7c9ea04c 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -8,17 +8,12 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/01_pem_pgbouncer_server_agent_connection/ --- -Each PEM agent connects to the PEM database server using the SSL certificates for each user. For example, an agent with `ID#1` connects to the PEM database server using the agent1 user. +In the default configuration, each PEM agent connects to the PEM database server using the SSL certificates for each user. For example, an agent with `ID#1` connects to the PEM database server using the agent1 user. ![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) -Prior to PEM version 7.5, the following limitations disallowed the use of the connection pooler between the PEM server and PEM agent: - -- The PEM agent uses an SSL certificate to connect to the PEM database server. -- It uses an individual user identifier when connecting to the PEM database server. - -EDB modified the PEM agent to allow the agent to use a common database user (instead of the dedicated agent users) to connect to the PEM database server. +In a PgBouncer-enabled environment, each PEM agent connects to PgBouncer using SSL certificates via the `pem_agent_user1` user. Upon authentication, PgBouncer manages connection requests efficiently, for example, by reusing existing connections, limiting the number of active connections. ![Connecting to pgBouncer.](../../images/pem_db_with_pgbouncer.png) -We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. PEM agents can connect to PgBouncer using SSL certificates. +We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. From 653a3d12db8f968f4619d801cccf521a87c4c7ee Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 22 Aug 2024 15:39:54 +0200 Subject: [PATCH 19/39] minor clean-up corrections --- .../configuring_the_pem_agent.mdx | 28 +++++++++---------- .../9/considerations/pem_pgbouncer/index.mdx | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index 6391bc91e1b..97eb8f3530b 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -82,26 +82,26 @@ heartbeat_connection=false If you're using an existing PEM agent, you can copy the SSL certificate and key files to the target machine and reuse the files. You must modify the files, adding a new parameter and replacing some parameters in the existing `agent.cfg` file. -Add a line to use agent_user as the agent: +1. Add a line to use agent_user as the agent: -```ini -agent_user=pem_agent_user1 -``` + ```ini + agent_user=pem_agent_user1 + ``` -Update the port to specify the PgBouncer port: +1. Update the port to specify the PgBouncer port: -```ini -pem_port=6432 -``` + ```ini + pem_port=6432 + ``` -Update the certificate and key path locations: +1. Update the certificate and key path locations: -```ini -agent_ssl_key=/root/.pem/pem_agent_user1.key -agent_ssl_crt=/root/.pem/pem_agent_user1.crt -``` + ```ini + agent_ssl_key=/root/.pem/pem_agent_user1.key + agent_ssl_crt=/root/.pem/pem_agent_user1.crt + ``` -As an alternative, you can run the agent self-registration script. However, that process creates a new agent id. If run the agent self-registration script, you must replace the new agent id with the existing id and disable the entry for the new agent id in the `pem.agent` table. For example: +As an alternative, you can run the agent self-registration script. However, that process creates a new agent id. If you run the agent self-registration script, you must replace the new agent id with the existing id and disable the entry for the new agent id in the `pem.agent` table. For example: ```sql pem=# UPDATE pem.agent SET active = false WHERE id = ; diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx index 32c71a2edda..ecdd5a57885 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/index.mdx @@ -24,4 +24,4 @@ You can use PgBouncer as a connection pooler for limiting the number of connecti - [Configuring PgBouncer](configuring_pgBouncer) provides detailed information about configuring PgBouncer to allow it to work with the PEM database server. - [Configuring the PEM agent](configuring_the_pem_agent) provides detailed information about configuring a PEM agent to connect to PgBouncer. -For detailed information about using the PEM web interface, see the [Accessing the web interface ](../../pem_web_interface). \ No newline at end of file +For detailed information about using the PEM web interface, see the [Accessing the web interface](../../pem_web_interface). \ No newline at end of file From e605062ba05fc8e639e751084d9d086a552ed3be Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 23 Aug 2024 13:51:06 +0200 Subject: [PATCH 20/39] Reworked text around diagrams in PEM server and agent page + added wording regarding the postgres/enterprisedb user for PgBouncer config --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 25 +++++++++++-------- ..._agent_connection_management_mechanism.mdx | 17 ++++++++++--- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index 513d9d82d0a..9bc02d047b1 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -13,37 +13,40 @@ You must configure PgBouncer to work with the PEM database server. ## Prerequisite -- You have installed [EDB PgBouncer](/pgbouncer/latest/installing) if you are running EDB Postgres Advanced Server. +- If you are running EDB Postgres Advanced Server, you have installed [EDB PgBouncer](/pgbouncer/latest/installing). Or -- You have installed open-source [PgBouncer](https://www.pgbouncer.org/install.html) if you are running EDB Postgres Extended Server or PostgreSQL. +- If you are running EDB Postgres Extended Server or PostgreSQL, you have installed community [PgBouncer](https://www.pgbouncer.org/install.html). -## Location of PgBouncer directories +### EDB PgBouncer and PgBouncer installation considerations -The name and location of the directories and files in the configuration steps depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed PgBouncer from the community repo, replace the names of the files and directories in the example with the PgBouncer values. +The name and location of the directories and files in the configuration steps, as well as the user, depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed community PgBouncer from the community repo or the EDB repo, replace the names of the files and directories in the worked example with the PgBouncer values. -| Name | PgBouncer | EDB PgBouncer | -|---------------------|-----------------------------|--------------------------------------| +| Name | PgBouncer | EDB PgBouncer | +|---------------------|-----------------------------|---------------------------------------| | PgBouncer directory | `/etc/pgbouncer` | `/etc/edb/pgbouncer<1.x>` | -| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | +| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | | HBA file | `(/etc/pgbouncer/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | -| Service file | `pgbouncer` | `edb-pgbouncer-<1.x>` | +| Service file | `pgbouncer` | `edb-pgbouncer-<1.x>` | +| User | `postgres` | `enterprisedb` | ## Configuring PgBouncer -This example runs EDB PgBouncer as the `enterprisedb` system user and outlines the process of configuring PgBouncer. +This example configures EDB PgBouncer with the `enterprisedb` system user. + +If you are running community PgBouncer, replace the names of the directories, files and user as explained in [Location of PgBouncer directories](#edb-pgbouncer-and-pgbouncer-installation-considerations) . 1. Open a terminal window and navigate to the PgBouncer directory. -1. Change the owner of the `etc` directory for PgBouncer (where `pgbouncer.ini` resides) to `enterprisedb`, and change the directory permissions to `0700`: +1. Change the owner of the `etc` directory for PgBouncer (where `edb-pgbouncer.ini` resides) to `enterprisedb`, and change the directory permissions to `0700`: ```shell $ chown -R enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x> $ chmod 0700 /etc/edb/pgbouncer<1.x> ``` -1. Change the contents of the `pgbouncer.ini` or `edb-pgbouncer.ini` file: +1. Change the contents of the `edb-pgbouncer.ini` file: ```ini [databases] diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 1bc7c9ea04c..b40f9aae109 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -8,12 +8,23 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/01_pem_pgbouncer_server_agent_connection/ --- -In the default configuration, each PEM agent connects to the PEM database server using the SSL certificates for each user. For example, an agent with `ID#1` connects to the PEM database server using the agent1 user. +## PEM Agent connections without PgBouncer + +In the default configuration, each PEM agent connects to the PEM database server directly using SSL for encryption. Each PEM agent uses its own dedicated user for the connection. + +For example, a user agent with `ID#1` connects to the PEM database server as agent1. ![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) -In a PgBouncer-enabled environment, each PEM agent connects to PgBouncer using SSL certificates via the `pem_agent_user1` user. Upon authentication, PgBouncer manages connection requests efficiently, for example, by reusing existing connections, limiting the number of active connections. +## PEM Agent connections with PgBouncer + +In a PgBouncer-enabled environment, PEM agents cannot connect to the PEM database server directly. PEM agents must use a proxy user that you configure specifically for the connection to PgBouncer. In the example, the proxy user handling all PEM agent connections is called `pem_agent_user1`. + +Once the PEM agents connect to PgBouncer using SSL, PgBouncer is responsible for managing connection requests to the PEM database server. PgBouncer uses the rules you have configured for connection pooling to manage the incoming connection requests, for example, by respecting the established maximum number of active connections. ![Connecting to pgBouncer.](../../images/pem_db_with_pgbouncer.png) -We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. +!!!note + We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. +!!! + From 0bddd65d43eb576342965a7138b5df5cd5f222cf Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 23 Aug 2024 14:18:16 +0200 Subject: [PATCH 21/39] Set env variable for DATA_DIR --- .../preparing_the_pem_database_server.mdx | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 39fa6e44193..757dc3e5ba6 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -125,7 +125,30 @@ This example shows how to prepare the PEM database serverwith the `enterprisedb` ## Creating the SSL key and certificate for PgBouncer-PEM authentication -Create a key and certificate for the `pem_agent_pool` group role. Then, move the files to the PgBouncer instance to allow bilateral authentication between the PEM instance and PgBouncer. +Create a key and certificate for the `pem_agent_pool` group role. Then, move the files to the PgBouncer instance to allow authentication between the PEM instance and PgBouncer. + +This example creates and configures the SSL files for EDB Postgres Advanced Server on RHEL. +See [SSL and user directories for other operating systems](#ssl-and-user-directories-for-other-operating-systems) if your operating system is different or you are using another Postgres distribution. + +1. Set the `$DATA_DIR` environment variable to your data directory: + + ```shell + export DATA_DIR=/var/lib/edb/as16/data + ``` + +
Data directories for other environments +
+ Here are some examples of other default data directories per operating system and Postgres version. + + | Postgres version | RHEL/Rocky Linux/AlmaLinux/SLES | Debian/Ubuntu | + |------------------------------------|---------------------------------|--------------------------| + | EDB Postgres
Advanced Server | /var/lib/edb/as16/data | /var/lib/edb-as/16/main | + | EDB Postgres
Extended Server | /var/lib/edb/edb-pge/16/data | /var/lib/edb-pge/16/main | + | PostgreSQL | /var/lib/edb/pgsql/16/data | /etc/postgresql/16/main | + +
+ +
1. Create the signing key with openssl: @@ -142,7 +165,7 @@ Create a key and certificate for the `pem_agent_pool` group role. Then, move the 1. Use the PEM CA and key to sign the CSR: ``` - openssl x509 -req -days 365 -in pem_agent_pool.csr -CA /var/lib/edb/as16/data/ca_certificate.crt -CAkey /var/lib/edb/as16/data/ca_key.key -CAcreateserial -out pem_agent_pool.crt + openssl x509 -req -days 365 -in pem_agent_pool.csr -CA $DATA_DIR/ca_certificate.crt -CAkey $DATA_DIR/ca_key.key -CAcreateserial -out pem_agent_pool.crt ``` 1. Move the created key and certificate to a path the `enterprisedb` user can access. @@ -156,3 +179,29 @@ Create a key and certificate for the `pem_agent_pool` group role. Then, move the chmod 0644 /var/lib/edb/.postgresql/pem_agent_pool.crt chown enterprisedb:enterprisedb /var/lib/edb/.postgresql/pem_agent_pool.* ``` + +## SSL and user directories for other operating systems + +For **EDB Postgres Advanced Server** installations: + +| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | +|---------------------|-------------------------------------------|--------------------------------------------| +| Certificate | /var/lib/edb/as16/data/ca_certificate.crt | /var/lib/edb-as/16/main/ca_certificate.crt | +| Key | /var/lib/edb/as16/data/ca_key.key | /var/lib/edb-as/16/main/ca_key.key | +| 'enterprisedb' home | /var/lib/edb/.postgresql | /var/lib/edb-as/.postgresql | + +For **PostgreSQL** installations: + +| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | +|----------------------|-----------------------------------------------|--------------------------------------------| +| Certificate location | /var/lib/edb/pgsql/16/data/ca_certificate.crt | /etc/postgresql/16/main/ca_certificate.crt | +| Key location | /var/lib/edb/pgsql/16/data/ca_key.key | /etc/postgresql/16/main/ca_key.key | +| 'postgres' home | /var/lib/pgsql/.postgresql | /var/lib/postgresql/.postgresql | + +For **EDB Postgres Extended Server** installations: + +| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | +|----------------------|-------------------------------------------------|---------------------------------------------| +| Certificate location | /var/lib/edb/edb-pge/16/data/ca_certificate.crt | /var/lib/edb-pge/16/main/ca_certificate.crt | +| Key location | /var/lib/edb/edb-pge/16/data/ca_key.key | /var/lib/edb-pge/16/main/ca_key.key | +| 'postgres' home | /var/lib/pgsql/.postgresql | /var/lib/postgresql/.postgresql | From 6d840769cb6728184c97eaa2ab7efc74e17ffee0 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 23 Aug 2024 14:47:42 +0200 Subject: [PATCH 22/39] Set env variable for USER_HOME & minor edits --- ..._agent_connection_management_mechanism.mdx | 4 +- .../preparing_the_pem_database_server.mdx | 75 +++++++++---------- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index b40f9aae109..518ac0996df 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -8,7 +8,7 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/01_pem_pgbouncer_server_agent_connection/ --- -## PEM Agent connections without PgBouncer +## Without PgBouncer In the default configuration, each PEM agent connects to the PEM database server directly using SSL for encryption. Each PEM agent uses its own dedicated user for the connection. @@ -16,7 +16,7 @@ For example, a user agent with `ID#1` connects to the PEM database server as age ![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) -## PEM Agent connections with PgBouncer +## With PgBouncer In a PgBouncer-enabled environment, PEM agents cannot connect to the PEM database server directly. PEM agents must use a proxy user that you configure specifically for the connection to PgBouncer. In the example, the proxy user handling all PEM agent connections is called `pem_agent_user1`. diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 757dc3e5ba6..971fbc9b343 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -125,26 +125,45 @@ This example shows how to prepare the PEM database serverwith the `enterprisedb` ## Creating the SSL key and certificate for PgBouncer-PEM authentication -Create a key and certificate for the `pem_agent_pool` group role. Then, move the files to the PgBouncer instance to allow authentication between the PEM instance and PgBouncer. +Create a key and certificate for the `pem_agent_pool` group role. Then, move the files to the PgBouncer instance to allow authentication between the PEM database server and PgBouncer. -This example creates and configures the SSL files for EDB Postgres Advanced Server on RHEL. -See [SSL and user directories for other operating systems](#ssl-and-user-directories-for-other-operating-systems) if your operating system is different or you are using another Postgres distribution. +This example runs EDB Postgres Advanced Server on RHEL. When setting your environment variables, choose the correct directories according to your operating system and Postgres distribution. -1. Set the `$DATA_DIR` environment variable to your data directory: +1. Set the `$DATA_DIR` environment variable to your data directory: ```shell export DATA_DIR=/var/lib/edb/as16/data ``` -
Data directories for other environments +
Data directories per OS and Postgres version
Here are some examples of other default data directories per operating system and Postgres version. - | Postgres version | RHEL/Rocky Linux/AlmaLinux/SLES | Debian/Ubuntu | - |------------------------------------|---------------------------------|--------------------------| - | EDB Postgres
Advanced Server | /var/lib/edb/as16/data | /var/lib/edb-as/16/main | - | EDB Postgres
Extended Server | /var/lib/edb/edb-pge/16/data | /var/lib/edb-pge/16/main | - | PostgreSQL | /var/lib/edb/pgsql/16/data | /etc/postgresql/16/main | + | Postgres version | RHEL/Rocky Linux/AlmaLinux/SLES | Debian/Ubuntu | + |---------------------------------------|---------------------------------|--------------------------| + | EDB Postgres
Advanced Server 16 | /var/lib/edb/as16/data | /var/lib/edb-as/16/main | + | EDB Postgres
Extended Server 16 | /var/lib/edb/edb-pge/16/data | /var/lib/edb-pge/16/main | + | PostgreSQL 16 | /var/lib/edb/pgsql/16/data | /etc/postgresql/16/main | + +
+ +
+ +1. Set the `$USER_HOME` environment variable to the home directory accesible to the user: + + ```shell + export USER_HOME=/var/lib/edb + ``` + +
User home directories per OS and Postgres version +
+ Here are some examples of other default home directories per operating system and Postgres version. + + | Postgres version | RHEL/Rocky Linux/AlmaLinux/SLES | Debian/Ubuntu | + |---------------------------------------|---------------------------------|---------------------| + | EDB Postgres
Advanced Server 16 | /var/lib/edb | /var/lib/edb-as | + | EDB Postgres
Extended Server 16 | /var/lib/pgsql | /var/lib/postgresql | + | PostgreSQL 16 | /var/lib/pgsql | /var/lib/postgresql |
@@ -173,35 +192,9 @@ See [SSL and user directories for other operating systems](#ssl-and-user-directo In this example, create a folder called `~/.postgresql` in the home directory of the `enterprisedb` user and ensure it has permissions: ``` - mkdir -p /var/lib/edb/.postgresql - mv pem_agent_pool.key pem_agent_pool.crt /var/lib/edb/.postgresql - chmod 0600 /var/lib/edb/.postgresql/pem_agent_pool.key - chmod 0644 /var/lib/edb/.postgresql/pem_agent_pool.crt - chown enterprisedb:enterprisedb /var/lib/edb/.postgresql/pem_agent_pool.* + mkdir -p $USER_HOME/.postgresql + mv pem_agent_pool.key pem_agent_pool.crt $USER_HOME/.postgresql + chmod 0600 $USER_HOME/.postgresql/pem_agent_pool.key + chmod 0644 $USER_HOME/.postgresql/pem_agent_pool.crt + chown enterprisedb:enterprisedb $USER_HOME/.postgresql/pem_agent_pool.* ``` - -## SSL and user directories for other operating systems - -For **EDB Postgres Advanced Server** installations: - -| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | -|---------------------|-------------------------------------------|--------------------------------------------| -| Certificate | /var/lib/edb/as16/data/ca_certificate.crt | /var/lib/edb-as/16/main/ca_certificate.crt | -| Key | /var/lib/edb/as16/data/ca_key.key | /var/lib/edb-as/16/main/ca_key.key | -| 'enterprisedb' home | /var/lib/edb/.postgresql | /var/lib/edb-as/.postgresql | - -For **PostgreSQL** installations: - -| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | -|----------------------|-----------------------------------------------|--------------------------------------------| -| Certificate location | /var/lib/edb/pgsql/16/data/ca_certificate.crt | /etc/postgresql/16/main/ca_certificate.crt | -| Key location | /var/lib/edb/pgsql/16/data/ca_key.key | /etc/postgresql/16/main/ca_key.key | -| 'postgres' home | /var/lib/pgsql/.postgresql | /var/lib/postgresql/.postgresql | - -For **EDB Postgres Extended Server** installations: - -| Directory | RHEL/Rocky Linux/AlmaLinux/SLES systems | Debian/Ubuntu systems | -|----------------------|-------------------------------------------------|---------------------------------------------| -| Certificate location | /var/lib/edb/edb-pge/16/data/ca_certificate.crt | /var/lib/edb-pge/16/main/ca_certificate.crt | -| Key location | /var/lib/edb/edb-pge/16/data/ca_key.key | /var/lib/edb-pge/16/main/ca_key.key | -| 'postgres' home | /var/lib/pgsql/.postgresql | /var/lib/postgresql/.postgresql | From bfe54c98e8b9122748f7ba947412778cc3a5fc97 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Tue, 27 Aug 2024 08:52:59 +0200 Subject: [PATCH 23/39] Removing $ from code snippets to allow executing commands upon copy-paste --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 14 +++++++------- .../pem_pgbouncer/configuring_the_pem_agent.mdx | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index 9bc02d047b1..b1121654281 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -21,7 +21,7 @@ You must configure PgBouncer to work with the PEM database server. ### EDB PgBouncer and PgBouncer installation considerations -The name and location of the directories and files in the configuration steps, as well as the user, depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed community PgBouncer from the community repo or the EDB repo, replace the names of the files and directories in the worked example with the PgBouncer values. +The name and location of the directories and files in the configuration steps, as well as the user, depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed community PgBouncer (whether you install it from the community repo or the EDB repo), replace the names of the files and directories in the worked example with the values for PgBouncer. | Name | PgBouncer | EDB PgBouncer | |---------------------|-----------------------------|---------------------------------------| @@ -42,8 +42,8 @@ If you are running community PgBouncer, replace the names of the directories, fi 1. Change the owner of the `etc` directory for PgBouncer (where `edb-pgbouncer.ini` resides) to `enterprisedb`, and change the directory permissions to `0700`: ```shell - $ chown -R enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x> - $ chmod 0700 /etc/edb/pgbouncer<1.x> + chown -R enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x> + chmod 0700 /etc/edb/pgbouncer<1.x> ``` 1. Change the contents of the `edb-pgbouncer.ini` file: @@ -120,14 +120,14 @@ If you are running community PgBouncer, replace the names of the directories, fi 1. Change the owner of the HBA file `(/etc/edb/pgbouncer<1.x>/hba_file)` to `enterprisedb`, and change the directory permissions to `0600`: ```shell - $ chown enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x>/hba_file - $ chmod 0600 /etc/edb/pgbouncer<1.x>/hba_file + chown enterprisedb:enterprisedb /etc/edb/pgbouncer<1.x>/hba_file + chmod 0600 /etc/edb/pgbouncer<1.x>/hba_file ``` 1. Enable the PgBouncer service, and start the service: ```shell - $ systemctl enable edb-pgbouncer-<1.x> + systemctl enable edb-pgbouncer-<1.x> __OUTPUT__ Created symlink from /etc/systemd/system/multi-user.target.wants/edb-pgbouncer-<1.x>.service @@ -135,5 +135,5 @@ If you are running community PgBouncer, replace the names of the directories, fi ``` ```shell - $ systemctl start edb-pgbouncer-<1.x> + systemctl start edb-pgbouncer-<1.x> ``` diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index 97eb8f3530b..22a1fd9e9fc 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -23,7 +23,7 @@ Now you can choose to [configure a new PEM agent](#configuring-a-new-pem-agent) After installing the PEM agent, configure it to work with a particular PEM database server. Use the following command: ```shell -$ PGSSLMODE=require PEM_SERVER_PASSWORD=pem_admin1_password \ +PGSSLMODE=require PEM_SERVER_PASSWORD=pem_admin1_password \ /usr/edb/pem/agent/bin/pemworker \ --register-agent \ --pem-server 172.16.254.22 \ @@ -50,7 +50,7 @@ The PEM agent uses the keys to connect to the PEM database server as pem_agent_u A line mentioning the agent-user to use appears in the `agent.cfg` configuration file. For example: ```ini -$ cat /usr/edb/pem/agent/etc/agent.cfg +cat /usr/edb/pem/agent/etc/agent.cfg [PEM/agent] pem_host=172.16.254.22 pem_port=6432 From 9ada96f227b3fed6f78f95297706eaff9052a43c Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 29 Aug 2024 12:14:31 +0200 Subject: [PATCH 24/39] Implementing feedback by Dave: wordings and typos --- .../9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx | 4 ++-- .../pem_pgbouncer/configuring_the_pem_agent.mdx | 2 +- .../pem_server_pem_agent_connection_management_mechanism.mdx | 4 ++-- .../pem_pgbouncer/preparing_the_pem_database_server.mdx | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index b1121654281..427705f0041 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -11,7 +11,7 @@ redirects: You must configure PgBouncer to work with the PEM database server. -## Prerequisite +## Prerequisits - If you are running EDB Postgres Advanced Server, you have installed [EDB PgBouncer](/pgbouncer/latest/installing). @@ -63,7 +63,7 @@ If you are running community PgBouncer, replace the names of the directories, fi listen_addr = * ;; Agent needs to use this port to connect the pem database now listen_port = 6432 - ;; Set to require to support SSL Certificate authentication + ;; Set to require to ensure SSL certificates are used for connections ;; for PEM Agents client_tls_sslmode = require ;; These are the root.crt, server.key, server.crt files present diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index 22a1fd9e9fc..b494ccc2dc8 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -8,7 +8,7 @@ redirects: - /pem/latest/pem_online_help/09_toc_pem_configure_pgbouncer/04_pem_pgbouncer_configuring_pem_agent/ --- -## Prerequisite +## Prerequisites You have [installed the PEM agent](../../installing_pem_agent/). diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 518ac0996df..508b3c0ac63 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -12,7 +12,7 @@ redirects: In the default configuration, each PEM agent connects to the PEM database server directly using SSL for encryption. Each PEM agent uses its own dedicated user for the connection. -For example, a user agent with `ID#1` connects to the PEM database server as agent1. +For example, a user agent with ID 1 connects to the PEM database server as agent1. ![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) @@ -25,6 +25,6 @@ Once the PEM agents connect to PgBouncer using SSL, PgBouncer is responsible for ![Connecting to pgBouncer.](../../images/pem_db_with_pgbouncer.png) !!!note - We recommend using PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. + Use PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. !!! diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 971fbc9b343..0682ce30cc9 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -10,7 +10,7 @@ redirects: You must configure dedicated users and create an SSL key and certificate on the PEM database server to enable connection pooling for PEM with PgBouncer. -This example shows how to prepare the PEM database serverwith the `enterprisedb` user on a RHEL-based operating system with EDB Postgres Advanced Server version 16. The location of your data, the configuration and key files, and the user you employ to perform the configuration are different depending on your OS and Postgres distribution. +This example shows how to prepare the PEM database server with the `enterprisedb` user on a RHEL-based operating system with EDB Postgres Advanced Server version 16. The location of your data, the configuration and key files, and the user you employ to perform the configuration may differ depending on your OS and Postgres distribution. ## Prerequisites @@ -100,7 +100,7 @@ This example shows how to prepare the PEM database serverwith the `enterprisedb` 1. Allow the pgbouncer user to connect to the `pem` database using the SSL authentication method by adding the `hostssl pem` entry in the `pg_hba.conf` file of the PEM database server. - In the list of records, ensure you place the `hostssl pem` entry before any other records assigned to the `+pem_agent` user. + In the list of rules, ensure you place the `hostssl pem` entry before any other rules assigned to the `+pem_agent` user. ```shell # Allow the PEM agent proxy user (used by pgbouncer) From 8f5c53ede2e6ba6facfa70d36f3e2442b19c9bf2 Mon Sep 17 00:00:00 2001 From: nidhibhammar <59045594+nidhibhammar@users.noreply.github.com> Date: Mon, 2 Sep 2024 13:08:39 +0530 Subject: [PATCH 25/39] PEM - Probes target type fix Added remaining probe examples to the Target type table --- .../pem/9/monitoring_performance/probes.mdx | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/product_docs/docs/pem/9/monitoring_performance/probes.mdx b/product_docs/docs/pem/9/monitoring_performance/probes.mdx index 2a8e9b3609c..ae222f17b39 100644 --- a/product_docs/docs/pem/9/monitoring_performance/probes.mdx +++ b/product_docs/docs/pem/9/monitoring_performance/probes.mdx @@ -178,18 +178,18 @@ Use the **General** tab to modify the definition of an existing probe or to spec - The **Mandatory columns** column indicates the coloumns you must configure in the probe query to ensure the required data is collected. - The **Probe examples** column provides some existing probes you can explore to better understand how probes are used in practice. - | Target type | Execution level | Mandatory columns | Probe examples | - |-------------|-----------------|------------------------------------------------------|----------------| - | Agent | Agent | None | cpu_usage | - | Server | Server | None | | - | Database | Database | None | | - | Schema | Database | schema_name | | - | Table | Database | schema_name, table_name | | - | Index | Database | schema_name, index_name | index_size | - | Sequence | Database | schema_name, sequence_name | | - | View | Database | schema_name, view_name | | - | Function | Database | schema_name, arg_types, function_type, function_name | | - | Extension | Extension | None | Extension | + | Target type | Execution level | Mandatory columns | Probe examples | + |-------------|-----------------|------------------------------------------------------|---------------------| + | Agent | Agent | None | cpu_usage | + | Server | Server | None | server_info | + | Database | Database | None | database_size | + | Schema | Database | schema_name | oc_extension | + | Table | Database | schema_name, table_name | table_size | + | Index | Database | schema_name, index_name | index_size | + | Sequence | Database | schema_name, sequence_name | oc_sequence | + | View | Database | schema_name, view_name | mview_size | + | Function | Database | schema_name, arg_types, function_type, function_name | function_statistics | + | Extension | Extension | None | bdr_node_summary | !!!note - The custom probes set to a database or larger target type (including schema, table, index, view, sequence, and functions) collect the information at the database level. From 15323170bb522c4542569d2ce72081d624e453be Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Wed, 4 Sep 2024 13:31:34 +0200 Subject: [PATCH 26/39] Add explanations to roles --- .../pem_pgbouncer/configuring_pgBouncer.mdx | 14 +++++++------- .../preparing_the_pem_database_server.mdx | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index 427705f0041..cd4f007b481 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -23,13 +23,13 @@ You must configure PgBouncer to work with the PEM database server. The name and location of the directories and files in the configuration steps, as well as the user, depend on whether you installed the community version of PgBouncer or EDB PgBouncer. If you have installed community PgBouncer (whether you install it from the community repo or the EDB repo), replace the names of the files and directories in the worked example with the values for PgBouncer. -| Name | PgBouncer | EDB PgBouncer | -|---------------------|-----------------------------|---------------------------------------| -| PgBouncer directory | `/etc/pgbouncer` | `/etc/edb/pgbouncer<1.x>` | -| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | -| HBA file | `(/etc/pgbouncer/hba_file)` | `(/etc/edb/pgbouncer<1.x>/hba_file)` | -| Service file | `pgbouncer` | `edb-pgbouncer-<1.x>` | -| User | `postgres` | `enterprisedb` | +| Name | PgBouncer | EDB PgBouncer | +|---------------------|---------------------------|------------------------------------| +| PgBouncer directory | `/etc/pgbouncer` | `/etc/edb/pgbouncer<1.x>` | +| ini file | `pgbouncer.ini` | `edb-pgbouncer.ini` | +| HBA file | `/etc/pgbouncer/hba_file` | `/etc/edb/pgbouncer<1.x>/hba_file` | +| Service file | `pgbouncer` | `edb-pgbouncer-<1.x>` | +| User | `postgres` | `enterprisedb` | ## Configuring PgBouncer diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx index 0682ce30cc9..0ddeb82c21c 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/preparing_the_pem_database_server.mdx @@ -26,7 +26,7 @@ This example shows how to prepare the PEM database server with the `enterprisedb ## Creating users and roles for PgBouncer-PEM connections -1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership: +1. Create a dedicated user named pgbouncer with `pem_agent_pool` membership. This user will serve connections from PgBouncer to the PEM database by forwarding all agent database queries. ```sql CREATE ROLE pgbouncer PASSWORD 'ANY_PASSWORD' LOGIN; @@ -40,7 +40,7 @@ This example shows how to prepare the PEM database server with the `enterprisedb GRANT ROLE ``` -1. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool role` membership: +1. Create a user named pem_admin1 (not a superuser) with `pem_admin` and `pem_agent_pool` role membership. This user is used to register the agent to the PEM server and manage access to the PEM database. ```sql CREATE ROLE pem_admin1 PASSWORD 'ANY_PASSWORD' LOGIN CREATEROLE; @@ -84,7 +84,7 @@ This example shows how to prepare the PEM database server with the `enterprisedb GRANT ``` -1. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1: +1. Use the `pem.create_proxy_agent_user(varchar)` function to create a user named pem_agent_user1. This proxy user will serve connections between all Agents and PgBouncer. ```sql SELECT pem.create_proxy_agent_user('pem_agent_user1'); From 955532818417e942d5f6ee9609e0eb6622af8b43 Mon Sep 17 00:00:00 2001 From: Betsy Gitelman Date: Tue, 27 Aug 2024 14:47:50 -0400 Subject: [PATCH 27/39] Edits to MTK: Added new pages for MTK options file feature #5959 --- .../08_mtk_command_options.mdx | 2 +- .../creating_txt_file.mdx | 42 +++++++++---------- .../executing_migration_with_txt.mdx | 32 +++++++------- .../mtk_command_options_in_file/index.mdx | 10 ++--- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/08_mtk_command_options.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/08_mtk_command_options.mdx index bef88723b51..bd0f32dbe2a 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/08_mtk_command_options.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/08_mtk_command_options.mdx @@ -33,7 +33,7 @@ The command options that work with Migration Toolkit are grouped by their behavi !!!note - If you are using several options for a migration, or have to specify a long list of objects for an option, consider using the `-optionsFile` option to specify the values in a separate text file. See [Specifying options using a file](mtk_command_options_in_file) for more information. + If you're using several options for a migration or have to specify a long list of objects for an option, consider using the `-optionsFile` option to specify the values in a separate text file. See [Specifying options using a file](mtk_command_options_in_file) for more information. !!! ## Offline migration options diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx index 9dd43c71e54..a05f52d11e5 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx @@ -4,19 +4,19 @@ navTitle: "Creating the options file" deepToC: true --- -To specify a text file that contains command line options and values, you must create a `` file with the options you want to execute, and then run the `runMTK.sh` command with the `-optionsFile ` argument. +To specify a text file that contains command line options and values, create a `.options` file with the options you want to execute. Then run the `runMTK.sh` command with the `-optionsFile .options` argument. !!!note - You can store the `` file in the same folder as the MTK utility, or in a directory of your choice. + You can store the `.options` file in the same folder as the Migration Toolkit utility or in a directory of your choice. !!! ## Creating the text file - guidelines -Create a text file on a location that is accessible by the user performing the migration. Then, add an option per line following these guidelines: +Create a text file in a location that's accessible by the user performing the migration. Then, add one option per line following these guidelines. ### Skip the dash -Use the same option syntax as in the command line options without the dash sign (-). The options file recognizes each line as a new parameter and doesn't require a dash as a delimiter. +Use the same option syntax as in the command line options without the dash (-). The options file recognizes each line as a new parameter and doesn't require a dash as a delimiter. | Option in CLI | Supported syntax for the option in file | |----------------|-----------------------------------------| @@ -24,19 +24,19 @@ Use the same option syntax as in the command line options without the dash sign ### Use a line per option -Use a new line for each option. Use a line per key-value pair. +Use a new line for each option. Use one line per key-value pair. | Option in CLI | Supported syntax for the option in file | |----------------------------------------------------------------|---------------------------------------------------------------| | `-tables hr -allViews`
`-excludeViews all_emp,acct_list` | tables hr
allViews
excludeViews all_emp,acct_list | -### Specify values for an option in a single or multiple lines +### Specify values for an option in a single line or multiple lines You can add a single line with an option and multiple values for that option. You can add several lines for the same option, each time with a different value. -And you can combine both approaches, because MTK will combine all lines that have the same option. +And you can combine both approaches, because Migration Toolkit combines all lines that specify the same option. | Option in CLI | Supported syntax for the option in file | |--------------------------|-------------------------------------------------| @@ -45,8 +45,8 @@ And you can combine both approaches, because MTK will combine all lines that hav | | tables TAB1
tables TAB2,TAB3 | !!!note - The duplication of options is only supported for the options file.
- When using `-tables` in the command line more than one time, only the last parameter is executed. For example, `./runMTK.sh -tables TAB2 -tables TAB3` only includes the _TAB3_ table in the migration. + Duplicating options is supported only for the options file.
+ When using `-tables` at the command line more than one time, only the last parameter is executed. For example, `./runMTK.sh -tables TAB2 -tables TAB3` includes only the `TAB3` table in the migration. !!! ### Use a space or equal sign to provide values @@ -58,25 +58,25 @@ For options that require values (key-value pairs), separate the option from the | `-views all_emp` | views=all_emp | | | views all_emp | -### Don't specify an option in the file and command line +### Don't specify an option in the file and at the command line -Don't include an option in the options file if you're specifying it as a flag in the command line. +Don't include an option in the options file if you're specifying it as a flag at the command line. -Specifying an option in both the command line and in the text file will cause the migration to fail. +Specifying an option both at the command line and in the text file causes the migration to fail. ## Order of processing Migration Toolkit reads command line options and option files in the order you provide them when running the command. -For example, if you run the following command, MTK first recognizes the `-sourcedbtype oracle` option, then reads the contents of `example.options` in order from top to bottom. Last parameter is always `schema_name`. +For example, if you run the following command, Migration Toolkit first recognizes the `-sourcedbtype oracle` option, and then reads the contents of `example.options` in order from top to bottom. The last parameter is always `schema_name`. ```shell runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name ``` -If you want an option to be executed last, you can either put it at the end of the `example.options` file and add no further options after `-optionsFile` in the command line. +If you want an option to be executed last, you can either put it at the end of the `.options` file or add no further options after `-optionsFile` at the command line. -Schema definition (`schema_name` or `-allSchemas`) must always be the last parameter on the command line. +Schema definition (`schema_name` or `-allSchemas`) must always be the last parameter at the command line: ```shell runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name @@ -89,7 +89,7 @@ tables=TAB1 dataOnly ``` -Or you can place the options flag (-dataOnly in this case) in the command line after specifying the `-optionsFile`: +Or you can use the options flag (`-dataOnly` in this case) at the command line after specifying the `-optionsFile`: ```shell runMTK.sh -sourcedbtype oracle -optionsFile example.options -dataOnly schema_name @@ -97,12 +97,12 @@ runMTK.sh -sourcedbtype oracle -optionsFile example.options -dataOnly schema_nam ## Other considerations -- **Commenting out**: Use a pound sign (#) to comment out a line. MTK won’t execute that line. +- **Commenting out** — Use a pound sign (#) to comment out a line. Migration Toolkit doesn't execute lines that are commented out. -- **Limitations**: Do not use quote marks. +- **Limitations** — Don't use quote marks. -- **Space processing**: MTK removes spaces between an option and its value. When you separate an option from its value with a space, only one space is treated as a delimiter. +- **Space processing** — Migration Toolkit removes spaces between an option and its value. When you separate an option from its value with a space, only one space is treated as a delimiter. - For example, to run the `-views all_emp, mgmt_list, acct_list` command line option, you can add _views all_emp,mgmt_list,acct_list_ OR _views = all_emp,mgmt_list,acct_list_ in the options file. + For example, to run the `-views all_emp, mgmt_list, acct_list` command line option, you can add `views all_emp,mgmt_list,acct_list` or `views = all_emp,mgmt_list,acct_list` in the options file. -After you have created the options file, [execute the migration with the options file](executing_migration_with_txt). +After you create the options file, [execute the migration with it](executing_migration_with_txt). diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx index 323f0562ec3..b0316ab34b8 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx @@ -4,31 +4,31 @@ navTitle: "Executing a migration with the options file" deepToC: true --- -After you have created the options file, execute the migration command by referencing the options file: +After you create the options file, reference it when executing the migration command: ```shell -./runMTK.sh -optionsFile +./runMTK.sh -optionsFile .options ``` !!!note - Provide the full path for `` if you are running the MTK command from a different folder than where you have stored the file. + Provide the full path for `.options` if you're running the Migration Toolkit command from a different folder than where you stored the file. !!! -You can specify all options in the file, or use a mix of command line and options file to specify the migration parameters. +You can specify all options in the file or use a mix of the command line and the options file to specify the migration parameters. ## Provide the scope for the schema migration -You must specify the scope of the schemas to be migrated. +Specify the scope of the schemas to migrate. -- If you want to migrate all schemas: Add the `-allSchemas` option in the command line or add `allSchemas` in your options file. Regardless of which method you use, this option must be the last parameter. +- If you want to migrate all schemas: Add the `-allSchemas` option at the command line or add `allSchemas` in your options file. Regardless of the method you use, this option must be the last parameter. -- If you want to specify one or several schemas: Specify the schemas you want to migrate in the command line with no preceding option. They have to be the last parameter in the command line. +- If you want to specify one or several schemas: Specify the schemas you want to migrate at the command line with no preceding option. Schema specifications must be the last parameter at the command line. -Here are some examples for specifying all options in the file: +Here are some examples for specifying all options in the file. ## Migrate a schema with specific tables -Content of the `` text file: +Content of the `example.options` text file: ``` tables comp_schema.emp,comp_schema.dept,finance_schema.acctg @@ -46,15 +46,15 @@ Command line equivalent: ./runMTK.sh -tables comp_schema.emp,comp_schema.dept,finance_schema.acctg schema_name ``` -## Use Options File to exclude tables and include functions +## Use options file to exclude tables and include functions -The options file can be used to separate table names from configuration options. -It may also contain comments to clarify why parameters are used. +You can use the options file to separate table names from configuration options. +You can also use comments to clarify why parameters are used. -In this example we are excluding tables and including functions. -It shows some extra comments, to explain why tables are excluded. +This example excludes tables and includes functions. +It shows some comments to explain why tables are excluded. -Content of the `` text file: +Content of the `excludeInclude.options` text file: ``` # finances @@ -84,7 +84,7 @@ Command line equivalent: Run an offline migration with all schemas. -Content of the `` text file: +Content of the `example.options` text file: ``` offlineMigration file_dest diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx index 29b6a5513b2..6dc74c1e8ba 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx @@ -4,15 +4,13 @@ navTitle: "Specifying options using a file" deepToC: true --- -EDB supports using a text file to specify several [database migration options](../08_mtk_command_options.mdx) instead of manually entering them in the command line. This is particularly helpful when: +EDB supports using a text file to specify several [database migration options](../08_mtk_command_options.mdx) instead of manually entering them at the command line. This is particularly helpful when: -- You have to specify a large number of command line options for complex migration scenarios. Entering all options in the command line becomes troublesome, or you might have even reached command line length limits. +- You have to specify a large number of command line options or a large number of values for complex migration scenarios. Entering all options or values at the command line becomes cumbersome, or you might reach command line length limits. -- You have to specify a large number of values for a command line option for complex migration scenarios. Entering all values in the command line becomes troublesome, or you might have even reached command line length limits. +- You want to apply the same migration options and values to several different databases, or you might want to automate database migration. Running `runMTK.sh` with several options and values repeatedly at the command line becomes prone to errors. -- You want to apply the same migration options and values to several different databases, or you might even consider automating the migration of databases. Running `runMTK.sh` with several options and values repeatedly in the command line becomes prone to errors. - -The `-optionsFile ` argument allows you to reference a text file with all options and values, making the migration process easier. +The `-optionsFile .options` argument allows you to reference a text file with all options and values, making the migration process easier. ## Execute a migration using an options file From 3b36232165ed256c9a674e6d017c1b2540a31e9b Mon Sep 17 00:00:00 2001 From: Betsy Gitelman Date: Thu, 29 Aug 2024 13:22:16 -0400 Subject: [PATCH 28/39] Edits to Added pages for MTK options file feature PR5959 --- .../creating_txt_file.mdx | 14 +++++++------- .../executing_migration_with_txt.mdx | 10 +++++----- .../mtk_command_options_in_file/index.mdx | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx index a05f52d11e5..97dcb260baa 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx @@ -4,10 +4,10 @@ navTitle: "Creating the options file" deepToC: true --- -To specify a text file that contains command line options and values, create a `.options` file with the options you want to execute. Then run the `runMTK.sh` command with the `-optionsFile .options` argument. +To specify a text file that contains command line options and values, create an `` file with the options you want to execute. Then run the `runMTK.sh` command with the `-optionsFile ` argument. !!!note - You can store the `.options` file in the same folder as the Migration Toolkit utility or in a directory of your choice. + You can store the `` file in the same folder as the Migration Toolkit utility or in a directory of your choice. !!! ## Creating the text file - guidelines @@ -51,7 +51,7 @@ And you can combine both approaches, because Migration Toolkit combines all line ### Use a space or equal sign to provide values -For options that require values (key-value pairs), separate the option from the value with a space or an equal sign (=). Both options are supported. +For options that require values (key-value pairs), separate the option from the value with a space or an equal sign (=). Both syntaxes are supported. | Option in CLI | Supported syntax for the option in file | |------------------|-----------------------------------------| @@ -74,15 +74,15 @@ For example, if you run the following command, Migration Toolkit first recognize runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name ``` -If you want an option to be executed last, you can either put it at the end of the `.options` file or add no further options after `-optionsFile` at the command line. +If you want an option to be executed last, you can either put it at the end of `` or add no options after `-optionsFile` at the command line. Schema definition (`schema_name` or `-allSchemas`) must always be the last parameter at the command line: ```shell -runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name +runMTK.sh -sourcedbtype oracle -optionsFile schema_name ``` -Where the content of the `example.options` file is: +Where the content of the `` file is: ``` tables=TAB1 @@ -92,7 +92,7 @@ dataOnly Or you can use the options flag (`-dataOnly` in this case) at the command line after specifying the `-optionsFile`: ```shell -runMTK.sh -sourcedbtype oracle -optionsFile example.options -dataOnly schema_name +runMTK.sh -sourcedbtype oracle -optionsFile -dataOnly schema_name ``` ## Other considerations diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx index b0316ab34b8..b538f371de1 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx @@ -7,22 +7,22 @@ deepToC: true After you create the options file, reference it when executing the migration command: ```shell -./runMTK.sh -optionsFile .options +./runMTK.sh -optionsFile ``` !!!note - Provide the full path for `.options` if you're running the Migration Toolkit command from a different folder than where you stored the file. + Provide the full path for `` if you're running the Migration Toolkit command from a different folder from where you stored the file. !!! You can specify all options in the file or use a mix of the command line and the options file to specify the migration parameters. ## Provide the scope for the schema migration -Specify the scope of the schemas to migrate. +Specify the scope of the schemas to migrate: -- If you want to migrate all schemas: Add the `-allSchemas` option at the command line or add `allSchemas` in your options file. Regardless of the method you use, this option must be the last parameter. +- If you want to migrate all schemas, add the `-allSchemas` option at the command line or add `allSchemas` in your options file. For both methods, this option must be the last parameter. -- If you want to specify one or several schemas: Specify the schemas you want to migrate at the command line with no preceding option. Schema specifications must be the last parameter at the command line. +- If you want to specify one or several schemas, specify the schemas you want to migrate at the command line with no preceding option. Schema specifications must be the last parameter at the command line. Here are some examples for specifying all options in the file. diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx index 6dc74c1e8ba..9f20f004445 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx @@ -10,7 +10,7 @@ EDB supports using a text file to specify several [database migration options](. - You want to apply the same migration options and values to several different databases, or you might want to automate database migration. Running `runMTK.sh` with several options and values repeatedly at the command line becomes prone to errors. -The `-optionsFile .options` argument allows you to reference a text file with all options and values, making the migration process easier. +The `-optionsFile ` argument allows you to reference a text file with all options and values, making the migration process easier. ## Execute a migration using an options file @@ -19,4 +19,3 @@ To perform a migration with a file that specifies the command options: 1. [Create a text file with the command options](creating_txt_file). 1. [Perform a migration by invoking the file with the command options](executing_migration_with_txt). - From a58a8838dd1a7c36cedb08679f81feff176530ea Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Fri, 30 Aug 2024 10:10:53 +0200 Subject: [PATCH 29/39] Reworked the Order of processing section --- .../creating_txt_file.mdx | 20 ++++++++++++------- .../executing_migration_with_txt.mdx | 2 +- .../mtk_command_options_in_file/index.mdx | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx index 97dcb260baa..b1f2b7a0d87 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx @@ -66,20 +66,20 @@ Specifying an option both at the command line and in the text file causes the mi ## Order of processing -Migration Toolkit reads command line options and option files in the order you provide them when running the command. +Migration Toolkit reads command line options and option files in the order you provide them when running the command. Ensure you add the [schema scope](executing_migration_with_txt/#provide-the-scope-for-the-schema-migration) (`schema_name` or `-allSchemas`) as the last parameter at the command line. -For example, if you run the following command, Migration Toolkit first recognizes the `-sourcedbtype oracle` option, and then reads the contents of `example.options` in order from top to bottom. The last parameter is always `schema_name`. +For example, if you run the following command, Migration Toolkit first recognizes the `-sourcedbtype oracle` option, and then reads the contents of `example.options` in order from top to bottom. The last parameter is the schema scope (`` or `-allSchemas`). ```shell runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name ``` -If you want an option to be executed last, you can either put it at the end of `` or add no options after `-optionsFile` at the command line. +Using an options file means that you can employ different syntaxes to perform a migration where parameters are executed in the same way. The following alternatives perform the same migration. -Schema definition (`schema_name` or `-allSchemas`) must always be the last parameter at the command line: +**Alternative 1** ```shell -runMTK.sh -sourcedbtype oracle -optionsFile schema_name +runMTK.sh -sourcedbtype oracle -optionsFile ``` Where the content of the `` file is: @@ -89,10 +89,16 @@ tables=TAB1 dataOnly ``` -Or you can use the options flag (`-dataOnly` in this case) at the command line after specifying the `-optionsFile`: +**Alternative 2** ```shell -runMTK.sh -sourcedbtype oracle -optionsFile -dataOnly schema_name +runMTK.sh -sourcedbtype oracle -optionsFile -dataOnly +``` + +Where the content of the `` file is: + +``` +tables=TAB1 ``` ## Other considerations diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx index b538f371de1..65bede836c3 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx @@ -7,7 +7,7 @@ deepToC: true After you create the options file, reference it when executing the migration command: ```shell -./runMTK.sh -optionsFile +./runMTK.sh -optionsFile ``` !!!note diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx index 9f20f004445..ae74cad94cb 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/index.mdx @@ -10,7 +10,7 @@ EDB supports using a text file to specify several [database migration options](. - You want to apply the same migration options and values to several different databases, or you might want to automate database migration. Running `runMTK.sh` with several options and values repeatedly at the command line becomes prone to errors. -The `-optionsFile ` argument allows you to reference a text file with all options and values, making the migration process easier. +The `-optionsFile ` argument allows you to reference a text file with options and values, making the migration process easier. ## Execute a migration using an options file From 739911153ee2e7e54a101ea1a7c591e556f39715 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Mon, 2 Sep 2024 09:51:40 +0200 Subject: [PATCH 30/39] changed example.options to options_textfile --- .../mtk_command_options_in_file/creating_txt_file.mdx | 4 ++-- .../executing_migration_with_txt.mdx | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx index b1f2b7a0d87..669c2ef1bfc 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/creating_txt_file.mdx @@ -68,10 +68,10 @@ Specifying an option both at the command line and in the text file causes the mi Migration Toolkit reads command line options and option files in the order you provide them when running the command. Ensure you add the [schema scope](executing_migration_with_txt/#provide-the-scope-for-the-schema-migration) (`schema_name` or `-allSchemas`) as the last parameter at the command line. -For example, if you run the following command, Migration Toolkit first recognizes the `-sourcedbtype oracle` option, and then reads the contents of `example.options` in order from top to bottom. The last parameter is the schema scope (`` or `-allSchemas`). +For example, if you run the following command, Migration Toolkit first recognizes the `-sourcedbtype oracle` option, and then reads the contents of `options_textfile` in order from top to bottom. The last parameter is the schema scope (`` or `-allSchemas`). ```shell -runMTK.sh -sourcedbtype oracle -optionsFile example.options schema_name +runMTK.sh -sourcedbtype oracle -optionsFile options_textfile schema_name ``` Using an options file means that you can employ different syntaxes to perform a migration where parameters are executed in the same way. The following alternatives perform the same migration. diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx index 65bede836c3..f08916ddedc 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx @@ -28,7 +28,7 @@ Here are some examples for specifying all options in the file. ## Migrate a schema with specific tables -Content of the `example.options` text file: +Content of the `options_textfile` text file: ``` tables comp_schema.emp,comp_schema.dept,finance_schema.acctg @@ -37,7 +37,7 @@ tables comp_schema.emp,comp_schema.dept,finance_schema.acctg Syntax of the migration command: ```shell -./runMTK.sh -optionsFile example.options schema_name +./runMTK.sh -optionsFile options_textfile schema_name ``` Command line equivalent: @@ -84,7 +84,7 @@ Command line equivalent: Run an offline migration with all schemas. -Content of the `example.options` text file: +Content of the `options_textfile` text file: ``` offlineMigration file_dest @@ -95,7 +95,7 @@ schemaOnly Syntax of the migration command: ```shell -./runMTK.sh -optionsFile example.options -allSchemas +./runMTK.sh -optionsFile options_textfile -allSchemas ``` Command line equivalent: From a1a919c378b7a1c2cd8c5115484a31e5e0c69921 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Wed, 4 Sep 2024 16:03:26 +0200 Subject: [PATCH 31/39] Applied suggestion from Matt --- .../executing_migration_with_txt.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx index f08916ddedc..0c9d94eb0b2 100644 --- a/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx +++ b/product_docs/docs/migration_toolkit/55/07_invoking_mtk/mtk_command_options_in_file/executing_migration_with_txt.mdx @@ -22,7 +22,7 @@ Specify the scope of the schemas to migrate: - If you want to migrate all schemas, add the `-allSchemas` option at the command line or add `allSchemas` in your options file. For both methods, this option must be the last parameter. -- If you want to specify one or several schemas, specify the schemas you want to migrate at the command line with no preceding option. Schema specifications must be the last parameter at the command line. +- If you want to specify a subset of schemas, specify the schemas you want to migrate at the command line with no preceding option and as a comma-separated list. Schema specifications must be the last parameter at the command line. Here are some examples for specifying all options in the file. From 9bf851b870fe928f58829d4c3c682d861d5df0e9 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 5 Sep 2024 10:15:31 +0200 Subject: [PATCH 32/39] Corrected diagrams --- .../pem_server_pem_agent_connection_management_mechanism.mdx | 4 ++-- product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png | 3 --- product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png | 3 --- product_docs/docs/pem/9/images/pem_with_pgbouncer.png | 3 +++ product_docs/docs/pem/9/images/pem_without_pgbouncer.png | 3 +++ 5 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png delete mode 100644 product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png create mode 100644 product_docs/docs/pem/9/images/pem_with_pgbouncer.png create mode 100644 product_docs/docs/pem/9/images/pem_without_pgbouncer.png diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx index 508b3c0ac63..80fe1f6630b 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/pem_server_pem_agent_connection_management_mechanism.mdx @@ -14,7 +14,7 @@ In the default configuration, each PEM agent connects to the PEM database server For example, a user agent with ID 1 connects to the PEM database server as agent1. -![Connecting to the PEM database without PgBouncer](../../images/pem_db_without_pgbouncer.png) +![Connecting to the PEM database without PgBouncer](../../images/pem_without_pgbouncer.png) ## With PgBouncer @@ -22,7 +22,7 @@ In a PgBouncer-enabled environment, PEM agents cannot connect to the PEM databas Once the PEM agents connect to PgBouncer using SSL, PgBouncer is responsible for managing connection requests to the PEM database server. PgBouncer uses the rules you have configured for connection pooling to manage the incoming connection requests, for example, by respecting the established maximum number of active connections. -![Connecting to pgBouncer.](../../images/pem_db_with_pgbouncer.png) +![Connecting to pgBouncer.](../../images/pem_with_pgbouncer.png) !!!note Use PgBouncer version 1.9.0 or later as the connection pooler. Versions 1.9.0 or later support cert authentication. diff --git a/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png b/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png deleted file mode 100644 index 90c673d2235..00000000000 --- a/product_docs/docs/pem/9/images/pem_db_with_pgbouncer.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8a84aca5d03e2e30abd75ce4f9c14b1b6c45587a3dc81aaecada7dbe18eb0704 -size 189178 diff --git a/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png b/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png deleted file mode 100644 index 86530287faa..00000000000 --- a/product_docs/docs/pem/9/images/pem_db_without_pgbouncer.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0e8a00e26842de54ecfca3b0cf50769ccba035a967aeb0a8604a594c0744c561 -size 216058 diff --git a/product_docs/docs/pem/9/images/pem_with_pgbouncer.png b/product_docs/docs/pem/9/images/pem_with_pgbouncer.png new file mode 100644 index 00000000000..21c0437c063 --- /dev/null +++ b/product_docs/docs/pem/9/images/pem_with_pgbouncer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0da3b93bf4127a528f10a1c58ef48c267cf66531dad13efec2aee8c17da527b9 +size 188773 diff --git a/product_docs/docs/pem/9/images/pem_without_pgbouncer.png b/product_docs/docs/pem/9/images/pem_without_pgbouncer.png new file mode 100644 index 00000000000..18e9d3ee630 --- /dev/null +++ b/product_docs/docs/pem/9/images/pem_without_pgbouncer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ecb7464eb33bf09494d885685a7a8f91909ce2761a434c47bd1f905605c574 +size 215177 From abae6e12858261555ef5d84a80f6f3b7b4be0eb0 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 5 Sep 2024 12:36:37 +0200 Subject: [PATCH 33/39] SMTP, SNMP and webhook clarification --- .../considerations/pem_pgbouncer/configuring_the_pem_agent.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index b494ccc2dc8..f10877e21e9 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -13,7 +13,7 @@ redirects: You have [installed the PEM agent](../../installing_pem_agent/). !!!note - Don't configure the PEM agent responsible for sending SNMP notifications with PgBouncer. For example, if the default PEM agent installed with PEM server is used for SNMP notifications, don't configure it with PgBouncer. + Don’t configure PEM agents with `enable_smtp`, `enable_snmp`, or `enable_webhook` set to `true` in the `agent.cfg` file to connect through PgBouncer. SNMP, SMTP, and Webhook spoolers use the LISTEN/NOTIFY mechanism provided by Postgres to send notifications asynchronously. Since PgBouncer doesn’t support the LISTEN/NOTIFY mechanism in transaction mode, connecting the agent to PgBouncer can cause notifications to be delayed or not delivered at all. Instead, connect the PEM agent directly to the PEM backend database. !!! Now you can choose to [configure a new PEM agent](#configuring-a-new-pem-agent) or [use an existing PEM agent](#configuring-an-existing-pem-agent) for PgBouncer. From cca416903d7dabd3a4d607cbcc71b6a643b6edc8 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 5 Sep 2024 13:46:26 +0200 Subject: [PATCH 34/39] typos --- .../9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx | 2 +- .../considerations/pem_pgbouncer/configuring_the_pem_agent.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx index cd4f007b481..fca3b2256ca 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_pgBouncer.mdx @@ -11,7 +11,7 @@ redirects: You must configure PgBouncer to work with the PEM database server. -## Prerequisits +## Prerequisites - If you are running EDB Postgres Advanced Server, you have installed [EDB PgBouncer](/pgbouncer/latest/installing). diff --git a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx index f10877e21e9..9f234f89268 100644 --- a/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx +++ b/product_docs/docs/pem/9/considerations/pem_pgbouncer/configuring_the_pem_agent.mdx @@ -13,7 +13,7 @@ redirects: You have [installed the PEM agent](../../installing_pem_agent/). !!!note - Don’t configure PEM agents with `enable_smtp`, `enable_snmp`, or `enable_webhook` set to `true` in the `agent.cfg` file to connect through PgBouncer. SNMP, SMTP, and Webhook spoolers use the LISTEN/NOTIFY mechanism provided by Postgres to send notifications asynchronously. Since PgBouncer doesn’t support the LISTEN/NOTIFY mechanism in transaction mode, connecting the agent to PgBouncer can cause notifications to be delayed or not delivered at all. Instead, connect the PEM agent directly to the PEM backend database. + Do not configure PEM agents with `enable_smtp`, `enable_snmp`, or `enable_webhook` set to `true` in the `agent.cfg` file to connect through PgBouncer. SNMP, SMTP, and Webhook spoolers use the LISTEN/NOTIFY mechanism provided by Postgres to send notifications asynchronously. Since PgBouncer doesn’t support the LISTEN/NOTIFY mechanism in transaction mode, connecting the agent to PgBouncer can cause notifications to be delayed or not delivered at all. Instead, connect the PEM agent directly to the PEM backend database. !!! Now you can choose to [configure a new PEM agent](#configuring-a-new-pem-agent) or [use an existing PEM agent](#configuring-an-existing-pem-agent) for PgBouncer. From c66ad72e079290825758957a4de6dce2d363f66a Mon Sep 17 00:00:00 2001 From: Josh Heyer Date: Tue, 3 Sep 2024 22:56:53 +0000 Subject: [PATCH 35/39] Fix a couple of images Also, add image URL checking to the link checker --- .../organizations/identity_provider/index.mdx | 2 +- .../1/identify_images/private_registries.mdx | 2 +- tools/automation/actions/link-check/index.js | 49 ++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/advocacy_docs/edb-postgres-ai/console/using/organizations/identity_provider/index.mdx b/advocacy_docs/edb-postgres-ai/console/using/organizations/identity_provider/index.mdx index b64ded13bca..e40a26f6a21 100644 --- a/advocacy_docs/edb-postgres-ai/console/using/organizations/identity_provider/index.mdx +++ b/advocacy_docs/edb-postgres-ai/console/using/organizations/identity_provider/index.mdx @@ -59,7 +59,7 @@ On the Set Up Identity Provider page: For example: - ![](../images/nameID.png) + ![](images/nameID.png) 3. In the **SAML settings** section, enter the configuration information for your preferred SAML identity provider: | Field | Description | diff --git a/product_docs/docs/postgres_distributed_for_kubernetes/1/identify_images/private_registries.mdx b/product_docs/docs/postgres_distributed_for_kubernetes/1/identify_images/private_registries.mdx index 87b57313330..445666507a6 100644 --- a/product_docs/docs/postgres_distributed_for_kubernetes/1/identify_images/private_registries.mdx +++ b/product_docs/docs/postgres_distributed_for_kubernetes/1/identify_images/private_registries.mdx @@ -43,7 +43,7 @@ log in to the EDB container registry, for example, through `docker login` or a In the [repos page in EDB](https://www.enterprisedb.com/repos-downloads), is an EDB Repos 2.0 section where a repo token appears obscured. -![EDB Repo Portal](images/edb-repo-portal.png) +![EDB Repo Portal](../images/edb-repo-portal.png) Next to the repo token is a **Copy Token** button to copy the token and an eye icon for looking at the content of the token. diff --git a/tools/automation/actions/link-check/index.js b/tools/automation/actions/link-check/index.js index b3b97d0edfb..8642e709984 100644 --- a/tools/automation/actions/link-check/index.js +++ b/tools/automation/actions/link-check/index.js @@ -16,6 +16,7 @@ import GithubSlugger from "github-slugger"; import toVfile from "to-vfile"; const { read, write } = toVfile; +const imageExts = [".png", ".svg", ".jpg", ".jpeg", ".gif"]; const docsUrl = "https://www.enterprisedb.com/docs"; // add path here to ignore link warnings const noWarnPaths = [ @@ -158,6 +159,30 @@ async function main() { await scanner.run(ast, input); } + const imageFiles = await glob( + imageExts.flatMap((ext) => [ + "product_docs/**/*" + ext, + "advocacy_docs/**/*" + ext, + ]), + ); + + for (const sourcePath of imageFiles) { + const metadata = { + canonical: fsPathToURLPath(sourcePath), + index: false, + slugs: [], + redirects: [], + source: sourcePath, + }; + allValidUrlPaths.set(metadata.canonical, metadata); + if (isVersioned(sourcePath)) { + const splitPath = metadata.canonical.split(path.posix.sep); + metadata.product = splitPath[1]; + metadata.version = splitPath[2]; + allValidUrlPaths.set(latestVersionURLPath(sourcePath), metadata); + } + } + // compile product versions const productVersions = {}; @@ -356,9 +381,16 @@ function cleanup() { const mapUrlToCanonical = (url, position) => { let test = normalizeUrl(url, metadata.canonical, metadata.index); + if ( + test.href === + docsUrl + "/edb-postgres-ai/analytics/images/level-50.png" + ) + debugger; if (!test.href.startsWith(docsUrl)) return url; if (test.href === docsUrl) return url; - if (path.posix.extname(test.pathname)) return url; + const ext = path.posix.extname(test.pathname); + const isImageUrl = imageExts.includes(ext); + if (ext && !isImageUrl) return url; metadata.linksChecked = metadata.linksChecked || 0 + 1; @@ -440,7 +472,7 @@ function cleanup() { return url; }; - visitParents(tree, ["link", "element"], (node) => { + visitParents(tree, ["link", "image", "element"], (node) => { try { if ( node.type === "element" && @@ -451,7 +483,7 @@ function cleanup() { node.properties.href, node.position, ); - else if (node.type === "link") + else if (node.type === "link" || node.type === "image") node.url = mapUrlToCanonical(node.url, node.position); } catch (e) { file.message(e, node.position); @@ -494,11 +526,14 @@ function fsPathToURLPath(fsPath) { // 2. strip trailing index.mdx // 3. strip trailing .mdx // 4. strip trailing / + // URL encode const docsLocations = /product_docs\/docs|advocacy_docs/; - return fsPath - .split(docsLocations)[1] - .replace(/\/index\.mdx$|\.mdx$/, "") - .replace(/\/$/, ""); + return encodeURI( + fsPath + .split(docsLocations)[1] + .replace(/\/index\.mdx$|\.mdx$/, "") + .replace(/\/$/, ""), + ); } function latestVersionURLPath(fsPath) { From b4dbfb3ae49b4eba674730c83a7d618df14238f7 Mon Sep 17 00:00:00 2001 From: Josh Heyer Date: Thu, 5 Sep 2024 16:11:39 +0000 Subject: [PATCH 36/39] Avoid re-creating sync'd jira issues ...by retrieving all pages of results for the already-synced query! --- tools/automation/actions/jira-sync/jira.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/automation/actions/jira-sync/jira.js b/tools/automation/actions/jira-sync/jira.js index 2195b953052..e25784ff0eb 100644 --- a/tools/automation/actions/jira-sync/jira.js +++ b/tools/automation/actions/jira-sync/jira.js @@ -121,15 +121,15 @@ async function loadGHIssues(issueNumber) { return ret; } -async function loadSynchedJiraIssues() { +async function loadSynchedJiraIssues(accumulateIssues) { console.log(`Loading synched Jira issues`); - const query = `summary ~ "\\"Docs GH #\\"" order by created DESC`; + const query = `summary ~ "\\"Docs GH #\\"" order by created ASC`; try { const response = await fetch( `https://enterprisedb.atlassian.net/rest/api/3/search?jql=${encodeURIComponent( query, - )}`, + )}&startAt=${(accumulateIssues || []).length}&maxResults=100`, { method: "GET", headers: { @@ -139,7 +139,10 @@ async function loadSynchedJiraIssues() { }, ); const json = await response.json(); - return json?.issues; + accumulateIssues = [...(accumulateIssues || []), ...(json?.issues || [])]; + if (json?.total > json?.startAt + json?.maxResults) + return loadSynchedJiraIssues(accumulateIssues); + return accumulateIssues; } catch (err) { console.error(err); } From 7b1860767314d29ca185f4c94fcbc618e7774c4a Mon Sep 17 00:00:00 2001 From: Josh Heyer Date: Fri, 6 Sep 2024 14:26:07 +0000 Subject: [PATCH 37/39] index image paths correctly when running as action --- tools/automation/actions/link-check/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/automation/actions/link-check/index.js b/tools/automation/actions/link-check/index.js index 8642e709984..c4d64add6ff 100644 --- a/tools/automation/actions/link-check/index.js +++ b/tools/automation/actions/link-check/index.js @@ -161,8 +161,8 @@ async function main() { const imageFiles = await glob( imageExts.flatMap((ext) => [ - "product_docs/**/*" + ext, - "advocacy_docs/**/*" + ext, + path.resolve(basePath, "product_docs/**/*" + ext), + path.resolve(basePath, "advocacy_docs/**/*" + ext), ]), ); From 00b3de585a8c614a9c4e02ce1cda2c1aa5ef7409 Mon Sep 17 00:00:00 2001 From: gvasquezvargas Date: Thu, 5 Sep 2024 15:39:01 +0200 Subject: [PATCH 38/39] TDE: Fixed INPUT and OUTPUT code and made command executable --- product_docs/docs/tde/15/enabling_tde.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/product_docs/docs/tde/15/enabling_tde.mdx b/product_docs/docs/tde/15/enabling_tde.mdx index 32177b2061f..79b24445228 100644 --- a/product_docs/docs/tde/15/enabling_tde.mdx +++ b/product_docs/docs/tde/15/enabling_tde.mdx @@ -114,7 +114,8 @@ You can find out whether TDE is present on a server by querying the `data_encryp A value of 0 means TDE isn't enabled. Any nonzero value reflects the version of TDE in use. Currently, when TDE is enabled, this value is 1. ```sql -# select data_encryption_version from pg_control_init(); +select data_encryption_version from pg_control_init(); +__OUTPUT__ data_encryption_version ------------------------- 1 From 61c6f25822f930f433b63c52a72d84e395e3b5e8 Mon Sep 17 00:00:00 2001 From: Simon Notley <43099400+sonotley@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:42:59 +0100 Subject: [PATCH 39/39] TPA 23.34.1 rel notes --- product_docs/docs/tpa/23/rel_notes/index.mdx | 2 ++ .../docs/tpa/23/rel_notes/tpa_23.34.1_rel_notes.mdx | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 product_docs/docs/tpa/23/rel_notes/tpa_23.34.1_rel_notes.mdx diff --git a/product_docs/docs/tpa/23/rel_notes/index.mdx b/product_docs/docs/tpa/23/rel_notes/index.mdx index 086be0dc0a5..70411fab830 100644 --- a/product_docs/docs/tpa/23/rel_notes/index.mdx +++ b/product_docs/docs/tpa/23/rel_notes/index.mdx @@ -2,6 +2,7 @@ title: Trusted Postgres Architect release notes navTitle: "Release notes" navigation: + - tpa_23.34.1_rel_notes - tpa_23.34_rel_notes - tpa_23.33_rel_notes - tpa_23.32_rel_notes @@ -32,6 +33,7 @@ The Trusted Postgres Architect documentation describes the latest version of Tru | Version | Release date | | ---------------------------- | ------------ | +| [23.35](tpa_23.34.1_rel_notes) | 09 Sep 2024 | | [23.34](tpa_23.34_rel_notes) | 22 Aug 2024 | | [23.33](tpa_23.33_rel_notes) | 24 Jun 2024 | | [23.32](tpa_23.32_rel_notes) | 15 May 2024 | diff --git a/product_docs/docs/tpa/23/rel_notes/tpa_23.34.1_rel_notes.mdx b/product_docs/docs/tpa/23/rel_notes/tpa_23.34.1_rel_notes.mdx new file mode 100644 index 00000000000..0d8e21cbce7 --- /dev/null +++ b/product_docs/docs/tpa/23/rel_notes/tpa_23.34.1_rel_notes.mdx @@ -0,0 +1,12 @@ +--- +title: Trusted Postgres Architect 23.34.1 release notes +navTitle: "Version 23.34.1" +--- + +Released: 9 September 2024 + +Trusted Postgres Architect 23.34.1 is a bug fix release which resolves the following issues: + +| Type | Description | +|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Bug Fix | Fixed an issue whereby running deploy after a switchover fails for nodes with `efm-witness` role. The `upstream-primary` for EFM nodes is determined using the facts gathered from Postgres. This previously failed for nodes with `efm-witness` roles since they do not have Postgres. The task to determine upstream-primary is now run only on nodes with `primary` or `replica` roles. | \ No newline at end of file