From 37aa53ec6fb83d0593119918d581ee0c563d13e1 Mon Sep 17 00:00:00 2001 From: francoughlin Date: Fri, 7 Jul 2023 16:15:48 -0400 Subject: [PATCH 01/13] EPAS reorg: Phase 2 EDB Resource Manager topic restructure Breaking up the EDB Resource Manager topic into multiple subtopics --- .../10_edb_resource_manager.mdx | 1060 ----------------- .../cpu_usage_throttling.mdx | 325 +++++ .../creating_resource_groups.mdx | 278 +++++ .../dirty_buffer_throttling.mdx | 394 ++++++ .../edb_resource_manager_key_concepts.mdx | 19 + .../edb_resource_manager_system_catalogs.mdx | 31 + .../10_edb_resource_manager/index.mdx | 34 + 7 files changed, 1081 insertions(+), 1060 deletions(-) delete mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/cpu_usage_throttling.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/dirty_buffer_throttling.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_system_catalogs.mdx create mode 100644 product_docs/docs/epas/15/database_administration/10_edb_resource_manager/index.mdx diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager.mdx deleted file mode 100644 index 14a45e13342..00000000000 --- a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager.mdx +++ /dev/null @@ -1,1060 +0,0 @@ ---- -title: "Throttling CPU and I/O at the process level" -navTitle: "EDB Resource Manager" -description: "How to use EDB Resource Manager to control the use of operating system resources used by EPAS processes" -legacyRedirectsGenerated: - # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.37.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.35.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.34.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.33.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.36.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.083.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.082.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.079.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.081.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.080.html" -redirects: - - ../../epas_guide/10_edb_resource_manager #generated for docs/epas/reorg-role-use-case-mode ---- - - - -EDB Resource Manager is an EDB Postgres Advanced Server feature that lets you control the use of operating system resources used by EDB Postgres Advanced Server processes. - -This capability allows you to protect the system from processes that might uncontrollably overuse and monopolize certain system resources. - -The following are some key points about using EDB Resource Manager: - -- The basic component of EDB Resource Manager is a *resource group*. A resource group is a named, global group. It's available to all databases in an EDB Postgres Advanced Server instance, and you can define various resource usage limits on it. EDB Postgres Advanced Server processes that are assigned as members of a given resource group are then controlled by EDB Resource Manager. This configuration keeps the aggregate resource use of all processes in the group near the limits defined on the group. -- Data definition language commands are used to create, alter, and drop resource groups. Only a database user with superuser privileges can use these commands. -- *Resource type parameters* define the desired aggregate consumption level of all processes belonging to a resource group. You use different resource type parameters for the different types of system resources currently supported by EDB Resource Manager. -- You can create multiple resource groups, each with different settings for its resource type parameters, which defines different consumption levels for each resource group. -- EDB Resource Manager throttles processes in a resource group to keep resource consumption near the limits defined by the resource type parameters. If multiple resource type parameters have defined settings in a resource group, the actual resource consumption might be significantly lower for certain resource types than their defined resource type parameter settings. This lower consumption happens because EDB Resource Manager throttles processes, attempting to keep all resources with defined resource type settings within their defined limits. -- The definitions of available resource groups and their resource type settings are stored in a shared global system catalog. Thus, all databases in a given EDB Postgres Advanced Server instance can use resource groups. -- The `edb_max_resource_groups` configuration parameter sets the maximum number of resource groups that can be active at the same time as running processes. The default setting is 16 resource groups. Changes to this parameter take effect when you restart the database server. -- Use the `SET edb_resource_group TO group_name` command to assign the current process to a specified resource group. Use the `RESET edb_resource_group` command or `SET edb_resource_group` to `DEFAULT` to remove the current process from a resource group. -- You can assign a default resource group to a role using the `ALTER ROLE ... SET` command or to a database using the `ALTER DATABASE ... SET` command. You can assign the entire database server instance a default resource group by setting the parameter in the `postgresql.conf` file. -- To include resource groups in a backup file of the database server instance, use the `pg_dumpall` backup utility with default settings. That is, don't specify any of the `--globals-only`, `--roles-only`, or `--tablespaces-only` options. - -## Creating and managing resource groups - -Use these data definition language commands to create and manage resource groups. - -### CREATE RESOURCE GROUP - -Use the `CREATE RESOURCE GROUP` command to create a new resource group. - -```sql -CREATE RESOURCE GROUP ; -``` - -#### Description - -The `CREATE RESOURCE GROUP` command creates a resource group with the specified name. You can then define resource limits on the group with the `ALTER RESOURCE GROUP` command. The resource group is accessible from all databases in the EDB Postgres Advanced Server instance. - -To use the `CREATE RESOURCE GROUP` command, you must have superuser privileges. - -#### Parameters - -`group_name` - -The name of the resource group. - -#### Example - -This example creates three resource groups named `resgrp_a`, `resgrp_b`, and `resgrp_c`: - -```sql -edb=# CREATE RESOURCE GROUP resgrp_a; -CREATE RESOURCE GROUP -edb=# CREATE RESOURCE GROUP resgrp_b; -CREATE RESOURCE GROUP -edb=# CREATE RESOURCE GROUP resgrp_c; -CREATE RESOURCE GROUP -``` - -This query shows the entries for the resource groups in the `edb_resource_group` catalog: - -```sql -edb=# SELECT * FROM edb_resource_group; -__OUTPUT__ - rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit ------------+------------------+-------------------- - resgrp_a | 0 | 0 - resgrp_b | 0 | 0 - resgrp_c | 0 | 0 -(3 rows) -``` - -### ALTER RESOURCE GROUP - -Use the `ALTER RESOURCE GROUP` command to change the attributes of an existing resource group. The command syntax comes in three forms. - -This form renames the resource group: - -```sql -ALTER RESOURCE GROUP RENAME TO ; -``` - -This form assigns a resource type to the resource group: - -```sql -ALTER RESOURCE GROUP SET - { TO | = } { | DEFAULT }; -``` - -This form resets the assignment of a resource type to its default in the group: - -```sql -ALTER RESOURCE GROUP RESET ; -``` - -#### Description - -The `ALTER RESOURCE GROUP` command changes certain attributes of an existing resource group. - -The form with the `RENAME TO` clause assigns a new name to an existing resource group. - -The form with the `SET resource_type TO` clause assigns the specified literal value to a resource type. Or, when you specify `DEFAULT`, it resets the resource type. Resetting a resource type means that the resource group has no defined limit on that resource type. - -The form with the `RESET resource_type` clause resets the resource type for the group. - -To use the `ALTER RESOURCE GROUP` command, you must have superuser privileges. - -#### Parameters - -`group_name` - -The name of the resource group to alter. - -`new_name` - -The new name to assign to the resource group. - -`resource_type` - -Specifies the type of resource to which to set a usage value. - -`value | DEFAULT` - -When `value` is specified, the literal value to assign to `resource_type`. Specify `DEFAULT` to reset the assignment of `resource_type` for the resource group. - -#### Example - -These examples show the use of the `ALTER RESOURCE GROUP` command: - -```sql -edb=# ALTER RESOURCE GROUP resgrp_a RENAME TO newgrp; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_b SET cpu_rate_limit = .5; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_b SET dirty_rate_limit = 6144; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_c RESET cpu_rate_limit; -ALTER RESOURCE GROUP -``` - -This query shows the results of the `ALTER RESOURCE GROUP` commands to the entries in the `edb_resource_group` catalog: - -```sql -edb=# SELECT * FROM edb_resource_group; -__OUTPUT__ - rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit ------------+------------------+-------------------- - newgrp | 0 | 0 - resgrp_b | 0.5 | 6144 - resgrp_c | 0 | 0 -(3 rows) -``` - -### DROP RESOURCE GROUP - -Use the `DROP RESOURCE GROUP` command to remove a resource group. - -```sql -DROP RESOURCE GROUP [ IF EXISTS ] ; -``` - -#### Description - -The `DROP RESOURCE GROUP` command removes a resource group with the specified name. - -To use the `DROP RESOURCE GROUP` command, you must have superuser privileges. - -#### Parameters - -`group_name` - -The name of the resource group to remove. - -`IF EXISTS` - -Don't throw an error if the resource group doesn't exist. Instead, issue a notice. - -#### Example - -This example removes the resource group `newgrp`: - -```sql -edb=# DROP RESOURCE GROUP newgrp; -DROP RESOURCE GROUP -``` - -### Assigning a process to a resource group - -Use the `SET edb_resource_group TO group_name` command to assign the current process to a specified resource group: - -```sql -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) -``` - -The resource type settings of the group take effect on the current process immediately. If you use the command to change the resource group assigned to the current process, the resource type settings of the newly assigned group take effect immediately. - -You can include processes in a resource group by default by assigning a default resource group to roles, databases, or an entire database server instance. - -You can assign a default resource group to a role using the `ALTER ROLE ... SET` command. For more information about the `ALTER ROLE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-alterrole.html). - -You can assign a default resource group to a database by using the `ALTER DATABASE ... SET` command. For more information about the `ALTER DATABASE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-alterdatabase.html). - -You can assign the entire database server instance a default resource group by setting the `edb_resource_group` configuration parameter in the `postgresql.conf` file: - -```ini -# - EDB Resource Manager - -#edb_max_resource_groups = 16 # 0-65536 (change requires restart) -edb_resource_group = 'resgrp_b' -``` - -If you change `edb_resource_group` in the `postgresql.conf` file, reload the configuration file to make it take effect on the database server instance. - -### Removing a process from a resource group - -Set `edb_resource_group` to `DEFAULT` or use `RESET edb_resource_group` to remove the current process from a resource group: - -```sql -edb=# SET edb_resource_group TO DEFAULT; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- -(1 row) -``` - -To remove a default resource group from a role, use the `ALTER ROLE ... RESET` form of the `ALTER ROLE` command. - -To remove a default resource group from a database, use the `ALTER DATABASE ... RESET` form of the `ALTER DATABASE` command. - -To remove a default resource group from the database server instance, set the `edb_resource_group` configuration parameter to an empty string in the `postgresql.conf` file. Then, reload the configuration file. - -### Monitoring processes in resource groups - -After you create resource groups, you can get the number of processes actively using these resource groups from the view `edb_all_resource_groups`. - -The following are the columns in `edb_all_resource_groups`: - -- **group_name.** Name of the resource group. -- **active_processes.** Number of active processes in the resource group. -- **cpu_rate_limit.** The value of the CPU rate limit resource type assigned to the resource group. -- **per_process_cpu_rate_limit.** The CPU rate limit that applies to an individual active process in the resource group. -- **dirty_rate_limit.** The value of the dirty rate limit resource type assigned to the resource group. -- **per_process_dirty_rate_limit.** The dirty rate limit that applies to an individual active process in the resource group. - -!!! Note - Columns `per_process_cpu_rate_limit` and `per_process_dirty_rate_limit` don't show the actual resource consumption used by the processes. They indicate how `EDB Resource Manager` sets the resource limit for an individual process based on the number of active processes in the resource group. - -This example shows `edb_all_resource_groups` when resource group `resgrp_a` contains no active processes, resource group `resgrp_b` contains two active processes, and resource group `resgrp_c` contains one active process: - -```sql -edb=# SELECT * FROM edb_all_resource_groups ORDER BY group_name; -__OUTPUT__ --[ RECORD 1 ]-----------------+------------------ - group_name | resgrp_a - active_processes | 0 - cpu_rate_limit | 0.5 - per_process_cpu_rate_limit | - dirty_rate_limit | 12288 - per_process_dirty_rate_limit | --[ RECORD 2 ]-----------------+------------------ - group_name | resgrp_b - active_processes | 2 - cpu_rate_limit | 0.4 - per_process_cpu_rate_limit | 0.195694289022895 - dirty_rate_limit | 6144 - per_process_dirty_rate_limit | 3785.92924684337 --[ RECORD 3 ]-----------------+------------------ - group_name | resgrp_c - active_processes | 1 - cpu_rate_limit | 0.3 - per_process_cpu_rate_limit | 0.292342129631091 - dirty_rate_limit | 3072 - per_process_dirty_rate_limit | 3072 -``` - -The CPU rate limit and dirty rate limit settings that are assigned to these resource groups are: - -```sql -edb=# SELECT * FROM edb_resource_group; -__OUTPUT__ - rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit ------------+------------------+-------------------- - resgrp_a | 0.5 | 12288 - resgrp_b | 0.4 | 6144 - resgrp_c | 0.3 | 3072 -(3 rows) -``` - -In the `edb_all_resource_groups` view, the `per_process_cpu_rate_limit` and `per_process_dirty_rate_limit` values are roughly the corresponding CPU rate limit and dirty rate limit divided by the number of active processes. - -## CPU usage throttling - -Control CPU use of a resource group by setting the `cpu_rate_limit` resource type parameter. - -Set the `cpu_rate_limit` parameter to the fraction of CPU time over wall-clock time to which the combined, simultaneous CPU usage of all processes in the group must not exceed. The value assigned to `cpu_rate_limit` is typically less than or equal to 1. - -On multicore systems, you can apply the `cpu_rate_limit` to more than one CPU core by setting it to greater than 1. For example, if `cpu_rate_limit` is set to 2.0, you use 100% of two CPUs. - -The valid range of the `cpu_rate_limit` parameter is 0 to 1.67772e+07. A setting of 0 means no CPU rate limit was set for the resource group. - -When the value is multiplied by 100, you can also interpret the `cpu_rate_limit` as the CPU usage percentage for a resource group. - -EDB Resource Manager uses *CPU throttling* to keep the aggregate CPU usage of all processes in the group within the limit specified by the `cpu_rate_limit` parameter. A process in the group might be interrupted and put into sleep mode for a short time to maintain the defined limit. When and how such interruptions occur is defined by a proprietary algorithm used by EDB Resource Manager. - -### Setting the CPU rate limit for a resource group - -Use the `ALTER RESOURCE GROUP` command with the `SET cpu_rate_limit` clause to set the CPU rate limit for a resource group. - -In this example, the CPU usage limit is set to 50% for `resgrp_a`, 40% for `resgrp_b`, and 30% for `resgrp_c`. This means that the combined CPU usage of all processes assigned to `resgrp_a` is maintained at approximately 50%. Similarly, for all processes in `resgrp_b`, the combined CPU usage is kept to approximately 40%, and so on. - -```sql -edb=# ALTER RESOURCE GROUP resgrp_a SET cpu_rate_limit TO .5; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_b SET cpu_rate_limit TO .4; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_c SET cpu_rate_limit TO .3; -ALTER RESOURCE GROUP -``` - -This query shows the settings of `cpu_rate_limit` in the catalog: - -```sql -edb=# SELECT rgrpname, rgrpcpuratelimit FROM edb_resource_group; -__OUTPUT__ - rgrpname | rgrpcpuratelimit -----------+------------------ - resgrp_a | 0.5 - resgrp_b | 0.4 - resgrp_c | 0.3 -(3 rows) -``` - -Changing the `cpu_rate_limit` of a resource group affects new processes that are assigned to the group. It also immediately affects any currently running processes that are members of the group. That is, if the `cpu_rate_limit` is changed from .5 to .3, currently running processes in the group are throttled downward so that the aggregate group CPU usage is near 30% instead of 50%. - -To show the effect of setting the CPU rate limit for resource groups, the following `psql` command-line examples use a CPU-intensive calculation of 20000 factorial (multiplication of 20000 \* 19999 \* 19998, and so on) performed by the query `SELECT 20000!`. - -The resource groups with the CPU rate limit settings shown in the previous query are used in these examples. - -### Example: Single process in a single group - -This example shows that the current process is set to use resource group `resgrp_b`. The factorial calculation then starts. - -```sql -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) -edb=# SELECT 20000!; -``` - -In a second session, the Linux `top` command is used to display the CPU usage as shown under the `%CPU` column. The following is a snapshot at an arbitrary point in time, as the `top` command output periodically changes: - -```shell -$ top -top - 16:37:03 up 4:15, 7 users, load average: 0.49, 0.20, 0.38 -Tasks: 202 total, 1 running, 201 sleeping, 0 stopped, 0 zombie -Cpu(s): 42.7%us, 2.3%sy, 0.0%ni, 55.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 -Mem: 1025624k total, 791160k used, 234464k free, 23400k buffers -Swap: 103420k total, 13404k used, 90016k free, 373504k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -28915 enterpri 20 0 195m 5900 4212 S 39.9 0.6 3:36.98 edb-postgres - 1033 root 20 0 171m 77m 2960 S 1.0 7.8 3:43.96 Xorg - 3040 user 20 0 278m 22m 14m S 1.0 2.2 3:41.72 knotify4 - . - . - . -``` - -The row where `edb-postgres` appears under the `COMMAND` column shows the `psql` session performing the factorial calculation. The CPU usage of the session shown under the `%CPU` column is 39.9, which is close to the 40% CPU limit set for resource group `resgrp_b`. - -By contrast, if the `psql` session is removed from the resource group and the factorial calculation is performed again, the CPU usage is much higher. - -```sql -edb=# SET edb_resource_group TO DEFAULT; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - -(1 row) - -edb=# SELECT 20000!; -``` - -Under the `%CPU` column for `edb-postgres`, the CPU usage is now 93.6, which is significantly higher than the 39.9 when the process was part of the resource group: - -```shell -$ top -top - 16:43:03 up 4:21, 7 users, load average: 0.66, 0.33, 0.37 -Tasks: 202 total, 5 running, 197 sleeping, 0 stopped, 0 zombie -Cpu(s): 96.7%us, 3.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 -Mem: 1025624k total, 791228k used, 234396k free, 23560k buffers -Swap: 103420k total, 13404k used, 90016k free, 373508k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -28915 enterpri 20 0 195m 5900 4212 R 93.6 0.6 5:01.56 edb-postgres - 1033 root 20 0 171m 77m 2960 S 1.0 7.8 3:48.15 Xorg - 2907 user 20 0 98.7m 11m 9100 S 0.3 1.2 0:46.51 vmware-user-lo - . - . - . -``` - -### Example: Multiple processes in a single group - -As stated previously, the CPU rate limit applies to the aggregate of all processes in the resource group. This concept is shown in the following example. - -The factorial calculation is performed simultaneously in two separate `psql` sessions, each of which was added to resource group `resgrp_b` that has `cpu_rate_limit` set to .4 (CPU usage of 40%). - -#### Session 1 - -```sql -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) - -edb=# SELECT 20000!; -``` - -#### Session 2 - -```sql -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) - -edb=# SELECT 20000!; -``` - -A third session monitors the CPU usage: - -```shell -$ top -top - 16:53:03 up 4:31, 7 users, load average: 0.31, 0.19, 0.27 -Tasks: 202 total, 1 running, 201 sleeping, 0 stopped, 0 zombie -Cpu(s): 41.2%us, 3.0%sy, 0.0%ni, 55.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 -Mem: 1025624k total, 792020k used, 233604k free, 23844k buffers -Swap: 103420k total, 13404k used, 90016k free, 373508k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -29857 enterpri 20 0 195m 4708 3312 S 19.9 0.5 0:57.35 edb-postgres -28915 enterpri 20 0 195m 5900 4212 S 19.6 0.6 5:35.49 edb-postgres - 3040 user 20 0 278m 22m 14m S 1.0 2.2 3:54.99 knotify4 - 1033 root 20 0 171m 78m 2960 S 0.3 7.8 3:55.71 Xorg - . - . - . -``` - -Two new processes named `edb-postgres` have `%CPU` values of 19.9 and 19.6. The sum is close to the 40% CPU usage set for resource group `resgrp_b`. - -This command sequence displays the sum of all `edb-postgres` processes sampled over half-second time intervals. This example shows how the total CPU usage of the processes in the resource group changes over time as EDB Resource Manager throttles the processes to keep the total resource group CPU usage near 40%. - -```shell -$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM -+= $9} END { print SUM / 2 }'; done -37.2 -39.1 -38.9 -38.3 -44.7 -39.2 -42.5 -39.1 -39.2 -39.2 -41 -42.85 -46.1 - . - . - . -``` - -### Example: Multiple processes in multiple groups - -In this example, two additional `psql` sessions are used along with the previous two sessions. The third and fourth sessions perform the same factorial calculation in resource group `resgrp_c` with a `cpu_rate_limit` of `.3` (30% CPU usage). - -#### Session 3 - -```sql -edb=# SET edb_resource_group TO resgrp_c; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_c -(1 row) - -edb=# SELECT 20000!; -``` - -#### Session 4 - -```sql -edb=# SET edb_resource_group TO resgrp_c; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_c -(1 row) -edb=# SELECT 20000!; -``` - -The `top` command displays the following output: - -```shell -$ top -top - 17:45:09 up 5:23, 8 users, load average: 0.47, 0.17, 0.26 -Tasks: 203 total, 4 running, 199 sleeping, 0 stopped, 0 zombie -Cpu(s): 70.2%us, 0.0%sy, 0.0%ni, 29.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 -Mem: 1025624k total, 806140k used, 219484k free, 25296k buffers -Swap: 103420k total, 13404k used, 90016k free, 374092k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -29857 enterpri 20 0 195m 4820 3324 S 19.9 0.5 4:25.02 edb-postgres -28915 enterpri 20 0 195m 5900 4212 R 19.6 0.6 9:07.50 edb-postgres -29023 enterpri 20 0 195m 4744 3248 R 16.3 0.5 4:01.73 edb-postgres -11019 enterpri 20 0 195m 4120 2764 R 15.3 0.4 0:04.92 edb-postgres - 2907 user 20 0 98.7m 12m 9112 S 1.3 1.2 0:56.54 vmware-user-lo - 3040 user 20 0 278m 22m 14m S 1.3 2.2 4:38.73 knotify4 -``` - -The two resource groups in use have CPU usage limits of 40% and 30%. The sum of the `%CPU` column for the first two `edb-postgres` processes is 39.5 (approximately 40%, which is the limit for `resgrp_b`). The sum of the `%CPU` column for the third and fourth `edb-postgres` processes is 31.6 (approximately 30%, which is the limit for `resgrp_c`). - -The sum of the CPU usage limits of the two resource groups to which these processes belong is 70%. The following output shows that the sum of the four processes borders around 70%: - -```shell -$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM -+= $9} END { print SUM / 2 }'; done -61.8 -76.4 -72.6 -69.55 -64.55 -79.95 -68.55 -71.25 -74.85 -62 -74.85 -76.9 -72.4 -65.9 -74.9 -68.25 -``` - -By contrast, if three sessions are processing. where two sessions remain in `resgrp_b` but the third session doesn't belong to any resource group, the `top` command shows the following output: - -```shell -$ top -top - 17:24:55 up 5:03, 7 users, load average: 1.00, 0.41, 0.38 -Tasks: 199 total, 3 running, 196 sleeping, 0 stopped, 0 zombie -Cpu(s): 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 -Mem: 1025624k total, 797692k used, 227932k free, 24724k buffers -Swap: 103420k total, 13404k used, 90016k free, 374068k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -29023 enterpri 20 0 195m 4744 3248 R 58.6 0.5 2:53.75 edb-postgres -28915 enterpri 20 0 195m 5900 4212 S 18.9 0.6 7:58.45 edb-postgres -29857 enterpri 20 0 195m 4820 3324 S 18.9 0.5 3:14.85 edb-postgres - 1033 root 20 0 174m 81m 2960 S 1.7 8.2 4:26.50 Xorg - 3040 user 20 0 278m 22m 14m S 1.0 2.2 4:21.20 knotify4 -``` - -The second and third `edb-postgres` processes belonging to the resource group where the CPU usage is limited to 40% have a total CPU usage of 37.8. However, the first `edb-postgres` process has a 58.6% CPU usage, as it isn't within a resource group. It basically uses the remaining available CPU resources on the system. - -Likewise, the following output shows the sum of all three sessions is around 95%, since one of the sessions has no set limit on its CPU usage: - -```shell -$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM -+= $9} END { print SUM / 2 }'; done -96 -90.35 -92.55 -96.4 -94.1 -90.7 -95.7 -95.45 -93.65 -87.95 -96.75 -94.25 -95.45 -97.35 -92.9 -96.05 -96.25 -94.95 - . - . - . -``` - -## Dirty buffer throttling - -To control writing to shared buffers, set the `dirty_rate_limit` resource type parameter. - -Set the `dirty_rate_limit` parameter to the number of kilobytes per second for the combined rate at which all the processes in the group write to, or “dirty”, the shared buffers. An example setting is 3072 kilobytes per seconds. - -The valid range of the `dirty_rate_limit` parameter is 0 to 1.67772e+07. A setting of 0 means no dirty rate limit was set for the resource group. - -EDB Resource Manager uses *dirty buffer throttling* to keep the aggregate shared buffer writing rate of all processes in the group near the limit specified by the `dirty_rate_limit` parameter. A process in the group might be interrupted and put into sleep mode for a short time to maintain the defined limit. When and how such interruptions occur is defined by a proprietary algorithm used by EDB Resource Manager. - -### Setting the dirty rate limit for a resource group - -Use the `ALTER RESOURCE GROUP` command with the `SET dirty_rate_limit` clause to set the dirty rate limit for a resource group. - -In this example, the dirty rate limit is set to 12288 kilobytes per second for `resgrp_a`, 6144 kilobytes per second for `resgrp_b`, and 3072 kilobytes per second for `resgrp_c`. This means that the combined writing rate to the shared buffer of all processes assigned to `resgrp_a` is maintained at approximately 12288 kilobytes per second. Similarly, for all processes in `resgrp_b`, the combined writing rate to the shared buffer is kept to approximately 6144 kilobytes per second, and so on. - -```sql -edb=# ALTER RESOURCE GROUP resgrp_a SET dirty_rate_limit TO 12288; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_b SET dirty_rate_limit TO 6144; -ALTER RESOURCE GROUP -edb=# ALTER RESOURCE GROUP resgrp_c SET dirty_rate_limit TO 3072; -ALTER RESOURCE GROUP -``` - -This query shows the settings of `dirty_rate_limit` in the catalog; - -```sql -edb=# SELECT rgrpname, rgrpdirtyratelimit FROM edb_resource_group; -__OUTPUT__ - rgrpname | rgrpdirtyratelimit ------------+-------------------- - resgrp_a | 12288 - resgrp_b | 6144 - resgrp_c | 3072 -(3 rows) -``` - -Changing the `dirty_rate_limit` of a resource group affects new processes that are assigned to the group. Any currently running processes that are members of the group are also immediately affected by the change. That is, if the `dirty_rate_limit` is changed from 12288 to 3072, currently running processes in the group are throttled downward so that the aggregate group dirty rate is near 3072 kilobytes per second instead of 12288 kilobytes per second. - -To show the effect of setting the dirty rate limit for resource groups, the examples use the following table for intensive I/O operations: - -```sql -CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -``` - -The `FILLFACTOR = 10` clause results in `INSERT` commands packing rows up to only 10% per page. The result is a larger sampling of dirty shared blocks for the purpose of these examples. - -The `pg_stat_statements` module is used to display the number of shared buffer blocks that are dirtied by a SQL command and the amount of time the command took to execute. This information is used to calculate the actual kilobytes per second writing rate for the SQL command and thus compare it to the dirty rate limit set for a resource group. - -To use the `pg_stat_statements` module: - -1. In the `postgresql.conf` file, add `$libdir/pg_stat_statements` to the `shared_preload_libraries` configuration parameter: - - ```ini - shared_preload_libraries = '$libdir/dbms_pipe,$libdir/edb_gen,$libdir/pg_stat_statements' - ``` - -2. Restart the database server. - -3. Use the `CREATE EXTENSION` command to finish creating the `pg_stat_statements` module: - - ```sql - edb=# CREATE EXTENSION pg_stat_statements SCHEMA public; - CREATE EXTENSION - ``` - - The `pg_stat_statements_reset()` function clears out the `pg_stat_statements` view for clarity of each example. - -The resource groups with the dirty rate limit settings shown in the previous query are used in these examples. - - - -### Example: Single process in a single group - -This sequence of commands creates table `t1`. The current process is set to use resource group `resgrp_b`. The `pg_stat_statements` view is cleared out by running the `pg_stat_statements_reset()` function. - -The `INSERT` command then generates a series of integers from 1 to 10,000 to populate the table and dirty approximately 10,000 blocks: - -```sql -edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) -``` -```sql -edb=# SELECT pg_stat_statements_reset(); -__OUTPUT__ - pg_stat_statements_reset --------------------------- - -(1 row) - -edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -The following shows the results from the `INSERT` command: - -```sql -edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM -pg_stat_statements; -__OUTPUT__ --[ RECORD 1 ]--------+-------------------------------------------------- - query | INSERT INTO t1 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 13496.184 - shared_blks_dirtied | 10003 -``` - -The actual dirty rate is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 13496.184 ms, which yields *0.74117247 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *741.17247 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *6072 kilobytes per second*. - -The actual dirty rate of 6072 kilobytes per second is close to the dirty rate limit for the resource group, which is 6144 kilobytes per second. - -By contrast, if you repeat the steps without the process belonging to any resource group, the dirty buffer rate is much higher: - -```sql -edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - -(1 row) -``` -```sql -edb=# SELECT pg_stat_statements_reset(); -__OUTPUT__ - pg_stat_statements_reset --------------------------- - -(1 row) - -edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -The following shows the results from the `INSERT` command without the use of a resource group: - -```sql -edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM -pg_stat_statements; -__OUTPUT__ --[ RECORD 1 ]--------+-------------------------------------------------- - query | INSERT INTO t1 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 2432.165 - shared_blks_dirtied | 10003 -``` - -The total time was only 2432.165 milliseconds, compared to 13496.184 milliseconds when using a resource group with a dirty rate limit set to 6144 kilobytes per second. - -The actual dirty rate without the use of a resource group is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 2432.165 ms, which yields *4.112797 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *4112.797 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *33692 kilobytes per second*. - -The actual dirty rate of 33692 kilobytes per second is much higher than when the resource group with a dirty rate limit of 6144 kilobytes per second was used. - - - -### Example: Multiple processes in a single group - -As stated previously, the dirty rate limit applies to the aggregate of all processes in the resource group. This concept is illustrated in the following example. - -For this example. the inserts are performed simultaneously on two different tables in two separate `psql` sessions, each of which was added to resource group `resgrp_b` that has a `dirty_rate_limit` set to 6144 kilobytes per second. - -#### Session 1 - -```sql -edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) - -edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -#### Session 2 - -```sql -edb=# CREATE TABLE t2 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SET edb_resource_group TO resgrp_b; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_b -(1 row) -``` -```sql -edb=# SELECT pg_stat_statements_reset(); -__OUTPUT__ - pg_stat_statements_reset --------------------------- -(1 row) - -edb=# INSERT INTO t2 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -!!! Note - The `INSERT` commands in session 1 and session 2 started after the `SELECT pg_stat_statements_reset()` command in session 2 ran. - -The following shows the results from the `INSERT` commands in the two sessions. `RECORD 3` shows the results from session 1. `RECORD 2` shows the results from session 2. - -```sql -edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM -pg_stat_statements; -__OUTPUT__ --[ RECORD 1 ]--------+-------------------------------------------------- - query | SELECT pg_stat_statements_reset(); - rows | 1 - total_time | 0.43 - shared_blks_dirtied | 0 --[ RECORD 2 ]--------+-------------------------------------------------- - query | INSERT INTO t2 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 30591.551 - shared_blks_dirtied | 10003 --[ RECORD 3 ]--------+-------------------------------------------------- - query | INSERT INTO t1 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 33215.334 - shared_blks_dirtied | 10003 -``` - -The total time was 33215.334 milliseconds for session 1 and 30591.551 milliseconds for session 2. When only one session was active in the same resource group, as shown in the first example, the time was 13496.184 milliseconds. Thus, more active processes in the resource group result in a slower dirty rate for each active process in the group. The following calculations show this. - -The actual dirty rate for session 1 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 33215.334 ms, which yields *0.30115609 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *301.15609 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2467 kilobytes per second*. - -The actual dirty rate for session 2 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 30591.551 ms, which yields *0.32698571 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *326.98571 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2679 kilobytes per second*. - -The combined dirty rate from session 1 (2467 kilobytes per second) and from session 2 (2679 kilobytes per second) yields 5146 kilobytes per second, which is below the set dirty rate limit of the resource group (6144 kilobytes per seconds). - - - -### Example: Multiple processes in multiple groups - -In this example, two additional `psql` sessions are used along with the previous two sessions. The third and fourth sessions perform the same `INSERT` command in resource group `resgrp_c` with a `dirty_rate_limit` of 3072 kilobytes per second. - -Sessions 1 and 2 are repeated as shown in the prior example using resource group `resgrp_b` with a `dirty_rate_limit` of 6144 kilobytes per second. - -#### Session 3 - -```sql -edb=# CREATE TABLE t3 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SET edb_resource_group TO resgrp_c; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - - -resgrp_c -(1 row) - -edb=# INSERT INTO t3 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -#### Session 4 - -```sql -edb=# CREATE TABLE t4 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); -CREATE TABLE -edb=# SET edb_resource_group TO resgrp_c; -SET -edb=# SHOW edb_resource_group; -__OUTPUT__ - edb_resource_group --------------------- - resgrp_c -(1 row) -``` -```sql -edb=# SELECT pg_stat_statements_reset(); -__OUTPUT__ - pg_stat_statements_reset --------------------------- - -(1 row) - -edb=# INSERT INTO t4 VALUES (generate_series (1,10000), 'aaa'); -INSERT 0 10000 -``` - -!!! Note - The `INSERT` commands in all four sessions started after the `SELECT pg_stat_statements_reset()` command in session 4 ran. - -The following shows the results from the `INSERT` commands in the four sessions: - -- `RECORD 3` shows the results from session 1. `RECORD 2` shows the results from session 2. - -- `RECORD 4` shows the results from session 3. `RECORD 5` shows the results from session 4. - -```sql -edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM -pg_stat_statements; -__OUTPUT__ --[ RECORD 1 ]--------+-------------------------------------------------- - query | SELECT pg_stat_statements_reset(); - rows | 1 - total_time | 0.467 - shared_blks_dirtied | 0 --[ RECORD 2 ]--------+-------------------------------------------------- - query | INSERT INTO t2 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 31343.458 - shared_blks_dirtied | 10003 --[ RECORD 3 ]--------+-------------------------------------------------- - query | INSERT INTO t1 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 28407.435 - shared_blks_dirtied | 10003 --[ RECORD 4 ]--------+-------------------------------------------------- - query | INSERT INTO t3 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 52727.846 - shared_blks_dirtied | 10003 --[ RECORD 5 ]--------+-------------------------------------------------- - query | INSERT INTO t4 VALUES (generate_series (?,?), ?); - rows | 10000 - total_time | 56063.697 - shared_blks_dirtied | 10003 -``` - -The times of session 1 (28407.435) and session 2 (31343.458) are close to each other, as they are both in the same resource group with `dirty_rate_limit` set to 6144. These times differe from the times of session 3 (52727.846) and session 4 (56063.697), which are in the resource group with `dirty_rate_limit` set to 3072. The latter group has a slower dirty rate limit, so the expected processing time is longer, as is the case for sessions 3 and 4. - -The actual dirty rate for session 1 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 28407.435 ms, which yields *0.35212612 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *352.12612 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2885 kilobytes per second*. - -The actual dirty rate for session 2 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 31343.458 ms, which yields *0.31914156 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *319.14156 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2614 kilobytes per second*. - -The combined dirty rate from session 1 (2885 kilobytes per second) and from session 2 (2614 kilobytes per second) yields 5499 kilobytes per second, which is near the set dirty rate limit of the resource group (6144 kilobytes per seconds). - -The actual dirty rate for session 3 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 52727.846 ms, which yields *0.18971001 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *189.71001 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *1554 kilobytes per second*. - -The actual dirty rate for session 4 is calculated as follows: - -- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 56063.697 ms, which yields *0.17842205 blocks per millisecond*. -- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *178.42205 blocks per second*. -- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *1462 kilobytes per second*. - -The combined dirty rate from session 3 (1554 kilobytes per second) and from session 4 (1462 kilobytes per second) yields 3016 kilobytes per second, which is near the set dirty rate limit of the resource group (3072 kilobytes per seconds). - -This example shows how EDB Resource Manager keeps the aggregate dirty rate of the active processes in its groups close to the dirty rate limit set for each group. - - - -## System catalogs - -System catalogs store the resource group information used by EDB Resource Manager. - -### edb_all_resource_groups - -The following table lists the information available in the `edb_all_resource_groups` catalog. - -| Column | Type | Description | -| ------------------------------ | --------- | ---------------------------------------------------------------------------- | -| `group_name` | `name` | The name of the resource group. | -| `active_processes` | `integer` | Number of currently active processes in the resource group. | -| `cpu_rate_limit` | `float8` | Maximum CPU rate limit for the resource group. `0` means no limit. | -| `per_process_cpu_rate_limit` | `float8` | Maximum CPU rate limit per currently active process in the resource group. | -| `dirty_rate_limit` | `float8` | Maximum dirty rate limit for a resource group. `0` means no limit. | -| `per_process_dirty_rate_limit` | `float8` | Maximum dirty rate limit per currently active process in the resource group. | - - - -### edb_resource_group - -The following table lists the information available in the `edb_resource_group` catalog. - -| Column | Type | Description | -| -------------------- | -------- | ------------------------------------------------------------------ | -| `rgrpname` | `name` | The name of the resource group. | -| `rgrpcpuratelimit` | `float8` | Maximum CPU rate limit for a resource group. `0` means no limit. | -| `rgrpdirtyratelimit` | `float8` | Maximum dirty rate limit for a resource group. `0` means no limit. | diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/cpu_usage_throttling.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/cpu_usage_throttling.mdx new file mode 100644 index 00000000000..f8cba087c68 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/cpu_usage_throttling.mdx @@ -0,0 +1,325 @@ +--- +title: "CPU usaqe throttling" +description: "How to use EDB Resource Manager to control the CPU use of a resource group" +--- + +Control CPU use of a resource group by setting the `cpu_rate_limit` resource type parameter. + +Set the `cpu_rate_limit` parameter to the fraction of CPU time over wall-clock time to which the combined, simultaneous CPU usage of all processes in the group must not exceed. The value assigned to `cpu_rate_limit` is typically less than or equal to 1. + +On multicore systems, you can apply the `cpu_rate_limit` to more than one CPU core by setting it to greater than 1. For example, if `cpu_rate_limit` is set to 2.0, you use 100% of two CPUs. The valid range of the `cpu_rate_limit` parameter is 0 to 1.67772e+07. A setting of 0 means no CPU rate limit was set for the resource group. + +When the value is multiplied by 100, you can also interpret the `cpu_rate_limit` as the CPU usage percentage for a resource group. + +EDB Resource Manager uses *CPU throttling* to keep the aggregate CPU usage of all processes in the group within the limit specified by the `cpu_rate_limit` parameter. A process in the group might be interrupted and put into sleep mode for a short time to maintain the defined limit. When and how such interruptions occur is defined by a proprietary algorithm used by EDB Resource Manager. + +## Setting the CPU rate limit for a resource group + +Use the `ALTER RESOURCE GROUP` command with the `SET cpu_rate_limit` clause to set the CPU rate limit for a resource group. + +In this example, the CPU usage limit is set to 50% for `resgrp_a`, 40% for `resgrp_b`, and 30% for `resgrp_c`. This means that the combined CPU usage of all processes assigned to `resgrp_a` is maintained at approximately 50%. Similarly, for all processes in `resgrp_b`, the combined CPU usage is kept to approximately 40%, and so on. + +```sql +edb=# ALTER RESOURCE GROUP resgrp_a SET cpu_rate_limit TO .5; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_b SET cpu_rate_limit TO .4; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_c SET cpu_rate_limit TO .3; +ALTER RESOURCE GROUP +``` + +This query shows the settings of `cpu_rate_limit` in the catalog: + +```sql +edb=# SELECT rgrpname, rgrpcpuratelimit FROM edb_resource_group; +__OUTPUT__ + rgrpname | rgrpcpuratelimit +----------+------------------ + resgrp_a | 0.5 + resgrp_b | 0.4 + resgrp_c | 0.3 +(3 rows) +``` + +Changing the `cpu_rate_limit` of a resource group affects new processes that are assigned to the group. It also immediately affects any currently running processes that are members of the group. That is, if the `cpu_rate_limit` is changed from .5 to .3, currently running processes in the group are throttled downward so that the aggregate group CPU usage is near 30% instead of 50%. + +To show the effect of setting the CPU rate limit for resource groups, the following `psql` command-line examples use a CPU-intensive calculation of 20000 factorial (multiplication of 20000 \* 19999 \* 19998, and so on) performed by the query `SELECT 20000!`. + +The resource groups with the CPU rate limit settings shown in the previous query are used in these examples. + +## Example: Single process in a single group + +This example shows that the current process is set to use resource group `resgrp_b`. The factorial calculation then starts. + +```sql +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) +edb=# SELECT 20000!; +``` + +In a second session, the Linux `top` command is used to display the CPU usage as shown under the `%CPU` column. The following is a snapshot at an arbitrary point in time, as the `top` command output periodically changes: + +```shell +$ top +top - 16:37:03 up 4:15, 7 users, load average: 0.49, 0.20, 0.38 +Tasks: 202 total, 1 running, 201 sleeping, 0 stopped, 0 zombie +Cpu(s): 42.7%us, 2.3%sy, 0.0%ni, 55.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 +Mem: 1025624k total, 791160k used, 234464k free, 23400k buffers +Swap: 103420k total, 13404k used, 90016k free, 373504k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +28915 enterpri 20 0 195m 5900 4212 S 39.9 0.6 3:36.98 edb-postgres + 1033 root 20 0 171m 77m 2960 S 1.0 7.8 3:43.96 Xorg + 3040 user 20 0 278m 22m 14m S 1.0 2.2 3:41.72 knotify4 + . + . + . +``` + +The row where `edb-postgres` appears under the `COMMAND` column shows the `psql` session performing the factorial calculation. The CPU usage of the session shown under the `%CPU` column is 39.9, which is close to the 40% CPU limit set for resource group `resgrp_b`. + +By contrast, if the `psql` session is removed from the resource group and the factorial calculation is performed again, the CPU usage is much higher. + +```sql +edb=# SET edb_resource_group TO DEFAULT; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + +(1 row) + +edb=# SELECT 20000!; +``` + +Under the `%CPU` column for `edb-postgres`, the CPU usage is now 93.6, which is significantly higher than the 39.9 when the process was part of the resource group: + +```shell +$ top +top - 16:43:03 up 4:21, 7 users, load average: 0.66, 0.33, 0.37 +Tasks: 202 total, 5 running, 197 sleeping, 0 stopped, 0 zombie +Cpu(s): 96.7%us, 3.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 +Mem: 1025624k total, 791228k used, 234396k free, 23560k buffers +Swap: 103420k total, 13404k used, 90016k free, 373508k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +28915 enterpri 20 0 195m 5900 4212 R 93.6 0.6 5:01.56 edb-postgres + 1033 root 20 0 171m 77m 2960 S 1.0 7.8 3:48.15 Xorg + 2907 user 20 0 98.7m 11m 9100 S 0.3 1.2 0:46.51 vmware-user-lo + . + . + . +``` + +## Example: Multiple processes in a single group + +As stated previously, the CPU rate limit applies to the aggregate of all processes in the resource group. This concept is shown in the following example. + +The factorial calculation is performed simultaneously in two separate `psql` sessions, each of which was added to resource group `resgrp_b` that has `cpu_rate_limit` set to .4 (CPU usage of 40%). + +### Session 1 + +```sql +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) + +edb=# SELECT 20000!; +``` + +### Session 2 + +```sql +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) + +edb=# SELECT 20000!; +``` + +A third session monitors the CPU usage: + +```shell +$ top +top - 16:53:03 up 4:31, 7 users, load average: 0.31, 0.19, 0.27 +Tasks: 202 total, 1 running, 201 sleeping, 0 stopped, 0 zombie +Cpu(s): 41.2%us, 3.0%sy, 0.0%ni, 55.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 +Mem: 1025624k total, 792020k used, 233604k free, 23844k buffers +Swap: 103420k total, 13404k used, 90016k free, 373508k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +29857 enterpri 20 0 195m 4708 3312 S 19.9 0.5 0:57.35 edb-postgres +28915 enterpri 20 0 195m 5900 4212 S 19.6 0.6 5:35.49 edb-postgres + 3040 user 20 0 278m 22m 14m S 1.0 2.2 3:54.99 knotify4 + 1033 root 20 0 171m 78m 2960 S 0.3 7.8 3:55.71 Xorg + . + . + . +``` + +Two new processes named `edb-postgres` have `%CPU` values of 19.9 and 19.6. The sum is close to the 40% CPU usage set for resource group `resgrp_b`. + +This command sequence displays the sum of all `edb-postgres` processes sampled over half-second time intervals. This example shows how the total CPU usage of the processes in the resource group changes over time as EDB Resource Manager throttles the processes to keep the total resource group CPU usage near 40%. + +```shell +$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM ++= $9} END { print SUM / 2 }'; done +37.2 +39.1 +38.9 +38.3 +44.7 +39.2 +42.5 +39.1 +39.2 +39.2 +41 +42.85 +46.1 + . + . + . +``` + +## Example: Multiple processes in multiple groups + +In this example, two additional `psql` sessions are used along with the previous two sessions. The third and fourth sessions perform the same factorial calculation in resource group `resgrp_c` with a `cpu_rate_limit` of `.3` (30% CPU usage). + +### Session 3 + +```sql +edb=# SET edb_resource_group TO resgrp_c; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_c +(1 row) + +edb=# SELECT 20000!; +``` + +### Session 4 + +```sql +edb=# SET edb_resource_group TO resgrp_c; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_c +(1 row) +edb=# SELECT 20000!; +``` + +The `top` command displays the following output: + +```shell +$ top +top - 17:45:09 up 5:23, 8 users, load average: 0.47, 0.17, 0.26 +Tasks: 203 total, 4 running, 199 sleeping, 0 stopped, 0 zombie +Cpu(s): 70.2%us, 0.0%sy, 0.0%ni, 29.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 +Mem: 1025624k total, 806140k used, 219484k free, 25296k buffers +Swap: 103420k total, 13404k used, 90016k free, 374092k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +29857 enterpri 20 0 195m 4820 3324 S 19.9 0.5 4:25.02 edb-postgres +28915 enterpri 20 0 195m 5900 4212 R 19.6 0.6 9:07.50 edb-postgres +29023 enterpri 20 0 195m 4744 3248 R 16.3 0.5 4:01.73 edb-postgres +11019 enterpri 20 0 195m 4120 2764 R 15.3 0.4 0:04.92 edb-postgres + 2907 user 20 0 98.7m 12m 9112 S 1.3 1.2 0:56.54 vmware-user-lo + 3040 user 20 0 278m 22m 14m S 1.3 2.2 4:38.73 knotify4 +``` + +The two resource groups in use have CPU usage limits of 40% and 30%. The sum of the `%CPU` column for the first two `edb-postgres` processes is 39.5 (approximately 40%, which is the limit for `resgrp_b`). The sum of the `%CPU` column for the third and fourth `edb-postgres` processes is 31.6 (approximately 30%, which is the limit for `resgrp_c`). + +The sum of the CPU usage limits of the two resource groups to which these processes belong is 70%. The following output shows that the sum of the four processes borders around 70%: + +```shell +$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM ++= $9} END { print SUM / 2 }'; done +61.8 +76.4 +72.6 +69.55 +64.55 +79.95 +68.55 +71.25 +74.85 +62 +74.85 +76.9 +72.4 +65.9 +74.9 +68.25 +``` + +By contrast, if three sessions are processing. where two sessions remain in `resgrp_b` but the third session doesn't belong to any resource group, the `top` command shows the following output: + +```shell +$ top +top - 17:24:55 up 5:03, 7 users, load average: 1.00, 0.41, 0.38 +Tasks: 199 total, 3 running, 196 sleeping, 0 stopped, 0 zombie +Cpu(s): 99.7%us, 0.3%sy, 0.0%ni, 0.0%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0 +Mem: 1025624k total, 797692k used, 227932k free, 24724k buffers +Swap: 103420k total, 13404k used, 90016k free, 374068k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND +29023 enterpri 20 0 195m 4744 3248 R 58.6 0.5 2:53.75 edb-postgres +28915 enterpri 20 0 195m 5900 4212 S 18.9 0.6 7:58.45 edb-postgres +29857 enterpri 20 0 195m 4820 3324 S 18.9 0.5 3:14.85 edb-postgres + 1033 root 20 0 174m 81m 2960 S 1.7 8.2 4:26.50 Xorg + 3040 user 20 0 278m 22m 14m S 1.0 2.2 4:21.20 knotify4 +``` + +The second and third `edb-postgres` processes belonging to the resource group where the CPU usage is limited to 40% have a total CPU usage of 37.8. However, the first `edb-postgres` process has a 58.6% CPU usage, as it isn't within a resource group. It basically uses the remaining available CPU resources on the system. + +Likewise, the following output shows the sum of all three sessions is around 95%, since one of the sessions has no set limit on its CPU usage: + +```shell +$ while [[ 1 -eq 1 ]]; do top -d0.5 -b -n2 | grep edb-postgres | awk '{ SUM ++= $9} END { print SUM / 2 }'; done +96 +90.35 +92.55 +96.4 +94.1 +90.7 +95.7 +95.45 +93.65 +87.95 +96.75 +94.25 +95.45 +97.35 +92.9 +96.05 +96.25 +94.95 + . + . + . +``` diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx new file mode 100644 index 00000000000..e8f967e11ec --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx @@ -0,0 +1,278 @@ +--- +title: "Creating resource groups" +description: "How to use EDB Resource Manager to create and manage resource groups" +--- + +Use these data definition language commands to create and manage resource groups. + +## CREATE RESOURCE GROUP + +Use the `CREATE RESOURCE GROUP` command to create a new resource group. + +```sql +CREATE RESOURCE GROUP ; +``` + +### Description + +The `CREATE RESOURCE GROUP` command creates a resource group with the specified name. You can then define resource limits on the group with the `ALTER RESOURCE GROUP` command. The resource group is accessible from all databases in the EDB Postgres Advanced Server instance. + +To use the `CREATE RESOURCE GROUP` command, you must have superuser privileges. + +### Parameters + +`group_name` + +The name of the resource group. + +### Example + +This example creates three resource groups named `resgrp_a`, `resgrp_b`, and `resgrp_c`: + +```sql +edb=# CREATE RESOURCE GROUP resgrp_a; +CREATE RESOURCE GROUP +edb=# CREATE RESOURCE GROUP resgrp_b; +CREATE RESOURCE GROUP +edb=# CREATE RESOURCE GROUP resgrp_c; +CREATE RESOURCE GROUP +``` + +This query shows the entries for the resource groups in the `edb_resource_group` catalog: + +```sql +edb=# SELECT * FROM edb_resource_group; +__OUTPUT__ + rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit +-----------+------------------+-------------------- + resgrp_a | 0 | 0 + resgrp_b | 0 | 0 + resgrp_c | 0 | 0 +(3 rows) +``` + +## ALTER RESOURCE GROUP + +Use the `ALTER RESOURCE GROUP` command to change the attributes of an existing resource group. The command syntax comes in three forms. + +This form renames the resource group: + +```sql +ALTER RESOURCE GROUP RENAME TO ; +``` + +This form assigns a resource type to the resource group: + +```sql +ALTER RESOURCE GROUP SET + { TO | = } { | DEFAULT }; +``` + +This form resets the assignment of a resource type to its default in the group: + +```sql +ALTER RESOURCE GROUP RESET ; +``` + +### Description + +The `ALTER RESOURCE GROUP` command changes certain attributes of an existing resource group. + +The form with the `RENAME TO` clause assigns a new name to an existing resource group. + +The form with the `SET resource_type TO` clause assigns the specified literal value to a resource type. Or, when you specify `DEFAULT`, it resets the resource type. Resetting a resource type means that the resource group has no defined limit on that resource type. + +The form with the `RESET resource_type` clause resets the resource type for the group. + +To use the `ALTER RESOURCE GROUP` command, you must have superuser privileges. + +### Parameters + +`group_name` + +The name of the resource group to alter. + +`new_name` + +The new name to assign to the resource group. + +`resource_type` + +Specifies the type of resource to which to set a usage value. + +`value | DEFAULT` + +When `value` is specified, the literal value to assign to `resource_type`. Specify `DEFAULT` to reset the assignment of `resource_type` for the resource group. + +### Example + +These examples show the use of the `ALTER RESOURCE GROUP` command: + +```sql +edb=# ALTER RESOURCE GROUP resgrp_a RENAME TO newgrp; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_b SET cpu_rate_limit = .5; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_b SET dirty_rate_limit = 6144; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_c RESET cpu_rate_limit; +ALTER RESOURCE GROUP +``` + +This query shows the results of the `ALTER RESOURCE GROUP` commands to the entries in the `edb_resource_group` catalog: + +```sql +edb=# SELECT * FROM edb_resource_group; +__OUTPUT__ + rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit +-----------+------------------+-------------------- + newgrp | 0 | 0 + resgrp_b | 0.5 | 6144 + resgrp_c | 0 | 0 +(3 rows) +``` + +## DROP RESOURCE GROUP + +Use the `DROP RESOURCE GROUP` command to remove a resource group. + +```sql +DROP RESOURCE GROUP [ IF EXISTS ] ; +``` + +### Description + +The `DROP RESOURCE GROUP` command removes a resource group with the specified name. + +To use the `DROP RESOURCE GROUP` command, you must have superuser privileges. + +### Parameters + +`group_name` + +The name of the resource group to remove. + +`IF EXISTS` + +Don't throw an error if the resource group doesn't exist. Instead, issue a notice. + +### Example + +This example removes the resource group `newgrp`: + +```sql +edb=# DROP RESOURCE GROUP newgrp; +DROP RESOURCE GROUP +``` + +## Assigning a process to a resource group + +Use the `SET edb_resource_group TO group_name` command to assign the current process to a specified resource group: + +```sql +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) +``` + +The resource type settings of the group take effect on the current process immediately. If you use the command to change the resource group assigned to the current process, the resource type settings of the newly assigned group take effect immediately. + +You can include processes in a resource group by default by assigning a default resource group to roles, databases, or an entire database server instance. + +You can assign a default resource group to a role using the `ALTER ROLE ... SET` command. For more information about the `ALTER ROLE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-alterrole.html). + +You can assign a default resource group to a database by using the `ALTER DATABASE ... SET` command. For more information about the `ALTER DATABASE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-alterdatabase.html). + +You can assign the entire database server instance a default resource group by setting the `edb_resource_group` configuration parameter in the `postgresql.conf` file: + +```ini +# - EDB Resource Manager - +#edb_max_resource_groups = 16 # 0-65536 (change requires restart) +edb_resource_group = 'resgrp_b' +``` + +If you change `edb_resource_group` in the `postgresql.conf` file, reload the configuration file to make it take effect on the database server instance. + +## Removing a process from a resource group + +Set `edb_resource_group` to `DEFAULT` or use `RESET edb_resource_group` to remove the current process from a resource group: + +```sql +edb=# SET edb_resource_group TO DEFAULT; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- +(1 row) +``` + +To remove a default resource group from a role, use the `ALTER ROLE ... RESET` form of the `ALTER ROLE` command. + +To remove a default resource group from a database, use the `ALTER DATABASE ... RESET` form of the `ALTER DATABASE` command. + +To remove a default resource group from the database server instance, set the `edb_resource_group` configuration parameter to an empty string in the `postgresql.conf` file. Then, reload the configuration file. + +## Monitoring processes in resource groups + +After you create resource groups, you can get the number of processes actively using these resource groups from the view `edb_all_resource_groups`. + +The following are the columns in `edb_all_resource_groups`: + +- **group_name.** Name of the resource group. +- **active_processes.** Number of active processes in the resource group. +- **cpu_rate_limit.** The value of the CPU rate limit resource type assigned to the resource group. +- **per_process_cpu_rate_limit.** The CPU rate limit that applies to an individual active process in the resource group. +- **dirty_rate_limit.** The value of the dirty rate limit resource type assigned to the resource group. +- **per_process_dirty_rate_limit.** The dirty rate limit that applies to an individual active process in the resource group. + +!!! Note + Columns `per_process_cpu_rate_limit` and `per_process_dirty_rate_limit` don't show the actual resource consumption used by the processes. They indicate how `EDB Resource Manager` sets the resource limit for an individual process based on the number of active processes in the resource group. + +This example shows `edb_all_resource_groups` when resource group `resgrp_a` contains no active processes, resource group `resgrp_b` contains two active processes, and resource group `resgrp_c` contains one active process: + +```sql +edb=# SELECT * FROM edb_all_resource_groups ORDER BY group_name; +__OUTPUT__ +-[ RECORD 1 ]-----------------+------------------ + group_name | resgrp_a + active_processes | 0 + cpu_rate_limit | 0.5 + per_process_cpu_rate_limit | + dirty_rate_limit | 12288 + per_process_dirty_rate_limit | +-[ RECORD 2 ]-----------------+------------------ + group_name | resgrp_b + active_processes | 2 + cpu_rate_limit | 0.4 + per_process_cpu_rate_limit | 0.195694289022895 + dirty_rate_limit | 6144 + per_process_dirty_rate_limit | 3785.92924684337 +-[ RECORD 3 ]-----------------+------------------ + group_name | resgrp_c + active_processes | 1 + cpu_rate_limit | 0.3 + per_process_cpu_rate_limit | 0.292342129631091 + dirty_rate_limit | 3072 + per_process_dirty_rate_limit | 3072 +``` + +The CPU rate limit and dirty rate limit settings that are assigned to these resource groups are: + +```sql +edb=# SELECT * FROM edb_resource_group; +__OUTPUT__ + rgrpname | rgrpcpuratelimit | rgrpdirtyratelimit +-----------+------------------+-------------------- + resgrp_a | 0.5 | 12288 + resgrp_b | 0.4 | 6144 + resgrp_c | 0.3 | 3072 +(3 rows) +``` + +In the `edb_all_resource_groups` view, the `per_process_cpu_rate_limit` and `per_process_dirty_rate_limit` values are roughly the corresponding CPU rate limit and dirty rate limit divided by the number of active processes. \ No newline at end of file diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/dirty_buffer_throttling.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/dirty_buffer_throttling.mdx new file mode 100644 index 00000000000..697969f9b56 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/dirty_buffer_throttling.mdx @@ -0,0 +1,394 @@ +--- +title: "Dirty buffer throttling" +description: "How to use EDB Resource Manager to control writing to shared buffers" +--- + +To control writing to shared buffers, set the `dirty_rate_limit` resource type parameter. + +Set the `dirty_rate_limit` parameter to the number of kilobytes per second for the combined rate at which all the processes in the group write to, or “dirty”, the shared buffers. An example setting is 3072 kilobytes per seconds. + +The valid range of the `dirty_rate_limit` parameter is 0 to 1.67772e+07. A setting of 0 means no dirty rate limit was set for the resource group. + +EDB Resource Manager uses *dirty buffer throttling* to keep the aggregate shared buffer writing rate of all processes in the group near the limit specified by the `dirty_rate_limit` parameter. A process in the group might be interrupted and put into sleep mode for a short time to maintain the defined limit. When and how such interruptions occur is defined by a proprietary algorithm used by EDB Resource Manager. + +## Setting the dirty rate limit for a resource group + +Use the `ALTER RESOURCE GROUP` command with the `SET dirty_rate_limit` clause to set the dirty rate limit for a resource group. + +In this example, the dirty rate limit is set to 12288 kilobytes per second for `resgrp_a`, 6144 kilobytes per second for `resgrp_b`, and 3072 kilobytes per second for `resgrp_c`. This means that the combined writing rate to the shared buffer of all processes assigned to `resgrp_a` is maintained at approximately 12288 kilobytes per second. Similarly, for all processes in `resgrp_b`, the combined writing rate to the shared buffer is kept to approximately 6144 kilobytes per second, and so on. + +```sql +edb=# ALTER RESOURCE GROUP resgrp_a SET dirty_rate_limit TO 12288; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_b SET dirty_rate_limit TO 6144; +ALTER RESOURCE GROUP +edb=# ALTER RESOURCE GROUP resgrp_c SET dirty_rate_limit TO 3072; +ALTER RESOURCE GROUP +``` + +This query shows the settings of `dirty_rate_limit` in the catalog; + +```sql +edb=# SELECT rgrpname, rgrpdirtyratelimit FROM edb_resource_group; +__OUTPUT__ + rgrpname | rgrpdirtyratelimit +-----------+-------------------- + resgrp_a | 12288 + resgrp_b | 6144 + resgrp_c | 3072 +(3 rows) +``` + +Changing the `dirty_rate_limit` of a resource group affects new processes that are assigned to the group. Any currently running processes that are members of the group are also immediately affected by the change. That is, if the `dirty_rate_limit` is changed from 12288 to 3072, currently running processes in the group are throttled downward so that the aggregate group dirty rate is near 3072 kilobytes per second instead of 12288 kilobytes per second. + +To show the effect of setting the dirty rate limit for resource groups, the examples use the following table for intensive I/O operations: + +```sql +CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +``` + +The `FILLFACTOR = 10` clause results in `INSERT` commands packing rows up to only 10% per page. The result is a larger sampling of dirty shared blocks for the purpose of these examples. + +The `pg_stat_statements` module is used to display the number of shared buffer blocks that are dirtied by a SQL command and the amount of time the command took to execute. This information is used to calculate the actual kilobytes per second writing rate for the SQL command and thus compare it to the dirty rate limit set for a resource group. + +To use the `pg_stat_statements` module: + +1. In the `postgresql.conf` file, add `$libdir/pg_stat_statements` to the `shared_preload_libraries` configuration parameter: + + ```ini + shared_preload_libraries = '$libdir/dbms_pipe,$libdir/edb_gen,$libdir/pg_stat_statements' + ``` + +2. Restart the database server. + +3. Use the `CREATE EXTENSION` command to finish creating the `pg_stat_statements` module: + + ```sql + edb=# CREATE EXTENSION pg_stat_statements SCHEMA public; + CREATE EXTENSION + ``` + + The `pg_stat_statements_reset()` function clears out the `pg_stat_statements` view for clarity of each example. + +The resource groups with the dirty rate limit settings shown in the previous query are used in these examples. + + + +## Example: Single process in a single group + +This sequence of commands creates table `t1`. The current process is set to use resource group `resgrp_b`. The `pg_stat_statements` view is cleared out by running the `pg_stat_statements_reset()` function. + +The `INSERT` command then generates a series of integers from 1 to 10,000 to populate the table and dirty approximately 10,000 blocks: + +```sql +edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) +``` +```sql +edb=# SELECT pg_stat_statements_reset(); +__OUTPUT__ + pg_stat_statements_reset +-------------------------- + +(1 row) + +edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +The following shows the results from the `INSERT` command: + +```sql +edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM +pg_stat_statements; +__OUTPUT__ +-[ RECORD 1 ]--------+-------------------------------------------------- + query | INSERT INTO t1 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 13496.184 + shared_blks_dirtied | 10003 +``` + +The actual dirty rate is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 13496.184 ms, which yields *0.74117247 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *741.17247 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *6072 kilobytes per second*. + +The actual dirty rate of 6072 kilobytes per second is close to the dirty rate limit for the resource group, which is 6144 kilobytes per second. + +By contrast, if you repeat the steps without the process belonging to any resource group, the dirty buffer rate is much higher: + +```sql +edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + +(1 row) +``` +```sql +edb=# SELECT pg_stat_statements_reset(); +__OUTPUT__ + pg_stat_statements_reset +-------------------------- + +(1 row) + +edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +The following shows the results from the `INSERT` command without the use of a resource group: + +```sql +edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM +pg_stat_statements; +__OUTPUT__ +-[ RECORD 1 ]--------+-------------------------------------------------- + query | INSERT INTO t1 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 2432.165 + shared_blks_dirtied | 10003 +``` + +The total time was only 2432.165 milliseconds, compared to 13496.184 milliseconds when using a resource group with a dirty rate limit set to 6144 kilobytes per second. + +The actual dirty rate without the use of a resource group is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 2432.165 ms, which yields *4.112797 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *4112.797 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *33692 kilobytes per second*. + +The actual dirty rate of 33692 kilobytes per second is much higher than when the resource group with a dirty rate limit of 6144 kilobytes per second was used. + + + +## Example: Multiple processes in a single group + +As stated previously, the dirty rate limit applies to the aggregate of all processes in the resource group. This concept is illustrated in the following example. + +For this example. the inserts are performed simultaneously on two different tables in two separate `psql` sessions, each of which was added to resource group `resgrp_b` that has a `dirty_rate_limit` set to 6144 kilobytes per second. + +### Session 1 + +```sql +edb=# CREATE TABLE t1 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) + +edb=# INSERT INTO t1 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +### Session 2 + +```sql +edb=# CREATE TABLE t2 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SET edb_resource_group TO resgrp_b; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_b +(1 row) +``` +```sql +edb=# SELECT pg_stat_statements_reset(); +__OUTPUT__ + pg_stat_statements_reset +-------------------------- +(1 row) + +edb=# INSERT INTO t2 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +!!! Note + The `INSERT` commands in session 1 and session 2 started after the `SELECT pg_stat_statements_reset()` command in session 2 ran. + +The following shows the results from the `INSERT` commands in the two sessions. `RECORD 3` shows the results from session 1. `RECORD 2` shows the results from session 2. + +```sql +edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM +pg_stat_statements; +__OUTPUT__ +-[ RECORD 1 ]--------+-------------------------------------------------- + query | SELECT pg_stat_statements_reset(); + rows | 1 + total_time | 0.43 + shared_blks_dirtied | 0 +-[ RECORD 2 ]--------+-------------------------------------------------- + query | INSERT INTO t2 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 30591.551 + shared_blks_dirtied | 10003 +-[ RECORD 3 ]--------+-------------------------------------------------- + query | INSERT INTO t1 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 33215.334 + shared_blks_dirtied | 10003 +``` + +The total time was 33215.334 milliseconds for session 1 and 30591.551 milliseconds for session 2. When only one session was active in the same resource group, as shown in the first example, the time was 13496.184 milliseconds. Thus, more active processes in the resource group result in a slower dirty rate for each active process in the group. The following calculations show this. + +The actual dirty rate for session 1 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 33215.334 ms, which yields *0.30115609 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *301.15609 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2467 kilobytes per second*. + +The actual dirty rate for session 2 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 30591.551 ms, which yields *0.32698571 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *326.98571 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2679 kilobytes per second*. + +The combined dirty rate from session 1 (2467 kilobytes per second) and from session 2 (2679 kilobytes per second) yields 5146 kilobytes per second, which is below the set dirty rate limit of the resource group (6144 kilobytes per seconds). + + + +## Example: Multiple processes in multiple groups + +In this example, two additional `psql` sessions are used along with the previous two sessions. The third and fourth sessions perform the same `INSERT` command in resource group `resgrp_c` with a `dirty_rate_limit` of 3072 kilobytes per second. + +Sessions 1 and 2 are repeated as shown in the prior example using resource group `resgrp_b` with a `dirty_rate_limit` of 6144 kilobytes per second. + +### Session 3 + +```sql +edb=# CREATE TABLE t3 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SET edb_resource_group TO resgrp_c; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + + +resgrp_c +(1 row) + +edb=# INSERT INTO t3 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +### Session 4 + +```sql +edb=# CREATE TABLE t4 (c1 INTEGER, c2 CHARACTER(500)) WITH (FILLFACTOR = 10); +CREATE TABLE +edb=# SET edb_resource_group TO resgrp_c; +SET +edb=# SHOW edb_resource_group; +__OUTPUT__ + edb_resource_group +-------------------- + resgrp_c +(1 row) +``` +```sql +edb=# SELECT pg_stat_statements_reset(); +__OUTPUT__ + pg_stat_statements_reset +-------------------------- + +(1 row) + +edb=# INSERT INTO t4 VALUES (generate_series (1,10000), 'aaa'); +INSERT 0 10000 +``` + +!!! Note + The `INSERT` commands in all four sessions started after the `SELECT pg_stat_statements_reset()` command in session 4 ran. + +The following shows the results from the `INSERT` commands in the four sessions: + +- `RECORD 3` shows the results from session 1. `RECORD 2` shows the results from session 2. + +- `RECORD 4` shows the results from session 3. `RECORD 5` shows the results from session 4. + +```sql +edb=# SELECT query, rows, total_time, shared_blks_dirtied FROM +pg_stat_statements; +__OUTPUT__ +-[ RECORD 1 ]--------+-------------------------------------------------- + query | SELECT pg_stat_statements_reset(); + rows | 1 + total_time | 0.467 + shared_blks_dirtied | 0 +-[ RECORD 2 ]--------+-------------------------------------------------- + query | INSERT INTO t2 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 31343.458 + shared_blks_dirtied | 10003 +-[ RECORD 3 ]--------+-------------------------------------------------- + query | INSERT INTO t1 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 28407.435 + shared_blks_dirtied | 10003 +-[ RECORD 4 ]--------+-------------------------------------------------- + query | INSERT INTO t3 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 52727.846 + shared_blks_dirtied | 10003 +-[ RECORD 5 ]--------+-------------------------------------------------- + query | INSERT INTO t4 VALUES (generate_series (?,?), ?); + rows | 10000 + total_time | 56063.697 + shared_blks_dirtied | 10003 +``` + +The times of session 1 (28407.435) and session 2 (31343.458) are close to each other, as they are both in the same resource group with `dirty_rate_limit` set to 6144. These times differe from the times of session 3 (52727.846) and session 4 (56063.697), which are in the resource group with `dirty_rate_limit` set to 3072. The latter group has a slower dirty rate limit, so the expected processing time is longer, as is the case for sessions 3 and 4. + +The actual dirty rate for session 1 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 28407.435 ms, which yields *0.35212612 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *352.12612 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2885 kilobytes per second*. + +The actual dirty rate for session 2 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 31343.458 ms, which yields *0.31914156 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *319.14156 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *2614 kilobytes per second*. + +The combined dirty rate from session 1 (2885 kilobytes per second) and from session 2 (2614 kilobytes per second) yields 5499 kilobytes per second, which is near the set dirty rate limit of the resource group (6144 kilobytes per seconds). + +The actual dirty rate for session 3 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 52727.846 ms, which yields *0.18971001 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *189.71001 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *1554 kilobytes per second*. + +The actual dirty rate for session 4 is calculated as follows: + +- The number of blocks dirtied per millisecond (ms) is 10003 blocks / 56063.697 ms, which yields *0.17842205 blocks per millisecond*. +- Multiply the result by 1000 to give the number of shared blocks dirtied per second (1 second = 1000 ms), which yields *178.42205 blocks per second*. +- Multiply the result by 8.192 to give the number of kilobytes dirtied per second (1 block = 8.192 kilobytes), which yields approximately *1462 kilobytes per second*. + +The combined dirty rate from session 3 (1554 kilobytes per second) and from session 4 (1462 kilobytes per second) yields 3016 kilobytes per second, which is near the set dirty rate limit of the resource group (3072 kilobytes per seconds). + +This example shows how EDB Resource Manager keeps the aggregate dirty rate of the active processes in its groups close to the dirty rate limit set for each group. + + diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx new file mode 100644 index 00000000000..d8d67277cf3 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx @@ -0,0 +1,19 @@ +--- +title: "Resource Manager key concepts" +description: "Describes key points about using EDB Resource Manager" +--- + +You use EDB Resource Manager to control the use of operating system resources used by EDB Postgres Advanced Server processes. + +The following are some key points about using EDB Resource Manager: + +- The basic component of EDB Resource Manager is a *resource group*. A resource group is a named, global group. It's available to all databases in an EDB Postgres Advanced Server instance, and you can define various resource usage limits on it. EDB Postgres Advanced Server processes that are assigned as members of a given resource group are then controlled by EDB Resource Manager. This configuration keeps the aggregate resource use of all processes in the group near the limits defined on the group. +- Data definition language commands are used to create, alter, and drop resource groups. Only a database user with superuser privileges can use these commands. +- *Resource type parameters* define the desired aggregate consumption level of all processes belonging to a resource group. You use different resource type parameters for the different types of system resources currently supported by EDB Resource Manager. +- You can create multiple resource groups, each with different settings for its resource type parameters, which defines different consumption levels for each resource group. +- EDB Resource Manager throttles processes in a resource group to keep resource consumption near the limits defined by the resource type parameters. If multiple resource type parameters have defined settings in a resource group, the actual resource consumption might be significantly lower for certain resource types than their defined resource type parameter settings. This lower consumption happens because EDB Resource Manager throttles processes, attempting to keep all resources with defined resource type settings within their defined limits. +- The definitions of available resource groups and their resource type settings are stored in a shared global system catalog. Thus, all databases in a given EDB Postgres Advanced Server instance can use resource groups. +- The `edb_max_resource_groups` configuration parameter sets the maximum number of resource groups that can be active at the same time as running processes. The default setting is 16 resource groups. Changes to this parameter take effect when you restart the database server. +- Use the `SET edb_resource_group TO group_name` command to assign the current process to a specified resource group. Use the `RESET edb_resource_group` command or `SET edb_resource_group` to `DEFAULT` to remove the current process from a resource group. +- You can assign a default resource group to a role using the `ALTER ROLE ... SET` command or to a database using the `ALTER DATABASE ... SET` command. You can assign the entire database server instance a default resource group by setting the parameter in the `postgresql.conf` file. +- To include resource groups in a backup file of the database server instance, use the `pg_dumpall` backup utility with default settings. That is, don't specify any of the `--globals-only`, `--roles-only`, or `--tablespaces-only` options. diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_system_catalogs.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_system_catalogs.mdx new file mode 100644 index 00000000000..fbb3e32661a --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_system_catalogs.mdx @@ -0,0 +1,31 @@ +--- +title: "System catalogs" +description: "Describes the system catalogs that store the resource group information used by EDB Resource Manager" +--- + +System catalogs store the resource group information used by EDB Resource Manager. + +## edb_all_resource_groups + +The following table lists the information available in the `edb_all_resource_groups` catalog. + +| Column | Type | Description | +| ------------------------------ | --------- | ---------------------------------------------------------------------------- | +| `group_name` | `name` | The name of the resource group. | +| `active_processes` | `integer` | Number of currently active processes in the resource group. | +| `cpu_rate_limit` | `float8` | Maximum CPU rate limit for the resource group. `0` means no limit. | +| `per_process_cpu_rate_limit` | `float8` | Maximum CPU rate limit per currently active process in the resource group. | +| `dirty_rate_limit` | `float8` | Maximum dirty rate limit for a resource group. `0` means no limit. | +| `per_process_dirty_rate_limit` | `float8` | Maximum dirty rate limit per currently active process in the resource group. | + + + +## edb_resource_group + +The following table lists the information available in the `edb_resource_group` catalog. + +| Column | Type | Description | +| -------------------- | -------- | ------------------------------------------------------------------ | +| `rgrpname` | `name` | The name of the resource group. | +| `rgrpcpuratelimit` | `float8` | Maximum CPU rate limit for a resource group. `0` means no limit. | +| `rgrpdirtyratelimit` | `float8` | Maximum dirty rate limit for a resource group. `0` means no limit. | diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/index.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/index.mdx new file mode 100644 index 00000000000..41fb57fbdc7 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/index.mdx @@ -0,0 +1,34 @@ +--- +title: "Throttling CPU and I/O at the process level" +navTitle: "EDB Resource Manager" +indexCards: simple +description: "How to use EDB Resource Manager to control the use of operating system resources used by EPAS processes" +navigation: +- edb_resource_manager_key_concepts +- creating_resource_groups +- cpu_usage_throttling +- dirty_buffer_throttling +- edb_resource_manager_system_catalogs +legacyRedirectsGenerated: + # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.37.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.35.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.34.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.33.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.6/EDB_Postgres_Advanced_Server_Guide.1.36.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.083.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.082.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.079.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.081.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.080.html" +redirects: + - ../../epas_guide/10_edb_resource_manager #generated for docs/epas/reorg-role-use-case-mode +--- + + + +EDB Resource Manager is an EDB Postgres Advanced Server feature that lets you control the use of operating system resources used by EDB Postgres Advanced Server processes. + +This capability allows you to protect the system from processes that might uncontrollably overuse and monopolize certain system resources. The following topics discuss the capabilities of the EDB Resource Manager. + + From 10b330fad32a5b57c4c18bffd42baca1fa5c01c0 Mon Sep 17 00:00:00 2001 From: francoughlin Date: Mon, 10 Jul 2023 11:04:22 -0400 Subject: [PATCH 02/13] EPAS reorg: Phase 2 upgrading restructure Added new How pg_upgrade works topic, consolidated limitations into upgrade considerations topic, added descriptions to all topics, and made an editing pass through the branch --- .../docs/epas/15/upgrading/03_limitations.mdx | 20 +++++++++-- .../01_performing_an_upgrade/index.mdx | 1 + .../02_invoking_pg_upgrade/index.mdx | 1 + .../03_upgrading_to_advanced_server.mdx | 1 + .../04_upgrading_a_pgAgent_installation.mdx | 1 + .../05_pg_upgrade_troubleshooting.mdx | 3 +- .../06_reverting_to_the_old_cluster.mdx | 1 + .../how_pg_upgrade_works.mdx | 30 ++++++++++++++++ .../index.mdx | 34 +++++++------------ 9 files changed, 68 insertions(+), 24 deletions(-) create mode 100644 product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/how_pg_upgrade_works.mdx diff --git a/product_docs/docs/epas/15/upgrading/03_limitations.mdx b/product_docs/docs/epas/15/upgrading/03_limitations.mdx index a7683677573..e3e42f4d636 100644 --- a/product_docs/docs/epas/15/upgrading/03_limitations.mdx +++ b/product_docs/docs/epas/15/upgrading/03_limitations.mdx @@ -5,11 +5,27 @@ redirects: - /epas/latest/epas_upgrade_guide/03_limitations/ --- -Consider the following when upgrading EDB Postgres Advanced Server: +The `pg_upgrade` executable is distributed with EDB Postgres Advanced Server and is installed as part of the Database Server component. You don't need to further install or configure it. The `pg_upgrade` utility performs an in-place transfer of existing data between EDB Postgres Advanced Server and any later version. + +Several factors determine if an in-place upgrade is practical: + +- The on-disk representation of user-defined tables must not change between the original version and the upgraded version. +- The on-disk representation of data types must not change between the original version and the upgraded version. +- To upgrade between major versions of EDB Postgres Advanced Server with `pg_upgrade`, both versions must share a common binary representation for each data type. Therefore, you can't use `pg_upgrade` to migrate from a 32-bit to a 64-bit Linux platform. + +Before performing a version upgrade, `pg_upgrade` verifies that the old cluster and the new cluster are compatible. +If the upgrade involves a change in the on-disk representation of database objects or data, or if it involves a change in the binary representation of data types, `pg_upgrade` can't perform the upgrade. To upgrade, you have to `pg_dump` the old data and then import that data to the new cluster, as described below. + +1. Export all of your data using `pg_dump`. +1. Install the new release. +1. Run `initdb` to create a new cluster. +1. Import your old data. + +Consider the following when upgrading EDB Postgres Advanced Server: - The `pg_upgrade` utility can't upgrade a partitioned table if a foreign key refers to the partitioned table. -- If you're upgrading from the version 9.4 server or a lower version of EDB Postgres Advanced Server, and you use partitioned tables that include a `SUBPARTITION BY` clause, you must use `pg_dump` and `pg_restore` to upgrade an existing EDB Postgres Advanced Server installation to a later version of EDB Postgres Advanced Server. To upgrade, you must: +- If you're upgrading from the **version 9.4 server or a lower version** of EDB Postgres Advanced Server, and you use partitioned tables that include a `SUBPARTITION BY` clause, you must use `pg_dump` and `pg_restore` to upgrade an existing EDB Postgres Advanced Server installation to a later version of EDB Postgres Advanced Server. To upgrade, you must: 1. Use `pg_dump` to preserve the content of the subpartitioned table. 2. Drop the table from the EDB Postgres Advanced Server 9.4 database or a lower version of EDB Postgres Advanced Server database. 3. Use `pg_upgrade` to upgrade the rest of the EDB Postgres Advanced Server database to a more recent version. diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/01_performing_an_upgrade/index.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/01_performing_an_upgrade/index.mdx index ef480377cea..345b2ab8efd 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/01_performing_an_upgrade/index.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/01_performing_an_upgrade/index.mdx @@ -1,5 +1,6 @@ --- title: "Performing an upgrade" +description: "Describes how to upgrade an earlier version of EDB Postgres Advanced Server to the current version" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/01_performing_an_upgrade/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/02_invoking_pg_upgrade/index.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/02_invoking_pg_upgrade/index.mdx index 39953d6b9e6..25902c95112 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/02_invoking_pg_upgrade/index.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/02_invoking_pg_upgrade/index.mdx @@ -1,5 +1,6 @@ --- title: "Invoking pg_upgrade" +description: "Describes how to invoke the pg_upgrade utility" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/02_invoking_pg_upgrade/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/03_upgrading_to_advanced_server.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/03_upgrading_to_advanced_server.mdx index 37320973723..b5f8837ab8e 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/03_upgrading_to_advanced_server.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/03_upgrading_to_advanced_server.mdx @@ -1,5 +1,6 @@ --- title: "Upgrading to EDB Postgres Advanced Server 15" +description: "Describes how to upgrade into a cluster" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/03_upgrading_to_advanced_server/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/04_upgrading_a_pgAgent_installation.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/04_upgrading_a_pgAgent_installation.mdx index d27bce0e21b..04f08483afb 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/04_upgrading_a_pgAgent_installation.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/04_upgrading_a_pgAgent_installation.mdx @@ -1,5 +1,6 @@ --- title: "Upgrading a pgAgent installation" +description: "Describes how to use the script provided with the EDB Postgres Advanced Server installer to update pgAgent" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/04_upgrading_a_pgAgent_installation/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/05_pg_upgrade_troubleshooting.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/05_pg_upgrade_troubleshooting.mdx index dade9bc417e..bc0848f006c 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/05_pg_upgrade_troubleshooting.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/05_pg_upgrade_troubleshooting.mdx @@ -1,5 +1,6 @@ --- -title: "pg_upgrade troubleshooting" +title: "Troubleshooting pg_upgrade" +description: "Provides tips for problems you might encounter during upgrade" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/05_pg_upgrade_troubleshooting/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/06_reverting_to_the_old_cluster.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/06_reverting_to_the_old_cluster.mdx index 10fa13ed52c..9b4fa3ec285 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/06_reverting_to_the_old_cluster.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/06_reverting_to_the_old_cluster.mdx @@ -1,5 +1,6 @@ --- title: "Reverting to the old cluster" +description: "Describes the various method you can use to revert to a previous cluster" redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/06_reverting_to_the_old_cluster/ --- diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/how_pg_upgrade_works.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/how_pg_upgrade_works.mdx new file mode 100644 index 00000000000..c2e74ab7d77 --- /dev/null +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/how_pg_upgrade_works.mdx @@ -0,0 +1,30 @@ +--- +title: "How pg_upgrade works" +description: "Describes how the upgrade utility works" +--- + +The `pg_upgrade` executable is distributed with EDB Postgres Advanced Server and is installed as part of the Database Server component. You don't need to further install or configure it. + +The `pg_upgrade` utility performs an in-place transfer of existing data between EDB Postgres Advanced Server and any later version. + +Before performing a version upgrade, `pg_upgrade` verifies that the old cluster and the new cluster are compatible. + +When `pg_upgrade` starts, it performs a compatibility check to ensure that all required executables are present and contain the expected version numbers. The verification process also checks the old and new `$PGDATA` directories to ensure that the expected files and subdirectories are in place. If the verification process succeeds, `pg_upgrade` starts the old `postmaster` and runs `pg_dumpall --schema-only` to capture the metadata contained in the old cluster. The script produced by `pg_dumpall` is used in later to re-create all user-defined objects in the new cluster. + +The script produced by `pg_dumpall` re-creates only user-defined objects and not system-defined objects. The new cluster already contains the system-defined objects created by the latest version of EDB Postgres Advanced Server. + +After extracting the metadata from the old cluster, `pg_upgrade` performs the bookkeeping tasks required to sync the new cluster with the existing data. + +`pg_upgrade` runs the `pg_dumpall` script against the new cluster to create empty database objects of the same shape and type as those found in the old cluster. Then, `pg_upgrade` links or copies each table and index from the old cluster to the new cluster. + +If you're upgrading to EDB Postgres Advanced Server and installed the `edb_dblink_oci` or `edb_dblink_libpq` extension, drop the extension before performing an upgrade. To drop the extension, connect to the server with the psql or PEM client, and invoke the commands: + +```sql +DROP EXTENSION edb_dblink_oci; +DROP EXTENSION edb_dblink_libpq; +``` + +When you finish upgrading, you can use the `CREATE EXTENSION` command to add the current versions of the extensions to your installation. + +!!! Note + If the upgrade involves a change in the on-disk representation of database objects or data, or if it involves a change in the binary representation of data types, `pg_upgrade` can't perform the upgrade. To upgrade, you have to `pg_dump` the old data and then import that data to the new cluster. diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx index 3f7c0f70c6c..4a1e32679fb 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx @@ -1,37 +1,29 @@ --- title: "Upgrading an installation with pg_upgrade" description: "Describes using pg_upgrade to migrate from an earlier version of EDB Postgres Advanced Server to the newest version" +indexCards: simple +navigation: +- how_pg_upgrade_works +- 01_performing_an_upgrade +- 02_invoking_pg_upgrade +- 03_upgrading_to_advanced_server +- 04_upgrading_a_pgAgent_installation +- 05_pg_upgrade_troubleshooting +- 06_reverting_to_the_old_cluster redirects: - /epas/latest/epas_upgrade_guide/04_upgrading_an_installation_with_pg_upgrade/ --- -While minor upgrades between versions are fairly simple and require only installing new executables, past major version upgrades were both expensive and time consuming. `pg_upgrade` eases migration between any version of EDB Postgres Advanced Server (version 9.0 or later) and any later release of EDB Postgres Advanced Server that's supported on the same platform. - -Without `pg_upgrade`, to migrate from an earlier version of EDB Postgres Advanced Server to the newest version: - -1. Export all of your data using `pg_dump`. -1. Install the new release. -1. Run `initdb` to create a new cluster. -1. Import your old data. - !!! Note - `pg_upgrade` can reduce both the amount of time and the disk space required for many major-version upgrades. + Review the [upgrade considerations](/epas/latest/upgrading/03_limitations/) prior to performing an upgrade with the `pg_upgrade` utility. -The `pg_upgrade` utility performs an in-place transfer of existing data between EDB Postgres Advanced Server and any later version. - -Several factors determine if an in-place upgrade is practical: - -- The on-disk representation of user-defined tables must not change between the original version and the upgraded version. -- The on-disk representation of data types must not change between the original version and the upgraded version. -- To upgrade between major versions of EDB Postgres Advanced Server with `pg_upgrade`, both versions must share a common binary representation for each data type. Therefore, you can't use `pg_upgrade` to migrate from a 32-bit to a 64-bit Linux platform. - -Before performing a version upgrade, `pg_upgrade` verifies that the old cluster and the new cluster are compatible. +While minor upgrades between versions are fairly simple and require only installing new executables, past major version upgrades were both expensive and time consuming. `pg_upgrade` eases migration between any version of EDB Postgres Advanced Server (version 9.0 or later) and any later release of EDB Postgres Advanced Server that's supported on the same platform. -If the upgrade involves a change in the on-disk representation of database objects or data, or if it involves a change in the binary representation of data types, `pg_upgrade` can't perform the upgrade. To upgrade, you have to `pg_dump` the old data and then import that data to the new cluster. +Using `pg_upgrade` can reduce both the amount of time and the disk space required for many major-version upgrades. -The `pg_upgrade` executable is distributed with EDB Postgres Advanced Server and is installed as part of the Database Server component. You don't need to further install or configure it. +The following topics walk you through the process of upgrading with `pg_upgrade`.
From d8d8c4506ed8e86c954b4159683d40f8d5ee2653 Mon Sep 17 00:00:00 2001 From: francoughlin Date: Mon, 10 Jul 2023 16:09:15 -0400 Subject: [PATCH 03/13] EPAS reorg: Phase 2 EDBLoader restructure Added new overview and compatibility topic, broke out all sections into separate topics, added descriptions to all topics --- .../database_administration/02_edb_loader.mdx | 1596 ----------------- .../building_the_control_file.mdx | 411 +++++ .../02_edb_loader/control_file_examples.mdx | 484 +++++ .../02_edb_loader/conventional_path_load.mdx | 102 ++ .../02_edb_loader/data_loading_methods.mdx | 24 + .../02_edb_loader/direct_path_load.mdx | 41 + .../edb_loader_overview_and_restrictions.mdx | 29 + .../02_edb_loader/index.mdx | 58 + .../02_edb_loader/invoking_edb_loader.mdx | 287 +++ .../parallel_direct_path_load.mdx | 147 ++ .../performing_remote_loading.mdx | 35 + .../02_edb_loader/using_edb_loader.mdx | 30 + .../index.mdx | 2 - product_docs/docs/epas/15/upgrading/index.mdx | 2 - 14 files changed, 1648 insertions(+), 1600 deletions(-) delete mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx create mode 100644 product_docs/docs/epas/15/database_administration/02_edb_loader/using_edb_loader.mdx diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader.mdx deleted file mode 100644 index 87ffed2f87c..00000000000 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader.mdx +++ /dev/null @@ -1,1596 +0,0 @@ ---- -title: "Loading bulk data" -navTitle: "EDB*Loader" -description: "How to use EDB*Loader, the high-performance bulk data loader" -legacyRedirectsGenerated: - # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.16.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.15.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.11.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.12.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.08.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.14.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.13.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.10.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.09.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.314.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.315.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.311.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.312.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.313.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.309.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.310.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.308.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.307.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.092.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.093.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.091.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.089.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.087.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.090.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.088.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.086.html" - - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.085.html" -redirects: - - ../../epas_compat_tools_guide/02_edb_loader #generated for docs/epas/reorg-role-use-case-mode ---- - -EDB\*Loader is a high-performance bulk data loader that provides an interface compatible with Oracle databases for EDB Postgres Advanced Server. The EDB\*Loader command line utility loads data from an input source, typically a file, into one or more tables using a subset of the parameters offered by Oracle SQL\*Loader. - -EDB\*Loader features include: - -- Support for the Oracle SQL\*Loader data loading methods (conventional path load, direct path load, and parallel direct path load) -- Syntax for control file directives compatible with Oracle SQL\*Loader -- Input data with delimiter-separated or fixed-width fields -- Bad file for collecting rejected records -- Loading of multiple target tables -- Discard file for collecting records that don't meet the selection criteria of any target table -- Log file for recording the EDB\*Loader session and any error messages -- Data loading from standard input and remote loading, particularly useful for large data sources on remote hosts - -!!! Note - The following are important version compatibility restrictions between the EDB\*Loader client and the database server. - - - When you invoke the EDB\*Loader program (called `edbldr`), you pass in parameters and directive information to the database server. We strongly recommend that you use the version 14 EDB\*Loader client (the edbldr program supplied with EDB Postgres Advanced Server 14) to load data only into version 14 of the database server. In general, use the same version for the EDB\*Loader client and database server. - - - Using EDB\*Loader with connection poolers such as PgPool-II and PgBouncer isn't supported. EDB\*Loader must connect directly to EDB Postgres Advanced Server version 14. Alternatively, the following are some of the commands you can use for loading data through connection poolers: - - ```shell - psql \copy - jdbc copyIn - psycopg2 copy_from - ``` - - - -## Data loading methods - -As with Oracle SQL\*Loader, EDB\*Loader supports three data loading methods: - -- Conventional path load -- Direct path load -- Parallel direct path load - -Conventional path load is the default method used by EDB\*Loader. Use basic insert processing to add rows to the table. - -The advantage of a conventional path load is that table constraints and database objects defined on the table are enforced during a conventional path load. Table constraints and database objects include primary keys, not null constraints, check constraints, unique indexes, foreign key constraints, triggers, and so on. - -One exception is that the EDB Postgres Advanced Server rules defined on the table aren't enforced. EDB\*Loader can load tables on which rules are defined. However, the rules aren't executed. As a consequence, you can't load partitioned tables implemented using rules with EDB\*Loader. - -!!! Note - Create EDB Postgres Advanced Server rules using the `CREATE RULE` command. EDB Postgres Advanced Server rules aren't the same database objects as rules and rule sets used in Oracle. - -EDB\*Loader also supports direct path loads. A direct path load is faster than a conventional path load but requires removing most types of constraints and triggers from the table. For more information, see [Direct path load](#direct-path-load). - -EDB\*Loader supports parallel direct path loads. A parallel direct path load provides even greater performance improvement by permitting multiple EDB\*Loader sessions to run simultaneously to load a single table. For more information, see [Parallel direct path load](#parallel-direct-path-load). - - - -## General usage - -EDB\*Loader can load data files with either delimiter-separated or fixed-width fields in single-byte or multibyte character sets. The delimiter can be a string consisting of one or more single-byte or multibyte characters. Data file encoding and the database encoding can differ. Character set conversion of the data file to the database encoding is supported. - -Each EDB\*Loader session runs as a single, independent transaction. If an error occurs during the EDB\*Loader session that aborts the transaction, all changes made during the session are rolled back. - -Generally, formatting errors in the data file don't result in an aborted transaction. Instead, the badly formatted records are written to a text file called the *bad file*. The reason for the error is recorded in the *log file*. - -Records causing database integrity errors result in an aborted transaction and rollback. As with formatting errors, the record causing the error is written to the bad file and the reason is recorded in the log file. - -!!! Note - EDB\*Loader differs from Oracle SQL\*Loader in that a database integrity error results in a rollback in EDB\*Loader. In Oracle SQL\*Loader, only the record causing the error is rejected. Records that were previously inserted into the table are retained, and loading continues after the rejected record. - -The following are examples of types of formatting errors that don't abort the transaction: - -- Attempt to load non-numeric value into a numeric column -- Numeric value too large for a numeric column -- Character value too long for the maximum length of a character column -- Attempt to load improperly formatted date value into a date column - -The following are examples of types of database errors that abort the transaction and result in the rollback of all changes made in the EDB\*Loader session: - -- Violating a unique constraint such as a primary key or unique index -- Violating a referential integrity constraint -- Violating a check constraint -- Error thrown by a trigger fired as a result of inserting rows - - - -## Building the EDB\*Loader control file - -When you invoke EDB\*Loader, the list of arguments provided must include the name of a control file. The control file includes the instructions that EDB\*Loader uses to load the tables from the input data file. The control file includes information such as: - -- The name of the input data file containing the data to load -- The name of the tables to load from the data file -- Names of the columns in the tables and their corresponding field placement in the data file -- Specification of whether the data file uses a delimiter string to separate the fields or if the fields occupy fixed column positions -- Optional selection criteria to choose the records from the data file to load into a given table -- The name of the file that collects illegally formatted records -- The name of the discard file that collects records that don't meet the selection criteria of any table - -The syntax for the EDB\*Loader control file is: - -```sql -[ OPTIONS ( [, ] ...) ] -LOAD DATA - [ CHARACTERSET ] - [ INFILE '{ | }' ] - [ BADFILE '' ] - [ DISCARDFILE '' ] - [ { DISCARDMAX | DISCARDS } ] -[ INSERT | APPEND | REPLACE | TRUNCATE ] -[ PRESERVE BLANKS ] -{ INTO TABLE - [ WHEN [ AND ] ...] - [ FIELDS TERMINATED BY '' - [ OPTIONALLY ENCLOSED BY '' ] ] - [ RECORDS DELIMITED BY '' ] - [ TRAILING NULLCOLS ] - ( [, ] ...) -} ... -``` - -Where `field_def` defines a field in the specified `data_file`. The field describes the location, data format, or value of the data to insert into `column_name` of `target_table`. The syntax of `field_def` is: - -```sql - { - CONSTANT | - FILLER [ POSITION () ] [ ] | - BOUNDFILLER [ POSITION () ] [ ] | - [ POSITION () ] [ ] - [ NULLIF [ AND ] ...] - [ PRESERVE BLANKS ] [ "" ] -} -``` - -Where `fieldtype` is one of: - -```sql -CHAR [()] | DATE [()] | TIMESTAMP [()] [ "" ] | -INTEGER EXTERNAL [()] | -FLOAT EXTERNAL [()] | DECIMAL EXTERNAL [()] | -ZONED EXTERNAL [()] | ZONED [( [,])] -``` - -### Description - -The specification of `data_file`, `bad_file`, and `discard_file` can include the full directory path or a relative directory path to the file name. If the file name is specified alone or with a relative directory path, the file is then assumed to exist, in the case of `data_file`, relative to the current working directory from which you invoke `edbldr. In the case of `bad_file` or `discard_file`, it's created. - -You can include references to environment variables in the EDB\*Loader control file when referring to a directory path or file name. Environment variable references are formatted differently on Windows systems than on Linux systems: - -- On Linux, the format is `$ENV_VARIABLE` or `${ENV_VARIABLE}`. -- On Windows, the format is `%ENV_VARIABLE%`. - -Where `ENV_VARIABLE` is the environment variable that's set to the directory path or file name. - -Set the `EDBLDR_ENV_STYLE` environment variable to interpret environment variable references as Windows-styled references or Linux-styled references regardless of the operating system on which EDB\*Loader resides. You can use this environment variable to create portable control files for EDB\*Loader. - -- On a Windows system, set `EDBLDR_ENV_STYLE` to `linux` or `unix` to recognize Linux-style references in the control file. -- On a Linux system, set `EDBLDR_ENV_STYLE` to `windows` to recognize Windows-style references in the control file. - -The operating system account `enterprisedb` must have read permission on the directory and file specified by `data_file`. - -The operating system account enterprisedb must have write permission on the directories where `bad_file` and `discard_file` are written. - -!!! Note - The file names for `data_file`, `bad_file`, and `discard_file` must have extensions `.dat`, `.bad`, and `.dsc`, respectively. If the provided file name doesn't have an extension, EDB\*Loader assumes the actual file name includes the appropriate extension. - -Suppose an EDB\*Loader session results in data format errors, the `BADFILE` clause isn't specified, and the BAD parameter isn't given on the command line when `edbldr` is invoked. In this case, a bad file is created with the name `control_file_base.bad` in the directory from which `edbldr` is invoked. `control_file_base` is the base name of the control file used in the `edbldr` session. - -If all of the following conditions are true, the discard file isn't created even if the EDB\*Loader session results in discarded records: - -- The `DISCARDFILE` clause for specifying the discard file isn't included in the control file. -- The `DISCARD` parameter for specifying the discard file isn't included on the command line. -- The `DISCARDMAX` clause for specifying the maximum number of discarded records isn't included in the control file. -- The `DISCARDS` clause for specifying the maximum number of discarded records isn't included in the control file. -- The `DISCARDMAX` parameter for specifying the maximum number of discarded records isn't included on the command line. - -Suppose you don't specify the `DISCARDFILE` clause and the `DISCARD` parameter for explicitly specifying the discard file name, but you do specify `DISCARDMAX` or `DISCARDS`. In this case, the EDB\*Loader session creates a discard file using the data file name with an extension of `.dsc`. - -!!! Note - The keywords `DISCARD` and `DISCARDS` differ. `DISCARD` is an EDB\*Loader command line parameter used to specify the discard file name (see [General usage](#general-usage)). `DISCARDS` is a clause of the `LOAD DATA` directive that can appear only in the control file. Keywords `DISCARDS` and `DISCARDMAX` provide the same functionality of specifying the maximum number of discarded records allowed before terminating the EDB\*Loader session. Records loaded into the database before terminating the EDB\*Loader session due to exceeding the `DISCARDS` or `DISCARDMAX` settings are kept in the database. They aren't rolled back. - -Specifying one of `INSERT`, `APPEND`, `REPLACE`, or `TRUNCATE` establishes the default action for adding rows to target tables. The default action is `INSERT`. - -If you specify the `FIELDS TERMINATED BY` clause, then you can't specify the `POSITION (start:end)` clause for any `field_def`. Alternatively, if you don't specify the `FIELDS TERMINATED BY` clause, then every `field_def` must contain the `POSITION (start:end)` clause, the `fieldtype(length)` clause, or the `CONSTANT` clause. - -### Parameters - -`OPTIONS param=value` - - Use the `OPTIONS` clause to specify `param=value` pairs that represent an EDB\*Loader directive. If you specify a parameter in the `OPTIONS` clause and on the command line when `edbldr` is invoked, the command line setting takes effect. - - Specify one or more of the following parameter/value pairs: - -- `DIRECT= { FALSE | TRUE }` - - With `DIRECT` set to `TRUE`, EDB\*Loader performs a direct path load instead of a conventional path load. The default value of `DIRECT` is `FALSE`. - - Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or can potentially contain references to rows that were deleted. Direct inserts to load data into replicated tables aren't supported. - - For information on direct path loads see, [Direct path load](#direct-path-load). - -- `ERRORS=error_count` - - `error_count` specifies the number of errors permitted before aborting the EDB\*Loader session. The default is `50`. - -- `FREEZE= { FALSE | TRUE }` - - Set `FREEZE` to `TRUE` to copy the data with the rows `frozen`. A tuple guaranteed to be visible to all current and future transactions is marked as frozen to prevent transaction ID wraparound. For more information about frozen tuples, see the [PostgreSQL core documentation]( } 'val' [ ) ] -``` - - This conditional clause is used for the `WHEN` clause, which is part of the `INTO TABLE target_table` clause. It's also used for the `NULLIF` clause, which is part of the field definition denoted as `field_def` in the syntax diagram. - - `start` and `end` are positive integers specifying the column positions in `data_file` that mark the beginning and end of a field to compare with the constant `val`. The first character in each record begins with a `start` value of `1`. - - `column_name` specifies the name assigned to a field definition of the data file as defined by `field_def` in the syntax diagram. - - Using (`start`:`end`) or `column_name` defines the portion of the record in `data_file` to compare with the value specified by `val` to evaluate as either true or false. - - All characters used in the `field_condition` text (particularly in the `val` string) must be valid in the database encoding. For performing data conversion, EDB\*Loader first converts the characters in `val` string to the database encoding and then to the data file encoding. - - In the `WHEN field_condition [ AND field_condition ]` clause, if all such conditions evaluate to `TRUE` for a given record, then EDB\*Loader attempts to insert that record into `target_table`. If the insert operation fails, the record is written to `bad_file`. - - Suppose, for a given record, that none of the `WHEN` clauses evaluate to `TRUE` for all `INTO TABLE` clauses. The record is written to `discard_file` if a discard file was specified for the EDB\*Loader session. - - See the description of the `NULLIF` clause in this parameters list for the effect of `field_condition` on this clause. - -`termstring` - - String of one or more characters that separates each field in `data_file`. The characters can be single byte or multibyte. However, they must be valid in the database encoding. Two consecutive appearances of `termstring` with no intervening character results in the corresponding column being set to null. - -`enclstring` - - String of one or more characters used to enclose a field value in `data_file`. The characters can be single byte or multibyte, However, they must be valid in the database encoding. Use `enclstring` on fields where `termstring` appears as part of the data. - -`delimstring` - - String of one or more characters that separates each record in `data_file`. The characters can be single byte or multibyte. However, they must be valid in the database encoding. Two consecutive appearances of `delimstring` with no intervening character results in no corresponding row being loaded into the table. You must also terminate the last record (that is, the end of the data file) with the `delimstring` characters. Otherwise, the final record isn't loaded into the table. - - !!! Note - The `RECORDS DELIMITED BY 'delimstring'` clause isn't compatible with Oracle databases. - -`TRAILING NULLCOLS` - - If you specify `TRAILING NULLCOLS`, then the columns in the column list for which there's no data in `data_file` for a given record are set to null when the row is inserted. This option applies only to one or more consecutive columns at the end of the column list. - - If fields are omitted at the end of a record and you don't specify `TRAILING NULLCOLS`, EDB\*Loader assumes the record contains formatting errors and writes it to the bad file. - -`column_name` - - Name of a column in `target_table` into which to insert a field value defined by `field_def`. If the field definition includes the `FILLER` or `BOUNDFILLER` clause, then `column_name` isn't required as the name of a column in the table. It can be any identifier name since the `FILLER` and `BOUNDFILLER` clauses prevent loading the field data into a table column. - -`CONSTANT val` - - Specifies a constant that's type-compatible with the column data type to which it's assigned in a field definition. You can use single or double quotes around `val`. If `val` contains white space, then you must use quotation marks around it. - - The use of the `CONSTANT` clause determines the value to assign to a column in each inserted row. No other clause can appear in the same field definition. - - If you use the `TERMINATED BY` clause to delimit the fields in `data_file`, there must be no delimited field in `data_file` corresponding to any field definition with a `CONSTANT` clause. In other words, EDB\*Loader assumes there's no field in `data_file` for any field definition with a `CONSTANT` clause. - -`FILLER` - - Specifies not to load the data in the field defined by the field definition into the associated column if the identifier of the field definition is an actual column name in the table. In this case, the column is set to null. Use of the `FILLER` or `BOUNDFILLER` clause is the only circumstance in which you don't have to identify the field definition with an actual column name. - - Unlike the `BOUNDFILLER` clause, you can't reference an identifier defined with the `FILLER` clause in a SQL expression. See the discussion of the `expr` parameter. - -`BOUNDFILLER` - - Specifies not to load the data in the field defined by the field definition into the associated column if the identifier of the field definition is an actual column name in the table. In this case, the column is set to null. Use of the `FILLER` or `BOUNDFILLER` clause is the only circumstance in which you don't have to identify the field definition with an actual column name. - - Unlike the `FILLER` clause, a SQL expression can reference an identifier defined with the `BOUNDFILLER` clause. See the discussion of the `expr` parameter. - -`POSITION (start:end)` - - Defines the location of the field in a record in a fixed-width field data file. `start` and `end` are positive integers. The first character in the record has a start value of `1`. - -```sql -CHAR [()] | DATE [()] | TIMESTAMP [()] [ "" ] | -INTEGER EXTERNAL [()] | -FLOAT EXTERNAL [()] | DECIMAL EXTERNAL [()] | -ZONED EXTERNAL [()] | ZONED [([,])] -``` - - Field type that describes the format of the data field in `data_file`. - - !!! Note - Specifying a field type is optional for descriptive purposes and has no effect on whether EDB\*Loader successfully inserts the data in the field into the table column. Successful loading depends on the compatibility of the column data type and the field value. For example, a column with data type `NUMBER(7,2)` successfully accepts a field containing `2600`. However, if the field contains a value such as `26XX`, the insertion fails, and the record is written to `bad_file`. - -`ZONED` data is not human readable. `ZONED` data is stored in an internal format where each digit is encoded in a separate nibble/nybble/4-bit field. In each `ZONED` value, the last byte contains a single digit in the high-order 4 bits and the sign in the low-order 4 bits. - -`length` - - Specifies the length of the value to load into the associated column. - - If you specify the `POSITION` (`start`:`end`) clause with a `fieldtype(length)` clause, then the ending position of the field is overridden by the specified `length` value. That is, the length of the value to load into the column is determined by the `length` value beginning at the `start` position and not by the `end` position of the `POSITION` (`start`:`end`) clause. Thus, the value to load into the column might be shorter than the field defined by `POSITION` (`start`:`end`). Or, it might go beyond the end position, depending on the specified `length` size. - - If you specify the `FIELDS TERMINATED BY 'termstring'` clause as part of the `INTO TABLE` clause, and a field definition contains the `fieldtype(length)` clause, then a record is accepted. However, the specified `length` values must be greater than or equal to the field lengths as determined by the `termstring` characters enclosing all such fields of the record. If the specified `length` value is less than a field length as determined by the enclosing `termstring` characters for any such field, then the record is rejected. - - If you don't specify the `FIELDS TERMINATED BY 'termstring'` clause, and you don't include the `POSITION` (`start`:`end`) clause with a field containing the `fieldtype(length)` clause, then the starting position of this field begins with the next character following the ending position of the preceding field. The ending position of the preceding field is either: - - - The end of its `length` value if the preceding field contains the `fieldtype(length)` clause - - Its `end` parameter if the field contains the `POSITION` (`start`:`end`) clause without the `fieldtype(length)` clause - -`precision` - - Use `precision` to specify the length of the `ZONED` value. - - If the `precision` value specified for `ZONED` conflicts with the length calculated by the server based on information provided with the `POSITION` clause, EDB\*Loader uses the value specified for `precision`. - -`scale` - - Specifies the number of digits to the right of the decimal point in a `ZONED` value. - -`datemask` - - Specifies the ordering and abbreviation of the day, month, and year components of a date field. - - !!! Note - If you specify the `DATE` or `TIMESTAMP` field type with a SQL expression for the column, then you must specify `datemask` after `DATE` or `TIMESTAMP` and before the SQL expression. See the discussion of the `expr` parameter. - - When using the `TIMESTAMP` field datatype, if you specify `time_stamp timestamp "yyyymmddhh24miss"`, the `datemask` is converted to the SQL expression. However, in the case of `time_stamp timestamp "select to_timestamp(:time_stamp, 'yyyymmddhh24miss')"`, EDB\*Loader can't differentiate between datemask and the SQL expression. It treats the third field (SQL expression in the example) as datemask and prepares the SQL expression, which isn't valid. Where: - -- `first field` specifies the column name. -- `second field` specifies the datatype. -- `third field` specifies the datemask. - -If you want to provide an SQL expression, then a workaround is to specify the datemask and SQL expression using the `TO_CHAR` function as: - - ```sql - time_stamp timestamp "yyyymmddhh24miss" "to_char(to_timestamp(:time_stamp, 'yyyymmddhh24miss'), 'yyyymmddhh24miss')" - ``` - -`NULLIF field_condition [ AND field_condition ] ...` - - See the description of `field_condition` in this parameter list for the syntax of `field_condition`. - - If all field conditions evaluate to `TRUE`, then the column identified by `column_name` in the field definition is set to null. If any field condition evaluates to `FALSE`, then the column is set to the appropriate value as normally occurs according to the field definition. - -`PRESERVE BLANKS` - - The `PRESERVE BLANKS` option works only with the `OPTIONALLY ENCLOSED BY` clause and retains leading and trailing whitespaces for both delimited and predetermined size fields. - - In case of `NO PRESERVE BLANKS`, if the fields are delimited, then only leading whitespaces are omitted. If any trailing whitespaces are present, they are left intact. In the case of predetermined-sized fields with `NO PRESERVE BLANKS`, the trailing whitespaces are omitted, and any leading whitespaces are left intact. - - !!! Note - If you don't provide `PRESERVE BLANKS` or `NO PRESERVE BLANKS`, then the behavior defaults to `NO PRESERVE BLANKS`. This option doesn't work for ideographic whitespaces. - -`expr` - - A SQL expression returning a scalar value that's type-compatible with the column data type to which it's assigned in a field definition. Use double quotes around `expr`. `expr` can contain a reference to any column in the field list except for fields with the `FILLER` clause. Prefix the column name using a colon (`:`). - - `expr` can also consist of a SQL `SELECT` statement. If you use a `SELECT` statement: - -- Enclose the `SELECT` statement in parentheses, that is, `(SELECT ...)`. -- The select list must consist of one expression following the `SELECT` keyword. -- The result set must not return more than one row. If no rows are returned, then the returned value of the resulting expression is null. - - The following is the syntax for a `SELECT` statement: - - ```sql - "(SELECT [ FROM [ WHERE ] ])" - ``` - - !!! Note - Omitting the `FROM table_list` clause isn't compatible with Oracle databases. If you don't need to specify any tables, using the `FROM DUAL` clause is compatible with Oracle databases. - -## EDB\* Loader control file examples - -The following are some examples of control files and their corresponding data files. - -### Delimiter-separated field data file - -This control file uses a delimiter-separated data file that appends rows to the `emp` table. The `APPEND` clause is used to allow inserting additional rows into the `emp` table. - - -```sql -LOAD DATA - INFILE 'emp.dat' - BADFILE 'emp.bad' - APPEND - INTO TABLE emp - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' - TRAILING NULLCOLS - ( - empno, - ename, - job, - mgr, - hiredate, - sal, - deptno, - comm - ) -``` - -The following is the corresponding delimiter-separated data file: - -```sql -9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20 -9102,PETERSON,SALESMAN,7698,20-DEC-10,2600.00,30,2300.00 -9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00 -9104,"JONES, JR.",MANAGER,7839,02-APR-09,7975.00,20 -``` - -The use of the `TRAILING NULLCOLS` clause allows you to omit the last field supplying the `comm` column from the first and last records. The `comm` column is set to null for the rows inserted from these records. - -Double quotation marks surround the value `JONES, JR.` in the last record since the comma delimiter character is part of the field value. - -This query displays the rows added to the table after the EDB\*Loader session: - -```sql -SELECT * FROM emp WHERE empno > 9100; -__OUTPUT__ - empno| ename | job | mgr| hiredate | sal | comm |deptno -------+-----------+---------+----+-------------------+-------+--------+------ - 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00|1980.00| | 20 - 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 - 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 - 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00|7975.00| | 20 -(4 rows) -``` - -### Fixed-width field data file - -This control file loads the same rows into the `emp` table. It uses a data file containing fixed-width fields. The `FIELDS TERMINATED BY` and `OPTIONALLY ENCLOSED BY` clauses are absent. Instead, each field includes the `POSITION` clause. - -```sql -LOAD DATA - INFILE 'emp_fixed.dat' - BADFILE 'emp_fixed.bad' - APPEND - INTO TABLE emp - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23), - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal POSITION (39:46), - deptno POSITION (47:48), - comm POSITION (49:56) - ) -``` - -The following is the corresponding data file containing fixed-width fields: - -```sql -9101ROGERS CLERK 790217-DEC-10 1980.0020 -9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 -9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 -9104JONES, JR.MANAGER 783902-APR-09 7975.0020 -``` - -### Single physical record data file – RECORDS DELIMITED BY clause - -This control file loads the same rows into the `emp` table but uses a data file with one physical record. Terminate each record loaded as a row in the table using a semicolon (`;`). The `RECORDS DELIMITED BY` clause specifies this value. - -```sql -LOAD DATA - INFILE 'emp_recdelim.dat' - BADFILE 'emp_recdelim.bad' - APPEND - INTO TABLE emp - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' - RECORDS DELIMITED BY ';' - TRAILING NULLCOLS - ( - empno, - ename, - job, - mgr, - hiredate, - sal, - deptno, - comm - ) -``` - -The following is the corresponding data file. The content is a single physical record in the data file. The record delimiter character is included following the last record, that is, at the end of the file. - -```sql -9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20,;9102,PETERSON,SALESMAN,7698,20-DEC-10, -2600.00,30,2300.00;9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00;9104,"JONES, -JR.",MANAGER,7839,02-APR-09,7975.00,20,; -``` - -### FILLER clause - -This control file uses the `FILLER` clause in the data fields for the `sal` and `comm` columns. EDB\*Loader ignores the values in these fields and sets the corresponding columns to null. - -```sql -LOAD DATA - INFILE 'emp_fixed.dat' - BADFILE 'emp_fixed.bad' - APPEND - INTO TABLE emp - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23), - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal FILLER POSITION (39:46), - deptno POSITION (47:48), - comm FILLER POSITION (49:56) - ) -``` - -Using the same fixed-width data file as in the prior fixed-width field example, the resulting rows in the table appear as follows: - -```sql -SELECT * FROM emp WHERE empno > 9100; -__OUTPUT__ - empno| ename | job | mgr| hiredate | sal | comm |deptno -------+-----------+---------+----+-------------------+-------+--------+------ - 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00| | | 20 - 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00| | | 30 - 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00| | | 30 - 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00| | | 20 -(4 rows) -``` - -### BOUNDFILLER clause - -This control file uses the `BOUNDFILLER` clause in the data fields for the `job` and `mgr` columns. EDB\*Loader ignores the values in these fields and sets the corresponding columns to null in the same manner as the `FILLER` clause. However, unlike columns with the `FILLER` clause, you can use columns with the `BOUNDFILLER` clause in an expression, as shown for column `jobdesc`. - -```sql -LOAD DATA - INFILE 'emp.dat' - BADFILE 'emp.bad' - APPEND - INTO TABLE empjob - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' - TRAILING NULLCOLS - ( - empno, - ename, - job BOUNDFILLER, - mgr BOUNDFILLER, - hiredate FILLER, - sal FILLER, - deptno FILLER, - comm FILLER, - jobdesc ":job || ' for manager ' || :mgr" - ) -``` - -The following is the delimiter-separated data file used in this example: - -```sql -9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20 -9102,PETERSON,SALESMAN,7698,20-DEC-10,2600.00,30,2300.00 -9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00 -9104,"JONES, JR.",MANAGER,7839,02-APR-09,7975.00,20 -``` - -The following table is loaded using the preceding control file and data file: - -```sql -CREATE TABLE empjob ( - empno NUMBER(4) NOT NULL CONSTRAINT empjob_pk PRIMARY KEY, - ename VARCHAR2(10), - job VARCHAR2(9), - mgr NUMBER(4), - jobdesc VARCHAR2(25) -); -``` - -The resulting rows in the table appear as follows: - -```sql -SELECT * FROM empjob; -__OUTPUT__ - empno | ename | job | mgr | jobdesc --------+------------+-----+-----+--------------------------- - 9101 | ROGERS | | | CLERK for manager 7902 - 9102 | PETERSON | | | SALESMAN for manager 7698 - 9103 | WARREN | | | SALESMAN for manager 7698 - 9104 | JONES, JR. | | | MANAGER for manager 7839 -(4 rows) -``` - -### Field types with length specification - -This control file contains the field type clauses with the length specification: - -```sql -LOAD DATA - INFILE 'emp_fixed.dat' - BADFILE 'emp_fixed.bad' - APPEND - INTO TABLE emp - TRAILING NULLCOLS - ( - empno CHAR(4), - ename CHAR(10), - job POSITION (15:23) CHAR(9), - mgr INTEGER EXTERNAL(4), - hiredate DATE(11) "DD-MON-YY", - sal DECIMAL EXTERNAL(8), - deptno POSITION (47:48), - comm POSITION (49:56) DECIMAL EXTERNAL(8) - ) -``` - -!!! Note - You can use the `POSITION` clause and the `fieldtype(length)` clause individually or in combination as long as each field definition contains at least one of the two clauses. - -The following is the corresponding data file containing fixed-width fields: - -```sql -9101ROGERS CLERK 790217-DEC-10 1980.0020 -9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 -9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 -9104JONES, JR. MANAGER 783902-APR-09 7975.0020 -``` - -The resulting rows in the table appear as follows: - -```sql -SELECT * FROM emp WHERE empno > 9100; -__OUTPUT__ - empno| ename | job | mgr| hiredate | sal | comm |deptno -------+-----------+---------+----+-------------------+-------+--------+------ - 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00|1980.00| | 20 - 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 - 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 - 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00|7975.00| | 20 -(4 rows) -``` - -### NULLIF clause - -This example uses the `NULLIF` clause on the `sal` column to set it to null for employees of job `MANAGER`. It also uses the clause on the `comm` column to set it to null if the employee isn't a `SALESMAN` and isn't in department `30`. In other words, a `comm` value is accepted if the employee is a `SALESMAN` or is a member of department `30`. - -The following is the control file: - -```sql -LOAD DATA - INFILE 'emp_fixed_2.dat' - BADFILE 'emp_fixed_2.bad' - APPEND - INTO TABLE emp - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23), - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal POSITION (39:46) NULLIF job = 'MANAGER', - deptno POSITION (47:48), - comm POSITION (49:56) NULLIF job <> 'SALESMAN' AND deptno <> '30' - ) -``` - -The following is the corresponding data file: - -```sql -9101ROGERS CLERK 790217-DEC-10 1980.0020 -9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 -9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 -9104JONES, JR. MANAGER 783902-APR-09 7975.0020 -9105ARNOLDS CLERK 778213-SEP-10 3750.0030 800.00 -9106JACKSON ANALYST 756603-JAN-11 4500.0040 2000.00 -9107MAXWELL SALESMAN 769820-DEC-10 2600.0010 1600.00 -``` - -The resulting rows in the table appear as follows: - -```sql -SELECT empno, ename, job, NVL(TO_CHAR(sal),'--null--') "sal", - NVL(TO_CHAR(comm),'--null--') "comm", deptno FROM emp WHERE empno > 9100; -__OUTPUT__ - empno | ename | job | sal | comm | deptno --------+------------+----------+----------+----------+------- - 9101 | ROGERS | CLERK | 1980.00 | --null-- | 20 - 9102 | PETERSON | SALESMAN | 2600.00 | 2300.00 | 30 - 9103 | WARREN | SALESMAN | 5250.00 | 2500.00 | 30 - 9104 | JONES, JR. | MANAGER | --null-- | --null-- | 20 - 9105 | ARNOLDS | CLERK | 3750.00 | 800.00 | 30 - 9106 | JACKSON | ANALYST | 4500.00 | --null-- | 40 - 9107 | MAXWELL | SALESMAN | 2600.00 | 1600.00 | 10 -(7 rows) -``` - -!!! Note - The `sal` column for employee `JONES, JR.` is null since the job is `MANAGER`. - -The `comm` values from the data file for employees `PETERSON`, `WARREN`, `ARNOLDS`, and `MAXWELL` are all loaded into the `comm` column of the `emp` table since these employees are either `SALESMAN` or members of department `30`. - -The `comm` value of `2000.00` in the data file for employee `JACKSON` is ignored, and the `comm` column of the `emp` table is set to null. This employee isn't a `SALESMAN` or a member of department `30`. - -### SELECT statement in a field expression - -This example uses a `SELECT` statement in the expression of the field definition to return the value to load into the column: - -```sql -LOAD DATA - INFILE 'emp_fixed.dat' - BADFILE 'emp_fixed.bad' - APPEND - INTO TABLE emp - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23) "(SELECT dname FROM dept WHERE deptno = :deptno)", - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal POSITION (39:46), - deptno POSITION (47:48), - comm POSITION (49:56) - ) -``` - -The following is the content of the `dept` table used in the `SELECT` statement: - -```sql -SELECT * FROM dept; -__OUTPUT__ - deptno | dname | loc ----------+------------+--------- - 10 | ACCOUNTING | NEW YORK - 20 | RESEARCH | DALLAS - 30 | SALES | CHICAGO - 40 | OPERATIONS | BOSTON -(4 rows) -``` - -The following is the corresponding data file: - -```sql -9101ROGERS CLERK 790217-DEC-10 1980.0020 -9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 -9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 -9104JONES, JR. MANAGER 783902-APR-09 7975.0020 -``` - -The resulting rows in the table appear as follows: - -```sql -SELECT * FROM emp WHERE empno > 9100; -__OUTPUT__ - empno| ename | job | mgr| hiredate | sal | comm |deptno -------+-----------+---------+----+-------------------+-------+--------+------ - 9101| ROGERS | RESEARCH|7902| 17-DEC-10 00:00:00|1980.00| | 20 - 9102| PETERSON | SALES |7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 - 9103| WARREN | SALES |7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 - 9104| JONES, JR.| RESEARCH|7839| 02-APR-09 00:00:00|7975.00| | 20 -(4 rows) -``` - -!!! Note - The `job` column contains the value from the `dname` column of the `dept` table returned by the `SELECT` statement instead of the job name from the data file. - -### Multiple INTO TABLE clauses - -This example uses multiple `INTO TABLE` clauses. For this example, two empty tables are created with the same data definition as the `emp` table. The following `CREATE TABLE` commands create these two empty tables, without inserting rows from the original `emp` table: - -```sql -CREATE TABLE emp_research AS SELECT * FROM emp WHERE deptno = 99; -CREATE TABLE emp_sales AS SELECT * FROM emp WHERE deptno = 99; -``` - -This control file contains two `INTO TABLE` clauses. Without an `APPEND` clause, it uses the default operation of `INSERT`. For this operation, the tables `emp_research` and `emp_sales` must be empty. - -```sql -LOAD DATA - INFILE 'emp_multitbl.dat' - BADFILE 'emp_multitbl.bad' - DISCARDFILE 'emp_multitbl.dsc' - INTO TABLE emp_research - WHEN (47:48) = '20' - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23), - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal POSITION (39:46), - deptno CONSTANT '20', - comm POSITION (49:56) - ) - INTO TABLE emp_sales - WHEN (47:48) = '30' - TRAILING NULLCOLS - ( - empno POSITION (1:4), - ename POSITION (5:14), - job POSITION (15:23), - mgr POSITION (24:27), - hiredate POSITION (28:38), - sal POSITION (39:46), - deptno CONSTANT '30', - comm POSITION (49:56) "ROUND(:comm + (:sal * .25), 0)" - ) -``` - -The `WHEN` clauses specify that when the field designated by columns 47 through 48 contains `20`, the record is inserted into the `emp_research` table. When that same field contains `30`, the record is inserted into the `emp_sales` table. If neither condition is true, the record is written to the discard file `emp_multitbl.dsc`. - -The `CONSTANT` clause is given for column `deptno`, so the specified constant value is inserted into `deptno` for each record. When the `CONSTANT` clause is used, it must be the only clause in the field definition other than the column name to which the constant value is assigned. - -Column `comm` of the `emp_sales` table is assigned a SQL expression. Expressions can reference column names by prefixing the column name with a colon character (`:`). - -The following is the corresponding data file: - -```sql -9101ROGERS CLERK 790217-DEC-10 1980.0020 -9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 -9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 -9104JONES, JR. MANAGER 783902-APR-09 7975.0020 -9105ARNOLDS CLERK 778213-SEP-10 3750.0010 -9106JACKSON ANALYST 756603-JAN-11 4500.0040 -``` - -The records for employees `ARNOLDS` and `JACKSON` contain `10` and `40` in columns 47 through 48, which don't satisfy any of the `WHEN` clauses. EDB\*Loader writes these two records to the discard file, `emp_multitbl.dsc`, with the following content: - -```sql -9105ARNOLDS CLERK 778213-SEP-10 3750.0010 -9106JACKSON ANALYST 756603-JAN-11 4500.0040 -``` - -The following are the rows loaded into the `emp_research` and `emp_sales` tables: - -```sql -SELECT * FROM emp_research; -__OUTPUT__ -empno | ename | job | mgr | hiredate | sal | comm | deptno --------+------------+---------+------+--------------------+---------+------+------- - 9101 | ROGERS | CLERK | 7902 | 17-DEC-10 00:00:00 | 1980.00 | | 20.00 - 9104 | JONES, JR. | MANAGER | 7839 | 02-APR-09 00:00:00 | 7975.00 | | 20.00 -(2 rows) -``` -```sql -SELECT * FROM emp_sales; -__OUTPUT__ -empno | ename | job | mgr | hiredate | sal | comm | deptno --------+----------+----------+------+--------------------+---------+---------+-------- - 9102 | PETERSON | SALESMAN | 7698 | 20-DEC-10 00:00:00 | 2600.00 | 2950.00 | 30.00 - 9103 | WARREN | SALESMAN | 7698 | 22-DEC-10 00:00:00 | 5250.00 | 3813.00 | 30.00 -(2 rows) -``` - - - -## Invoking EDB\*Loader - -You can run EDB\*Loader as superuser or as a normal user. Use the following command to invoke EDB\*Loader from the command line: - -```sql -edbldr [ -d ] [ -p ] [ -h ] -[ USERID={ | / | | / } ] -[ { -c | connstr= } ] - CONTROL= -[ DATA= ] -[ BAD=] -[ DISCARD= ] -[ DISCARDMAX= ] -[ HANDLE_CONFLICTS={ FALSE | TRUE } ] -[ LOG= ] -[ PARFILE= ] -[ DIRECT={ FALSE | TRUE } ] -[ FREEZE={ FALSE | TRUE } ] -[ ERRORS= ] -[ PARALLEL={ FALSE | TRUE } ] -[ ROWS= ] -[ SKIP= ] -[ SKIP_INDEX_MAINTENANCE={ FALSE | TRUE } ] -[ edb_resource_group= ] -``` - -### Description - -If you omit the `-d` option, the `-p` option, or the `-h` option, the defaults for the database, port, and host are determined by the same rules as other EDB Postgres Advanced Server utility programs, such as `edb-psql`. - -You can specify parameters listed in the syntax diagram in a *parameter file*. Exeptions include the `-d` option, `-p` option, `-h` option, and the `PARFILE` parameter. Specify the parameter file on the command line when you invoke `edbldr` using `PARFILE=param_file`. You can specify some parameters in the `OPTIONS` clause in the control file. For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -You can include the full directory path or a relative directory path to the file name when specifying `control_file`, `data_file`, `bad_file`, `discard_file`, `log_file`, and `param_file`. If you specify the file name alone or with a relative directory path, the file is assumed to exist in the case of `control_file`, `data_file`, or `param_file` relative to the current working directory from which `edbldr` is invoked. In the case of `bad_file`, `discard_file`, or `log_file`, the file is created. - -!!! Note - The control file must exist in the character set encoding of the client where `edbldr` is invoked. If the client is in an encoding different from the database encoding, then you must set the `PGCLIENTENCODING` environment variable on the client to the client’s encoding prior to invoking `edbldr`. This technique ensures character set conversion between the client and the database server is done correctly. - -The operating system account used to invoke `edbldr` must have read permission on the directories and files specified by `control_file`, `data_file`, and `param_file`. - -The operating system account `enterprisedb` must have write permission on the directories where `bad_file`, `discard_file`, and `log_file` are written. - -!!! Note - The file names for `control_file`, `data_file`, `bad_file`, `discard_file`, and `log_file` must include the extensions `.ctl`, `.dat`, `.bad`, `.dsc`, and `.log`, respectively. If the provided file name doesn't contain an extension, EDB\*Loader assumes the actual file name includes the appropriate extension. - -### Parameters - -`dbname` - - Name of the database containing the tables to load. - -`port` - - Port number on which the database server is accepting connections. - -`host` - - IP address of the host on which the database server is running. - -`USERID={ username/password | username/ | username | / }` - - EDB\*Loader connects to the database with `username`. `username` must be a superuser or a username with the required privileges. `password` is the password for `username`. - - If you omit the `USERID` parameter, EDB\*Loader prompts for `username` and `password`. If you specify `USERID=username/`, then EDB\*Loader either: - - Uses the password file specified by environment variable `PGPASSFILE` if `PGPASSFILE` is set - - Uses the `.pgpass` password file (`pgpass.conf` on Windows systems) if `PGPASSFILE` isn't set - - If you specify `USERID=username`, then EDB\*Loader prompts for `password`. If you specify `USERID=/`, the connection is attempted using the operating system account as the user name. - - !!! Note - EDB\*Loader ignores the EDB Postgres Advanced Server connection environment variables `PGUSER` and `PGPASSWORD`. See the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/libpq-pgpass.html) for information on the `PGPASSFILE` environment variable and the password file. - -`-c CONNECTION_STRING` - -`connstr=CONNECTION_STRING` - - The `-c` or `connstr=` option allows you to specify all the connection parameters supported by libpq. With this option, you can also specify SSL connection parameters or other connection parameters supported by libpq. If you provide connection options such as `-d`, `-h`, `-p` or `userid=dbuser/dbpass` separately, they might override the values provided by the `-c` or `connstr=` option. - -`CONTROL=control_file` - - `control_file` specifies the name of the control file containing EDB\*Loader directives. If you don't specify a file extension, an extension of `.ctl` is assumed. - - For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -`DATA=data_file` - - `data_file` specifies the name of the file containing the data to load into the target table. If you don't specify a file extension, an extension of `.dat` is assumed. Specifying a `data_file` on the command line overrides the `INFILE` clause specified in the control file. - - For more information about `data_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -`BAD=bad_file` - - `bad_file` specifies the name of a file that receives input data records that can't be loaded due to errors. Specifying a `bad_file` on the command line overrides any `BADFILE` clause specified in the control file. - - For more information about `bad_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -`DISCARD=discard_file` - - `discard_file` is the name of the file that receives input data records that don't meet any table’s selection criteria. Specifying a `discard_file` on the command line overrides the `DISCARDFILE` clause in the control file. - - For more information about `discard_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -`DISCARDMAX=max_discard_recs` - - `max_discard_recs` is the maximum number of discarded records that can be encountered from the input data records before terminating the EDB\*Loader session. Specifying `max_discard_recs` on the command line overrides the `DISCARDMAX` or `DISCARDS` clause in the control file. - - For more information about `max_discard_recs`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). - -`HANDLE_CONFLICTS={ FALSE | TRUE }` - - If any record insertion fails due to a unique constraint violation, EDB\*Loader aborts the entire operation. You can instruct EDB\*Loader to instead move the duplicate record to the `BAD` file and continue processing by setting `HANDLE_CONFLICTS` to `TRUE`. This behavior applies only if indexes are present. By default, `HANDLE_CONFLICTS` is set to `FALSE`. - - Setting `HANDLE_CONFLICTS` to `TRUE` isn't supported with direct path loading. If you set this parameter to `TRUE` when direct path loading, EDB\*Loader throws an error. - -`LOG=log_file` - - `log_file` specifies the name of the file in which EDB\*Loader records the results of the EDB\*Loader session. - - If you omit the `LOG` parameter, EDB\*Loader creates a log file with the name `control_file_base.log` in the directory from which `edbldr` is invoked. `control_file_base` is the base name of the control file used in the EDB\*Loader session. The operating system account `enterprisedb` must have write permission on the directory where the log file is written. - -`PARFILE=param_file` - - `param_file` specifies the name of the file that contains command line parameters for the EDB\*Loader session. You can specify command line parameters listed in this section in `param_file` instead of on the command line. Exceptions are the `-d`, `-p`, and `-h` options, and the `PARFILE` parameter. - - Any parameter given in `param_file` overrides the same parameter supplied on the command line before the `PARFILE` option. Any parameter given on the command line that appears after the `PARFILE` option overrides the same parameter given in `param_file`. - - !!! Note - Unlike other EDB\*Loader files, there's no default file name or extension assumed for `param_file`. However, by Oracle SQL\*Loader convention, `.par` is typically used as an extension. It isn't required. - -`DIRECT= { FALSE | TRUE }` - - If `DIRECT` is set to `TRUE` EDB\*Loader performs a direct path load instead of a conventional path load. The default value of `DIRECT` is `FALSE`. - - Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or might contain references to rows that were deleted. EnterpriseDB doesn't support direct inserts to load data into replicated tables. - - For information about direct path loads, see [Direct path load](#direct-path-load). - -`FREEZE= { FALSE | TRUE }` - - Set `FREEZE` to `TRUE` to copy the data with the rows `frozen`. A tuple guaranteed to be visible to all current and future transactions is marked as frozen to prevent transaction ID wraparound. For more information about frozen tuples, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/routine-vacuuming.html). - - You must specify a data-loading type of `TRUNCATE` in the control file when using the `FREEZE` option. `FREEZE` isn't supported for direct loading. - - By default, `FREEZE` is `FALSE`. - -`ERRORS=error_count` - - `error_count` specifies the number of errors permitted before aborting the EDB\*Loader session. The default is `50`. - -`PARALLEL= { FALSE | TRUE }` - - Set `PARALLEL` to `TRUE` to indicate that this EDB\*Loader session is one of a number of concurrent EDB\*Loader sessions participating in a parallel direct path load. The default value of `PARALLEL` is `FALSE`. - - When `PARALLEL` is `TRUE`, the `DIRECT` parameter must also be set to `TRUE`. - - For more information about parallel direct path loads, see [Parallel direct path load](#parallel-direct-path-load). - -`ROWS=n` - - `n` specifies the number of rows that EDB\*Loader commits before loading the next set of `n` rows. - -`SKIP=skip_count` - - Number of records at the beginning of the input data file to skip before loading begins. The default is `0`. - -`SKIP_INDEX_MAINTENANCE= { FALSE | TRUE }` - - If set to `TRUE`, index maintenance isn't performed as part of a direct path load, and indexes on the loaded table are marked as invalid. The default value of `SKIP_INDEX_MAINTENANCE` is `FALSE`. - - During a parallel direct path load, target table indexes aren't updated. They are marked as invalid after the load is complete. - - You can use the `REINDEX` command to rebuild an index. For more information about the `REINDEX` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-reindex.htm). - -`edb_resource_group=group_name` - - `group_name` specifies the name of an EDB Resource Manager resource group to which to assign the EDB\*Loader session. - - Any default resource group that was assigned to the session is overridden by the resource group given by the `edb_resource_group` parameter specified on the `edbldr` command line. An example of such a group is a database user running the EDB\*Loader session who was assigned a default resource group with the `ALTER ROLE ... SET` `edb_resource_group` command. - -### Examples - -This example invokes EDB\*Loader using a control file named `emp.ctl` to load a table in database `edb`. The file is located in the current working directory. - -```shell -$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password -CONTROL=emp.ctl -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. - -Successfully loaded (4) records -``` - -In this example, EDB\*Loader prompts for the user name and password since they are omitted from the command line. In addition, the files for the bad file and log file are specified with the `BAD` and `LOG` command line parameters. - -```shell -$ /usr/edb/as14/bin/edbldr -d edb CONTROL=emp.ctl BAD=/tmp/emp.bad -LOG=/tmp/emp.log -Enter the user name : enterprisedb -Enter the password : -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. - -Successfully loaded (4) records -``` - -This example runs EDB\*Loader using a parameter file located in the current working directory. The `SKIP` and `ERRORS` parameter default values are specified in the parameter file in addition the `CONTROL`, `BAD`, and `LOG` files. The parameter file, `emp.par`, contains the following: - -```ini -CONTROL=emp.ctl -BAD=/tmp/emp.bad -LOG=/tmp/emp.log -SKIP=1 -ERRORS=10 -``` - -Invoke EDB\*Loader with the parameter file: - -```shell -$ /usr/edb/as14/bin/edbldr -d edb PARFILE=emp.par -Enter the user name : enterprisedb -Enter the password : -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. - -Successfully loaded (3) records -``` - -This example invokes EDB\*Loader using a `connstr=` option that uses the `emp.ctl` control file located in the current working directory to load a table in a database named `edb`: - -```shell -$ /usr/edb/as14/bin/edbldr connstr=\"sslmode=verify-ca sslcompression=0 -host=127.0.0.1 dbname=edb port=5444 user=enterprisedb\" CONTROL=emp.ctl -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. - -Successfully loaded (4) records -``` - -This example invokes EDB\*Loader using a normal user. For this example, one empty table `bar` is created and a normal user `bob` is created. The `bob` user is granted all privileges on the table `bar`. The CREATE TABLE command creates the empty table. The CREATE USER command creates the user and the GRANT command gives required privileges to the user `bob` on the `bar` table: - -```sql -CREATE TABLE bar(i int); -CREATE USER bob identified by '123'; -GRANT ALL on bar TO bob; -``` - -The control file and data file: - -```shell -## Control file -EDBAS/ - (master) $ cat /tmp/edbldr.ctl -LOAD DATA INFILE '/tmp/edbldr.dat' -truncate into table bar -( -i position(1:1) -) - -## Data file -EDBAS/ - (master) $ cat /tmp/edbldr.dat -1 -2 -3 -5 -``` - -Invoke EDB*Loader: - -```shell -EDBAS/ - (master) $ /usr/edb/as15/bin/edbldr -d edb userid=bob/123 control=/tmp/edbldr.ctl -EDB*Loader: Copyright (c) 2007-2022, EnterpriseDB Corporation. - -Successfully loaded (4) records -``` - - - -### Exit codes - -When EDB\*Loader exits, it returns one of the following codes: - -| Exit code | Description | -| --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | Indicates that all rows loaded successfully. | -| `1` | Indicates that EDB\*Loader encountered command line or syntax errors or aborted the load operation due to an unrecoverable error. | -| `2` | Indicates that the load completed, but some (or all) rows were rejected or discarded. | -| `3` | Indicates that EDB\*Loader encountered fatal errors (such as OS errors). This class of errors is equivalent to the `FATAL` or `PANIC` severity levels of PostgreSQL errors. | - - - -## Direct path load - -During a direct path load, EDB\*Loader writes the data directly to the database pages, which is then synchronized to disk. The insert processing associated with a conventional path load is bypassed, resulting in a performance improvement. - -Bypassing insert processing reduces the types of constraints on the target table. The following types of constraints are permitted on the target table of a direct path load: - -- Primary key -- Not null constraints -- Indexes (unique or non-unique) - -The following are restrictions on the target table of a direct path load: - -- Triggers aren't permitted. -- Check constraints aren't permitted. -- Foreign key constraints on the target table referencing another table aren't permitted. -- Foreign key constraints on other tables referencing the target table aren't permitted. -- You must not partition the table. -- Rules can exist on the target table, but they aren't executed. - -!!! Note - Currently, a direct path load in EDB\*Loader is more restrictive than in Oracle SQL\*Loader. The preceding restrictions don't apply to Oracle SQL\*Loader in most cases. The following restrictions apply to a control file used in a direct path load: - - - Multiple table loads aren't supported. You can specify only one `INTO TABLE` clause in the control file. - - You can't use SQL expressions in the data field definitions of the `INTO TABLE` clause. - - The `FREEZE` option isn't supported for direct path loading. - -To run a direct path load, add the `DIRECT=TRUE` option: - -```shell -$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password -CONTROL=emp.ctl DIRECT=TRUE -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. - -Successfully loaded (4) records -``` - - - -## Parallel direct path load - -You can further improve the performance of a direct path load by distributing the loading process over two or more sessions running concurrently. Each session runs a direct path load into the same table. - -Since the same table is loaded from multiple sessions, the input records to load into the table must be divided among several data files. This way, each EDB\*Loader session uses its own data file, and the same record isn't loaded into the table more than once. - -The target table of a parallel direct path load is under the same restrictions as a direct path load run in a single session. - -The following are the restrictions on the target table of a direct path load: - -- Triggers aren't permitted. -- Check constraints aren't permitted. -- Foreign key constraints on the target table referencing another table aren't permitted. -- Foreign key constraints on other tables referencing the target table aren't permitted. -- You must not partition the table. -- Rules can exist on the target table, but they aren't executed. - -In addition, you must specify the `APPEND` clause in the control file used by each EDB\*Loader session. - -To run a parallel direct path load, run EDB\*Loader in a separate session for each participant of the parallel direct path load. You must include the `DIRECT=TRUE` and `PARALLEL=TRUE` parameters when invoking each such EDB\*Loader session. - -Each EDB\*Loader session runs as an independent transaction. Aborting and rolling back changes of one of the parallel sessions doesn't affect the loading done by the other parallel sessions. - -!!! Note - In a parallel direct path load, each EDB\*Loader session reserves a fixed number of blocks in the target table using turns. Some of the blocks in the last allocated chunk might not be used, and those blocks remain uninitialized. A later use of the `VACUUM` command on the target table might show warnings about these uninitialized blocks, such as the following: - -```text -WARNING: relation "emp" page 98264 is uninitialized --- fixing - -WARNING: relation "emp" page 98265 is uninitialized --- fixing - -WARNING: relation "emp" page 98266 is uninitialized --- fixing -``` - -This behavior is expected and doesn't indicate data corruption. - -Indexes on the target table aren't updated during a parallel direct path load. They are therefore marked as invalid after the load is complete. You must use the `REINDEX` command to rebuild the indexes. - -This example shows the use of a parallel direct path load on the `emp` table. - -!!! Note - If you attempt a parallel direct path load on the sample `emp` table provided with EDB Postgres Advanced Server, you must first remove the triggers and constraints referencing the `emp` table. In addition, the primary key column, `empno`, was expanded from `NUMBER(4)` to `NUMBER` in this example to allow for inserting more rows. - -This is the control file used in the first session: - -```sql -LOAD DATA - INFILE '/home/user/loader/emp_parallel_1.dat' - APPEND - INTO TABLE emp - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' - TRAILING NULLCOLS - ( - empno, - ename, - job, - mgr, - hiredate, - sal, - deptno, - comm - ) -``` - -You must specify the `APPEND` clause in the control file for a parallel direct path load. - -This example invokes EDB\*Loader in the first session. You must specify the `DIRECT=TRUE` and `PARALLEL=TRUE` parameters. - -```shell -$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password -CONTROL=emp_parallel_1.ctl DIRECT=TRUE PARALLEL=TRUE -WARNING: index maintenance will be skipped with PARALLEL load -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. -``` - -The control file used for the second session appears as follows. It's the same as the one used in the first session, but it uses a different data file. - -```sql -LOAD DATA - INFILE '/home/user/loader/emp_parallel_2.dat' - APPEND - INTO TABLE emp - FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' - TRAILING NULLCOLS - ( - empno, - ename, - job, - mgr, - hiredate, - sal, - deptno, - comm - ) -``` - -This control file is used in a second session: - -```shell -$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password -CONTROL=emp_parallel_2.ctl DIRECT=TRUE PARALLEL=TRUE -WARNING: index maintenance will be skipped with PARALLEL load -EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. -``` - -EDB\*Loader displays the following message in each session when the load operation completes: - -```text -Successfully loaded (10000) records -``` - -This query shows that the index on the emp table was marked `INVALID`: - -```sql -SELECT index_name, status FROM user_indexes WHERE table_name = 'EMP'; -__OUTPUT__ - index_name | status --------------+--------- - EMP_PK | INVALID - (1 row) -``` - -!!! Note - `user_indexes` is the view of indexes compatible with Oracle databases owned by the current user. - -Queries on the `emp` table don't use the index unless you rebuild it using the `REINDEX` command: - -```sql -REINDEX INDEX emp_pk; -``` - -A later query on `user_indexes` shows that the index is now marked as `VALID`: - -```sql -SELECT index_name, status FROM user_indexes WHERE table_name = 'EMP'; -__OUTPUT__ - index_name | status --------------+-------- - EMP_PK | VALID - (1 row) -``` - - - -## Remote loading - -EDB\*Loader supports a feature called *remote loading*. In remote loading, the database containing the table to load is running on a database server on a host different from the one where EDB\*Loader is invoked with the input data source. - -This feature is useful if you have a large amount of data to load, and you don't want to create a large data file on the host running the database server. - -In addition, you can use the standard input feature to pipe the data from the data source, such as another program or script, directly to EDB\*Loader. EDB\*Loader then loads the table in the remote database. This feature bypasses having to create a data file on disk for EDB\*Loader. - -Performing remote loading along with using standard input requires the following: - -- The `edbldr` program must be installed on the client host on which to invoke it with the data source for the EDB\*Loader session. -- The control file must contain the clause `INFILE 'stdin'` so you can pipe the data directly into EDB\*Loader’s standard input. For information on the `INFILE` clause and the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). -- All files used by EDB\*Loader, such as the control file, bad file, discard file, and log file, must reside on or be created on the client host on which `edbldr` is invoked. -- When invoking EDB\*Loader, use the `-h` option to specify the IP address of the remote database server. For more information, see [Invoking EDB\*Loader](#invoking-edbloader). -- Use the operating system pipe operator (`|`) or input redirection operator (`<`) to supply the input data to EDB\*Loader. - -This example loads a database running on a database server at `192.168.1.14` using data piped from a source named `datasource`: - -```shell -datasource | ./edbldr -d edb -h 192.168.1.14 USERID=enterprisedb/password -CONTROL=remote.ctl -``` - -This example also shows how you can use standard input: - -```shell -./edbldr -d edb -h 192.168.1.14 USERID=enterprisedb/password -CONTROL=remote.ctl < datasource -``` - - - -## Updating a table with a conventional path load - -You can use EDB\*Loader with a conventional path load to update the rows in a table, merging new data with the existing data. When you invoke EDB\*Loader to perform an update, the server searches the table for an existing row with a matching primary key: - -- If the server locates a row with a matching key, it replaces the existing row with the new row. -- If the server doesn't locate a row with a matching key, it adds the new row to the table. - -To use EDB\*Loader to update a table, the table must have a primary key. You can't use EDB\*Loader to update a partitioned table. - -To perform `UPDATE`, use the same steps as when performing a conventional path load: - -1. Create a data file that contains the rows you want to update or insert. -2. Define a control file that uses the `INFILE` keyword to specify the name of the data file. For information about building the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). -3. Invoke EDB\*Loader, specifying the database name, connection information, and the name of the control file. For information about invoking EDB\*Loader, see [Invoking EDB\*Loader](#invoking-edbloader). - -This example uses the `emp` table that's distributed with the EDB Postgres Advanced Server sample data. By default, the table contains: - -```sql -edb=# select * from emp; -__OUTPUT__ -empno|ename | job | mgr | hiredate | sal | comm | deptno ------+------+---------+------+--------------------+--------- +-------+-------- -7369 |SMITH |CLERK | 7902 | 17-DEC-80 00:00:00 | 800.00 | | 20 -7499 |ALLEN |SALESMAN | 7698 | 20-FEB-81 00:00:00 | 1600.00 |300.00 | 30 -7521 |WARD |SALESMAN | 7698 | 22-FEB-81 00:00:00 | 1250.00 |500.00 | 30 -7566 |JONES |MANAGER | 7839 | 02-APR-81 00:00:00 | 2975.00 | | 20 -7654 |MARTIN|SALESMAN | 7698 | 28-SEP-81 00:00:00 | 1250.00 |1400.00| 30 -7698 |BLAKE |MANAGER | 7839 | 01-MAY-81 00:00:00 | 2850.00 | | 30 -7782 |CLARK |MANAGER | 7839 | 09-JUN-81 00:00:00 | 2450.00 | | 10 -7788 |SCOTT |ANALYST | 7566 | 19-APR-87 00:00:00 | 3000.00 | | 20 -7839 |KING |PRESIDENT| | 17-NOV-81 00:00:00 | 5000.00 | | 10 -7844 |TURNER|SALESMAN | 7698 | 08-SEP-81 00:00:00 | 1500.00 | 0.00 | 30 -7876 |ADAMS |CLERK | 7788 | 23-MAY-87 00:00:00 | 1100.00 | | 20 -7900 |JAMES |CLERK | 7698 | 03-DEC-81 00:00:00 | 950.00 | | 30 -7902 |FORD |ANALYST | 7566 | 03-DEC-81 00:00:00 | 3000.00 | | 20 -7934 |MILLER|CLERK | 7782 | 23-JAN-82 00:00:00 | 1300.00 | | 10 -(14 rows) -``` - -This control file (`emp_update.ctl`) specifies the fields in the table in a comma-delimited list. The control file performs an `UPDATE` on the `emp` table. - -```sql -LOAD DATA - INFILE 'emp_update.dat' - BADFILE 'emp_update.bad' - DISCARDFILE 'emp_update.dsc' -UPDATE INTO TABLE emp -FIELDS TERMINATED BY "," -(empno, ename, job, mgr, hiredate, sal, comm, deptno) -``` - -The data that's being updated or inserted is saved in the `emp_update.dat` file. `emp_update.dat` contains: - -```sql -7521,WARD,MANAGER,7839,22-FEB-81 00:00:00,3000.00,0.00,30 -7566,JONES,MANAGER,7839,02-APR-81 00:00:00,3500.00,0.00,20 -7903,BAKER,SALESMAN,7521,10-JUN-13 00:00:00,1800.00,500.00,20 -7904,MILLS,SALESMAN,7839,13-JUN-13 00:00:00,1800.00,500.00,20 -7654,MARTIN,SALESMAN,7698,28-SEP-81 00:00:00,1500.00,400.00,30 -``` - -Invoke EDB\*Loader, specifying the name of the database (`edb`), the name of a database user and their associated password, and the name of the control file (`emp_update.ctl`): - -```shell -edbldr -d edb userid=user_name/password control=emp_update.ctl -``` - -After performing the update, the `emp` table contains: - -```sql -edb=# select * from emp; -__OUTPUT__ -empno|ename | job | mgr | hiredate | sal | comm | deptno ------+------+---------+------+--------------------+---------+--------+-------- -7369 |SMITH |CLERK | 7902 | 17-DEC-80 00:00:00 | 800.00 | | 20 -7499 |ALLEN |SALESMAN | 7698 | 20-FEB-81 00:00:00 | 1600.00 | 300.00 | 30 -7521 |WARD |MANAGER | 7839 | 22-FEB-81 00:00:00 | 3000.00 | 0.00 | 30 -7566 |JONES |MANAGER | 7839 | 02-APR-81 00:00:00 | 3500.00 | 0.00 | 20 -7654 |MARTIN|SALESMAN | 7698 | 28-SEP-81 00:00:00 | 1500.00 | 400.00 | 30 -7698 |BLAKE |MANAGER | 7839 | 01-MAY-81 00:00:00 | 2850.00 | | 30 -7782 |CLARK |MANAGER | 7839 | 09-JUN-81 00:00:00 | 2450.00 | | 10 -7788 |SCOTT |ANALYST | 7566 | 19-APR-87 00:00:00 | 3000.00 | | 20 -7839 |KING |PRESIDENT| | 17-NOV-81 00:00:00 | 5000.00 | | 10 -7844 |TURNER|SALESMAN | 7698 | 08-SEP-81 00:00:00 | 1500.00 | 0.00 | 30 -7876 |ADAMS |CLERK | 7788 | 23-MAY-87 00:00:00 | 1100.00 | | 20 -7900 |JAMES |CLERK | 7698 | 03-DEC-81 00:00:00 | 950.00 | | 30 -7902 |FORD |ANALYST | 7566 | 03-DEC-81 00:00:00 | 3000.00 | | 20 -7903 |BAKER |SALESMAN | 7521 | 10-JUN-13 00:00:00 | 1800.00 | 500.00 | 20 -7904 |MILLS |SALESMAN | 7839 | 13-JUN-13 00:00:00 | 1800.00 | 500.00 | 20 -7934 |MILLER|CLERK | 7782 | 23-JAN-82 00:00:00 | 1300.00 | | 10 -(16 rows) -``` - -The rows containing information for the three employees that are currently in the `emp` table are updated, while rows are added for the new employees (`BAKER` and `MILLS`). diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx new file mode 100644 index 00000000000..e9cdff16eea --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx @@ -0,0 +1,411 @@ +--- +title: "Building the EDB*Loader control file" +navTitle: "Building the control file" +description: "Provides information necessary to build a control file" +--- + + + +When you invoke EDB\*Loader, the list of arguments provided must include the name of a control file. The control file includes the instructions that EDB\*Loader uses to load the tables from the input data file. The control file includes information such as: + +- The name of the input data file containing the data to load +- The name of the tables to load from the data file +- Names of the columns in the tables and their corresponding field placement in the data file +- Specification of whether the data file uses a delimiter string to separate the fields or if the fields occupy fixed column positions +- Optional selection criteria to choose the records from the data file to load into a given table +- The name of the file that collects illegally formatted records +- The name of the discard file that collects records that don't meet the selection criteria of any table + +## Control file syntax + +The syntax for the EDB\*Loader control file is: + +```sql +[ OPTIONS ( [, ] ...) ] +LOAD DATA + [ CHARACTERSET ] + [ INFILE '{ | }' ] + [ BADFILE '' ] + [ DISCARDFILE '' ] + [ { DISCARDMAX | DISCARDS } ] +[ INSERT | APPEND | REPLACE | TRUNCATE ] +[ PRESERVE BLANKS ] +{ INTO TABLE + [ WHEN [ AND ] ...] + [ FIELDS TERMINATED BY '' + [ OPTIONALLY ENCLOSED BY '' ] ] + [ RECORDS DELIMITED BY '' ] + [ TRAILING NULLCOLS ] + ( [, ] ...) +} ... +``` + +Where `field_def` defines a field in the specified `data_file`. The field describes the location, data format, or value of the data to insert into `column_name` of `target_table`. The syntax of `field_def` is: + +```sql + { + CONSTANT | + FILLER [ POSITION () ] [ ] | + BOUNDFILLER [ POSITION () ] [ ] | + [ POSITION () ] [ ] + [ NULLIF [ AND ] ...] + [ PRESERVE BLANKS ] [ "" ] +} +``` + +Where `fieldtype` is one of: + +```sql +CHAR [()] | DATE [()] | TIMESTAMP [()] [ "" ] | +INTEGER EXTERNAL [()] | +FLOAT EXTERNAL [()] | DECIMAL EXTERNAL [()] | +ZONED EXTERNAL [()] | ZONED [( [,])] +``` + +## Description + +The specification of `data_file`, `bad_file`, and `discard_file` can include the full directory path or a relative directory path to the file name. If the file name is specified alone or with a relative directory path, the file is then assumed to exist, in the case of `data_file`, relative to the current working directory from which you invoke `edbldr. In the case of `bad_file` or `discard_file`, it's created. + +You can include references to environment variables in the EDB\*Loader control file when referring to a directory path or file name. Environment variable references are formatted differently on Windows systems than on Linux systems: + +- On Linux, the format is `$ENV_VARIABLE` or `${ENV_VARIABLE}`. +- On Windows, the format is `%ENV_VARIABLE%`. + +Where `ENV_VARIABLE` is the environment variable that's set to the directory path or file name. + +Set the `EDBLDR_ENV_STYLE` environment variable to interpret environment variable references as Windows-styled references or Linux-styled references regardless of the operating system on which EDB\*Loader resides. You can use this environment variable to create portable control files for EDB\*Loader. + +- On a Windows system, set `EDBLDR_ENV_STYLE` to `linux` or `unix` to recognize Linux-style references in the control file. +- On a Linux system, set `EDBLDR_ENV_STYLE` to `windows` to recognize Windows-style references in the control file. + +The operating system account `enterprisedb` must have read permission on the directory and file specified by `data_file`. + +The operating system account enterprisedb must have write permission on the directories where `bad_file` and `discard_file` are written. + +!!! Note + The file names for `data_file`, `bad_file`, and `discard_file` must have extensions `.dat`, `.bad`, and `.dsc`, respectively. If the provided file name doesn't have an extension, EDB\*Loader assumes the actual file name includes the appropriate extension. + +Suppose an EDB\*Loader session results in data format errors, the `BADFILE` clause isn't specified, and the BAD parameter isn't given on the command line when `edbldr` is invoked. In this case, a bad file is created with the name `control_file_base.bad` in the directory from which `edbldr` is invoked. `control_file_base` is the base name of the control file used in the `edbldr` session. + +If all of the following conditions are true, the discard file isn't created even if the EDB\*Loader session results in discarded records: + +- The `DISCARDFILE` clause for specifying the discard file isn't included in the control file. +- The `DISCARD` parameter for specifying the discard file isn't included on the command line. +- The `DISCARDMAX` clause for specifying the maximum number of discarded records isn't included in the control file. +- The `DISCARDS` clause for specifying the maximum number of discarded records isn't included in the control file. +- The `DISCARDMAX` parameter for specifying the maximum number of discarded records isn't included on the command line. + +Suppose you don't specify the `DISCARDFILE` clause and the `DISCARD` parameter for explicitly specifying the discard file name, but you do specify `DISCARDMAX` or `DISCARDS`. In this case, the EDB\*Loader session creates a discard file using the data file name with an extension of `.dsc`. + +!!! Note + The keywords `DISCARD` and `DISCARDS` differ. `DISCARD` is an EDB\*Loader command line parameter used to specify the discard file name (see [General usage](#general-usage)). `DISCARDS` is a clause of the `LOAD DATA` directive that can appear only in the control file. Keywords `DISCARDS` and `DISCARDMAX` provide the same functionality of specifying the maximum number of discarded records allowed before terminating the EDB\*Loader session. Records loaded into the database before terminating the EDB\*Loader session due to exceeding the `DISCARDS` or `DISCARDMAX` settings are kept in the database. They aren't rolled back. + +Specifying one of `INSERT`, `APPEND`, `REPLACE`, or `TRUNCATE` establishes the default action for adding rows to target tables. The default action is `INSERT`. + +If you specify the `FIELDS TERMINATED BY` clause, then you can't specify the `POSITION (start:end)` clause for any `field_def`. Alternatively, if you don't specify the `FIELDS TERMINATED BY` clause, then every `field_def` must contain the `POSITION (start:end)` clause, the `fieldtype(length)` clause, or the `CONSTANT` clause. + +## Parameters + +`OPTIONS param=value` + + Use the `OPTIONS` clause to specify `param=value` pairs that represent an EDB\*Loader directive. If you specify a parameter in the `OPTIONS` clause and on the command line when `edbldr` is invoked, the command line setting takes effect. + + Specify one or more of the following parameter/value pairs: + +- `DIRECT= { FALSE | TRUE }` + + With `DIRECT` set to `TRUE`, EDB\*Loader performs a direct path load instead of a conventional path load. The default value of `DIRECT` is `FALSE`. + + Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or can potentially contain references to rows that were deleted. Direct inserts to load data into replicated tables aren't supported. + + For information on direct path loads see, [Direct path load](#direct-path-load). + +- `ERRORS=error_count` + + `error_count` specifies the number of errors permitted before aborting the EDB\*Loader session. The default is `50`. + +- `FREEZE= { FALSE | TRUE }` + + Set `FREEZE` to `TRUE` to copy the data with the rows `frozen`. A tuple guaranteed to be visible to all current and future transactions is marked as frozen to prevent transaction ID wraparound. For more information about frozen tuples, see the [PostgreSQL core documentation]( } 'val' [ ) ] +``` + + This conditional clause is used for the `WHEN` clause, which is part of the `INTO TABLE target_table` clause. It's also used for the `NULLIF` clause, which is part of the field definition denoted as `field_def` in the syntax diagram. + + `start` and `end` are positive integers specifying the column positions in `data_file` that mark the beginning and end of a field to compare with the constant `val`. The first character in each record begins with a `start` value of `1`. + + `column_name` specifies the name assigned to a field definition of the data file as defined by `field_def` in the syntax diagram. + + Using (`start`:`end`) or `column_name` defines the portion of the record in `data_file` to compare with the value specified by `val` to evaluate as either true or false. + + All characters used in the `field_condition` text (particularly in the `val` string) must be valid in the database encoding. For performing data conversion, EDB\*Loader first converts the characters in `val` string to the database encoding and then to the data file encoding. + + In the `WHEN field_condition [ AND field_condition ]` clause, if all such conditions evaluate to `TRUE` for a given record, then EDB\*Loader attempts to insert that record into `target_table`. If the insert operation fails, the record is written to `bad_file`. + + Suppose, for a given record, that none of the `WHEN` clauses evaluate to `TRUE` for all `INTO TABLE` clauses. The record is written to `discard_file` if a discard file was specified for the EDB\*Loader session. + + See the description of the `NULLIF` clause in this parameters list for the effect of `field_condition` on this clause. + +`termstring` + + String of one or more characters that separates each field in `data_file`. The characters can be single byte or multibyte. However, they must be valid in the database encoding. Two consecutive appearances of `termstring` with no intervening character results in the corresponding column being set to null. + +`enclstring` + + String of one or more characters used to enclose a field value in `data_file`. The characters can be single byte or multibyte, However, they must be valid in the database encoding. Use `enclstring` on fields where `termstring` appears as part of the data. + +`delimstring` + + String of one or more characters that separates each record in `data_file`. The characters can be single byte or multibyte. However, they must be valid in the database encoding. Two consecutive appearances of `delimstring` with no intervening character results in no corresponding row being loaded into the table. You must also terminate the last record (that is, the end of the data file) with the `delimstring` characters. Otherwise, the final record isn't loaded into the table. + + !!! Note + The `RECORDS DELIMITED BY 'delimstring'` clause isn't compatible with Oracle databases. + +`TRAILING NULLCOLS` + + If you specify `TRAILING NULLCOLS`, then the columns in the column list for which there's no data in `data_file` for a given record are set to null when the row is inserted. This option applies only to one or more consecutive columns at the end of the column list. + + If fields are omitted at the end of a record and you don't specify `TRAILING NULLCOLS`, EDB\*Loader assumes the record contains formatting errors and writes it to the bad file. + +`column_name` + + Name of a column in `target_table` into which to insert a field value defined by `field_def`. If the field definition includes the `FILLER` or `BOUNDFILLER` clause, then `column_name` isn't required as the name of a column in the table. It can be any identifier name since the `FILLER` and `BOUNDFILLER` clauses prevent loading the field data into a table column. + +`CONSTANT val` + + Specifies a constant that's type-compatible with the column data type to which it's assigned in a field definition. You can use single or double quotes around `val`. If `val` contains white space, then you must use quotation marks around it. + + The use of the `CONSTANT` clause determines the value to assign to a column in each inserted row. No other clause can appear in the same field definition. + + If you use the `TERMINATED BY` clause to delimit the fields in `data_file`, there must be no delimited field in `data_file` corresponding to any field definition with a `CONSTANT` clause. In other words, EDB\*Loader assumes there's no field in `data_file` for any field definition with a `CONSTANT` clause. + +`FILLER` + + Specifies not to load the data in the field defined by the field definition into the associated column if the identifier of the field definition is an actual column name in the table. In this case, the column is set to null. Use of the `FILLER` or `BOUNDFILLER` clause is the only circumstance in which you don't have to identify the field definition with an actual column name. + + Unlike the `BOUNDFILLER` clause, you can't reference an identifier defined with the `FILLER` clause in a SQL expression. See the discussion of the `expr` parameter. + +`BOUNDFILLER` + + Specifies not to load the data in the field defined by the field definition into the associated column if the identifier of the field definition is an actual column name in the table. In this case, the column is set to null. Use of the `FILLER` or `BOUNDFILLER` clause is the only circumstance in which you don't have to identify the field definition with an actual column name. + + Unlike the `FILLER` clause, a SQL expression can reference an identifier defined with the `BOUNDFILLER` clause. See the discussion of the `expr` parameter. + +`POSITION (start:end)` + + Defines the location of the field in a record in a fixed-width field data file. `start` and `end` are positive integers. The first character in the record has a start value of `1`. + +```sql +CHAR [()] | DATE [()] | TIMESTAMP [()] [ "" ] | +INTEGER EXTERNAL [()] | +FLOAT EXTERNAL [()] | DECIMAL EXTERNAL [()] | +ZONED EXTERNAL [()] | ZONED [([,])] +``` + + Field type that describes the format of the data field in `data_file`. + + !!! Note + Specifying a field type is optional for descriptive purposes and has no effect on whether EDB\*Loader successfully inserts the data in the field into the table column. Successful loading depends on the compatibility of the column data type and the field value. For example, a column with data type `NUMBER(7,2)` successfully accepts a field containing `2600`. However, if the field contains a value such as `26XX`, the insertion fails, and the record is written to `bad_file`. + +`ZONED` data is not human readable. `ZONED` data is stored in an internal format where each digit is encoded in a separate nibble/nybble/4-bit field. In each `ZONED` value, the last byte contains a single digit in the high-order 4 bits and the sign in the low-order 4 bits. + +`length` + + Specifies the length of the value to load into the associated column. + + If you specify the `POSITION` (`start`:`end`) clause with a `fieldtype(length)` clause, then the ending position of the field is overridden by the specified `length` value. That is, the length of the value to load into the column is determined by the `length` value beginning at the `start` position and not by the `end` position of the `POSITION` (`start`:`end`) clause. Thus, the value to load into the column might be shorter than the field defined by `POSITION` (`start`:`end`). Or, it might go beyond the end position, depending on the specified `length` size. + + If you specify the `FIELDS TERMINATED BY 'termstring'` clause as part of the `INTO TABLE` clause, and a field definition contains the `fieldtype(length)` clause, then a record is accepted. However, the specified `length` values must be greater than or equal to the field lengths as determined by the `termstring` characters enclosing all such fields of the record. If the specified `length` value is less than a field length as determined by the enclosing `termstring` characters for any such field, then the record is rejected. + + If you don't specify the `FIELDS TERMINATED BY 'termstring'` clause, and you don't include the `POSITION` (`start`:`end`) clause with a field containing the `fieldtype(length)` clause, then the starting position of this field begins with the next character following the ending position of the preceding field. The ending position of the preceding field is either: + + - The end of its `length` value if the preceding field contains the `fieldtype(length)` clause + - Its `end` parameter if the field contains the `POSITION` (`start`:`end`) clause without the `fieldtype(length)` clause + +`precision` + + Use `precision` to specify the length of the `ZONED` value. + + If the `precision` value specified for `ZONED` conflicts with the length calculated by the server based on information provided with the `POSITION` clause, EDB\*Loader uses the value specified for `precision`. + +`scale` + + Specifies the number of digits to the right of the decimal point in a `ZONED` value. + +`datemask` + + Specifies the ordering and abbreviation of the day, month, and year components of a date field. + + !!! Note + If you specify the `DATE` or `TIMESTAMP` field type with a SQL expression for the column, then you must specify `datemask` after `DATE` or `TIMESTAMP` and before the SQL expression. See the discussion of the `expr` parameter. + + When using the `TIMESTAMP` field datatype, if you specify `time_stamp timestamp "yyyymmddhh24miss"`, the `datemask` is converted to the SQL expression. However, in the case of `time_stamp timestamp "select to_timestamp(:time_stamp, 'yyyymmddhh24miss')"`, EDB\*Loader can't differentiate between datemask and the SQL expression. It treats the third field (SQL expression in the example) as datemask and prepares the SQL expression, which isn't valid. Where: + +- `first field` specifies the column name. +- `second field` specifies the datatype. +- `third field` specifies the datemask. + +If you want to provide an SQL expression, then a workaround is to specify the datemask and SQL expression using the `TO_CHAR` function as: + + ```sql + time_stamp timestamp "yyyymmddhh24miss" "to_char(to_timestamp(:time_stamp, 'yyyymmddhh24miss'), 'yyyymmddhh24miss')" + ``` + +`NULLIF field_condition [ AND field_condition ] ...` + + See the description of `field_condition` in this parameter list for the syntax of `field_condition`. + + If all field conditions evaluate to `TRUE`, then the column identified by `column_name` in the field definition is set to null. If any field condition evaluates to `FALSE`, then the column is set to the appropriate value as normally occurs according to the field definition. + +`PRESERVE BLANKS` + + The `PRESERVE BLANKS` option works only with the `OPTIONALLY ENCLOSED BY` clause and retains leading and trailing whitespaces for both delimited and predetermined size fields. + + In case of `NO PRESERVE BLANKS`, if the fields are delimited, then only leading whitespaces are omitted. If any trailing whitespaces are present, they are left intact. In the case of predetermined-sized fields with `NO PRESERVE BLANKS`, the trailing whitespaces are omitted, and any leading whitespaces are left intact. + + !!! Note + If you don't provide `PRESERVE BLANKS` or `NO PRESERVE BLANKS`, then the behavior defaults to `NO PRESERVE BLANKS`. This option doesn't work for ideographic whitespaces. + +`expr` + + A SQL expression returning a scalar value that's type-compatible with the column data type to which it's assigned in a field definition. Use double quotes around `expr`. `expr` can contain a reference to any column in the field list except for fields with the `FILLER` clause. Prefix the column name using a colon (`:`). + + `expr` can also consist of a SQL `SELECT` statement. If you use a `SELECT` statement: + +- Enclose the `SELECT` statement in parentheses, that is, `(SELECT ...)`. +- The select list must consist of one expression following the `SELECT` keyword. +- The result set must not return more than one row. If no rows are returned, then the returned value of the resulting expression is null. + + The following is the syntax for a `SELECT` statement: + + ```sql + "(SELECT [ FROM [ WHERE ] ])" + ``` + + !!! Note + Omitting the `FROM table_list` clause isn't compatible with Oracle databases. If you don't need to specify any tables, using the `FROM DUAL` clause is compatible with Oracle databases. diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx new file mode 100644 index 00000000000..346082985dd --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx @@ -0,0 +1,484 @@ +--- +title: "EDB* Loader control file examples" +navTitle: "Control file examples" +description: "Provides examples of control files and their corresponding data files" +--- + +The following are some examples of control files and their corresponding data files. + +## Delimiter-separated field data file + +This control file uses a delimiter-separated data file that appends rows to the `emp` table. The `APPEND` clause is used to allow inserting additional rows into the `emp` table. + + +```sql +LOAD DATA + INFILE 'emp.dat' + BADFILE 'emp.bad' + APPEND + INTO TABLE emp + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' + TRAILING NULLCOLS + ( + empno, + ename, + job, + mgr, + hiredate, + sal, + deptno, + comm + ) +``` + +The following is the corresponding delimiter-separated data file: + +```sql +9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20 +9102,PETERSON,SALESMAN,7698,20-DEC-10,2600.00,30,2300.00 +9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00 +9104,"JONES, JR.",MANAGER,7839,02-APR-09,7975.00,20 +``` + +The use of the `TRAILING NULLCOLS` clause allows you to omit the last field supplying the `comm` column from the first and last records. The `comm` column is set to null for the rows inserted from these records. + +Double quotation marks surround the value `JONES, JR.` in the last record since the comma delimiter character is part of the field value. + +This query displays the rows added to the table after the EDB\*Loader session: + +```sql +SELECT * FROM emp WHERE empno > 9100; +__OUTPUT__ + empno| ename | job | mgr| hiredate | sal | comm |deptno +------+-----------+---------+----+-------------------+-------+--------+------ + 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00|1980.00| | 20 + 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 + 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 + 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00|7975.00| | 20 +(4 rows) +``` + +## Fixed-width field data file + +This control file loads the same rows into the `emp` table. It uses a data file containing fixed-width fields. The `FIELDS TERMINATED BY` and `OPTIONALLY ENCLOSED BY` clauses are absent. Instead, each field includes the `POSITION` clause. + +```sql +LOAD DATA + INFILE 'emp_fixed.dat' + BADFILE 'emp_fixed.bad' + APPEND + INTO TABLE emp + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23), + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal POSITION (39:46), + deptno POSITION (47:48), + comm POSITION (49:56) + ) +``` + +The following is the corresponding data file containing fixed-width fields: + +```sql +9101ROGERS CLERK 790217-DEC-10 1980.0020 +9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 +9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 +9104JONES, JR.MANAGER 783902-APR-09 7975.0020 +``` + +## Single physical record data file – RECORDS DELIMITED BY clause + +This control file loads the same rows into the `emp` table but uses a data file with one physical record. Terminate each record loaded as a row in the table using a semicolon (`;`). The `RECORDS DELIMITED BY` clause specifies this value. + +```sql +LOAD DATA + INFILE 'emp_recdelim.dat' + BADFILE 'emp_recdelim.bad' + APPEND + INTO TABLE emp + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' + RECORDS DELIMITED BY ';' + TRAILING NULLCOLS + ( + empno, + ename, + job, + mgr, + hiredate, + sal, + deptno, + comm + ) +``` + +The following is the corresponding data file. The content is a single physical record in the data file. The record delimiter character is included following the last record, that is, at the end of the file. + +```sql +9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20,;9102,PETERSON,SALESMAN,7698,20-DEC-10, +2600.00,30,2300.00;9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00;9104,"JONES, +JR.",MANAGER,7839,02-APR-09,7975.00,20,; +``` + +## FILLER clause + +This control file uses the `FILLER` clause in the data fields for the `sal` and `comm` columns. EDB\*Loader ignores the values in these fields and sets the corresponding columns to null. + +```sql +LOAD DATA + INFILE 'emp_fixed.dat' + BADFILE 'emp_fixed.bad' + APPEND + INTO TABLE emp + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23), + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal FILLER POSITION (39:46), + deptno POSITION (47:48), + comm FILLER POSITION (49:56) + ) +``` + +Using the same fixed-width data file as in the prior fixed-width field example, the resulting rows in the table appear as follows: + +```sql +SELECT * FROM emp WHERE empno > 9100; +__OUTPUT__ + empno| ename | job | mgr| hiredate | sal | comm |deptno +------+-----------+---------+----+-------------------+-------+--------+------ + 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00| | | 20 + 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00| | | 30 + 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00| | | 30 + 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00| | | 20 +(4 rows) +``` + +## BOUNDFILLER clause + +This control file uses the `BOUNDFILLER` clause in the data fields for the `job` and `mgr` columns. EDB\*Loader ignores the values in these fields and sets the corresponding columns to null in the same manner as the `FILLER` clause. However, unlike columns with the `FILLER` clause, you can use columns with the `BOUNDFILLER` clause in an expression, as shown for column `jobdesc`. + +```sql +LOAD DATA + INFILE 'emp.dat' + BADFILE 'emp.bad' + APPEND + INTO TABLE empjob + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' + TRAILING NULLCOLS + ( + empno, + ename, + job BOUNDFILLER, + mgr BOUNDFILLER, + hiredate FILLER, + sal FILLER, + deptno FILLER, + comm FILLER, + jobdesc ":job || ' for manager ' || :mgr" + ) +``` + +The following is the delimiter-separated data file used in this example: + +```sql +9101,ROGERS,CLERK,7902,17-DEC-10,1980.00,20 +9102,PETERSON,SALESMAN,7698,20-DEC-10,2600.00,30,2300.00 +9103,WARREN,SALESMAN,7698,22-DEC-10,5250.00,30,2500.00 +9104,"JONES, JR.",MANAGER,7839,02-APR-09,7975.00,20 +``` + +The following table is loaded using the preceding control file and data file: + +```sql +CREATE TABLE empjob ( + empno NUMBER(4) NOT NULL CONSTRAINT empjob_pk PRIMARY KEY, + ename VARCHAR2(10), + job VARCHAR2(9), + mgr NUMBER(4), + jobdesc VARCHAR2(25) +); +``` + +The resulting rows in the table appear as follows: + +```sql +SELECT * FROM empjob; +__OUTPUT__ + empno | ename | job | mgr | jobdesc +-------+------------+-----+-----+--------------------------- + 9101 | ROGERS | | | CLERK for manager 7902 + 9102 | PETERSON | | | SALESMAN for manager 7698 + 9103 | WARREN | | | SALESMAN for manager 7698 + 9104 | JONES, JR. | | | MANAGER for manager 7839 +(4 rows) +``` + +## Field types with length specification + +This control file contains the field type clauses with the length specification: + +```sql +LOAD DATA + INFILE 'emp_fixed.dat' + BADFILE 'emp_fixed.bad' + APPEND + INTO TABLE emp + TRAILING NULLCOLS + ( + empno CHAR(4), + ename CHAR(10), + job POSITION (15:23) CHAR(9), + mgr INTEGER EXTERNAL(4), + hiredate DATE(11) "DD-MON-YY", + sal DECIMAL EXTERNAL(8), + deptno POSITION (47:48), + comm POSITION (49:56) DECIMAL EXTERNAL(8) + ) +``` + +!!! Note + You can use the `POSITION` clause and the `fieldtype(length)` clause individually or in combination as long as each field definition contains at least one of the two clauses. + +The following is the corresponding data file containing fixed-width fields: + +```sql +9101ROGERS CLERK 790217-DEC-10 1980.0020 +9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 +9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 +9104JONES, JR. MANAGER 783902-APR-09 7975.0020 +``` + +The resulting rows in the table appear as follows: + +```sql +SELECT * FROM emp WHERE empno > 9100; +__OUTPUT__ + empno| ename | job | mgr| hiredate | sal | comm |deptno +------+-----------+---------+----+-------------------+-------+--------+------ + 9101| ROGERS | CLERK |7902| 17-DEC-10 00:00:00|1980.00| | 20 + 9102| PETERSON | SALESMAN|7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 + 9103| WARREN | SALESMAN|7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 + 9104| JONES, JR.| MANAGER |7839| 02-APR-09 00:00:00|7975.00| | 20 +(4 rows) +``` + +## NULLIF clause + +This example uses the `NULLIF` clause on the `sal` column to set it to null for employees of job `MANAGER`. It also uses the clause on the `comm` column to set it to null if the employee isn't a `SALESMAN` and isn't in department `30`. In other words, a `comm` value is accepted if the employee is a `SALESMAN` or is a member of department `30`. + +The following is the control file: + +```sql +LOAD DATA + INFILE 'emp_fixed_2.dat' + BADFILE 'emp_fixed_2.bad' + APPEND + INTO TABLE emp + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23), + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal POSITION (39:46) NULLIF job = 'MANAGER', + deptno POSITION (47:48), + comm POSITION (49:56) NULLIF job <> 'SALESMAN' AND deptno <> '30' + ) +``` + +The following is the corresponding data file: + +```sql +9101ROGERS CLERK 790217-DEC-10 1980.0020 +9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 +9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 +9104JONES, JR. MANAGER 783902-APR-09 7975.0020 +9105ARNOLDS CLERK 778213-SEP-10 3750.0030 800.00 +9106JACKSON ANALYST 756603-JAN-11 4500.0040 2000.00 +9107MAXWELL SALESMAN 769820-DEC-10 2600.0010 1600.00 +``` + +The resulting rows in the table appear as follows: + +```sql +SELECT empno, ename, job, NVL(TO_CHAR(sal),'--null--') "sal", + NVL(TO_CHAR(comm),'--null--') "comm", deptno FROM emp WHERE empno > 9100; +__OUTPUT__ + empno | ename | job | sal | comm | deptno +-------+------------+----------+----------+----------+------- + 9101 | ROGERS | CLERK | 1980.00 | --null-- | 20 + 9102 | PETERSON | SALESMAN | 2600.00 | 2300.00 | 30 + 9103 | WARREN | SALESMAN | 5250.00 | 2500.00 | 30 + 9104 | JONES, JR. | MANAGER | --null-- | --null-- | 20 + 9105 | ARNOLDS | CLERK | 3750.00 | 800.00 | 30 + 9106 | JACKSON | ANALYST | 4500.00 | --null-- | 40 + 9107 | MAXWELL | SALESMAN | 2600.00 | 1600.00 | 10 +(7 rows) +``` + +!!! Note + The `sal` column for employee `JONES, JR.` is null since the job is `MANAGER`. + +The `comm` values from the data file for employees `PETERSON`, `WARREN`, `ARNOLDS`, and `MAXWELL` are all loaded into the `comm` column of the `emp` table since these employees are either `SALESMAN` or members of department `30`. + +The `comm` value of `2000.00` in the data file for employee `JACKSON` is ignored, and the `comm` column of the `emp` table is set to null. This employee isn't a `SALESMAN` or a member of department `30`. + +## SELECT statement in a field expression + +This example uses a `SELECT` statement in the expression of the field definition to return the value to load into the column: + +```sql +LOAD DATA + INFILE 'emp_fixed.dat' + BADFILE 'emp_fixed.bad' + APPEND + INTO TABLE emp + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23) "(SELECT dname FROM dept WHERE deptno = :deptno)", + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal POSITION (39:46), + deptno POSITION (47:48), + comm POSITION (49:56) + ) +``` + +The following is the content of the `dept` table used in the `SELECT` statement: + +```sql +SELECT * FROM dept; +__OUTPUT__ + deptno | dname | loc +---------+------------+--------- + 10 | ACCOUNTING | NEW YORK + 20 | RESEARCH | DALLAS + 30 | SALES | CHICAGO + 40 | OPERATIONS | BOSTON +(4 rows) +``` + +The following is the corresponding data file: + +```sql +9101ROGERS CLERK 790217-DEC-10 1980.0020 +9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 +9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 +9104JONES, JR. MANAGER 783902-APR-09 7975.0020 +``` + +The resulting rows in the table appear as follows: + +```sql +SELECT * FROM emp WHERE empno > 9100; +__OUTPUT__ + empno| ename | job | mgr| hiredate | sal | comm |deptno +------+-----------+---------+----+-------------------+-------+--------+------ + 9101| ROGERS | RESEARCH|7902| 17-DEC-10 00:00:00|1980.00| | 20 + 9102| PETERSON | SALES |7698| 20-DEC-10 00:00:00|2600.00| 2300.00| 30 + 9103| WARREN | SALES |7698| 22-DEC-10 00:00:00|5250.00| 2500.00| 30 + 9104| JONES, JR.| RESEARCH|7839| 02-APR-09 00:00:00|7975.00| | 20 +(4 rows) +``` + +!!! Note + The `job` column contains the value from the `dname` column of the `dept` table returned by the `SELECT` statement instead of the job name from the data file. + +## Multiple INTO TABLE clauses + +This example uses multiple `INTO TABLE` clauses. For this example, two empty tables are created with the same data definition as the `emp` table. The following `CREATE TABLE` commands create these two empty tables, without inserting rows from the original `emp` table: + +```sql +CREATE TABLE emp_research AS SELECT * FROM emp WHERE deptno = 99; +CREATE TABLE emp_sales AS SELECT * FROM emp WHERE deptno = 99; +``` + +This control file contains two `INTO TABLE` clauses. Without an `APPEND` clause, it uses the default operation of `INSERT`. For this operation, the tables `emp_research` and `emp_sales` must be empty. + +```sql +LOAD DATA + INFILE 'emp_multitbl.dat' + BADFILE 'emp_multitbl.bad' + DISCARDFILE 'emp_multitbl.dsc' + INTO TABLE emp_research + WHEN (47:48) = '20' + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23), + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal POSITION (39:46), + deptno CONSTANT '20', + comm POSITION (49:56) + ) + INTO TABLE emp_sales + WHEN (47:48) = '30' + TRAILING NULLCOLS + ( + empno POSITION (1:4), + ename POSITION (5:14), + job POSITION (15:23), + mgr POSITION (24:27), + hiredate POSITION (28:38), + sal POSITION (39:46), + deptno CONSTANT '30', + comm POSITION (49:56) "ROUND(:comm + (:sal * .25), 0)" + ) +``` + +The `WHEN` clauses specify that when the field designated by columns 47 through 48 contains `20`, the record is inserted into the `emp_research` table. When that same field contains `30`, the record is inserted into the `emp_sales` table. If neither condition is true, the record is written to the discard file `emp_multitbl.dsc`. + +The `CONSTANT` clause is given for column `deptno`, so the specified constant value is inserted into `deptno` for each record. When the `CONSTANT` clause is used, it must be the only clause in the field definition other than the column name to which the constant value is assigned. + +Column `comm` of the `emp_sales` table is assigned a SQL expression. Expressions can reference column names by prefixing the column name with a colon character (`:`). + +The following is the corresponding data file: + +```sql +9101ROGERS CLERK 790217-DEC-10 1980.0020 +9102PETERSON SALESMAN 769820-DEC-10 2600.0030 2300.00 +9103WARREN SALESMAN 769822-DEC-10 5250.0030 2500.00 +9104JONES, JR. MANAGER 783902-APR-09 7975.0020 +9105ARNOLDS CLERK 778213-SEP-10 3750.0010 +9106JACKSON ANALYST 756603-JAN-11 4500.0040 +``` + +The records for employees `ARNOLDS` and `JACKSON` contain `10` and `40` in columns 47 through 48, which don't satisfy any of the `WHEN` clauses. EDB\*Loader writes these two records to the discard file, `emp_multitbl.dsc`, with the following content: + +```sql +9105ARNOLDS CLERK 778213-SEP-10 3750.0010 +9106JACKSON ANALYST 756603-JAN-11 4500.0040 +``` + +The following are the rows loaded into the `emp_research` and `emp_sales` tables: + +```sql +SELECT * FROM emp_research; +__OUTPUT__ +empno | ename | job | mgr | hiredate | sal | comm | deptno +-------+------------+---------+------+--------------------+---------+------+------- + 9101 | ROGERS | CLERK | 7902 | 17-DEC-10 00:00:00 | 1980.00 | | 20.00 + 9104 | JONES, JR. | MANAGER | 7839 | 02-APR-09 00:00:00 | 7975.00 | | 20.00 +(2 rows) +``` +```sql +SELECT * FROM emp_sales; +__OUTPUT__ +empno | ename | job | mgr | hiredate | sal | comm | deptno +-------+----------+----------+------+--------------------+---------+---------+-------- + 9102 | PETERSON | SALESMAN | 7698 | 20-DEC-10 00:00:00 | 2600.00 | 2950.00 | 30.00 + 9103 | WARREN | SALESMAN | 7698 | 22-DEC-10 00:00:00 | 5250.00 | 3813.00 | 30.00 +(2 rows) +``` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx new file mode 100644 index 00000000000..ffb9b4f2a02 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx @@ -0,0 +1,102 @@ +--- +title: "Updating a table with a conventional path load" +navTitle: "Conventional path load" +description: "Describes how to use EDB*Loader with a conventional path load to update the rows in a table" +--- + + + +You can use EDB\*Loader with a conventional path load to update the rows in a table, merging new data with the existing data. When you invoke EDB\*Loader to perform an update, the server searches the table for an existing row with a matching primary key: + +- If the server locates a row with a matching key, it replaces the existing row with the new row. +- If the server doesn't locate a row with a matching key, it adds the new row to the table. + +To use EDB\*Loader to update a table, the table must have a primary key. You can't use EDB\*Loader to update a partitioned table. + +To perform `UPDATE`, use the same steps as when performing a conventional path load: + +1. Create a data file that contains the rows you want to update or insert. +2. Define a control file that uses the `INFILE` keyword to specify the name of the data file. For information about building the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). +3. Invoke EDB\*Loader, specifying the database name, connection information, and the name of the control file. For information about invoking EDB\*Loader, see [Invoking EDB\*Loader](#invoking-edbloader). + +This example uses the `emp` table that's distributed with the EDB Postgres Advanced Server sample data. By default, the table contains: + +```sql +edb=# select * from emp; +__OUTPUT__ +empno|ename | job | mgr | hiredate | sal | comm | deptno +-----+------+---------+------+--------------------+--------- +-------+-------- +7369 |SMITH |CLERK | 7902 | 17-DEC-80 00:00:00 | 800.00 | | 20 +7499 |ALLEN |SALESMAN | 7698 | 20-FEB-81 00:00:00 | 1600.00 |300.00 | 30 +7521 |WARD |SALESMAN | 7698 | 22-FEB-81 00:00:00 | 1250.00 |500.00 | 30 +7566 |JONES |MANAGER | 7839 | 02-APR-81 00:00:00 | 2975.00 | | 20 +7654 |MARTIN|SALESMAN | 7698 | 28-SEP-81 00:00:00 | 1250.00 |1400.00| 30 +7698 |BLAKE |MANAGER | 7839 | 01-MAY-81 00:00:00 | 2850.00 | | 30 +7782 |CLARK |MANAGER | 7839 | 09-JUN-81 00:00:00 | 2450.00 | | 10 +7788 |SCOTT |ANALYST | 7566 | 19-APR-87 00:00:00 | 3000.00 | | 20 +7839 |KING |PRESIDENT| | 17-NOV-81 00:00:00 | 5000.00 | | 10 +7844 |TURNER|SALESMAN | 7698 | 08-SEP-81 00:00:00 | 1500.00 | 0.00 | 30 +7876 |ADAMS |CLERK | 7788 | 23-MAY-87 00:00:00 | 1100.00 | | 20 +7900 |JAMES |CLERK | 7698 | 03-DEC-81 00:00:00 | 950.00 | | 30 +7902 |FORD |ANALYST | 7566 | 03-DEC-81 00:00:00 | 3000.00 | | 20 +7934 |MILLER|CLERK | 7782 | 23-JAN-82 00:00:00 | 1300.00 | | 10 +(14 rows) +``` + +This control file (`emp_update.ctl`) specifies the fields in the table in a comma-delimited list. The control file performs an `UPDATE` on the `emp` table. + +```sql +LOAD DATA + INFILE 'emp_update.dat' + BADFILE 'emp_update.bad' + DISCARDFILE 'emp_update.dsc' +UPDATE INTO TABLE emp +FIELDS TERMINATED BY "," +(empno, ename, job, mgr, hiredate, sal, comm, deptno) +``` + +The data that's being updated or inserted is saved in the `emp_update.dat` file. `emp_update.dat` contains: + +```sql +7521,WARD,MANAGER,7839,22-FEB-81 00:00:00,3000.00,0.00,30 +7566,JONES,MANAGER,7839,02-APR-81 00:00:00,3500.00,0.00,20 +7903,BAKER,SALESMAN,7521,10-JUN-13 00:00:00,1800.00,500.00,20 +7904,MILLS,SALESMAN,7839,13-JUN-13 00:00:00,1800.00,500.00,20 +7654,MARTIN,SALESMAN,7698,28-SEP-81 00:00:00,1500.00,400.00,30 +``` + +Invoke EDB\*Loader, specifying the name of the database (`edb`), the name of a database user and their associated password, and the name of the control file (`emp_update.ctl`): + +```shell +edbldr -d edb userid=user_name/password control=emp_update.ctl +``` + +After performing the update, the `emp` table contains: + +```sql +edb=# select * from emp; +__OUTPUT__ +empno|ename | job | mgr | hiredate | sal | comm | deptno +-----+------+---------+------+--------------------+---------+--------+-------- +7369 |SMITH |CLERK | 7902 | 17-DEC-80 00:00:00 | 800.00 | | 20 +7499 |ALLEN |SALESMAN | 7698 | 20-FEB-81 00:00:00 | 1600.00 | 300.00 | 30 +7521 |WARD |MANAGER | 7839 | 22-FEB-81 00:00:00 | 3000.00 | 0.00 | 30 +7566 |JONES |MANAGER | 7839 | 02-APR-81 00:00:00 | 3500.00 | 0.00 | 20 +7654 |MARTIN|SALESMAN | 7698 | 28-SEP-81 00:00:00 | 1500.00 | 400.00 | 30 +7698 |BLAKE |MANAGER | 7839 | 01-MAY-81 00:00:00 | 2850.00 | | 30 +7782 |CLARK |MANAGER | 7839 | 09-JUN-81 00:00:00 | 2450.00 | | 10 +7788 |SCOTT |ANALYST | 7566 | 19-APR-87 00:00:00 | 3000.00 | | 20 +7839 |KING |PRESIDENT| | 17-NOV-81 00:00:00 | 5000.00 | | 10 +7844 |TURNER|SALESMAN | 7698 | 08-SEP-81 00:00:00 | 1500.00 | 0.00 | 30 +7876 |ADAMS |CLERK | 7788 | 23-MAY-87 00:00:00 | 1100.00 | | 20 +7900 |JAMES |CLERK | 7698 | 03-DEC-81 00:00:00 | 950.00 | | 30 +7902 |FORD |ANALYST | 7566 | 03-DEC-81 00:00:00 | 3000.00 | | 20 +7903 |BAKER |SALESMAN | 7521 | 10-JUN-13 00:00:00 | 1800.00 | 500.00 | 20 +7904 |MILLS |SALESMAN | 7839 | 13-JUN-13 00:00:00 | 1800.00 | 500.00 | 20 +7934 |MILLER|CLERK | 7782 | 23-JAN-82 00:00:00 | 1300.00 | | 10 +(16 rows) +``` + +The rows containing information for the three employees that are currently in the `emp` table are updated, while rows are added for the new employees (`BAKER` and `MILLS`). + + diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx new file mode 100644 index 00000000000..795c496e279 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx @@ -0,0 +1,24 @@ +--- +title: "Overview of data loading methods" +navTitle: "Data loading methods" +description: "Description of the data loading methods supported by EDB*Loader" +--- + +As with Oracle SQL\*Loader, EDB\*Loader supports three data loading methods: + +- Conventional path load +- Direct path load +- Parallel direct path load + +Conventional path load is the default method used by EDB\*Loader. Use basic insert processing to add rows to the table. + +The advantage of a conventional path load is that table constraints and database objects defined on the table are enforced during a conventional path load. Table constraints and database objects include primary keys, not null constraints, check constraints, unique indexes, foreign key constraints, triggers, and so on. + +One exception is that the EDB Postgres Advanced Server rules defined on the table aren't enforced. EDB\*Loader can load tables on which rules are defined. However, the rules aren't executed. As a consequence, you can't load partitioned tables implemented using rules with EDB\*Loader. + +!!! Note + Create EDB Postgres Advanced Server rules using the `CREATE RULE` command. EDB Postgres Advanced Server rules aren't the same database objects as rules and rule sets used in Oracle. + +EDB\*Loader also supports direct path loads. A direct path load is faster than a conventional path load but requires removing most types of constraints and triggers from the table. For more information, see [Direct path load](#direct-path-load). + +EDB\*Loader supports parallel direct path loads. A parallel direct path load provides even greater performance improvement by permitting multiple EDB\*Loader sessions to run simultaneously to load a single table. For more information, see [Parallel direct path load](#parallel-direct-path-load). diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx new file mode 100644 index 00000000000..0d5d5251d14 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx @@ -0,0 +1,41 @@ +--- +title: "Running a direct path load" +navTitle: "Direct path load" +description: "Describes how to use EDB\*Loader to write data directly to the database pages" +--- + + + +During a direct path load, EDB\*Loader writes the data directly to the database pages, which is then synchronized to disk. The insert processing associated with a conventional path load is bypassed, resulting in a performance improvement. + +Bypassing insert processing reduces the types of constraints on the target table. The following types of constraints are permitted on the target table of a direct path load: + +- Primary key +- Not null constraints +- Indexes (unique or non-unique) + +The following are restrictions on the target table of a direct path load: + +- Triggers aren't permitted. +- Check constraints aren't permitted. +- Foreign key constraints on the target table referencing another table aren't permitted. +- Foreign key constraints on other tables referencing the target table aren't permitted. +- You must not partition the table. +- Rules can exist on the target table, but they aren't executed. + +!!! Note + Currently, a direct path load in EDB\*Loader is more restrictive than in Oracle SQL\*Loader. The preceding restrictions don't apply to Oracle SQL\*Loader in most cases. The following restrictions apply to a control file used in a direct path load: + + - Multiple table loads aren't supported. You can specify only one `INTO TABLE` clause in the control file. + - You can't use SQL expressions in the data field definitions of the `INTO TABLE` clause. + - The `FREEZE` option isn't supported for direct path loading. + +To run a direct path load, add the `DIRECT=TRUE` option: + +```shell +$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password +CONTROL=emp.ctl DIRECT=TRUE +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. + +Successfully loaded (4) records +``` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx new file mode 100644 index 00000000000..cef7b8f2f7b --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx @@ -0,0 +1,29 @@ +--- +title: "EDB*Loader overview and compatability" +navTitle: "Overview and compatability" +description: "Provides an overview of the key features of EDB*Loader as well as important compatability information" +--- + +EDB\*Loader features include: + +- Support for the Oracle SQL\*Loader data loading methods (conventional path load, direct path load, and parallel direct path load) +- Syntax for control file directives compatible with Oracle SQL\*Loader +- Input data with delimiter-separated or fixed-width fields +- Bad file for collecting rejected records +- Loading of multiple target tables +- Discard file for collecting records that don't meet the selection criteria of any target table +- Log file for recording the EDB\*Loader session and any error messages +- Data loading from standard input and remote loading, particularly useful for large data sources on remote hosts + +!!! Note + The following are important version compatibility restrictions between the EDB\*Loader client and the database server. + + - When you invoke the EDB\*Loader program (called `edbldr`), you pass in parameters and directive information to the database server. We strongly recommend that you use the version 14 EDB\*Loader client (the edbldr program supplied with EDB Postgres Advanced Server 14) to load data only into version 14 of the database server. In general, use the same version for the EDB\*Loader client and database server. + + - Using EDB\*Loader with connection poolers such as PgPool-II and PgBouncer isn't supported. EDB\*Loader must connect directly to EDB Postgres Advanced Server version 14. Alternatively, the following are some of the commands you can use for loading data through connection poolers: + + ```shell + psql \copy + jdbc copyIn + psycopg2 copy_from + ``` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx new file mode 100644 index 00000000000..e66365e1da8 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx @@ -0,0 +1,58 @@ +--- +title: "Loading bulk data" +navTitle: "EDB*Loader" +indexCards: simple +description: "How to use EDB*Loader, the high-performance bulk data loader" +navigation: +- edb_loader_overview_and_restrictions +- data_loading_methods +- using_edb_loader +- building_the_control_file +- control_file_examples +- invoking_edb_loader +- direct_path_load +- parallel_direct_path_load +- performing_remote_loading +- conventional_path_load +legacyRedirectsGenerated: + # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.16.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.15.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.11.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.12.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.08.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.14.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.13.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.10.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.09.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.314.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.315.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.311.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.312.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.313.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.309.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.310.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.308.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-guide/9.5/Database_Compatibility_for_Oracle_Developers_Guide.1.307.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.092.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.093.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.091.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.089.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.087.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.090.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.088.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.086.html" + - "/edb-docs/d/edb-postgres-advanced-server/user-guides/user-guide/9.5/EDB_Postgres_Enterprise_Guide.1.085.html" +redirects: + - ../../epas_compat_tools_guide/02_edb_loader #generated for docs/epas/reorg-role-use-case-mode +--- + +EDB\*Loader is a high-performance bulk data loader that provides an interface compatible with Oracle databases for EDB Postgres Advanced Server. The EDB\*Loader command line utility loads data from an input source, typically a file, into one or more tables using a subset of the parameters offered by Oracle SQL\*Loader. + + + + + + + + diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx new file mode 100644 index 00000000000..99498b05e52 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx @@ -0,0 +1,287 @@ +--- +title: "Invoking EDB*Loader" +description: "Describes how to run EDB*Loader from the command line" +--- + + + +You can run EDB\*Loader as superuser or as a normal user. Use the following command to invoke EDB\*Loader from the command line: + +```sql +edbldr [ -d ] [ -p ] [ -h ] +[ USERID={ | / | | / } ] +[ { -c | connstr= } ] + CONTROL= +[ DATA= ] +[ BAD=] +[ DISCARD= ] +[ DISCARDMAX= ] +[ HANDLE_CONFLICTS={ FALSE | TRUE } ] +[ LOG= ] +[ PARFILE= ] +[ DIRECT={ FALSE | TRUE } ] +[ FREEZE={ FALSE | TRUE } ] +[ ERRORS= ] +[ PARALLEL={ FALSE | TRUE } ] +[ ROWS= ] +[ SKIP= ] +[ SKIP_INDEX_MAINTENANCE={ FALSE | TRUE } ] +[ edb_resource_group= ] +``` + +## Description + +If you omit the `-d` option, the `-p` option, or the `-h` option, the defaults for the database, port, and host are determined by the same rules as other EDB Postgres Advanced Server utility programs, such as `edb-psql`. + +You can specify parameters listed in the syntax diagram in a *parameter file*. Exeptions include the `-d` option, `-p` option, `-h` option, and the `PARFILE` parameter. Specify the parameter file on the command line when you invoke `edbldr` using `PARFILE=param_file`. You can specify some parameters in the `OPTIONS` clause in the control file. For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +You can include the full directory path or a relative directory path to the file name when specifying `control_file`, `data_file`, `bad_file`, `discard_file`, `log_file`, and `param_file`. If you specify the file name alone or with a relative directory path, the file is assumed to exist in the case of `control_file`, `data_file`, or `param_file` relative to the current working directory from which `edbldr` is invoked. In the case of `bad_file`, `discard_file`, or `log_file`, the file is created. + +!!! Note + The control file must exist in the character set encoding of the client where `edbldr` is invoked. If the client is in an encoding different from the database encoding, then you must set the `PGCLIENTENCODING` environment variable on the client to the client’s encoding prior to invoking `edbldr`. This technique ensures character set conversion between the client and the database server is done correctly. + +The operating system account used to invoke `edbldr` must have read permission on the directories and files specified by `control_file`, `data_file`, and `param_file`. + +The operating system account `enterprisedb` must have write permission on the directories where `bad_file`, `discard_file`, and `log_file` are written. + +!!! Note + The file names for `control_file`, `data_file`, `bad_file`, `discard_file`, and `log_file` must include the extensions `.ctl`, `.dat`, `.bad`, `.dsc`, and `.log`, respectively. If the provided file name doesn't contain an extension, EDB\*Loader assumes the actual file name includes the appropriate extension. + +## Parameters + +`dbname` + + Name of the database containing the tables to load. + +`port` + + Port number on which the database server is accepting connections. + +`host` + + IP address of the host on which the database server is running. + +`USERID={ username/password | username/ | username | / }` + + EDB\*Loader connects to the database with `username`. `username` must be a superuser or a username with the required privileges. `password` is the password for `username`. + + If you omit the `USERID` parameter, EDB\*Loader prompts for `username` and `password`. If you specify `USERID=username/`, then EDB\*Loader either: + - Uses the password file specified by environment variable `PGPASSFILE` if `PGPASSFILE` is set + - Uses the `.pgpass` password file (`pgpass.conf` on Windows systems) if `PGPASSFILE` isn't set + + If you specify `USERID=username`, then EDB\*Loader prompts for `password`. If you specify `USERID=/`, the connection is attempted using the operating system account as the user name. + + !!! Note + EDB\*Loader ignores the EDB Postgres Advanced Server connection environment variables `PGUSER` and `PGPASSWORD`. See the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/libpq-pgpass.html) for information on the `PGPASSFILE` environment variable and the password file. + +`-c CONNECTION_STRING` + +`connstr=CONNECTION_STRING` + + The `-c` or `connstr=` option allows you to specify all the connection parameters supported by libpq. With this option, you can also specify SSL connection parameters or other connection parameters supported by libpq. If you provide connection options such as `-d`, `-h`, `-p` or `userid=dbuser/dbpass` separately, they might override the values provided by the `-c` or `connstr=` option. + +`CONTROL=control_file` + + `control_file` specifies the name of the control file containing EDB\*Loader directives. If you don't specify a file extension, an extension of `.ctl` is assumed. + + For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +`DATA=data_file` + + `data_file` specifies the name of the file containing the data to load into the target table. If you don't specify a file extension, an extension of `.dat` is assumed. Specifying a `data_file` on the command line overrides the `INFILE` clause specified in the control file. + + For more information about `data_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +`BAD=bad_file` + + `bad_file` specifies the name of a file that receives input data records that can't be loaded due to errors. Specifying a `bad_file` on the command line overrides any `BADFILE` clause specified in the control file. + + For more information about `bad_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +`DISCARD=discard_file` + + `discard_file` is the name of the file that receives input data records that don't meet any table’s selection criteria. Specifying a `discard_file` on the command line overrides the `DISCARDFILE` clause in the control file. + + For more information about `discard_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +`DISCARDMAX=max_discard_recs` + + `max_discard_recs` is the maximum number of discarded records that can be encountered from the input data records before terminating the EDB\*Loader session. Specifying `max_discard_recs` on the command line overrides the `DISCARDMAX` or `DISCARDS` clause in the control file. + + For more information about `max_discard_recs`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + +`HANDLE_CONFLICTS={ FALSE | TRUE }` + + If any record insertion fails due to a unique constraint violation, EDB\*Loader aborts the entire operation. You can instruct EDB\*Loader to instead move the duplicate record to the `BAD` file and continue processing by setting `HANDLE_CONFLICTS` to `TRUE`. This behavior applies only if indexes are present. By default, `HANDLE_CONFLICTS` is set to `FALSE`. + + Setting `HANDLE_CONFLICTS` to `TRUE` isn't supported with direct path loading. If you set this parameter to `TRUE` when direct path loading, EDB\*Loader throws an error. + +`LOG=log_file` + + `log_file` specifies the name of the file in which EDB\*Loader records the results of the EDB\*Loader session. + + If you omit the `LOG` parameter, EDB\*Loader creates a log file with the name `control_file_base.log` in the directory from which `edbldr` is invoked. `control_file_base` is the base name of the control file used in the EDB\*Loader session. The operating system account `enterprisedb` must have write permission on the directory where the log file is written. + +`PARFILE=param_file` + + `param_file` specifies the name of the file that contains command line parameters for the EDB\*Loader session. You can specify command line parameters listed in this section in `param_file` instead of on the command line. Exceptions are the `-d`, `-p`, and `-h` options, and the `PARFILE` parameter. + + Any parameter given in `param_file` overrides the same parameter supplied on the command line before the `PARFILE` option. Any parameter given on the command line that appears after the `PARFILE` option overrides the same parameter given in `param_file`. + + !!! Note + Unlike other EDB\*Loader files, there's no default file name or extension assumed for `param_file`. However, by Oracle SQL\*Loader convention, `.par` is typically used as an extension. It isn't required. + +`DIRECT= { FALSE | TRUE }` + + If `DIRECT` is set to `TRUE` EDB\*Loader performs a direct path load instead of a conventional path load. The default value of `DIRECT` is `FALSE`. + + Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or might contain references to rows that were deleted. EnterpriseDB doesn't support direct inserts to load data into replicated tables. + + For information about direct path loads, see [Direct path load](#direct-path-load). + +`FREEZE= { FALSE | TRUE }` + + Set `FREEZE` to `TRUE` to copy the data with the rows `frozen`. A tuple guaranteed to be visible to all current and future transactions is marked as frozen to prevent transaction ID wraparound. For more information about frozen tuples, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/routine-vacuuming.html). + + You must specify a data-loading type of `TRUNCATE` in the control file when using the `FREEZE` option. `FREEZE` isn't supported for direct loading. + + By default, `FREEZE` is `FALSE`. + +`ERRORS=error_count` + + `error_count` specifies the number of errors permitted before aborting the EDB\*Loader session. The default is `50`. + +`PARALLEL= { FALSE | TRUE }` + + Set `PARALLEL` to `TRUE` to indicate that this EDB\*Loader session is one of a number of concurrent EDB\*Loader sessions participating in a parallel direct path load. The default value of `PARALLEL` is `FALSE`. + + When `PARALLEL` is `TRUE`, the `DIRECT` parameter must also be set to `TRUE`. + + For more information about parallel direct path loads, see [Parallel direct path load](#parallel-direct-path-load). + +`ROWS=n` + + `n` specifies the number of rows that EDB\*Loader commits before loading the next set of `n` rows. + +`SKIP=skip_count` + + Number of records at the beginning of the input data file to skip before loading begins. The default is `0`. + +`SKIP_INDEX_MAINTENANCE= { FALSE | TRUE }` + + If set to `TRUE`, index maintenance isn't performed as part of a direct path load, and indexes on the loaded table are marked as invalid. The default value of `SKIP_INDEX_MAINTENANCE` is `FALSE`. + + During a parallel direct path load, target table indexes aren't updated. They are marked as invalid after the load is complete. + + You can use the `REINDEX` command to rebuild an index. For more information about the `REINDEX` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-reindex.htm). + +`edb_resource_group=group_name` + + `group_name` specifies the name of an EDB Resource Manager resource group to which to assign the EDB\*Loader session. + + Any default resource group that was assigned to the session is overridden by the resource group given by the `edb_resource_group` parameter specified on the `edbldr` command line. An example of such a group is a database user running the EDB\*Loader session who was assigned a default resource group with the `ALTER ROLE ... SET` `edb_resource_group` command. + +## Examples + +This example invokes EDB\*Loader using a control file named `emp.ctl` to load a table in database `edb`. The file is located in the current working directory. + +```shell +$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password +CONTROL=emp.ctl +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. + +Successfully loaded (4) records +``` + +In this example, EDB\*Loader prompts for the user name and password since they are omitted from the command line. In addition, the files for the bad file and log file are specified with the `BAD` and `LOG` command line parameters. + +```shell +$ /usr/edb/as14/bin/edbldr -d edb CONTROL=emp.ctl BAD=/tmp/emp.bad +LOG=/tmp/emp.log +Enter the user name : enterprisedb +Enter the password : +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. + +Successfully loaded (4) records +``` + +This example runs EDB\*Loader using a parameter file located in the current working directory. The `SKIP` and `ERRORS` parameter default values are specified in the parameter file in addition the `CONTROL`, `BAD`, and `LOG` files. The parameter file, `emp.par`, contains the following: + +```ini +CONTROL=emp.ctl +BAD=/tmp/emp.bad +LOG=/tmp/emp.log +SKIP=1 +ERRORS=10 +``` + +Invoke EDB\*Loader with the parameter file: + +```shell +$ /usr/edb/as14/bin/edbldr -d edb PARFILE=emp.par +Enter the user name : enterprisedb +Enter the password : +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. + +Successfully loaded (3) records +``` + +This example invokes EDB\*Loader using a `connstr=` option that uses the `emp.ctl` control file located in the current working directory to load a table in a database named `edb`: + +```shell +$ /usr/edb/as14/bin/edbldr connstr=\"sslmode=verify-ca sslcompression=0 +host=127.0.0.1 dbname=edb port=5444 user=enterprisedb\" CONTROL=emp.ctl +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. + +Successfully loaded (4) records +``` + +This example invokes EDB\*Loader using a normal user. For this example, one empty table `bar` is created and a normal user `bob` is created. The `bob` user is granted all privileges on the table `bar`. The CREATE TABLE command creates the empty table. The CREATE USER command creates the user and the GRANT command gives required privileges to the user `bob` on the `bar` table: + +```sql +CREATE TABLE bar(i int); +CREATE USER bob identified by '123'; +GRANT ALL on bar TO bob; +``` + +The control file and data file: + +```shell +## Control file +EDBAS/ - (master) $ cat /tmp/edbldr.ctl +LOAD DATA INFILE '/tmp/edbldr.dat' +truncate into table bar +( +i position(1:1) +) + +## Data file +EDBAS/ - (master) $ cat /tmp/edbldr.dat +1 +2 +3 +5 +``` + +Invoke EDB*Loader: + +```shell +EDBAS/ - (master) $ /usr/edb/as15/bin/edbldr -d edb userid=bob/123 control=/tmp/edbldr.ctl +EDB*Loader: Copyright (c) 2007-2022, EnterpriseDB Corporation. + +Successfully loaded (4) records +``` + + + +## Exit codes + +When EDB\*Loader exits, it returns one of the following codes: + +| Exit code | Description | +| --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | Indicates that all rows loaded successfully. | +| `1` | Indicates that EDB\*Loader encountered command line or syntax errors or aborted the load operation due to an unrecoverable error. | +| `2` | Indicates that the load completed, but some (or all) rows were rejected or discarded. | +| `3` | Indicates that EDB\*Loader encountered fatal errors (such as OS errors). This class of errors is equivalent to the `FATAL` or `PANIC` severity levels of PostgreSQL errors. | + diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx new file mode 100644 index 00000000000..462108389d2 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx @@ -0,0 +1,147 @@ +--- +title: "Running a parallel direct path load" +navTitle: "Parallel direct path load" +description: "Describes how to run a parallel direct path load to distribute the loading process over two or more sessions" +--- + + + +You can further improve the performance of a direct path load by distributing the loading process over two or more sessions running concurrently. Each session runs a direct path load into the same table. + +Since the same table is loaded from multiple sessions, the input records to load into the table must be divided among several data files. This way, each EDB\*Loader session uses its own data file, and the same record isn't loaded into the table more than once. + +The target table of a parallel direct path load is under the same restrictions as a direct path load run in a single session. + +The following are the restrictions on the target table of a direct path load: + +- Triggers aren't permitted. +- Check constraints aren't permitted. +- Foreign key constraints on the target table referencing another table aren't permitted. +- Foreign key constraints on other tables referencing the target table aren't permitted. +- You must not partition the table. +- Rules can exist on the target table, but they aren't executed. + +In addition, you must specify the `APPEND` clause in the control file used by each EDB\*Loader session. + +To run a parallel direct path load, run EDB\*Loader in a separate session for each participant of the parallel direct path load. You must include the `DIRECT=TRUE` and `PARALLEL=TRUE` parameters when invoking each such EDB\*Loader session. + +Each EDB\*Loader session runs as an independent transaction. Aborting and rolling back changes of one of the parallel sessions doesn't affect the loading done by the other parallel sessions. + +!!! Note + In a parallel direct path load, each EDB\*Loader session reserves a fixed number of blocks in the target table using turns. Some of the blocks in the last allocated chunk might not be used, and those blocks remain uninitialized. A later use of the `VACUUM` command on the target table might show warnings about these uninitialized blocks, such as the following: + +```text +WARNING: relation "emp" page 98264 is uninitialized --- fixing + +WARNING: relation "emp" page 98265 is uninitialized --- fixing + +WARNING: relation "emp" page 98266 is uninitialized --- fixing +``` + +This behavior is expected and doesn't indicate data corruption. + +Indexes on the target table aren't updated during a parallel direct path load. They are therefore marked as invalid after the load is complete. You must use the `REINDEX` command to rebuild the indexes. + +This example shows the use of a parallel direct path load on the `emp` table. + +!!! Note + If you attempt a parallel direct path load on the sample `emp` table provided with EDB Postgres Advanced Server, you must first remove the triggers and constraints referencing the `emp` table. In addition, the primary key column, `empno`, was expanded from `NUMBER(4)` to `NUMBER` in this example to allow for inserting more rows. + +This is the control file used in the first session: + +```sql +LOAD DATA + INFILE '/home/user/loader/emp_parallel_1.dat' + APPEND + INTO TABLE emp + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' + TRAILING NULLCOLS + ( + empno, + ename, + job, + mgr, + hiredate, + sal, + deptno, + comm + ) +``` + +You must specify the `APPEND` clause in the control file for a parallel direct path load. + +This example invokes EDB\*Loader in the first session. You must specify the `DIRECT=TRUE` and `PARALLEL=TRUE` parameters. + +```shell +$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password +CONTROL=emp_parallel_1.ctl DIRECT=TRUE PARALLEL=TRUE +WARNING: index maintenance will be skipped with PARALLEL load +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. +``` + +The control file used for the second session appears as follows. It's the same as the one used in the first session, but it uses a different data file. + +```sql +LOAD DATA + INFILE '/home/user/loader/emp_parallel_2.dat' + APPEND + INTO TABLE emp + FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' + TRAILING NULLCOLS + ( + empno, + ename, + job, + mgr, + hiredate, + sal, + deptno, + comm + ) +``` + +This control file is used in a second session: + +```shell +$ /usr/edb/as14/bin/edbldr -d edb USERID=enterprisedb/password +CONTROL=emp_parallel_2.ctl DIRECT=TRUE PARALLEL=TRUE +WARNING: index maintenance will be skipped with PARALLEL load +EDB*Loader: Copyright (c) 2007-2021, EnterpriseDB Corporation. +``` + +EDB\*Loader displays the following message in each session when the load operation completes: + +```text +Successfully loaded (10000) records +``` + +This query shows that the index on the emp table was marked `INVALID`: + +```sql +SELECT index_name, status FROM user_indexes WHERE table_name = 'EMP'; +__OUTPUT__ + index_name | status +-------------+--------- + EMP_PK | INVALID + (1 row) +``` + +!!! Note + `user_indexes` is the view of indexes compatible with Oracle databases owned by the current user. + +Queries on the `emp` table don't use the index unless you rebuild it using the `REINDEX` command: + +```sql +REINDEX INDEX emp_pk; +``` + +A later query on `user_indexes` shows that the index is now marked as `VALID`: + +```sql +SELECT index_name, status FROM user_indexes WHERE table_name = 'EMP'; +__OUTPUT__ + index_name | status +-------------+-------- + EMP_PK | VALID + (1 row) +``` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx new file mode 100644 index 00000000000..5515638df3a --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx @@ -0,0 +1,35 @@ +--- +title: "Performing remote loading" +navTitle: "Remote loading" +description: "Describes how to load a database running on a database server using data piped from a different source" +--- + + + +EDB\*Loader supports a feature called *remote loading*. In remote loading, the database containing the table to load is running on a database server on a host different from the one where EDB\*Loader is invoked with the input data source. + +This feature is useful if you have a large amount of data to load, and you don't want to create a large data file on the host running the database server. + +In addition, you can use the standard input feature to pipe the data from the data source, such as another program or script, directly to EDB\*Loader. EDB\*Loader then loads the table in the remote database. This feature bypasses having to create a data file on disk for EDB\*Loader. + +Performing remote loading along with using standard input requires the following: + +- The `edbldr` program must be installed on the client host on which to invoke it with the data source for the EDB\*Loader session. +- The control file must contain the clause `INFILE 'stdin'` so you can pipe the data directly into EDB\*Loader’s standard input. For information on the `INFILE` clause and the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). +- All files used by EDB\*Loader, such as the control file, bad file, discard file, and log file, must reside on or be created on the client host on which `edbldr` is invoked. +- When invoking EDB\*Loader, use the `-h` option to specify the IP address of the remote database server. For more information, see [Invoking EDB\*Loader](#invoking-edbloader). +- Use the operating system pipe operator (`|`) or input redirection operator (`<`) to supply the input data to EDB\*Loader. + +This example loads a database running on a database server at `192.168.1.14` using data piped from a source named `datasource`: + +```shell +datasource | ./edbldr -d edb -h 192.168.1.14 USERID=enterprisedb/password +CONTROL=remote.ctl +``` + +This example also shows how you can use standard input: + +```shell +./edbldr -d edb -h 192.168.1.14 USERID=enterprisedb/password +CONTROL=remote.ctl < datasource +``` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/using_edb_loader.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/using_edb_loader.mdx new file mode 100644 index 00000000000..a2cc9db8437 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/using_edb_loader.mdx @@ -0,0 +1,30 @@ +--- +title: "Using EDB*Loader" +description: "Describes the general usage of the EDB*Loader utility" +--- + + +EDB\*Loader can load data files with either delimiter-separated or fixed-width fields in single-byte or multibyte character sets. The delimiter can be a string consisting of one or more single-byte or multibyte characters. Data file encoding and the database encoding can differ. Character set conversion of the data file to the database encoding is supported. + +Each EDB\*Loader session runs as a single, independent transaction. If an error occurs during the EDB\*Loader session that aborts the transaction, all changes made during the session are rolled back. + +Generally, formatting errors in the data file don't result in an aborted transaction. Instead, the badly formatted records are written to a text file called the *bad file*. The reason for the error is recorded in the *log file*. + +Records causing database integrity errors result in an aborted transaction and rollback. As with formatting errors, the record causing the error is written to the bad file and the reason is recorded in the log file. + +!!! Note + EDB\*Loader differs from Oracle SQL\*Loader in that a database integrity error results in a rollback in EDB\*Loader. In Oracle SQL\*Loader, only the record causing the error is rejected. Records that were previously inserted into the table are retained, and loading continues after the rejected record. + +The following are examples of types of formatting errors that don't abort the transaction: + +- Attempt to load non-numeric value into a numeric column +- Numeric value too large for a numeric column +- Character value too long for the maximum length of a character column +- Attempt to load improperly formatted date value into a date column + +The following are examples of types of database errors that abort the transaction and result in the rollback of all changes made in the EDB\*Loader session: + +- Violating a unique constraint such as a primary key or unique index +- Violating a referential integrity constraint +- Violating a check constraint +- Error thrown by a trigger fired as a result of inserting rows \ No newline at end of file diff --git a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx index 4a1e32679fb..3ff09c347aa 100644 --- a/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx +++ b/product_docs/docs/epas/15/upgrading/04_upgrading_an_installation_with_pg_upgrade/index.mdx @@ -23,8 +23,6 @@ While minor upgrades between versions are fairly simple and require only install Using `pg_upgrade` can reduce both the amount of time and the disk space required for many major-version upgrades. -The following topics walk you through the process of upgrading with `pg_upgrade`. -
performing_an_upgrade invoking_pg_upgrade upgrading_to_advanced_server upgrading_a_pgAgent_installation pg_upgrade_troubleshooting reverting_to_the_old_cluster diff --git a/product_docs/docs/epas/15/upgrading/index.mdx b/product_docs/docs/epas/15/upgrading/index.mdx index 0d4d5fd08a5..097888a1cff 100644 --- a/product_docs/docs/epas/15/upgrading/index.mdx +++ b/product_docs/docs/epas/15/upgrading/index.mdx @@ -11,5 +11,3 @@ Upgrading EDB Postgres Advanced Server involves: - `pg_upgrade` to upgrade from an earlier version of EDB Postgres Advanced Server to the latest version. - `yum` to perform a minor version upgrade on a Linux host. - `StackBuilder Plus` to perform a minor version upgrade on a Windows host. - -The following topics provide instructions for upgrading EDB Postgres Advanced Server. \ No newline at end of file From 2d1bd84f7bf380ea3587f0157920d2ed96a562ae Mon Sep 17 00:00:00 2001 From: francoughlin Date: Tue, 11 Jul 2023 13:27:12 -0400 Subject: [PATCH 04/13] EPAS reorg: Phase 2 EDB Clone Schema restructure Added new overview topics, broke out all sections into separate topics, added descriptions to all topics --- .../edb_loader_overview_and_restrictions.mdx | 4 +- .../edb_resource_manager_key_concepts.mdx | 3 +- .../14_edb_clone_schema.mdx | 1098 ----------------- .../checking_the_status.mdx | 45 + .../copying_a_remote_schema.mdx | 324 +++++ .../14_edb_clone_schema/copying_a_schema.mdx | 306 +++++ .../edb_clone_schema_overview.mdx | 80 ++ .../14_edb_clone_schema/index.mdx | 25 + .../performing_cleanup_tasks.mdx | 68 + .../setting_up_edb_clone_schema.mdx | 302 +++++ 10 files changed, 1154 insertions(+), 1101 deletions(-) delete mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/checking_the_status.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_schema.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/index.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/performing_cleanup_tasks.mdx create mode 100644 product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx index cef7b8f2f7b..6b7d53ef263 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/edb_loader_overview_and_restrictions.mdx @@ -1,6 +1,6 @@ --- -title: "EDB*Loader overview and compatability" -navTitle: "Overview and compatability" +title: "EDB*Loader key concepts and compatability" +navTitle: "Key concepts and compatability" description: "Provides an overview of the key features of EDB*Loader as well as important compatability information" --- diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx index d8d67277cf3..6d56b7d0947 100644 --- a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/edb_resource_manager_key_concepts.mdx @@ -1,5 +1,6 @@ --- -title: "Resource Manager key concepts" +title: "EDB Resource Manager key concepts" +navTitle: "Key concepts" description: "Describes key points about using EDB Resource Manager" --- diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema.mdx deleted file mode 100644 index a51a5b0d7ba..00000000000 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema.mdx +++ /dev/null @@ -1,1098 +0,0 @@ ---- -title: "Copying a database" -navTitle: "EDB clone schema" -description: "How to use the EDB Clone Schema module to copy a schema and its database objects from a source database to a target database" -redirects: - - ../../epas_guide/14_edb_clone_schema #generated for docs/epas/reorg-role-use-case-mode ---- - - - -EDB Clone Schema is an extension module for EDB Postgres Advanced Server that allows you to copy a schema and its database objects from a local or remote database (the source database) to a receiving database (the target database). - -The source and target databases can be either: - -- The same physical database -- Different databases in the same database cluster -- Separate databases running under different database clusters on separate database server hosts - -Use the following functions with EDB Clone Schema: - -- `localcopyschema`. This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](#localcopyschema) for more information. -- `localcopyschema_nb`. This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](#localcopyschema_nb) for more information. -- `remotecopyschema`. This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](#remotecopyschema) for more information. -- `remotecopyschema_nb`. This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](#remotecopyschema_nb) for more information. -- `process_status_from_log`. This function displays the status of the cloning functions. The information is obtained from a log file you specify when invoking a cloning function. See [process_status_from_log](#process_status_from_log) for more information. -- `remove_log_file_and_job`. This function deletes the log file created by a cloning function. You can also use this function to delete a job created by the non-blocking form of the function. See [remove_log_file_and_job](#remove_log_file_and_job) for more information. - -You can clone these database objects from one schema to another: - -- Data types -- Tables including partitioned tables, excluding foreign tables -- Indexes -- Constraints -- Sequences -- View definitions -- Materialized views -- Private synonyms -- Table triggers, but excluding event triggers -- Rules -- Functions -- Procedures -- Packages -- Comments for all supported object types -- Access control lists (ACLs) for all supported object types - -You can't clone the following database objects: - -- Large objects (Postgres `LOBs` and `BFILEs`) -- Logical replication attributes for a table -- Database links -- Foreign data wrappers -- Foreign tables -- Event triggers -- Extensions - -For cloning objects that rely on extensions, see the limitations that follow. - -- Row-level security -- Policies -- Operator class - -The following limitations apply: - -- EDB Clone Schema is supported on EDB Postgres Advanced Server when you specify a dialect of **Compatible with Oracle** on the EDB Postgres Advanced Server Dialect dialog box during installation. It's also supported when you include the `--redwood-like` keywords during a text-mode installation or cluster initialization. -- The source code in functions, procedures, triggers, packages, and so on, aren't modified after being copied to the target schema. If such programs contain coded references to objects with schema names, the programs might fail when invoked in the target schema if such schema names are no longer consistent in the target schema. -- Cross-schema object dependencies aren't resolved. If an object in the target schema depends on an object in another schema, this dependency isn't resolved by the cloning functions. -- For remote cloning, if an object in the source schema depends on an extension, then you must create this extension in the public schema of the remote database before invoking the remote cloning function. -- At most, 16 copy jobs can run in parallel to clone schemas. Each job can have at most 16 worker processes to copy table data in parallel. -- You can't cancel queries run by background workers. - -## Setup process - -You must install several extensions along with the PL/Perl language on any database used as the source or target database by an EDB Clone Schema function. - -In addition, it might help to modify some configuration parameters in the `postgresql.conf` file of the database servers. - -### Installing extensions and PL/Perl - -Perform this installation on any database to be used as the source or target database by an EDB Clone Schema function. - -1. Install the following extensions on the database: - - - `postgres_fdw` - - `dblink` - - `adminpack` - - `pgagent` - - Ensure that pgAgent is installed before creating the `pgagent` extension. On Linux, you can use the `edb-asxx-pgagent` RPM package, where `xx` is the EDB Postgres Advanced Server version number to install pgAgent. On Windows, use StackBuilder Plus to download and install pgAgent. - - You can install the extensions using the following commands: - - ```sql - CREATE EXTENSION postgres_fdw SCHEMA public; - CREATE EXTENSION dblink SCHEMA public; - CREATE EXTENSION adminpack; - CREATE EXTENSION pgagent; - ``` - - For more information about using the `CREATE EXTENSION` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createextension.html). - -1. Modify the `postgresql.conf` file by adding `$libdir/parallel_clone` to the `shared_preload_libraries` configuration parameter as shown in this example: - - ```ini - shared_preload_libraries = '$libdir/dbms_pipe,$libdir/dbms_aq,$libdir/parallel_clone' - ``` - -1. Install the Perl procedural language (PL/Perl) on the database, and run the `CREATE TRUSTED LANGUAGE plperl` command. For Linux, install PL/Perl using the `edb-asxx-server-plperl` RPM package, where `xx` is the EDB Postgres Advanced Server version number. For Windows, use the EDB Postgres Language Pack. For information on EDB Language Pack, see the [EDB Postgres Language Pack](/language_pack/latest). - -1. Connect to the database as a superuser and run the following command: - - ```sql - CREATE TRUSTED LANGUAGE plperl; - ``` - -For more information about using the `CREATE LANGUAGE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createlanguage.html). - - - -### Setting configuration parameters - -You might need to modify configuration parameters in the `postgresql.conf` file. - -#### Performance configuration parameters - -You might need to tune the system for copying a large schema as part of one transaction. Tuning of configuration parameters is for the source database server referenced in a cloning function. - -You might need to tune the following configuration parameters in the `postgresql.conf` file: - -- `work_mem`. Specifies the amount of memory for internal sort operations and hash tables to use before writing to temporary disk files. -- `maintenance_work_mem`. Specifies the maximum amount of memory for maintenance operations such as `VACUUM`, `CREATE INDEX`, and `ALTER TABLE ADD FOREIGN KEY` to use. -- `max_worker_processes`. Sets the maximum number of background processes that the system can support. -- `checkpoint_timeout`. Maximum time between automatic WAL checkpoints, in seconds. -- `checkpoint_completion_target`. Specifies the target of checkpoint completion as a fraction of total time between checkpoints. -- `checkpoint_flush_after`. Whenever more than `checkpoint_flush_after` bytes are written while performing a checkpoint, attempt to force the OS to issue these writes to the underlying storage. -- `max_wal_size`. Maximum size to let the WAL grow to between automatic WAL checkpoints. -- `max_locks_per_transaction`. Controls the average number of object locks allocated for each transaction. Individual transactions can lock more objects as long as the locks of all transactions fit in the lock table. - -For information about the configuration parameters, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/runtime-config.html). - -#### Status logging - -Status logging by the cloning functions creates log files in the directory specified by the `log_directory` parameter in the `postgresql.conf` file for the database server to which you're connected when invoking the cloning function. - -The default location is `PGDATA/log`, as shown by the following: - -```ini -#log_directory = 'log' # directory where log files are written, - # can be absolute or relative to PGDATA -``` - -This directory must exist before running a cloning function. - -The name of the log file is determined by what you specify in the parameter list when invoking the cloning function. - -To display the status from a log file, use the `process_status_from_log` function. - -To delete a log file, use the `remove_log_file_and_job` function, or delete it manually from the log directory. - -### Installing EDB Clone Schema - -Install the EDB Clone Schema on any database to be used as the source or target database by an EDB Clone Schema function. - -1. If you previously installed an older version of the `edb_cloneschema` extension, run the following command: - -**Step 1:** If you had previously installed an older version of the `edb_cloneschema` extension, then you must run the following command: - -```sql -DROP EXTENSION parallel_clone CASCADE; -``` - - This command also drops the `edb_cloneschema` extension. - -1. Install the extensions. Make sure that you create the `parallel_clone` extension before creating the `edb_cloneschema` extension. - - ```sql - CREATE EXTENSION parallel_clone SCHEMA public; - - CREATE EXTENSION edb_cloneschema; - ``` - -### Creating the foreign servers and user mappings - -When using one of the local cloning functions `localcopyschema` or `localcopyschema_nb`, one of the required parameters includes a single, foreign server. This server is for identifying the database server and its database that's the source and receiver of the cloned schema. - -When using one of the remote cloning functions `remotecopyschema` or `remotecopyschema_nb`, two of the required parameters include two foreign servers. The foreign server specified as the first parameter identifies the source database server and its database that's the provider of the cloned schema. The foreign server specified as the second parameter identifies the target database server and its database that's the receiver of the cloned schema. - -For each foreign server, you must create a user mapping. When a selected database superuser invokes a cloning function, that superuser must be mapped to a database user name and password that has access to the foreign server that's specified as a parameter in the cloning function. - -For general information about foreign data, foreign servers, and user mappings, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/ddl-foreign-data.html). - -#### Foreign server and user mapping for local cloning functions - -For the `localcopyschema` and `localcopyschema_nb` functions, the source and target schemas are both in the same database of the same database server. Thus, you must define and specify only one foreign server for these functions. This foreign server is also referred to as the *local server* because this server is the one to which you must be connected when invoking the `localcopyschema` or `localcopyschema_nb` function. - -The user mapping defines the connection and authentication information for the foreign server. You must create this foreign server and user mapping in the database of the local server in which the cloning occurs. - -The database user for whom the user mapping is defined must be a superuser and connected to the local server when invoking an EDB Clone Schema function. - -This example creates the foreign server for the database containing the schema to clone and to receive the cloned schema: - -```sql -CREATE SERVER local_server FOREIGN DATA WRAPPER postgres_fdw - OPTIONS( - host 'localhost', - port '5444', - dbname 'edb' -); -``` - -For more information about using the `CREATE SERVER` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createserver.html). - -The user mapping for this server is the following: - -```sql -CREATE USER MAPPING FOR enterprisedb SERVER local_server - OPTIONS ( - user 'enterprisedb', - password 'password' -); -``` - -For more information about using the `CREATE USER MAPPING` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/sql-createusermapping.html). - -The following `psql` commands show the foreign server and user mapping: - -```sql -edb=# \des+ -__OUTPUT__ -List of foreign servers --[ RECORD 1 ]--------+---------------------------------------------- -Name | local_server -Owner | enterprisedb -Foreign-data wrapper | postgres_fdw -Access privileges | -Type | -Version | -FDW options | (host 'localhost', port '5444', dbname 'edb') -Description | -``` -```sql -edb=# \deu+ -__OUTPUT__ - List of user mappings - Server | User name | FDW options ---------------+--------------+---------------------------------------------- - local_server | enterprisedb | ("user" 'enterprisedb', password 'password') -(1 row) -``` - -When database superuser `enterprisedb` invokes a cloning function, the database user `enterprisedb` with its password is used to connect to `local_server` on the `localhost` with port `5444` to database `edb`. - -In this case, the mapped database user, `enterprisedb`, and the database user, `enterprisedb`, used to connect to the local `edb` database are the same database user. However, that's not required. - -For specific use of these foreign server and user mapping examples, see the example given in [`localcopyschema`](#localcopyschema). - -#### Foreign server and user mapping for remote cloning functions - -For the `remotecopyschema` and `remotecopyschema_nb` functions, the source and target schemas are in different databases of either the same or different database servers. You must define and specify two foreign servers for these functions. - -The foreign server defining the originating database server and its database containing the source schema to clone is referred to as the *source server* or the *remote server*. - -The foreign server defining the database server and its database to receive the schema to clone is referred to as the *target server* or the *local server*. - -The target server is also referred to as the local server because this server is the one to which you must be connected when invoking the `remotecopyschema` or `remotecopyschema_nb` function. - -The user mappings define the connection and authentication information for the foreign servers. - -You must create all of these foreign servers and user mappings in the target database of the target/local server. - -The database user for whom the user mappings are defined must be a superuser and the user connected to the local server when invoking an EDB Clone Schema function. - -This example creates the foreign server for the local, target database that receives the cloned schema: - -```sql -CREATE SERVER tgt_server FOREIGN DATA WRAPPER postgres_fdw - OPTIONS( - host 'localhost', - port '5444', - dbname 'tgtdb' -); -``` - -The following is the user mapping for this server: - -```sql -CREATE USER MAPPING FOR enterprisedb SERVER tgt_server - OPTIONS ( - user 'tgtuser', - password 'tgtpassword' -); -``` - -This example creates the foreign server for the remote, source database that's the source for the cloned schema: - -```sql -CREATE SERVER src_server FOREIGN DATA WRAPPER postgres_fdw - OPTIONS( - host '192.168.2.28', - port '5444', - dbname 'srcdb' -); -``` - -The following is the user mapping for this server: - -```sql -CREATE USER MAPPING FOR enterprisedb SERVER src_server - OPTIONS ( - user 'srcuser', - password 'srcpassword' -); -``` - -The following `psql` commands show the foreign servers and user mappings: - -```sql -tgtdb=# \des+ -__OUTPUT__ -List of foreign servers --[ RECORD 1 ]--------+--------------------------------------------------- -Name | src_server -Owner | tgtuser -Foreign-data wrapper | postgres_fdw -Access privileges | -Type | -Version | -FDW options | (host '192.168.2.28', port '5444', dbname 'srcdb') -Description | --[ RECORD 2 ]--------+--------------------------------------------------- -Name | tgt_server -Owner | tgtuser -Foreign-data wrapper | postgres_fdw -Access privileges | -Type | -Version | -FDW options | (host 'localhost', port '5444', dbname 'tgtdb') -Description | -``` -```sql -tgtdb=# \deu+ -__OUTPUT__ - List of user mappings - Server | User name | FDW options -------------+--------------+-------------------------------------------- - src_server | enterprisedb | ("user" 'srcuser', password 'srcpassword') - tgt_server | enterprisedb | ("user" 'tgtuser', password 'tgtpassword') -(2 rows) -``` - -When database superuser `enterprisedb` invokes a cloning function, the database user `tgtuser` with password `tgtpassword` is used to connect to `tgt_server` on the `localhost` with port `5444` to database `tgtdb`. - -In addition, database user `srcuser` with password `srcpassword` connects to `src_server` on host `192.168.2.28` with port `5444` to database `srcdb`. - -!!! Note - Be sure the `pg_hba.conf` file of the database server running the source database `srcdb` has an appropriate entry. This entry must permit connection from the target server location (address `192.168.2.27` in the following example) with the database user `srcuser` that was included in the user mapping for the foreign server `src_server` defining the source server and database. - -```shell -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all md5 -# IPv4 local connections: -host srcdb srcuser 192.168.2.27/32 md5 -``` - -For specific use of these foreign server and user mapping examples, see the example given in [`remotecopyschema`](#remotecopyschema). - -## EDB Clone Schema functions - -The EDB Clone Schema functions are created in the `edb_util` schema when the `parallel_clone` and `edb_cloneschema` extensions are installed. - -Verify the following conditions before using an EDB Clone Schema function: - -- You're connected to the target or local database as the database superuser defined in the `CREATE USER MAPPING` command for the foreign server of the target or local database. -- The `edb_util` schema is in the search path, or invoke the cloning function with the `edb_util` prefix. -- The target schema doesn't exist in the target database. -- When using the remote copy functions, if the `on_tblspace` parameter is set to `true`, then the target database cluster contains all tablespaces that are referenced by objects in the source schema. Otherwise, creating the DDL statements for those database objects fails in the target schema, which causes a failure of the cloning process. -- When using the remote copy functions, if you set the `copy_acls` parameter to `true`, then all roles that have `GRANT` privileges on objects in the source schema exist in the target database cluster. Otherwise granting privileges to those roles fails in the target schema, which causes a failure of the cloning process. -- pgAgent is running against the target database if you're using the non-blocking form of the function. - -pgAgent is provided as a component with EDB Postgres Advanced Server. For information about pgAgent, see the [pgAdmin documentation](https://www.pgadmin.org/docs/pgadmin4/dev/pgagent.html). - -### localcopyschema - - - -The `localcopyschema` function copies a schema and its database objects in a local database specified in the `source_fdw` foreign server from the source schema to the specified target schema in the same database. - -```sql -localcopyschema( - TEXT, - TEXT, - TEXT, - TEXT - [, BOOLEAN - [, BOOLEAN - [, BOOLEAN - [, INTEGER ]]]] -) -``` - -The function returns a Boolean value. If the function succeeds, then `true` is returned. If the function fails, then `false` is returned. - -The `source_fdw, source_schema, target_schema`, and `log_filename` are required parameters while all other parameters are optional. - -#### Parameters - -`source_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. - -`source_schema` - -Name of the schema from which to clone database objects. - -`target_schema` - -Name of the schema into which to clone database objects from the source schema. - -`log_filename` - -Name of the log file in which information from the function is recorded. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. - -`on_tblspace` - -Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. - -`verbose_on` - -Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. - -`copy_acls` - -Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. - -`worker_count` - -Number of background workers to perform the clone in parallel. The default value is `1`. - -#### Example - -This example shows the cloning of schema `edb` containing a set of database objects to target schema `edbcopy`. Both schemas are in database `edb` as defined by `local_server`. - -The example is for the following environment: - -- Host on which the database server is running: `localhost` -- Port of the database server: `5444` -- Database source/target of the clone: `edb` -- Foreign server (`local_server`) and user mapping with the information of the preceding bullet points -- Source schema: `edb` -- Target schema: `edbcopy` -- Database superuser to invoke `localcopyschema: enterprisedb` - -Before invoking the function, database user `enterprisedb` connects to to database `edb`: - -```sql -edb=# SET search_path TO "$user",public,edb_util; -SET -edb=# SHOW search_path; -__OUTPUT__ - search_path ---------------------------- - "$user", public, edb_util -(1 row) -``` -```sql -edb=# SELECT localcopyschema ('local_server','edb','edbcopy','clone_edb_edbcopy'); -__OUTPUT__ - localcopyschema ------------------ - t -(1 row) -``` - -The following displays the logging status using the `process_status_from_log` function: - -```sql -edb=# SELECT process_status_from_log('clone_edb_edbcopy'); -__OUTPUT__ - process_status_from_log ------------------------------------------------------------------------------------------------- - (FINISH,"2017-06-29 11:07:36.830783-04",3855,INFO,"STAGE: FINAL","successfully cloned schema") -(1 row) -``` - -After the clone is complete, the following shows some of the database objects copied to the `edbcopy` schema: - -```sql -edb=# SET search_path TO edbcopy; -SET -edb=# \dt+ -__OUTPUT__ - List of relations - Schema | Name | Type | Owner | Size | Description ----------+---------+-------+--------------+------------+------------- - edbcopy | dept | table | enterprisedb | 8192 bytes | - edbcopy | emp | table | enterprisedb | 8192 bytes | - edbcopy | jobhist | table | enterprisedb | 8192 bytes | -(3 rows) -``` -```sql -edb=# \dv -__OUTPUT__ - List of relations - Schema | Name | Type | Owner ----------+----------+------+-------------- - edbcopy | salesemp | view | enterprisedb -(1 row) -``` -```sql -edb=# \di -__OUTPUT__ - List of relations - Schema | Name | Type | Owner | Table ----------+---------------+-------+--------------+--------- - edbcopy | dept_dname_uq | index | enterprisedb | dept - edbcopy | dept_pk | index | enterprisedb | dept - edbcopy | emp_pk | index | enterprisedb | emp - edbcopy | jobhist_pk | index | enterprisedb | jobhist -(4 rows) -``` -```sql -edb=# \ds -__OUTPUT__ - List of relations - Schema | Name | Type | Owner ----------+------------+----------+-------------- - edbcopy | next_empno | sequence | enterprisedb -(1 row) -``` -```sql -edb=# SELECT DISTINCT schema_name, name, type FROM user_source WHERE -schema_name = 'EDBCOPY' ORDER BY type, name; -__OUTPUT__ - schema_name | name | type --------------+--------------------------------+-------------- - EDBCOPY | EMP_COMP | FUNCTION - EDBCOPY | HIRE_CLERK | FUNCTION - EDBCOPY | HIRE_SALESMAN | FUNCTION - EDBCOPY | NEW_EMPNO | FUNCTION - EDBCOPY | EMP_ADMIN | PACKAGE - EDBCOPY | EMP_ADMIN | PACKAGE BODY - EDBCOPY | EMP_QUERY | PROCEDURE - EDBCOPY | EMP_QUERY_CALLER | PROCEDURE - EDBCOPY | LIST_EMP | PROCEDURE - EDBCOPY | SELECT_EMP | PROCEDURE - EDBCOPY | EMP_SAL_TRIG | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_19991" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_19992" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_19999" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_20000" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_20004" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_a_20005" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_19993" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_19994" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_20001" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_20002" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_20006" | TRIGGER - EDBCOPY | "RI_ConstraintTrigger_c_20007" | TRIGGER - EDBCOPY | USER_AUDIT_TRIG | TRIGGER -(24 rows) -``` - -### localcopyschema_nb - - - -The `localcopyschema_nb` function copies a schema and its database objects in a local database specified in the `source_fdw` foreign server from the source schema to the specified target schema in the same database. The copy occurs in a non-blocking manner as a job submitted to pgAgent. - -```sql -localcopyschema_nb( - TEXT, - TEXT, - TEXT, - TEXT - [, BOOLEAN - [, BOOLEAN - [, BOOLEAN - [, INTEGER ]]]] -) -``` - -The function returns an `INTEGER` value job ID for the job submitted to pgAgent. If the function fails, then null is returned. - -The `source_fdw`, `source`, `target`, and `log_filename` parameters are required. All other parameters are optional. - -After the pgAgent job completes, remove it with the `remove_log_file_and_job` function. - -#### Parameters - -`source_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. - -`source` - -Name of the schema from which to clone database objects. - -`target` - -Name of the schema into which to clone database objects from the source schema. - -`log_filename` - -Name of the log file in which to record information from the function. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. - -`on_tblspace` - -Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. - -`verbose_on` - -Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. - -`copy_acls` - -Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. - -`worker_count` - -Number of background workers to perform the clone in parallel. The default value is `1`. - -#### Example - -The same cloning operation is performed as the example in [`localcopyschema`](#localcopyschema) but using the non-blocking function `localcopyschema_nb`. - -You can use this command to see whether pgAgent is running on the appropriate local database: - -```shell -[root@localhost ~]# ps -ef | grep pgagent -root 4518 1 0 11:35 pts/1 00:00:00 pgagent -s /tmp/pgagent_edb_log -hostaddr=127.0.0.1 port=5444 dbname=edb user=enterprisedb password=password -root 4525 4399 0 11:35 pts/1 00:00:00 grep --color=auto pgagent -``` - -If pgAgent isn't running, you can start it as shown by the following. The `pgagent` program file is located in the `bin` subdirectory of the EDB Postgres Advanced Server installation directory. - -```shell -[root@localhost bin]# ./pgagent -l 2 -s /tmp/pgagent_edb_log hostaddr=127.0.0.1 port=5444 -dbname=edb user=enterprisedb password=password -``` - -!!! Note - The `pgagent -l 2` option starts pgAgent in `DEBUG` mode, which logs continuous debugging information into the log file specified with the `-s` option. Use a lower value for the `-l` option, or omit it entirely to record less information. - -The `localcopyschema_nb` function returns the job ID shown as `4` in the example. - -```sql -edb=# SELECT edb_util.localcopyschema_nb ('local_server','edb','edbcopy','clone_edb_edbcopy'); -__OUTPUT__ - localcopyschema_nb --------------------- - 4 -(1 row) -``` - -The following displays the job status: - -```sql -edb=# SELECT edb_util.process_status_from_log('clone_edb_edbcopy'); -__OUTPUT__ - process_status_from_log ---------------------------------------------------------------------------------------------------- - (FINISH,"29-JUN-17 11:39:11.620093 -04:00",4618,INFO,"STAGE: FINAL","successfully cloned schema") -(1 row) -``` - -The following removes the pgAgent job: - -```sql -edb=# SELECT edb_util.remove_log_file_and_job (4); -__OUTPUT__ - remove_log_file_and_job -------------------------- - t -(1 row) -``` - -### remotecopyschema - - - -The `remotecopyschema` function copies a schema and its database objects from a source schema in the remote source database specified in the `source_fdw` foreign server to a target schema in the local target database specified in the `target_fdw` foreign server: - -```sql -remotecopyschema( - TEXT, - TEXT, - TEXT, - TEXT, - TEXT - [, BOOLEAN - [, BOOLEAN - [, BOOLEAN - [, INTEGER ]]]] -) -``` - -The function returns a Boolean value. If the function succeeds, then `true` is returned. If the function fails, then `false` is returned. - -The `source_fdw`, `target_fdw`, `source_schema`, `target_schema`, and `log_filename` are required parameters. All other parameters are optional. - -#### Parameters - -`source_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. - -`target_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper to which to clone database objects. - -`source_schema` - -Name of the schema from which to clone database objects. - -`target_schema` - -Name of the schema into which to clone database objects from the source schema. - -`log_filename` - -Name of the log file in which information from the function is recorded. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. - -`on_tblspace` - -Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. - -!!! Note - If you specify `true` and a database object has a `TABLESPACE` clause, the tablespace must exist in the target database cluseter. Otherwise, the cloning function fails. - -`verbose_on` - -Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. - -`copy_acls` - -Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. - -!!! Note - If you specify `true`, a role with `GRANT` privilege must exist in the target database cluster. Otherwise, the cloning function fails. - -`worker_count` - -Number of background workers to perform the clone in parallel. The default value is `1`. - -#### Example - -This example shows cloning schema `srcschema` in database `srcdb` (as defined by `src_server`) to target schema `tgtschema` in database `tgtdb` (as defined by `tgt_server`). - -The source server environment: - -- Host on which the source database server is running: `192.168.2.28` -- Port of the source database server: `5444` -- Database source of the clone: `srcdb` -- Foreign server (`src_server`) and user mapping with the information of the preceding bullet points -- Source schema: `srcschema` - -The target server environment: - -- Host on which the target database server is running: `localhost` -- Port of the target database server: `5444` -- Database target of the clone: `tgtdb` -- Foreign server (`tgt_server`) and user mapping with the information of the preceding bullet points -- Target schema: `tgtschema` -- Database superuser to invoke `remotecopyschema: enterprisedb` - -Before invoking the function, the connection database user `enterprisedb` connects to database `tgtdb`. A `worker_count` of `4` is specified for this function. - -```sql -tgtdb=# SELECT edb_util.remotecopyschema -('src_server','tgt_server','srcschema','tgtschema','clone_rmt_src_tgt',worker_count => 4); -__OUTPUT__ - remotecopyschema ------------------- - t -(1 row) -``` - -The following displays the status from the log file during various points in the cloning process: - -```sql -tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); -__OUTPUT__ - process_status_from_log - ------------------------------------------------------------------------------------------------------ --------------------------------------- - (RUNNING,"28-JUN-17 13:18:05.299953 -04:00",4021,INFO,"STAGE: DATA-COPY","[0][0] successfully - copied data in [tgtschema.pgbench_tellers] -") - (1 row) -``` -```sql -tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); -__OUTPUT__ - process_status_from_log - ------------------------------------------------------------------------------------------------------ ---------------------------------------- - (RUNNING,"28-JUN-17 13:18:06.634364 -04:00",4022,INFO,"STAGE: DATA-COPY","[0][1] successfully - copied data in [tgtschema.pgbench_history] -") - (1 row) -``` -```sql -tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); -__OUTPUT__ - process_status_from_log - ------------------------------------------------------------------------------------------------------- -------------------------------------- - (RUNNING,"28-JUN-17 13:18:10.550393 -04:00",4039,INFO,"STAGE: POST-DATA","CREATE PRIMARY KEY - CONSTRAINT pgbench_tellers_pkey successful" -) - (1 row) -``` -```sql -tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); -__OUTPUT__ - process_status_from_log -------------------------------------------------------------------------------------------------------- ---------------------------------- - (FINISH,"28-JUN-17 13:18:12.019627 -04:00",4039,INFO,"STAGE: FINAL","successfully clone - schema into tgtschema") - (1 row) -``` - -The following shows the cloned tables: - -```sql -tgtdb=# \dt+ -__OUTPUT__ - List of relations - Schema | Name | Type | Owner | Size | Description ------------+------------------+-------+--------------+------------+------------- - tgtschema | pgbench_accounts | table | enterprisedb | 256 MB | - tgtschema | pgbench_branches | table | enterprisedb | 8192 bytes | - tgtschema | pgbench_history | table | enterprisedb | 25 MB | - tgtschema | pgbench_tellers | table | enterprisedb | 16 kB | -(4 rows) -``` - -When the `remotecopyschema` function was invoked, four background workers were specified. - -The following portion of the log file `clone_rmt_src_tgt` shows the status of the parallel data copying operation using four background workers: - -```text -Wed Jun 28 13:18:05.232949 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] table count [4] -Wed Jun 28 13:18:05.233321 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][0] worker started to -copy data -Wed Jun 28 13:18:05.233640 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][1] worker started to -copy data -Wed Jun 28 13:18:05.233919 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][2] worker started to -copy data -Wed Jun 28 13:18:05.234231 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][3] worker started to -copy data -Wed Jun 28 13:18:05.298174 2017 EDT: [4024] INFO: [STAGE: DATA-COPY] [0][3] successfully -copied data in [tgtschema.pgbench_branches] -Wed Jun 28 13:18:05.299913 2017 EDT: [4021] INFO: [STAGE: DATA-COPY] [0][0] successfully -copied data in [tgtschema.pgbench_tellers] -Wed Jun 28 13:18:06.634310 2017 EDT: [4022] INFO: [STAGE: DATA-COPY] [0][1] successfully -copied data in [tgtschema.pgbench_history] -Wed Jun 28 13:18:10.477333 2017 EDT: [4023] INFO: [STAGE: DATA-COPY] [0][2] successfully -copied data in [tgtschema.pgbench_accounts] -Wed Jun 28 13:18:10.477609 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] all workers finished -[4] -Wed Jun 28 13:18:10.477654 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] copy done [4] tables -Wed Jun 28 13:18:10.493938 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] successfully copied data -into tgtschema -``` - -The `DATA-COPY` log message includes two square-bracket numbers, for example, `[0][3]`. The first number is the job index. The second number is the worker index. The worker index values range from 0 to 3 for the four background workers. - -In case two clone schema jobs are running in parallel, the first log file has `0` as the job index, and the second has `1` as the job index. - -### remotecopyschema_nb - - - -The `remotecopyschema_nb` function copies a schema and its database objects from a source schema in the remote source database specified in the `source_fdw` foreign server to a target schema in the local target database specified in the `target_fdw` foreign server. Copying occurs in a non-blocking manner as a job submitted to pgAgent. - -```sql -remotecopyschema_nb( - TEXT, - TEXT, - TEXT, - TEXT, - TEXT - [, BOOLEAN - [, BOOLEAN - [, BOOLEAN - [, INTEGER ]]]] -) -``` - -The function returns an `INTEGER` value job ID for the job submitted to pgAgent. If the function fails, then null is returned. - -The `source_fdw`, `target_fdw`, `source`, `target`, and `log_filename` parameters are required. All other parameters are optional. - -After the pgAgent job is complete, remove it with the `remove_log_file_and_job` function. - -#### Parameters - -`source_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. - -`target_fdw` - -Name of the foreign server managed by the `postgres_fdw` foreign data wrapper to which to clone database objects. - -`source` - -Name of the schema from which to clone database objects. - -`target` - -Name of the schema into which to clone database objects from the source schema. - -`log_filename` - -Name of the log file in which to record information from the function. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. - -`on_tblspace` - -Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. - -!!! Note - If you specify `true` is specified and a database object has a `TABLESPACE` clause, that tablespace must exist in the target database cluster. Otherwise, the cloning function fails. - -`verbose_on` - -Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. - -`copy_acls` - -Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. - -!!! Note - If you specify `true`, a role with `GRANT` privilege must exist in the target database cluster. Otherwise the cloning function fails. - -`worker_count` - -Number of background workers to perform the clone in parallel. The default value is `1`. - -#### Example - -The same cloning operation is performed as the example in [`remotecopyschema`](#remotecopyschema) but using the non-blocking function `remotecopyschema_nb`. - -The following command starts pgAgent on the target database `tgtdb`. The `pgagent` program file is located in the `bin` subdirectory of the EDB Postgres Advanced Server installation directory. - -```shell -[root@localhost bin]# ./pgagent -l 1 -s /tmp/pgagent_tgtdb_log hostaddr=127.0.0.1 port=5444 -user=enterprisedb dbname=tgtdb password=password -``` - -The `remotecopyschema_nb` function returns the job ID shown as `2` in the example: - -```sql -tgtdb=# SELECT edb_util.remotecopyschema_nb -('src_server','tgt_server','srcschema','tgtschema','clone_rmt_src_tgt',worker_count => 4); -__OUTPUT__ - remotecopyschema_nb ---------------------- - 2 -(1 row) -``` - -The following shows the completed status of the job: - -```sql -tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); -__OUTPUT__ - process_status_from_log --------------------------------------------------------------------------------------------------------------------- - (FINISH,"29-JUN-17 current:16:00.100284 -04:00",3849,INFO,"STAGE: FINAL","successfully clone schema into tgtschema") -(1 row) -``` - -The following removes the log file and the pgAgent job: - -```sql -tgtdb=# SELECT edb_util.remove_log_file_and_job ('clone_rmt_src_tgt',2); -__OUTPUT__ - remove_log_file_and_job -------------------------- - t -(1 row) -``` - -### process_status_from_log - - - -The `process_status_from_log` function provides the status of a cloning function from its log file: - -```sql -process_status_from_log ( - TEXT -) -``` - -The function returns the following fields from the log file: - -| Field name | Description | -| ---------------- | --------------------------------------------------------------------------------------------------------- | -| `status` | Displays either `STARTING`, `RUNNING`, `FINISH`, or `FAILED`. | -| `execution_time` | When the command was executed. Displayed in timestamp format. | -| `pid` | Session process ID in which clone schema is getting called. | -| `level` | Displays either `INFO`, `ERROR`, or `SUCCESSFUL`. | -| `stage` | Displays either `STARTUP`, `INITIAL`, `DDL-COLLECTION`, `PRE-DATA`, `DATA-COPY`, `POST-DATA`, or `FINAL`. | -| `message` | Information respective to each command or failure. | - -#### Parameters - -`log_file` - -Name of the log file recording the cloning of a schema as specified when the cloning function was invoked. - -#### Example - -The following shows the use of the `process_status_from_log` function: - -```sql -edb=# SELECT edb_util.process_status_from_log('clone_edb_edbcopy'); -__OUTPUT__ - process_status_from_log ---------------------------------------------------------------------------------------------------- - (FINISH,"26-JUN-17 11:57:03.214458 -04:00",3691,INFO,"STAGE: FINAL","successfully cloned schema") -(1 row) -``` - -### remove_log_file_and_job - - - -The `remove_log_file_and_job` function performs cleanup tasks by removing the log files created by the schema cloning functions and the jobs created by the non-blocking functions. - -```sql -remove_log_file_and_job ( - { TEXT | - INTEGER | - TEXT, INTEGER - } -) -``` - -You can specify values for either or both of the two parameters when invoking the `remove_log_file_and_job` function: - -- If you specify only `log_file`, then the function removes only the log file. -- If you specify only `job_id`, then the function removes only the job. -- If you specify both, then the function removes only the log file and the job. - -#### Parameters - -`log_file` - -Name of the log file to remove. - -`job_id` - -Job ID of the job to remove. - -#### Example - -This example removes only the log file, given the log file name: - -```sql -edb=# SELECT edb_util.remove_log_file_and_job ('clone_edb_edbcopy'); -__OUTPUT__ - remove_log_file_and_job -------------------------- - t -(1 row) -``` - -This example removes only the job, given the job ID: - -```sql -edb=# SELECT edb_util.remove_log_file_and_job (3); -__OUTPUT__ - remove_log_file_and_job -------------------------- - t -(1 row) -``` - -This example removes the log file and the job, given both values: - -```sql -tgtdb=# SELECT edb_util.remove_log_file_and_job ('clone_rmt_src_tgt',2); -__OUTPUT__ - remove_log_file_and_job -------------------------- - t -(1 row) -``` diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/checking_the_status.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/checking_the_status.mdx new file mode 100644 index 00000000000..b5a9e7eadb3 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/checking_the_status.mdx @@ -0,0 +1,45 @@ +--- +title: "Checking the status of the cloning process" +navTitle: "Checking status" +description: "Describes how to obtain the status of the cloning function" +--- + + + +The `process_status_from_log` function provides the status of a cloning function from its log file: + +```sql +process_status_from_log ( + TEXT +) +``` + +The function returns the following fields from the log file: + +| Field name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------- | +| `status` | Displays either `STARTING`, `RUNNING`, `FINISH`, or `FAILED`. | +| `execution_time` | When the command was executed. Displayed in timestamp format. | +| `pid` | Session process ID in which clone schema is getting called. | +| `level` | Displays either `INFO`, `ERROR`, or `SUCCESSFUL`. | +| `stage` | Displays either `STARTUP`, `INITIAL`, `DDL-COLLECTION`, `PRE-DATA`, `DATA-COPY`, `POST-DATA`, or `FINAL`. | +| `message` | Information respective to each command or failure. | + +## Parameters + +`log_file` + +Name of the log file recording the cloning of a schema as specified when the cloning function was invoked. + +## Example + +The following shows the use of the `process_status_from_log` function: + +```sql +edb=# SELECT edb_util.process_status_from_log('clone_edb_edbcopy'); +__OUTPUT__ + process_status_from_log +--------------------------------------------------------------------------------------------------- + (FINISH,"26-JUN-17 11:57:03.214458 -04:00",3691,INFO,"STAGE: FINAL","successfully cloned schema") +(1 row) +``` diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx new file mode 100644 index 00000000000..64427307156 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx @@ -0,0 +1,324 @@ +--- +title: "Copying database objcts from a remote source" +navTitle: "Copying a remote database" +description: "Describes how to copy a database when the source schema and the copy will reside in separate databases" +--- + +There are two functions you can use with EDB Clone Schema to perform a local copy of a schema and its database objects: + +- `remotecopyschema` — This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](#remotecopyschema) for more information. +- `remotecopyschema_nb` — This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](#remotecopyschema_nb) for more information. + +## remotecopyschema + + + +The `remotecopyschema` function copies a schema and its database objects from a source schema in the remote source database specified in the `source_fdw` foreign server to a target schema in the local target database specified in the `target_fdw` foreign server: + +```sql +remotecopyschema( + TEXT, + TEXT, + TEXT, + TEXT, + TEXT + [, BOOLEAN + [, BOOLEAN + [, BOOLEAN + [, INTEGER ]]]] +) +``` + +The function returns a Boolean value. If the function succeeds, then `true` is returned. If the function fails, then `false` is returned. + +The `source_fdw`, `target_fdw`, `source_schema`, `target_schema`, and `log_filename` are required parameters. All other parameters are optional. + +### Parameters + +`source_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. + +`target_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper to which to clone database objects. + +`source_schema` + +Name of the schema from which to clone database objects. + +`target_schema` + +Name of the schema into which to clone database objects from the source schema. + +`log_filename` + +Name of the log file in which information from the function is recorded. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. + +`on_tblspace` + +Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. + +!!! Note + If you specify `true` and a database object has a `TABLESPACE` clause, the tablespace must exist in the target database cluseter. Otherwise, the cloning function fails. + +`verbose_on` + +Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. + +`copy_acls` + +Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. + +!!! Note + If you specify `true`, a role with `GRANT` privilege must exist in the target database cluster. Otherwise, the cloning function fails. + +`worker_count` + +Number of background workers to perform the clone in parallel. The default value is `1`. + +### Example + +This example shows cloning schema `srcschema` in database `srcdb` (as defined by `src_server`) to target schema `tgtschema` in database `tgtdb` (as defined by `tgt_server`). + +The source server environment: + +- Host on which the source database server is running: `192.168.2.28` +- Port of the source database server: `5444` +- Database source of the clone: `srcdb` +- Foreign server (`src_server`) and user mapping with the information of the preceding bullet points +- Source schema: `srcschema` + +The target server environment: + +- Host on which the target database server is running: `localhost` +- Port of the target database server: `5444` +- Database target of the clone: `tgtdb` +- Foreign server (`tgt_server`) and user mapping with the information of the preceding bullet points +- Target schema: `tgtschema` +- Database superuser to invoke `remotecopyschema: enterprisedb` + +Before invoking the function, the connection database user `enterprisedb` connects to database `tgtdb`. A `worker_count` of `4` is specified for this function. + +```sql +tgtdb=# SELECT edb_util.remotecopyschema +('src_server','tgt_server','srcschema','tgtschema','clone_rmt_src_tgt',worker_count => 4); +__OUTPUT__ + remotecopyschema +------------------ + t +(1 row) +``` + +The following displays the status from the log file during various points in the cloning process: + +```sql +tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); +__OUTPUT__ + process_status_from_log + +----------------------------------------------------------------------------------------------------- +-------------------------------------- + (RUNNING,"28-JUN-17 13:18:05.299953 -04:00",4021,INFO,"STAGE: DATA-COPY","[0][0] successfully + copied data in [tgtschema.pgbench_tellers] +") + (1 row) +``` +```sql +tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); +__OUTPUT__ + process_status_from_log + +----------------------------------------------------------------------------------------------------- +--------------------------------------- + (RUNNING,"28-JUN-17 13:18:06.634364 -04:00",4022,INFO,"STAGE: DATA-COPY","[0][1] successfully + copied data in [tgtschema.pgbench_history] +") + (1 row) +``` +```sql +tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); +__OUTPUT__ + process_status_from_log + +------------------------------------------------------------------------------------------------------ +------------------------------------- + (RUNNING,"28-JUN-17 13:18:10.550393 -04:00",4039,INFO,"STAGE: POST-DATA","CREATE PRIMARY KEY + CONSTRAINT pgbench_tellers_pkey successful" +) + (1 row) +``` +```sql +tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); +__OUTPUT__ + process_status_from_log +------------------------------------------------------------------------------------------------------- +--------------------------------- + (FINISH,"28-JUN-17 13:18:12.019627 -04:00",4039,INFO,"STAGE: FINAL","successfully clone + schema into tgtschema") + (1 row) +``` + +The following shows the cloned tables: + +```sql +tgtdb=# \dt+ +__OUTPUT__ + List of relations + Schema | Name | Type | Owner | Size | Description +-----------+------------------+-------+--------------+------------+------------- + tgtschema | pgbench_accounts | table | enterprisedb | 256 MB | + tgtschema | pgbench_branches | table | enterprisedb | 8192 bytes | + tgtschema | pgbench_history | table | enterprisedb | 25 MB | + tgtschema | pgbench_tellers | table | enterprisedb | 16 kB | +(4 rows) +``` + +When the `remotecopyschema` function was invoked, four background workers were specified. + +The following portion of the log file `clone_rmt_src_tgt` shows the status of the parallel data copying operation using four background workers: + +```text +Wed Jun 28 13:18:05.232949 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] table count [4] +Wed Jun 28 13:18:05.233321 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][0] worker started to +copy data +Wed Jun 28 13:18:05.233640 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][1] worker started to +copy data +Wed Jun 28 13:18:05.233919 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][2] worker started to +copy data +Wed Jun 28 13:18:05.234231 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0][3] worker started to +copy data +Wed Jun 28 13:18:05.298174 2017 EDT: [4024] INFO: [STAGE: DATA-COPY] [0][3] successfully +copied data in [tgtschema.pgbench_branches] +Wed Jun 28 13:18:05.299913 2017 EDT: [4021] INFO: [STAGE: DATA-COPY] [0][0] successfully +copied data in [tgtschema.pgbench_tellers] +Wed Jun 28 13:18:06.634310 2017 EDT: [4022] INFO: [STAGE: DATA-COPY] [0][1] successfully +copied data in [tgtschema.pgbench_history] +Wed Jun 28 13:18:10.477333 2017 EDT: [4023] INFO: [STAGE: DATA-COPY] [0][2] successfully +copied data in [tgtschema.pgbench_accounts] +Wed Jun 28 13:18:10.477609 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] all workers finished +[4] +Wed Jun 28 13:18:10.477654 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] [0] copy done [4] tables +Wed Jun 28 13:18:10.493938 2017 EDT: [4019] INFO: [STAGE: DATA-COPY] successfully copied data +into tgtschema +``` + +The `DATA-COPY` log message includes two square-bracket numbers, for example, `[0][3]`. The first number is the job index. The second number is the worker index. The worker index values range from 0 to 3 for the four background workers. + +In case two clone schema jobs are running in parallel, the first log file has `0` as the job index, and the second has `1` as the job index. + +## remotecopyschema_nb + + + +The `remotecopyschema_nb` function copies a schema and its database objects from a source schema in the remote source database specified in the `source_fdw` foreign server to a target schema in the local target database specified in the `target_fdw` foreign server. Copying occurs in a non-blocking manner as a job submitted to pgAgent. + +```sql +remotecopyschema_nb( + TEXT, + TEXT, + TEXT, + TEXT, + TEXT + [, BOOLEAN + [, BOOLEAN + [, BOOLEAN + [, INTEGER ]]]] +) +``` + +The function returns an `INTEGER` value job ID for the job submitted to pgAgent. If the function fails, then null is returned. + +The `source_fdw`, `target_fdw`, `source`, `target`, and `log_filename` parameters are required. All other parameters are optional. + +After the pgAgent job is complete, remove it with the `remove_log_file_and_job` function. + +### Parameters + +`source_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. + +`target_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper to which to clone database objects. + +`source` + +Name of the schema from which to clone database objects. + +`target` + +Name of the schema into which to clone database objects from the source schema. + +`log_filename` + +Name of the log file in which to record information from the function. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. + +`on_tblspace` + +Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. + +!!! Note + If you specify `true` is specified and a database object has a `TABLESPACE` clause, that tablespace must exist in the target database cluster. Otherwise, the cloning function fails. + +`verbose_on` + +Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. + +`copy_acls` + +Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. + +!!! Note + If you specify `true`, a role with `GRANT` privilege must exist in the target database cluster. Otherwise the cloning function fails. + +`worker_count` + +Number of background workers to perform the clone in parallel. The default value is `1`. + +### Example + +The same cloning operation is performed as the example in [`remotecopyschema`](#remotecopyschema) but using the non-blocking function `remotecopyschema_nb`. + +The following command starts pgAgent on the target database `tgtdb`. The `pgagent` program file is located in the `bin` subdirectory of the EDB Postgres Advanced Server installation directory. + +```shell +[root@localhost bin]# ./pgagent -l 1 -s /tmp/pgagent_tgtdb_log hostaddr=127.0.0.1 port=5444 +user=enterprisedb dbname=tgtdb password=password +``` + +The `remotecopyschema_nb` function returns the job ID shown as `2` in the example: + +```sql +tgtdb=# SELECT edb_util.remotecopyschema_nb +('src_server','tgt_server','srcschema','tgtschema','clone_rmt_src_tgt',worker_count => 4); +__OUTPUT__ + remotecopyschema_nb +--------------------- + 2 +(1 row) +``` + +The following shows the completed status of the job: + +```sql +tgtdb=# SELECT edb_util.process_status_from_log('clone_rmt_src_tgt'); +__OUTPUT__ + process_status_from_log +-------------------------------------------------------------------------------------------------------------------- + (FINISH,"29-JUN-17 current:16:00.100284 -04:00",3849,INFO,"STAGE: FINAL","successfully clone schema into tgtschema") +(1 row) +``` + +The following removes the log file and the pgAgent job: + +```sql +tgtdb=# SELECT edb_util.remove_log_file_and_job ('clone_rmt_src_tgt',2); +__OUTPUT__ + remove_log_file_and_job +------------------------- + t +(1 row) +``` + diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_schema.mdx new file mode 100644 index 00000000000..f0a18dcec9b --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_schema.mdx @@ -0,0 +1,306 @@ +--- +title: "Copying database objects from a local source to a target" +navTitle: "Copying a local database" +description: "Describes how to copy database objects from a source database into the same database but with a different schema name" +--- + +There are two functions you can use with EDB Clone Schema to perform a local copy of a schema and its database objects: + +- `localcopyschema` — This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](#localcopyschema) for more information. +- `localcopyschema_nb` — This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](#localcopyschema_nb) for more information. + +## localcopyschema + + + +The `localcopyschema` function copies a schema and its database objects in a local database specified in the `source_fdw` foreign server from the source schema to the specified target schema in the same database. + +```sql +localcopyschema( + TEXT, + TEXT, + TEXT, + TEXT + [, BOOLEAN + [, BOOLEAN + [, BOOLEAN + [, INTEGER ]]]] +) +``` + +The function returns a Boolean value. If the function succeeds, then `true` is returned. If the function fails, then `false` is returned. + +The `source_fdw, source_schema, target_schema`, and `log_filename` are required parameters while all other parameters are optional. + +### Parameters + +`source_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. + +`source_schema` + +Name of the schema from which to clone database objects. + +`target_schema` + +Name of the schema into which to clone database objects from the source schema. + +`log_filename` + +Name of the log file in which information from the function is recorded. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. + +`on_tblspace` + +Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. + +`verbose_on` + +Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. + +`copy_acls` + +Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. + +`worker_count` + +Number of background workers to perform the clone in parallel. The default value is `1`. + +### Example + +This example shows the cloning of schema `edb` containing a set of database objects to target schema `edbcopy`. Both schemas are in database `edb` as defined by `local_server`. + +The example is for the following environment: + +- Host on which the database server is running: `localhost` +- Port of the database server: `5444` +- Database source/target of the clone: `edb` +- Foreign server (`local_server`) and user mapping with the information of the preceding bullet points +- Source schema: `edb` +- Target schema: `edbcopy` +- Database superuser to invoke `localcopyschema: enterprisedb` + +Before invoking the function, database user `enterprisedb` connects to to database `edb`: + +```sql +edb=# SET search_path TO "$user",public,edb_util; +SET +edb=# SHOW search_path; +__OUTPUT__ + search_path +--------------------------- + "$user", public, edb_util +(1 row) +``` +```sql +edb=# SELECT localcopyschema ('local_server','edb','edbcopy','clone_edb_edbcopy'); +__OUTPUT__ + localcopyschema +----------------- + t +(1 row) +``` + +The following displays the logging status using the `process_status_from_log` function: + +```sql +edb=# SELECT process_status_from_log('clone_edb_edbcopy'); +__OUTPUT__ + process_status_from_log +------------------------------------------------------------------------------------------------ + (FINISH,"2017-06-29 11:07:36.830783-04",3855,INFO,"STAGE: FINAL","successfully cloned schema") +(1 row) +``` + +After the clone is complete, the following shows some of the database objects copied to the `edbcopy` schema: + +```sql +edb=# SET search_path TO edbcopy; +SET +edb=# \dt+ +__OUTPUT__ + List of relations + Schema | Name | Type | Owner | Size | Description +---------+---------+-------+--------------+------------+------------- + edbcopy | dept | table | enterprisedb | 8192 bytes | + edbcopy | emp | table | enterprisedb | 8192 bytes | + edbcopy | jobhist | table | enterprisedb | 8192 bytes | +(3 rows) +``` +```sql +edb=# \dv +__OUTPUT__ + List of relations + Schema | Name | Type | Owner +---------+----------+------+-------------- + edbcopy | salesemp | view | enterprisedb +(1 row) +``` +```sql +edb=# \di +__OUTPUT__ + List of relations + Schema | Name | Type | Owner | Table +---------+---------------+-------+--------------+--------- + edbcopy | dept_dname_uq | index | enterprisedb | dept + edbcopy | dept_pk | index | enterprisedb | dept + edbcopy | emp_pk | index | enterprisedb | emp + edbcopy | jobhist_pk | index | enterprisedb | jobhist +(4 rows) +``` +```sql +edb=# \ds +__OUTPUT__ + List of relations + Schema | Name | Type | Owner +---------+------------+----------+-------------- + edbcopy | next_empno | sequence | enterprisedb +(1 row) +``` +```sql +edb=# SELECT DISTINCT schema_name, name, type FROM user_source WHERE +schema_name = 'EDBCOPY' ORDER BY type, name; +__OUTPUT__ + schema_name | name | type +-------------+--------------------------------+-------------- + EDBCOPY | EMP_COMP | FUNCTION + EDBCOPY | HIRE_CLERK | FUNCTION + EDBCOPY | HIRE_SALESMAN | FUNCTION + EDBCOPY | NEW_EMPNO | FUNCTION + EDBCOPY | EMP_ADMIN | PACKAGE + EDBCOPY | EMP_ADMIN | PACKAGE BODY + EDBCOPY | EMP_QUERY | PROCEDURE + EDBCOPY | EMP_QUERY_CALLER | PROCEDURE + EDBCOPY | LIST_EMP | PROCEDURE + EDBCOPY | SELECT_EMP | PROCEDURE + EDBCOPY | EMP_SAL_TRIG | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_19991" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_19992" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_19999" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_20000" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_20004" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_a_20005" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_19993" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_19994" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_20001" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_20002" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_20006" | TRIGGER + EDBCOPY | "RI_ConstraintTrigger_c_20007" | TRIGGER + EDBCOPY | USER_AUDIT_TRIG | TRIGGER +(24 rows) +``` + +## localcopyschema_nb + + + +The `localcopyschema_nb` function copies a schema and its database objects in a local database specified in the `source_fdw` foreign server from the source schema to the specified target schema in the same database. The copy occurs in a non-blocking manner as a job submitted to pgAgent. + +```sql +localcopyschema_nb( + TEXT, + TEXT, + TEXT, + TEXT + [, BOOLEAN + [, BOOLEAN + [, BOOLEAN + [, INTEGER ]]]] +) +``` + +The function returns an `INTEGER` value job ID for the job submitted to pgAgent. If the function fails, then null is returned. + +The `source_fdw`, `source`, `target`, and `log_filename` parameters are required. All other parameters are optional. + +After the pgAgent job completes, remove it with the `remove_log_file_and_job` function. + +### Parameters + +`source_fdw` + +Name of the foreign server managed by the `postgres_fdw` foreign data wrapper from which to clone database objects. + +`source` + +Name of the schema from which to clone database objects. + +`target` + +Name of the schema into which to clone database objects from the source schema. + +`log_filename` + +Name of the log file in which to record information from the function. The log file is created under the directory specified by the `log_directory` configuration parameter in the `postgresql.conf` file. + +`on_tblspace` + +Boolean value to specify whether to create database objects in their tablespaces. If `false`, then the `TABLESPACE` clause isn't included in the applicable `CREATE` DDL statement when added to the target schema. If `true`, then the `TABLESPACE` clause is included in the `CREATE` DDL statement when added to the target schema. The default value is `false`. + +`verbose_on` + +Boolean value to specify whether to print the DDLs in `log_filename` when creating objects in the target schema. If `false`, then DDLs aren't printed. If `true`, then DDLs are printed. The default value is `false`. + +`copy_acls` + +Boolean value to specify whether to include the access control list (ACL) while creating objects in the target schema. The access control list is the set of `GRANT` privilege statements. If `false`, then the access control list isn't included for the target schema. If `true`, then the access control list is included for the target schema. The default value is `false`. + +`worker_count` + +Number of background workers to perform the clone in parallel. The default value is `1`. + +### Example + +The same cloning operation is performed as the example in [`localcopyschema`](#localcopyschema) but using the non-blocking function `localcopyschema_nb`. + +You can use this command to see whether pgAgent is running on the appropriate local database: + +```shell +[root@localhost ~]# ps -ef | grep pgagent +root 4518 1 0 11:35 pts/1 00:00:00 pgagent -s /tmp/pgagent_edb_log +hostaddr=127.0.0.1 port=5444 dbname=edb user=enterprisedb password=password +root 4525 4399 0 11:35 pts/1 00:00:00 grep --color=auto pgagent +``` + +If pgAgent isn't running, you can start it as shown by the following. The `pgagent` program file is located in the `bin` subdirectory of the EDB Postgres Advanced Server installation directory. + +```shell +[root@localhost bin]# ./pgagent -l 2 -s /tmp/pgagent_edb_log hostaddr=127.0.0.1 port=5444 +dbname=edb user=enterprisedb password=password +``` + +!!! Note + The `pgagent -l 2` option starts pgAgent in `DEBUG` mode, which logs continuous debugging information into the log file specified with the `-s` option. Use a lower value for the `-l` option, or omit it entirely to record less information. + +The `localcopyschema_nb` function returns the job ID shown as `4` in the example. + +```sql +edb=# SELECT edb_util.localcopyschema_nb ('local_server','edb','edbcopy','clone_edb_edbcopy'); +__OUTPUT__ + localcopyschema_nb +-------------------- + 4 +(1 row) +``` + +The following displays the job status: + +```sql +edb=# SELECT edb_util.process_status_from_log('clone_edb_edbcopy'); +__OUTPUT__ + process_status_from_log +--------------------------------------------------------------------------------------------------- + (FINISH,"29-JUN-17 11:39:11.620093 -04:00",4618,INFO,"STAGE: FINAL","successfully cloned schema") +(1 row) +``` + +The following removes the pgAgent job: + +```sql +edb=# SELECT edb_util.remove_log_file_and_job (4); +__OUTPUT__ + remove_log_file_and_job +------------------------- + t +(1 row) +``` diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx new file mode 100644 index 00000000000..3e0fe08aaea --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx @@ -0,0 +1,80 @@ +--- +title: "EDB Clone Schema key concepts and limitations" +navTitle: "Overview and limitations" +description: "Provides an overview of the key features of EDB Clone Schema as well as important information about limitations" +--- + +## EDB Clone Schema functions + +The EDB Clone Schema functions are created in the `edb_util` schema when the `parallel_clone` and `edb_cloneschema` extensions are installed. + +## Prerequisites + +Verify the following conditions before using an EDB Clone Schema function: + +- You're connected to the target or local database as the database superuser defined in the `CREATE USER MAPPING` command for the foreign server of the target or local database. +- The `edb_util` schema is in the search path, or invoke the cloning function with the `edb_util` prefix. +- The target schema doesn't exist in the target database. +- When using the remote copy functions, if the `on_tblspace` parameter is set to `true`, then the target database cluster contains all tablespaces that are referenced by objects in the source schema. Otherwise, creating the DDL statements for those database objects fails in the target schema, which causes a failure of the cloning process. +- When using the remote copy functions, if you set the `copy_acls` parameter to `true`, then all roles that have `GRANT` privileges on objects in the source schema exist in the target database cluster. Otherwise granting privileges to those roles fails in the target schema, which causes a failure of the cloning process. +- pgAgent is running against the target database if you're using the non-blocking form of the function. + +pgAgent is provided as a component with EDB Postgres Advanced Server. For information about pgAgent, see the [pgAdmin documentation](https://www.pgadmin.org/docs/pgadmin4/dev/pgagent.html). + +## Overview of the functions + +You use the following functions with EDB Clone Schema: + +- `localcopyschema`. This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](#localcopyschema) for more information. +- `localcopyschema_nb`. This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](#localcopyschema_nb) for more information. +- `remotecopyschema`. This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](#remotecopyschema) for more information. +- `remotecopyschema_nb`. This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](#remotecopyschema_nb) for more information. +- `process_status_from_log`. This function displays the status of the cloning functions. The information is obtained from a log file you specify when invoking a cloning function. See [process_status_from_log](#process_status_from_log) for more information. +- `remove_log_file_and_job`. This function deletes the log file created by a cloning function. You can also use this function to delete a job created by the non-blocking form of the function. See [remove_log_file_and_job](#remove_log_file_and_job) for more information. + +## List of supported database objects + +You can clone these database objects from one schema to another: + +- Data types +- Tables including partitioned tables, excluding foreign tables +- Indexes +- Constraints +- Sequences +- View definitions +- Materialized views +- Private synonyms +- Table triggers, but excluding event triggers +- Rules +- Functions +- Procedures +- Packages +- Comments for all supported object types +- Access control lists (ACLs) for all supported object types + +You can't clone the following database objects: + +- Large objects (Postgres `LOBs` and `BFILEs`) +- Logical replication attributes for a table +- Database links +- Foreign data wrappers +- Foreign tables +- Event triggers +- Extensions + +For cloning objects that rely on extensions, see the limitations that follow. + +- Row-level security +- Policies +- Operator class + +## Limitations + +The following limitations apply: + +- EDB Clone Schema is supported on EDB Postgres Advanced Server when you specify a dialect of **Compatible with Oracle** on the EDB Postgres Advanced Server Dialect dialog box during installation. It's also supported when you include the `--redwood-like` keywords during a text-mode installation or cluster initialization. +- The source code in functions, procedures, triggers, packages, and so on, aren't modified after being copied to the target schema. If such programs contain coded references to objects with schema names, the programs might fail when invoked in the target schema if such schema names are no longer consistent in the target schema. +- Cross-schema object dependencies aren't resolved. If an object in the target schema depends on an object in another schema, this dependency isn't resolved by the cloning functions. +- For remote cloning, if an object in the source schema depends on an extension, then you must create this extension in the public schema of the remote database before invoking the remote cloning function. +- At most, 16 copy jobs can run in parallel to clone schemas. Each job can have at most 16 worker processes to copy table data in parallel. +- You can't cancel queries run by background workers. diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/index.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/index.mdx new file mode 100644 index 00000000000..9a22a607e4f --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/index.mdx @@ -0,0 +1,25 @@ +--- +title: "Copying a database" +navTitle: "EDB Clone Schema" +indexCards: simple +description: "How to use the EDB Clone Schema module to copy a schema and its database objects from a source database to a target database" +navigation: +- edb_clone_schema_overview +- setting_up_edb_clone_schema +- copying_a_schema +- copying_a_remote_schema +- checking_the_status +- performing_cleanup_tasks +redirects: + - ../../epas_guide/14_edb_clone_schema #generated for docs/epas/reorg-role-use-case-mode +--- + + + +EDB Clone Schema is an extension module for EDB Postgres Advanced Server that allows you to copy a schema and its database objects from a local or remote database (the source database) to a receiving database (the target database). + +The source and target databases can be either: + +- The same physical database +- Different databases in the same database cluster +- Separate databases running under different database clusters on separate database server hosts diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/performing_cleanup_tasks.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/performing_cleanup_tasks.mdx new file mode 100644 index 00000000000..dd2784bd469 --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/performing_cleanup_tasks.mdx @@ -0,0 +1,68 @@ +--- +title: "Performing cleanup tasks" +description: "Describes how to cleanup log files" +--- + + + +The `remove_log_file_and_job` function performs cleanup tasks by removing the log files created by the schema cloning functions and the jobs created by the non-blocking functions. + +```sql +remove_log_file_and_job ( + { TEXT | + INTEGER | + TEXT, INTEGER + } +) +``` + +You can specify values for either or both of the two parameters when invoking the `remove_log_file_and_job` function: + +- If you specify only `log_file`, then the function removes only the log file. +- If you specify only `job_id`, then the function removes only the job. +- If you specify both, then the function removes only the log file and the job. + +## Parameters + +`log_file` + +Name of the log file to remove. + +`job_id` + +Job ID of the job to remove. + +## Example + +This example removes only the log file, given the log file name: + +```sql +edb=# SELECT edb_util.remove_log_file_and_job ('clone_edb_edbcopy'); +__OUTPUT__ + remove_log_file_and_job +------------------------- + t +(1 row) +``` + +This example removes only the job, given the job ID: + +```sql +edb=# SELECT edb_util.remove_log_file_and_job (3); +__OUTPUT__ + remove_log_file_and_job +------------------------- + t +(1 row) +``` + +This example removes the log file and the job, given both values: + +```sql +tgtdb=# SELECT edb_util.remove_log_file_and_job ('clone_rmt_src_tgt',2); +__OUTPUT__ + remove_log_file_and_job +------------------------- + t +(1 row) +``` diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx new file mode 100644 index 00000000000..28f522ca92b --- /dev/null +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx @@ -0,0 +1,302 @@ +--- +title: "Setting up EDB Clone Schema" +description: "Describes how to install and configure the EDB Clone Schema extension" +--- + +To use EDB Clone Schema, you must first install several extensions along with the PL/Perl language on any database used as the source or target database by an EDB Clone Schema function. + +In addition, it might help to modify some configuration parameters in the `postgresql.conf` file of the database servers. + +## Installing extensions + +Perform this installation on any database to be used as the source or target database by an EDB Clone Schema function. + +1. Install the following extensions on the database: + - `postgres_fdw` + - `dblink` + - `adminpack` + - `pgagent` +1. Ensure that pgAgent is installed before creating the `pgagent` extension. On Linux, you can use the `edb-asxx-pgagent` RPM package, where `xx` is the EDB Postgres Advanced Server version number to install pgAgent. On Windows, use StackBuilder Plus to download and install pgAgent. + + You can install the extensions using the following commands: + + ```sql + CREATE EXTENSION postgres_fdw SCHEMA public; + CREATE EXTENSION dblink SCHEMA public; + CREATE EXTENSION adminpack; + CREATE EXTENSION pgagent; + ``` + + For more information about using the `CREATE EXTENSION` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createextension.html). + +## Installing extensions and PL/Perl + +1. Modify the `postgresql.conf` file by adding `$libdir/parallel_clone` to the `shared_preload_libraries` configuration parameter as shown in this example: + + ```ini + shared_preload_libraries = '$libdir/dbms_pipe,$libdir/dbms_aq,$libdir/parallel_clone' + ``` + +## Installing PL/Perl + +1. Install the Perl procedural language (PL/Perl) on the database, and run the `CREATE TRUSTED LANGUAGE plperl` command. For Linux, install PL/Perl using the `edb-asxx-server-plperl` RPM package, where `xx` is the EDB Postgres Advanced Server version number. For Windows, use the EDB Postgres Language Pack. For information on EDB Language Pack, see the [EDB Postgres Language Pack](/language_pack/latest). + +1. Connect to the database as a superuser and run the following command: + + ```sql + CREATE TRUSTED LANGUAGE plperl; + ``` + +For more information about using the `CREATE LANGUAGE` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createlanguage.html). + + + +## Setting configuration parameters + +You might need to modify configuration parameters in the `postgresql.conf` file. + +### Performance configuration parameters + +You might need to tune the system for copying a large schema as part of one transaction. Tuning of configuration parameters is for the source database server referenced in a cloning function. + +You might need to tune the following configuration parameters in the `postgresql.conf` file: + +- `work_mem`. Specifies the amount of memory for internal sort operations and hash tables to use before writing to temporary disk files. +- `maintenance_work_mem`. Specifies the maximum amount of memory for maintenance operations such as `VACUUM`, `CREATE INDEX`, and `ALTER TABLE ADD FOREIGN KEY` to use. +- `max_worker_processes`. Sets the maximum number of background processes that the system can support. +- `checkpoint_timeout`. Maximum time between automatic WAL checkpoints, in seconds. +- `checkpoint_completion_target`. Specifies the target of checkpoint completion as a fraction of total time between checkpoints. +- `checkpoint_flush_after`. Whenever more than `checkpoint_flush_after` bytes are written while performing a checkpoint, attempt to force the OS to issue these writes to the underlying storage. +- `max_wal_size`. Maximum size to let the WAL grow to between automatic WAL checkpoints. +- `max_locks_per_transaction`. Controls the average number of object locks allocated for each transaction. Individual transactions can lock more objects as long as the locks of all transactions fit in the lock table. + +For information about the configuration parameters, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/runtime-config.html). + +### Status logging + +Status logging by the cloning functions creates log files in the directory specified by the `log_directory` parameter in the `postgresql.conf` file for the database server to which you're connected when invoking the cloning function. + +The default location is `PGDATA/log`, as shown by the following: + +```ini +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +``` + +This directory must exist before running a cloning function. + +The name of the log file is determined by what you specify in the parameter list when invoking the cloning function. + +To display the status from a log file, use the `process_status_from_log` function. + +To delete a log file, use the `remove_log_file_and_job` function, or delete it manually from the log directory. + +## Installing EDB Clone Schema + +Install the EDB Clone Schema on any database to be used as the source or target database by an EDB Clone Schema function. + +1. If you previously installed an older version of the `edb_cloneschema` extension, run the following command: + +**Step 1:** If you had previously installed an older version of the `edb_cloneschema` extension, then you must run the following command: + +```sql +DROP EXTENSION parallel_clone CASCADE; +``` + + This command also drops the `edb_cloneschema` extension. + +1. Install the extensions. Make sure that you create the `parallel_clone` extension before creating the `edb_cloneschema` extension. + + ```sql + CREATE EXTENSION parallel_clone SCHEMA public; + + CREATE EXTENSION edb_cloneschema; + ``` + +## Creating the foreign servers and user mappings + +When using one of the local cloning functions `localcopyschema` or `localcopyschema_nb`, one of the required parameters includes a single, foreign server. This server is for identifying the database server and its database that's the source and receiver of the cloned schema. + +When using one of the remote cloning functions `remotecopyschema` or `remotecopyschema_nb`, two of the required parameters include two foreign servers. The foreign server specified as the first parameter identifies the source database server and its database that's the provider of the cloned schema. The foreign server specified as the second parameter identifies the target database server and its database that's the receiver of the cloned schema. + +For each foreign server, you must create a user mapping. When a selected database superuser invokes a cloning function, that superuser must be mapped to a database user name and password that has access to the foreign server that's specified as a parameter in the cloning function. + +For general information about foreign data, foreign servers, and user mappings, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/ddl-foreign-data.html). + +### Foreign server and user mapping for local cloning functions + +For the `localcopyschema` and `localcopyschema_nb` functions, the source and target schemas are both in the same database of the same database server. Thus, you must define and specify only one foreign server for these functions. This foreign server is also referred to as the *local server* because this server is the one to which you must be connected when invoking the `localcopyschema` or `localcopyschema_nb` function. + +The user mapping defines the connection and authentication information for the foreign server. You must create this foreign server and user mapping in the database of the local server in which the cloning occurs. + +The database user for whom the user mapping is defined must be a superuser and connected to the local server when invoking an EDB Clone Schema function. + +This example creates the foreign server for the database containing the schema to clone and to receive the cloned schema: + +```sql +CREATE SERVER local_server FOREIGN DATA WRAPPER postgres_fdw + OPTIONS( + host 'localhost', + port '5444', + dbname 'edb' +); +``` + +For more information about using the `CREATE SERVER` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createserver.html). + +The user mapping for this server is the following: + +```sql +CREATE USER MAPPING FOR enterprisedb SERVER local_server + OPTIONS ( + user 'enterprisedb', + password 'password' +); +``` + +For more information about using the `CREATE USER MAPPING` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/sql-createusermapping.html). + +The following `psql` commands show the foreign server and user mapping: + +```sql +edb=# \des+ +__OUTPUT__ +List of foreign servers +-[ RECORD 1 ]--------+---------------------------------------------- +Name | local_server +Owner | enterprisedb +Foreign-data wrapper | postgres_fdw +Access privileges | +Type | +Version | +FDW options | (host 'localhost', port '5444', dbname 'edb') +Description | +``` +```sql +edb=# \deu+ +__OUTPUT__ + List of user mappings + Server | User name | FDW options +--------------+--------------+---------------------------------------------- + local_server | enterprisedb | ("user" 'enterprisedb', password 'password') +(1 row) +``` + +When database superuser `enterprisedb` invokes a cloning function, the database user `enterprisedb` with its password is used to connect to `local_server` on the `localhost` with port `5444` to database `edb`. + +In this case, the mapped database user, `enterprisedb`, and the database user, `enterprisedb`, used to connect to the local `edb` database are the same database user. However, that's not required. + +For specific use of these foreign server and user mapping examples, see the example given in [`localcopyschema`](#localcopyschema). + +### Foreign server and user mapping for remote cloning functions + +For the `remotecopyschema` and `remotecopyschema_nb` functions, the source and target schemas are in different databases of either the same or different database servers. You must define and specify two foreign servers for these functions. + +The foreign server defining the originating database server and its database containing the source schema to clone is referred to as the *source server* or the *remote server*. + +The foreign server defining the database server and its database to receive the schema to clone is referred to as the *target server* or the *local server*. + +The target server is also referred to as the local server because this server is the one to which you must be connected when invoking the `remotecopyschema` or `remotecopyschema_nb` function. + +The user mappings define the connection and authentication information for the foreign servers. + +You must create all of these foreign servers and user mappings in the target database of the target/local server. + +The database user for whom the user mappings are defined must be a superuser and the user connected to the local server when invoking an EDB Clone Schema function. + +This example creates the foreign server for the local, target database that receives the cloned schema: + +```sql +CREATE SERVER tgt_server FOREIGN DATA WRAPPER postgres_fdw + OPTIONS( + host 'localhost', + port '5444', + dbname 'tgtdb' +); +``` + +The following is the user mapping for this server: + +```sql +CREATE USER MAPPING FOR enterprisedb SERVER tgt_server + OPTIONS ( + user 'tgtuser', + password 'tgtpassword' +); +``` + +This example creates the foreign server for the remote, source database that's the source for the cloned schema: + +```sql +CREATE SERVER src_server FOREIGN DATA WRAPPER postgres_fdw + OPTIONS( + host '192.168.2.28', + port '5444', + dbname 'srcdb' +); +``` + +The following is the user mapping for this server: + +```sql +CREATE USER MAPPING FOR enterprisedb SERVER src_server + OPTIONS ( + user 'srcuser', + password 'srcpassword' +); +``` + +The following `psql` commands show the foreign servers and user mappings: + +```sql +tgtdb=# \des+ +__OUTPUT__ +List of foreign servers +-[ RECORD 1 ]--------+--------------------------------------------------- +Name | src_server +Owner | tgtuser +Foreign-data wrapper | postgres_fdw +Access privileges | +Type | +Version | +FDW options | (host '192.168.2.28', port '5444', dbname 'srcdb') +Description | +-[ RECORD 2 ]--------+--------------------------------------------------- +Name | tgt_server +Owner | tgtuser +Foreign-data wrapper | postgres_fdw +Access privileges | +Type | +Version | +FDW options | (host 'localhost', port '5444', dbname 'tgtdb') +Description | +``` +```sql +tgtdb=# \deu+ +__OUTPUT__ + List of user mappings + Server | User name | FDW options +------------+--------------+-------------------------------------------- + src_server | enterprisedb | ("user" 'srcuser', password 'srcpassword') + tgt_server | enterprisedb | ("user" 'tgtuser', password 'tgtpassword') +(2 rows) +``` + +When database superuser `enterprisedb` invokes a cloning function, the database user `tgtuser` with password `tgtpassword` is used to connect to `tgt_server` on the `localhost` with port `5444` to database `tgtdb`. + +In addition, database user `srcuser` with password `srcpassword` connects to `src_server` on host `192.168.2.28` with port `5444` to database `srcdb`. + +!!! Note + Be sure the `pg_hba.conf` file of the database server running the source database `srcdb` has an appropriate entry. This entry must permit connection from the target server location (address `192.168.2.27` in the following example) with the database user `srcuser` that was included in the user mapping for the foreign server `src_server` defining the source server and database. + +```shell +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all md5 +# IPv4 local connections: +host srcdb srcuser 192.168.2.27/32 md5 +``` + +For specific use of these foreign server and user mapping examples, see the example given in [`remotecopyschema`](#remotecopyschema). + From 2df9ff48a27d17f16137b8b103ca0974fdd68f06 Mon Sep 17 00:00:00 2001 From: Fran Coughlin <132373434+francoughlin@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:58:17 -0400 Subject: [PATCH 05/13] Update direct_path_load.mdx Fixed typo --- .../database_administration/02_edb_loader/direct_path_load.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx index 0d5d5251d14..af0ade12ca2 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx @@ -1,7 +1,7 @@ --- title: "Running a direct path load" navTitle: "Direct path load" -description: "Describes how to use EDB\*Loader to write data directly to the database pages" +description: "Describes how to use EDB*Loader to write data directly to the database pages" --- From e2ff14a2feb7d7cdd439cddc34ee009a41851dd0 Mon Sep 17 00:00:00 2001 From: francoughlin Date: Tue, 11 Jul 2023 15:37:44 -0400 Subject: [PATCH 06/13] Final edits for EDB clone schema Minor updates to overview and setup --- .../copying_a_remote_schema.mdx | 2 +- .../edb_clone_schema_overview.mdx | 4 ++-- .../setting_up_edb_clone_schema.mdx | 22 +++++++------------ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx index 64427307156..8c7275a8364 100644 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/copying_a_remote_schema.mdx @@ -4,7 +4,7 @@ navTitle: "Copying a remote database" description: "Describes how to copy a database when the source schema and the copy will reside in separate databases" --- -There are two functions you can use with EDB Clone Schema to perform a local copy of a schema and its database objects: +There are two functions you can use with EDB Clone Schema to perform a remote copy of a schema and its database objects: - `remotecopyschema` — This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](#remotecopyschema) for more information. - `remotecopyschema_nb` — This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](#remotecopyschema_nb) for more information. diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx index 3e0fe08aaea..6d883f32d37 100644 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx @@ -19,11 +19,11 @@ Verify the following conditions before using an EDB Clone Schema function: - When using the remote copy functions, if you set the `copy_acls` parameter to `true`, then all roles that have `GRANT` privileges on objects in the source schema exist in the target database cluster. Otherwise granting privileges to those roles fails in the target schema, which causes a failure of the cloning process. - pgAgent is running against the target database if you're using the non-blocking form of the function. -pgAgent is provided as a component with EDB Postgres Advanced Server. For information about pgAgent, see the [pgAdmin documentation](https://www.pgadmin.org/docs/pgadmin4/dev/pgagent.html). +EDB Postgres Advanced Server includes pgAgent as a component. For information about pgAgent, see the [pgAdmin documentation](https://www.pgadmin.org/docs/pgadmin4/dev/pgagent.html). ## Overview of the functions -You use the following functions with EDB Clone Schema: +Use the following functions with EDB Clone Schema: - `localcopyschema`. This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](#localcopyschema) for more information. - `localcopyschema_nb`. This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](#localcopyschema_nb) for more information. diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx index 28f522ca92b..088075e152c 100644 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx @@ -11,11 +11,7 @@ In addition, it might help to modify some configuration parameters in the `postg Perform this installation on any database to be used as the source or target database by an EDB Clone Schema function. -1. Install the following extensions on the database: - - `postgres_fdw` - - `dblink` - - `adminpack` - - `pgagent` +1. Install the following extensions on the database: `postgres_fdw`, `dblink`, `adminpack` and `pgagent`. 1. Ensure that pgAgent is installed before creating the `pgagent` extension. On Linux, you can use the `edb-asxx-pgagent` RPM package, where `xx` is the EDB Postgres Advanced Server version number to install pgAgent. On Windows, use StackBuilder Plus to download and install pgAgent. You can install the extensions using the following commands: @@ -29,9 +25,9 @@ Perform this installation on any database to be used as the source or target dat For more information about using the `CREATE EXTENSION` command, see the [PostgreSQL core documentation](https://www.postgresql.org/docs/current/static/sql-createextension.html). -## Installing extensions and PL/Perl +## Modifying the configuration file -1. Modify the `postgresql.conf` file by adding `$libdir/parallel_clone` to the `shared_preload_libraries` configuration parameter as shown in this example: +Modify the `postgresql.conf` file by adding `$libdir/parallel_clone` to the `shared_preload_libraries` configuration parameter as shown in this example: ```ini shared_preload_libraries = '$libdir/dbms_pipe,$libdir/dbms_aq,$libdir/parallel_clone' @@ -95,13 +91,11 @@ To delete a log file, use the `remove_log_file_and_job` function, or delete it m Install the EDB Clone Schema on any database to be used as the source or target database by an EDB Clone Schema function. -1. If you previously installed an older version of the `edb_cloneschema` extension, run the following command: +1. If you previously installed an older version of the `edb_cloneschema` extension, run the following command: -**Step 1:** If you had previously installed an older version of the `edb_cloneschema` extension, then you must run the following command: - -```sql -DROP EXTENSION parallel_clone CASCADE; -``` + ```sql + DROP EXTENSION parallel_clone CASCADE; + ``` This command also drops the `edb_cloneschema` extension. @@ -125,7 +119,7 @@ For general information about foreign data, foreign servers, and user mappings, ### Foreign server and user mapping for local cloning functions -For the `localcopyschema` and `localcopyschema_nb` functions, the source and target schemas are both in the same database of the same database server. Thus, you must define and specify only one foreign server for these functions. This foreign server is also referred to as the *local server* because this server is the one to which you must be connected when invoking the `localcopyschema` or `localcopyschema_nb` function. +For the `localcopyschema` and `localcopyschema_nb` functions, the source and target schemas are both in the same database of the same database server. You must define and specify only one foreign server for these functions. This foreign server is also referred to as the *local server* because this server is the one to which you must be connected when invoking the `localcopyschema` or `localcopyschema_nb` function. The user mapping defines the connection and authentication information for the foreign server. You must create this foreign server and user mapping in the database of the local server in which the cloning occurs. From f7856f39118b71921c33861f1f54abe42b29dcb3 Mon Sep 17 00:00:00 2001 From: nidhibhammar <59045594+nidhibhammar@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:40:31 +0530 Subject: [PATCH 07/13] Update product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx --- .../02_edb_loader/building_the_control_file.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx index e9cdff16eea..d770e7a1358 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx @@ -126,7 +126,7 @@ If you specify the `FIELDS TERMINATED BY` clause, then you can't specify the `PO - `FREEZE= { FALSE | TRUE }` - Set `FREEZE` to `TRUE` to copy the data with the rows `frozen`. A tuple guaranteed to be visible to all current and future transactions is marked as frozen to prevent transaction ID wraparound. For more information about frozen tuples, see the [PostgreSQL core documentation]( Date: Fri, 14 Jul 2023 09:49:40 +0530 Subject: [PATCH 08/13] Update product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx --- .../02_edb_loader/control_file_examples.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx index 346082985dd..12278ee581d 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx @@ -1,5 +1,5 @@ --- -title: "EDB* Loader control file examples" +title: "EDB*Loader control file examples" navTitle: "Control file examples" description: "Provides examples of control files and their corresponding data files" --- From cac640bbda4acf5591e0e08184bbaaf60a886c86 Mon Sep 17 00:00:00 2001 From: Chris Estes <106166814+ccestes@users.noreply.github.com> Date: Fri, 14 Jul 2023 11:05:28 -0400 Subject: [PATCH 09/13] fix --- .../release/getting_started/creating_a_cluster/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product_docs/docs/biganimal/release/getting_started/creating_a_cluster/index.mdx b/product_docs/docs/biganimal/release/getting_started/creating_a_cluster/index.mdx index 2f87d427d72..ed4ff82e237 100644 --- a/product_docs/docs/biganimal/release/getting_started/creating_a_cluster/index.mdx +++ b/product_docs/docs/biganimal/release/getting_started/creating_a_cluster/index.mdx @@ -54,8 +54,8 @@ The following options aren't available when creating your cluster: 1. Select the type of cloud account for your cluster. You can choose BigAnimal's cloud account or set up your own. For more information, see [Deployment options](../../overview/deployment_options). -!!! Note - If you use BigAnimal's cloud account, your estimated cloud infrastructure costs are calculated into your cluster's estimated monthly price shown at the bottom of the page. + !!! Note + If you use BigAnimal's cloud account, your estimated cloud infrastructure costs are calculated into your cluster's estimated monthly price shown at the bottom of the page. 1. Select the cloud provider for your cluster. If you're using your own account and haven't connected it to BigAnimal yet, see [Connecting to your cloud](/biganimal/latest/getting_started/02_connecting_to_your_cloud). From 933e000be8ab12ba35362b184364ace65d4dc6bd Mon Sep 17 00:00:00 2001 From: francoughlin Date: Fri, 14 Jul 2023 12:11:58 -0400 Subject: [PATCH 10/13] Addressing comments from PR review Restructured EDB*Loader per Nidhi's comments; fixed broken links --- .../control_file_examples.mdx | 0 .../index.mdx} | 6 ++--- .../02_edb_loader/data_loading_methods.mdx | 4 ++-- .../02_edb_loader/index.mdx | 5 ----- .../conventional_path_load.mdx | 4 ++-- .../direct_path_load.mdx | 0 .../index.mdx} | 22 ++++++++++++------- .../parallel_direct_path_load.mdx | 0 .../performing_remote_loading.mdx | 4 ++-- .../creating_resource_groups.mdx | 2 +- .../edb_clone_schema_overview.mdx | 12 +++++----- .../setting_up_edb_clone_schema.mdx | 4 ++-- 12 files changed, 32 insertions(+), 31 deletions(-) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{ => building_the_control_file}/control_file_examples.mdx (100%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{building_the_control_file.mdx => building_the_control_file/index.mdx} (97%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{ => invoking_edb_loader}/conventional_path_load.mdx (96%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{ => invoking_edb_loader}/direct_path_load.mdx (100%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{invoking_edb_loader.mdx => invoking_edb_loader/index.mdx} (94%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{ => invoking_edb_loader}/parallel_direct_path_load.mdx (100%) rename product_docs/docs/epas/15/database_administration/02_edb_loader/{ => invoking_edb_loader}/performing_remote_loading.mdx (89%) diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file/control_file_examples.mdx similarity index 100% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/control_file_examples.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file/control_file_examples.mdx diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file/index.mdx similarity index 97% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file/index.mdx index d770e7a1358..6f146088e73 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/building_the_control_file/index.mdx @@ -98,7 +98,7 @@ If all of the following conditions are true, the discard file isn't created even Suppose you don't specify the `DISCARDFILE` clause and the `DISCARD` parameter for explicitly specifying the discard file name, but you do specify `DISCARDMAX` or `DISCARDS`. In this case, the EDB\*Loader session creates a discard file using the data file name with an extension of `.dsc`. !!! Note - The keywords `DISCARD` and `DISCARDS` differ. `DISCARD` is an EDB\*Loader command line parameter used to specify the discard file name (see [General usage](#general-usage)). `DISCARDS` is a clause of the `LOAD DATA` directive that can appear only in the control file. Keywords `DISCARDS` and `DISCARDMAX` provide the same functionality of specifying the maximum number of discarded records allowed before terminating the EDB\*Loader session. Records loaded into the database before terminating the EDB\*Loader session due to exceeding the `DISCARDS` or `DISCARDMAX` settings are kept in the database. They aren't rolled back. + The keywords `DISCARD` and `DISCARDS` differ. `DISCARD` is an EDB\*Loader command line parameter used to specify the discard file name. `DISCARDS` is a clause of the `LOAD DATA` directive that can appear only in the control file. Keywords `DISCARDS` and `DISCARDMAX` provide the same functionality of specifying the maximum number of discarded records allowed before terminating the EDB\*Loader session. Records loaded into the database before terminating the EDB\*Loader session due to exceeding the `DISCARDS` or `DISCARDMAX` settings are kept in the database. They aren't rolled back. Specifying one of `INSERT`, `APPEND`, `REPLACE`, or `TRUNCATE` establishes the default action for adding rows to target tables. The default action is `INSERT`. @@ -118,7 +118,7 @@ If you specify the `FIELDS TERMINATED BY` clause, then you can't specify the `PO Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or can potentially contain references to rows that were deleted. Direct inserts to load data into replicated tables aren't supported. - For information on direct path loads see, [Direct path load](#direct-path-load). + For information on direct path loads see, [Direct path load](/epas/latest/database_administration/02_edb_loader/invoking_edb_loader/direct_path_load/). - `ERRORS=error_count` @@ -136,7 +136,7 @@ If you specify the `FIELDS TERMINATED BY` clause, then you can't specify the `PO Set `PARALLEL` to `TRUE` to indicate that this EDB\*Loader session is one of a number of concurrent EDB\*Loader sessions participating in a parallel direct path load. The default value of `PARALLEL` is `FALSE`. - When `PARALLEL` is `TRUE`, you must also set the `DIRECT` parameter to `TRUE`. For more information about parallel direct path loads, see [Parallel direct path load](#parallel-direct-path-load). + When `PARALLEL` is `TRUE`, you must also set the `DIRECT` parameter to `TRUE`. For more information about parallel direct path loads, see [Parallel direct path load](/epas/latest/database_administration/02_edb_loader/invoking_edb_loader/parallel_direct_path_load). - `ROWS=n` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx index 795c496e279..3ec1342a52a 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/data_loading_methods.mdx @@ -19,6 +19,6 @@ One exception is that the EDB Postgres Advanced Server rules defined on the tabl !!! Note Create EDB Postgres Advanced Server rules using the `CREATE RULE` command. EDB Postgres Advanced Server rules aren't the same database objects as rules and rule sets used in Oracle. -EDB\*Loader also supports direct path loads. A direct path load is faster than a conventional path load but requires removing most types of constraints and triggers from the table. For more information, see [Direct path load](#direct-path-load). +EDB\*Loader also supports direct path loads. A direct path load is faster than a conventional path load but requires removing most types of constraints and triggers from the table. For more information, see [Direct path load](invoking_edb_loader/direct_path_load.mdx). -EDB\*Loader supports parallel direct path loads. A parallel direct path load provides even greater performance improvement by permitting multiple EDB\*Loader sessions to run simultaneously to load a single table. For more information, see [Parallel direct path load](#parallel-direct-path-load). +EDB\*Loader supports parallel direct path loads. A parallel direct path load provides even greater performance improvement by permitting multiple EDB\*Loader sessions to run simultaneously to load a single table. For more information, see [Parallel direct path load](invoking_edb_loader/parallel_direct_path_load.mdx). diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx index e66365e1da8..7f44b638519 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/index.mdx @@ -8,12 +8,7 @@ navigation: - data_loading_methods - using_edb_loader - building_the_control_file -- control_file_examples - invoking_edb_loader -- direct_path_load -- parallel_direct_path_load -- performing_remote_loading -- conventional_path_load legacyRedirectsGenerated: # This list is generated by a script. If you need add entries, use the `legacyRedirects` key. - "/edb-docs/d/edb-postgres-advanced-server/user-guides/database-compatibility-for-oracle-developers-tools-and-utilities-guide/9.6/DB_Compat_for_Oracle_Dev_Tools_Guide.1.16.html" diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/conventional_path_load.mdx similarity index 96% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/conventional_path_load.mdx index ffb9b4f2a02..289b2092443 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/conventional_path_load.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/conventional_path_load.mdx @@ -16,8 +16,8 @@ To use EDB\*Loader to update a table, the table must have a primary key. You can To perform `UPDATE`, use the same steps as when performing a conventional path load: 1. Create a data file that contains the rows you want to update or insert. -2. Define a control file that uses the `INFILE` keyword to specify the name of the data file. For information about building the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). -3. Invoke EDB\*Loader, specifying the database name, connection information, and the name of the control file. For information about invoking EDB\*Loader, see [Invoking EDB\*Loader](#invoking-edbloader). +2. Define a control file that uses the `INFILE` keyword to specify the name of the data file. For information about building the EDB\*Loader control file, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). +3. Invoke EDB\*Loader, specifying the database name, connection information, and the name of the control file. For information about invoking EDB\*Loader, see [Invoking EDB\*Loader](/epas/latest/database_administration/02_edb_loader/invoking_edb_loader/). This example uses the `emp` table that's distributed with the EDB Postgres Advanced Server sample data. By default, the table contains: diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/direct_path_load.mdx similarity index 100% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/direct_path_load.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/direct_path_load.mdx diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/index.mdx similarity index 94% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/index.mdx index 99498b05e52..5e57e1936b5 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/index.mdx @@ -1,6 +1,12 @@ --- title: "Invoking EDB*Loader" description: "Describes how to run EDB*Loader from the command line" +indexCards: simple +navigation: +- conventional_path_load +- direct_path_load +- parallel_direct_path_load +- performing_remote_loading --- @@ -33,7 +39,7 @@ edbldr [ -d ] [ -p ] [ -h ] If you omit the `-d` option, the `-p` option, or the `-h` option, the defaults for the database, port, and host are determined by the same rules as other EDB Postgres Advanced Server utility programs, such as `edb-psql`. -You can specify parameters listed in the syntax diagram in a *parameter file*. Exeptions include the `-d` option, `-p` option, `-h` option, and the `PARFILE` parameter. Specify the parameter file on the command line when you invoke `edbldr` using `PARFILE=param_file`. You can specify some parameters in the `OPTIONS` clause in the control file. For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). +You can specify parameters listed in the syntax diagram in a *parameter file*. Exeptions include the `-d` option, `-p` option, `-h` option, and the `PARFILE` parameter. Specify the parameter file on the command line when you invoke `edbldr` using `PARFILE=param_file`. You can specify some parameters in the `OPTIONS` clause in the control file. For more information on the control file, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). You can include the full directory path or a relative directory path to the file name when specifying `control_file`, `data_file`, `bad_file`, `discard_file`, `log_file`, and `param_file`. If you specify the file name alone or with a relative directory path, the file is assumed to exist in the case of `control_file`, `data_file`, or `param_file` relative to the current working directory from which `edbldr` is invoked. In the case of `bad_file`, `discard_file`, or `log_file`, the file is created. @@ -84,31 +90,31 @@ The operating system account `enterprisedb` must have write permission on the di `control_file` specifies the name of the control file containing EDB\*Loader directives. If you don't specify a file extension, an extension of `.ctl` is assumed. - For more information on the control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + For more information on the control file, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). `DATA=data_file` `data_file` specifies the name of the file containing the data to load into the target table. If you don't specify a file extension, an extension of `.dat` is assumed. Specifying a `data_file` on the command line overrides the `INFILE` clause specified in the control file. - For more information about `data_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + For more information about `data_file`, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). `BAD=bad_file` `bad_file` specifies the name of a file that receives input data records that can't be loaded due to errors. Specifying a `bad_file` on the command line overrides any `BADFILE` clause specified in the control file. - For more information about `bad_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + For more information about `bad_file`, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). `DISCARD=discard_file` `discard_file` is the name of the file that receives input data records that don't meet any table’s selection criteria. Specifying a `discard_file` on the command line overrides the `DISCARDFILE` clause in the control file. - For more information about `discard_file`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + For more information about `discard_file`, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). `DISCARDMAX=max_discard_recs` `max_discard_recs` is the maximum number of discarded records that can be encountered from the input data records before terminating the EDB\*Loader session. Specifying `max_discard_recs` on the command line overrides the `DISCARDMAX` or `DISCARDS` clause in the control file. - For more information about `max_discard_recs`, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). + For more information about `max_discard_recs`, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). `HANDLE_CONFLICTS={ FALSE | TRUE }` @@ -137,7 +143,7 @@ The operating system account `enterprisedb` must have write permission on the di Don't set `DIRECT=true` when loading the data into a replicated table. If you're using EDB\*Loader to load data into a replicated table and set `DIRECT=true`, indexes might omit rows that are in a table or might contain references to rows that were deleted. EnterpriseDB doesn't support direct inserts to load data into replicated tables. - For information about direct path loads, see [Direct path load](#direct-path-load). + For information about direct path loads, see [Direct path load](direct_path_load). `FREEZE= { FALSE | TRUE }` @@ -157,7 +163,7 @@ The operating system account `enterprisedb` must have write permission on the di When `PARALLEL` is `TRUE`, the `DIRECT` parameter must also be set to `TRUE`. - For more information about parallel direct path loads, see [Parallel direct path load](#parallel-direct-path-load). + For more information about parallel direct path loads, see [Parallel direct path load](parallel_direct_path_load). `ROWS=n` diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/parallel_direct_path_load.mdx similarity index 100% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/parallel_direct_path_load.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/parallel_direct_path_load.mdx diff --git a/product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/performing_remote_loading.mdx similarity index 89% rename from product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx rename to product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/performing_remote_loading.mdx index 5515638df3a..7339c9a3748 100644 --- a/product_docs/docs/epas/15/database_administration/02_edb_loader/performing_remote_loading.mdx +++ b/product_docs/docs/epas/15/database_administration/02_edb_loader/invoking_edb_loader/performing_remote_loading.mdx @@ -15,9 +15,9 @@ In addition, you can use the standard input feature to pipe the data from the da Performing remote loading along with using standard input requires the following: - The `edbldr` program must be installed on the client host on which to invoke it with the data source for the EDB\*Loader session. -- The control file must contain the clause `INFILE 'stdin'` so you can pipe the data directly into EDB\*Loader’s standard input. For information on the `INFILE` clause and the EDB\*Loader control file, see [Building the EDB\*Loader control file](#building-the-edbloader-control-file). +- The control file must contain the clause `INFILE 'stdin'` so you can pipe the data directly into EDB\*Loader’s standard input. For information on the `INFILE` clause and the EDB\*Loader control file, see [Building the EDB\*Loader control file](/epas/latest/database_administration/02_edb_loader/building_the_control_file/). - All files used by EDB\*Loader, such as the control file, bad file, discard file, and log file, must reside on or be created on the client host on which `edbldr` is invoked. -- When invoking EDB\*Loader, use the `-h` option to specify the IP address of the remote database server. For more information, see [Invoking EDB\*Loader](#invoking-edbloader). +- When invoking EDB\*Loader, use the `-h` option to specify the IP address of the remote database server. For more information, see [Invoking EDB\*Loader](/epas/latest/database_administration/02_edb_loader/invoking_edb_loader/). - Use the operating system pipe operator (`|`) or input redirection operator (`<`) to supply the input data to EDB\*Loader. This example loads a database running on a database server at `192.168.1.14` using data piped from a source named `datasource`: diff --git a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx index e8f967e11ec..1a4d7516e2e 100644 --- a/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx +++ b/product_docs/docs/epas/15/database_administration/10_edb_resource_manager/creating_resource_groups.mdx @@ -1,5 +1,5 @@ --- -title: "Creating resource groups" +title: "Working with resource groups" description: "How to use EDB Resource Manager to create and manage resource groups" --- diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx index 6d883f32d37..d293972d470 100644 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/edb_clone_schema_overview.mdx @@ -25,12 +25,12 @@ EDB Postgres Advanced Server includes pgAgent as a component. For information ab Use the following functions with EDB Clone Schema: -- `localcopyschema`. This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](#localcopyschema) for more information. -- `localcopyschema_nb`. This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](#localcopyschema_nb) for more information. -- `remotecopyschema`. This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](#remotecopyschema) for more information. -- `remotecopyschema_nb`. This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](#remotecopyschema_nb) for more information. -- `process_status_from_log`. This function displays the status of the cloning functions. The information is obtained from a log file you specify when invoking a cloning function. See [process_status_from_log](#process_status_from_log) for more information. -- `remove_log_file_and_job`. This function deletes the log file created by a cloning function. You can also use this function to delete a job created by the non-blocking form of the function. See [remove_log_file_and_job](#remove_log_file_and_job) for more information. +- `localcopyschema`. This function copies a schema and its database objects from a source database into the same database (the target) but with a different schema name from the original. Use this function when the source schema and the copy will reside within the same database. See [localcopyschema](copying_a_schema) for more information. +- `localcopyschema_nb`. This function performs the same purpose as `localcopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is referred to as a *non-blocking* function. See [localcopyschema_nb](copying_a_schema) for more information. +- `remotecopyschema`. This function copies a schema and its database objects from a source database to a different target database. Use this function when the source schema and the copy will reside in separate databases. The separate databases can reside in the same EDB Postgres Advanced Server database clusters or in different ones. See [remotecopyschema](copying_a_remote_schema) for more information. +- `remotecopyschema_nb`. This function performs the same purpose as `remotecopyschema` but as a background job, which frees up the terminal from which the function was initiated. This function is a non-blocking function. See [remotecopyschema_nb](copying_a_remote_schema) for more information. +- `process_status_from_log`. This function displays the status of the cloning functions. The information is obtained from a log file you specify when invoking a cloning function. See [process_status_from_log](checking_the_status) for more information. +- `remove_log_file_and_job`. This function deletes the log file created by a cloning function. You can also use this function to delete a job created by the non-blocking form of the function. See [remove_log_file_and_job](performing_cleanup_tasks) for more information. ## List of supported database objects diff --git a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx index 088075e152c..d0e18be352d 100644 --- a/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx +++ b/product_docs/docs/epas/15/database_administration/14_edb_clone_schema/setting_up_edb_clone_schema.mdx @@ -180,7 +180,7 @@ When database superuser `enterprisedb` invokes a cloning function, the database In this case, the mapped database user, `enterprisedb`, and the database user, `enterprisedb`, used to connect to the local `edb` database are the same database user. However, that's not required. -For specific use of these foreign server and user mapping examples, see the example given in [`localcopyschema`](#localcopyschema). +For specific use of these foreign server and user mapping examples, see the example given in [`localcopyschema`](copying_a_schema). ### Foreign server and user mapping for remote cloning functions @@ -292,5 +292,5 @@ local all all md5 host srcdb srcuser 192.168.2.27/32 md5 ``` -For specific use of these foreign server and user mapping examples, see the example given in [`remotecopyschema`](#remotecopyschema). +For specific use of these foreign server and user mapping examples, see the example given in [`remotecopyschema`](copying_a_remote_schema). From 9bb6d03a79b6be9c181284bd97eaf031ff8f41dc Mon Sep 17 00:00:00 2001 From: nidhibhammar <59045594+nidhibhammar@users.noreply.github.com> Date: Fri, 14 Jul 2023 22:44:24 +0530 Subject: [PATCH 11/13] PEM - 9.2.2 and 8.6.3 release notes update --- product_docs/docs/pem/8/pem_rel_notes/index.mdx | 1 + product_docs/docs/pem/9/pem_rel_notes/index.mdx | 1 + 2 files changed, 2 insertions(+) diff --git a/product_docs/docs/pem/8/pem_rel_notes/index.mdx b/product_docs/docs/pem/8/pem_rel_notes/index.mdx index 1dac6a6cef9..8445e730d8a 100644 --- a/product_docs/docs/pem/8/pem_rel_notes/index.mdx +++ b/product_docs/docs/pem/8/pem_rel_notes/index.mdx @@ -9,6 +9,7 @@ The Postgres Enterprise Manager (PEM) documentation describes the latest version | Version | Release Date | Upstream Merges | Accessibility Conformance | | ------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| [8.6.3](863_rel_notes) | 14 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [8.6.1](861_rel_notes) | 31 Mar 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [8.6.0](02_860_rel_notes) | 29 Sep 2022 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [8.5.0](03_850_rel_notes) | 23 Jun 2022 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | diff --git a/product_docs/docs/pem/9/pem_rel_notes/index.mdx b/product_docs/docs/pem/9/pem_rel_notes/index.mdx index a813876341c..accaa75b35a 100644 --- a/product_docs/docs/pem/9/pem_rel_notes/index.mdx +++ b/product_docs/docs/pem/9/pem_rel_notes/index.mdx @@ -13,6 +13,7 @@ The Postgres Enterprise Manager (PEM) documentation describes the latest version | Version | Release Date | Upstream Merges | Accessibility Conformance | | ------------------------- | ------------ | --------------------------------------------------------------------------| --------------------------------------------------------------------------------------------------- | +| [9.2.2](922_rel_notes) | 03 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.2.1](921_rel_notes) | 03 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.2.0](920_rel_notes) | 24 May 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.1.1](911_rel_notes) | 31 Mar 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | From e5482f198d67ee24bae03f7f6cacd6b67716f3e9 Mon Sep 17 00:00:00 2001 From: Dee Dee Rothery <83650384+drothery-edb@users.noreply.github.com> Date: Fri, 14 Jul 2023 13:32:31 -0400 Subject: [PATCH 12/13] Update product_docs/docs/pem/9/pem_rel_notes/index.mdx --- product_docs/docs/pem/9/pem_rel_notes/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product_docs/docs/pem/9/pem_rel_notes/index.mdx b/product_docs/docs/pem/9/pem_rel_notes/index.mdx index accaa75b35a..778ae3c93c2 100644 --- a/product_docs/docs/pem/9/pem_rel_notes/index.mdx +++ b/product_docs/docs/pem/9/pem_rel_notes/index.mdx @@ -13,7 +13,7 @@ The Postgres Enterprise Manager (PEM) documentation describes the latest version | Version | Release Date | Upstream Merges | Accessibility Conformance | | ------------------------- | ------------ | --------------------------------------------------------------------------| --------------------------------------------------------------------------------------------------- | -| [9.2.2](922_rel_notes) | 03 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | +| [9.2.2](922_rel_notes) | 14 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.2.1](921_rel_notes) | 03 Jul 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.2.0](920_rel_notes) | 24 May 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | | [9.1.1](911_rel_notes) | 31 Mar 2023 | NA | [Conformance Report](https://www.enterprisedb.com/accessibility) | From f2243453a94811ac4110bbd3bf366437501d02de Mon Sep 17 00:00:00 2001 From: drothery-edb Date: Fri, 14 Jul 2023 14:39:45 -0400 Subject: [PATCH 13/13] BigAnimal: update CLI commands for connecting to AWS --- .../02_connecting_to_your_cloud/connecting_aws.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/product_docs/docs/biganimal/release/getting_started/02_connecting_to_your_cloud/connecting_aws.mdx b/product_docs/docs/biganimal/release/getting_started/02_connecting_to_your_cloud/connecting_aws.mdx index 8c508effa7a..f98f9bb2e2b 100644 --- a/product_docs/docs/biganimal/release/getting_started/02_connecting_to_your_cloud/connecting_aws.mdx +++ b/product_docs/docs/biganimal/release/getting_started/02_connecting_to_your_cloud/connecting_aws.mdx @@ -23,13 +23,13 @@ To connect your cloud: 1. Create a BigAnimal CLI credential: ```shell - biganimal create-credential --name --address portal.biganimal.com --port 443 + biganimal credential create --name ``` 1. To set up your cloud provider, run the `setup-csp` command: ```shell - biganimal setup-csp + biganimal cloud-provider setup ``` The command checks for cloud account readiness and displays the results. @@ -44,6 +44,6 @@ To connect your cloud: 1. Connect your cloud account to BigAnimal: ```shell - biganimal connect-csp --provider aws --project + biganimal cloud-provider connect --provider aws --project ``` After your cloud account is successfully connected to BigAnimal, you and other users with the correct permissions can create clusters. \ No newline at end of file