From 4f4281415a04c9523371770049b67edadd561e45 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Mon, 8 Jan 2024 17:44:53 +0100 Subject: [PATCH 1/4] add sample cdc Signed-off-by: Eric Vergnaud --- postgres-cdc-example/Dockerfile | 16 ++++++++++++++ postgres-cdc-example/README.md | 21 +++++++++++++++++++ postgres-cdc-example/docker-compose.yml | 28 +++++++++++++++++++++++++ postgres-cdc-example/startup.sh | 16 ++++++++++++++ 4 files changed, 81 insertions(+) create mode 100644 postgres-cdc-example/Dockerfile create mode 100644 postgres-cdc-example/README.md create mode 100644 postgres-cdc-example/docker-compose.yml create mode 100755 postgres-cdc-example/startup.sh diff --git a/postgres-cdc-example/Dockerfile b/postgres-cdc-example/Dockerfile new file mode 100644 index 0000000000..be29822cb4 --- /dev/null +++ b/postgres-cdc-example/Dockerfile @@ -0,0 +1,16 @@ +FROM postgres:12.16 + +RUN apt-get update +RUN apt-get install -y postgresql-12-wal2json +RUN apt-get install -y redis-tools + +COPY startup.sh . + +ENV CDC_DB_NAME=CDC_DB_NAME + +ENV REDIS_HOST=REDIS_HOST +ENV REDIS_PORT=6379 +ENV REDIS_CHANNEL=REDIS_CHANNEL + +USER postgres +CMD ./startup.sh \ No newline at end of file diff --git a/postgres-cdc-example/README.md b/postgres-cdc-example/README.md new file mode 100644 index 0000000000..7dab71a0f3 --- /dev/null +++ b/postgres-cdc-example/README.md @@ -0,0 +1,21 @@ +This is a working CDC example for a local Postgres 12 / Redis. +It relies on wal2json for change notifications, see https://github.com/eulerto/wal2json. +It uses redis-cli to publish the notifications. + +Demo usage: + - the CDC'ed database must pre-exist, it should first be created as follows + - run `cd cdc` + - run `docker run -p 5432:5432 -v ./postgres-data:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:12.16` + - using a postgresql client, create the 'test_db' database and a table + - Ctrl-C the docker container started above + - now run `docker compose up -d` + - checking configuration: + + - from docker desktop, open a terminal to redis container + - from that terminal, run `redis-cli SUBSCRIBE test_db_changes` + - using a postgresql client, perform some inserts, updates or deletes in the 'test_db' database + +Real usage: + - you need to configure the env variables of the postgres service in the docker-compose.yml file + - the database name should be set to the messages repository name configured for the mediator + diff --git a/postgres-cdc-example/docker-compose.yml b/postgres-cdc-example/docker-compose.yml new file mode 100644 index 0000000000..2acc449913 --- /dev/null +++ b/postgres-cdc-example/docker-compose.yml @@ -0,0 +1,28 @@ +version: '3' + +services: + postgres: + build: . + image: pgsql-cdc/latest + container_name: cdc-pgsql + platform: linux/amd64 + ports: + - 5432:5432 + volumes: + - ./postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - CDC_DB_NAME=test_db + - REDIS_HOST=host.docker.internal + - REDIS_PORT=6379 + - REDIS_CHANNEL=test_db_changes + + redis: + image: redis:7.2.3 + container_name: cdc-redis + platform: linux/amd64 + ports: + - 6379:6379 + + diff --git a/postgres-cdc-example/startup.sh b/postgres-cdc-example/startup.sh new file mode 100755 index 0000000000..caa74939c9 --- /dev/null +++ b/postgres-cdc-example/startup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# run postgres in the background so we can run pg_recvlogical +postgres -c wal_level=logical -c max_wal_senders=1 -c shared_preload_libraries=wal2json & +# ensure postgres is fully initialized +sleep 10s +# create replication slot +pg_recvlogical -d test_db --slot messages_slot --create-slot --if-not-exists -P wal2json +# need a function for publishing otherwise bash shouts +publish_to_redis() { + redis-cli -h $REDIS_HOST -p $REDIS_PORT PUBLISH $REDIS_CHANNEL $1 +} +# listen to replication messages and publish them to redis +pg_recvlogical -d test_db --slot messages_slot --start -o pretty-print=0 -f - | while read message; do publish_to_redis $message ; done +# uncomment the following if you comment the previous to keep the container running +# tail -f /dev/null From d83e8a7111526db4d8667d2fc5f588a9ce14bca8 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Mon, 8 Jan 2024 17:58:17 +0100 Subject: [PATCH 2/4] use env variable and improve comments Signed-off-by: Eric Vergnaud --- postgres-cdc-example/startup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postgres-cdc-example/startup.sh b/postgres-cdc-example/startup.sh index caa74939c9..ba89552636 100755 --- a/postgres-cdc-example/startup.sh +++ b/postgres-cdc-example/startup.sh @@ -2,15 +2,15 @@ # run postgres in the background so we can run pg_recvlogical postgres -c wal_level=logical -c max_wal_senders=1 -c shared_preload_libraries=wal2json & -# ensure postgres is fully initialized +# ensure postgres is fully initialized before running pg_recvlogical sleep 10s -# create replication slot -pg_recvlogical -d test_db --slot messages_slot --create-slot --if-not-exists -P wal2json +# create replication slot if required +pg_recvlogical -d test_db --slot ${CDC_DB_NAME}_slot --create-slot --if-not-exists -P wal2json # need a function for publishing otherwise bash shouts publish_to_redis() { redis-cli -h $REDIS_HOST -p $REDIS_PORT PUBLISH $REDIS_CHANNEL $1 } # listen to replication messages and publish them to redis -pg_recvlogical -d test_db --slot messages_slot --start -o pretty-print=0 -f - | while read message; do publish_to_redis $message ; done +pg_recvlogical -d ${CDC_DB_NAME} --slot ${CDC_DB_NAME}_slot --start -o pretty-print=0 -f - | while read message; do publish_to_redis $message ; done # uncomment the following if you comment the previous to keep the container running # tail -f /dev/null From 17ecee5b51dbd8ad41c380cb828f5299a58a5b5b Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Thu, 11 Jan 2024 10:34:15 +0100 Subject: [PATCH 3/4] Move sample under examples Signed-off-by: Eric Vergnaud --- {postgres-cdc-example => samples/postgres-cdc}/Dockerfile | 0 {postgres-cdc-example => samples/postgres-cdc}/README.md | 0 {postgres-cdc-example => samples/postgres-cdc}/docker-compose.yml | 0 {postgres-cdc-example => samples/postgres-cdc}/startup.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {postgres-cdc-example => samples/postgres-cdc}/Dockerfile (100%) rename {postgres-cdc-example => samples/postgres-cdc}/README.md (100%) rename {postgres-cdc-example => samples/postgres-cdc}/docker-compose.yml (100%) rename {postgres-cdc-example => samples/postgres-cdc}/startup.sh (100%) diff --git a/postgres-cdc-example/Dockerfile b/samples/postgres-cdc/Dockerfile similarity index 100% rename from postgres-cdc-example/Dockerfile rename to samples/postgres-cdc/Dockerfile diff --git a/postgres-cdc-example/README.md b/samples/postgres-cdc/README.md similarity index 100% rename from postgres-cdc-example/README.md rename to samples/postgres-cdc/README.md diff --git a/postgres-cdc-example/docker-compose.yml b/samples/postgres-cdc/docker-compose.yml similarity index 100% rename from postgres-cdc-example/docker-compose.yml rename to samples/postgres-cdc/docker-compose.yml diff --git a/postgres-cdc-example/startup.sh b/samples/postgres-cdc/startup.sh similarity index 100% rename from postgres-cdc-example/startup.sh rename to samples/postgres-cdc/startup.sh From 0a969e0ede7141f607f114ce7232c7c068337d00 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Fri, 12 Jan 2024 10:22:43 +0100 Subject: [PATCH 4/4] apply formatting rules Signed-off-by: Eric Vergnaud --- samples/postgres-cdc/README.md | 25 +++++++++++++------------ samples/postgres-cdc/docker-compose.yml | 2 -- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/samples/postgres-cdc/README.md b/samples/postgres-cdc/README.md index 7dab71a0f3..7be4fa332d 100644 --- a/samples/postgres-cdc/README.md +++ b/samples/postgres-cdc/README.md @@ -3,19 +3,20 @@ It relies on wal2json for change notifications, see https://github.com/eulerto/w It uses redis-cli to publish the notifications. Demo usage: - - the CDC'ed database must pre-exist, it should first be created as follows - - run `cd cdc` - - run `docker run -p 5432:5432 -v ./postgres-data:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:12.16` - - using a postgresql client, create the 'test_db' database and a table - - Ctrl-C the docker container started above - - now run `docker compose up -d` - - checking configuration: - - from docker desktop, open a terminal to redis container - - from that terminal, run `redis-cli SUBSCRIBE test_db_changes` - - using a postgresql client, perform some inserts, updates or deletes in the 'test_db' database +- the CDC'ed database must pre-exist, it should first be created as follows +- run `cd cdc` +- run `docker run -p 5432:5432 -v ./postgres-data:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:12.16` +- using a postgresql client, create the 'test_db' database and a table +- Ctrl-C the docker container started above +- now run `docker compose up -d` +- checking configuration: + + - from docker desktop, open a terminal to redis container + - from that terminal, run `redis-cli SUBSCRIBE test_db_changes` + - using a postgresql client, perform some inserts, updates or deletes in the 'test_db' database Real usage: - - you need to configure the env variables of the postgres service in the docker-compose.yml file - - the database name should be set to the messages repository name configured for the mediator +- you need to configure the env variables of the postgres service in the docker-compose.yml file +- the database name should be set to the messages repository name configured for the mediator diff --git a/samples/postgres-cdc/docker-compose.yml b/samples/postgres-cdc/docker-compose.yml index 2acc449913..d0fd978920 100644 --- a/samples/postgres-cdc/docker-compose.yml +++ b/samples/postgres-cdc/docker-compose.yml @@ -24,5 +24,3 @@ services: platform: linux/amd64 ports: - 6379:6379 - -