diff --git a/.gitignore b/.gitignore index f805144c..3301c376 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__/ # Distribution / packaging .Python env/ +venv/ bin/ build/ develop-eggs/ diff --git a/TEST.rst b/TEST.rst index 0c233a54..7e145d4a 100644 --- a/TEST.rst +++ b/TEST.rst @@ -2,12 +2,43 @@ Tests ===== +Test Container +============== + +Instead of installing the necessary requirements onto your host machine, the `test_container/` directory contains +instructions for building a docker image with all the necessary requirements for running the tests across all +supported python versions. When you are finished the container and associated data can be removed. + +Install and run the container:: + + $ ./test_container/build.sh + $ ./test_container/run.sh + +Run the tests inside the container:: + + # tox --workdir /output -vv + +Remove the container and associated data:: + + $ sudo rm -rf test_container/output + $ docker image rm geoalchemy2 + $ docker system prune + + +Host System +=========== + +If you have a Linux system that you want to run the tests on instead of the test container, follow these steps: + + Install system dependencies -=========================== +--------------------------- + +(instructions for Ubuntu 22.04) Install PostgreSQL and PostGIS:: - $ sudo apt-get install postgresql postgis + $ sudo apt-get install postgresql postgresql-14-postgis-3 postgresql-14-postgis-3-scripts Install the Python and PostgreSQL development packages:: @@ -29,7 +60,7 @@ Install the Python dependencies:: Or you can use the Conda environment provided in the `GeoAlchemy2_dev.yml` file. Set up the PostGIS database -=========================== +--------------------------- Create the ``gis`` role:: @@ -51,7 +82,7 @@ With PostGIS 3 enable PostGIS Raster as well:: $ sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis_raster;" Set the path to the SpatiaLite module -===================================== +------------------------------------- By default the SpatiaLite functional tests are not run. To run them the ``SPATIALITE_LIBRARY_PATH`` environment variable must be set. @@ -62,7 +93,7 @@ the SpatiaLite library is ``/usr/lib/x86_64-linux-gnu/mod_spatialite.so``, so yo $ export SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so" Set up the MySQL database -========================= +------------------------- Create the ``gis`` role:: @@ -71,10 +102,10 @@ Create the ``gis`` role:: Create the ``gis`` database:: - $ mysql -u gis -p -e "CREATE DATABASE gis;" + $ mysql -u gis --password=gis -e "CREATE DATABASE gis;" Run Tests -========= +--------- To run the tests:: diff --git a/test_container/.dockerignore b/test_container/.dockerignore new file mode 100644 index 00000000..16be8f21 --- /dev/null +++ b/test_container/.dockerignore @@ -0,0 +1 @@ +/output/ diff --git a/test_container/.gitignore b/test_container/.gitignore new file mode 100644 index 00000000..16be8f21 --- /dev/null +++ b/test_container/.gitignore @@ -0,0 +1 @@ +/output/ diff --git a/test_container/Dockerfile b/test_container/Dockerfile new file mode 100644 index 00000000..15a098c9 --- /dev/null +++ b/test_container/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:22.04 + +COPY ./helpers/install_requirements.sh / +RUN /install_requirements.sh + +COPY ./helpers/init_postgres.sh / +env PGDATA="/var/lib/postgresql/data" +env POSTGRES_PATH="/usr/lib/postgresql/14" +RUN su postgres -c /init_postgres.sh + +ENV SPATIALITE_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu/mod_spatialite.so" + +COPY ./helpers/init_mysql.sh / +RUN /init_mysql.sh + +COPY ./helpers/entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] diff --git a/test_container/build.sh b/test_container/build.sh new file mode 100755 index 00000000..50895354 --- /dev/null +++ b/test_container/build.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +cd "${SCRIPT_DIR}" +docker build -t geoalchemy2 . diff --git a/test_container/helpers/entrypoint.sh b/test_container/helpers/entrypoint.sh new file mode 100755 index 00000000..9689dfd8 --- /dev/null +++ b/test_container/helpers/entrypoint.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +echo "starting postgres" +(su postgres -c '${POSTGRES_PATH}/bin/pg_ctl start' > /dev/null 2>&1) & + +while ! ${POSTGRES_PATH}/bin/pg_isready --quiet; do + sleep 0.2 +done + +echo "starting mysql" +/etc/init.d/mysql start + +echo "waiting for mysql to start" +while ! mysqladmin ping -h 127.0.0.1 --silent; do + sleep 0.2 +done + +echo "###############################" +echo "GeoAlchemy2 Test Container" +echo "" +echo 'run tests with `tox --workdir /output -vv`' +echo "###############################" + +############################### +# workarounds to get the tests working while mounting the code in as read-only +mkdir /geoalchemy2 +find /geoalchemy2_read_only -mindepth 1 -maxdepth 1 | while read -r item; do + ln -s "${item}" "/geoalchemy2/$(basename "${item}")" +done + +cd /geoalchemy2 + +# remove links that would cause issues if they are present and read-only +rm -f .mypy_cache .eggs *.egg-info .git .gitignore doc reports + +# copy these items instead of symlinking +cp -r /geoalchemy2_read_only/doc /geoalchemy2/doc +cp /geoalchemy2_read_only/.gitignore ./ + +# store reports in the output directory +mkdir -p /output/reports +ln -s /output/reports /geoalchemy2/reports + +# to allow pre-commit to run +git config --global init.defaultBranch master +git config --global user.email "user@example.com" +git config --global user.name "user" +git init > /dev/null +git add --all +git commit -m "dummy commit" > /dev/null + +export MYPY_CACHE_DIR=/output/.mypy_cache + +############################### + +exec bash diff --git a/test_container/helpers/init_mysql.sh b/test_container/helpers/init_mysql.sh new file mode 100755 index 00000000..a6b7c86f --- /dev/null +++ b/test_container/helpers/init_mysql.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -e + +if [ $(whoami) != "root" ]; then + echo "must run as the root user" + exit 1 +fi + +/etc/init.d/mysql start + +echo "waiting for mysql to start" +while ! mysqladmin ping -h 127.0.0.1 --silent; do + sleep 0.2 +done + +echo "Create the 'gis' role" +mysql -e "CREATE USER 'gis'@'%' IDENTIFIED BY 'gis';" +mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'gis'@'%' WITH GRANT OPTION;" + +echo "Create the 'gis' database" +mysql -u gis --password=gis -e "CREATE DATABASE gis;" diff --git a/test_container/helpers/init_postgres.sh b/test_container/helpers/init_postgres.sh new file mode 100755 index 00000000..2de14287 --- /dev/null +++ b/test_container/helpers/init_postgres.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -e + +if [ $(whoami) != "postgres" ]; then + echo "must run as the postgres user" + exit 1 +fi + +${POSTGRES_PATH}/bin/initdb --auth=trust --username=postgres -E 'UTF-8' + +# by default only listens on localhost so host cannot connect +echo "listen_addresses = '0.0.0.0'" >> "${PGDATA}/postgresql.conf" + +(${POSTGRES_PATH}/bin/pg_ctl start > /dev/null 2>&1) & + +while ! ${POSTGRES_PATH}/bin/pg_isready --quiet; do + sleep 0.2 +done + +echo "Create the 'gis' role" +psql -c "CREATE ROLE gis PASSWORD 'gis' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN;" + +echo "Create the 'gis' database" +createdb -E UTF-8 gis +psql -d gis -c 'CREATE SCHEMA gis;' +psql -c 'GRANT CREATE ON DATABASE gis TO "gis";' +psql -d gis -c 'GRANT USAGE,CREATE ON SCHEMA gis TO "gis";' + + +echo "Enable PostGIS for the 'gis' database" +psql -d gis -c "CREATE EXTENSION postgis;" + +echo "With PostGIS 3 enable PostGIS Raster as well" +psql -d gis -c "CREATE EXTENSION postgis_raster;" + + +${POSTGRES_PATH}/bin/pg_ctl stop +sleep 1 diff --git a/test_container/helpers/install_requirements.sh b/test_container/helpers/install_requirements.sh new file mode 100755 index 00000000..2d8aa7af --- /dev/null +++ b/test_container/helpers/install_requirements.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -e + +# based on geoalchemy2/TEST.rst +packages=( + # for managing virtual environments + tox + git + pypy3 + pypy3-dev + pypy3-venv + python3.7 + python3.7-dev + python3.7-venv + python3.8 + python3.8-dev + python3.8-venv + python3.9 + python3.9-dev + python3.9-venv + python3.10 + python3.10-dev + python3.10-venv + python3.11 + python3.11-dev + python3.11-venv + + # PostgreSQL and PostGIS + postgresql + postgresql-14-postgis-3 + postgresql-14-postgis-3-scripts + libpq-dev + libgeos-dev + + # SpatiaLite + libsqlite3-mod-spatialite + + # MySQL + mysql-client + mysql-server + default-libmysqlclient-dev + + # mysqlclient requirements + # https://github.com/PyMySQL/mysqlclient#linux + default-libmysqlclient-dev + build-essential + pkg-config + + # rasterio requirements with pypy + libgdal-dev +) + +export DEBIAN_FRONTEND=noninteractive + +apt-get update -y +apt-get install --no-install-recommends -y software-properties-common gnupg2 +add-apt-repository ppa:deadsnakes/ppa +apt-get update -y + +apt-get install --no-install-recommends -y "${packages[@]}" + +# clear the package list cache (populated with apt-get update) +rm -rf /var/lib/apt/lists/* diff --git a/test_container/run.sh b/test_container/run.sh new file mode 100755 index 00000000..6a6e815c --- /dev/null +++ b/test_container/run.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +mkdir -p "${SCRIPT_DIR}/output" + +docker run --rm -it \ + --mount type=bind,source="${ROOT}",target=/geoalchemy2_read_only,ro \ + --mount type=bind,source="${SCRIPT_DIR}/output",target=/output \ + geoalchemy2