From 797ffce7386f1f5ef983d6d89f24b13fe5d7cd02 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 5 Aug 2024 20:05:24 +0000 Subject: [PATCH 1/7] task: fix src paths and docker multistage builds --- .dockerignore | 22 ++- .env.template | 2 +- config-hk-download.yaml | 2 +- config-hk-process.yaml | 2 +- deploy/Dockerfile | 48 +++--- pack.sh | 10 ++ poetry.lock | 152 ++++++++++-------- pyproject.toml | 12 +- run-docker.sh | 8 +- src/imap_db/alembic.ini | 2 +- src/imap_db/migrations/env.py | 3 +- src/imap_mag/DB.py | 8 +- src/imap_mag/imapProcessing.py | 39 ++++- src/imap_mag/main.py | 11 +- .../calibration/CalibrationApplicator.py | 8 +- src/mag_toolkit/calibration/Calibrator.py | 4 +- .../calibration/calibrationFormatProcessor.py | 2 +- 17 files changed, 210 insertions(+), 125 deletions(-) diff --git a/.dockerignore b/.dockerignore index a9eb655..024c919 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,18 @@ +# Ignore everything +* + +# Allow files and directories +!/dist +#!/dist/file.txt + +# Ignore unnecessary files inside allowed directories +# This should go after the allowed directories +**/*~ +**/*.log +**/.DS_Store +**/Thumbs.db + +#Ignore python crap **/__pycache__ **/.venv **/.classpath @@ -17,7 +32,6 @@ **/*.jfm **/bin **/build - **/.work **/htmlcov **/output @@ -30,17 +44,13 @@ **/obj **/secrets.dev.yaml **/values.dev.yaml - test-results.xml .coverage coverage.xml .coveragerc -*.sh - poetry.* pyproject.* pyrightconfig.json README.md - LICENSE -README.md +CHANGELOG.md diff --git a/.env.template b/.env.template index b7e0965..2475852 100644 --- a/.env.template +++ b/.env.template @@ -1,5 +1,5 @@ # DB Connection string -SQLALCHEMY_URL=postgresql://USER_HERE:PASSWORD_HERE@db:5432/imap +SQLALCHEMY_URL=postgresql+psycopg://USER_HERE:PASSWORD_HERE@db:5432/imap # API credentials fpr LASP PODA API - See https://lasp.colorado.edu/ops/imap/poda/ WEBPODA_AUTH_CODE= diff --git a/config-hk-download.yaml b/config-hk-download.yaml index 8e3c264..cc371d0 100644 --- a/config-hk-download.yaml +++ b/config-hk-download.yaml @@ -8,4 +8,4 @@ destination: filename: power.pkts packet-definition: - hk: src/imap_mag/xtce/tlm_20240724.xml + hk: xtce/tlm_20240724.xml diff --git a/config-hk-process.yaml b/config-hk-process.yaml index 9f4fc4b..bd0477f 100644 --- a/config-hk-process.yaml +++ b/config-hk-process.yaml @@ -8,4 +8,4 @@ destination: filename: result.csv packet-definition: - hk: src/imap_mag/xtce/tlm_20240724.xml + hk: xtce/tlm_20240724.xml diff --git a/deploy/Dockerfile b/deploy/Dockerfile index f4fcb97..9cf384d 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,25 +1,36 @@ -# For more information, please refer to https://aka.ms/vscode-docker-python -FROM python:3.12-slim - ARG PYTHON_VERSION="3.12" -ARG TOOL_PACKAGE="imap_mag-*.tar.gz" +FROM python:${PYTHON_VERSION}-slim AS compile-image + +ARG TOOL_PACKAGE="imap_mag-*.tar.gz" +ARG PYTHON_VERSION # Keeps Python from generating .pyc files in the container ENV PYTHONDONTWRITEBYTECODE=1 # Turns off buffering for easier container logging ENV PYTHONUNBUFFERED=1 -COPY deploy/entrypoint.sh /app/entrypoint.sh -COPY *.yaml /app/ - -#TODO: this is a hack mounting the src folder to the container -COPY src/imap_mag/xtce/*.xml /app/src/imap_mag/xtce/ - -COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} /app/python${PYTHON_VERSION}/ +# Install the postgres client and any other compile time dependencies needed to build our app +RUN apt-get update && apt-get install -y libpq-dev gcc # Creates a non-root user with an explicit UID and adds permission to access the /app folder # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers +RUN adduser -u 5678 --disabled-password --gecos "" appuser + +USER appuser + +# first restore the python dependencies in a single cache layer - should get faster builds as this changes rarely +COPY dist/python${PYTHON_VERSION}/requirements.txt . +RUN python3 -m pip install --user -r requirements.txt + +# now install the imap-mag package as the user +COPY dist/python${PYTHON_VERSION}/${TOOL_PACKAGE} python${PYTHON_VERSION}/ +RUN python3 -m pip install --user python${PYTHON_VERSION}/${TOOL_PACKAGE} + +# create the runtime image and copy in the installed packed from compile-image +FROM python:${PYTHON_VERSION}-slim AS runtime-image +COPY --from=compile-image /home/appuser/.local /home/appuser/.local +COPY dist/docker /app RUN adduser -u 5678 --disabled-password --gecos "" appuser && \ chown -R appuser /app && \ @@ -27,21 +38,18 @@ RUN adduser -u 5678 --disabled-password --gecos "" appuser && \ chown -R appuser /data && \ chmod +x /app/entrypoint.sh -# Install the postgres client and any other dependencies needed to install our app -RUN apt-get update && apt-get install -y libpq-dev gcc - WORKDIR /app USER appuser -RUN ls -la . -RUN python3 -m pip install --user pipx && \ - python3 -m pipx install --python python${PYTHON_VERSION} python${PYTHON_VERSION}/${TOOL_PACKAGE} - +# Make sure scripts in .local are usable: ENV PATH="$PATH:/home/appuser/.local/bin" +# DELETE ME +RUN ls -la . + # Now the imap-mag CLI is available on the path and -# the 3.12 python package for imap-mag is intalled at /app/python3.12/imap_mag-[VERSION].tar.gz +# the 3.12 python package for imap-mag is intalled at /home/appuser/.local/bin # During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug CMD ["/app/entrypoint.sh"] -#CMD ["python", "src/main.py"] + diff --git a/pack.sh b/pack.sh index a4a8927..9d02d82 100755 --- a/pack.sh +++ b/pack.sh @@ -16,6 +16,16 @@ echo "Packing version $(poetry version --short) for $(python3 --version) into $O poetry lock poetry build +# output a requierments.txt file used by docker during the build +poetry self add poetry-plugin-export +poetry export --format=requirements.txt > dist/requirements.txt + # move the files into a folder with the python version mkdir -p dist/python$PYTHON_VERSION find dist/ -maxdepth 1 -type f -name '*' -exec mv {} dist/python$PYTHON_VERSION \; + +# setup a docker folder that will be copied into the docker output at /app +dockerFolder=dist/docker +mkdir -p $dockerFolder +cp deploy/entrypoint.sh $dockerFolder +cp *.yaml $dockerFolder diff --git a/poetry.lock b/poetry.lock index b42b8ff..8f5dc2b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -984,25 +984,89 @@ pyyaml = ">=5.1" virtualenv = ">=20.10.0" [[package]] -name = "psycopg2" -version = "2.9.9" -description = "psycopg2 - Python-PostgreSQL Database Adapter" +name = "psycopg" +version = "3.2.1" +description = "PostgreSQL database adapter for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "psycopg-3.2.1-py3-none-any.whl", hash = "sha256:ece385fb413a37db332f97c49208b36cf030ff02b199d7635ed2fbd378724175"}, + {file = "psycopg-3.2.1.tar.gz", hash = "sha256:dc8da6dc8729dacacda3cc2f17d2c9397a70a66cf0d2b69c91065d60d5f00cb7"}, +] + +[package.dependencies] +psycopg-binary = {version = "3.2.1", optional = true, markers = "implementation_name != \"pypy\" and extra == \"binary\""} +typing-extensions = ">=4.4" +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.2.1)"] +c = ["psycopg-c (==3.2.1)"] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.6)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.6)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + +[[package]] +name = "psycopg-binary" +version = "3.2.1" +description = "PostgreSQL database adapter for Python -- C optimisation distribution" +optional = false +python-versions = ">=3.8" files = [ - {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, - {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, - {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, - {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, - {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, - {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, - {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, - {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, - {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, - {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, - {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, - {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:cad2de17804c4cfee8640ae2b279d616bb9e4734ac3c17c13db5e40982bd710d"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:592b27d6c46a40f9eeaaeea7c1fef6f3c60b02c634365eb649b2d880669f149f"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a997efbaadb5e1a294fb5760e2f5643d7b8e4e3fe6cb6f09e6d605fd28e0291"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1d2b6438fb83376f43ebb798bf0ad5e57bc56c03c9c29c85bc15405c8c0ac5a"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1f087bd84bdcac78bf9f024ebdbfacd07fc0a23ec8191448a50679e2ac4a19e"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:415c3b72ea32119163255c6504085f374e47ae7345f14bc3f0ef1f6e0976a879"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f092114f10f81fb6bae544a0ec027eb720e2d9c74a4fcdaa9dd3899873136935"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06a7aae34edfe179ddc04da005e083ff6c6b0020000399a2cbf0a7121a8a22ea"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0b018631e5c80ce9bc210b71ea885932f9cca6db131e4df505653d7e3873a938"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f8a509aeaac364fa965454e80cd110fe6d48ba2c80f56c9b8563423f0b5c3cfd"}, + {file = "psycopg_binary-3.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:413977d18412ff83486eeb5875eb00b185a9391c57febac45b8993bf9c0ff489"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:62b1b7b07e00ee490afb39c0a47d8282a9c2822c7cfed9553a04b0058adf7e7f"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8afb07114ea9b924a4a0305ceb15354ccf0ef3c0e14d54b8dbeb03e50182dd7"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40bb515d042f6a345714ec0403df68ccf13f73b05e567837d80c886c7c9d3805"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6418712ba63cebb0c88c050b3997185b0ef54173b36568522d5634ac06153040"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:101472468d59c74bb8565fab603e032803fd533d16be4b2d13da1bab8deb32a3"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa3931f308ab4a479d0ee22dc04bea867a6365cac0172e5ddcba359da043854b"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dc314a47d44fe1a8069b075a64abffad347a3a1d8652fed1bab5d3baea37acb2"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cc304a46be1e291031148d9d95c12451ffe783ff0cc72f18e2cc7ec43cdb8c68"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f9e13600647087df5928875559f0eb8f496f53e6278b7da9511b4b3d0aff960"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b140182830c76c74d17eba27df3755a46442ce8d4fb299e7f1cf2f74a87c877b"}, + {file = "psycopg_binary-3.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:3c838806eeb99af39f934b7999e35f947a8e577997cc892c12b5053a97a9057f"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7066d3dca196ed0dc6172f9777b2d62e4f138705886be656cccff2d555234d60"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:28ada5f610468c57d8a4a055a8ea915d0085a43d794266c4f3b9d02f4288f4db"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e8213bf50af073b1aa8dc3cff123bfeedac86332a16c1b7274910bc88a847c7"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74d623261655a169bc84a9669890975c229f2fa6e19a7f2d10a77675dcf1a707"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42781ba94e8842ee98bca5a7d0c44cc9d067500fedca2d6a90fa3609b6d16b42"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e6669091d09f8ba36e10ce678a6d9916e110446236a9b92346464a3565635e"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b09e8a576a2ac69d695032ee76f31e03b30781828b5dd6d18c6a009e5a3d1c35"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8f28ff0cb9f1defdc4a6f8c958bf6787274247e7dfeca811f6e2f56602695fb1"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4c84fcac8a3a3479ac14673095cc4e1fdba2935499f72c436785ac679bec0d1a"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:950fd666ec9e9fe6a8eeb2b5a8f17301790e518953730ad44d715b59ffdbc67f"}, + {file = "psycopg_binary-3.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:334046a937bb086c36e2c6889fe327f9f29bfc085d678f70fac0b0618949f674"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:1d6833f607f3fc7b22226a9e121235d3b84c0eda1d3caab174673ef698f63788"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d353e028b8f848b9784450fc2abf149d53a738d451eab3ee4c85703438128b9"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f34e369891f77d0738e5d25727c307d06d5344948771e5379ea29c76c6d84555"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ab58213cc976a1666f66bc1cb2e602315cd753b7981a8e17237ac2a185bd4a1"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0104a72a17aa84b3b7dcab6c84826c595355bf54bb6ea6d284dcb06d99c6801"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:059cbd4e6da2337e17707178fe49464ed01de867dc86c677b30751755ec1dc51"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:73f9c9b984be9c322b5ec1515b12df1ee5896029f5e72d46160eb6517438659c"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:af0469c00f24c4bec18c3d2ede124bf62688d88d1b8a5f3c3edc2f61046fe0d7"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:463d55345f73ff391df8177a185ad57b552915ad33f5cc2b31b930500c068b22"}, + {file = "psycopg_binary-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:302b86f92c0d76e99fe1b5c22c492ae519ce8b98b88d37ef74fda4c9e24c6b46"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:0879b5d76b7d48678d31278242aaf951bc2d69ca4e4d7cef117e4bbf7bfefda9"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f99e59f8a5f4dcd9cbdec445f3d8ac950a492fc0e211032384d6992ed3c17eb7"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84837e99353d16c6980603b362d0f03302d4b06c71672a6651f38df8a482923d"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ce965caf618061817f66c0906f0452aef966c293ae0933d4fa5a16ea6eaf5bb"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78c2007caf3c90f08685c5378e3ceb142bafd5636be7495f7d86ec8a977eaeef"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7a84b5eb194a258116154b2a4ff2962ea60ea52de089508db23a51d3d6b1c7d1"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4a42b8f9ab39affcd5249b45cac763ac3cf12df962b67e23fd15a2ee2932afe5"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:788ffc43d7517c13e624c83e0e553b7b8823c9655e18296566d36a829bfb373f"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:21927f41c4d722ae8eb30d62a6ce732c398eac230509af5ba1749a337f8a63e2"}, + {file = "psycopg_binary-3.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:921f0c7f39590763d64a619de84d1b142587acc70fd11cbb5ba8fa39786f3073"}, ] [[package]] @@ -1467,60 +1531,12 @@ bitstring = ">=4.0.1" [[package]] name = "sqlalchemy" -version = "2.0.31" +version = "2.0.32" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, - {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, - {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, + {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, ] [package.dependencies] @@ -1840,4 +1856,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "9f5ddbc86d7476e8e2f31f45bb941b5679d3f3976c92a4a5b744c8ab24ecc224" +content-hash = "00a92fb906cc93d1a11fd65e4e3ed1b04161ed8cbc4fca3c41d518d0b39839f9" diff --git a/pyproject.toml b/pyproject.toml index ad093bd..da156cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,9 +9,9 @@ description = "Process IMAP data" authors = ["alastairtree"] readme = "README.md" packages = [ - { include = "src/imap_mag" }, - { include = "src/imap_db" }, - { include = "src/mag_toolkit" }, + { include = "imap_mag", from = "src" }, + { include = "imap_db", from = "src" }, + { include = "mag_toolkit", from = "src" }, ] [tool.poetry.dependencies] @@ -24,13 +24,13 @@ xarray = "^2024.7.0" numpy = "^2.0.1" typer = "^0.12.3" sqlalchemy = "^2.0.31" -psycopg2 = "^2.9.9" alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" imap-data-access = "^0.7.0" cdflib = "^1.3.1" +psycopg = {extras = ["binary"], version = "^3.2.1"} [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" @@ -44,8 +44,8 @@ testcontainers = "^4.7.2" [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` -imap-mag = 'src.imap_mag.main:app' -imap-db = 'src.imap_db.main:app' +imap-mag = 'imap_mag.main:app' +imap-db = 'imap_db.main:app' [tool.pytest.ini_options] pythonpath = [ diff --git a/run-docker.sh b/run-docker.sh index c7122b6..abc52e7 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -4,9 +4,13 @@ set -e IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" +# docker run --rm -it \ +# --entrypoint /bin/bash \ +# --env-file dev.env \ +# -v /mnt/imap-data:/data \ +# $IMAGE_NAME + docker run --rm -it \ - --entrypoint /bin/sh \ --env-file dev.env \ -v /mnt/imap-data:/data \ $IMAGE_NAME - diff --git a/src/imap_db/alembic.ini b/src/imap_db/alembic.ini index e98fde2..3692529 100644 --- a/src/imap_db/alembic.ini +++ b/src/imap_db/alembic.ini @@ -5,7 +5,7 @@ script_location = src/imap_db/migrations #sqlalchemy.url = postgresql://postgres:postgres@db/imap -sqlalchemy.url = postgresql://postgres:postgres@host.docker.internal:5432/imap +sqlalchemy.url = postgresql+psycopg://postgres:postgres@host.docker.internal:5432/imap # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time diff --git a/src/imap_db/migrations/env.py b/src/imap_db/migrations/env.py index 8dcfe34..b5add4c 100644 --- a/src/imap_db/migrations/env.py +++ b/src/imap_db/migrations/env.py @@ -1,10 +1,9 @@ from logging.config import fileConfig from alembic import context +from imap_db.model import Base from sqlalchemy import engine_from_config, pool -from src.imap_db.model import Base - # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 9dff8f3..3ca15e0 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,10 +1,9 @@ import os +from imap_db.model import File from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from src.imap_db.model import File - class DB: def __init__(self, db_url=None): @@ -12,6 +11,11 @@ def __init__(self, db_url=None): if db_url is None and env_url is not None: db_url = env_url + if db_url is None: + raise ValueError( + "No database URL provided. Consider setting SQLALCHEMY_URL environment variable." + ) + self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) diff --git a/src/imap_mag/imapProcessing.py b/src/imap_mag/imapProcessing.py index fb800ee..6331906 100644 --- a/src/imap_mag/imapProcessing.py +++ b/src/imap_mag/imapProcessing.py @@ -1,6 +1,7 @@ import abc import collections import logging +import os import re from pathlib import Path @@ -34,10 +35,44 @@ class HKProcessor(FileProcessor): xtcePacketDefinition: Path def initialize(self, config: appConfig.AppConfig) -> None: - self.xtcePacketDefinition = config.packet_definition.hk + # first try the file path as is, then in the same directory as the module, then fallback to a default + pythonModuleRelativePath = Path( + os.path.join(os.path.dirname(__file__), config.packet_definition.hk) + ) + defaultFallbackPath = Path("tlm.xml") + logging.debug( + "Trying XTCE packet definition file from these paths in turn: \n %s\n %s\n %s\n", + config.packet_definition.hk, + pythonModuleRelativePath, + defaultFallbackPath, + ) + if ( + config.packet_definition is not None + and config.packet_definition.hk is not None + and config.packet_definition.hk.exists() + ): + logging.debug( + "Using XTCE packet definition file from relative path: %s", + config.packet_definition.hk, + ) + self.xtcePacketDefinition = config.packet_definition.hk + # otherwise try path relative to the module + elif pythonModuleRelativePath.exists(): + logging.debug( + "Using XTCE packet definition file from module path: %s", + pythonModuleRelativePath, + ) + self.xtcePacketDefinition = pythonModuleRelativePath + else: + logging.debug( + "Using XTCE packet definition file from default path: %s", + defaultFallbackPath, + ) + self.xtcePacketDefinition = defaultFallbackPath + if not self.xtcePacketDefinition.exists(): raise FileNotFoundError( - f"XTCE packet definition file not found: {self.xtcePacketDefinition}" + f"XTCE packet definition file not found: {config.packet_definition.hk}" ) def process(self, file: Path) -> Path: diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index ddfe553..1763ac6 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -13,14 +13,13 @@ # config import yaml - -from src.imap_db.model import File -from src.mag_toolkit import CDFLoader -from src.mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator -from src.mag_toolkit.calibration.calibrationFormatProcessor import ( +from imap_db.model import File +from mag_toolkit import CDFLoader +from mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator +from mag_toolkit.calibration.calibrationFormatProcessor import ( CalibrationFormatProcessor, ) -from src.mag_toolkit.calibration.Calibrator import ( +from mag_toolkit.calibration.Calibrator import ( Calibrator, CalibratorType, SpinAxisCalibrator, diff --git a/src/mag_toolkit/calibration/CalibrationApplicator.py b/src/mag_toolkit/calibration/CalibrationApplicator.py index 6c5c7d2..6d7de36 100644 --- a/src/mag_toolkit/calibration/CalibrationApplicator.py +++ b/src/mag_toolkit/calibration/CalibrationApplicator.py @@ -3,12 +3,12 @@ import numpy as np -from src.mag_toolkit.calibration.CalibrationExceptions import CalibrationValidityError -from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat -from src.mag_toolkit.calibration.calibrationFormatProcessor import ( +from ..CDFLoader import load_cdf, write_cdf +from .CalibrationExceptions import CalibrationValidityError +from .calibrationFormat import CalibrationFormat +from .calibrationFormatProcessor import ( CalibrationFormatProcessor, ) -from src.mag_toolkit.CDFLoader import load_cdf, write_cdf class CalibrationApplicator: diff --git a/src/mag_toolkit/calibration/Calibrator.py b/src/mag_toolkit/calibration/Calibrator.py index 75b857b..c5b5126 100644 --- a/src/mag_toolkit/calibration/Calibrator.py +++ b/src/mag_toolkit/calibration/Calibrator.py @@ -2,8 +2,8 @@ from datetime import datetime from enum import Enum -from src.mag_toolkit.calibration import MatlabWrapper -from src.mag_toolkit.calibration.calibrationFormat import ( +from . import MatlabWrapper +from .calibrationFormat import ( CalibrationFormat, OffsetCollection, SingleCalibration, diff --git a/src/mag_toolkit/calibration/calibrationFormatProcessor.py b/src/mag_toolkit/calibration/calibrationFormatProcessor.py index fa53d1a..11f0a74 100644 --- a/src/mag_toolkit/calibration/calibrationFormatProcessor.py +++ b/src/mag_toolkit/calibration/calibrationFormatProcessor.py @@ -4,7 +4,7 @@ import yaml from pydantic import ValidationError -from src.mag_toolkit.calibration.calibrationFormat import CalibrationFormat +from .calibrationFormat import CalibrationFormat class CalibrationFormatProcessor: From f27c1423bbafba87f4d010f9ab1ad28825a8a8ac Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 5 Aug 2024 20:05:55 +0000 Subject: [PATCH 2/7] task: tidy dockerfile --- deploy/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 9cf384d..95e4c1f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -44,9 +44,6 @@ USER appuser # Make sure scripts in .local are usable: ENV PATH="$PATH:/home/appuser/.local/bin" -# DELETE ME -RUN ls -la . - # Now the imap-mag CLI is available on the path and # the 3.12 python package for imap-mag is intalled at /home/appuser/.local/bin From f321588893a1b61fa6be775fade0fdae10fabbb8 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Tue, 6 Aug 2024 08:55:44 +0000 Subject: [PATCH 3/7] feat: update precommit --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10cbade..080c7f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: hooks: - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.5.6 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -36,7 +36,7 @@ repos: - id: markdownlint-fix args: [--ignore, LICENSE.md] - repo: https://github.com/python-poetry/poetry - rev: "1.8" + rev: "1.8.0" hooks: - id: poetry-check - id: poetry-lock From 618011533a2d3e3966b8effd0a307fbad98723c5 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Tue, 6 Aug 2024 12:04:51 +0100 Subject: [PATCH 4/7] task: remove poetry lock pre-commit as it fails in CI for no good reason --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 080c7f0..3f73e6b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,4 +39,3 @@ repos: rev: "1.8.0" hooks: - id: poetry-check - - id: poetry-lock From a21be3b1b9c4e865524a33531a91879d037b1670 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Tue, 6 Aug 2024 14:16:29 +0000 Subject: [PATCH 5/7] bug: name local docker image so can run pipeline platform --- build-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-docker.sh b/build-docker.sh index d87c162..251899b 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -4,7 +4,7 @@ set -e CLI_TOOL="imap_mag" TOOL_PYTHON_VERSION="${TOOL_PYTHON_VERSION:-python3.12}" TOOL_PACKAGE="${TOOL_PACKAGE:-$CLI_TOOL-*.tar.gz}" -IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" +IMAGE_NAME="${IMAGE_NAME:-ghcr.io/imperialcollegelondon/imap-pipeline-core:latest-local-dev}" if [ ! -f dist/$TOOL_PYTHON_VERSION/$TOOL_PACKAGE ] then From 8ea1e492cce75f82537301e4ce1d3758e7731101 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Tue, 6 Aug 2024 14:16:44 +0000 Subject: [PATCH 6/7] bug: better error handling for missing files --- src/imap_mag/main.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index 1763ac6..cba66fd 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -94,13 +94,17 @@ def hello(name: str): print(f"Hello {name}") -def prepareWorkFile(file, configFile): +def prepareWorkFile(file, configFile) -> Path | None: logging.debug(f"Grabbing file matching {file} in {configFile.source.folder}") # get all files in \\RDS.IMPERIAL.AC.UK\rds\project\solarorbitermagnetometer\live\SO-MAG-Web\quicklooks_py\ files = [] folder = configFile.source.folder + if not folder.exists(): + logging.warning(f"Folder {folder} does not exist") + return None + # if pattern contains a % if "%" in file: updatedFile = datetime.now().strftime(file) @@ -150,7 +154,12 @@ def process( workFile = prepareWorkFile(file, configFile) - # TODO: do something with the data! + if workFile is None: + logging.critical( + "Unable to find a file to process in %s", configFile.source.folder + ) + raise typer.Abort() + fileProcessor = imapProcessing.dispatchFile(workFile) fileProcessor.initialize(configFile) result = fileProcessor.process(workFile) @@ -267,6 +276,13 @@ def calibrate( configFile: appConfig.AppConfig = commandInit(config) workFile = prepareWorkFile(input, configFile) + + if workFile is None: + logging.critical( + "Unable to find a file to process in %s", configFile.source.folder + ) + raise typer.Abort() + calibrator: Calibrator match method: From 8057d74bd6c4770a492b43b94a16f092d448989f Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Fri, 9 Aug 2024 15:58:39 +0000 Subject: [PATCH 7/7] task: improve run-docker.sh --- run-docker.sh | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/run-docker.sh b/run-docker.sh index abc52e7..04960e7 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -1,16 +1,32 @@ #!/bin/bash set -e -IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" +# Example: +# ./run-docker.sh +# ./run-docker.sh -i // for interactive debugging +# ./run-docker.sh -i -e VARIABLE=some_value // for interactive debugging with VARIABLE env var set +IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" -# docker run --rm -it \ -# --entrypoint /bin/bash \ -# --env-file dev.env \ -# -v /mnt/imap-data:/data \ -# $IMAGE_NAME +echo "Running $IMAGE_NAME with dev.env file" -docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME +# check if the argument "DEBUG" or "-i" is passed +if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then + echo "Overriding entrypoint to be an interactive bash shell for debugging" + docker run --rm -it \ + --entrypoint /bin/bash \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +elif [ -z "$1" ]; then # no args passed + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +else + echo "Extra arguments: $@" + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $@ $IMAGE_NAME +fi