diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6474864..eb6693c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,6 +34,7 @@ "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}", "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}", "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}", + "SQLALCHEMY_URL": "${localEnv:SQLALCHEMY_URL}", // Define WireMock variables to connect Docker outside of Docker. "WIREMOCK_DIND": "1", "TESTCONTAINERS_HOST_OVERRIDE": "host.docker.internal" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 811e3a2..4c6d408 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,11 +16,14 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +# write to checks/pull-request extra permission needed by 5monkeys/cobertura-action to post coverage stats +# write packages needed by docker image step permissions: id-token: write contents: write checks: write packages: write + pull-requests: write env: PREFERED_PYTHON_VERSION: '3.12' @@ -150,6 +153,13 @@ jobs: path: 'test-results.xml' reporter: java-junit + - name: Coverage Report + uses: 5monkeys/cobertura-action@v14 + with: + report_name: Coverage Report (${{ matrix.python-versions }}) + path: "coverage.xml" + minimum_coverage: 80 + - name: Create Release ${{github.ref_name}} & upload artifacts uses: softprops/action-gh-release@v2 if: ${{ startsWith(github.ref, 'refs/tags/') }} @@ -160,6 +170,44 @@ jobs: files: | dist/${{ env.PACKAGE_NAME }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }}.zip + test_on_windows: + strategy: + matrix: + python-versions: ['3.10', '3.11', '3.12'] + os: [windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-versions }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install + + - name: Run tests + run: poetry run pytest -s --cov-config=.coveragerc --cov=src --cov-append --cov-report=xml --cov-report term-missing --cov-report=html --junitxml=test-results.xml tests + + - name: Upload Coverage report + uses: actions/upload-artifact@v4 + if: matrix.python-versions == env.PREFERED_PYTHON_VERSION + with: + name: CoverageReport_${{ matrix.os }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }} + path: htmlcov + if-no-files-found: error + + - name: Test Report + uses: dorny/test-reporter@v1 + if: success() || failure() + with: + name: Test Results (${{ matrix.os }}) (${{ matrix.python-versions }}) + path: 'test-results.xml' + reporter: java-junit + + build_single_file_binary: strategy: matrix: diff --git a/.gitignore b/.gitignore index 7a6fd75..ae89700 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ site/ .work /output dev.env +debug diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 95e4c1f..0fc51a4 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,7 +11,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # Install the postgres client and any other compile time dependencies needed to build our app -RUN apt-get update && apt-get install -y libpq-dev gcc +RUN apt-get update && apt-get install -y libpq-dev gcc git # Creates a non-root user with an explicit UID and adds permission to access the /app folder # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers diff --git a/pack.sh b/pack.sh index 9d02d82..d290346 100755 --- a/pack.sh +++ b/pack.sh @@ -18,7 +18,7 @@ poetry build # output a requierments.txt file used by docker during the build poetry self add poetry-plugin-export -poetry export --format=requirements.txt > dist/requirements.txt +poetry export --without-hashes --format=requirements.txt > dist/requirements.txt # move the files into a folder with the python version mkdir -p dist/python$PYTHON_VERSION diff --git a/poetry.lock b/poetry.lock index 8f5dc2b..d2f7638 100644 --- a/poetry.lock +++ b/poetry.lock @@ -599,12 +599,12 @@ files = [ [[package]] name = "imap-data-access" -version = "0.7.0" +version = "0.9.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" files = [ - {file = "imap_data_access-0.7.0.tar.gz", hash = "sha256:f0db935949d048394fc554b308b1e4a1572a18acd41636462d37c309c7cb4c9d"}, + {file = "imap_data_access-0.9.0.tar.gz", hash = "sha256:997084118c85455d1c977d5640a8654717a1f8adf39eaeccfc3e124b5efd3c4f"}, ] [package.extras] @@ -787,56 +787,63 @@ files = [ [[package]] name = "numpy" -version = "2.0.1" +version = "2.1.0" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" -files = [ - {file = "numpy-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fbb536eac80e27a2793ffd787895242b7f18ef792563d742c2d673bfcb75134"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ff563d43c69b1baba77af455dd0a839df8d25e8590e79c90fcbe1499ebde42"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:1b902ce0e0a5bb7704556a217c4f63a7974f8f43e090aff03fcf262e0b135e02"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:f1659887361a7151f89e79b276ed8dff3d75877df906328f14d8bb40bb4f5101"}, - {file = "numpy-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4658c398d65d1b25e1760de3157011a80375da861709abd7cef3bad65d6543f9"}, - {file = "numpy-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4127d4303b9ac9f94ca0441138acead39928938660ca58329fe156f84b9f3015"}, - {file = "numpy-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e5eeca8067ad04bc8a2a8731183d51d7cbaac66d86085d5f4766ee6bf19c7f87"}, - {file = "numpy-2.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9adbd9bb520c866e1bfd7e10e1880a1f7749f1f6e5017686a5fbb9b72cf69f82"}, - {file = "numpy-2.0.1-cp310-cp310-win32.whl", hash = "sha256:7b9853803278db3bdcc6cd5beca37815b133e9e77ff3d4733c247414e78eb8d1"}, - {file = "numpy-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:81b0893a39bc5b865b8bf89e9ad7807e16717f19868e9d234bdaf9b1f1393868"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75b4e316c5902d8163ef9d423b1c3f2f6252226d1aa5cd8a0a03a7d01ffc6268"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6e4eeb6eb2fced786e32e6d8df9e755ce5be920d17f7ce00bc38fcde8ccdbf9e"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a1e01dcaab205fbece13c1410253a9eea1b1c9b61d237b6fa59bcc46e8e89343"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a8fc2de81ad835d999113ddf87d1ea2b0f4704cbd947c948d2f5513deafe5a7b"}, - {file = "numpy-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a3d94942c331dd4e0e1147f7a8699a4aa47dffc11bf8a1523c12af8b2e91bbe"}, - {file = "numpy-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15eb4eca47d36ec3f78cde0a3a2ee24cf05ca7396ef808dda2c0ddad7c2bde67"}, - {file = "numpy-2.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b83e16a5511d1b1f8a88cbabb1a6f6a499f82c062a4251892d9ad5d609863fb7"}, - {file = "numpy-2.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f87fec1f9bc1efd23f4227becff04bd0e979e23ca50cc92ec88b38489db3b55"}, - {file = "numpy-2.0.1-cp311-cp311-win32.whl", hash = "sha256:36d3a9405fd7c511804dc56fc32974fa5533bdeb3cd1604d6b8ff1d292b819c4"}, - {file = "numpy-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:08458fbf403bff5e2b45f08eda195d4b0c9b35682311da5a5a0a0925b11b9bd8"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bf4e6f4a2a2e26655717a1983ef6324f2664d7011f6ef7482e8c0b3d51e82ac"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6fddc5fe258d3328cd8e3d7d3e02234c5d70e01ebe377a6ab92adb14039cb4"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5daab361be6ddeb299a918a7c0864fa8618af66019138263247af405018b04e1"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:ea2326a4dca88e4a274ba3a4405eb6c6467d3ffbd8c7d38632502eaae3820587"}, - {file = "numpy-2.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529af13c5f4b7a932fb0e1911d3a75da204eff023ee5e0e79c1751564221a5c8"}, - {file = "numpy-2.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6790654cb13eab303d8402354fabd47472b24635700f631f041bd0b65e37298a"}, - {file = "numpy-2.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cbab9fc9c391700e3e1287666dfd82d8666d10e69a6c4a09ab97574c0b7ee0a7"}, - {file = "numpy-2.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:99d0d92a5e3613c33a5f01db206a33f8fdf3d71f2912b0de1739894668b7a93b"}, - {file = "numpy-2.0.1-cp312-cp312-win32.whl", hash = "sha256:173a00b9995f73b79eb0191129f2455f1e34c203f559dd118636858cc452a1bf"}, - {file = "numpy-2.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:bb2124fdc6e62baae159ebcfa368708867eb56806804d005860b6007388df171"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc085b28d62ff4009364e7ca34b80a9a080cbd97c2c0630bb5f7f770dae9414"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8fae4ebbf95a179c1156fab0b142b74e4ba4204c87bde8d3d8b6f9c34c5825ef"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:72dc22e9ec8f6eaa206deb1b1355eb2e253899d7347f5e2fae5f0af613741d06"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:ec87f5f8aca726117a1c9b7083e7656a9d0d606eec7299cc067bb83d26f16e0c"}, - {file = "numpy-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f682ea61a88479d9498bf2091fdcd722b090724b08b31d63e022adc063bad59"}, - {file = "numpy-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8efc84f01c1cd7e34b3fb310183e72fcdf55293ee736d679b6d35b35d80bba26"}, - {file = "numpy-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3fdabe3e2a52bc4eff8dc7a5044342f8bd9f11ef0934fcd3289a788c0eb10018"}, - {file = "numpy-2.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:24a0e1befbfa14615b49ba9659d3d8818a0f4d8a1c5822af8696706fbda7310c"}, - {file = "numpy-2.0.1-cp39-cp39-win32.whl", hash = "sha256:f9cf5ea551aec449206954b075db819f52adc1638d46a6738253a712d553c7b4"}, - {file = "numpy-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:e9e81fa9017eaa416c056e5d9e71be93d05e2c3c2ab308d23307a8bc4443c368"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61728fba1e464f789b11deb78a57805c70b2ed02343560456190d0501ba37b0f"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:12f5d865d60fb9734e60a60f1d5afa6d962d8d4467c120a1c0cda6eb2964437d"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eacf3291e263d5a67d8c1a581a8ebbcfd6447204ef58828caf69a5e3e8c75990"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2c3a346ae20cfd80b6cfd3e60dc179963ef2ea58da5ec074fd3d9e7a1e7ba97f"}, - {file = "numpy-2.0.1.tar.gz", hash = "sha256:485b87235796410c3519a699cfe1faab097e509e90ebb05dcd098db2ae87e7b3"}, +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b"}, + {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b"}, + {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f"}, + {file = "numpy-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84"}, + {file = "numpy-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33"}, + {file = "numpy-2.1.0-cp310-cp310-win32.whl", hash = "sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211"}, + {file = "numpy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, + {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, + {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, + {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, + {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, + {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, + {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8"}, + {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745"}, + {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111"}, + {file = "numpy-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0"}, + {file = "numpy-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574"}, + {file = "numpy-2.1.0-cp312-cp312-win32.whl", hash = "sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02"}, + {file = "numpy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8ab81ccd753859ab89e67199b9da62c543850f819993761c1e94a75a814ed667"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:442596f01913656d579309edcd179a2a2f9977d9a14ff41d042475280fc7f34e"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:848c6b5cad9898e4b9ef251b6f934fa34630371f2e916261070a4eb9092ffd33"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:54c6a63e9d81efe64bfb7bcb0ec64332a87d0b87575f6009c8ba67ea6374770b"}, + {file = "numpy-2.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:652e92fc409e278abdd61e9505649e3938f6d04ce7ef1953f2ec598a50e7c195"}, + {file = "numpy-2.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ab32eb9170bf8ffcbb14f11613f4a0b108d3ffee0832457c5d4808233ba8977"}, + {file = "numpy-2.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:8fb49a0ba4d8f41198ae2d52118b050fd34dace4b8f3fb0ee34e23eb4ae775b1"}, + {file = "numpy-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44e44973262dc3ae79e9063a1284a73e09d01b894b534a769732ccd46c28cc62"}, + {file = "numpy-2.1.0-cp313-cp313-win32.whl", hash = "sha256:ab83adc099ec62e044b1fbb3a05499fa1e99f6d53a1dde102b2d85eff66ed324"}, + {file = "numpy-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:de844aaa4815b78f6023832590d77da0e3b6805c644c33ce94a1e449f16d6ab5"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:343e3e152bf5a087511cd325e3b7ecfd5b92d369e80e74c12cd87826e263ec06"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f07fa2f15dabe91259828ce7d71b5ca9e2eb7c8c26baa822c825ce43552f4883"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5474dad8c86ee9ba9bb776f4b99ef2d41b3b8f4e0d199d4f7304728ed34d0300"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1f817c71683fd1bb5cff1529a1d085a57f02ccd2ebc5cd2c566f9a01118e3b7d"}, + {file = "numpy-2.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a3336fbfa0d38d3deacd3fe7f3d07e13597f29c13abf4d15c3b6dc2291cbbdd"}, + {file = "numpy-2.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a894c51fd8c4e834f00ac742abad73fc485df1062f1b875661a3c1e1fb1c2f6"}, + {file = "numpy-2.1.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:9156ca1f79fc4acc226696e95bfcc2b486f165a6a59ebe22b2c1f82ab190384a"}, + {file = "numpy-2.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:624884b572dff8ca8f60fab591413f077471de64e376b17d291b19f56504b2bb"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, + {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, ] [[package]] @@ -1205,23 +1212,23 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyinstaller" -version = "6.9.0" +version = "6.10.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false -python-versions = "<3.13,>=3.8" +python-versions = "<3.14,>=3.8" files = [ - {file = "pyinstaller-6.9.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:5ced2e83acf222b936ea94abc5a5cc96588705654b39138af8fb321d9cf2b954"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:f18a3d551834ef8fb7830d48d4cc1527004d0e6b51ded7181e78374ad6111846"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_i686.whl", hash = "sha256:f2fc568de3d6d2a176716a3fc9f20da06d351e8bea5ddd10ecb5659fce3a05b0"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:a0f378f64ad0655d11ade9fde7877e7573fd3d5066231608ce7dfa9040faecdd"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:7bf0c13c5a8560c89540746ae742f4f4b82290e95a6b478374d9f34959fe25d6"}, - {file = "pyinstaller-6.9.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:da994aba14c5686db88796684de265a8665733b4df09b939f7ebdf097d18df72"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:4e3e50743c091a06e6d01c59bdd6d03967b453ee5384a9e790759be4129db4a4"}, - {file = "pyinstaller-6.9.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b041be2fe78da47a269604d62c940d68c62f9a3913bdf64af4123f7689d47099"}, - {file = "pyinstaller-6.9.0-py3-none-win32.whl", hash = "sha256:2bf4de17a1c63c0b797b38e13bfb4d03b5ee7c0a68e28b915a7eaacf6b76087f"}, - {file = "pyinstaller-6.9.0-py3-none-win_amd64.whl", hash = "sha256:43709c70b1da8441a730327a8ed362bfcfdc3d42c1bf89f3e2b0a163cc4e7d33"}, - {file = "pyinstaller-6.9.0-py3-none-win_arm64.whl", hash = "sha256:f15c1ef11ed5ceb32447dfbdab687017d6adbef7fc32aa359d584369bfe56eda"}, - {file = "pyinstaller-6.9.0.tar.gz", hash = "sha256:f4a75c552facc2e2a370f1e422b971b5e5cdb4058ff38cea0235aa21fc0b378f"}, + {file = "pyinstaller-6.10.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:d60fb22859e11483af735aec115fdde09467cdbb29edd9844839f2c920b748c0"}, + {file = "pyinstaller-6.10.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:46d75359668993ddd98630a3669dc5249f3c446e35239b43bc7f4155bc574748"}, + {file = "pyinstaller-6.10.0-py3-none-manylinux2014_i686.whl", hash = "sha256:3398a98fa17d47ccb31f8779ecbdacec025f7adb2f22757a54b706ac8b4fe906"}, + {file = "pyinstaller-6.10.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e9989f354ae4ed8a3bec7bdb37ae0d170751d6520e500f049c7cd0632d31d5c3"}, + {file = "pyinstaller-6.10.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:b7c90c91921b3749083115b28f30f40abf2bb481ceff196d2b2ce0eaa2b3d429"}, + {file = "pyinstaller-6.10.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf876d7d93b8b4f28d1ad57fa24645cf43119c79e985dd5e5f7a801245e6f53"}, + {file = "pyinstaller-6.10.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:db05e3f2f10f9f78c56f1fb163d9cb453433429fe4281218ebaf1ebfd39ba942"}, + {file = "pyinstaller-6.10.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:28eca3817f176fdc19747e1afcf434f13bb9f17a644f611be2c5a61b1f498ed7"}, + {file = "pyinstaller-6.10.0-py3-none-win32.whl", hash = "sha256:703e041718987e46ba0568a2c71ecf2459fddef57cf9edf3efeed4a53e3dae3f"}, + {file = "pyinstaller-6.10.0-py3-none-win_amd64.whl", hash = "sha256:95b55966e563e8b8f31a43882aea10169e9a11fdf38e626d86a2907b640c0701"}, + {file = "pyinstaller-6.10.0-py3-none-win_arm64.whl", hash = "sha256:308e0a8670c9c9ac0cebbf1bbb492e71b6675606f2ec78bc4adfc830d209e087"}, + {file = "pyinstaller-6.10.0.tar.gz", hash = "sha256:143840f8056ff7b910bf8f16f6cd92cc10a6c2680bb76d0a25d558d543d21270"}, ] [package.dependencies] @@ -1229,7 +1236,7 @@ altgraph = "*" macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} packaging = ">=22.0" pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} -pyinstaller-hooks-contrib = ">=2024.7" +pyinstaller-hooks-contrib = ">=2024.8" pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" @@ -1239,13 +1246,13 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.7" +version = "2024.8" description = "Community maintained hooks for PyInstaller" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyinstaller_hooks_contrib-2024.7-py2.py3-none-any.whl", hash = "sha256:8bf0775771fbaf96bcd2f4dfd6f7ae6c1dd1b1efe254c7e50477b3c08e7841d8"}, - {file = "pyinstaller_hooks_contrib-2024.7.tar.gz", hash = "sha256:fd5f37dcf99bece184e40642af88be16a9b89613ecb958a8bd1136634fc9fac5"}, + {file = "pyinstaller_hooks_contrib-2024.8-py3-none-any.whl", hash = "sha256:0057fe9a5c398d3f580e73e58793a1d4a8315ca91c3df01efea1c14ed557825a"}, + {file = "pyinstaller_hooks_contrib-2024.8.tar.gz", hash = "sha256:29b68d878ab739e967055b56a93eb9b58e529d5b054fbab7a2f2bacf80cef3e2"}, ] [package.dependencies] @@ -1292,6 +1299,23 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1342,73 +1366,75 @@ files = [ [[package]] name = "pywin32-ctypes" -version = "0.2.2" +version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" files = [ - {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, - {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, ] [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -1452,46 +1478,46 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.5.6" +version = "0.5.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.6-py3-none-linux_armv6l.whl", hash = "sha256:a0ef5930799a05522985b9cec8290b185952f3fcd86c1772c3bdbd732667fdcd"}, - {file = "ruff-0.5.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b652dc14f6ef5d1552821e006f747802cc32d98d5509349e168f6bf0ee9f8f42"}, - {file = "ruff-0.5.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:80521b88d26a45e871f31e4b88938fd87db7011bb961d8afd2664982dfc3641a"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9bc8f328a9f1309ae80e4d392836e7dbc77303b38ed4a7112699e63d3b066ab"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d394940f61f7720ad371ddedf14722ee1d6250fd8d020f5ea5a86e7be217daf"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111a99cdb02f69ddb2571e2756e017a1496c2c3a2aeefe7b988ddab38b416d36"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e395daba77a79f6dc0d07311f94cc0560375ca20c06f354c7c99af3bf4560c5d"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c476acb43c3c51e3c614a2e878ee1589655fa02dab19fe2db0423a06d6a5b1b6"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2ff8003f5252fd68425fd53d27c1f08b201d7ed714bb31a55c9ac1d4c13e2eb"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c94e084ba3eaa80c2172918c2ca2eb2230c3f15925f4ed8b6297260c6ef179ad"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1f77c1c3aa0669fb230b06fb24ffa3e879391a3ba3f15e3d633a752da5a3e670"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f908148c93c02873210a52cad75a6eda856b2cbb72250370ce3afef6fb99b1ed"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:563a7ae61ad284187d3071d9041c08019975693ff655438d8d4be26e492760bd"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:94fe60869bfbf0521e04fd62b74cbca21cbc5beb67cbb75ab33fe8c174f54414"}, - {file = "ruff-0.5.6-py3-none-win32.whl", hash = "sha256:e6a584c1de6f8591c2570e171cc7ce482bb983d49c70ddf014393cd39e9dfaed"}, - {file = "ruff-0.5.6-py3-none-win_amd64.whl", hash = "sha256:d7fe7dccb1a89dc66785d7aa0ac283b2269712d8ed19c63af908fdccca5ccc1a"}, - {file = "ruff-0.5.6-py3-none-win_arm64.whl", hash = "sha256:57c6c0dd997b31b536bff49b9eee5ed3194d60605a4427f735eeb1f9c1b8d264"}, - {file = "ruff-0.5.6.tar.gz", hash = "sha256:07c9e3c2a8e1fe377dd460371c3462671a728c981c3205a5217291422209f642"}, + {file = "ruff-0.5.7-py3-none-linux_armv6l.whl", hash = "sha256:548992d342fc404ee2e15a242cdbea4f8e39a52f2e7752d0e4cbe88d2d2f416a"}, + {file = "ruff-0.5.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00cc8872331055ee017c4f1071a8a31ca0809ccc0657da1d154a1d2abac5c0be"}, + {file = "ruff-0.5.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eaf3d86a1fdac1aec8a3417a63587d93f906c678bb9ed0b796da7b59c1114a1e"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a01c34400097b06cf8a6e61b35d6d456d5bd1ae6961542de18ec81eaf33b4cb8"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcc8054f1a717e2213500edaddcf1dbb0abad40d98e1bd9d0ad364f75c763eea"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f70284e73f36558ef51602254451e50dd6cc479f8b6f8413a95fcb5db4a55fc"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a78ad870ae3c460394fc95437d43deb5c04b5c29297815a2a1de028903f19692"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ccd078c66a8e419475174bfe60a69adb36ce04f8d4e91b006f1329d5cd44bcf"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e31c9bad4ebf8fdb77b59cae75814440731060a09a0e0077d559a556453acbb"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d796327eed8e168164346b769dd9a27a70e0298d667b4ecee6877ce8095ec8e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a09ea2c3f7778cc635e7f6edf57d566a8ee8f485f3c4454db7771efb692c499"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a36d8dcf55b3a3bc353270d544fb170d75d2dff41eba5df57b4e0b67a95bb64e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9369c218f789eefbd1b8d82a8cf25017b523ac47d96b2f531eba73770971c9e5"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b88ca3db7eb377eb24fb7c82840546fb7acef75af4a74bd36e9ceb37a890257e"}, + {file = "ruff-0.5.7-py3-none-win32.whl", hash = "sha256:33d61fc0e902198a3e55719f4be6b375b28f860b09c281e4bdbf783c0566576a"}, + {file = "ruff-0.5.7-py3-none-win_amd64.whl", hash = "sha256:083bbcbe6fadb93cd86709037acc510f86eed5a314203079df174c40bbbca6b3"}, + {file = "ruff-0.5.7-py3-none-win_arm64.whl", hash = "sha256:2dca26154ff9571995107221d0aeaad0e75a77b5a682d6236cf89a58c70b76f4"}, + {file = "ruff-0.5.7.tar.gz", hash = "sha256:8dfc0a458797f5d9fb622dd0efc52d796f23f0a1493a9527f4e49a550ae9a7e5"}, ] [[package]] name = "setuptools" -version = "72.1.0" +version = "73.0.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-73.0.1-py3-none-any.whl", hash = "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e"}, + {file = "setuptools-73.0.1.tar.gz", hash = "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] [[package]] name = "shellingham" @@ -1536,6 +1562,54 @@ description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0c9045ecc2e4db59bfc97b20516dfdf8e41d910ac6fb667ebd3a79ea54084619"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1467940318e4a860afd546ef61fefb98a14d935cd6817ed07a228c7f7c62f389"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5954463675cb15db8d4b521f3566a017c8789222b8316b1e6934c811018ee08b"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167e7497035c303ae50651b351c28dc22a40bb98fbdb8468cdc971821b1ae533"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27dfb676ac02529fb6e343b3a482303f16e6bc3a4d868b73935b8792edb52d0"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf2360a5e0f7bd75fa80431bf8ebcfb920c9f885e7956c7efde89031695cafb8"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win32.whl", hash = "sha256:306fe44e754a91cd9d600a6b070c1f2fadbb4a1a257b8781ccf33c7067fd3e4d"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:99db65e6f3ab42e06c318f15c98f59a436f1c78179e6a6f40f529c8cc7100b22"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21b053be28a8a414f2ddd401f1be8361e41032d2ef5884b2f31d31cb723e559f"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b178e875a7a25b5938b53b006598ee7645172fccafe1c291a706e93f48499ff5"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a40ee2cc7ea653645bd4cf024326dea2076673fc9d3d33f20f6c81db83e1d"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295ff8689544f7ee7e819529633d058bd458c1fd7f7e3eebd0f9268ebc56c2a0"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:49496b68cd190a147118af585173ee624114dfb2e0297558c460ad7495f9dfe2"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd9b73c5c15f0ec5ce18128b1fe9157ddd0044abc373e6ecd5ba376a7e5d961"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win32.whl", hash = "sha256:9365a3da32dabd3e69e06b972b1ffb0c89668994c7e8e75ce21d3e5e69ddef28"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win_amd64.whl", hash = "sha256:8bd63d051f4f313b102a2af1cbc8b80f061bf78f3d5bd0843ff70b5859e27924"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bab3db192a0c35e3c9d1560eb8332463e29e5507dbd822e29a0a3c48c0a8d92"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:19d98f4f58b13900d8dec4ed09dd09ef292208ee44cc9c2fe01c1f0a2fe440e9"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd33c61513cb1b7371fd40cf221256456d26a56284e7d19d1f0b9f1eb7dd7e8"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6ba0497c1d066dd004e0f02a92426ca2df20fac08728d03f67f6960271feec"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b6be53e4fde0065524f1a0a7929b10e9280987b320716c1509478b712a7688c"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:916a798f62f410c0b80b63683c8061f5ebe237b0f4ad778739304253353bc1cb"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win32.whl", hash = "sha256:31983018b74908ebc6c996a16ad3690301a23befb643093fcfe85efd292e384d"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win_amd64.whl", hash = "sha256:4363ed245a6231f2e2957cccdda3c776265a75851f4753c60f3004b90e69bfeb"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8afd5b26570bf41c35c0121801479958b4446751a3971fb9a480c1afd85558e"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c750987fc876813f27b60d619b987b057eb4896b81117f73bb8d9918c14f1cad"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0102afff4890f651ed91120c1120065663506b760da4e7823913ebd3258be"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:78c03d0f8a5ab4f3034c0e8482cfcc415a3ec6193491cfa1c643ed707d476f16"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:3bd1cae7519283ff525e64645ebd7a3e0283f3c038f461ecc1c7b040a0c932a1"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win32.whl", hash = "sha256:01438ebcdc566d58c93af0171c74ec28efe6a29184b773e378a385e6215389da"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:4979dc80fbbc9d2ef569e71e0896990bc94df2b9fdbd878290bd129b65ab579c"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c742be912f57586ac43af38b3848f7688863a403dfb220193a882ea60e1ec3a"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:62e23d0ac103bcf1c5555b6c88c114089587bc64d048fef5bbdb58dfd26f96da"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:251f0d1108aab8ea7b9aadbd07fb47fb8e3a5838dde34aa95a3349876b5a1f1d"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef18a84e5116340e38eca3e7f9eeaaef62738891422e7c2a0b80feab165905f"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3eb6a97a1d39976f360b10ff208c73afb6a4de86dd2a6212ddf65c4a6a2347d5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0c1c9b673d21477cec17ab10bc4decb1322843ba35b481585facd88203754fc5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win32.whl", hash = "sha256:c41a2b9ca80ee555decc605bd3c4520cc6fef9abde8fd66b1cf65126a6922d65"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:8a37e4d265033c897892279e8adf505c8b6b4075f2b40d77afb31f7185cd6ecd"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fec964fba2ef46476312a03ec8c425956b05c20220a1a03703537824b5e8e1"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:328429aecaba2aee3d71e11f2477c14eec5990fb6d0e884107935f7fb6001632"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85a01b5599e790e76ac3fe3aa2f26e1feba56270023d6afd5550ed63c68552b3"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf04784797dcdf4c0aa952c8d234fa01974c4729db55c45732520ce12dd95b4"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4488120becf9b71b3ac718f4138269a6be99a42fe023ec457896ba4f80749525"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14e09e083a5796d513918a66f3d6aedbc131e39e80875afe81d98a03312889e6"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win32.whl", hash = "sha256:0d322cc9c9b2154ba7e82f7bf25ecc7c36fbe2d82e2933b3642fc095a52cfc78"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:7dd8583df2f98dea28b5cd53a1beac963f4f9d087888d75f22fcc93a07cf8d84"}, + {file = "SQLAlchemy-2.0.32-py3-none-any.whl", hash = "sha256:e567a8793a692451f706b363ccf3c45e056b67d90ead58c3bc9471af5d212202"}, {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, ] @@ -1598,13 +1672,13 @@ url = ["furl (>=0.4.1)"] [[package]] name = "testcontainers" -version = "4.7.2" +version = "4.8.0" description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "testcontainers-4.7.2-py3-none-any.whl", hash = "sha256:23b13cf8078f615a08c75197f227796d90c46df92d2b282ae7c39b1fc1a9c9ed"}, - {file = "testcontainers-4.7.2.tar.gz", hash = "sha256:9976b1cdcdeb9feeae6a477073e7c8b02cd40ea44f1daa34b5da6d2c918dff0d"}, + {file = "testcontainers-4.8.0-py3-none-any.whl", hash = "sha256:0b85d787e5b1f8b32042704d23b6c54787bf6751d2d3cfee2c031349ef2eea30"}, + {file = "testcontainers-4.8.0.tar.gz", hash = "sha256:56153bb5938694844f0e6bd0cf82e19dd6a6516bc29881440e273939201a42d5"}, ] [package.dependencies] @@ -1615,10 +1689,12 @@ wrapt = "*" [package.extras] arangodb = ["python-arango (>=7.8,<8.0)"] +aws = ["boto3", "httpx"] azurite = ["azure-storage-blob (>=12.19,<13.0)"] chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] cosmosdb = ["azure-cosmos"] +db2 = ["ibm_db_sa", "sqlalchemy"] generic = ["httpx"] google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] influxdb = ["influxdb", "influxdb-client"] @@ -1639,6 +1715,7 @@ qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] +scylla = ["cassandra-driver (==3.29.1)"] selenium = ["selenium"] sftp = ["cryptography"] test-module-import = ["httpx"] @@ -1658,13 +1735,13 @@ files = [ [[package]] name = "typer" -version = "0.12.3" +version = "0.12.4" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, - {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, + {file = "typer-0.12.4-py3-none-any.whl", hash = "sha256:819aa03699f438397e876aa12b0d63766864ecba1b579092cc9fe35d886e34b6"}, + {file = "typer-0.12.4.tar.gz", hash = "sha256:c9c1613ed6a166162705b3347b8d10b661ccc5d95692654d0fb628118f2c34e6"}, ] [package.dependencies] @@ -1737,19 +1814,23 @@ name = "wiremock" version = "2.6.1" description = "Wiremock Admin API Client" optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "wiremock-2.6.1-py3-none-any.whl", hash = "sha256:417a803b0bba3ab6240410aedb4de15a32581fb29b1310b05289b4aa1a7c9ffd"}, - {file = "wiremock-2.6.1.tar.gz", hash = "sha256:89b64d763a68a1808274aa4daf802f7ce3f9bff2a18ac6bf8923c997a21d67c1"}, -] +python-versions = "^3.7 | ^3.8 | ^3.9 | ^3.10 | ^3.11" +files = [] +develop = false [package.dependencies] -importlib-resources = ">=5.12.0,<6.0.0" -requests = ">=2.20.0,<3.0.0" +importlib-resources = "^5.12.0" +requests = "^2.20.0" [package.extras] testing = ["docker (>=6.1.0,<7.0.0)", "testcontainers (>=3.7.1,<4.0.0)"] +[package.source] +type = "git" +url = "https://github.com/ImperialCollegeLondon/python-wiremock.git" +reference = "fix-test-containers-on-windows" +resolved_reference = "fc2df60d0ff13e22f2cf5995d11596f83869a0ea" + [[package]] name = "wrapt" version = "1.16.0" @@ -1856,4 +1937,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "00a92fb906cc93d1a11fd65e4e3ed1b04161ed8cbc4fca3c41d518d0b39839f9" +content-hash = "46cf132984999ff68385aae9bfd5317548d568a149d5f15042b9909753929740" diff --git a/pyproject.toml b/pyproject.toml index da156cd..8ac647c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] requires-python = ">=3.10" name = "imap-mag" +version = "0.1.0" [tool.poetry] name = "imap-mag" @@ -28,19 +29,20 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.7.0" +imap-data-access = "^0.9.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" pytest-cov = "^5.0.0" +pytest-mock = "^3.14.0" pyinstaller = "^6.5.0" pre-commit = "^3.8.0" ruff = "^0.5.4" -wiremock = "^2.6.1" docker = "^7.1.0" testcontainers = "^4.7.2" +wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` diff --git a/run-docker.sh b/run-docker.sh index 04960e7..cc172c3 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -20,9 +20,9 @@ if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then $IMAGE_NAME elif [ -z "$1" ]; then # no args passed docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME else echo "Extra arguments: $@" docker run --rm -it \ diff --git a/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py new file mode 100644 index 0000000..fbcc999 --- /dev/null +++ b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py @@ -0,0 +1,52 @@ +"""Added version, hash, date and software version columns + +Revision ID: 669111c45c37 +Revises: d0457f3e98c8 +Create Date: 2024-08-09 13:35:21.578940 + +""" + +from datetime import datetime + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "669111c45c37" +down_revision = "d0457f3e98c8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "files", sa.Column("version", sa.Integer(), nullable=False, default=0) + ) + op.add_column( + "files", sa.Column("hash", sa.String(length=64), nullable=False, default="") + ) + op.add_column( + "files", + sa.Column( + "date", sa.DateTime(), nullable=False, default=datetime.fromtimestamp(0) + ), + ) + op.add_column( + "files", + sa.Column( + "software_version", sa.String(length=16), nullable=False, default="0.0.0" + ), + ) + op.create_unique_constraint(None, "files", ["path"]) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, "files", type_="unique") + op.drop_column("files", "software_version") + op.drop_column("files", "date") + op.drop_column("files", "hash") + op.drop_column("files", "version") + # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py index 6cfbcb6..644f9a3 100644 --- a/src/imap_db/model.py +++ b/src/imap_db/model.py @@ -1,4 +1,6 @@ -from sqlalchemy import String +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -8,9 +10,14 @@ class Base(DeclarativeBase): class File(Base): __tablename__ = "files" + id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(128)) - path: Mapped[str] = mapped_column(String(256)) + path: Mapped[str] = mapped_column(String(256), unique=True) + version: Mapped[int] = mapped_column(Integer()) + hash: Mapped[str] = mapped_column(String(64)) + date: Mapped[datetime] = mapped_column(DateTime()) + software_version: Mapped[str] = mapped_column(String(16)) def __repr__(self) -> str: return f"" diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 3ca15e0..a3c4bfd 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,11 +1,34 @@ +import abc +import logging import os +from pathlib import Path +import typer from imap_db.model import File from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from imap_mag import __version__ +from imap_mag.outputManager import IFileMetadataProvider, IOutputManager, generate_hash + + +class IDatabase(abc.ABC): + """Interface for database manager.""" + + def insert_file(self, file: File) -> None: + """Insert a file into the database.""" + self.insert_files([file]) + pass + + @abc.abstractmethod + def insert_files(self, files: list[File]) -> None: + """Insert a list of files into the database.""" + pass + + +class Database(IDatabase): + """Database manager.""" -class DB: def __init__(self, db_url=None): env_url = os.getenv("SQLALCHEMY_URL") if db_url is None and env_url is not None: @@ -16,10 +39,12 @@ def __init__(self, db_url=None): "No database URL provided. Consider setting SQLALCHEMY_URL environment variable." ) + # TODO: Check database is available + self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - def insert_files(self, files: list[File]): + def insert_files(self, files: list[File]) -> None: session = self.Session() try: for file in files: @@ -40,3 +65,60 @@ def insert_files(self, files: list[File]): raise e finally: session.close() + + +class DatabaseOutputManager(IOutputManager): + """Decorator for adding files to database as well as output.""" + + __output_manager: IOutputManager + __database: IDatabase + + def __init__( + self, output_manager: IOutputManager, database: Database | None = None + ): + """Initialize database and output manager.""" + + self.__output_manager = output_manager + + if database is None: + self.__database = Database() + else: + self.__database = database + + def add_file( + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: + (destination_file, metadata_provider) = self.__output_manager.add_file( + original_file, metadata_provider + ) + + file_hash: str = generate_hash(original_file) + + if not ( + destination_file.exists() and (generate_hash(destination_file) == file_hash) + ): + logging.error( + f"File {destination_file} does not exist or is not the same as original {original_file}." + ) + destination_file.unlink(missing_ok=True) + raise typer.Abort() + + logging.info(f"Inserting {destination_file} into database.") + + try: + self.__database.insert_file( + File( + name=destination_file.name, + path=destination_file.absolute().as_posix(), + version=metadata_provider.version, + hash=file_hash, + date=metadata_provider.date, + software_version=__version__, + ) + ) + except Exception as e: + logging.error(f"Error inserting {destination_file} into database: {e}") + destination_file.unlink() + raise e + + return (destination_file, metadata_provider) diff --git a/src/imap_mag/__init__.py b/src/imap_mag/__init__.py index e69de29..f3ebd37 100644 --- a/src/imap_mag/__init__.py +++ b/src/imap_mag/__init__.py @@ -0,0 +1,13 @@ +"""The main module for project.""" + +from importlib.metadata import PackageNotFoundError, version + + +def get_version() -> str: + try: + return version("imap-mag") + except PackageNotFoundError: + print("IMAP MAG CLI Version unknown, not installed via pip.") + + +__version__ = get_version() diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index 5d4250a..b15a050 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -1,6 +1,4 @@ import logging -import os -import shutil from pathlib import Path from typing import Optional @@ -8,7 +6,9 @@ import pandas as pd import typer -from . import appConfig +from .appConfig import Destination +from .DB import DatabaseOutputManager +from .outputManager import IFileMetadataProvider, IOutputManager, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -56,18 +56,41 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() -def copyFileToDestination(filePath: Path, destination: appConfig.Destination) -> None: +def getOutputManager(destination: Destination) -> IOutputManager: + """Retrieve output manager based on destination.""" + + output_manager = OutputManager(destination.folder) + + if destination.export_to_database: + output_manager = DatabaseOutputManager(output_manager) + + return output_manager + + +def copyFileToDestination( + file_path: Path, + destination: Destination, + output_manager: Optional[OutputManager] = None, +) -> tuple[Path, IFileMetadataProvider]: """Copy file to destination folder.""" - destinationFile = Path(destination.folder) + class SimpleMetadataProvider(IFileMetadataProvider): + """Simple metadata provider for compatibility.""" - if not destinationFile.exists(): - logging.debug(f"Creating destination folder {destinationFile}.") - os.makedirs(destinationFile) + def __init__(self, filename: str) -> None: + self.filename = filename - if destination.filename: - destinationFile = destinationFile / destination.filename + def get_folder_structure(self) -> str: + return "" - logging.info(f"Copying {filePath} to {destinationFile.absolute()}") - completed = shutil.copy2(filePath, destinationFile) - logging.info(f"Copy complete: {completed}") + def get_file_name(self) -> str: + return self.filename + + destination_folder = Path(destination.folder) + + if output_manager is None: + output_manager: OutputManager = OutputManager(destination_folder) + + return output_manager.add_file( + file_path, SimpleMetadataProvider(destination.filename) + ) diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py new file mode 100644 index 0000000..efbaaf0 --- /dev/null +++ b/src/imap_mag/cli/fetchBinary.py @@ -0,0 +1,81 @@ +"""Program to retrieve and process MAG binary files.""" + +import typing +from datetime import datetime +from pathlib import Path + +import pandas as pd +import typing_extensions + +from ..client.webPODA import WebPODA +from ..outputManager import IOutputManager + + +class FetchBinaryOptions(typing.TypedDict): + """Options for WebPODA interactions.""" + + packet: str + start_date: datetime + end_date: datetime + + +class FetchBinary: + """Manage WebPODA data.""" + + __MAG_PREFIX: str = "mag_" + + __web_poda: WebPODA + __output_manager: IOutputManager | None + + def __init__( + self, + web_poda: WebPODA, + output_manager: IOutputManager | None = None, + ) -> None: + """Initialize WebPODA interface.""" + + self.__web_poda = web_poda + self.__output_manager = output_manager + + def download_binaries( + self, **options: typing_extensions.Unpack[FetchBinaryOptions] + ) -> list[Path]: + """Retrieve WebPODA data.""" + + downloaded = [] + + date_range: pd.DatetimeIndex = pd.date_range( + start=options["start_date"], + end=options["end_date"], + freq="D", + normalize=True, + ) + + dates = date_range.to_pydatetime().tolist() + if len(dates) == 1: + dates += [ + pd.Timestamp(dates[0] + pd.Timedelta(days=1)) + .normalize() + .to_pydatetime() + ] + + for d in range(len(dates) - 1): + file: Path = self.__web_poda.download( + packet=options["packet"], start_date=dates[d], end_date=dates[d + 1] + ) + + if file.stat().st_size > 0: + if self.__output_manager is not None: + self.__output_manager.add_default_file( + file, + descriptor=options["packet"] + .lower() + .strip(self.__MAG_PREFIX) + .replace("_", "-"), + date=dates[d], + extension="pkts", + ) + + downloaded += [file] + + return downloaded diff --git a/src/imap_mag/cli/fetchScience.py b/src/imap_mag/cli/fetchScience.py index 49fd3e9..9c4ad93 100644 --- a/src/imap_mag/cli/fetchScience.py +++ b/src/imap_mag/cli/fetchScience.py @@ -1,14 +1,15 @@ """Program to retrieve and process MAG CDF files.""" import typing +from datetime import datetime from enum import Enum from pathlib import Path import pandas as pd import typing_extensions -from .. import appUtils from ..client.sdcDataAccess import ISDCDataAccess +from ..outputManager import IOutputManager class MAGMode(str, Enum): @@ -25,28 +26,30 @@ class FetchScienceOptions(typing.TypedDict): """Options for SOC interactions.""" level: str - start_date: str - end_date: str - output_dir: str + start_date: datetime + end_date: datetime class FetchScience: """Manage SOC data.""" + __data_access: ISDCDataAccess + __output_manager: IOutputManager | None + __modes: list[MAGMode] __sensor: list[MAGSensor] - __data_access: ISDCDataAccess - def __init__( self, data_access: ISDCDataAccess, - modes: list[MAGMode] = ["norm", "burst"], - sensors: list[MAGSensor] = ["magi", "mago"], + output_manager: IOutputManager | None = None, + modes: list[MAGMode] = [MAGMode.Normal, MAGMode.Burst], + sensors: list[MAGSensor] = [MAGSensor.IBS, MAGSensor.OBS], ) -> None: """Initialize SDC interface.""" self.__data_access = data_access + self.__output_manager = output_manager self.__modes = modes self.__sensor = sensors @@ -59,8 +62,8 @@ def download_latest_science( for mode in self.__modes: date_range: pd.DatetimeIndex = pd.date_range( - start=appUtils.convertToDatetime(options["start_date"]), - end=appUtils.convertToDatetime(options["end_date"]), + start=options["start_date"], + end=options["end_date"], freq="D", normalize=True, ) @@ -69,9 +72,9 @@ def download_latest_science( for sensor in self.__sensor: file_details = self.__data_access.get_filename( level=options["level"], - descriptor=str(mode) + "-" + str(sensor), + descriptor=mode.value + "-" + sensor.value, start_date=date, - end_date=None, + end_date=date, version="latest", extension="cdf", ) @@ -82,4 +85,13 @@ def download_latest_science( self.__data_access.download(file["file_path"]) ] + if self.__output_manager is not None: + self.__output_manager.add_default_file( + downloaded[-1], + level=options["level"], + descriptor=file["descriptor"], + date=date, + extension="cdf", + ) + return downloaded diff --git a/src/imap_mag/client/sdcDataAccess.py b/src/imap_mag/client/sdcDataAccess.py index 88d277d..cafd3d3 100644 --- a/src/imap_mag/client/sdcDataAccess.py +++ b/src/imap_mag/client/sdcDataAccess.py @@ -2,9 +2,9 @@ import abc import logging -import pathlib import typing from datetime import datetime +from pathlib import Path import imap_data_access import typing_extensions @@ -44,7 +44,7 @@ class ISDCDataAccess(abc.ABC): @abc.abstractmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: """Get file path for data from imap-data-access.""" pass @@ -68,7 +68,7 @@ def get_filename( pass @abc.abstractmethod - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: """Download data from imap-data-access.""" pass @@ -76,10 +76,10 @@ def download(self, file_name: str) -> pathlib.Path: class SDCDataAccess(ISDCDataAccess): """Class for uploading and downloading MAG data via imap-data-access.""" - def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: + def __init__(self, data_dir: Path, sdc_url: str | None = None) -> None: """Initialize SDC API client.""" - imap_data_access.config["DATA_DIR"] = pathlib.Path(data_dir) + imap_data_access.config["DATA_DIR"] = data_dir imap_data_access.config["DATA_ACCESS_URL"] = ( sdc_url or "https://api.dev.imap-mission.com" ) @@ -87,7 +87,7 @@ def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: @staticmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: science_file = imap_data_access.ScienceFilePath.generate_from_inputs( instrument="mag", data_level=options["level"], @@ -136,6 +136,6 @@ def get_filename( return file_details - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: logging.debug(f"Downloading {file_name} from imap-data-access.") - return pathlib.Path(imap_data_access.download(file_name)) + return imap_data_access.download(file_name) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index cba66fd..4dfb7a2 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -13,7 +13,6 @@ # config import yaml -from imap_db.model import File from mag_toolkit import CDFLoader from mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator from mag_toolkit.calibration.calibrationFormatProcessor import ( @@ -26,7 +25,8 @@ SpinPlaneCalibrator, ) -from . import DB, appConfig, appLogging, appUtils, imapProcessing +from . import appConfig, appLogging, appUtils, imapProcessing +from .cli.fetchBinary import FetchBinary from .cli.fetchScience import FetchScience from .client.sdcDataAccess import SDCDataAccess from .client.webPODA import WebPODA @@ -35,7 +35,7 @@ globalState = {"verbose": False} -def commandInit(config: Path) -> appConfig.AppConfig: +def commandInit(config: Path | None) -> appConfig.AppConfig: # load and verify the config file if config is None: logging.critical("No config file") @@ -182,6 +182,8 @@ def fetch_binary( end_date: Annotated[str, typer.Option(help="End date for the download")], config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): + """Download binary data from WebPODA.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: @@ -189,6 +191,9 @@ def fetch_binary( raise typer.Abort() packet: str = appUtils.getPacketFromApID(apid) + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading raw packet {packet} from {start_date} to {end_date}.") poda = WebPODA( @@ -196,13 +201,12 @@ def fetch_binary( configFile.work_folder, configFile.api.webpoda_url if configFile.api else None, ) - result: str = poda.download( - packet=packet, - start_date=appUtils.convertToDatetime(start_date), - end_date=appUtils.convertToDatetime(end_date), - ) + output_manager = appUtils.getOutputManager(configFile.destination) - appUtils.copyFileToDestination(result, configFile.destination) + fetch_binary = FetchBinary(poda, output_manager) + fetch_binary.download_binaries( + packet=packet, start_date=start_date, end_date=end_date + ) class LevelEnum(str, Enum): @@ -227,39 +231,32 @@ def fetch_science( level: Annotated[ LevelEnum, typer.Option(help="Level to download") ] = LevelEnum.level_2, - config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), + config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): + """Download science data from the SDC.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: logging.critical("No SDC_AUTH_CODE API key provided") raise typer.Abort() + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading {level} science from {start_date} to {end_date}.") data_access = SDCDataAccess( - data_dir=str(configFile.work_folder), + data_dir=configFile.work_folder, sdc_url=configFile.api.sdc_url if configFile.api else None, ) + output_manager = appUtils.getOutputManager(configFile.destination) - fetch_science = FetchScience(data_access) - files = fetch_science.download_latest_science( + fetch_science = FetchScience(data_access, output_manager) + fetch_science.download_latest_science( level=level.value, start_date=start_date, end_date=end_date ) - records = [] - for file in files: - records.append(File(name=file.name, path=file.absolute().as_posix())) - - for file in files: - appUtils.copyFileToDestination(file, configFile.destination) - - if configFile.destination.export_to_database: - db = DB.DB() - db.insert_files(records) - - logging.info(f"Downloaded {len(files)} files and saved to database") - # imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf @app.command() diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py new file mode 100644 index 0000000..5f667b4 --- /dev/null +++ b/src/imap_mag/outputManager.py @@ -0,0 +1,147 @@ +import abc +import hashlib +import logging +import shutil +import typing +from datetime import datetime +from pathlib import Path + +import typer + + +def generate_hash(file: Path) -> str: + return hashlib.md5(file.read_bytes()).hexdigest() + + +class IFileMetadataProvider(abc.ABC): + """Interface for metadata providers.""" + + version: int = 0 + + @abc.abstractmethod + def get_folder_structure(self) -> str: + """Retrieve folder structure.""" + + @abc.abstractmethod + def get_file_name(self) -> str: + """Retireve file name.""" + + +class DatastoreScienceFilepathGenerator(IFileMetadataProvider): + """Metadata for output files.""" + + prefix: str | None = "imap_mag" + level: str | None = None + descriptor: str + date: datetime + extension: str + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + def get_folder_structure(self) -> str: + if self.date is None: + logging.error("No 'date' defined. Cannot generate folder structure.") + raise typer.Abort() + + return self.date.strftime("%Y/%m/%d") + + def get_file_name(self) -> str: + if any(x is None for x in ["descriptor", "date", "version", "extension"]): + logging.error( + "No 'descriptor', 'date', 'version', or 'extension' defined. Cannot generate file name." + ) + raise typer.Abort() + + descriptor = self.descriptor + + if self.level is not None: + descriptor = f"{self.level}_{descriptor}" + + if self.prefix is not None: + descriptor = f"{self.prefix}_{descriptor}" + + return f"{descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" + + +class IOutputManager(abc.ABC): + """Interface for output managers.""" + + @abc.abstractmethod + def add_file( + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: + """Add file to output location.""" + + def add_default_file( + self, original_file: Path, **metadata: typing.Any + ) -> tuple[Path, IFileMetadataProvider]: + return self.add_file( + original_file, DatastoreScienceFilepathGenerator(**metadata) + ) + + +class OutputManager(IOutputManager): + """Manage output files.""" + + location: Path + + def __init__(self, location: Path) -> None: + self.location = location + + def add_file( + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: + """Add file to output location.""" + + if not self.location.exists(): + logging.debug(f"Output location does not exist. Creating {self.location}.") + self.location.mkdir(parents=True, exist_ok=True) + + destination_file: Path = self.__assemble_full_path(metadata_provider) + + if not destination_file.parent.exists(): + logging.debug( + f"Output folder structure does not exist. Creating {destination_file.parent}." + ) + destination_file.parent.mkdir(parents=True, exist_ok=True) + + if destination_file.exists(): + if generate_hash(destination_file) == generate_hash(original_file): + logging.info(f"File {destination_file} already exists and is the same.") + return (destination_file, metadata_provider) + + metadata_provider.version = self.__get_next_available_version( + destination_file, metadata_provider + ) + destination_file = self.__assemble_full_path(metadata_provider) + + logging.info(f"Copying {original_file} to {destination_file.absolute()}.") + destination = shutil.copy2(original_file, destination_file) + logging.info(f"Copied to {destination}.") + + return (destination_file, metadata_provider) + + def __assemble_full_path(self, metadata_provider: IFileMetadataProvider) -> Path: + """Assemble full path from metadata.""" + + return ( + self.location + / metadata_provider.get_folder_structure() + / metadata_provider.get_file_name() + ) + + def __get_next_available_version( + self, destination_file: Path, metadata_provider: IFileMetadataProvider + ) -> int: + """Find a viable version for a file.""" + + while destination_file.exists(): + logging.debug( + f"File {destination_file} already exists and is different. Increasing version to {metadata_provider.version}." + ) + metadata_provider.version += 1 + destination_file = self.__assemble_full_path(metadata_provider) + + return metadata_provider.version diff --git a/tests/config/calibration_application_config.yaml b/tests/config/calibration_application_config.yaml index b748acb..f6e3448 100644 --- a/tests/config/calibration_application_config.yaml +++ b/tests/config/calibration_application_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: L2.cdf \ No newline at end of file + filename: L2.cdf + export-to-database: false diff --git a/tests/config/calibration_config.yaml b/tests/config/calibration_config.yaml index 5ed5230..a09829b 100644 --- a/tests/config/calibration_config.yaml +++ b/tests/config/calibration_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: calibration.json \ No newline at end of file + filename: calibration.json + export-to-database: false diff --git a/tests/config/hk_process.yaml b/tests/config/hk_process.yaml index f3d4af1..8a72ea8 100644 --- a/tests/config/hk_process.yaml +++ b/tests/config/hk_process.yaml @@ -6,6 +6,7 @@ work-folder: .work destination: folder: output/ filename: result.csv + export-to-database: false packet-definition: hk: src/imap_mag/xtce/tlm_20240724.xml diff --git a/tests/testUtils.py b/tests/testUtils.py index c233b09..bdb846d 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1,8 +1,32 @@ import os -from pathlib import Path, PosixPath +from pathlib import Path, PosixPath, WindowsPath import imap_mag.appConfig as appConfig +import pytest import yaml +from imap_mag import appLogging + + +@pytest.fixture(autouse=True) +def enableLogging(): + appLogging.set_up_logging( + console_log_output="stdout", + console_log_level="debug", + console_log_color=True, + logfile_file="debug", + logfile_log_level="debug", + logfile_log_color=False, + log_line_template="%(color_on)s[%(asctime)s] [%(levelname)-8s] %(message)s%(color_off)s", + console_log_line_template="%(color_on)s%(message)s%(color_off)s", + ) + yield + + +@pytest.fixture(autouse=True) +def tidyDataFolders(): + os.system("rm -rf .work") + os.system("rm -rf output/*") + yield def create_serialize_config( @@ -35,6 +59,23 @@ def create_serialize_config( yaml.add_representer( PosixPath, lambda dumper, data: dumper.represent_str(str(data)) ) + yaml.add_representer( + WindowsPath, lambda dumper, data: dumper.represent_str(str(data)) + ) yaml.dump(config.model_dump(by_alias=True), f) return (config, config_file) + + +def create_test_file(file_path: Path, content: str | None = None) -> Path: + """Create a file with the given content.""" + + file_path.unlink(missing_ok=True) + file_path.parent.mkdir(parents=True, exist_ok=True) + + file_path.touch() + + if content is not None: + file_path.write_text(content) + + return file_path diff --git a/tests/test_appUtils.py b/tests/test_appUtils.py new file mode 100644 index 0000000..71cf2fc --- /dev/null +++ b/tests/test_appUtils.py @@ -0,0 +1,17 @@ +"""Tests for app utilities.""" + +import pytest +import typer +from imap_mag.appUtils import convertToDatetime, getPacketFromApID + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def test_get_packet_from_apid_errors_on_invalid_apid() -> None: + with pytest.raises(typer.Abort): + getPacketFromApID(12345) + + +def test_convert_to_datetime_on_invalid_datetime() -> None: + with pytest.raises(typer.Abort): + convertToDatetime("ABCDEF") diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..acf8ff6 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,148 @@ +"""Tests for database classes.""" + +import hashlib +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +import typer +from imap_db.model import File +from imap_mag import __version__ +from imap_mag.DB import DatabaseOutputManager, IDatabase +from imap_mag.outputManager import DatastoreScienceFilepathGenerator, IOutputManager + +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +@pytest.fixture +def mock_database() -> mock.Mock: + """Fixture for a mock IDatabase instance.""" + return mock.create_autospec(IDatabase, spec_set=True) + + +def test_database_output_manager_writes_to_database( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DatastoreScienceFilepathGenerator( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + def check_inserted_file(file: File): + # Two instances of `File` will never be equal, so we check the attributes. + assert file.name == "test_file.txt" + assert file.path == test_file.absolute().as_posix() + assert file.version == 1 + assert file.hash == hashlib.md5(b"some content").hexdigest() + assert file.date == datetime(2025, 5, 2) + assert file.software_version == __version__ + + mock_database.insert_file.side_effect = lambda file: check_inserted_file(file) + + # Exercise. + (actual_file, actual_metadata_provider) = database_manager.add_file( + original_file, metadata_provider + ) + + # Verify. + mock_output_manager.add_file.assert_called_once_with( + original_file, metadata_provider + ) + + assert actual_file == test_file + assert actual_metadata_provider == metadata_provider + + +def test_database_output_manager_errors_when_destination_file_is_not_found( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DatastoreScienceFilepathGenerator( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + test_file.unlink(missing_ok=True) + + mock_output_manager.add_file.side_effect = lambda *_: ( + test_file, + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_destination_file_different_hash( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DatastoreScienceFilepathGenerator( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some other content"), + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_database_error( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DatastoreScienceFilepathGenerator( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + mock_database.insert_file.side_effect = ArithmeticError("Database error") + + # Exercise and verify. + with pytest.raises(ArithmeticError): + database_manager.add_file(original_file, metadata_provider) diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py new file mode 100644 index 0000000..5c149a7 --- /dev/null +++ b/tests/test_fetchBinary.py @@ -0,0 +1,86 @@ +"""Tests for `FetchBinary` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchBinary import FetchBinary +from imap_mag.client.webPODA import IWebPODA +from imap_mag.outputManager import IOutputManager + +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_poda() -> mock.Mock: + """Fixture for a mock IWebPODA instance.""" + return mock.create_autospec(IWebPODA, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_binary_empty_download_not_added_to_output( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, None) + + # Exercise. + actual_downloaded: list[Path] = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + mock_output_manager.add_default_file.assert_not_called() + + assert actual_downloaded == [] + + +def test_fetch_binary_with_same_start_end_date( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, "content") + + # Exercise. + actual_downloaded: list[Path] = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + descriptor="hsk-pw", + date=datetime(2025, 5, 2), + extension="pkts", + ) + + assert actual_downloaded == [test_file] diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py new file mode 100644 index 0000000..05f64f7 --- /dev/null +++ b/tests/test_fetchScience.py @@ -0,0 +1,107 @@ +"""Tests for `FetchScience` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchScience import FetchScience, MAGMode, MAGSensor +from imap_mag.client.sdcDataAccess import ISDCDataAccess +from imap_mag.outputManager import IOutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_soc() -> mock.Mock: + """Fixture for a mock ISDCDataAccess instance.""" + return mock.create_autospec(ISDCDataAccess, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_science_no_matching_files( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + mock_soc.get_filename.side_effect = lambda **_: {} # return empty dictionary + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + + mock_soc.download.assert_not_called() + mock_output_manager.add_default_file.assert_not_called() + + assert actual_downloaded == [] + + +def test_fetch_science_with_same_start_end_date( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + test_file = Path(tempfile.gettempdir()) / "test_file" + + mock_soc.get_filename.side_effect = lambda **_: [ + { + "file_path": test_file.absolute(), + "descriptor": "norm-mago", + } + ] + mock_soc.download.side_effect = lambda file_path: file_path + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + mock_soc.download.assert_called_once_with( + test_file.absolute(), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + level="l1b", + descriptor="norm-mago", + date=datetime(2025, 5, 2), + extension="cdf", + ) + + assert actual_downloaded == [test_file] diff --git a/tests/test_main.py b/tests/test_main.py index 16a4066..6e36f5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -11,19 +11,12 @@ from imap_mag.main import app from typer.testing import CliRunner -from .testUtils import create_serialize_config +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 from .wiremockUtils import wiremock_manager # noqa: F401 runner = CliRunner() -@pytest.fixture(autouse=True) -def tidyDataFolders(): - os.system("rm -rf .work") - os.system("rm -rf output/*") - yield - - def test_app_says_hello(): result = runner.invoke(app, ["hello", "Bob"]) @@ -80,6 +73,10 @@ def test_process_with_binary_hk_converts_to_csv(): assert expectedNumRows == len(lines) +@pytest.mark.skipif( + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner", +) def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Set up. binary_file = os.path.abspath("tests/data/2025/MAG_HSK_PW.pkts") @@ -114,15 +111,19 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/power.pkts").exists() + assert Path("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts").exists() with ( - open("output/power.pkts", "rb") as output, + open("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts", "rb") as output, open(binary_file, "rb") as input, ): assert output.read() == input.read() +@pytest.mark.skipif( + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner", +) def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. query_response: list[dict[str, str]] = [ @@ -143,7 +144,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 ) wiremock_manager.add_string_mapping( - "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&version=latest&extension=cdf", + "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&end_date=20250502&extension=cdf", json.dumps(query_response), priority=1, ) @@ -154,7 +155,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 wiremock_manager.add_string_mapping( re.escape("/query?instrument=mag&data_level=l1b&descriptor=") + ".*" - + re.escape("&start_date=20250502&version=latest&extension=cdf"), + + re.escape("&start_date=20250502&end_date=20250502&extension=cdf"), json.dumps({}), is_pattern=True, priority=2, @@ -187,10 +188,12 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/result.cdf").exists() + assert Path("output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf").exists() with ( - open("output/result.cdf", "rb") as output, + open( + "output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf", "rb" + ) as output, open(cdf_file, "rb") as input, ): assert output.read() == input.read() diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py new file mode 100644 index 0000000..f235f22 --- /dev/null +++ b/tests/test_outputManager.py @@ -0,0 +1,113 @@ +"""Tests for `OutputManager` class.""" + +from datetime import datetime +from pathlib import Path + +from imap_mag.outputManager import IFileMetadataProvider, OutputManager + +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 + + +def test_copy_new_file(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = create_test_file(Path(".work/some_test_file.txt")) + + # Exercise. + manager.add_default_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt").exists() + + +def test_copy_file_same_content(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = create_test_file(Path(".work/some_test_file.txt"), "some content") + existing_file = create_test_file( + Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt"), "some content" + ) + + existing_modification_time = existing_file.stat().st_mtime + + # Exercise. + manager.add_default_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert not Path("output/2025/05/02/imap_mag_pwr_20250502_v001.txt").exists() + assert existing_file.stat().st_mtime == existing_modification_time + + +def test_copy_file_existing_versions(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = create_test_file(Path(".work/some_test_file.txt"), "some content") + + for version in range(2): + create_test_file( + Path(f"output/2025/05/02/imap_mag_pwr_20250502_v{version:03}.txt") + ) + + # Exercise. + manager.add_default_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v002.txt").exists() + + +def test_copy_file_forced_version(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = create_test_file(Path(".work/some_test_file.txt")) + + # Exercise. + manager.add_default_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + version=3, + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v003.txt").exists() + + +class TestMetadataProvider(IFileMetadataProvider): + def get_folder_structure(self) -> str: + return "abc" + + def get_file_name(self) -> str: + return "def" + + +def test_copy_file_custom_providers(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = create_test_file(Path(".work/some_test_file.txt")) + + # Exercise. + manager.add_file(original_file, TestMetadataProvider()) + + # Verify. + assert Path("output/abc/def").exists() diff --git a/tests/test_sdcDataAccess.py b/tests/test_sdcDataAccess.py new file mode 100644 index 0000000..fe30835 --- /dev/null +++ b/tests/test_sdcDataAccess.py @@ -0,0 +1,43 @@ +"""Tests for `SDCDataAccess` class.""" + +import os +from datetime import datetime +from pathlib import Path + +import imap_data_access +from imap_mag.client.sdcDataAccess import SDCDataAccess + +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 +from .wiremockUtils import wiremock_manager # noqa: F401 + + +def test_sdc_data_access_constructor_sets_config() -> None: + # Set up. + data_dir = "some_test_folder" + data_access_url = "https://some_test_url" + + # Exercise. + _ = SDCDataAccess(data_dir, data_access_url) + + # Verify. + assert imap_data_access.config["DATA_DIR"] == data_dir + assert imap_data_access.config["DATA_ACCESS_URL"] == data_access_url + + +def test_get_file_path_builds_file_path() -> None: + # Set up. + data_access = SDCDataAccess("some_test_folder") + + # Exercise. + (file_name, file_path) = data_access.get_file_path( + level="l1b", + descriptor="norm-magi", + start_date=datetime(2025, 5, 2), + version="v002", + ) + + # Verify. + assert file_name == Path("imap_mag_l1b_norm-magi_20250502_v002.cdf") + assert file_path == Path( + os.path.join("some_test_folder", "imap", "mag", "l1b", "2025", "05", file_name) + )