From fe26c603cb5589af77f0161e146cf30112bb05c2 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 12:52:31 +0000 Subject: [PATCH 001/102] feat: add class to manage outputs --- .gitignore | 1 + poetry.lock | 100 +++++++++++++-------------- src/imap_mag/main.py | 6 +- src/imap_mag/outputManager.py | 126 ++++++++++++++++++++++++++++++++++ tests/testUtils.py | 24 +++++++ tests/test_main.py | 10 +-- tests/test_outputManager.py | 123 +++++++++++++++++++++++++++++++++ 7 files changed, 330 insertions(+), 60 deletions(-) create mode 100644 src/imap_mag/outputManager.py create mode 100644 tests/test_outputManager.py diff --git a/.gitignore b/.gitignore index 7a6fd75..ae89700 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ site/ .work /output dev.env +debug diff --git a/poetry.lock b/poetry.lock index b42b8ff..8eb2f9f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1467,60 +1467,60 @@ bitstring = ">=4.0.1" [[package]] name = "sqlalchemy" -version = "2.0.31" +version = "2.0.32" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, - {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, - {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, - {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, - {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, - {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, - {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, - {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, - {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0c9045ecc2e4db59bfc97b20516dfdf8e41d910ac6fb667ebd3a79ea54084619"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1467940318e4a860afd546ef61fefb98a14d935cd6817ed07a228c7f7c62f389"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5954463675cb15db8d4b521f3566a017c8789222b8316b1e6934c811018ee08b"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167e7497035c303ae50651b351c28dc22a40bb98fbdb8468cdc971821b1ae533"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27dfb676ac02529fb6e343b3a482303f16e6bc3a4d868b73935b8792edb52d0"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf2360a5e0f7bd75fa80431bf8ebcfb920c9f885e7956c7efde89031695cafb8"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win32.whl", hash = "sha256:306fe44e754a91cd9d600a6b070c1f2fadbb4a1a257b8781ccf33c7067fd3e4d"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:99db65e6f3ab42e06c318f15c98f59a436f1c78179e6a6f40f529c8cc7100b22"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21b053be28a8a414f2ddd401f1be8361e41032d2ef5884b2f31d31cb723e559f"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b178e875a7a25b5938b53b006598ee7645172fccafe1c291a706e93f48499ff5"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a40ee2cc7ea653645bd4cf024326dea2076673fc9d3d33f20f6c81db83e1d"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295ff8689544f7ee7e819529633d058bd458c1fd7f7e3eebd0f9268ebc56c2a0"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:49496b68cd190a147118af585173ee624114dfb2e0297558c460ad7495f9dfe2"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd9b73c5c15f0ec5ce18128b1fe9157ddd0044abc373e6ecd5ba376a7e5d961"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win32.whl", hash = "sha256:9365a3da32dabd3e69e06b972b1ffb0c89668994c7e8e75ce21d3e5e69ddef28"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win_amd64.whl", hash = "sha256:8bd63d051f4f313b102a2af1cbc8b80f061bf78f3d5bd0843ff70b5859e27924"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bab3db192a0c35e3c9d1560eb8332463e29e5507dbd822e29a0a3c48c0a8d92"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:19d98f4f58b13900d8dec4ed09dd09ef292208ee44cc9c2fe01c1f0a2fe440e9"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd33c61513cb1b7371fd40cf221256456d26a56284e7d19d1f0b9f1eb7dd7e8"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6ba0497c1d066dd004e0f02a92426ca2df20fac08728d03f67f6960271feec"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b6be53e4fde0065524f1a0a7929b10e9280987b320716c1509478b712a7688c"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:916a798f62f410c0b80b63683c8061f5ebe237b0f4ad778739304253353bc1cb"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win32.whl", hash = "sha256:31983018b74908ebc6c996a16ad3690301a23befb643093fcfe85efd292e384d"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win_amd64.whl", hash = "sha256:4363ed245a6231f2e2957cccdda3c776265a75851f4753c60f3004b90e69bfeb"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8afd5b26570bf41c35c0121801479958b4446751a3971fb9a480c1afd85558e"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c750987fc876813f27b60d619b987b057eb4896b81117f73bb8d9918c14f1cad"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0102afff4890f651ed91120c1120065663506b760da4e7823913ebd3258be"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:78c03d0f8a5ab4f3034c0e8482cfcc415a3ec6193491cfa1c643ed707d476f16"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:3bd1cae7519283ff525e64645ebd7a3e0283f3c038f461ecc1c7b040a0c932a1"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win32.whl", hash = "sha256:01438ebcdc566d58c93af0171c74ec28efe6a29184b773e378a385e6215389da"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:4979dc80fbbc9d2ef569e71e0896990bc94df2b9fdbd878290bd129b65ab579c"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c742be912f57586ac43af38b3848f7688863a403dfb220193a882ea60e1ec3a"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:62e23d0ac103bcf1c5555b6c88c114089587bc64d048fef5bbdb58dfd26f96da"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:251f0d1108aab8ea7b9aadbd07fb47fb8e3a5838dde34aa95a3349876b5a1f1d"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef18a84e5116340e38eca3e7f9eeaaef62738891422e7c2a0b80feab165905f"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3eb6a97a1d39976f360b10ff208c73afb6a4de86dd2a6212ddf65c4a6a2347d5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0c1c9b673d21477cec17ab10bc4decb1322843ba35b481585facd88203754fc5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win32.whl", hash = "sha256:c41a2b9ca80ee555decc605bd3c4520cc6fef9abde8fd66b1cf65126a6922d65"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:8a37e4d265033c897892279e8adf505c8b6b4075f2b40d77afb31f7185cd6ecd"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fec964fba2ef46476312a03ec8c425956b05c20220a1a03703537824b5e8e1"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:328429aecaba2aee3d71e11f2477c14eec5990fb6d0e884107935f7fb6001632"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85a01b5599e790e76ac3fe3aa2f26e1feba56270023d6afd5550ed63c68552b3"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf04784797dcdf4c0aa952c8d234fa01974c4729db55c45732520ce12dd95b4"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4488120becf9b71b3ac718f4138269a6be99a42fe023ec457896ba4f80749525"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14e09e083a5796d513918a66f3d6aedbc131e39e80875afe81d98a03312889e6"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win32.whl", hash = "sha256:0d322cc9c9b2154ba7e82f7bf25ecc7c36fbe2d82e2933b3642fc095a52cfc78"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:7dd8583df2f98dea28b5cd53a1beac963f4f9d087888d75f22fcc93a07cf8d84"}, + {file = "SQLAlchemy-2.0.32-py3-none-any.whl", hash = "sha256:e567a8793a692451f706b363ccf3c45e056b67d90ead58c3bc9471af5d212202"}, + {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, ] [package.dependencies] diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index ddfe553..95740e3 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -36,7 +36,7 @@ globalState = {"verbose": False} -def commandInit(config: Path) -> appConfig.AppConfig: +def commandInit(config: Path | None) -> appConfig.AppConfig: # load and verify the config file if config is None: logging.critical("No config file") @@ -174,6 +174,8 @@ def fetch_binary( end_date: Annotated[str, typer.Option(help="End date for the download")], config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): + """Download binary data from WebPODA.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: @@ -221,6 +223,8 @@ def fetch_science( ] = LevelEnum.level_2, config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), ): + """Download science data from the SDC.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py new file mode 100644 index 0000000..26a49ca --- /dev/null +++ b/src/imap_mag/outputManager.py @@ -0,0 +1,126 @@ +import hashlib +import logging +import shutil +import typing +from datetime import datetime +from pathlib import Path + +import typer +import typing_extensions + + +class OutputMetadata(typing.TypedDict): + """Metadata for output files.""" + + data_level: str | None + descriptor: str + date: datetime + version: str | None + extension: str + + +class OutputManager: + """Manage output files.""" + + @staticmethod + def get_folder_structure(**metadata: OutputMetadata) -> str: + """Retrieve folder structure from metadata.""" + + if "date" not in metadata: + logging.error(f"Metadata must contain key 'date'. Got: {metadata.keys()}") + raise typer.Abort() + + return metadata["date"].strftime("%Y/%m/%d") + + @staticmethod + def get_file_name(**metadata: OutputMetadata) -> str: + """Retireve file name from metadata.""" + + if {"descriptor", "date", "version", "extension"} > metadata.keys(): + logging.error( + f"Metadata must contain keys 'descriptor', 'date', 'version', 'extension'. Got: {metadata.keys()}" + ) + raise typer.Abort() + + return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-{metadata['version']}.{metadata['extension']}" + + """Output location.""" + location: Path + """Function returning folder structure pattern.""" + folder_structure_provider: typing.Callable[..., str] = get_folder_structure + """Function returning file name pattern.""" + file_name_provider: typing.Callable[..., str] = get_file_name + + def __init__( + self, + location: Path, + *, + folder_structure_provider: typing.Callable[..., str] | None = None, + file_name_provider: typing.Callable[..., str] | None = None, + ) -> None: + self.location = location + + if folder_structure_provider is not None: + self.folder_structure_provider = folder_structure_provider + + if file_name_provider is not None: + self.file_name_provider = file_name_provider + + def add_file( + self, original_file: Path, **metadata: typing_extensions.Unpack[OutputMetadata] + ) -> Path: + """Add file to output location.""" + + if ("version" not in metadata) or (metadata["version"] is None): + logging.debug("No version provided. Setting to 'v000'.") + metadata["version"] = "v000" + + if not self.location.exists(): + self.location.mkdir(parents=True, exist_ok=True) + + destination_file: Path = self.__assemble_full_path(**metadata) + + if not destination_file.parent.exists(): + destination_file.parent.mkdir(parents=True, exist_ok=True) + + if destination_file.exists(): + if ( + hashlib.md5(destination_file.read_bytes()).hexdigest() + == hashlib.md5(original_file.read_bytes()).hexdigest() + ): + logging.info(f"File {destination_file} already exists and is the same.") + return destination_file + + metadata["version"] = self.__find_viable_version( + destination_file, **metadata + ) + destination_file = self.__assemble_full_path(**metadata) + + logging.info(f"Copying {original_file} to {destination_file.absolute()}.") + destination = shutil.copy2(original_file, destination_file) + logging.info(f"Copied to {destination}.") + + return destination_file + + def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: + """Assemble full path from metadata.""" + + return ( + self.location + / self.folder_structure_provider(**metadata) + / self.file_name_provider(**metadata) + ) + + def __find_viable_version( + self, destination_file: Path, **metadata: OutputMetadata + ) -> str: + """Find a viable version for a file.""" + + while destination_file.exists(): + logging.info( + f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." + ) + metadata["version"] = "v%03d" % (int(metadata["version"][1:]) + 1) + destination_file = self.__assemble_full_path(**metadata) + + return metadata["version"] diff --git a/tests/testUtils.py b/tests/testUtils.py index c233b09..fffca72 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -2,7 +2,31 @@ from pathlib import Path, PosixPath import imap_mag.appConfig as appConfig +import pytest import yaml +from imap_mag import appLogging + + +@pytest.fixture(autouse=True) +def enableLogging(): + appLogging.set_up_logging( + console_log_output="stdout", + console_log_level="debug", + console_log_color=True, + logfile_file="debug", + logfile_log_level="debug", + logfile_log_color=False, + log_line_template="%(color_on)s[%(asctime)s] [%(levelname)-8s] %(message)s%(color_off)s", + console_log_line_template="%(color_on)s%(message)s%(color_off)s", + ) + yield + + +@pytest.fixture(autouse=True) +def tidyDataFolders(): + os.system("rm -rf .work") + os.system("rm -rf output/*") + yield def create_serialize_config( diff --git a/tests/test_main.py b/tests/test_main.py index 16a4066..10a9df2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,23 +7,15 @@ import re from pathlib import Path -import pytest from imap_mag.main import app from typer.testing import CliRunner -from .testUtils import create_serialize_config +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 from .wiremockUtils import wiremock_manager # noqa: F401 runner = CliRunner() -@pytest.fixture(autouse=True) -def tidyDataFolders(): - os.system("rm -rf .work") - os.system("rm -rf output/*") - yield - - def test_app_says_hello(): result = runner.invoke(app, ["hello", "Bob"]) diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py new file mode 100644 index 0000000..3a57596 --- /dev/null +++ b/tests/test_outputManager.py @@ -0,0 +1,123 @@ +"""Tests for `OutputManager` class.""" + +from datetime import datetime +from pathlib import Path + +from imap_mag.outputManager import OutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def test_copy_new_file(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v000.txt").exists() + + +def test_copy_file_same_content(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + original_file.write_bytes(b"some content") + + existing_file = Path("output/2025/05/02/pwr-20250502-v000.txt") + existing_file.parent.mkdir(parents=True, exist_ok=True) + existing_file.touch() + existing_file.write_bytes(b"some content") + + existing_modification_time = existing_file.stat().st_mtime + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert not Path("output/2025/05/02/pwr-20250502-v001.txt").exists() + assert existing_file.stat().st_mtime == existing_modification_time + + +def test_copy_file_existing_versions(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + original_file.write_bytes(b"some content") + + for version in range(2): + existing_file = Path(f"output/2025/05/02/pwr-20250502-v{version:03}.txt") + existing_file.parent.mkdir(parents=True, exist_ok=True) + existing_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v002.txt").exists() + + +def test_copy_file_forced_version(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + version="v003", + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() + + +def test_copy_file_custom_providers(): + # Set up. + manager = OutputManager( + Path("output"), + folder_structure_provider=lambda **_: "abc", + file_name_provider=lambda **_: "def", + ) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file(original_file) + + # Verify. + assert Path("output/abc/def").exists() From 231e0c28a5beed69ae6b7e0404a24a3db91a3dc6 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 14:33:55 +0000 Subject: [PATCH 002/102] refactor: version is an `int` not a `str` --- src/imap_mag/outputManager.py | 10 +++++----- tests/test_outputManager.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 26a49ca..b72cffc 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -15,7 +15,7 @@ class OutputMetadata(typing.TypedDict): data_level: str | None descriptor: str date: datetime - version: str | None + version: int | None extension: str @@ -42,7 +42,7 @@ def get_file_name(**metadata: OutputMetadata) -> str: ) raise typer.Abort() - return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-{metadata['version']}.{metadata['extension']}" + return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-v{metadata['version']:03}.{metadata['extension']}" """Output location.""" location: Path @@ -73,7 +73,7 @@ def add_file( if ("version" not in metadata) or (metadata["version"] is None): logging.debug("No version provided. Setting to 'v000'.") - metadata["version"] = "v000" + metadata["version"] = 0 if not self.location.exists(): self.location.mkdir(parents=True, exist_ok=True) @@ -113,14 +113,14 @@ def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: def __find_viable_version( self, destination_file: Path, **metadata: OutputMetadata - ) -> str: + ) -> int: """Find a viable version for a file.""" while destination_file.exists(): logging.info( f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." ) - metadata["version"] = "v%03d" % (int(metadata["version"][1:]) + 1) + metadata["version"] += 1 destination_file = self.__assemble_full_path(**metadata) return metadata["version"] diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 3a57596..4ad826e 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -96,7 +96,7 @@ def test_copy_file_forced_version(): original_file, descriptor="pwr", date=datetime(2025, 5, 2), - version="v003", + version=3, extension="txt", ) From 3741a1f25b3655f3cc23bd7c537a158ec300d02a Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 15:09:41 +0000 Subject: [PATCH 003/102] refactor: replace custom logic with `OutputManager` --- src/imap_mag/appUtils.py | 29 +++++++++++++++-------------- src/imap_mag/outputManager.py | 4 ++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index 5d4250a..dfa1258 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -1,6 +1,4 @@ import logging -import os -import shutil from pathlib import Path from typing import Optional @@ -8,7 +6,8 @@ import pandas as pd import typer -from . import appConfig +from .appConfig import Destination +from .outputManager import OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -56,18 +55,20 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() -def copyFileToDestination(filePath: Path, destination: appConfig.Destination) -> None: +def copyFileToDestination( + file_path: Path, + destination: Destination, + output_manager: Optional[OutputManager] = None, +) -> Path: """Copy file to destination folder.""" - destinationFile = Path(destination.folder) + destination_folder = Path(destination.folder) - if not destinationFile.exists(): - logging.debug(f"Creating destination folder {destinationFile}.") - os.makedirs(destinationFile) + if output_manager is None: + output_manager: OutputManager = OutputManager( + destination_folder, + folder_structure_provider=lambda **_: "", + file_name_provider=lambda **_: file_path.name, + ) - if destination.filename: - destinationFile = destinationFile / destination.filename - - logging.info(f"Copying {filePath} to {destinationFile.absolute()}") - completed = shutil.copy2(filePath, destinationFile) - logging.info(f"Copy complete: {completed}") + output_manager.add_file(file_path) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index b72cffc..4d91260 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -76,11 +76,15 @@ def add_file( metadata["version"] = 0 if not self.location.exists(): + logging.debug(f"Output location does not exist. Creating {self.location}.") self.location.mkdir(parents=True, exist_ok=True) destination_file: Path = self.__assemble_full_path(**metadata) if not destination_file.parent.exists(): + logging.debug( + f"Output folder structure does not exist. Creating {destination_file.parent}." + ) destination_file.parent.mkdir(parents=True, exist_ok=True) if destination_file.exists(): From fdcd9ba2f4c44e9f8f1cc4041ca219e7eda65cc9 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 08:20:08 +0000 Subject: [PATCH 004/102] refactor: convert "pattern" methods to separate "provider" class --- src/imap_mag/appUtils.py | 6 +- src/imap_mag/outputManager.py | 125 ++++++++++++++++++---------------- tests/test_outputManager.py | 26 ++++--- 3 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index dfa1258..435e8e0 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -7,7 +7,7 @@ import typer from .appConfig import Destination -from .outputManager import OutputManager +from .outputManager import IMetadataProvider, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -59,7 +59,7 @@ def copyFileToDestination( file_path: Path, destination: Destination, output_manager: Optional[OutputManager] = None, -) -> Path: +) -> tuple[Path, IMetadataProvider]: """Copy file to destination folder.""" destination_folder = Path(destination.folder) @@ -71,4 +71,4 @@ def copyFileToDestination( file_name_provider=lambda **_: file_path.name, ) - output_manager.add_file(file_path) + return output_manager.add_file(file_path) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 4d91260..aac4f35 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -1,3 +1,4 @@ +import abc import hashlib import logging import shutil @@ -6,80 +7,84 @@ from pathlib import Path import typer -import typing_extensions -class OutputMetadata(typing.TypedDict): +class IMetadataProvider(abc.ABC): + """Interface for metadata providers.""" + + version: int = 0 + + @abc.abstractmethod + def get_folder_structure(self) -> str: + """Retrieve folder structure.""" + + @abc.abstractmethod + def get_file_name(self) -> str: + """Retireve file name.""" + + +class DefaultMetadataProvider(IMetadataProvider): """Metadata for output files.""" - data_level: str | None + data_level: str | None = None descriptor: str date: datetime - version: int | None extension: str + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) -class OutputManager: - """Manage output files.""" - - @staticmethod - def get_folder_structure(**metadata: OutputMetadata) -> str: - """Retrieve folder structure from metadata.""" - - if "date" not in metadata: - logging.error(f"Metadata must contain key 'date'. Got: {metadata.keys()}") + def get_folder_structure(self) -> str: + if self.date is None: + logging.error("No 'date' defined. Cannot generate folder structure.") raise typer.Abort() - return metadata["date"].strftime("%Y/%m/%d") - - @staticmethod - def get_file_name(**metadata: OutputMetadata) -> str: - """Retireve file name from metadata.""" + return self.date.strftime("%Y/%m/%d") - if {"descriptor", "date", "version", "extension"} > metadata.keys(): + def get_file_name(self) -> str: + if any(x is None for x in ["descriptor", "date", "version", "extension"]): logging.error( - f"Metadata must contain keys 'descriptor', 'date', 'version', 'extension'. Got: {metadata.keys()}" + "No 'descriptor', 'date', 'version', or 'extension' defined. Cannot generate file name." ) raise typer.Abort() - return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-v{metadata['version']:03}.{metadata['extension']}" - - """Output location.""" - location: Path - """Function returning folder structure pattern.""" - folder_structure_provider: typing.Callable[..., str] = get_folder_structure - """Function returning file name pattern.""" - file_name_provider: typing.Callable[..., str] = get_file_name - - def __init__( - self, - location: Path, - *, - folder_structure_provider: typing.Callable[..., str] | None = None, - file_name_provider: typing.Callable[..., str] | None = None, - ) -> None: - self.location = location + return f"{self.descriptor}-{self.date.strftime('%Y%m%d')}-v{self.version:03}.{self.extension}" - if folder_structure_provider is not None: - self.folder_structure_provider = folder_structure_provider - if file_name_provider is not None: - self.file_name_provider = file_name_provider +class IOutputManager(abc.ABC): + """Interface for output managers.""" + @abc.abstractmethod def add_file( - self, original_file: Path, **metadata: typing_extensions.Unpack[OutputMetadata] - ) -> Path: + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: """Add file to output location.""" - if ("version" not in metadata) or (metadata["version"] is None): - logging.debug("No version provided. Setting to 'v000'.") - metadata["version"] = 0 + def add_default_file( + self, original_file: Path, **metadata: typing.Any + ) -> tuple[Path, IMetadataProvider]: + return self.add_file(original_file, DefaultMetadataProvider(**metadata)) + + +class OutputManager(IOutputManager): + """Manage output files.""" + + location: Path + + def __init__(self, location: Path) -> None: + self.location = location + + def add_file( + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: + """Add file to output location.""" if not self.location.exists(): logging.debug(f"Output location does not exist. Creating {self.location}.") self.location.mkdir(parents=True, exist_ok=True) - destination_file: Path = self.__assemble_full_path(**metadata) + destination_file: Path = self.__assemble_full_path(metadata_provider) if not destination_file.parent.exists(): logging.debug( @@ -93,38 +98,38 @@ def add_file( == hashlib.md5(original_file.read_bytes()).hexdigest() ): logging.info(f"File {destination_file} already exists and is the same.") - return destination_file + return (destination_file, metadata_provider) - metadata["version"] = self.__find_viable_version( - destination_file, **metadata + metadata_provider.version = self.__find_viable_version( + destination_file, metadata_provider ) - destination_file = self.__assemble_full_path(**metadata) + destination_file = self.__assemble_full_path(metadata_provider) logging.info(f"Copying {original_file} to {destination_file.absolute()}.") destination = shutil.copy2(original_file, destination_file) logging.info(f"Copied to {destination}.") - return destination_file + return (destination_file, metadata_provider) - def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: + def __assemble_full_path(self, metadata_provider: IMetadataProvider) -> Path: """Assemble full path from metadata.""" return ( self.location - / self.folder_structure_provider(**metadata) - / self.file_name_provider(**metadata) + / metadata_provider.get_folder_structure() + / metadata_provider.get_file_name() ) def __find_viable_version( - self, destination_file: Path, **metadata: OutputMetadata + self, destination_file: Path, metadata_provider: IMetadataProvider ) -> int: """Find a viable version for a file.""" while destination_file.exists(): logging.info( - f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." + f"File {destination_file} already exists and is different. Increasing version to {metadata_provider.version}." ) - metadata["version"] += 1 - destination_file = self.__assemble_full_path(**metadata) + metadata_provider.version += 1 + destination_file = self.__assemble_full_path(metadata_provider) - return metadata["version"] + return metadata_provider.version diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 4ad826e..3d10cd4 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import Path -from imap_mag.outputManager import OutputManager +from imap_mag.outputManager import IMetadataProvider, OutputManager from .testUtils import enableLogging, tidyDataFolders # noqa: F401 @@ -17,7 +17,7 @@ def test_copy_new_file(): original_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -45,7 +45,7 @@ def test_copy_file_same_content(): existing_modification_time = existing_file.stat().st_mtime # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -72,7 +72,7 @@ def test_copy_file_existing_versions(): existing_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -92,7 +92,7 @@ def test_copy_file_forced_version(): original_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -104,20 +104,24 @@ def test_copy_file_forced_version(): assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() +class TestMetadataProvider(IMetadataProvider): + def get_folder_structure(self) -> str: + return "abc" + + def get_file_name(self) -> str: + return "def" + + def test_copy_file_custom_providers(): # Set up. - manager = OutputManager( - Path("output"), - folder_structure_provider=lambda **_: "abc", - file_name_provider=lambda **_: "def", - ) + manager = OutputManager(Path("output")) original_file = Path(".work/some_test_file.txt") original_file.parent.mkdir(parents=True, exist_ok=True) original_file.touch() # Exercise. - manager.add_file(original_file) + manager.add_file(original_file, TestMetadataProvider()) # Verify. assert Path("output/abc/def").exists() From 8ca6ca32975dabe03c3ff661b9b1b045827f7cf0 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 08:20:35 +0000 Subject: [PATCH 005/102] build: single source app version --- poetry.lock | 125 +++++++++++++++++++++------------------ pyproject.toml | 4 +- src/imap_mag/__init__.py | 7 +++ 3 files changed, 79 insertions(+), 57 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8eb2f9f..fbf6cdb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -599,12 +599,12 @@ files = [ [[package]] name = "imap-data-access" -version = "0.7.0" +version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" files = [ - {file = "imap_data_access-0.7.0.tar.gz", hash = "sha256:f0db935949d048394fc554b308b1e4a1572a18acd41636462d37c309c7cb4c9d"}, + {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, ] [package.extras] @@ -1289,62 +1289,64 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -1440,6 +1442,17 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "single-version" +version = "1.6.0" +description = "Small utility to define version string for Poetry-style Python project." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "single_version-1.6.0-py3-none-any.whl", hash = "sha256:67a2734e728b9554750e867b33591f3ad9509ccb851bb3047ced7bfe68429ecd"}, + {file = "single_version-1.6.0.tar.gz", hash = "sha256:3b1fb6e9bd2c88268948d9191c78b63ddd3c07554c1f07cd8a85aedf2486e4fc"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1840,4 +1853,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "9f5ddbc86d7476e8e2f31f45bb941b5679d3f3976c92a4a5b744c8ab24ecc224" +content-hash = "40096a30f5c8ed7a5daaaaa5da50d0872d3ae7ab77e08256087fb8e44fb8a779" diff --git a/pyproject.toml b/pyproject.toml index ad093bd..2b0edfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] requires-python = ">=3.10" name = "imap-mag" +version = "0.0.1" [tool.poetry] name = "imap-mag" @@ -29,8 +30,9 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.7.0" +imap-data-access = "^0.8.0" cdflib = "^1.3.1" +single-version = "^1.6.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/src/imap_mag/__init__.py b/src/imap_mag/__init__.py index e69de29..ad47412 100644 --- a/src/imap_mag/__init__.py +++ b/src/imap_mag/__init__.py @@ -0,0 +1,7 @@ +"""The main module for project.""" + +from pathlib import Path + +from single_version import get_version + +__version__ = get_version("imap-mag", Path(__file__).parent.parent) From b4087a22dc500aca367681671fdea68e02e1c585 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 10:15:17 +0000 Subject: [PATCH 006/102] style: replace `-` with `_` in file names --- src/imap_mag/outputManager.py | 2 +- tests/test_outputManager.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index aac4f35..4347bee 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -49,7 +49,7 @@ def get_file_name(self) -> str: ) raise typer.Abort() - return f"{self.descriptor}-{self.date.strftime('%Y%m%d')}-v{self.version:03}.{self.extension}" + return f"{self.descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" class IOutputManager(abc.ABC): diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 3d10cd4..9466c03 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -25,7 +25,7 @@ def test_copy_new_file(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v000.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v000.txt").exists() def test_copy_file_same_content(): @@ -37,7 +37,7 @@ def test_copy_file_same_content(): original_file.touch() original_file.write_bytes(b"some content") - existing_file = Path("output/2025/05/02/pwr-20250502-v000.txt") + existing_file = Path("output/2025/05/02/pwr_20250502_v000.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() existing_file.write_bytes(b"some content") @@ -53,7 +53,7 @@ def test_copy_file_same_content(): ) # Verify. - assert not Path("output/2025/05/02/pwr-20250502-v001.txt").exists() + assert not Path("output/2025/05/02/pwr_20250502_v001.txt").exists() assert existing_file.stat().st_mtime == existing_modification_time @@ -67,7 +67,7 @@ def test_copy_file_existing_versions(): original_file.write_bytes(b"some content") for version in range(2): - existing_file = Path(f"output/2025/05/02/pwr-20250502-v{version:03}.txt") + existing_file = Path(f"output/2025/05/02/pwr_20250502_v{version:03}.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() @@ -80,7 +80,7 @@ def test_copy_file_existing_versions(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v002.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v002.txt").exists() def test_copy_file_forced_version(): @@ -101,7 +101,7 @@ def test_copy_file_forced_version(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v003.txt").exists() class TestMetadataProvider(IMetadataProvider): From da4c44b3987ad343ccdbd96e67e03db3e76acf45 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 11:00:12 +0000 Subject: [PATCH 007/102] task: add support for `prefix` and `level` in output metadata --- src/imap_mag/outputManager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 4347bee..e46a4d0 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -26,7 +26,8 @@ def get_file_name(self) -> str: class DefaultMetadataProvider(IMetadataProvider): """Metadata for output files.""" - data_level: str | None = None + prefix: str | None = "imap_mag" + level: str | None = None descriptor: str date: datetime extension: str @@ -49,7 +50,15 @@ def get_file_name(self) -> str: ) raise typer.Abort() - return f"{self.descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" + descriptor = self.descriptor + + if self.level is not None: + descriptor = f"{self.level}_{descriptor}" + + if self.prefix is not None: + descriptor = f"{self.prefix}_{descriptor}" + + return f"{descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" class IOutputManager(abc.ABC): From 99773b5ec2c0fc9110b87eb4c44d59860b706c1b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:04:31 +0000 Subject: [PATCH 008/102] feat: add more variables in database --- ...dded_version_date_and_software_version_.py | 34 +++++++++++++++++++ src/imap_db/model.py | 7 +++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py diff --git a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py b/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py new file mode 100644 index 0000000..ccb34bd --- /dev/null +++ b/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py @@ -0,0 +1,34 @@ +"""Added version, date and software version columns + +Revision ID: 77642de1867a +Revises: d0457f3e98c8 +Create Date: 2024-08-07 13:03:30.889623 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "77642de1867a" +down_revision = "d0457f3e98c8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("files", sa.Column("version", sa.Integer(), nullable=False)) + op.add_column("files", sa.Column("date", sa.DateTime(), nullable=False)) + op.add_column( + "files", sa.Column("software_version", sa.String(length=16), nullable=False) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("files", "software_version") + op.drop_column("files", "date") + op.drop_column("files", "version") + # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py index 6cfbcb6..c2be860 100644 --- a/src/imap_db/model.py +++ b/src/imap_db/model.py @@ -1,4 +1,6 @@ -from sqlalchemy import String +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -11,6 +13,9 @@ class File(Base): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(128)) path: Mapped[str] = mapped_column(String(256)) + version: Mapped[int] = mapped_column(Integer()) + date: Mapped[datetime] = mapped_column(DateTime()) + software_version: Mapped[str] = mapped_column(String(16)) def __repr__(self) -> str: return f"" From 7ede655c8e7b17a1e996e4279605a9dde6f075f0 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:23:39 +0000 Subject: [PATCH 009/102] feat!: use output manager to write to output and database BREAKING: some tests no longer pass --- .pre-commit-config.yaml | 4 +- src/imap_mag/DB.py | 33 +++++++++++++- src/imap_mag/cli/fetchBinary.py | 78 ++++++++++++++++++++++++++++++++ src/imap_mag/cli/fetchScience.py | 29 ++++++++---- src/imap_mag/main.py | 43 ++++++++---------- 5 files changed, 152 insertions(+), 35 deletions(-) create mode 100644 src/imap_mag/cli/fetchBinary.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10cbade..080c7f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: hooks: - id: check-github-workflows - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.5 + rev: v0.5.6 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -36,7 +36,7 @@ repos: - id: markdownlint-fix args: [--ignore, LICENSE.md] - repo: https://github.com/python-poetry/poetry - rev: "1.8" + rev: "1.8.0" hooks: - id: poetry-check - id: poetry-lock diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 9dff8f3..12e2aba 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,9 +1,12 @@ import os +from pathlib import Path from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from src.imap_db.model import File +from src.imap_mag import __version__ +from src.imap_mag.outputManager import IMetadataProvider, IOutputManager class DB: @@ -15,7 +18,10 @@ def __init__(self, db_url=None): self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - def insert_files(self, files: list[File]): + def insert_file(self, file: File) -> None: + self.insert_files([file]) + + def insert_files(self, files: list[File]) -> None: session = self.Session() try: for file in files: @@ -36,3 +42,28 @@ def insert_files(self, files: list[File]): raise e finally: session.close() + + +class DatabaseOutputManager(IOutputManager): + def __init__(self, output_manager: IOutputManager, db: DB = DB()): + self.output_manager = output_manager + self.db = db + + def add_file( + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: + (destination_file, metadata_provider) = self.output_manager.add_file( + original_file, metadata_provider + ) + + self.db.insert_file( + File( + name=destination_file.name, + path=destination_file.absolute().as_posix(), + version=metadata_provider.version, + date=metadata_provider.date, + software_version=__version__, + ) + ) + + return (destination_file, metadata_provider) diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py new file mode 100644 index 0000000..a450180 --- /dev/null +++ b/src/imap_mag/cli/fetchBinary.py @@ -0,0 +1,78 @@ +"""Program to retrieve and process MAG binary files.""" + +import typing +from datetime import datetime +from pathlib import Path + +import pandas as pd +import typing_extensions + +from ..client.webPODA import WebPODA +from ..outputManager import IOutputManager + + +class FetchBinaryOptions(typing.TypedDict): + """Options for WebPODA interactions.""" + + packet: str + start_date: datetime + end_date: datetime + + +class FetchBinary: + """Manage WebPODA data.""" + + __web_poda: WebPODA + __output_manager: IOutputManager | None + + def __init__( + self, + web_poda: WebPODA, + output_manager: IOutputManager | None = None, + ) -> None: + """Initialize WebPODA interface.""" + + self.__web_poda = web_poda + self.__output_manager = output_manager + + def download_binaries( + self, **options: typing_extensions.Unpack[FetchBinaryOptions] + ) -> list[Path]: + """Retrieve WebPODA data.""" + + downloaded = [] + + date_range: pd.DatetimeIndex = pd.date_range( + start=options["start_date"], + end=options["end_date"], + freq="D", + normalize=True, + ) + + dates = date_range.to_pydatetime().tolist() + if len(dates) == 1: + dates += [ + pd.Timestamp(dates[0] + pd.Timedelta(days=1)) + .normalize() + .to_pydatetime() + ] + + for d in range(len(dates) - 1): + file = self.__web_poda.download( + packet=options["packet"], start_date=dates[d], end_date=dates[d + 1] + ) + + if self.__output_manager is not None: + self.__output_manager.add_default_file( + file, + descriptor=options["packet"] + .lower() + .strip("mag_") + .replace("_", "-"), + date=dates[d], + extension="pkts", + ) + + downloaded += [file] + + return downloaded diff --git a/src/imap_mag/cli/fetchScience.py b/src/imap_mag/cli/fetchScience.py index 49fd3e9..6da8d93 100644 --- a/src/imap_mag/cli/fetchScience.py +++ b/src/imap_mag/cli/fetchScience.py @@ -1,14 +1,15 @@ """Program to retrieve and process MAG CDF files.""" import typing +from datetime import datetime from enum import Enum from pathlib import Path import pandas as pd import typing_extensions -from .. import appUtils from ..client.sdcDataAccess import ISDCDataAccess +from ..outputManager import IOutputManager class MAGMode(str, Enum): @@ -25,28 +26,31 @@ class FetchScienceOptions(typing.TypedDict): """Options for SOC interactions.""" level: str - start_date: str - end_date: str + start_date: datetime + end_date: datetime output_dir: str class FetchScience: """Manage SOC data.""" + __data_access: ISDCDataAccess + __output_manager: IOutputManager | None + __modes: list[MAGMode] __sensor: list[MAGSensor] - __data_access: ISDCDataAccess - def __init__( self, data_access: ISDCDataAccess, + output_manager: IOutputManager | None = None, modes: list[MAGMode] = ["norm", "burst"], sensors: list[MAGSensor] = ["magi", "mago"], ) -> None: """Initialize SDC interface.""" self.__data_access = data_access + self.__output_manager = output_manager self.__modes = modes self.__sensor = sensors @@ -59,8 +63,8 @@ def download_latest_science( for mode in self.__modes: date_range: pd.DatetimeIndex = pd.date_range( - start=appUtils.convertToDatetime(options["start_date"]), - end=appUtils.convertToDatetime(options["end_date"]), + start=options["start_date"], + end=options["end_date"], freq="D", normalize=True, ) @@ -71,7 +75,7 @@ def download_latest_science( level=options["level"], descriptor=str(mode) + "-" + str(sensor), start_date=date, - end_date=None, + end_date=date, version="latest", extension="cdf", ) @@ -82,4 +86,13 @@ def download_latest_science( self.__data_access.download(file["file_path"]) ] + if self.__output_manager is not None: + self.__output_manager.add_default_file( + downloaded[-1], + level=options["level"], + descriptor=file["descriptor"], + date=date, + extension="cdf", + ) + return downloaded diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index 95740e3..c4d0410 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -14,7 +14,6 @@ # config import yaml -from src.imap_db.model import File from src.mag_toolkit import CDFLoader from src.mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator from src.mag_toolkit.calibration.calibrationFormatProcessor import ( @@ -27,10 +26,13 @@ SpinPlaneCalibrator, ) -from . import DB, appConfig, appLogging, appUtils, imapProcessing +from . import appConfig, appLogging, appUtils, imapProcessing +from .cli.fetchBinary import FetchBinary from .cli.fetchScience import FetchScience from .client.sdcDataAccess import SDCDataAccess from .client.webPODA import WebPODA +from .DB import DatabaseOutputManager +from .outputManager import OutputManager app = typer.Typer() globalState = {"verbose": False} @@ -183,6 +185,9 @@ def fetch_binary( raise typer.Abort() packet: str = appUtils.getPacketFromApID(apid) + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading raw packet {packet} from {start_date} to {end_date}.") poda = WebPODA( @@ -190,13 +195,12 @@ def fetch_binary( configFile.work_folder, configFile.api.webpoda_url if configFile.api else None, ) - result: str = poda.download( - packet=packet, - start_date=appUtils.convertToDatetime(start_date), - end_date=appUtils.convertToDatetime(end_date), - ) + output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) - appUtils.copyFileToDestination(result, configFile.destination) + fetch_binary = FetchBinary(poda, output_manager) + fetch_binary.download_binaries( + packet=packet, start_date=start_date, end_date=end_date + ) class LevelEnum(str, Enum): @@ -221,7 +225,7 @@ def fetch_science( level: Annotated[ LevelEnum, typer.Option(help="Level to download") ] = LevelEnum.level_2, - config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), + config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): """Download science data from the SDC.""" @@ -231,31 +235,22 @@ def fetch_science( logging.critical("No SDC_AUTH_CODE API key provided") raise typer.Abort() + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading {level} science from {start_date} to {end_date}.") data_access = SDCDataAccess( data_dir=str(configFile.work_folder), sdc_url=configFile.api.sdc_url if configFile.api else None, ) + output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) - fetch_science = FetchScience(data_access) - files = fetch_science.download_latest_science( + fetch_science = FetchScience(data_access, output_manager) + fetch_science.download_latest_science( level=level.value, start_date=start_date, end_date=end_date ) - records = [] - for file in files: - records.append(File(name=file.name, path=file.absolute().as_posix())) - - for file in files: - appUtils.copyFileToDestination(file, configFile.destination) - - if configFile.destination.export_to_database: - db = DB.DB() - db.insert_files(records) - - logging.info(f"Downloaded {len(files)} files and saved to database") - # imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf @app.command() From 76afe1e54ae164c4e7e5a18e964bb5a73d4bb3e3 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:25:26 +0000 Subject: [PATCH 010/102] dev: add `SQLALCHEMY_URL` to dev container environment --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6474864..eb6693c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,6 +34,7 @@ "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}", "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}", "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}", + "SQLALCHEMY_URL": "${localEnv:SQLALCHEMY_URL}", // Define WireMock variables to connect Docker outside of Docker. "WIREMOCK_DIND": "1", "TESTCONTAINERS_HOST_OVERRIDE": "host.docker.internal" From f6927b1bb78034e147e49d499b4a1e7a11ad2117 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:32:15 +0000 Subject: [PATCH 011/102] test(fix): fix output manager and calibration tests --- src/imap_mag/appUtils.py | 22 ++++++++++++++++------ tests/test_outputManager.py | 14 ++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index 435e8e0..bc90af7 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -62,13 +62,23 @@ def copyFileToDestination( ) -> tuple[Path, IMetadataProvider]: """Copy file to destination folder.""" + class SimpleMetadataProvider(IMetadataProvider): + """Simple metadata provider for compatibility.""" + + def __init__(self, filename: str) -> None: + self.filename = filename + + def get_folder_structure(self) -> str: + return "" + + def get_file_name(self) -> str: + return self.filename + destination_folder = Path(destination.folder) if output_manager is None: - output_manager: OutputManager = OutputManager( - destination_folder, - folder_structure_provider=lambda **_: "", - file_name_provider=lambda **_: file_path.name, - ) + output_manager: OutputManager = OutputManager(destination_folder) - return output_manager.add_file(file_path) + return output_manager.add_file( + file_path, SimpleMetadataProvider(destination.filename) + ) diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 9466c03..912ef7b 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -25,7 +25,7 @@ def test_copy_new_file(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v000.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt").exists() def test_copy_file_same_content(): @@ -37,7 +37,7 @@ def test_copy_file_same_content(): original_file.touch() original_file.write_bytes(b"some content") - existing_file = Path("output/2025/05/02/pwr_20250502_v000.txt") + existing_file = Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() existing_file.write_bytes(b"some content") @@ -53,7 +53,7 @@ def test_copy_file_same_content(): ) # Verify. - assert not Path("output/2025/05/02/pwr_20250502_v001.txt").exists() + assert not Path("output/2025/05/02/imap_mag_pwr_20250502_v001.txt").exists() assert existing_file.stat().st_mtime == existing_modification_time @@ -67,7 +67,9 @@ def test_copy_file_existing_versions(): original_file.write_bytes(b"some content") for version in range(2): - existing_file = Path(f"output/2025/05/02/pwr_20250502_v{version:03}.txt") + existing_file = Path( + f"output/2025/05/02/imap_mag_pwr_20250502_v{version:03}.txt" + ) existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() @@ -80,7 +82,7 @@ def test_copy_file_existing_versions(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v002.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v002.txt").exists() def test_copy_file_forced_version(): @@ -101,7 +103,7 @@ def test_copy_file_forced_version(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v003.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v003.txt").exists() class TestMetadataProvider(IMetadataProvider): From 3eb23f1312cb78d39e32417d7dda4debec3cdc3e Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:45:25 +0000 Subject: [PATCH 012/102] fix: do not export to database in tests --- src/imap_mag/appUtils.py | 14 +++++++++++++- src/imap_mag/main.py | 6 ++---- tests/config/calibration_application_config.yaml | 3 ++- tests/config/calibration_config.yaml | 3 ++- tests/config/hk_process.yaml | 1 + 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index bc90af7..c87410c 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -7,7 +7,8 @@ import typer from .appConfig import Destination -from .outputManager import IMetadataProvider, OutputManager +from .DB import DatabaseOutputManager +from .outputManager import IMetadataProvider, IOutputManager, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -55,6 +56,17 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() +def getOutputManager(destination: Destination) -> IOutputManager: + """Retrieve output manager based on destination.""" + + output_manager = OutputManager(destination.folder) + + if destination.export_to_database: + output_manager = DatabaseOutputManager(output_manager) + + return output_manager + + def copyFileToDestination( file_path: Path, destination: Destination, diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index c4d0410..39cec92 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -31,8 +31,6 @@ from .cli.fetchScience import FetchScience from .client.sdcDataAccess import SDCDataAccess from .client.webPODA import WebPODA -from .DB import DatabaseOutputManager -from .outputManager import OutputManager app = typer.Typer() globalState = {"verbose": False} @@ -195,7 +193,7 @@ def fetch_binary( configFile.work_folder, configFile.api.webpoda_url if configFile.api else None, ) - output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) + output_manager = appUtils.getOutputManager(configFile.destination) fetch_binary = FetchBinary(poda, output_manager) fetch_binary.download_binaries( @@ -244,7 +242,7 @@ def fetch_science( data_dir=str(configFile.work_folder), sdc_url=configFile.api.sdc_url if configFile.api else None, ) - output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) + output_manager = appUtils.getOutputManager(configFile.destination) fetch_science = FetchScience(data_access, output_manager) fetch_science.download_latest_science( diff --git a/tests/config/calibration_application_config.yaml b/tests/config/calibration_application_config.yaml index b748acb..f6e3448 100644 --- a/tests/config/calibration_application_config.yaml +++ b/tests/config/calibration_application_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: L2.cdf \ No newline at end of file + filename: L2.cdf + export-to-database: false diff --git a/tests/config/calibration_config.yaml b/tests/config/calibration_config.yaml index 5ed5230..a09829b 100644 --- a/tests/config/calibration_config.yaml +++ b/tests/config/calibration_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: calibration.json \ No newline at end of file + filename: calibration.json + export-to-database: false diff --git a/tests/config/hk_process.yaml b/tests/config/hk_process.yaml index f3d4af1..8a72ea8 100644 --- a/tests/config/hk_process.yaml +++ b/tests/config/hk_process.yaml @@ -6,6 +6,7 @@ work-folder: .work destination: folder: output/ filename: result.csv + export-to-database: false packet-definition: hk: src/imap_mag/xtce/tlm_20240724.xml From 332cb246bc2fe3e31a194761da94dabbc9f89ab7 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 14:17:32 +0000 Subject: [PATCH 013/102] fix: HK and science tests --- tests/test_main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 10a9df2..05e7dac 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -106,10 +106,10 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/power.pkts").exists() + assert Path("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts").exists() with ( - open("output/power.pkts", "rb") as output, + open("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts", "rb") as output, open(binary_file, "rb") as input, ): assert output.read() == input.read() @@ -135,7 +135,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 ) wiremock_manager.add_string_mapping( - "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&version=latest&extension=cdf", + "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&end_date=20250502&extension=cdf", json.dumps(query_response), priority=1, ) @@ -146,7 +146,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 wiremock_manager.add_string_mapping( re.escape("/query?instrument=mag&data_level=l1b&descriptor=") + ".*" - + re.escape("&start_date=20250502&version=latest&extension=cdf"), + + re.escape("&start_date=20250502&end_date=20250502&extension=cdf"), json.dumps({}), is_pattern=True, priority=2, @@ -179,10 +179,12 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/result.cdf").exists() + assert Path("output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf").exists() with ( - open("output/result.cdf", "rb") as output, + open( + "output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf", "rb" + ) as output, open(cdf_file, "rb") as input, ): assert output.read() == input.read() From de14a6f22331087a32e620f99a1b562330f23cf6 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 14:32:32 +0000 Subject: [PATCH 014/102] fix: attempt to fix issue with database initialization --- src/imap_mag/DB.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 12e2aba..8e3038c 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -45,9 +45,15 @@ def insert_files(self, files: list[File]) -> None: class DatabaseOutputManager(IOutputManager): - def __init__(self, output_manager: IOutputManager, db: DB = DB()): + def __init__(self, output_manager: IOutputManager, db: DB | None = None): + """Initialize database and output manager.""" + self.output_manager = output_manager - self.db = db + + if db is None: + self.db = DB() + else: + self.db = db def add_file( self, original_file: Path, metadata_provider: IMetadataProvider From fdb5882a787805bfa8f34c07f2a75cc680445633 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Thu, 8 Aug 2024 10:43:00 +0000 Subject: [PATCH 015/102] build: add `pytest-mock` to dev dependencies --- poetry.lock | 19 ++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index fbf6cdb..bc3f4c2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1228,6 +1228,23 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1853,4 +1870,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "40096a30f5c8ed7a5daaaaa5da50d0872d3ae7ab77e08256087fb8e44fb8a779" +content-hash = "ddf04336d74e92f046fa216cb8c28f649f928da0dece3d9a193b7b93e9880631" diff --git a/pyproject.toml b/pyproject.toml index 2b0edfe..742627b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ single-version = "^1.6.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" pytest-cov = "^5.0.0" +pytest-mock = "^3.14.0" pyinstaller = "^6.5.0" pre-commit = "^3.8.0" ruff = "^0.5.4" From 8cbe6beff77f704c551b70ab961397a493dcbfe3 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Thu, 8 Aug 2024 16:40:29 +0000 Subject: [PATCH 016/102] test: add tests for `FetchBinary` class --- src/imap_mag/cli/fetchBinary.py | 27 +++++---- tests/test_fetchBinary.py | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 tests/test_fetchBinary.py diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py index a450180..ebf55f3 100644 --- a/src/imap_mag/cli/fetchBinary.py +++ b/src/imap_mag/cli/fetchBinary.py @@ -58,21 +58,22 @@ def download_binaries( ] for d in range(len(dates) - 1): - file = self.__web_poda.download( + file: Path = self.__web_poda.download( packet=options["packet"], start_date=dates[d], end_date=dates[d + 1] ) - if self.__output_manager is not None: - self.__output_manager.add_default_file( - file, - descriptor=options["packet"] - .lower() - .strip("mag_") - .replace("_", "-"), - date=dates[d], - extension="pkts", - ) - - downloaded += [file] + if file.stat().st_size > 0: + if self.__output_manager is not None: + self.__output_manager.add_default_file( + file, + descriptor=options["packet"] + .lower() + .strip("mag_") + .replace("_", "-"), + date=dates[d], + extension="pkts", + ) + + downloaded += [file] return downloaded diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py new file mode 100644 index 0000000..2efad0f --- /dev/null +++ b/tests/test_fetchBinary.py @@ -0,0 +1,101 @@ +"""Tests for `OutputManager` class.""" + +import os +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchBinary import FetchBinary +from imap_mag.client.webPODA import IWebPODA +from imap_mag.outputManager import IOutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def create_file(file_path: Path, content: str | None) -> Path: + """Create a file with the given content.""" + + file_path.parent.mkdir(parents=True, exist_ok=True) + + if file_path.exists(): + os.remove(file_path) + + with open(file_path, "w") as file: + if content is not None: + file.write(content) + + return file_path + + +@pytest.fixture +def mock_poda() -> mock.Mock: + """Fixture for a mock IWebPODA instance.""" + return mock.create_autospec(IWebPODA, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_binary_empty_download_not_added_to_output( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_file(test_file, None) + + # Exercise. + actual_downloaded: Path = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + assert not mock_output_manager.add_default_file.called + assert actual_downloaded == [] + + +def test_fetch_binary_with_same_start_end_date( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") + + # Exercise. + actual_downloaded: Path = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + descriptor="hsk-pw", + date=datetime(2025, 5, 2), + extension="pkts", + ) + + assert actual_downloaded == [test_file] From 527bcc686595de1292142a3b50a911cd9b970930 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 09:33:14 +0000 Subject: [PATCH 017/102] test: add unit tests for `FetchScience` --- src/imap_mag/cli/fetchScience.py | 7 +- tests/test_fetchBinary.py | 4 +- tests/test_fetchScience.py | 106 +++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/test_fetchScience.py diff --git a/src/imap_mag/cli/fetchScience.py b/src/imap_mag/cli/fetchScience.py index 6da8d93..9c4ad93 100644 --- a/src/imap_mag/cli/fetchScience.py +++ b/src/imap_mag/cli/fetchScience.py @@ -28,7 +28,6 @@ class FetchScienceOptions(typing.TypedDict): level: str start_date: datetime end_date: datetime - output_dir: str class FetchScience: @@ -44,8 +43,8 @@ def __init__( self, data_access: ISDCDataAccess, output_manager: IOutputManager | None = None, - modes: list[MAGMode] = ["norm", "burst"], - sensors: list[MAGSensor] = ["magi", "mago"], + modes: list[MAGMode] = [MAGMode.Normal, MAGMode.Burst], + sensors: list[MAGSensor] = [MAGSensor.IBS, MAGSensor.OBS], ) -> None: """Initialize SDC interface.""" @@ -73,7 +72,7 @@ def download_latest_science( for sensor in self.__sensor: file_details = self.__data_access.get_filename( level=options["level"], - descriptor=str(mode) + "-" + str(sensor), + descriptor=mode.value + "-" + sensor.value, start_date=date, end_date=date, version="latest", diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index 2efad0f..b68ae4f 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -51,7 +51,7 @@ def test_fetch_binary_empty_download_not_added_to_output( mock_poda.download.side_effect = lambda **_: create_file(test_file, None) # Exercise. - actual_downloaded: Path = fetchBinary.download_binaries( + actual_downloaded: list[Path] = fetchBinary.download_binaries( packet="MAG_HSK_PW", start_date=datetime(2025, 5, 2), end_date=datetime(2025, 5, 2), @@ -78,7 +78,7 @@ def test_fetch_binary_with_same_start_end_date( mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") # Exercise. - actual_downloaded: Path = fetchBinary.download_binaries( + actual_downloaded: list[Path] = fetchBinary.download_binaries( packet="MAG_HSK_PW", start_date=datetime(2025, 5, 2), end_date=datetime(2025, 5, 2), diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py new file mode 100644 index 0000000..d6177d6 --- /dev/null +++ b/tests/test_fetchScience.py @@ -0,0 +1,106 @@ +"""Tests for `OutputManager` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchScience import FetchScience, MAGMode, MAGSensor +from imap_mag.client.sdcDataAccess import ISDCDataAccess +from imap_mag.outputManager import IOutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_soc() -> mock.Mock: + """Fixture for a mock ISDCDataAccess instance.""" + return mock.create_autospec(ISDCDataAccess, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_science_no_matching_files( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + mock_soc.get_filename.side_effect = lambda **_: {} + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + + assert not mock_soc.download.called + assert not mock_output_manager.add_default_file.called + assert actual_downloaded == [] + + +def test_fetch_science_with_same_start_end_date( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + test_file = Path(tempfile.gettempdir()) / "test_file" + + mock_soc.get_filename.side_effect = lambda **_: [ + { + "file_path": test_file.absolute(), + "descriptor": "norm-mago", + } + ] + mock_soc.download.side_effect = lambda file_path: file_path + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + mock_soc.download.assert_called_once_with( + test_file.absolute(), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + level="l1b", + descriptor="norm-mago", + date=datetime(2025, 5, 2), + extension="cdf", + ) + + assert actual_downloaded == [test_file] From bc374d3025f410fd31481e52db89cb42d8318b5d Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 09:40:14 +0000 Subject: [PATCH 018/102] chore: update lock file --- poetry.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index bc3f4c2..defbf7e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1407,29 +1407,29 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.5.6" +version = "0.5.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.6-py3-none-linux_armv6l.whl", hash = "sha256:a0ef5930799a05522985b9cec8290b185952f3fcd86c1772c3bdbd732667fdcd"}, - {file = "ruff-0.5.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b652dc14f6ef5d1552821e006f747802cc32d98d5509349e168f6bf0ee9f8f42"}, - {file = "ruff-0.5.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:80521b88d26a45e871f31e4b88938fd87db7011bb961d8afd2664982dfc3641a"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9bc8f328a9f1309ae80e4d392836e7dbc77303b38ed4a7112699e63d3b066ab"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d394940f61f7720ad371ddedf14722ee1d6250fd8d020f5ea5a86e7be217daf"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111a99cdb02f69ddb2571e2756e017a1496c2c3a2aeefe7b988ddab38b416d36"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e395daba77a79f6dc0d07311f94cc0560375ca20c06f354c7c99af3bf4560c5d"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c476acb43c3c51e3c614a2e878ee1589655fa02dab19fe2db0423a06d6a5b1b6"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2ff8003f5252fd68425fd53d27c1f08b201d7ed714bb31a55c9ac1d4c13e2eb"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c94e084ba3eaa80c2172918c2ca2eb2230c3f15925f4ed8b6297260c6ef179ad"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1f77c1c3aa0669fb230b06fb24ffa3e879391a3ba3f15e3d633a752da5a3e670"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f908148c93c02873210a52cad75a6eda856b2cbb72250370ce3afef6fb99b1ed"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:563a7ae61ad284187d3071d9041c08019975693ff655438d8d4be26e492760bd"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:94fe60869bfbf0521e04fd62b74cbca21cbc5beb67cbb75ab33fe8c174f54414"}, - {file = "ruff-0.5.6-py3-none-win32.whl", hash = "sha256:e6a584c1de6f8591c2570e171cc7ce482bb983d49c70ddf014393cd39e9dfaed"}, - {file = "ruff-0.5.6-py3-none-win_amd64.whl", hash = "sha256:d7fe7dccb1a89dc66785d7aa0ac283b2269712d8ed19c63af908fdccca5ccc1a"}, - {file = "ruff-0.5.6-py3-none-win_arm64.whl", hash = "sha256:57c6c0dd997b31b536bff49b9eee5ed3194d60605a4427f735eeb1f9c1b8d264"}, - {file = "ruff-0.5.6.tar.gz", hash = "sha256:07c9e3c2a8e1fe377dd460371c3462671a728c981c3205a5217291422209f642"}, + {file = "ruff-0.5.7-py3-none-linux_armv6l.whl", hash = "sha256:548992d342fc404ee2e15a242cdbea4f8e39a52f2e7752d0e4cbe88d2d2f416a"}, + {file = "ruff-0.5.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00cc8872331055ee017c4f1071a8a31ca0809ccc0657da1d154a1d2abac5c0be"}, + {file = "ruff-0.5.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eaf3d86a1fdac1aec8a3417a63587d93f906c678bb9ed0b796da7b59c1114a1e"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a01c34400097b06cf8a6e61b35d6d456d5bd1ae6961542de18ec81eaf33b4cb8"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcc8054f1a717e2213500edaddcf1dbb0abad40d98e1bd9d0ad364f75c763eea"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f70284e73f36558ef51602254451e50dd6cc479f8b6f8413a95fcb5db4a55fc"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a78ad870ae3c460394fc95437d43deb5c04b5c29297815a2a1de028903f19692"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ccd078c66a8e419475174bfe60a69adb36ce04f8d4e91b006f1329d5cd44bcf"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e31c9bad4ebf8fdb77b59cae75814440731060a09a0e0077d559a556453acbb"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d796327eed8e168164346b769dd9a27a70e0298d667b4ecee6877ce8095ec8e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a09ea2c3f7778cc635e7f6edf57d566a8ee8f485f3c4454db7771efb692c499"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a36d8dcf55b3a3bc353270d544fb170d75d2dff41eba5df57b4e0b67a95bb64e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9369c218f789eefbd1b8d82a8cf25017b523ac47d96b2f531eba73770971c9e5"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b88ca3db7eb377eb24fb7c82840546fb7acef75af4a74bd36e9ceb37a890257e"}, + {file = "ruff-0.5.7-py3-none-win32.whl", hash = "sha256:33d61fc0e902198a3e55719f4be6b375b28f860b09c281e4bdbf783c0566576a"}, + {file = "ruff-0.5.7-py3-none-win_amd64.whl", hash = "sha256:083bbcbe6fadb93cd86709037acc510f86eed5a314203079df174c40bbbca6b3"}, + {file = "ruff-0.5.7-py3-none-win_arm64.whl", hash = "sha256:2dca26154ff9571995107221d0aeaad0e75a77b5a682d6236cf89a58c70b76f4"}, + {file = "ruff-0.5.7.tar.gz", hash = "sha256:8dfc0a458797f5d9fb622dd0efc52d796f23f0a1493a9527f4e49a550ae9a7e5"}, ] [[package]] From c7c274f9615760918227e50e5f613258a6603112 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 13:39:27 +0000 Subject: [PATCH 019/102] task: add `hash` to database and make `path` a unique constraint --- ...1c45c37_added_version_hash_date_and_software_.py} | 12 ++++++++---- src/imap_db/model.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) rename src/imap_db/migrations/versions/{2024_08_07-77642de1867a_added_version_date_and_software_version_.py => 2024_08_09-669111c45c37_added_version_hash_date_and_software_.py} (68%) diff --git a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py similarity index 68% rename from src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py rename to src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py index ccb34bd..8cdc34d 100644 --- a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py +++ b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py @@ -1,8 +1,8 @@ -"""Added version, date and software version columns +"""Added version, hash, date and software version columns -Revision ID: 77642de1867a +Revision ID: 669111c45c37 Revises: d0457f3e98c8 -Create Date: 2024-08-07 13:03:30.889623 +Create Date: 2024-08-09 13:35:21.578940 """ @@ -10,7 +10,7 @@ from alembic import op # revision identifiers, used by Alembic. -revision = "77642de1867a" +revision = "669111c45c37" down_revision = "d0457f3e98c8" branch_labels = None depends_on = None @@ -19,16 +19,20 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("files", sa.Column("version", sa.Integer(), nullable=False)) + op.add_column("files", sa.Column("hash", sa.String(length=64), nullable=False)) op.add_column("files", sa.Column("date", sa.DateTime(), nullable=False)) op.add_column( "files", sa.Column("software_version", sa.String(length=16), nullable=False) ) + op.create_unique_constraint(None, "files", ["path"]) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, "files", type_="unique") op.drop_column("files", "software_version") op.drop_column("files", "date") + op.drop_column("files", "hash") op.drop_column("files", "version") # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py index c2be860..644f9a3 100644 --- a/src/imap_db/model.py +++ b/src/imap_db/model.py @@ -10,10 +10,12 @@ class Base(DeclarativeBase): class File(Base): __tablename__ = "files" + id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(128)) - path: Mapped[str] = mapped_column(String(256)) + path: Mapped[str] = mapped_column(String(256), unique=True) version: Mapped[int] = mapped_column(Integer()) + hash: Mapped[str] = mapped_column(String(64)) date: Mapped[datetime] = mapped_column(DateTime()) software_version: Mapped[str] = mapped_column(String(16)) From e95ac5f65bb62156f42b3b588e48c5342ca059e1 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 13:40:05 +0000 Subject: [PATCH 020/102] feat: update database handler to check for error cases --- src/imap_mag/DB.py | 82 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 17 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 8e3038c..fcb9bde 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,6 +1,10 @@ +import abc +import hashlib +import logging import os from pathlib import Path +import typer from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -9,17 +13,34 @@ from src.imap_mag.outputManager import IMetadataProvider, IOutputManager -class DB: +class IDatabase(abc.ABC): + """Interface for database manager.""" + + def insert_file(self, file: File) -> None: + """Insert a file into the database.""" + self.insert_files([file]) + pass + + @abc.abstractmethod + def insert_files(self, files: list[File]) -> None: + """Insert a list of files into the database.""" + pass + + +class Database(IDatabase): + """Database manager.""" + def __init__(self, db_url=None): env_url = os.getenv("SQLALCHEMY_URL") if db_url is None and env_url is not None: db_url = env_url + # TODO: Check database is available + self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - def insert_file(self, file: File) -> None: - self.insert_files([file]) + logging.debug(f"Creating database with URL: {db_url}") def insert_files(self, files: list[File]) -> None: session = self.Session() @@ -45,31 +66,58 @@ def insert_files(self, files: list[File]) -> None: class DatabaseOutputManager(IOutputManager): - def __init__(self, output_manager: IOutputManager, db: DB | None = None): + """Decorator for adding files to database as well as output.""" + + __output_manager: IOutputManager + __database: IDatabase + + def __init__( + self, output_manager: IOutputManager, database: Database | None = None + ): """Initialize database and output manager.""" - self.output_manager = output_manager + self.__output_manager = output_manager - if db is None: - self.db = DB() + if database is None: + self.__database = Database() else: - self.db = db + self.__database = database def add_file( self, original_file: Path, metadata_provider: IMetadataProvider ) -> tuple[Path, IMetadataProvider]: - (destination_file, metadata_provider) = self.output_manager.add_file( + (destination_file, metadata_provider) = self.__output_manager.add_file( original_file, metadata_provider ) - self.db.insert_file( - File( - name=destination_file.name, - path=destination_file.absolute().as_posix(), - version=metadata_provider.version, - date=metadata_provider.date, - software_version=__version__, + file_hash: str = hashlib.md5(original_file.read_bytes()).hexdigest() + + if not ( + destination_file.exists() + and (hashlib.md5(destination_file.read_bytes()).hexdigest() == file_hash) + ): + logging.error( + f"File {destination_file} does not exist or is not the same as original {original_file}." ) - ) + destination_file.unlink(missing_ok=True) + typer.Abort() + + logging.info(f"Inserting {destination_file} into database.") + + try: + self.__database.insert_file( + File( + name=destination_file.name, + path=destination_file.absolute().as_posix(), + version=metadata_provider.version, + hash=file_hash, + date=metadata_provider.date, + software_version=__version__, + ) + ) + except Exception as e: + logging.error(f"Error inserting {destination_file} into database: {e}") + destination_file.unlink() + raise e return (destination_file, metadata_provider) From 87dce326e4e7fec035feb4bfe7467621cd9f8c2b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 14:58:22 +0000 Subject: [PATCH 021/102] fix: remove logging of URL --- src/imap_mag/DB.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index fcb9bde..37db166 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -40,8 +40,6 @@ def __init__(self, db_url=None): self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - logging.debug(f"Creating database with URL: {db_url}") - def insert_files(self, files: list[File]) -> None: session = self.Session() try: From 050d8b41477e4ec84303cd2abce61beb6ab19fb8 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 16:30:25 +0000 Subject: [PATCH 022/102] test: move `create_test_file` to utility and use `assert_not_called` for mocks --- tests/testUtils.py | 14 ++++++++++++++ tests/test_fetchBinary.py | 25 +++++-------------------- tests/test_fetchScience.py | 5 +++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/testUtils.py b/tests/testUtils.py index fffca72..b7f0234 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -62,3 +62,17 @@ def create_serialize_config( yaml.dump(config.model_dump(by_alias=True), f) return (config, config_file) + + +def create_test_file(file_path: Path, content: str | None) -> Path: + """Create a file with the given content.""" + + file_path.unlink(missing_ok=True) + file_path.parent.mkdir(parents=True, exist_ok=True) + + file_path.touch() + + if content is not None: + file_path.write_text(content) + + return file_path diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index b68ae4f..cda2146 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -1,6 +1,5 @@ """Tests for `OutputManager` class.""" -import os import tempfile from datetime import datetime from pathlib import Path @@ -11,22 +10,7 @@ from imap_mag.client.webPODA import IWebPODA from imap_mag.outputManager import IOutputManager -from .testUtils import enableLogging, tidyDataFolders # noqa: F401 - - -def create_file(file_path: Path, content: str | None) -> Path: - """Create a file with the given content.""" - - file_path.parent.mkdir(parents=True, exist_ok=True) - - if file_path.exists(): - os.remove(file_path) - - with open(file_path, "w") as file: - if content is not None: - file.write(content) - - return file_path +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 @pytest.fixture @@ -48,7 +32,7 @@ def test_fetch_binary_empty_download_not_added_to_output( fetchBinary = FetchBinary(mock_poda, mock_output_manager) test_file = Path(tempfile.gettempdir()) / "test_file" - mock_poda.download.side_effect = lambda **_: create_file(test_file, None) + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, None) # Exercise. actual_downloaded: list[Path] = fetchBinary.download_binaries( @@ -64,7 +48,8 @@ def test_fetch_binary_empty_download_not_added_to_output( end_date=datetime(2025, 5, 3), ) - assert not mock_output_manager.add_default_file.called + mock_output_manager.add_default_file.assert_not_called() + assert actual_downloaded == [] @@ -75,7 +60,7 @@ def test_fetch_binary_with_same_start_end_date( fetchBinary = FetchBinary(mock_poda, mock_output_manager) test_file = Path(tempfile.gettempdir()) / "test_file" - mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, "content") # Exercise. actual_downloaded: list[Path] = fetchBinary.download_binaries( diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py index d6177d6..a9f7a52 100644 --- a/tests/test_fetchScience.py +++ b/tests/test_fetchScience.py @@ -52,8 +52,9 @@ def test_fetch_science_no_matching_files( extension="cdf", ) - assert not mock_soc.download.called - assert not mock_output_manager.add_default_file.called + mock_soc.download.assert_not_called() + mock_output_manager.add_default_file.assert_not_called() + assert actual_downloaded == [] From fa82965809cf6267fde53cd9566183de8d0cbc1f Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 16:30:52 +0000 Subject: [PATCH 023/102] test: add some tests for database output manager --- src/imap_mag/DB.py | 2 +- tests/test_database.py | 118 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/test_database.py diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 37db166..c2f60d9 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -98,7 +98,7 @@ def add_file( f"File {destination_file} does not exist or is not the same as original {original_file}." ) destination_file.unlink(missing_ok=True) - typer.Abort() + raise typer.Abort() logging.info(f"Inserting {destination_file} into database.") diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..d4e967e --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,118 @@ +"""Tests for `OutputManager` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +import typer +from imap_mag.DB import DatabaseOutputManager, IDatabase +from imap_mag.outputManager import DefaultMetadataProvider, IOutputManager + +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +@pytest.fixture +def mock_database() -> mock.Mock: + """Fixture for a mock IDatabase instance.""" + return mock.create_autospec(IDatabase, spec_set=True) + + +def test_database_output_manager_writes_to_database( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + # Exercise. + (actual_file, actual_metadata_provider) = database_manager.add_file( + original_file, metadata_provider + ) + + # Verify. + mock_output_manager.add_file.assert_called_once_with( + original_file, metadata_provider + ) + # mock_database.insert_file.assert_called_once_with( + # File( + # name="test_file.txt", + # path=test_file.absolute().as_posix(), + # version=1, + # hash=hashlib.md5(b"some content").hexdigest(), + # date=datetime(2025, 5, 2), + # software_version=__version__, + # ) + # ) + + assert actual_file == test_file + assert actual_metadata_provider == metadata_provider + + +def test_database_output_manager_errors_destination_file_not_found( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + test_file.unlink(missing_ok=True) + + mock_output_manager.add_file.side_effect = lambda *_: ( + test_file, + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_destination_file_different_hash( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some other content"), + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) From 84770f3b93a23b38c28a431467cbcaa7528da971 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 08:58:22 +0000 Subject: [PATCH 024/102] test: add coverage for database error --- tests/test_database.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_database.py b/tests/test_database.py index d4e967e..884fcf8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -116,3 +116,29 @@ def test_database_output_manager_errors_destination_file_different_hash( # Exercise and verify. with pytest.raises(typer.Abort): database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_database_error( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + mock_database.insert_file.side_effect = ArithmeticError("Database error") + + # Exercise and verify. + with pytest.raises(ArithmeticError): + database_manager.add_file(original_file, metadata_provider) From 739765ba964026ddc6a9eaf3bfa554ba97c46848 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:05:01 +0000 Subject: [PATCH 025/102] build: change `imap-mag` version to initial version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 74d6d18..84e17e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] requires-python = ">=3.10" name = "imap-mag" -version = "0.0.1" +version = "0.1.0" [tool.poetry] name = "imap-mag" From 94831b2cf81ebf9ddfe5d0d3a83c6f2758daa2e1 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:13:41 +0000 Subject: [PATCH 026/102] test(fix): remove commented out text --- tests/test_database.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 884fcf8..3d2fae8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,6 @@ """Tests for `OutputManager` class.""" +import hashlib import tempfile from datetime import datetime from pathlib import Path @@ -7,6 +8,8 @@ import pytest import typer +from imap_db.model import File +from imap_mag import __version__ from imap_mag.DB import DatabaseOutputManager, IDatabase from imap_mag.outputManager import DefaultMetadataProvider, IOutputManager @@ -44,6 +47,17 @@ def test_database_output_manager_writes_to_database( metadata_provider, ) + def check_inserted_file(file: File): + # Two instances of `File` will never be equal, so we check the attributes. + assert file.name == "test_file.txt" + assert file.path == test_file.absolute().as_posix() + assert file.version == 1 + assert file.hash == hashlib.md5(b"some content").hexdigest() + assert file.date == datetime(2025, 5, 2) + assert file.software_version == __version__ + + mock_database.insert_file.side_effect = lambda file: check_inserted_file(file) + # Exercise. (actual_file, actual_metadata_provider) = database_manager.add_file( original_file, metadata_provider @@ -53,16 +67,6 @@ def test_database_output_manager_writes_to_database( mock_output_manager.add_file.assert_called_once_with( original_file, metadata_provider ) - # mock_database.insert_file.assert_called_once_with( - # File( - # name="test_file.txt", - # path=test_file.absolute().as_posix(), - # version=1, - # hash=hashlib.md5(b"some content").hexdigest(), - # date=datetime(2025, 5, 2), - # software_version=__version__, - # ) - # ) assert actual_file == test_file assert actual_metadata_provider == metadata_provider From bddeea88b8f405ad56ad0be99054ffa616551c36 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:18:14 +0000 Subject: [PATCH 027/102] fix: temporarely use custom version of imap-data-access to avoid issue with `version=latest` argument --- poetry.lock | 13 +++++++++---- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 880e69e..ceb61c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,14 +603,19 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [ - {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, -] +files = [] +develop = false [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] +[package.source] +type = "git" +url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" +reference = "HEAD" +resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" + [[package]] name = "importlib-resources" version = "5.13.0" @@ -1934,4 +1939,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "05034a7ae56f6ac95b6728ffa0c1af41cd021338cb3171e71ef0c67d376b6c19" +content-hash = "2646920291fe7954ce23f3b4b16e68b0a0b96daed4600ac03766e248d5480dcc" diff --git a/pyproject.toml b/pyproject.toml index 84e17e4..a46c768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.8.0" +imap-data-access = { git = "https://github.com/ImperialCollegeLondon/imap-data-access.git" } # TODO: replace with imap-data-access > 0.8.0 cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} From 3239efb06b89b76363b9ef7315f680c36bc18fbf Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:33:33 +0000 Subject: [PATCH 028/102] Revert "fix: temporarely use custom version of imap-data-access to avoid issue with `version=latest` argument" This reverts commit bddeea88b8f405ad56ad0be99054ffa616551c36. --- poetry.lock | 13 ++++--------- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index ceb61c9..880e69e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,19 +603,14 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [] -develop = false +files = [ + {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, +] [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] -[package.source] -type = "git" -url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" -reference = "HEAD" -resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" - [[package]] name = "importlib-resources" version = "5.13.0" @@ -1939,4 +1934,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "2646920291fe7954ce23f3b4b16e68b0a0b96daed4600ac03766e248d5480dcc" +content-hash = "05034a7ae56f6ac95b6728ffa0c1af41cd021338cb3171e71ef0c67d376b6c19" diff --git a/pyproject.toml b/pyproject.toml index a46c768..84e17e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = { git = "https://github.com/ImperialCollegeLondon/imap-data-access.git" } # TODO: replace with imap-data-access > 0.8.0 +imap-data-access = "^0.8.0" cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} From 164b61b4332cc2fe0a20894cb36295f93f24509b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:39:54 +0000 Subject: [PATCH 029/102] build: add coverage report to CI action --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 811e3a2..c77ad82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,13 @@ jobs: path: 'test-results.xml' reporter: java-junit + - name: Coverage Report + uses: 5monkeys/cobertura-action@master + with: + report_name: Coverage Report (${{ matrix.python-versions }}) + path: "coverage.xml" + minimum_coverage: 80 + - name: Create Release ${{github.ref_name}} & upload artifacts uses: softprops/action-gh-release@v2 if: ${{ startsWith(github.ref, 'refs/tags/') }} From 7c9ebd48bfc7e54c731eaf8352cf15b109740c4b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:40:56 +0000 Subject: [PATCH 030/102] test: skip failing test due to bug in `imap-data-access` --- tests/test_main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 05e7dac..92ef4ec 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,6 +7,7 @@ import re from pathlib import Path +import pytest from imap_mag.main import app from typer.testing import CliRunner @@ -115,6 +116,7 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() +@pytest.mark.skip(reason="Waiting for `imap-data-access` version > 0.8.0.") def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. query_response: list[dict[str, str]] = [ From 9145268c9d6b8ed217a58c21aaf00ee58e00717a Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:53:14 +0000 Subject: [PATCH 031/102] build(fix): use version instead of branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c77ad82..7f5faac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: reporter: java-junit - name: Coverage Report - uses: 5monkeys/cobertura-action@master + uses: 5monkeys/cobertura-action@v14 with: report_name: Coverage Report (${{ matrix.python-versions }}) path: "coverage.xml" From 67b1726da1c53ee5cf217503e8ecd1105fed291b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:29:44 +0000 Subject: [PATCH 032/102] test(fix): name of classes under test in comment --- tests/test_database.py | 2 +- tests/test_fetchBinary.py | 2 +- tests/test_fetchScience.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 3d2fae8..56d9ad3 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for database classes.""" import hashlib import tempfile diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index cda2146..5c149a7 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for `FetchBinary` class.""" import tempfile from datetime import datetime diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py index a9f7a52..65d91d8 100644 --- a/tests/test_fetchScience.py +++ b/tests/test_fetchScience.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for `FetchScience` class.""" import tempfile from datetime import datetime From 9540787e93c034459fe425ebe0a4ad1864482e40 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:54:23 +0000 Subject: [PATCH 033/102] test: add more tests to increase coverage --- src/imap_mag/client/sdcDataAccess.py | 16 +++++------ src/imap_mag/main.py | 2 +- tests/test_appUtils.py | 17 +++++++++++ tests/test_sdcDataAccess.py | 43 ++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 tests/test_appUtils.py create mode 100644 tests/test_sdcDataAccess.py diff --git a/src/imap_mag/client/sdcDataAccess.py b/src/imap_mag/client/sdcDataAccess.py index 88d277d..cafd3d3 100644 --- a/src/imap_mag/client/sdcDataAccess.py +++ b/src/imap_mag/client/sdcDataAccess.py @@ -2,9 +2,9 @@ import abc import logging -import pathlib import typing from datetime import datetime +from pathlib import Path import imap_data_access import typing_extensions @@ -44,7 +44,7 @@ class ISDCDataAccess(abc.ABC): @abc.abstractmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: """Get file path for data from imap-data-access.""" pass @@ -68,7 +68,7 @@ def get_filename( pass @abc.abstractmethod - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: """Download data from imap-data-access.""" pass @@ -76,10 +76,10 @@ def download(self, file_name: str) -> pathlib.Path: class SDCDataAccess(ISDCDataAccess): """Class for uploading and downloading MAG data via imap-data-access.""" - def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: + def __init__(self, data_dir: Path, sdc_url: str | None = None) -> None: """Initialize SDC API client.""" - imap_data_access.config["DATA_DIR"] = pathlib.Path(data_dir) + imap_data_access.config["DATA_DIR"] = data_dir imap_data_access.config["DATA_ACCESS_URL"] = ( sdc_url or "https://api.dev.imap-mission.com" ) @@ -87,7 +87,7 @@ def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: @staticmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: science_file = imap_data_access.ScienceFilePath.generate_from_inputs( instrument="mag", data_level=options["level"], @@ -136,6 +136,6 @@ def get_filename( return file_details - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: logging.debug(f"Downloading {file_name} from imap-data-access.") - return pathlib.Path(imap_data_access.download(file_name)) + return imap_data_access.download(file_name) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index 1fa375d..4dfb7a2 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -247,7 +247,7 @@ def fetch_science( logging.info(f"Downloading {level} science from {start_date} to {end_date}.") data_access = SDCDataAccess( - data_dir=str(configFile.work_folder), + data_dir=configFile.work_folder, sdc_url=configFile.api.sdc_url if configFile.api else None, ) output_manager = appUtils.getOutputManager(configFile.destination) diff --git a/tests/test_appUtils.py b/tests/test_appUtils.py new file mode 100644 index 0000000..bf0cd68 --- /dev/null +++ b/tests/test_appUtils.py @@ -0,0 +1,17 @@ +"""Tests for app utilities.""" + +import pytest +import typer +from imap_mag.appUtils import getPacketFromApID + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def test_get_packet_from_apid_errors_on_invalid_apid() -> None: + with pytest.raises(typer.Abort): + getPacketFromApID(12345) + + +def test_convert_to_datetime_on_invalid_datetime() -> None: + with pytest.raises(typer.Abort): + getPacketFromApID("ABCDEF") diff --git a/tests/test_sdcDataAccess.py b/tests/test_sdcDataAccess.py new file mode 100644 index 0000000..fe30835 --- /dev/null +++ b/tests/test_sdcDataAccess.py @@ -0,0 +1,43 @@ +"""Tests for `SDCDataAccess` class.""" + +import os +from datetime import datetime +from pathlib import Path + +import imap_data_access +from imap_mag.client.sdcDataAccess import SDCDataAccess + +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 +from .wiremockUtils import wiremock_manager # noqa: F401 + + +def test_sdc_data_access_constructor_sets_config() -> None: + # Set up. + data_dir = "some_test_folder" + data_access_url = "https://some_test_url" + + # Exercise. + _ = SDCDataAccess(data_dir, data_access_url) + + # Verify. + assert imap_data_access.config["DATA_DIR"] == data_dir + assert imap_data_access.config["DATA_ACCESS_URL"] == data_access_url + + +def test_get_file_path_builds_file_path() -> None: + # Set up. + data_access = SDCDataAccess("some_test_folder") + + # Exercise. + (file_name, file_path) = data_access.get_file_path( + level="l1b", + descriptor="norm-magi", + start_date=datetime(2025, 5, 2), + version="v002", + ) + + # Verify. + assert file_name == Path("imap_mag_l1b_norm-magi_20250502_v002.cdf") + assert file_path == Path( + os.path.join("some_test_folder", "imap", "mag", "l1b", "2025", "05", file_name) + ) From 39a30a6f005e95fc37ee7dc5e0c70aca6356ca3f Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:57:53 +0000 Subject: [PATCH 034/102] test(fix): actually use `convertToDatetime` --- tests/test_appUtils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_appUtils.py b/tests/test_appUtils.py index bf0cd68..71cf2fc 100644 --- a/tests/test_appUtils.py +++ b/tests/test_appUtils.py @@ -2,7 +2,7 @@ import pytest import typer -from imap_mag.appUtils import getPacketFromApID +from imap_mag.appUtils import convertToDatetime, getPacketFromApID from .testUtils import enableLogging, tidyDataFolders # noqa: F401 @@ -14,4 +14,4 @@ def test_get_packet_from_apid_errors_on_invalid_apid() -> None: def test_convert_to_datetime_on_invalid_datetime() -> None: with pytest.raises(typer.Abort): - getPacketFromApID("ABCDEF") + convertToDatetime("ABCDEF") From 65db0fff36c82622ff19b4e4c489a606f6424f14 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 09:12:04 +0000 Subject: [PATCH 035/102] fix: CI action failure, packaging failure, test failure --- .github/workflows/ci.yml | 4 ++++ pack.sh | 2 +- poetry.lock | 36 ++++++++++++++++++++++-------------- pyproject.toml | 2 +- tests/test_main.py | 2 -- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f5faac..9fe12f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,10 @@ jobs: python-versions: ['3.10', '3.11', '3.12'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: read + checks: write # map step outputs to job outputs so they can be share among jobs outputs: package_version: ${{ env.PACKAGE_VERSION }} diff --git a/pack.sh b/pack.sh index 9d02d82..d290346 100755 --- a/pack.sh +++ b/pack.sh @@ -18,7 +18,7 @@ poetry build # output a requierments.txt file used by docker during the build poetry self add poetry-plugin-export -poetry export --format=requirements.txt > dist/requirements.txt +poetry export --without-hashes --format=requirements.txt > dist/requirements.txt # move the files into a folder with the python version mkdir -p dist/python$PYTHON_VERSION diff --git a/poetry.lock b/poetry.lock index 880e69e..455d3c1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,14 +603,19 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [ - {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, -] +files = [] +develop = false [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] +[package.source] +type = "git" +url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" +reference = "main" +resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" + [[package]] name = "importlib-resources" version = "5.13.0" @@ -1359,13 +1364,13 @@ files = [ [[package]] name = "pywin32-ctypes" -version = "0.2.2" +version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" files = [ - {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, - {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, ] [[package]] @@ -1498,18 +1503,18 @@ files = [ [[package]] name = "setuptools" -version = "72.1.0" +version = "72.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, + {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, ] [package.extras] core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1676,13 +1681,13 @@ url = ["furl (>=0.4.1)"] [[package]] name = "testcontainers" -version = "4.7.2" +version = "4.8.0" description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "testcontainers-4.7.2-py3-none-any.whl", hash = "sha256:23b13cf8078f615a08c75197f227796d90c46df92d2b282ae7c39b1fc1a9c9ed"}, - {file = "testcontainers-4.7.2.tar.gz", hash = "sha256:9976b1cdcdeb9feeae6a477073e7c8b02cd40ea44f1daa34b5da6d2c918dff0d"}, + {file = "testcontainers-4.8.0-py3-none-any.whl", hash = "sha256:0b85d787e5b1f8b32042704d23b6c54787bf6751d2d3cfee2c031349ef2eea30"}, + {file = "testcontainers-4.8.0.tar.gz", hash = "sha256:56153bb5938694844f0e6bd0cf82e19dd6a6516bc29881440e273939201a42d5"}, ] [package.dependencies] @@ -1693,10 +1698,12 @@ wrapt = "*" [package.extras] arangodb = ["python-arango (>=7.8,<8.0)"] +aws = ["boto3", "httpx"] azurite = ["azure-storage-blob (>=12.19,<13.0)"] chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] cosmosdb = ["azure-cosmos"] +db2 = ["ibm_db_sa", "sqlalchemy"] generic = ["httpx"] google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] influxdb = ["influxdb", "influxdb-client"] @@ -1717,6 +1724,7 @@ qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] +scylla = ["cassandra-driver (==3.29.1)"] selenium = ["selenium"] sftp = ["cryptography"] test-module-import = ["httpx"] @@ -1934,4 +1942,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "05034a7ae56f6ac95b6728ffa0c1af41cd021338cb3171e71ef0c67d376b6c19" +content-hash = "10329b21c3e5c533981a46614c44553a7d5539133745209806cbd2fca5899149" diff --git a/pyproject.toml b/pyproject.toml index 84e17e4..075bf67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.8.0" +imap-data-access = {git = "https://github.com/ImperialCollegeLondon/imap-data-access.git", rev = "main"} cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} diff --git a/tests/test_main.py b/tests/test_main.py index 92ef4ec..05e7dac 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,7 +7,6 @@ import re from pathlib import Path -import pytest from imap_mag.main import app from typer.testing import CliRunner @@ -116,7 +115,6 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() -@pytest.mark.skip(reason="Waiting for `imap-data-access` version > 0.8.0.") def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. query_response: list[dict[str, str]] = [ From afa22f2b97e55952037e7e019a5abc8a4ff01d64 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 09:40:13 +0000 Subject: [PATCH 036/102] fix: more fixes to permissions for action and packaging --- .github/workflows/ci.yml | 4 +--- deploy/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe12f7..c8fdd32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,7 @@ jobs: os: [ubuntu-latest] runs-on: ${{ matrix.os }} permissions: - contents: read - actions: read - checks: write + pull-requests: write # map step outputs to job outputs so they can be share among jobs outputs: package_version: ${{ env.PACKAGE_VERSION }} diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 95e4c1f..0fc51a4 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,7 +11,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # Install the postgres client and any other compile time dependencies needed to build our app -RUN apt-get update && apt-get install -y libpq-dev gcc +RUN apt-get update && apt-get install -y libpq-dev gcc git # Creates a non-root user with an explicit UID and adds permission to access the /app folder # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers From cffe6853f03d5184c30169d51ba7f868266f20e6 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 10:18:54 +0000 Subject: [PATCH 037/102] Squashed commit of the following: commit ee06eb90cca2bd84988c5294ab91167d40bc8568 Author: Michele Facchinelli Date: Fri Aug 16 09:53:09 2024 +0000 fix: add packages permission commit d0bac86adbeca2ef394d24c4356bbee1b68ae40a Author: Michele Facchinelli Date: Fri Aug 16 09:47:17 2024 +0000 fix: add checks permissions commit 80b3b92677b91dbd75e5775882a476402948fd45 Author: Michele Facchinelli Date: Fri Aug 16 09:44:44 2024 +0000 fix: permissions for CI actions --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8fdd32..b9eb7da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,9 @@ jobs: os: [ubuntu-latest] runs-on: ${{ matrix.os }} permissions: + actions: write + checks: write + packages: write pull-requests: write # map step outputs to job outputs so they can be share among jobs outputs: From f342eb1255b89bdeed8cd17f432153b084151a19 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 5 Aug 2024 20:05:24 +0000 Subject: [PATCH 038/102] task: fix src paths and docker multistage builds --- deploy/Dockerfile | 3 +++ run-docker.sh | 64 +++++++++++++++++++++++------------------------ 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 95e4c1f..9cf384d 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -44,6 +44,9 @@ USER appuser # Make sure scripts in .local are usable: ENV PATH="$PATH:/home/appuser/.local/bin" +# DELETE ME +RUN ls -la . + # Now the imap-mag CLI is available on the path and # the 3.12 python package for imap-mag is intalled at /home/appuser/.local/bin diff --git a/run-docker.sh b/run-docker.sh index 04960e7..edda253 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -1,32 +1,32 @@ -#!/bin/bash -set -e - -# Example: -# ./run-docker.sh -# ./run-docker.sh -i // for interactive debugging -# ./run-docker.sh -i -e VARIABLE=some_value // for interactive debugging with VARIABLE env var set - -IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" - -echo "Running $IMAGE_NAME with dev.env file" - -# check if the argument "DEBUG" or "-i" is passed -if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then - echo "Overriding entrypoint to be an interactive bash shell for debugging" - docker run --rm -it \ - --entrypoint /bin/bash \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME -elif [ -z "$1" ]; then # no args passed - docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME -else - echo "Extra arguments: $@" - docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $@ $IMAGE_NAME -fi +#!/bin/bash +set -e + +# Example: +# ./run-docker.sh +# ./run-docker.sh -i // for interactive debugging +# ./run-docker.sh -i -e VARIABLE=some_value // for interactive debugging with VARIABLE env var set + +IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" + +echo "Running $IMAGE_NAME with dev.env file" + +# check if the argument "DEBUG" or "-i" is passed +if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then + echo "Overriding entrypoint to be an interactive bash shell for debugging" + docker run --rm -it \ + --entrypoint /bin/bash \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +elif [ -z "$1" ]; then # no args passed + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +else + echo "Extra arguments: $@" + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $@ $IMAGE_NAME +fi From dc720961d6f58ac19967d2b3d93b58ea458160d5 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Mon, 5 Aug 2024 20:05:55 +0000 Subject: [PATCH 039/102] task: tidy dockerfile --- deploy/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 9cf384d..95e4c1f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -44,9 +44,6 @@ USER appuser # Make sure scripts in .local are usable: ENV PATH="$PATH:/home/appuser/.local/bin" -# DELETE ME -RUN ls -la . - # Now the imap-mag CLI is available on the path and # the 3.12 python package for imap-mag is intalled at /home/appuser/.local/bin From 485781239e4d57391df1ebcb70b304b4d5a23d7f Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Thu, 8 Aug 2024 16:07:30 +0100 Subject: [PATCH 040/102] Initial add of CI step to test on Windows --- .github/workflows/ci.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 811e3a2..c82f620 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,44 @@ jobs: files: | dist/${{ env.PACKAGE_NAME }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }}.zip + test_on_windows: + strategy: + matrix: + python-versions: ['3.10', '3.11', '3.12'] + os: [windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-versions }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install + + - name: Run tests + run: poetry run pytest -s --cov-config=.coveragerc --cov=src --cov-append --cov-report=xml --cov-report term-missing --cov-report=html --junitxml=test-results.xml tests + + - name: Upload Coverage report + uses: actions/upload-artifact@v4 + if: matrix.python-versions == env.PREFERED_PYTHON_VERSION + with: + name: CoverageReport_${{ matrix.os }}_python${{matrix.python-versions}}_${{ env.PACKAGE_VERSION }} + path: htmlcov + if-no-files-found: error + + - name: Test Report + uses: dorny/test-reporter@v1 + if: success() || failure() + with: + name: Test Results (${{ matrix.os }}) (${{ matrix.python-versions }}) + path: 'test-results.xml' + reporter: java-junit + + build_single_file_binary: strategy: matrix: From 43d80836b863826deb77bd4a0cd12751e423a812 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Fri, 9 Aug 2024 15:50:46 +0100 Subject: [PATCH 041/102] install patched wiremock and fix Path usage --- poetry.lock | 68 +++++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- tests/wiremockUtils.py | 4 +-- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8f5dc2b..5b0087f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1536,6 +1536,54 @@ description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0c9045ecc2e4db59bfc97b20516dfdf8e41d910ac6fb667ebd3a79ea54084619"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1467940318e4a860afd546ef61fefb98a14d935cd6817ed07a228c7f7c62f389"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5954463675cb15db8d4b521f3566a017c8789222b8316b1e6934c811018ee08b"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:167e7497035c303ae50651b351c28dc22a40bb98fbdb8468cdc971821b1ae533"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b27dfb676ac02529fb6e343b3a482303f16e6bc3a4d868b73935b8792edb52d0"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bf2360a5e0f7bd75fa80431bf8ebcfb920c9f885e7956c7efde89031695cafb8"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win32.whl", hash = "sha256:306fe44e754a91cd9d600a6b070c1f2fadbb4a1a257b8781ccf33c7067fd3e4d"}, + {file = "SQLAlchemy-2.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:99db65e6f3ab42e06c318f15c98f59a436f1c78179e6a6f40f529c8cc7100b22"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21b053be28a8a414f2ddd401f1be8361e41032d2ef5884b2f31d31cb723e559f"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b178e875a7a25b5938b53b006598ee7645172fccafe1c291a706e93f48499ff5"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723a40ee2cc7ea653645bd4cf024326dea2076673fc9d3d33f20f6c81db83e1d"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295ff8689544f7ee7e819529633d058bd458c1fd7f7e3eebd0f9268ebc56c2a0"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:49496b68cd190a147118af585173ee624114dfb2e0297558c460ad7495f9dfe2"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd9b73c5c15f0ec5ce18128b1fe9157ddd0044abc373e6ecd5ba376a7e5d961"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win32.whl", hash = "sha256:9365a3da32dabd3e69e06b972b1ffb0c89668994c7e8e75ce21d3e5e69ddef28"}, + {file = "SQLAlchemy-2.0.32-cp311-cp311-win_amd64.whl", hash = "sha256:8bd63d051f4f313b102a2af1cbc8b80f061bf78f3d5bd0843ff70b5859e27924"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bab3db192a0c35e3c9d1560eb8332463e29e5507dbd822e29a0a3c48c0a8d92"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:19d98f4f58b13900d8dec4ed09dd09ef292208ee44cc9c2fe01c1f0a2fe440e9"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cd33c61513cb1b7371fd40cf221256456d26a56284e7d19d1f0b9f1eb7dd7e8"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6ba0497c1d066dd004e0f02a92426ca2df20fac08728d03f67f6960271feec"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2b6be53e4fde0065524f1a0a7929b10e9280987b320716c1509478b712a7688c"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:916a798f62f410c0b80b63683c8061f5ebe237b0f4ad778739304253353bc1cb"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win32.whl", hash = "sha256:31983018b74908ebc6c996a16ad3690301a23befb643093fcfe85efd292e384d"}, + {file = "SQLAlchemy-2.0.32-cp312-cp312-win_amd64.whl", hash = "sha256:4363ed245a6231f2e2957cccdda3c776265a75851f4753c60f3004b90e69bfeb"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b8afd5b26570bf41c35c0121801479958b4446751a3971fb9a480c1afd85558e"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c750987fc876813f27b60d619b987b057eb4896b81117f73bb8d9918c14f1cad"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ada0102afff4890f651ed91120c1120065663506b760da4e7823913ebd3258be"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:78c03d0f8a5ab4f3034c0e8482cfcc415a3ec6193491cfa1c643ed707d476f16"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:3bd1cae7519283ff525e64645ebd7a3e0283f3c038f461ecc1c7b040a0c932a1"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win32.whl", hash = "sha256:01438ebcdc566d58c93af0171c74ec28efe6a29184b773e378a385e6215389da"}, + {file = "SQLAlchemy-2.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:4979dc80fbbc9d2ef569e71e0896990bc94df2b9fdbd878290bd129b65ab579c"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c742be912f57586ac43af38b3848f7688863a403dfb220193a882ea60e1ec3a"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:62e23d0ac103bcf1c5555b6c88c114089587bc64d048fef5bbdb58dfd26f96da"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:251f0d1108aab8ea7b9aadbd07fb47fb8e3a5838dde34aa95a3349876b5a1f1d"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef18a84e5116340e38eca3e7f9eeaaef62738891422e7c2a0b80feab165905f"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3eb6a97a1d39976f360b10ff208c73afb6a4de86dd2a6212ddf65c4a6a2347d5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0c1c9b673d21477cec17ab10bc4decb1322843ba35b481585facd88203754fc5"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win32.whl", hash = "sha256:c41a2b9ca80ee555decc605bd3c4520cc6fef9abde8fd66b1cf65126a6922d65"}, + {file = "SQLAlchemy-2.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:8a37e4d265033c897892279e8adf505c8b6b4075f2b40d77afb31f7185cd6ecd"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:52fec964fba2ef46476312a03ec8c425956b05c20220a1a03703537824b5e8e1"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:328429aecaba2aee3d71e11f2477c14eec5990fb6d0e884107935f7fb6001632"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85a01b5599e790e76ac3fe3aa2f26e1feba56270023d6afd5550ed63c68552b3"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaf04784797dcdf4c0aa952c8d234fa01974c4729db55c45732520ce12dd95b4"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4488120becf9b71b3ac718f4138269a6be99a42fe023ec457896ba4f80749525"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:14e09e083a5796d513918a66f3d6aedbc131e39e80875afe81d98a03312889e6"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win32.whl", hash = "sha256:0d322cc9c9b2154ba7e82f7bf25ecc7c36fbe2d82e2933b3642fc095a52cfc78"}, + {file = "SQLAlchemy-2.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:7dd8583df2f98dea28b5cd53a1beac963f4f9d087888d75f22fcc93a07cf8d84"}, + {file = "SQLAlchemy-2.0.32-py3-none-any.whl", hash = "sha256:e567a8793a692451f706b363ccf3c45e056b67d90ead58c3bc9471af5d212202"}, {file = "SQLAlchemy-2.0.32.tar.gz", hash = "sha256:c1b88cc8b02b6a5f0efb0345a03672d4c897dc7d92585176f88c67346f565ea8"}, ] @@ -1737,19 +1785,23 @@ name = "wiremock" version = "2.6.1" description = "Wiremock Admin API Client" optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "wiremock-2.6.1-py3-none-any.whl", hash = "sha256:417a803b0bba3ab6240410aedb4de15a32581fb29b1310b05289b4aa1a7c9ffd"}, - {file = "wiremock-2.6.1.tar.gz", hash = "sha256:89b64d763a68a1808274aa4daf802f7ce3f9bff2a18ac6bf8923c997a21d67c1"}, -] +python-versions = "^3.7 | ^3.8 | ^3.9 | ^3.10 | ^3.11" +files = [] +develop = false [package.dependencies] -importlib-resources = ">=5.12.0,<6.0.0" -requests = ">=2.20.0,<3.0.0" +importlib-resources = "^5.12.0" +requests = "^2.20.0" [package.extras] testing = ["docker (>=6.1.0,<7.0.0)", "testcontainers (>=3.7.1,<4.0.0)"] +[package.source] +type = "git" +url = "git@github.com:ImperialCollegeLondon/python-wiremock.git" +reference = "fix-test-containers-on-windows" +resolved_reference = "d7cb44af3c3cf84e84b219094b3fc97a8ed06c5c" + [[package]] name = "wrapt" version = "1.16.0" @@ -1856,4 +1908,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "00a92fb906cc93d1a11fd65e4e3ed1b04161ed8cbc4fca3c41d518d0b39839f9" +content-hash = "0ad1f8fbdb650d2841254fdfe3a326f4390270ce10bef74584cc24e88778bd7b" diff --git a/pyproject.toml b/pyproject.toml index da156cd..78ea68d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ pandas = "^2.2.2" imap-data-access = "^0.7.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} +wiremock = {git = "git@github.com:ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" @@ -38,7 +39,6 @@ pytest-cov = "^5.0.0" pyinstaller = "^6.5.0" pre-commit = "^3.8.0" ruff = "^0.5.4" -wiremock = "^2.6.1" docker = "^7.1.0" testcontainers = "^4.7.2" diff --git a/tests/wiremockUtils.py b/tests/wiremockUtils.py index 66ceb82..20b8b70 100644 --- a/tests/wiremockUtils.py +++ b/tests/wiremockUtils.py @@ -1,5 +1,5 @@ import typing -from pathlib import Path +from pathlib import Path, PurePosixPath import pytest import typing_extensions @@ -55,7 +55,7 @@ def add_file_mapping( ) -> None: """Copy file to container and add WireMock mapping for it.""" - container_dir_path = Path("/home/wiremock/__files") + container_dir_path = PurePosixPath("/home/wiremock/__files") with open(host_path, "rb") as f: self.__mock_container.copy_files_to_container( From 65ebba6d11c1cbe659e1927e8935c2a7db449354 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Fri, 9 Aug 2024 15:59:34 +0100 Subject: [PATCH 042/102] clone from https rather than ssh --- poetry.lock | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5b0087f..d863f2d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1798,7 +1798,7 @@ testing = ["docker (>=6.1.0,<7.0.0)", "testcontainers (>=3.7.1,<4.0.0)"] [package.source] type = "git" -url = "git@github.com:ImperialCollegeLondon/python-wiremock.git" +url = "https://github.com/ImperialCollegeLondon/python-wiremock.git" reference = "fix-test-containers-on-windows" resolved_reference = "d7cb44af3c3cf84e84b219094b3fc97a8ed06c5c" @@ -1908,4 +1908,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "0ad1f8fbdb650d2841254fdfe3a326f4390270ce10bef74584cc24e88778bd7b" +content-hash = "7452752b3522e4513c287a99b753459abf9523839bb9c142a50837e8fd0e7a7b" diff --git a/pyproject.toml b/pyproject.toml index 78ea68d..11c3de1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ pandas = "^2.2.2" imap-data-access = "^0.7.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} -wiremock = {git = "git@github.com:ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} +wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" From 8dbd7dab2a219ef65ef1885deec024d28a08cda1 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Mon, 12 Aug 2024 16:21:57 +0100 Subject: [PATCH 043/102] fix: add yaml representer for a indowsPath --- tests/testUtils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/testUtils.py b/tests/testUtils.py index c233b09..07261a5 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1,6 +1,5 @@ import os -from pathlib import Path, PosixPath - +from pathlib import Path, PosixPath, WindowsPath import imap_mag.appConfig as appConfig import yaml @@ -35,6 +34,9 @@ def create_serialize_config( yaml.add_representer( PosixPath, lambda dumper, data: dumper.represent_str(str(data)) ) + yaml.add_representer( + WindowsPath, lambda dumper, data: dumper.represent_str(str(data)) + ) yaml.dump(config.model_dump(by_alias=True), f) return (config, config_file) From 455277f96948d1887baa898609f687dc972f30c4 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Mon, 12 Aug 2024 17:38:49 +0100 Subject: [PATCH 044/102] fix: remove hashes from requirements so git package can be installed --- pack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pack.sh b/pack.sh index 9d02d82..d290346 100755 --- a/pack.sh +++ b/pack.sh @@ -18,7 +18,7 @@ poetry build # output a requierments.txt file used by docker during the build poetry self add poetry-plugin-export -poetry export --format=requirements.txt > dist/requirements.txt +poetry export --without-hashes --format=requirements.txt > dist/requirements.txt # move the files into a folder with the python version mkdir -p dist/python$PYTHON_VERSION From 4e27257803c0a09c441077a969139149202b5e6b Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Mon, 12 Aug 2024 17:54:33 +0100 Subject: [PATCH 045/102] install git in dockerfile --- deploy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 95e4c1f..0fc51a4 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,7 +11,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # Install the postgres client and any other compile time dependencies needed to build our app -RUN apt-get update && apt-get install -y libpq-dev gcc +RUN apt-get update && apt-get install -y libpq-dev gcc git # Creates a non-root user with an explicit UID and adds permission to access the /app folder # For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers From 3a40b7653faeaa4c805975e7d4fa46f97623cfb9 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Mon, 12 Aug 2024 17:57:14 +0100 Subject: [PATCH 046/102] Correct formatting --- tests/testUtils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/testUtils.py b/tests/testUtils.py index 07261a5..c8e30a4 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1,5 +1,6 @@ import os from pathlib import Path, PosixPath, WindowsPath + import imap_mag.appConfig as appConfig import yaml From 39df053d247cfe4e9ace1f65df1ba3bab5b0f305 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Mon, 12 Aug 2024 18:05:27 +0100 Subject: [PATCH 047/102] update dependency on patched python wiremock to dev dependency --- poetry.lock | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index d863f2d..764b57a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1908,4 +1908,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "7452752b3522e4513c287a99b753459abf9523839bb9c142a50837e8fd0e7a7b" +content-hash = "b4b811611814a4876734b6cc3e2dc52c3795e22f317c321b8853f172165efa36" diff --git a/pyproject.toml b/pyproject.toml index 11c3de1..91a420f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ pandas = "^2.2.2" imap-data-access = "^0.7.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} -wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" @@ -41,6 +40,7 @@ pre-commit = "^3.8.0" ruff = "^0.5.4" docker = "^7.1.0" testcontainers = "^4.7.2" +wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "d7cb44af3c3cf84e84b219094b3fc97a8ed06c5c"} [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` From df730b891ec2d22381d4ddd431aa431d404b1bc1 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:06:51 +0100 Subject: [PATCH 048/102] Update wiremock to patch it so no path manipulations necessary --- poetry.lock | 4 ++-- pyproject.toml | 2 +- tests/wiremockUtils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 764b57a..b918bc3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1800,7 +1800,7 @@ testing = ["docker (>=6.1.0,<7.0.0)", "testcontainers (>=3.7.1,<4.0.0)"] type = "git" url = "https://github.com/ImperialCollegeLondon/python-wiremock.git" reference = "fix-test-containers-on-windows" -resolved_reference = "d7cb44af3c3cf84e84b219094b3fc97a8ed06c5c" +resolved_reference = "fc2df60d0ff13e22f2cf5995d11596f83869a0ea" [[package]] name = "wrapt" @@ -1908,4 +1908,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "b4b811611814a4876734b6cc3e2dc52c3795e22f317c321b8853f172165efa36" +content-hash = "1fc833bcb9a1cfef4dde9214e97c327cbec06ec6e6de6f423ccd8abcb054ae65" diff --git a/pyproject.toml b/pyproject.toml index 91a420f..84711db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ pre-commit = "^3.8.0" ruff = "^0.5.4" docker = "^7.1.0" testcontainers = "^4.7.2" -wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "d7cb44af3c3cf84e84b219094b3fc97a8ed06c5c"} +wiremock = {git = "https://github.com/ImperialCollegeLondon/python-wiremock.git", rev = "fix-test-containers-on-windows"} [tool.poetry.scripts] # can execute via poetry, e.g. `poetry run imap-mag hello world` diff --git a/tests/wiremockUtils.py b/tests/wiremockUtils.py index 20b8b70..5144340 100644 --- a/tests/wiremockUtils.py +++ b/tests/wiremockUtils.py @@ -55,7 +55,7 @@ def add_file_mapping( ) -> None: """Copy file to container and add WireMock mapping for it.""" - container_dir_path = PurePosixPath("/home/wiremock/__files") + container_dir_path = Path("/home/wiremock/__files") with open(host_path, "rb") as f: self.__mock_container.copy_files_to_container( From 8b98e37787266af102772a8721e6dfa5ea0249c3 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:20:43 +0100 Subject: [PATCH 049/102] Correct code formatting --- tests/wiremockUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wiremockUtils.py b/tests/wiremockUtils.py index 5144340..66ceb82 100644 --- a/tests/wiremockUtils.py +++ b/tests/wiremockUtils.py @@ -1,5 +1,5 @@ import typing -from pathlib import Path, PurePosixPath +from pathlib import Path import pytest import typing_extensions From d71e3598f2a43dbf91e6fc610af9b2828e4c30c1 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:38:46 +0100 Subject: [PATCH 050/102] Skip tests that will fail on Windows runner --- tests/test_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index 16a4066..d42b9bb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -79,7 +79,7 @@ def test_process_with_binary_hk_converts_to_csv(): assert expectedLastLine == lines[-1] assert expectedNumRows == len(lines) - +@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner") def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Set up. binary_file = os.path.abspath("tests/data/2025/MAG_HSK_PW.pkts") @@ -123,6 +123,7 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() +@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner") def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. query_response: list[dict[str, str]] = [ From e0eee44003811d8298690218ccbb9aa480080ff6 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:41:59 +0100 Subject: [PATCH 051/102] Fix ruff errors --- tests/test_main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index d42b9bb..d21c606 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -79,7 +79,10 @@ def test_process_with_binary_hk_converts_to_csv(): assert expectedLastLine == lines[-1] assert expectedNumRows == len(lines) -@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner") +@pytest.mark.skipif( + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner" +) def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Set up. binary_file = os.path.abspath("tests/data/2025/MAG_HSK_PW.pkts") @@ -123,7 +126,10 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() -@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner") +@pytest.mark.skipif( + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner" +) def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. query_response: list[dict[str, str]] = [ From 4f922b8837c263dee3955c69c389001b122794e8 Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:43:43 +0100 Subject: [PATCH 052/102] Fix ruff errors --- tests/test_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index d21c606..1b5fee7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -80,8 +80,8 @@ def test_process_with_binary_hk_converts_to_csv(): assert expectedNumRows == len(lines) @pytest.mark.skipif( - os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", - reason="Wiremock test containers will not work on Windows Github Runner" + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner" ) def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Set up. @@ -127,8 +127,8 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 @pytest.mark.skipif( - os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", - reason="Wiremock test containers will not work on Windows Github Runner" + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner" ) def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. From 4e8d20e1d7ff6928add58457f56d4061c11a2a1d Mon Sep 17 00:00:00 2001 From: Mhairi Finlayson Date: Tue, 13 Aug 2024 10:55:25 +0100 Subject: [PATCH 053/102] Running pre-commit --- tests/test_main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 1b5fee7..d479fdf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -79,9 +79,10 @@ def test_process_with_binary_hk_converts_to_csv(): assert expectedLastLine == lines[-1] assert expectedNumRows == len(lines) + @pytest.mark.skipif( - os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", - reason="Wiremock test containers will not work on Windows Github Runner" + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner", ) def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Set up. @@ -127,8 +128,8 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 @pytest.mark.skipif( - os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", - reason="Wiremock test containers will not work on Windows Github Runner" + os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", + reason="Wiremock test containers will not work on Windows Github Runner", ) def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Set up. From 1163a78a5be87331970636e23a9c6d2952705d91 Mon Sep 17 00:00:00 2001 From: Alastair Crabtree Date: Fri, 16 Aug 2024 10:21:33 +0000 Subject: [PATCH 054/102] chore: fix line ending in pre-commit --- run-docker.sh | 64 +++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/run-docker.sh b/run-docker.sh index edda253..cc172c3 100755 --- a/run-docker.sh +++ b/run-docker.sh @@ -1,32 +1,32 @@ -#!/bin/bash -set -e - -# Example: -# ./run-docker.sh -# ./run-docker.sh -i // for interactive debugging -# ./run-docker.sh -i -e VARIABLE=some_value // for interactive debugging with VARIABLE env var set - -IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" - -echo "Running $IMAGE_NAME with dev.env file" - -# check if the argument "DEBUG" or "-i" is passed -if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then - echo "Overriding entrypoint to be an interactive bash shell for debugging" - docker run --rm -it \ - --entrypoint /bin/bash \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME -elif [ -z "$1" ]; then # no args passed - docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $IMAGE_NAME -else - echo "Extra arguments: $@" - docker run --rm -it \ - --env-file dev.env \ - -v /mnt/imap-data:/data \ - $@ $IMAGE_NAME -fi +#!/bin/bash +set -e + +# Example: +# ./run-docker.sh +# ./run-docker.sh -i // for interactive debugging +# ./run-docker.sh -i -e VARIABLE=some_value // for interactive debugging with VARIABLE env var set + +IMAGE_NAME="${IMAGE_NAME:-imap-pipeline-core/imap-mag}" + +echo "Running $IMAGE_NAME with dev.env file" + +# check if the argument "DEBUG" or "-i" is passed +if [ "$1" == "debug" ] || [ "$1" == "DEBUG" ] || [ "$1" == "-i" ]; then + echo "Overriding entrypoint to be an interactive bash shell for debugging" + docker run --rm -it \ + --entrypoint /bin/bash \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +elif [ -z "$1" ]; then # no args passed + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $IMAGE_NAME +else + echo "Extra arguments: $@" + docker run --rm -it \ + --env-file dev.env \ + -v /mnt/imap-data:/data \ + $@ $IMAGE_NAME +fi From 1739fe64baee41843e78f288a7df5cf7f698ef56 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 12:52:31 +0000 Subject: [PATCH 055/102] feat: add class to manage outputs --- .gitignore | 1 + src/imap_mag/main.py | 6 +- src/imap_mag/outputManager.py | 126 ++++++++++++++++++++++++++++++++++ tests/testUtils.py | 24 +++++++ tests/test_main.py | 10 +-- tests/test_outputManager.py | 123 +++++++++++++++++++++++++++++++++ 6 files changed, 280 insertions(+), 10 deletions(-) create mode 100644 src/imap_mag/outputManager.py create mode 100644 tests/test_outputManager.py diff --git a/.gitignore b/.gitignore index 7a6fd75..ae89700 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,4 @@ site/ .work /output dev.env +debug diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index cba66fd..a797a2c 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -35,7 +35,7 @@ globalState = {"verbose": False} -def commandInit(config: Path) -> appConfig.AppConfig: +def commandInit(config: Path | None) -> appConfig.AppConfig: # load and verify the config file if config is None: logging.critical("No config file") @@ -182,6 +182,8 @@ def fetch_binary( end_date: Annotated[str, typer.Option(help="End date for the download")], config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): + """Download binary data from WebPODA.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: @@ -229,6 +231,8 @@ def fetch_science( ] = LevelEnum.level_2, config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), ): + """Download science data from the SDC.""" + configFile: appConfig.AppConfig = commandInit(config) if not auth_code: diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py new file mode 100644 index 0000000..26a49ca --- /dev/null +++ b/src/imap_mag/outputManager.py @@ -0,0 +1,126 @@ +import hashlib +import logging +import shutil +import typing +from datetime import datetime +from pathlib import Path + +import typer +import typing_extensions + + +class OutputMetadata(typing.TypedDict): + """Metadata for output files.""" + + data_level: str | None + descriptor: str + date: datetime + version: str | None + extension: str + + +class OutputManager: + """Manage output files.""" + + @staticmethod + def get_folder_structure(**metadata: OutputMetadata) -> str: + """Retrieve folder structure from metadata.""" + + if "date" not in metadata: + logging.error(f"Metadata must contain key 'date'. Got: {metadata.keys()}") + raise typer.Abort() + + return metadata["date"].strftime("%Y/%m/%d") + + @staticmethod + def get_file_name(**metadata: OutputMetadata) -> str: + """Retireve file name from metadata.""" + + if {"descriptor", "date", "version", "extension"} > metadata.keys(): + logging.error( + f"Metadata must contain keys 'descriptor', 'date', 'version', 'extension'. Got: {metadata.keys()}" + ) + raise typer.Abort() + + return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-{metadata['version']}.{metadata['extension']}" + + """Output location.""" + location: Path + """Function returning folder structure pattern.""" + folder_structure_provider: typing.Callable[..., str] = get_folder_structure + """Function returning file name pattern.""" + file_name_provider: typing.Callable[..., str] = get_file_name + + def __init__( + self, + location: Path, + *, + folder_structure_provider: typing.Callable[..., str] | None = None, + file_name_provider: typing.Callable[..., str] | None = None, + ) -> None: + self.location = location + + if folder_structure_provider is not None: + self.folder_structure_provider = folder_structure_provider + + if file_name_provider is not None: + self.file_name_provider = file_name_provider + + def add_file( + self, original_file: Path, **metadata: typing_extensions.Unpack[OutputMetadata] + ) -> Path: + """Add file to output location.""" + + if ("version" not in metadata) or (metadata["version"] is None): + logging.debug("No version provided. Setting to 'v000'.") + metadata["version"] = "v000" + + if not self.location.exists(): + self.location.mkdir(parents=True, exist_ok=True) + + destination_file: Path = self.__assemble_full_path(**metadata) + + if not destination_file.parent.exists(): + destination_file.parent.mkdir(parents=True, exist_ok=True) + + if destination_file.exists(): + if ( + hashlib.md5(destination_file.read_bytes()).hexdigest() + == hashlib.md5(original_file.read_bytes()).hexdigest() + ): + logging.info(f"File {destination_file} already exists and is the same.") + return destination_file + + metadata["version"] = self.__find_viable_version( + destination_file, **metadata + ) + destination_file = self.__assemble_full_path(**metadata) + + logging.info(f"Copying {original_file} to {destination_file.absolute()}.") + destination = shutil.copy2(original_file, destination_file) + logging.info(f"Copied to {destination}.") + + return destination_file + + def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: + """Assemble full path from metadata.""" + + return ( + self.location + / self.folder_structure_provider(**metadata) + / self.file_name_provider(**metadata) + ) + + def __find_viable_version( + self, destination_file: Path, **metadata: OutputMetadata + ) -> str: + """Find a viable version for a file.""" + + while destination_file.exists(): + logging.info( + f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." + ) + metadata["version"] = "v%03d" % (int(metadata["version"][1:]) + 1) + destination_file = self.__assemble_full_path(**metadata) + + return metadata["version"] diff --git a/tests/testUtils.py b/tests/testUtils.py index c8e30a4..2b16cfe 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -2,7 +2,31 @@ from pathlib import Path, PosixPath, WindowsPath import imap_mag.appConfig as appConfig +import pytest import yaml +from imap_mag import appLogging + + +@pytest.fixture(autouse=True) +def enableLogging(): + appLogging.set_up_logging( + console_log_output="stdout", + console_log_level="debug", + console_log_color=True, + logfile_file="debug", + logfile_log_level="debug", + logfile_log_color=False, + log_line_template="%(color_on)s[%(asctime)s] [%(levelname)-8s] %(message)s%(color_off)s", + console_log_line_template="%(color_on)s%(message)s%(color_off)s", + ) + yield + + +@pytest.fixture(autouse=True) +def tidyDataFolders(): + os.system("rm -rf .work") + os.system("rm -rf output/*") + yield def create_serialize_config( diff --git a/tests/test_main.py b/tests/test_main.py index d479fdf..6a6e28f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,23 +7,15 @@ import re from pathlib import Path -import pytest from imap_mag.main import app from typer.testing import CliRunner -from .testUtils import create_serialize_config +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 from .wiremockUtils import wiremock_manager # noqa: F401 runner = CliRunner() -@pytest.fixture(autouse=True) -def tidyDataFolders(): - os.system("rm -rf .work") - os.system("rm -rf output/*") - yield - - def test_app_says_hello(): result = runner.invoke(app, ["hello", "Bob"]) diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py new file mode 100644 index 0000000..3a57596 --- /dev/null +++ b/tests/test_outputManager.py @@ -0,0 +1,123 @@ +"""Tests for `OutputManager` class.""" + +from datetime import datetime +from pathlib import Path + +from imap_mag.outputManager import OutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def test_copy_new_file(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v000.txt").exists() + + +def test_copy_file_same_content(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + original_file.write_bytes(b"some content") + + existing_file = Path("output/2025/05/02/pwr-20250502-v000.txt") + existing_file.parent.mkdir(parents=True, exist_ok=True) + existing_file.touch() + existing_file.write_bytes(b"some content") + + existing_modification_time = existing_file.stat().st_mtime + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert not Path("output/2025/05/02/pwr-20250502-v001.txt").exists() + assert existing_file.stat().st_mtime == existing_modification_time + + +def test_copy_file_existing_versions(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + original_file.write_bytes(b"some content") + + for version in range(2): + existing_file = Path(f"output/2025/05/02/pwr-20250502-v{version:03}.txt") + existing_file.parent.mkdir(parents=True, exist_ok=True) + existing_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v002.txt").exists() + + +def test_copy_file_forced_version(): + # Set up. + manager = OutputManager(Path("output")) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file( + original_file, + descriptor="pwr", + date=datetime(2025, 5, 2), + version="v003", + extension="txt", + ) + + # Verify. + assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() + + +def test_copy_file_custom_providers(): + # Set up. + manager = OutputManager( + Path("output"), + folder_structure_provider=lambda **_: "abc", + file_name_provider=lambda **_: "def", + ) + + original_file = Path(".work/some_test_file.txt") + original_file.parent.mkdir(parents=True, exist_ok=True) + original_file.touch() + + # Exercise. + manager.add_file(original_file) + + # Verify. + assert Path("output/abc/def").exists() From fc771c311ab6f9493c99ae9e62a1fd91cac265c7 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 14:33:55 +0000 Subject: [PATCH 056/102] refactor: version is an `int` not a `str` --- src/imap_mag/outputManager.py | 10 +++++----- tests/test_outputManager.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 26a49ca..b72cffc 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -15,7 +15,7 @@ class OutputMetadata(typing.TypedDict): data_level: str | None descriptor: str date: datetime - version: str | None + version: int | None extension: str @@ -42,7 +42,7 @@ def get_file_name(**metadata: OutputMetadata) -> str: ) raise typer.Abort() - return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-{metadata['version']}.{metadata['extension']}" + return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-v{metadata['version']:03}.{metadata['extension']}" """Output location.""" location: Path @@ -73,7 +73,7 @@ def add_file( if ("version" not in metadata) or (metadata["version"] is None): logging.debug("No version provided. Setting to 'v000'.") - metadata["version"] = "v000" + metadata["version"] = 0 if not self.location.exists(): self.location.mkdir(parents=True, exist_ok=True) @@ -113,14 +113,14 @@ def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: def __find_viable_version( self, destination_file: Path, **metadata: OutputMetadata - ) -> str: + ) -> int: """Find a viable version for a file.""" while destination_file.exists(): logging.info( f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." ) - metadata["version"] = "v%03d" % (int(metadata["version"][1:]) + 1) + metadata["version"] += 1 destination_file = self.__assemble_full_path(**metadata) return metadata["version"] diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 3a57596..4ad826e 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -96,7 +96,7 @@ def test_copy_file_forced_version(): original_file, descriptor="pwr", date=datetime(2025, 5, 2), - version="v003", + version=3, extension="txt", ) From d1099d94a48b0a2246a1444fe6f024b637bec4a3 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Tue, 6 Aug 2024 15:09:41 +0000 Subject: [PATCH 057/102] refactor: replace custom logic with `OutputManager` --- src/imap_mag/appUtils.py | 29 +++++++++++++++-------------- src/imap_mag/outputManager.py | 4 ++++ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index 5d4250a..dfa1258 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -1,6 +1,4 @@ import logging -import os -import shutil from pathlib import Path from typing import Optional @@ -8,7 +6,8 @@ import pandas as pd import typer -from . import appConfig +from .appConfig import Destination +from .outputManager import OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -56,18 +55,20 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() -def copyFileToDestination(filePath: Path, destination: appConfig.Destination) -> None: +def copyFileToDestination( + file_path: Path, + destination: Destination, + output_manager: Optional[OutputManager] = None, +) -> Path: """Copy file to destination folder.""" - destinationFile = Path(destination.folder) + destination_folder = Path(destination.folder) - if not destinationFile.exists(): - logging.debug(f"Creating destination folder {destinationFile}.") - os.makedirs(destinationFile) + if output_manager is None: + output_manager: OutputManager = OutputManager( + destination_folder, + folder_structure_provider=lambda **_: "", + file_name_provider=lambda **_: file_path.name, + ) - if destination.filename: - destinationFile = destinationFile / destination.filename - - logging.info(f"Copying {filePath} to {destinationFile.absolute()}") - completed = shutil.copy2(filePath, destinationFile) - logging.info(f"Copy complete: {completed}") + output_manager.add_file(file_path) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index b72cffc..4d91260 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -76,11 +76,15 @@ def add_file( metadata["version"] = 0 if not self.location.exists(): + logging.debug(f"Output location does not exist. Creating {self.location}.") self.location.mkdir(parents=True, exist_ok=True) destination_file: Path = self.__assemble_full_path(**metadata) if not destination_file.parent.exists(): + logging.debug( + f"Output folder structure does not exist. Creating {destination_file.parent}." + ) destination_file.parent.mkdir(parents=True, exist_ok=True) if destination_file.exists(): From 4d66bed2f63d6c00180d183d286a3232e1f8447e Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 08:20:08 +0000 Subject: [PATCH 058/102] refactor: convert "pattern" methods to separate "provider" class --- src/imap_mag/appUtils.py | 6 +- src/imap_mag/outputManager.py | 125 ++++++++++++++++++---------------- tests/test_outputManager.py | 26 ++++--- 3 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index dfa1258..435e8e0 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -7,7 +7,7 @@ import typer from .appConfig import Destination -from .outputManager import OutputManager +from .outputManager import IMetadataProvider, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -59,7 +59,7 @@ def copyFileToDestination( file_path: Path, destination: Destination, output_manager: Optional[OutputManager] = None, -) -> Path: +) -> tuple[Path, IMetadataProvider]: """Copy file to destination folder.""" destination_folder = Path(destination.folder) @@ -71,4 +71,4 @@ def copyFileToDestination( file_name_provider=lambda **_: file_path.name, ) - output_manager.add_file(file_path) + return output_manager.add_file(file_path) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 4d91260..aac4f35 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -1,3 +1,4 @@ +import abc import hashlib import logging import shutil @@ -6,80 +7,84 @@ from pathlib import Path import typer -import typing_extensions -class OutputMetadata(typing.TypedDict): +class IMetadataProvider(abc.ABC): + """Interface for metadata providers.""" + + version: int = 0 + + @abc.abstractmethod + def get_folder_structure(self) -> str: + """Retrieve folder structure.""" + + @abc.abstractmethod + def get_file_name(self) -> str: + """Retireve file name.""" + + +class DefaultMetadataProvider(IMetadataProvider): """Metadata for output files.""" - data_level: str | None + data_level: str | None = None descriptor: str date: datetime - version: int | None extension: str + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) -class OutputManager: - """Manage output files.""" - - @staticmethod - def get_folder_structure(**metadata: OutputMetadata) -> str: - """Retrieve folder structure from metadata.""" - - if "date" not in metadata: - logging.error(f"Metadata must contain key 'date'. Got: {metadata.keys()}") + def get_folder_structure(self) -> str: + if self.date is None: + logging.error("No 'date' defined. Cannot generate folder structure.") raise typer.Abort() - return metadata["date"].strftime("%Y/%m/%d") - - @staticmethod - def get_file_name(**metadata: OutputMetadata) -> str: - """Retireve file name from metadata.""" + return self.date.strftime("%Y/%m/%d") - if {"descriptor", "date", "version", "extension"} > metadata.keys(): + def get_file_name(self) -> str: + if any(x is None for x in ["descriptor", "date", "version", "extension"]): logging.error( - f"Metadata must contain keys 'descriptor', 'date', 'version', 'extension'. Got: {metadata.keys()}" + "No 'descriptor', 'date', 'version', or 'extension' defined. Cannot generate file name." ) raise typer.Abort() - return f"{metadata['descriptor']}-{metadata['date'].strftime('%Y%m%d')}-v{metadata['version']:03}.{metadata['extension']}" - - """Output location.""" - location: Path - """Function returning folder structure pattern.""" - folder_structure_provider: typing.Callable[..., str] = get_folder_structure - """Function returning file name pattern.""" - file_name_provider: typing.Callable[..., str] = get_file_name - - def __init__( - self, - location: Path, - *, - folder_structure_provider: typing.Callable[..., str] | None = None, - file_name_provider: typing.Callable[..., str] | None = None, - ) -> None: - self.location = location + return f"{self.descriptor}-{self.date.strftime('%Y%m%d')}-v{self.version:03}.{self.extension}" - if folder_structure_provider is not None: - self.folder_structure_provider = folder_structure_provider - if file_name_provider is not None: - self.file_name_provider = file_name_provider +class IOutputManager(abc.ABC): + """Interface for output managers.""" + @abc.abstractmethod def add_file( - self, original_file: Path, **metadata: typing_extensions.Unpack[OutputMetadata] - ) -> Path: + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: """Add file to output location.""" - if ("version" not in metadata) or (metadata["version"] is None): - logging.debug("No version provided. Setting to 'v000'.") - metadata["version"] = 0 + def add_default_file( + self, original_file: Path, **metadata: typing.Any + ) -> tuple[Path, IMetadataProvider]: + return self.add_file(original_file, DefaultMetadataProvider(**metadata)) + + +class OutputManager(IOutputManager): + """Manage output files.""" + + location: Path + + def __init__(self, location: Path) -> None: + self.location = location + + def add_file( + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: + """Add file to output location.""" if not self.location.exists(): logging.debug(f"Output location does not exist. Creating {self.location}.") self.location.mkdir(parents=True, exist_ok=True) - destination_file: Path = self.__assemble_full_path(**metadata) + destination_file: Path = self.__assemble_full_path(metadata_provider) if not destination_file.parent.exists(): logging.debug( @@ -93,38 +98,38 @@ def add_file( == hashlib.md5(original_file.read_bytes()).hexdigest() ): logging.info(f"File {destination_file} already exists and is the same.") - return destination_file + return (destination_file, metadata_provider) - metadata["version"] = self.__find_viable_version( - destination_file, **metadata + metadata_provider.version = self.__find_viable_version( + destination_file, metadata_provider ) - destination_file = self.__assemble_full_path(**metadata) + destination_file = self.__assemble_full_path(metadata_provider) logging.info(f"Copying {original_file} to {destination_file.absolute()}.") destination = shutil.copy2(original_file, destination_file) logging.info(f"Copied to {destination}.") - return destination_file + return (destination_file, metadata_provider) - def __assemble_full_path(self, **metadata: OutputMetadata) -> Path: + def __assemble_full_path(self, metadata_provider: IMetadataProvider) -> Path: """Assemble full path from metadata.""" return ( self.location - / self.folder_structure_provider(**metadata) - / self.file_name_provider(**metadata) + / metadata_provider.get_folder_structure() + / metadata_provider.get_file_name() ) def __find_viable_version( - self, destination_file: Path, **metadata: OutputMetadata + self, destination_file: Path, metadata_provider: IMetadataProvider ) -> int: """Find a viable version for a file.""" while destination_file.exists(): logging.info( - f"File {destination_file} already exists and is different. Increasing version to {metadata['version']}." + f"File {destination_file} already exists and is different. Increasing version to {metadata_provider.version}." ) - metadata["version"] += 1 - destination_file = self.__assemble_full_path(**metadata) + metadata_provider.version += 1 + destination_file = self.__assemble_full_path(metadata_provider) - return metadata["version"] + return metadata_provider.version diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 4ad826e..3d10cd4 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import Path -from imap_mag.outputManager import OutputManager +from imap_mag.outputManager import IMetadataProvider, OutputManager from .testUtils import enableLogging, tidyDataFolders # noqa: F401 @@ -17,7 +17,7 @@ def test_copy_new_file(): original_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -45,7 +45,7 @@ def test_copy_file_same_content(): existing_modification_time = existing_file.stat().st_mtime # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -72,7 +72,7 @@ def test_copy_file_existing_versions(): existing_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -92,7 +92,7 @@ def test_copy_file_forced_version(): original_file.touch() # Exercise. - manager.add_file( + manager.add_default_file( original_file, descriptor="pwr", date=datetime(2025, 5, 2), @@ -104,20 +104,24 @@ def test_copy_file_forced_version(): assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() +class TestMetadataProvider(IMetadataProvider): + def get_folder_structure(self) -> str: + return "abc" + + def get_file_name(self) -> str: + return "def" + + def test_copy_file_custom_providers(): # Set up. - manager = OutputManager( - Path("output"), - folder_structure_provider=lambda **_: "abc", - file_name_provider=lambda **_: "def", - ) + manager = OutputManager(Path("output")) original_file = Path(".work/some_test_file.txt") original_file.parent.mkdir(parents=True, exist_ok=True) original_file.touch() # Exercise. - manager.add_file(original_file) + manager.add_file(original_file, TestMetadataProvider()) # Verify. assert Path("output/abc/def").exists() From 620da38ca74335d92c96d721e53eb046a556274f Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 08:20:35 +0000 Subject: [PATCH 059/102] build: single source app version --- poetry.lock | 125 +++++++++++++++++++++------------------ pyproject.toml | 4 +- src/imap_mag/__init__.py | 7 +++ 3 files changed, 79 insertions(+), 57 deletions(-) diff --git a/poetry.lock b/poetry.lock index b918bc3..504e515 100644 --- a/poetry.lock +++ b/poetry.lock @@ -599,12 +599,12 @@ files = [ [[package]] name = "imap-data-access" -version = "0.7.0" +version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" files = [ - {file = "imap_data_access-0.7.0.tar.gz", hash = "sha256:f0db935949d048394fc554b308b1e4a1572a18acd41636462d37c309c7cb4c9d"}, + {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, ] [package.extras] @@ -1353,62 +1353,64 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] @@ -1504,6 +1506,17 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "single-version" +version = "1.6.0" +description = "Small utility to define version string for Poetry-style Python project." +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "single_version-1.6.0-py3-none-any.whl", hash = "sha256:67a2734e728b9554750e867b33591f3ad9509ccb851bb3047ced7bfe68429ecd"}, + {file = "single_version-1.6.0.tar.gz", hash = "sha256:3b1fb6e9bd2c88268948d9191c78b63ddd3c07554c1f07cd8a85aedf2486e4fc"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1908,4 +1921,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "1fc833bcb9a1cfef4dde9214e97c327cbec06ec6e6de6f423ccd8abcb054ae65" +content-hash = "40096a30f5c8ed7a5daaaaa5da50d0872d3ae7ab77e08256087fb8e44fb8a779" diff --git a/pyproject.toml b/pyproject.toml index 84711db..57348e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] requires-python = ">=3.10" name = "imap-mag" +version = "0.0.1" [tool.poetry] name = "imap-mag" @@ -28,9 +29,10 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.7.0" +imap-data-access = "^0.8.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} +single-version = "^1.6.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/src/imap_mag/__init__.py b/src/imap_mag/__init__.py index e69de29..ad47412 100644 --- a/src/imap_mag/__init__.py +++ b/src/imap_mag/__init__.py @@ -0,0 +1,7 @@ +"""The main module for project.""" + +from pathlib import Path + +from single_version import get_version + +__version__ = get_version("imap-mag", Path(__file__).parent.parent) From 1a75c22aeeb101af40a993e13083e09fef2844d1 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 10:15:17 +0000 Subject: [PATCH 060/102] style: replace `-` with `_` in file names --- src/imap_mag/outputManager.py | 2 +- tests/test_outputManager.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index aac4f35..4347bee 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -49,7 +49,7 @@ def get_file_name(self) -> str: ) raise typer.Abort() - return f"{self.descriptor}-{self.date.strftime('%Y%m%d')}-v{self.version:03}.{self.extension}" + return f"{self.descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" class IOutputManager(abc.ABC): diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 3d10cd4..9466c03 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -25,7 +25,7 @@ def test_copy_new_file(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v000.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v000.txt").exists() def test_copy_file_same_content(): @@ -37,7 +37,7 @@ def test_copy_file_same_content(): original_file.touch() original_file.write_bytes(b"some content") - existing_file = Path("output/2025/05/02/pwr-20250502-v000.txt") + existing_file = Path("output/2025/05/02/pwr_20250502_v000.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() existing_file.write_bytes(b"some content") @@ -53,7 +53,7 @@ def test_copy_file_same_content(): ) # Verify. - assert not Path("output/2025/05/02/pwr-20250502-v001.txt").exists() + assert not Path("output/2025/05/02/pwr_20250502_v001.txt").exists() assert existing_file.stat().st_mtime == existing_modification_time @@ -67,7 +67,7 @@ def test_copy_file_existing_versions(): original_file.write_bytes(b"some content") for version in range(2): - existing_file = Path(f"output/2025/05/02/pwr-20250502-v{version:03}.txt") + existing_file = Path(f"output/2025/05/02/pwr_20250502_v{version:03}.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() @@ -80,7 +80,7 @@ def test_copy_file_existing_versions(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v002.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v002.txt").exists() def test_copy_file_forced_version(): @@ -101,7 +101,7 @@ def test_copy_file_forced_version(): ) # Verify. - assert Path("output/2025/05/02/pwr-20250502-v003.txt").exists() + assert Path("output/2025/05/02/pwr_20250502_v003.txt").exists() class TestMetadataProvider(IMetadataProvider): From cb4186c13379b3b3d1df9e44b69da39b8ede7c52 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 11:00:12 +0000 Subject: [PATCH 061/102] task: add support for `prefix` and `level` in output metadata --- src/imap_mag/outputManager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 4347bee..e46a4d0 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -26,7 +26,8 @@ def get_file_name(self) -> str: class DefaultMetadataProvider(IMetadataProvider): """Metadata for output files.""" - data_level: str | None = None + prefix: str | None = "imap_mag" + level: str | None = None descriptor: str date: datetime extension: str @@ -49,7 +50,15 @@ def get_file_name(self) -> str: ) raise typer.Abort() - return f"{self.descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" + descriptor = self.descriptor + + if self.level is not None: + descriptor = f"{self.level}_{descriptor}" + + if self.prefix is not None: + descriptor = f"{self.prefix}_{descriptor}" + + return f"{descriptor}_{self.date.strftime('%Y%m%d')}_v{self.version:03}.{self.extension}" class IOutputManager(abc.ABC): From cbe7e83e8f3c82709de7f655e729c78ea699997e Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:04:31 +0000 Subject: [PATCH 062/102] feat: add more variables in database --- ...dded_version_date_and_software_version_.py | 34 +++++++++++++++++++ src/imap_db/model.py | 7 +++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py diff --git a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py b/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py new file mode 100644 index 0000000..ccb34bd --- /dev/null +++ b/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py @@ -0,0 +1,34 @@ +"""Added version, date and software version columns + +Revision ID: 77642de1867a +Revises: d0457f3e98c8 +Create Date: 2024-08-07 13:03:30.889623 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "77642de1867a" +down_revision = "d0457f3e98c8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("files", sa.Column("version", sa.Integer(), nullable=False)) + op.add_column("files", sa.Column("date", sa.DateTime(), nullable=False)) + op.add_column( + "files", sa.Column("software_version", sa.String(length=16), nullable=False) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("files", "software_version") + op.drop_column("files", "date") + op.drop_column("files", "version") + # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py index 6cfbcb6..c2be860 100644 --- a/src/imap_db/model.py +++ b/src/imap_db/model.py @@ -1,4 +1,6 @@ -from sqlalchemy import String +from datetime import datetime + +from sqlalchemy import DateTime, Integer, String from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -11,6 +13,9 @@ class File(Base): id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(128)) path: Mapped[str] = mapped_column(String(256)) + version: Mapped[int] = mapped_column(Integer()) + date: Mapped[datetime] = mapped_column(DateTime()) + software_version: Mapped[str] = mapped_column(String(16)) def __repr__(self) -> str: return f"" From a69dd9b916dced069d0c4156ba224a6f7c3c15d4 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:23:39 +0000 Subject: [PATCH 063/102] feat!: use output manager to write to output and database BREAKING: some tests no longer pass --- src/imap_mag/DB.py | 35 +++++++++++++- src/imap_mag/cli/fetchBinary.py | 78 ++++++++++++++++++++++++++++++++ src/imap_mag/cli/fetchScience.py | 29 ++++++++---- src/imap_mag/main.py | 43 ++++++++---------- 4 files changed, 152 insertions(+), 33 deletions(-) create mode 100644 src/imap_mag/cli/fetchBinary.py diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 3ca15e0..b7a6f19 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,9 +1,14 @@ import os +from pathlib import Path from imap_db.model import File from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from src.imap_db.model import File +from src.imap_mag import __version__ +from src.imap_mag.outputManager import IMetadataProvider, IOutputManager + class DB: def __init__(self, db_url=None): @@ -19,7 +24,10 @@ def __init__(self, db_url=None): self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - def insert_files(self, files: list[File]): + def insert_file(self, file: File) -> None: + self.insert_files([file]) + + def insert_files(self, files: list[File]) -> None: session = self.Session() try: for file in files: @@ -40,3 +48,28 @@ def insert_files(self, files: list[File]): raise e finally: session.close() + + +class DatabaseOutputManager(IOutputManager): + def __init__(self, output_manager: IOutputManager, db: DB = DB()): + self.output_manager = output_manager + self.db = db + + def add_file( + self, original_file: Path, metadata_provider: IMetadataProvider + ) -> tuple[Path, IMetadataProvider]: + (destination_file, metadata_provider) = self.output_manager.add_file( + original_file, metadata_provider + ) + + self.db.insert_file( + File( + name=destination_file.name, + path=destination_file.absolute().as_posix(), + version=metadata_provider.version, + date=metadata_provider.date, + software_version=__version__, + ) + ) + + return (destination_file, metadata_provider) diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py new file mode 100644 index 0000000..a450180 --- /dev/null +++ b/src/imap_mag/cli/fetchBinary.py @@ -0,0 +1,78 @@ +"""Program to retrieve and process MAG binary files.""" + +import typing +from datetime import datetime +from pathlib import Path + +import pandas as pd +import typing_extensions + +from ..client.webPODA import WebPODA +from ..outputManager import IOutputManager + + +class FetchBinaryOptions(typing.TypedDict): + """Options for WebPODA interactions.""" + + packet: str + start_date: datetime + end_date: datetime + + +class FetchBinary: + """Manage WebPODA data.""" + + __web_poda: WebPODA + __output_manager: IOutputManager | None + + def __init__( + self, + web_poda: WebPODA, + output_manager: IOutputManager | None = None, + ) -> None: + """Initialize WebPODA interface.""" + + self.__web_poda = web_poda + self.__output_manager = output_manager + + def download_binaries( + self, **options: typing_extensions.Unpack[FetchBinaryOptions] + ) -> list[Path]: + """Retrieve WebPODA data.""" + + downloaded = [] + + date_range: pd.DatetimeIndex = pd.date_range( + start=options["start_date"], + end=options["end_date"], + freq="D", + normalize=True, + ) + + dates = date_range.to_pydatetime().tolist() + if len(dates) == 1: + dates += [ + pd.Timestamp(dates[0] + pd.Timedelta(days=1)) + .normalize() + .to_pydatetime() + ] + + for d in range(len(dates) - 1): + file = self.__web_poda.download( + packet=options["packet"], start_date=dates[d], end_date=dates[d + 1] + ) + + if self.__output_manager is not None: + self.__output_manager.add_default_file( + file, + descriptor=options["packet"] + .lower() + .strip("mag_") + .replace("_", "-"), + date=dates[d], + extension="pkts", + ) + + downloaded += [file] + + return downloaded diff --git a/src/imap_mag/cli/fetchScience.py b/src/imap_mag/cli/fetchScience.py index 49fd3e9..6da8d93 100644 --- a/src/imap_mag/cli/fetchScience.py +++ b/src/imap_mag/cli/fetchScience.py @@ -1,14 +1,15 @@ """Program to retrieve and process MAG CDF files.""" import typing +from datetime import datetime from enum import Enum from pathlib import Path import pandas as pd import typing_extensions -from .. import appUtils from ..client.sdcDataAccess import ISDCDataAccess +from ..outputManager import IOutputManager class MAGMode(str, Enum): @@ -25,28 +26,31 @@ class FetchScienceOptions(typing.TypedDict): """Options for SOC interactions.""" level: str - start_date: str - end_date: str + start_date: datetime + end_date: datetime output_dir: str class FetchScience: """Manage SOC data.""" + __data_access: ISDCDataAccess + __output_manager: IOutputManager | None + __modes: list[MAGMode] __sensor: list[MAGSensor] - __data_access: ISDCDataAccess - def __init__( self, data_access: ISDCDataAccess, + output_manager: IOutputManager | None = None, modes: list[MAGMode] = ["norm", "burst"], sensors: list[MAGSensor] = ["magi", "mago"], ) -> None: """Initialize SDC interface.""" self.__data_access = data_access + self.__output_manager = output_manager self.__modes = modes self.__sensor = sensors @@ -59,8 +63,8 @@ def download_latest_science( for mode in self.__modes: date_range: pd.DatetimeIndex = pd.date_range( - start=appUtils.convertToDatetime(options["start_date"]), - end=appUtils.convertToDatetime(options["end_date"]), + start=options["start_date"], + end=options["end_date"], freq="D", normalize=True, ) @@ -71,7 +75,7 @@ def download_latest_science( level=options["level"], descriptor=str(mode) + "-" + str(sensor), start_date=date, - end_date=None, + end_date=date, version="latest", extension="cdf", ) @@ -82,4 +86,13 @@ def download_latest_science( self.__data_access.download(file["file_path"]) ] + if self.__output_manager is not None: + self.__output_manager.add_default_file( + downloaded[-1], + level=options["level"], + descriptor=file["descriptor"], + date=date, + extension="cdf", + ) + return downloaded diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index a797a2c..593203e 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -13,7 +13,6 @@ # config import yaml -from imap_db.model import File from mag_toolkit import CDFLoader from mag_toolkit.calibration.CalibrationApplicator import CalibrationApplicator from mag_toolkit.calibration.calibrationFormatProcessor import ( @@ -26,10 +25,13 @@ SpinPlaneCalibrator, ) -from . import DB, appConfig, appLogging, appUtils, imapProcessing +from . import appConfig, appLogging, appUtils, imapProcessing +from .cli.fetchBinary import FetchBinary from .cli.fetchScience import FetchScience from .client.sdcDataAccess import SDCDataAccess from .client.webPODA import WebPODA +from .DB import DatabaseOutputManager +from .outputManager import OutputManager app = typer.Typer() globalState = {"verbose": False} @@ -191,6 +193,9 @@ def fetch_binary( raise typer.Abort() packet: str = appUtils.getPacketFromApID(apid) + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading raw packet {packet} from {start_date} to {end_date}.") poda = WebPODA( @@ -198,13 +203,12 @@ def fetch_binary( configFile.work_folder, configFile.api.webpoda_url if configFile.api else None, ) - result: str = poda.download( - packet=packet, - start_date=appUtils.convertToDatetime(start_date), - end_date=appUtils.convertToDatetime(end_date), - ) + output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) - appUtils.copyFileToDestination(result, configFile.destination) + fetch_binary = FetchBinary(poda, output_manager) + fetch_binary.download_binaries( + packet=packet, start_date=start_date, end_date=end_date + ) class LevelEnum(str, Enum): @@ -229,7 +233,7 @@ def fetch_science( level: Annotated[ LevelEnum, typer.Option(help="Level to download") ] = LevelEnum.level_2, - config: Annotated[Path, typer.Option()] = Path("config-sci.yaml"), + config: Annotated[Path, typer.Option()] = Path("config.yaml"), ): """Download science data from the SDC.""" @@ -239,31 +243,22 @@ def fetch_science( logging.critical("No SDC_AUTH_CODE API key provided") raise typer.Abort() + start_date = appUtils.convertToDatetime(start_date) + end_date = appUtils.convertToDatetime(end_date) + logging.info(f"Downloading {level} science from {start_date} to {end_date}.") data_access = SDCDataAccess( data_dir=str(configFile.work_folder), sdc_url=configFile.api.sdc_url if configFile.api else None, ) + output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) - fetch_science = FetchScience(data_access) - files = fetch_science.download_latest_science( + fetch_science = FetchScience(data_access, output_manager) + fetch_science.download_latest_science( level=level.value, start_date=start_date, end_date=end_date ) - records = [] - for file in files: - records.append(File(name=file.name, path=file.absolute().as_posix())) - - for file in files: - appUtils.copyFileToDestination(file, configFile.destination) - - if configFile.destination.export_to_database: - db = DB.DB() - db.insert_files(records) - - logging.info(f"Downloaded {len(files)} files and saved to database") - # imap-mag calibrate --config calibration_config.yaml --method SpinAxisCalibrator imap_mag_l1b_norm-mago_20250502_v000.cdf @app.command() From 5093519ba88443863a67aabd54bf082bd987a142 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:25:26 +0000 Subject: [PATCH 064/102] dev: add `SQLALCHEMY_URL` to dev container environment --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6474864..eb6693c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -34,6 +34,7 @@ "WEBPODA_AUTH_CODE": "${localEnv:WEBPODA_AUTH_CODE}", "SDC_AUTH_CODE": "${localEnv:SDC_AUTH_CODE}", "IMAP_DATA_ACCESS_URL": "${localEnv:IMAP_DATA_ACCESS_URL}", + "SQLALCHEMY_URL": "${localEnv:SQLALCHEMY_URL}", // Define WireMock variables to connect Docker outside of Docker. "WIREMOCK_DIND": "1", "TESTCONTAINERS_HOST_OVERRIDE": "host.docker.internal" From 5510d3aac6a7d6cb64fe4d4e836ad85f4e50217a Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:32:15 +0000 Subject: [PATCH 065/102] test(fix): fix output manager and calibration tests --- src/imap_mag/appUtils.py | 22 ++++++++++++++++------ tests/test_outputManager.py | 14 ++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index 435e8e0..bc90af7 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -62,13 +62,23 @@ def copyFileToDestination( ) -> tuple[Path, IMetadataProvider]: """Copy file to destination folder.""" + class SimpleMetadataProvider(IMetadataProvider): + """Simple metadata provider for compatibility.""" + + def __init__(self, filename: str) -> None: + self.filename = filename + + def get_folder_structure(self) -> str: + return "" + + def get_file_name(self) -> str: + return self.filename + destination_folder = Path(destination.folder) if output_manager is None: - output_manager: OutputManager = OutputManager( - destination_folder, - folder_structure_provider=lambda **_: "", - file_name_provider=lambda **_: file_path.name, - ) + output_manager: OutputManager = OutputManager(destination_folder) - return output_manager.add_file(file_path) + return output_manager.add_file( + file_path, SimpleMetadataProvider(destination.filename) + ) diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 9466c03..912ef7b 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -25,7 +25,7 @@ def test_copy_new_file(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v000.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt").exists() def test_copy_file_same_content(): @@ -37,7 +37,7 @@ def test_copy_file_same_content(): original_file.touch() original_file.write_bytes(b"some content") - existing_file = Path("output/2025/05/02/pwr_20250502_v000.txt") + existing_file = Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt") existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() existing_file.write_bytes(b"some content") @@ -53,7 +53,7 @@ def test_copy_file_same_content(): ) # Verify. - assert not Path("output/2025/05/02/pwr_20250502_v001.txt").exists() + assert not Path("output/2025/05/02/imap_mag_pwr_20250502_v001.txt").exists() assert existing_file.stat().st_mtime == existing_modification_time @@ -67,7 +67,9 @@ def test_copy_file_existing_versions(): original_file.write_bytes(b"some content") for version in range(2): - existing_file = Path(f"output/2025/05/02/pwr_20250502_v{version:03}.txt") + existing_file = Path( + f"output/2025/05/02/imap_mag_pwr_20250502_v{version:03}.txt" + ) existing_file.parent.mkdir(parents=True, exist_ok=True) existing_file.touch() @@ -80,7 +82,7 @@ def test_copy_file_existing_versions(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v002.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v002.txt").exists() def test_copy_file_forced_version(): @@ -101,7 +103,7 @@ def test_copy_file_forced_version(): ) # Verify. - assert Path("output/2025/05/02/pwr_20250502_v003.txt").exists() + assert Path("output/2025/05/02/imap_mag_pwr_20250502_v003.txt").exists() class TestMetadataProvider(IMetadataProvider): From 28ce147c4324f04d8b24059244472ae044acb342 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 13:45:25 +0000 Subject: [PATCH 066/102] fix: do not export to database in tests --- src/imap_mag/appUtils.py | 14 +++++++++++++- src/imap_mag/main.py | 6 ++---- tests/config/calibration_application_config.yaml | 3 ++- tests/config/calibration_config.yaml | 3 ++- tests/config/hk_process.yaml | 1 + 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index bc90af7..c87410c 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -7,7 +7,8 @@ import typer from .appConfig import Destination -from .outputManager import IMetadataProvider, OutputManager +from .DB import DatabaseOutputManager +from .outputManager import IMetadataProvider, IOutputManager, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -55,6 +56,17 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() +def getOutputManager(destination: Destination) -> IOutputManager: + """Retrieve output manager based on destination.""" + + output_manager = OutputManager(destination.folder) + + if destination.export_to_database: + output_manager = DatabaseOutputManager(output_manager) + + return output_manager + + def copyFileToDestination( file_path: Path, destination: Destination, diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index 593203e..1fa375d 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -30,8 +30,6 @@ from .cli.fetchScience import FetchScience from .client.sdcDataAccess import SDCDataAccess from .client.webPODA import WebPODA -from .DB import DatabaseOutputManager -from .outputManager import OutputManager app = typer.Typer() globalState = {"verbose": False} @@ -203,7 +201,7 @@ def fetch_binary( configFile.work_folder, configFile.api.webpoda_url if configFile.api else None, ) - output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) + output_manager = appUtils.getOutputManager(configFile.destination) fetch_binary = FetchBinary(poda, output_manager) fetch_binary.download_binaries( @@ -252,7 +250,7 @@ def fetch_science( data_dir=str(configFile.work_folder), sdc_url=configFile.api.sdc_url if configFile.api else None, ) - output_manager = DatabaseOutputManager(OutputManager(configFile.destination.folder)) + output_manager = appUtils.getOutputManager(configFile.destination) fetch_science = FetchScience(data_access, output_manager) fetch_science.download_latest_science( diff --git a/tests/config/calibration_application_config.yaml b/tests/config/calibration_application_config.yaml index b748acb..f6e3448 100644 --- a/tests/config/calibration_application_config.yaml +++ b/tests/config/calibration_application_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: L2.cdf \ No newline at end of file + filename: L2.cdf + export-to-database: false diff --git a/tests/config/calibration_config.yaml b/tests/config/calibration_config.yaml index 5ed5230..a09829b 100644 --- a/tests/config/calibration_config.yaml +++ b/tests/config/calibration_config.yaml @@ -5,4 +5,5 @@ work-folder: .work destination: folder: output/ - filename: calibration.json \ No newline at end of file + filename: calibration.json + export-to-database: false diff --git a/tests/config/hk_process.yaml b/tests/config/hk_process.yaml index f3d4af1..8a72ea8 100644 --- a/tests/config/hk_process.yaml +++ b/tests/config/hk_process.yaml @@ -6,6 +6,7 @@ work-folder: .work destination: folder: output/ filename: result.csv + export-to-database: false packet-definition: hk: src/imap_mag/xtce/tlm_20240724.xml From 9b353bc09766b5678960e28bfb649f643125bc2a Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 14:17:32 +0000 Subject: [PATCH 067/102] fix: HK and science tests --- tests/test_main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 6a6e28f..e435f4d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -110,10 +110,10 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/power.pkts").exists() + assert Path("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts").exists() with ( - open("output/power.pkts", "rb") as output, + open("output/2025/05/02/imap_mag_hsk-pw_20250502_v000.pkts", "rb") as output, open(binary_file, "rb") as input, ): assert output.read() == input.read() @@ -143,7 +143,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 ) wiremock_manager.add_string_mapping( - "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&version=latest&extension=cdf", + "/query?instrument=mag&data_level=l1b&descriptor=norm-magi&start_date=20250502&end_date=20250502&extension=cdf", json.dumps(query_response), priority=1, ) @@ -154,7 +154,7 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 wiremock_manager.add_string_mapping( re.escape("/query?instrument=mag&data_level=l1b&descriptor=") + ".*" - + re.escape("&start_date=20250502&version=latest&extension=cdf"), + + re.escape("&start_date=20250502&end_date=20250502&extension=cdf"), json.dumps({}), is_pattern=True, priority=2, @@ -187,10 +187,12 @@ def test_fetch_science_downloads_cdf_from_sdc(wiremock_manager): # noqa: F811 # Verify. assert result.exit_code == 0 - assert Path("output/result.cdf").exists() + assert Path("output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf").exists() with ( - open("output/result.cdf", "rb") as output, + open( + "output/2025/05/02/imap_mag_l1b_norm-magi_20250502_v000.cdf", "rb" + ) as output, open(cdf_file, "rb") as input, ): assert output.read() == input.read() From d780de8115194767d5272572242ff888540e2a7c Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Wed, 7 Aug 2024 14:32:32 +0000 Subject: [PATCH 068/102] fix: attempt to fix issue with database initialization --- src/imap_mag/DB.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index b7a6f19..b59788c 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -51,9 +51,15 @@ def insert_files(self, files: list[File]) -> None: class DatabaseOutputManager(IOutputManager): - def __init__(self, output_manager: IOutputManager, db: DB = DB()): + def __init__(self, output_manager: IOutputManager, db: DB | None = None): + """Initialize database and output manager.""" + self.output_manager = output_manager - self.db = db + + if db is None: + self.db = DB() + else: + self.db = db def add_file( self, original_file: Path, metadata_provider: IMetadataProvider From f58a6a7c5c40563091b24eb6dcfb92c4e31863a2 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Thu, 8 Aug 2024 10:43:00 +0000 Subject: [PATCH 069/102] build: add `pytest-mock` to dev dependencies --- poetry.lock | 19 ++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 504e515..b32eeca 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1292,6 +1292,23 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1921,4 +1938,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "40096a30f5c8ed7a5daaaaa5da50d0872d3ae7ab77e08256087fb8e44fb8a779" +content-hash = "ddf04336d74e92f046fa216cb8c28f649f928da0dece3d9a193b7b93e9880631" diff --git a/pyproject.toml b/pyproject.toml index 57348e4..d9b8979 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ single-version = "^1.6.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" pytest-cov = "^5.0.0" +pytest-mock = "^3.14.0" pyinstaller = "^6.5.0" pre-commit = "^3.8.0" ruff = "^0.5.4" From f7e9fd185079f9a07856dc51f183f112e4d545f0 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Thu, 8 Aug 2024 16:40:29 +0000 Subject: [PATCH 070/102] test: add tests for `FetchBinary` class --- src/imap_mag/cli/fetchBinary.py | 27 +++++---- tests/test_fetchBinary.py | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 tests/test_fetchBinary.py diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py index a450180..ebf55f3 100644 --- a/src/imap_mag/cli/fetchBinary.py +++ b/src/imap_mag/cli/fetchBinary.py @@ -58,21 +58,22 @@ def download_binaries( ] for d in range(len(dates) - 1): - file = self.__web_poda.download( + file: Path = self.__web_poda.download( packet=options["packet"], start_date=dates[d], end_date=dates[d + 1] ) - if self.__output_manager is not None: - self.__output_manager.add_default_file( - file, - descriptor=options["packet"] - .lower() - .strip("mag_") - .replace("_", "-"), - date=dates[d], - extension="pkts", - ) - - downloaded += [file] + if file.stat().st_size > 0: + if self.__output_manager is not None: + self.__output_manager.add_default_file( + file, + descriptor=options["packet"] + .lower() + .strip("mag_") + .replace("_", "-"), + date=dates[d], + extension="pkts", + ) + + downloaded += [file] return downloaded diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py new file mode 100644 index 0000000..2efad0f --- /dev/null +++ b/tests/test_fetchBinary.py @@ -0,0 +1,101 @@ +"""Tests for `OutputManager` class.""" + +import os +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchBinary import FetchBinary +from imap_mag.client.webPODA import IWebPODA +from imap_mag.outputManager import IOutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def create_file(file_path: Path, content: str | None) -> Path: + """Create a file with the given content.""" + + file_path.parent.mkdir(parents=True, exist_ok=True) + + if file_path.exists(): + os.remove(file_path) + + with open(file_path, "w") as file: + if content is not None: + file.write(content) + + return file_path + + +@pytest.fixture +def mock_poda() -> mock.Mock: + """Fixture for a mock IWebPODA instance.""" + return mock.create_autospec(IWebPODA, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_binary_empty_download_not_added_to_output( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_file(test_file, None) + + # Exercise. + actual_downloaded: Path = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + assert not mock_output_manager.add_default_file.called + assert actual_downloaded == [] + + +def test_fetch_binary_with_same_start_end_date( + mock_poda: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchBinary = FetchBinary(mock_poda, mock_output_manager) + + test_file = Path(tempfile.gettempdir()) / "test_file" + mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") + + # Exercise. + actual_downloaded: Path = fetchBinary.download_binaries( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_poda.download.assert_called_once_with( + packet="MAG_HSK_PW", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 3), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + descriptor="hsk-pw", + date=datetime(2025, 5, 2), + extension="pkts", + ) + + assert actual_downloaded == [test_file] From 1a6178a0d706ad3f7c24873e0527ab0c9b1185ad Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 09:33:14 +0000 Subject: [PATCH 071/102] test: add unit tests for `FetchScience` --- src/imap_mag/cli/fetchScience.py | 7 +- tests/test_fetchBinary.py | 4 +- tests/test_fetchScience.py | 106 +++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/test_fetchScience.py diff --git a/src/imap_mag/cli/fetchScience.py b/src/imap_mag/cli/fetchScience.py index 6da8d93..9c4ad93 100644 --- a/src/imap_mag/cli/fetchScience.py +++ b/src/imap_mag/cli/fetchScience.py @@ -28,7 +28,6 @@ class FetchScienceOptions(typing.TypedDict): level: str start_date: datetime end_date: datetime - output_dir: str class FetchScience: @@ -44,8 +43,8 @@ def __init__( self, data_access: ISDCDataAccess, output_manager: IOutputManager | None = None, - modes: list[MAGMode] = ["norm", "burst"], - sensors: list[MAGSensor] = ["magi", "mago"], + modes: list[MAGMode] = [MAGMode.Normal, MAGMode.Burst], + sensors: list[MAGSensor] = [MAGSensor.IBS, MAGSensor.OBS], ) -> None: """Initialize SDC interface.""" @@ -73,7 +72,7 @@ def download_latest_science( for sensor in self.__sensor: file_details = self.__data_access.get_filename( level=options["level"], - descriptor=str(mode) + "-" + str(sensor), + descriptor=mode.value + "-" + sensor.value, start_date=date, end_date=date, version="latest", diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index 2efad0f..b68ae4f 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -51,7 +51,7 @@ def test_fetch_binary_empty_download_not_added_to_output( mock_poda.download.side_effect = lambda **_: create_file(test_file, None) # Exercise. - actual_downloaded: Path = fetchBinary.download_binaries( + actual_downloaded: list[Path] = fetchBinary.download_binaries( packet="MAG_HSK_PW", start_date=datetime(2025, 5, 2), end_date=datetime(2025, 5, 2), @@ -78,7 +78,7 @@ def test_fetch_binary_with_same_start_end_date( mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") # Exercise. - actual_downloaded: Path = fetchBinary.download_binaries( + actual_downloaded: list[Path] = fetchBinary.download_binaries( packet="MAG_HSK_PW", start_date=datetime(2025, 5, 2), end_date=datetime(2025, 5, 2), diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py new file mode 100644 index 0000000..d6177d6 --- /dev/null +++ b/tests/test_fetchScience.py @@ -0,0 +1,106 @@ +"""Tests for `OutputManager` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +from imap_mag.cli.fetchScience import FetchScience, MAGMode, MAGSensor +from imap_mag.client.sdcDataAccess import ISDCDataAccess +from imap_mag.outputManager import IOutputManager + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_soc() -> mock.Mock: + """Fixture for a mock ISDCDataAccess instance.""" + return mock.create_autospec(ISDCDataAccess, spec_set=True) + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +def test_fetch_science_no_matching_files( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + mock_soc.get_filename.side_effect = lambda **_: {} + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + + assert not mock_soc.download.called + assert not mock_output_manager.add_default_file.called + assert actual_downloaded == [] + + +def test_fetch_science_with_same_start_end_date( + mock_soc: mock.Mock, mock_output_manager: mock.Mock +) -> None: + # Set up. + fetchScience = FetchScience( + mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] + ) + + test_file = Path(tempfile.gettempdir()) / "test_file" + + mock_soc.get_filename.side_effect = lambda **_: [ + { + "file_path": test_file.absolute(), + "descriptor": "norm-mago", + } + ] + mock_soc.download.side_effect = lambda file_path: file_path + + # Exercise. + actual_downloaded: list[Path] = fetchScience.download_latest_science( + level="l1b", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + ) + + # Verify. + mock_soc.get_filename.assert_called_once_with( + level="l1b", + descriptor="norm-mago", + start_date=datetime(2025, 5, 2), + end_date=datetime(2025, 5, 2), + version="latest", + extension="cdf", + ) + mock_soc.download.assert_called_once_with( + test_file.absolute(), + ) + + mock_output_manager.add_default_file.assert_called_once_with( + test_file, + level="l1b", + descriptor="norm-mago", + date=datetime(2025, 5, 2), + extension="cdf", + ) + + assert actual_downloaded == [test_file] From b8737aaa55f2d261e9e6c31a84d8563d872c4592 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 09:40:14 +0000 Subject: [PATCH 072/102] chore: update lock file --- poetry.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index b32eeca..3c5ee99 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1471,29 +1471,29 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.5.6" +version = "0.5.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.6-py3-none-linux_armv6l.whl", hash = "sha256:a0ef5930799a05522985b9cec8290b185952f3fcd86c1772c3bdbd732667fdcd"}, - {file = "ruff-0.5.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b652dc14f6ef5d1552821e006f747802cc32d98d5509349e168f6bf0ee9f8f42"}, - {file = "ruff-0.5.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:80521b88d26a45e871f31e4b88938fd87db7011bb961d8afd2664982dfc3641a"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9bc8f328a9f1309ae80e4d392836e7dbc77303b38ed4a7112699e63d3b066ab"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d394940f61f7720ad371ddedf14722ee1d6250fd8d020f5ea5a86e7be217daf"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111a99cdb02f69ddb2571e2756e017a1496c2c3a2aeefe7b988ddab38b416d36"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e395daba77a79f6dc0d07311f94cc0560375ca20c06f354c7c99af3bf4560c5d"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c476acb43c3c51e3c614a2e878ee1589655fa02dab19fe2db0423a06d6a5b1b6"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2ff8003f5252fd68425fd53d27c1f08b201d7ed714bb31a55c9ac1d4c13e2eb"}, - {file = "ruff-0.5.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c94e084ba3eaa80c2172918c2ca2eb2230c3f15925f4ed8b6297260c6ef179ad"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1f77c1c3aa0669fb230b06fb24ffa3e879391a3ba3f15e3d633a752da5a3e670"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f908148c93c02873210a52cad75a6eda856b2cbb72250370ce3afef6fb99b1ed"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:563a7ae61ad284187d3071d9041c08019975693ff655438d8d4be26e492760bd"}, - {file = "ruff-0.5.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:94fe60869bfbf0521e04fd62b74cbca21cbc5beb67cbb75ab33fe8c174f54414"}, - {file = "ruff-0.5.6-py3-none-win32.whl", hash = "sha256:e6a584c1de6f8591c2570e171cc7ce482bb983d49c70ddf014393cd39e9dfaed"}, - {file = "ruff-0.5.6-py3-none-win_amd64.whl", hash = "sha256:d7fe7dccb1a89dc66785d7aa0ac283b2269712d8ed19c63af908fdccca5ccc1a"}, - {file = "ruff-0.5.6-py3-none-win_arm64.whl", hash = "sha256:57c6c0dd997b31b536bff49b9eee5ed3194d60605a4427f735eeb1f9c1b8d264"}, - {file = "ruff-0.5.6.tar.gz", hash = "sha256:07c9e3c2a8e1fe377dd460371c3462671a728c981c3205a5217291422209f642"}, + {file = "ruff-0.5.7-py3-none-linux_armv6l.whl", hash = "sha256:548992d342fc404ee2e15a242cdbea4f8e39a52f2e7752d0e4cbe88d2d2f416a"}, + {file = "ruff-0.5.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00cc8872331055ee017c4f1071a8a31ca0809ccc0657da1d154a1d2abac5c0be"}, + {file = "ruff-0.5.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eaf3d86a1fdac1aec8a3417a63587d93f906c678bb9ed0b796da7b59c1114a1e"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a01c34400097b06cf8a6e61b35d6d456d5bd1ae6961542de18ec81eaf33b4cb8"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcc8054f1a717e2213500edaddcf1dbb0abad40d98e1bd9d0ad364f75c763eea"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f70284e73f36558ef51602254451e50dd6cc479f8b6f8413a95fcb5db4a55fc"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a78ad870ae3c460394fc95437d43deb5c04b5c29297815a2a1de028903f19692"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9ccd078c66a8e419475174bfe60a69adb36ce04f8d4e91b006f1329d5cd44bcf"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e31c9bad4ebf8fdb77b59cae75814440731060a09a0e0077d559a556453acbb"}, + {file = "ruff-0.5.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d796327eed8e168164346b769dd9a27a70e0298d667b4ecee6877ce8095ec8e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4a09ea2c3f7778cc635e7f6edf57d566a8ee8f485f3c4454db7771efb692c499"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a36d8dcf55b3a3bc353270d544fb170d75d2dff41eba5df57b4e0b67a95bb64e"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9369c218f789eefbd1b8d82a8cf25017b523ac47d96b2f531eba73770971c9e5"}, + {file = "ruff-0.5.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b88ca3db7eb377eb24fb7c82840546fb7acef75af4a74bd36e9ceb37a890257e"}, + {file = "ruff-0.5.7-py3-none-win32.whl", hash = "sha256:33d61fc0e902198a3e55719f4be6b375b28f860b09c281e4bdbf783c0566576a"}, + {file = "ruff-0.5.7-py3-none-win_amd64.whl", hash = "sha256:083bbcbe6fadb93cd86709037acc510f86eed5a314203079df174c40bbbca6b3"}, + {file = "ruff-0.5.7-py3-none-win_arm64.whl", hash = "sha256:2dca26154ff9571995107221d0aeaad0e75a77b5a682d6236cf89a58c70b76f4"}, + {file = "ruff-0.5.7.tar.gz", hash = "sha256:8dfc0a458797f5d9fb622dd0efc52d796f23f0a1493a9527f4e49a550ae9a7e5"}, ] [[package]] From 8a9236da81c83b72e1d39c692e090568d2ad447f Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 13:39:27 +0000 Subject: [PATCH 073/102] task: add `hash` to database and make `path` a unique constraint --- ...1c45c37_added_version_hash_date_and_software_.py} | 12 ++++++++---- src/imap_db/model.py | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) rename src/imap_db/migrations/versions/{2024_08_07-77642de1867a_added_version_date_and_software_version_.py => 2024_08_09-669111c45c37_added_version_hash_date_and_software_.py} (68%) diff --git a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py similarity index 68% rename from src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py rename to src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py index ccb34bd..8cdc34d 100644 --- a/src/imap_db/migrations/versions/2024_08_07-77642de1867a_added_version_date_and_software_version_.py +++ b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py @@ -1,8 +1,8 @@ -"""Added version, date and software version columns +"""Added version, hash, date and software version columns -Revision ID: 77642de1867a +Revision ID: 669111c45c37 Revises: d0457f3e98c8 -Create Date: 2024-08-07 13:03:30.889623 +Create Date: 2024-08-09 13:35:21.578940 """ @@ -10,7 +10,7 @@ from alembic import op # revision identifiers, used by Alembic. -revision = "77642de1867a" +revision = "669111c45c37" down_revision = "d0457f3e98c8" branch_labels = None depends_on = None @@ -19,16 +19,20 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("files", sa.Column("version", sa.Integer(), nullable=False)) + op.add_column("files", sa.Column("hash", sa.String(length=64), nullable=False)) op.add_column("files", sa.Column("date", sa.DateTime(), nullable=False)) op.add_column( "files", sa.Column("software_version", sa.String(length=16), nullable=False) ) + op.create_unique_constraint(None, "files", ["path"]) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, "files", type_="unique") op.drop_column("files", "software_version") op.drop_column("files", "date") + op.drop_column("files", "hash") op.drop_column("files", "version") # ### end Alembic commands ### diff --git a/src/imap_db/model.py b/src/imap_db/model.py index c2be860..644f9a3 100644 --- a/src/imap_db/model.py +++ b/src/imap_db/model.py @@ -10,10 +10,12 @@ class Base(DeclarativeBase): class File(Base): __tablename__ = "files" + id: Mapped[int] = mapped_column(primary_key=True) name: Mapped[str] = mapped_column(String(128)) - path: Mapped[str] = mapped_column(String(256)) + path: Mapped[str] = mapped_column(String(256), unique=True) version: Mapped[int] = mapped_column(Integer()) + hash: Mapped[str] = mapped_column(String(64)) date: Mapped[datetime] = mapped_column(DateTime()) software_version: Mapped[str] = mapped_column(String(16)) From 56d8fc1bb9f1ad4692703aafb314eeb52eda2170 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 13:40:05 +0000 Subject: [PATCH 074/102] feat: update database handler to check for error cases --- src/imap_mag/DB.py | 87 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index b59788c..fee1d4e 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,16 +1,35 @@ +import abc +import hashlib +import logging import os from pathlib import Path +import typer from imap_db.model import File from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from src.imap_db.model import File -from src.imap_mag import __version__ -from src.imap_mag.outputManager import IMetadataProvider, IOutputManager +from imap_mag import __version__ +from imap_mag.outputManager import IMetadataProvider, IOutputManager -class DB: +class IDatabase(abc.ABC): + """Interface for database manager.""" + + def insert_file(self, file: File) -> None: + """Insert a file into the database.""" + self.insert_files([file]) + pass + + @abc.abstractmethod + def insert_files(self, files: list[File]) -> None: + """Insert a list of files into the database.""" + pass + + +class Database(IDatabase): + """Database manager.""" + def __init__(self, db_url=None): env_url = os.getenv("SQLALCHEMY_URL") if db_url is None and env_url is not None: @@ -21,11 +40,12 @@ def __init__(self, db_url=None): "No database URL provided. Consider setting SQLALCHEMY_URL environment variable." ) + # TODO: Check database is available + self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - def insert_file(self, file: File) -> None: - self.insert_files([file]) + logging.debug(f"Creating database with URL: {db_url}") def insert_files(self, files: list[File]) -> None: session = self.Session() @@ -51,31 +71,58 @@ def insert_files(self, files: list[File]) -> None: class DatabaseOutputManager(IOutputManager): - def __init__(self, output_manager: IOutputManager, db: DB | None = None): + """Decorator for adding files to database as well as output.""" + + __output_manager: IOutputManager + __database: IDatabase + + def __init__( + self, output_manager: IOutputManager, database: Database | None = None + ): """Initialize database and output manager.""" - self.output_manager = output_manager + self.__output_manager = output_manager - if db is None: - self.db = DB() + if database is None: + self.__database = Database() else: - self.db = db + self.__database = database def add_file( self, original_file: Path, metadata_provider: IMetadataProvider ) -> tuple[Path, IMetadataProvider]: - (destination_file, metadata_provider) = self.output_manager.add_file( + (destination_file, metadata_provider) = self.__output_manager.add_file( original_file, metadata_provider ) - self.db.insert_file( - File( - name=destination_file.name, - path=destination_file.absolute().as_posix(), - version=metadata_provider.version, - date=metadata_provider.date, - software_version=__version__, + file_hash: str = hashlib.md5(original_file.read_bytes()).hexdigest() + + if not ( + destination_file.exists() + and (hashlib.md5(destination_file.read_bytes()).hexdigest() == file_hash) + ): + logging.error( + f"File {destination_file} does not exist or is not the same as original {original_file}." ) - ) + destination_file.unlink(missing_ok=True) + typer.Abort() + + logging.info(f"Inserting {destination_file} into database.") + + try: + self.__database.insert_file( + File( + name=destination_file.name, + path=destination_file.absolute().as_posix(), + version=metadata_provider.version, + hash=file_hash, + date=metadata_provider.date, + software_version=__version__, + ) + ) + except Exception as e: + logging.error(f"Error inserting {destination_file} into database: {e}") + destination_file.unlink() + raise e return (destination_file, metadata_provider) From a366cbc89420aadf919754e5e1e66245514f9b2b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 14:58:22 +0000 Subject: [PATCH 075/102] fix: remove logging of URL --- src/imap_mag/DB.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index fee1d4e..705c85c 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -45,8 +45,6 @@ def __init__(self, db_url=None): self.engine = create_engine(db_url) self.Session = sessionmaker(bind=self.engine) - logging.debug(f"Creating database with URL: {db_url}") - def insert_files(self, files: list[File]) -> None: session = self.Session() try: From 97ff8f26945fb67c29148201d59e2f7d61612e5c Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 16:30:25 +0000 Subject: [PATCH 076/102] test: move `create_test_file` to utility and use `assert_not_called` for mocks --- tests/testUtils.py | 14 ++++++++++++++ tests/test_fetchBinary.py | 25 +++++-------------------- tests/test_fetchScience.py | 5 +++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/testUtils.py b/tests/testUtils.py index 2b16cfe..3f2e56b 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -65,3 +65,17 @@ def create_serialize_config( yaml.dump(config.model_dump(by_alias=True), f) return (config, config_file) + + +def create_test_file(file_path: Path, content: str | None) -> Path: + """Create a file with the given content.""" + + file_path.unlink(missing_ok=True) + file_path.parent.mkdir(parents=True, exist_ok=True) + + file_path.touch() + + if content is not None: + file_path.write_text(content) + + return file_path diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index b68ae4f..cda2146 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -1,6 +1,5 @@ """Tests for `OutputManager` class.""" -import os import tempfile from datetime import datetime from pathlib import Path @@ -11,22 +10,7 @@ from imap_mag.client.webPODA import IWebPODA from imap_mag.outputManager import IOutputManager -from .testUtils import enableLogging, tidyDataFolders # noqa: F401 - - -def create_file(file_path: Path, content: str | None) -> Path: - """Create a file with the given content.""" - - file_path.parent.mkdir(parents=True, exist_ok=True) - - if file_path.exists(): - os.remove(file_path) - - with open(file_path, "w") as file: - if content is not None: - file.write(content) - - return file_path +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 @pytest.fixture @@ -48,7 +32,7 @@ def test_fetch_binary_empty_download_not_added_to_output( fetchBinary = FetchBinary(mock_poda, mock_output_manager) test_file = Path(tempfile.gettempdir()) / "test_file" - mock_poda.download.side_effect = lambda **_: create_file(test_file, None) + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, None) # Exercise. actual_downloaded: list[Path] = fetchBinary.download_binaries( @@ -64,7 +48,8 @@ def test_fetch_binary_empty_download_not_added_to_output( end_date=datetime(2025, 5, 3), ) - assert not mock_output_manager.add_default_file.called + mock_output_manager.add_default_file.assert_not_called() + assert actual_downloaded == [] @@ -75,7 +60,7 @@ def test_fetch_binary_with_same_start_end_date( fetchBinary = FetchBinary(mock_poda, mock_output_manager) test_file = Path(tempfile.gettempdir()) / "test_file" - mock_poda.download.side_effect = lambda **_: create_file(test_file, "content") + mock_poda.download.side_effect = lambda **_: create_test_file(test_file, "content") # Exercise. actual_downloaded: list[Path] = fetchBinary.download_binaries( diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py index d6177d6..a9f7a52 100644 --- a/tests/test_fetchScience.py +++ b/tests/test_fetchScience.py @@ -52,8 +52,9 @@ def test_fetch_science_no_matching_files( extension="cdf", ) - assert not mock_soc.download.called - assert not mock_output_manager.add_default_file.called + mock_soc.download.assert_not_called() + mock_output_manager.add_default_file.assert_not_called() + assert actual_downloaded == [] From c22489ba515b3fe23a2a896dc213188a4a7f9caa Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 9 Aug 2024 16:30:52 +0000 Subject: [PATCH 077/102] test: add some tests for database output manager --- src/imap_mag/DB.py | 2 +- tests/test_database.py | 118 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/test_database.py diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 705c85c..c0f7d98 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -103,7 +103,7 @@ def add_file( f"File {destination_file} does not exist or is not the same as original {original_file}." ) destination_file.unlink(missing_ok=True) - typer.Abort() + raise typer.Abort() logging.info(f"Inserting {destination_file} into database.") diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..d4e967e --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,118 @@ +"""Tests for `OutputManager` class.""" + +import tempfile +from datetime import datetime +from pathlib import Path +from unittest import mock + +import pytest +import typer +from imap_mag.DB import DatabaseOutputManager, IDatabase +from imap_mag.outputManager import DefaultMetadataProvider, IOutputManager + +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 + + +@pytest.fixture +def mock_output_manager() -> mock.Mock: + """Fixture for a mock IOutputManager instance.""" + return mock.create_autospec(IOutputManager, spec_set=True) + + +@pytest.fixture +def mock_database() -> mock.Mock: + """Fixture for a mock IDatabase instance.""" + return mock.create_autospec(IDatabase, spec_set=True) + + +def test_database_output_manager_writes_to_database( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + # Exercise. + (actual_file, actual_metadata_provider) = database_manager.add_file( + original_file, metadata_provider + ) + + # Verify. + mock_output_manager.add_file.assert_called_once_with( + original_file, metadata_provider + ) + # mock_database.insert_file.assert_called_once_with( + # File( + # name="test_file.txt", + # path=test_file.absolute().as_posix(), + # version=1, + # hash=hashlib.md5(b"some content").hexdigest(), + # date=datetime(2025, 5, 2), + # software_version=__version__, + # ) + # ) + + assert actual_file == test_file + assert actual_metadata_provider == metadata_provider + + +def test_database_output_manager_errors_destination_file_not_found( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + test_file.unlink(missing_ok=True) + + mock_output_manager.add_file.side_effect = lambda *_: ( + test_file, + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_destination_file_different_hash( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some other content"), + metadata_provider, + ) + + # Exercise and verify. + with pytest.raises(typer.Abort): + database_manager.add_file(original_file, metadata_provider) From 5dbf6dd400f5c5c0f6369f5e58d5086662b71c7b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 08:58:22 +0000 Subject: [PATCH 078/102] test: add coverage for database error --- tests/test_database.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_database.py b/tests/test_database.py index d4e967e..884fcf8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -116,3 +116,29 @@ def test_database_output_manager_errors_destination_file_different_hash( # Exercise and verify. with pytest.raises(typer.Abort): database_manager.add_file(original_file, metadata_provider) + + +def test_database_output_manager_errors_database_error( + mock_output_manager: mock.Mock, mock_database: mock.Mock +) -> None: + # Set up. + database_manager = DatabaseOutputManager(mock_output_manager, mock_database) + + original_file = create_test_file( + Path(tempfile.gettempdir()) / "some_file", "some content" + ) + metadata_provider = DefaultMetadataProvider( + version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" + ) + + test_file = Path(tempfile.gettempdir()) / "test_file.txt" + mock_output_manager.add_file.side_effect = lambda *_: ( + create_test_file(test_file, "some content"), + metadata_provider, + ) + + mock_database.insert_file.side_effect = ArithmeticError("Database error") + + # Exercise and verify. + with pytest.raises(ArithmeticError): + database_manager.add_file(original_file, metadata_provider) From 490a515cadb8ff4835a81926a198ba966300f58c Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:05:01 +0000 Subject: [PATCH 079/102] build: change `imap-mag` version to initial version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d9b8979..79e4c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] requires-python = ">=3.10" name = "imap-mag" -version = "0.0.1" +version = "0.1.0" [tool.poetry] name = "imap-mag" From d4ac011488d478cff291438282cda155c6d5b849 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:13:41 +0000 Subject: [PATCH 080/102] test(fix): remove commented out text --- tests/test_database.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 884fcf8..3d2fae8 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,5 +1,6 @@ """Tests for `OutputManager` class.""" +import hashlib import tempfile from datetime import datetime from pathlib import Path @@ -7,6 +8,8 @@ import pytest import typer +from imap_db.model import File +from imap_mag import __version__ from imap_mag.DB import DatabaseOutputManager, IDatabase from imap_mag.outputManager import DefaultMetadataProvider, IOutputManager @@ -44,6 +47,17 @@ def test_database_output_manager_writes_to_database( metadata_provider, ) + def check_inserted_file(file: File): + # Two instances of `File` will never be equal, so we check the attributes. + assert file.name == "test_file.txt" + assert file.path == test_file.absolute().as_posix() + assert file.version == 1 + assert file.hash == hashlib.md5(b"some content").hexdigest() + assert file.date == datetime(2025, 5, 2) + assert file.software_version == __version__ + + mock_database.insert_file.side_effect = lambda file: check_inserted_file(file) + # Exercise. (actual_file, actual_metadata_provider) = database_manager.add_file( original_file, metadata_provider @@ -53,16 +67,6 @@ def test_database_output_manager_writes_to_database( mock_output_manager.add_file.assert_called_once_with( original_file, metadata_provider ) - # mock_database.insert_file.assert_called_once_with( - # File( - # name="test_file.txt", - # path=test_file.absolute().as_posix(), - # version=1, - # hash=hashlib.md5(b"some content").hexdigest(), - # date=datetime(2025, 5, 2), - # software_version=__version__, - # ) - # ) assert actual_file == test_file assert actual_metadata_provider == metadata_provider From e4b545d2e5aeede225d2f877cd00072b45f70912 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:18:14 +0000 Subject: [PATCH 081/102] fix: temporarely use custom version of imap-data-access to avoid issue with `version=latest` argument --- poetry.lock | 13 +++++++++---- pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3c5ee99..27c8fb6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,14 +603,19 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [ - {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, -] +files = [] +develop = false [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] +[package.source] +type = "git" +url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" +reference = "HEAD" +resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" + [[package]] name = "importlib-resources" version = "5.13.0" @@ -1938,4 +1943,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "ddf04336d74e92f046fa216cb8c28f649f928da0dece3d9a193b7b93e9880631" +content-hash = "2646920291fe7954ce23f3b4b16e68b0a0b96daed4600ac03766e248d5480dcc" diff --git a/pyproject.toml b/pyproject.toml index 79e4c1e..5c1e1f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.8.0" +imap-data-access = { git = "https://github.com/ImperialCollegeLondon/imap-data-access.git" } # TODO: replace with imap-data-access > 0.8.0 cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} single-version = "^1.6.0" From 7114e0cb895e0d1ce225f101cbf01b49451687b4 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:33:33 +0000 Subject: [PATCH 082/102] Revert "fix: temporarely use custom version of imap-data-access to avoid issue with `version=latest` argument" This reverts commit bddeea88b8f405ad56ad0be99054ffa616551c36. --- poetry.lock | 13 ++++--------- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index 27c8fb6..87909d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,19 +603,14 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [] -develop = false +files = [ + {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, +] [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] -[package.source] -type = "git" -url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" -reference = "HEAD" -resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" - [[package]] name = "importlib-resources" version = "5.13.0" @@ -1943,4 +1938,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "2646920291fe7954ce23f3b4b16e68b0a0b96daed4600ac03766e248d5480dcc" +content-hash = "05034a7ae56f6ac95b6728ffa0c1af41cd021338cb3171e71ef0c67d376b6c19" diff --git a/pyproject.toml b/pyproject.toml index 5c1e1f5..79e4c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = { git = "https://github.com/ImperialCollegeLondon/imap-data-access.git" } # TODO: replace with imap-data-access > 0.8.0 +imap-data-access = "^0.8.0" cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} single-version = "^1.6.0" From f786a875804920e37511c69d77c52cde07665caa Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:39:54 +0000 Subject: [PATCH 083/102] build: add coverage report to CI action --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c82f620..054e69c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,6 +150,13 @@ jobs: path: 'test-results.xml' reporter: java-junit + - name: Coverage Report + uses: 5monkeys/cobertura-action@master + with: + report_name: Coverage Report (${{ matrix.python-versions }}) + path: "coverage.xml" + minimum_coverage: 80 + - name: Create Release ${{github.ref_name}} & upload artifacts uses: softprops/action-gh-release@v2 if: ${{ startsWith(github.ref, 'refs/tags/') }} From e9694c9f0e67ca00071df81ab93a809887d97b32 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:40:56 +0000 Subject: [PATCH 084/102] test: skip failing test due to bug in `imap-data-access` --- tests/test_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_main.py b/tests/test_main.py index e435f4d..6e36f5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,6 +7,7 @@ import re from pathlib import Path +import pytest from imap_mag.main import app from typer.testing import CliRunner From 3a82f360005d97d400e977627bcdd774f01f9583 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 09:53:14 +0000 Subject: [PATCH 085/102] build(fix): use version instead of branch --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 054e69c..bafe0c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,7 +151,7 @@ jobs: reporter: java-junit - name: Coverage Report - uses: 5monkeys/cobertura-action@master + uses: 5monkeys/cobertura-action@v14 with: report_name: Coverage Report (${{ matrix.python-versions }}) path: "coverage.xml" From 6c368f4eab6cb4a6e513ea84fa100ddebed89f3b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:29:44 +0000 Subject: [PATCH 086/102] test(fix): name of classes under test in comment --- tests/test_database.py | 2 +- tests/test_fetchBinary.py | 2 +- tests/test_fetchScience.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_database.py b/tests/test_database.py index 3d2fae8..56d9ad3 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for database classes.""" import hashlib import tempfile diff --git a/tests/test_fetchBinary.py b/tests/test_fetchBinary.py index cda2146..5c149a7 100644 --- a/tests/test_fetchBinary.py +++ b/tests/test_fetchBinary.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for `FetchBinary` class.""" import tempfile from datetime import datetime diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py index a9f7a52..65d91d8 100644 --- a/tests/test_fetchScience.py +++ b/tests/test_fetchScience.py @@ -1,4 +1,4 @@ -"""Tests for `OutputManager` class.""" +"""Tests for `FetchScience` class.""" import tempfile from datetime import datetime From 01360fb17e4062ad62d778d1b3979e7b7c4c0c81 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:54:23 +0000 Subject: [PATCH 087/102] test: add more tests to increase coverage --- src/imap_mag/client/sdcDataAccess.py | 16 +++++------ src/imap_mag/main.py | 2 +- tests/test_appUtils.py | 17 +++++++++++ tests/test_sdcDataAccess.py | 43 ++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 tests/test_appUtils.py create mode 100644 tests/test_sdcDataAccess.py diff --git a/src/imap_mag/client/sdcDataAccess.py b/src/imap_mag/client/sdcDataAccess.py index 88d277d..cafd3d3 100644 --- a/src/imap_mag/client/sdcDataAccess.py +++ b/src/imap_mag/client/sdcDataAccess.py @@ -2,9 +2,9 @@ import abc import logging -import pathlib import typing from datetime import datetime +from pathlib import Path import imap_data_access import typing_extensions @@ -44,7 +44,7 @@ class ISDCDataAccess(abc.ABC): @abc.abstractmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: """Get file path for data from imap-data-access.""" pass @@ -68,7 +68,7 @@ def get_filename( pass @abc.abstractmethod - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: """Download data from imap-data-access.""" pass @@ -76,10 +76,10 @@ def download(self, file_name: str) -> pathlib.Path: class SDCDataAccess(ISDCDataAccess): """Class for uploading and downloading MAG data via imap-data-access.""" - def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: + def __init__(self, data_dir: Path, sdc_url: str | None = None) -> None: """Initialize SDC API client.""" - imap_data_access.config["DATA_DIR"] = pathlib.Path(data_dir) + imap_data_access.config["DATA_DIR"] = data_dir imap_data_access.config["DATA_ACCESS_URL"] = ( sdc_url or "https://api.dev.imap-mission.com" ) @@ -87,7 +87,7 @@ def __init__(self, data_dir: str, sdc_url: str | None = None) -> None: @staticmethod def get_file_path( **options: typing_extensions.Unpack[FileOptions], - ) -> tuple[str, str]: + ) -> tuple[Path, Path]: science_file = imap_data_access.ScienceFilePath.generate_from_inputs( instrument="mag", data_level=options["level"], @@ -136,6 +136,6 @@ def get_filename( return file_details - def download(self, file_name: str) -> pathlib.Path: + def download(self, file_name: str) -> Path: logging.debug(f"Downloading {file_name} from imap-data-access.") - return pathlib.Path(imap_data_access.download(file_name)) + return imap_data_access.download(file_name) diff --git a/src/imap_mag/main.py b/src/imap_mag/main.py index 1fa375d..4dfb7a2 100644 --- a/src/imap_mag/main.py +++ b/src/imap_mag/main.py @@ -247,7 +247,7 @@ def fetch_science( logging.info(f"Downloading {level} science from {start_date} to {end_date}.") data_access = SDCDataAccess( - data_dir=str(configFile.work_folder), + data_dir=configFile.work_folder, sdc_url=configFile.api.sdc_url if configFile.api else None, ) output_manager = appUtils.getOutputManager(configFile.destination) diff --git a/tests/test_appUtils.py b/tests/test_appUtils.py new file mode 100644 index 0000000..bf0cd68 --- /dev/null +++ b/tests/test_appUtils.py @@ -0,0 +1,17 @@ +"""Tests for app utilities.""" + +import pytest +import typer +from imap_mag.appUtils import getPacketFromApID + +from .testUtils import enableLogging, tidyDataFolders # noqa: F401 + + +def test_get_packet_from_apid_errors_on_invalid_apid() -> None: + with pytest.raises(typer.Abort): + getPacketFromApID(12345) + + +def test_convert_to_datetime_on_invalid_datetime() -> None: + with pytest.raises(typer.Abort): + getPacketFromApID("ABCDEF") diff --git a/tests/test_sdcDataAccess.py b/tests/test_sdcDataAccess.py new file mode 100644 index 0000000..fe30835 --- /dev/null +++ b/tests/test_sdcDataAccess.py @@ -0,0 +1,43 @@ +"""Tests for `SDCDataAccess` class.""" + +import os +from datetime import datetime +from pathlib import Path + +import imap_data_access +from imap_mag.client.sdcDataAccess import SDCDataAccess + +from .testUtils import create_serialize_config, tidyDataFolders # noqa: F401 +from .wiremockUtils import wiremock_manager # noqa: F401 + + +def test_sdc_data_access_constructor_sets_config() -> None: + # Set up. + data_dir = "some_test_folder" + data_access_url = "https://some_test_url" + + # Exercise. + _ = SDCDataAccess(data_dir, data_access_url) + + # Verify. + assert imap_data_access.config["DATA_DIR"] == data_dir + assert imap_data_access.config["DATA_ACCESS_URL"] == data_access_url + + +def test_get_file_path_builds_file_path() -> None: + # Set up. + data_access = SDCDataAccess("some_test_folder") + + # Exercise. + (file_name, file_path) = data_access.get_file_path( + level="l1b", + descriptor="norm-magi", + start_date=datetime(2025, 5, 2), + version="v002", + ) + + # Verify. + assert file_name == Path("imap_mag_l1b_norm-magi_20250502_v002.cdf") + assert file_path == Path( + os.path.join("some_test_folder", "imap", "mag", "l1b", "2025", "05", file_name) + ) From 0a0a4556f689c2a9e3de6d514c7fd5f1129c37f3 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Mon, 12 Aug 2024 14:57:53 +0000 Subject: [PATCH 088/102] test(fix): actually use `convertToDatetime` --- tests/test_appUtils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_appUtils.py b/tests/test_appUtils.py index bf0cd68..71cf2fc 100644 --- a/tests/test_appUtils.py +++ b/tests/test_appUtils.py @@ -2,7 +2,7 @@ import pytest import typer -from imap_mag.appUtils import getPacketFromApID +from imap_mag.appUtils import convertToDatetime, getPacketFromApID from .testUtils import enableLogging, tidyDataFolders # noqa: F401 @@ -14,4 +14,4 @@ def test_get_packet_from_apid_errors_on_invalid_apid() -> None: def test_convert_to_datetime_on_invalid_datetime() -> None: with pytest.raises(typer.Abort): - getPacketFromApID("ABCDEF") + convertToDatetime("ABCDEF") From 91f0c346fe961cf0d22aa31ba7071646cf8e160f Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 09:12:04 +0000 Subject: [PATCH 089/102] fix: CI action failure, packaging failure, test failure --- .github/workflows/ci.yml | 4 ++++ poetry.lock | 36 ++++++++++++++++++++++-------------- pyproject.toml | 2 +- tests/test_main.py | 1 - 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bafe0c6..0d71a51 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,10 @@ jobs: python-versions: ['3.10', '3.11', '3.12'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} + permissions: + contents: read + actions: read + checks: write # map step outputs to job outputs so they can be share among jobs outputs: package_version: ${{ env.PACKAGE_VERSION }} diff --git a/poetry.lock b/poetry.lock index 87909d0..13f3a55 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,14 +603,19 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [ - {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, -] +files = [] +develop = false [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] +[package.source] +type = "git" +url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" +reference = "main" +resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" + [[package]] name = "importlib-resources" version = "5.13.0" @@ -1359,13 +1364,13 @@ files = [ [[package]] name = "pywin32-ctypes" -version = "0.2.2" +version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" files = [ - {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, - {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, ] [[package]] @@ -1498,18 +1503,18 @@ files = [ [[package]] name = "setuptools" -version = "72.1.0" +version = "72.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, + {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, ] [package.extras] core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -1676,13 +1681,13 @@ url = ["furl (>=0.4.1)"] [[package]] name = "testcontainers" -version = "4.7.2" +version = "4.8.0" description = "Python library for throwaway instances of anything that can run in a Docker container" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "testcontainers-4.7.2-py3-none-any.whl", hash = "sha256:23b13cf8078f615a08c75197f227796d90c46df92d2b282ae7c39b1fc1a9c9ed"}, - {file = "testcontainers-4.7.2.tar.gz", hash = "sha256:9976b1cdcdeb9feeae6a477073e7c8b02cd40ea44f1daa34b5da6d2c918dff0d"}, + {file = "testcontainers-4.8.0-py3-none-any.whl", hash = "sha256:0b85d787e5b1f8b32042704d23b6c54787bf6751d2d3cfee2c031349ef2eea30"}, + {file = "testcontainers-4.8.0.tar.gz", hash = "sha256:56153bb5938694844f0e6bd0cf82e19dd6a6516bc29881440e273939201a42d5"}, ] [package.dependencies] @@ -1693,10 +1698,12 @@ wrapt = "*" [package.extras] arangodb = ["python-arango (>=7.8,<8.0)"] +aws = ["boto3", "httpx"] azurite = ["azure-storage-blob (>=12.19,<13.0)"] chroma = ["chromadb-client"] clickhouse = ["clickhouse-driver"] cosmosdb = ["azure-cosmos"] +db2 = ["ibm_db_sa", "sqlalchemy"] generic = ["httpx"] google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"] influxdb = ["influxdb", "influxdb-client"] @@ -1717,6 +1724,7 @@ qdrant = ["qdrant-client"] rabbitmq = ["pika"] redis = ["redis"] registry = ["bcrypt"] +scylla = ["cassandra-driver (==3.29.1)"] selenium = ["selenium"] sftp = ["cryptography"] test-module-import = ["httpx"] @@ -1938,4 +1946,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "05034a7ae56f6ac95b6728ffa0c1af41cd021338cb3171e71ef0c67d376b6c19" +content-hash = "10329b21c3e5c533981a46614c44553a7d5539133745209806cbd2fca5899149" diff --git a/pyproject.toml b/pyproject.toml index 79e4c1e..ed4dfcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.8.0" +imap-data-access = {git = "https://github.com/ImperialCollegeLondon/imap-data-access.git", rev = "main"} cdflib = "^1.3.1" psycopg = {extras = ["binary"], version = "^3.2.1"} single-version = "^1.6.0" diff --git a/tests/test_main.py b/tests/test_main.py index 6e36f5f..e435f4d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,7 +7,6 @@ import re from pathlib import Path -import pytest from imap_mag.main import app from typer.testing import CliRunner From ea59ccb2097291fd164c4f1b8673604bcc6f9dea Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 09:40:13 +0000 Subject: [PATCH 090/102] fix: more fixes to permissions for action and packaging --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d71a51..5cd39a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,7 @@ jobs: os: [ubuntu-latest] runs-on: ${{ matrix.os }} permissions: - contents: read - actions: read - checks: write + pull-requests: write # map step outputs to job outputs so they can be share among jobs outputs: package_version: ${{ env.PACKAGE_VERSION }} From 693cd49f54b5b0ceb8584d2c62efa6330743c814 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 10:18:54 +0000 Subject: [PATCH 091/102] Squashed commit of the following: commit ee06eb90cca2bd84988c5294ab91167d40bc8568 Author: Michele Facchinelli Date: Fri Aug 16 09:53:09 2024 +0000 fix: add packages permission commit d0bac86adbeca2ef394d24c4356bbee1b68ae40a Author: Michele Facchinelli Date: Fri Aug 16 09:47:17 2024 +0000 fix: add checks permissions commit 80b3b92677b91dbd75e5775882a476402948fd45 Author: Michele Facchinelli Date: Fri Aug 16 09:44:44 2024 +0000 fix: permissions for CI actions --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cd39a3..a03ebb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,9 @@ jobs: os: [ubuntu-latest] runs-on: ${{ matrix.os }} permissions: + actions: write + checks: write + packages: write pull-requests: write # map step outputs to job outputs so they can be share among jobs outputs: From 9c86b45c94c2af602f771389fb344ac4b4693977 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 10:38:05 +0000 Subject: [PATCH 092/102] fix: merge issues --- poetry.lock | 2 +- pyproject.toml | 1 - tests/test_main.py | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4bc105e..98b14f5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1946,4 +1946,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "10329b21c3e5c533981a46614c44553a7d5539133745209806cbd2fca5899149" +content-hash = "dc458336c8a392c38e1974b8e013d3809e450ccfd8670193eff062938510c8e2" diff --git a/pyproject.toml b/pyproject.toml index 2ff4bf2..06ef9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ imap-data-access = {git = "https://github.com/ImperialCollegeLondon/imap-data-ac cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} -single-version = "^1.6.0" [tool.poetry.group.dev.dependencies] pytest = "^8.3.1" diff --git a/tests/test_main.py b/tests/test_main.py index e435f4d..6e36f5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,6 +7,7 @@ import re from pathlib import Path +import pytest from imap_mag.main import app from typer.testing import CliRunner From 36d8ecf951d88288776ebc7212af70db16fcef7e Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 10:45:18 +0000 Subject: [PATCH 093/102] build(fix): use existing permissions --- .github/workflows/ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a03ebb4..99a5d5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ permissions: contents: write checks: write packages: write + pull-requests: write env: PREFERED_PYTHON_VERSION: '3.12' @@ -38,11 +39,6 @@ jobs: python-versions: ['3.10', '3.11', '3.12'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} - permissions: - actions: write - checks: write - packages: write - pull-requests: write # map step outputs to job outputs so they can be share among jobs outputs: package_version: ${{ env.PACKAGE_VERSION }} From 6218f99a61b3bf7b6a333fb28c19a9644366e4e3 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 12:52:16 +0000 Subject: [PATCH 094/102] fix: install git on Docker image --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a5d5b..a42bd5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,6 +260,7 @@ jobs: image: ${{ matrix.image }} options: -v ${{ github.workspace }}:/src/ run: | + apt install git-all python -m pip install poetry python -m poetry self add poetry-pyinstaller-plugin python -m poetry install From 41decd0f9aa86d9dc0ce8ac26f6663924eddb32b Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 13:00:23 +0000 Subject: [PATCH 095/102] try: use git under wine --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a42bd5e..1a0b784 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,7 +260,7 @@ jobs: image: ${{ matrix.image }} options: -v ${{ github.workspace }}:/src/ run: | - apt install git-all + echo 'wine '\''C:\Program Files\Git\bin\git.exe'\'' "$@"' > /usr/bin/git python -m pip install poetry python -m poetry self add poetry-pyinstaller-plugin python -m poetry install From 844213b015af7e645ab31efad893b206b4bf3d51 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 13:07:32 +0000 Subject: [PATCH 096/102] try: force Docker run action to finish --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a0b784..558b90c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,6 +255,7 @@ jobs: shell: bash - uses: addnab/docker-run-action@v3 + continue-on-error: true with: registry: gcr.io image: ${{ matrix.image }} From a041109488c39821692426589500f261988589c8 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 13:10:30 +0000 Subject: [PATCH 097/102] try: more tests to make git work on Docker --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 558b90c..ca26a21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,6 +208,7 @@ jobs: build_single_file_binary: strategy: + fail-fast: false matrix: os: [linux, windows] include: @@ -255,13 +256,15 @@ jobs: shell: bash - uses: addnab/docker-run-action@v3 - continue-on-error: true with: registry: gcr.io image: ${{ matrix.image }} options: -v ${{ github.workspace }}:/src/ run: | - echo 'wine '\''C:\Program Files\Git\bin\git.exe'\'' "$@"' > /usr/bin/git + if [[ "$OSTYPE" == "msys" ]] + then + echo 'wine '\''C:\Program Files\Git\bin\git.exe'\'' "$@"' > /usr/bin/git + fi python -m pip install poetry python -m poetry self add poetry-pyinstaller-plugin python -m poetry install From 37fa4162f1b535cdf2b65aa316092665c87d9d79 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 16 Aug 2024 14:15:18 +0000 Subject: [PATCH 098/102] fix: give up on git-based imap-data-access and skip failing test --- .github/workflows/ci.yml | 5 ----- poetry.lock | 13 ++++--------- pyproject.toml | 2 +- tests/test_main.py | 1 + 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca26a21..99a5d5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,7 +208,6 @@ jobs: build_single_file_binary: strategy: - fail-fast: false matrix: os: [linux, windows] include: @@ -261,10 +260,6 @@ jobs: image: ${{ matrix.image }} options: -v ${{ github.workspace }}:/src/ run: | - if [[ "$OSTYPE" == "msys" ]] - then - echo 'wine '\''C:\Program Files\Git\bin\git.exe'\'' "$@"' > /usr/bin/git - fi python -m pip install poetry python -m poetry self add poetry-pyinstaller-plugin python -m poetry install diff --git a/poetry.lock b/poetry.lock index 98b14f5..4b251e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -603,19 +603,14 @@ version = "0.8.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" -files = [] -develop = false +files = [ + {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, +] [package.extras] dev = ["imap_data_access[test]", "pre-commit", "ruff"] test = ["pytest", "pytest-cov"] -[package.source] -type = "git" -url = "https://github.com/ImperialCollegeLondon/imap-data-access.git" -reference = "main" -resolved_reference = "c792c2f6d92512f2be4de0f4f796b409d634f843" - [[package]] name = "importlib-resources" version = "5.13.0" @@ -1946,4 +1941,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "dc458336c8a392c38e1974b8e013d3809e450ccfd8670193eff062938510c8e2" +content-hash = "361d2fdb0c606fd9c3f47d61e2a1d9e8a89a022fd8673edeb85b0c912404eba8" diff --git a/pyproject.toml b/pyproject.toml index 06ef9ba..e31fb8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = {git = "https://github.com/ImperialCollegeLondon/imap-data-access.git", rev = "main"} +imap-data-access = "^0.8.0" cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} diff --git a/tests/test_main.py b/tests/test_main.py index 6e36f5f..ed12490 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -120,6 +120,7 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() +@pytest.mark.skip("Wait for `imap-data-access` to release version > 0.8.0.") @pytest.mark.skipif( os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner", From 1bcb16a753f2ce476a53934b95f8a3129a123f36 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Thu, 22 Aug 2024 15:45:58 +0000 Subject: [PATCH 099/102] fix: update to latest `imap-data-access` and unfilter test --- poetry.lock | 125 ++++++++++++++++++++++++--------------------- pyproject.toml | 2 +- tests/test_main.py | 1 - 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4b251e2..de6e7c6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -599,12 +599,12 @@ files = [ [[package]] name = "imap-data-access" -version = "0.8.0" +version = "0.9.0" description = "IMAP SDC Data Access" optional = false python-versions = "*" files = [ - {file = "imap_data_access-0.8.0.tar.gz", hash = "sha256:bfcb96a6dc7c724662272bd83f5ea89a32eda4c4524bc2b94e12ee166a4a1194"}, + {file = "imap_data_access-0.9.0.tar.gz", hash = "sha256:997084118c85455d1c977d5640a8654717a1f8adf39eaeccfc3e124b5efd3c4f"}, ] [package.extras] @@ -787,56 +787,63 @@ files = [ [[package]] name = "numpy" -version = "2.0.1" +version = "2.1.0" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" -files = [ - {file = "numpy-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fbb536eac80e27a2793ffd787895242b7f18ef792563d742c2d673bfcb75134"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:69ff563d43c69b1baba77af455dd0a839df8d25e8590e79c90fcbe1499ebde42"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:1b902ce0e0a5bb7704556a217c4f63a7974f8f43e090aff03fcf262e0b135e02"}, - {file = "numpy-2.0.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:f1659887361a7151f89e79b276ed8dff3d75877df906328f14d8bb40bb4f5101"}, - {file = "numpy-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4658c398d65d1b25e1760de3157011a80375da861709abd7cef3bad65d6543f9"}, - {file = "numpy-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4127d4303b9ac9f94ca0441138acead39928938660ca58329fe156f84b9f3015"}, - {file = "numpy-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e5eeca8067ad04bc8a2a8731183d51d7cbaac66d86085d5f4766ee6bf19c7f87"}, - {file = "numpy-2.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:9adbd9bb520c866e1bfd7e10e1880a1f7749f1f6e5017686a5fbb9b72cf69f82"}, - {file = "numpy-2.0.1-cp310-cp310-win32.whl", hash = "sha256:7b9853803278db3bdcc6cd5beca37815b133e9e77ff3d4733c247414e78eb8d1"}, - {file = "numpy-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:81b0893a39bc5b865b8bf89e9ad7807e16717f19868e9d234bdaf9b1f1393868"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75b4e316c5902d8163ef9d423b1c3f2f6252226d1aa5cd8a0a03a7d01ffc6268"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6e4eeb6eb2fced786e32e6d8df9e755ce5be920d17f7ce00bc38fcde8ccdbf9e"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:a1e01dcaab205fbece13c1410253a9eea1b1c9b61d237b6fa59bcc46e8e89343"}, - {file = "numpy-2.0.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a8fc2de81ad835d999113ddf87d1ea2b0f4704cbd947c948d2f5513deafe5a7b"}, - {file = "numpy-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a3d94942c331dd4e0e1147f7a8699a4aa47dffc11bf8a1523c12af8b2e91bbe"}, - {file = "numpy-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15eb4eca47d36ec3f78cde0a3a2ee24cf05ca7396ef808dda2c0ddad7c2bde67"}, - {file = "numpy-2.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b83e16a5511d1b1f8a88cbabb1a6f6a499f82c062a4251892d9ad5d609863fb7"}, - {file = "numpy-2.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f87fec1f9bc1efd23f4227becff04bd0e979e23ca50cc92ec88b38489db3b55"}, - {file = "numpy-2.0.1-cp311-cp311-win32.whl", hash = "sha256:36d3a9405fd7c511804dc56fc32974fa5533bdeb3cd1604d6b8ff1d292b819c4"}, - {file = "numpy-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:08458fbf403bff5e2b45f08eda195d4b0c9b35682311da5a5a0a0925b11b9bd8"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6bf4e6f4a2a2e26655717a1983ef6324f2664d7011f6ef7482e8c0b3d51e82ac"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7d6fddc5fe258d3328cd8e3d7d3e02234c5d70e01ebe377a6ab92adb14039cb4"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5daab361be6ddeb299a918a7c0864fa8618af66019138263247af405018b04e1"}, - {file = "numpy-2.0.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:ea2326a4dca88e4a274ba3a4405eb6c6467d3ffbd8c7d38632502eaae3820587"}, - {file = "numpy-2.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529af13c5f4b7a932fb0e1911d3a75da204eff023ee5e0e79c1751564221a5c8"}, - {file = "numpy-2.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6790654cb13eab303d8402354fabd47472b24635700f631f041bd0b65e37298a"}, - {file = "numpy-2.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cbab9fc9c391700e3e1287666dfd82d8666d10e69a6c4a09ab97574c0b7ee0a7"}, - {file = "numpy-2.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:99d0d92a5e3613c33a5f01db206a33f8fdf3d71f2912b0de1739894668b7a93b"}, - {file = "numpy-2.0.1-cp312-cp312-win32.whl", hash = "sha256:173a00b9995f73b79eb0191129f2455f1e34c203f559dd118636858cc452a1bf"}, - {file = "numpy-2.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:bb2124fdc6e62baae159ebcfa368708867eb56806804d005860b6007388df171"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfc085b28d62ff4009364e7ca34b80a9a080cbd97c2c0630bb5f7f770dae9414"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8fae4ebbf95a179c1156fab0b142b74e4ba4204c87bde8d3d8b6f9c34c5825ef"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:72dc22e9ec8f6eaa206deb1b1355eb2e253899d7347f5e2fae5f0af613741d06"}, - {file = "numpy-2.0.1-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:ec87f5f8aca726117a1c9b7083e7656a9d0d606eec7299cc067bb83d26f16e0c"}, - {file = "numpy-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f682ea61a88479d9498bf2091fdcd722b090724b08b31d63e022adc063bad59"}, - {file = "numpy-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8efc84f01c1cd7e34b3fb310183e72fcdf55293ee736d679b6d35b35d80bba26"}, - {file = "numpy-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3fdabe3e2a52bc4eff8dc7a5044342f8bd9f11ef0934fcd3289a788c0eb10018"}, - {file = "numpy-2.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:24a0e1befbfa14615b49ba9659d3d8818a0f4d8a1c5822af8696706fbda7310c"}, - {file = "numpy-2.0.1-cp39-cp39-win32.whl", hash = "sha256:f9cf5ea551aec449206954b075db819f52adc1638d46a6738253a712d553c7b4"}, - {file = "numpy-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:e9e81fa9017eaa416c056e5d9e71be93d05e2c3c2ab308d23307a8bc4443c368"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:61728fba1e464f789b11deb78a57805c70b2ed02343560456190d0501ba37b0f"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:12f5d865d60fb9734e60a60f1d5afa6d962d8d4467c120a1c0cda6eb2964437d"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eacf3291e263d5a67d8c1a581a8ebbcfd6447204ef58828caf69a5e3e8c75990"}, - {file = "numpy-2.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2c3a346ae20cfd80b6cfd3e60dc179963ef2ea58da5ec074fd3d9e7a1e7ba97f"}, - {file = "numpy-2.0.1.tar.gz", hash = "sha256:485b87235796410c3519a699cfe1faab097e509e90ebb05dcd098db2ae87e7b3"}, +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673"}, + {file = "numpy-2.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b"}, + {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b"}, + {file = "numpy-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f"}, + {file = "numpy-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84"}, + {file = "numpy-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33"}, + {file = "numpy-2.1.0-cp310-cp310-win32.whl", hash = "sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211"}, + {file = "numpy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1"}, + {file = "numpy-2.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e"}, + {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb"}, + {file = "numpy-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3"}, + {file = "numpy-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36"}, + {file = "numpy-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd"}, + {file = "numpy-2.1.0-cp311-cp311-win32.whl", hash = "sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e"}, + {file = "numpy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1"}, + {file = "numpy-2.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8"}, + {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745"}, + {file = "numpy-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111"}, + {file = "numpy-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0"}, + {file = "numpy-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574"}, + {file = "numpy-2.1.0-cp312-cp312-win32.whl", hash = "sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02"}, + {file = "numpy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8ab81ccd753859ab89e67199b9da62c543850f819993761c1e94a75a814ed667"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:442596f01913656d579309edcd179a2a2f9977d9a14ff41d042475280fc7f34e"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:848c6b5cad9898e4b9ef251b6f934fa34630371f2e916261070a4eb9092ffd33"}, + {file = "numpy-2.1.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:54c6a63e9d81efe64bfb7bcb0ec64332a87d0b87575f6009c8ba67ea6374770b"}, + {file = "numpy-2.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:652e92fc409e278abdd61e9505649e3938f6d04ce7ef1953f2ec598a50e7c195"}, + {file = "numpy-2.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ab32eb9170bf8ffcbb14f11613f4a0b108d3ffee0832457c5d4808233ba8977"}, + {file = "numpy-2.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:8fb49a0ba4d8f41198ae2d52118b050fd34dace4b8f3fb0ee34e23eb4ae775b1"}, + {file = "numpy-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:44e44973262dc3ae79e9063a1284a73e09d01b894b534a769732ccd46c28cc62"}, + {file = "numpy-2.1.0-cp313-cp313-win32.whl", hash = "sha256:ab83adc099ec62e044b1fbb3a05499fa1e99f6d53a1dde102b2d85eff66ed324"}, + {file = "numpy-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:de844aaa4815b78f6023832590d77da0e3b6805c644c33ce94a1e449f16d6ab5"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:343e3e152bf5a087511cd325e3b7ecfd5b92d369e80e74c12cd87826e263ec06"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f07fa2f15dabe91259828ce7d71b5ca9e2eb7c8c26baa822c825ce43552f4883"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5474dad8c86ee9ba9bb776f4b99ef2d41b3b8f4e0d199d4f7304728ed34d0300"}, + {file = "numpy-2.1.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1f817c71683fd1bb5cff1529a1d085a57f02ccd2ebc5cd2c566f9a01118e3b7d"}, + {file = "numpy-2.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a3336fbfa0d38d3deacd3fe7f3d07e13597f29c13abf4d15c3b6dc2291cbbdd"}, + {file = "numpy-2.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a894c51fd8c4e834f00ac742abad73fc485df1062f1b875661a3c1e1fb1c2f6"}, + {file = "numpy-2.1.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:9156ca1f79fc4acc226696e95bfcc2b486f165a6a59ebe22b2c1f82ab190384a"}, + {file = "numpy-2.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:624884b572dff8ca8f60fab591413f077471de64e376b17d291b19f56504b2bb"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575"}, + {file = "numpy-2.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2"}, + {file = "numpy-2.1.0.tar.gz", hash = "sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2"}, ] [[package]] @@ -1498,19 +1505,19 @@ files = [ [[package]] name = "setuptools" -version = "72.2.0" +version = "73.0.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.2.0-py3-none-any.whl", hash = "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4"}, - {file = "setuptools-72.2.0.tar.gz", hash = "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9"}, + {file = "setuptools-73.0.1-py3-none-any.whl", hash = "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e"}, + {file = "setuptools-73.0.1.tar.gz", hash = "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] [[package]] name = "shellingham" @@ -1739,13 +1746,13 @@ files = [ [[package]] name = "typer" -version = "0.12.3" +version = "0.12.4" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.7" files = [ - {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, - {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, + {file = "typer-0.12.4-py3-none-any.whl", hash = "sha256:819aa03699f438397e876aa12b0d63766864ecba1b579092cc9fe35d886e34b6"}, + {file = "typer-0.12.4.tar.gz", hash = "sha256:c9c1613ed6a166162705b3347b8d10b661ccc5d95692654d0fb628118f2c34e6"}, ] [package.dependencies] @@ -1941,4 +1948,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "361d2fdb0c606fd9c3f47d61e2a1d9e8a89a022fd8673edeb85b0c912404eba8" +content-hash = "5e61cad3f957d9dabe11d074137198dab507d4ef007b072912a21b6239e733f3" diff --git a/pyproject.toml b/pyproject.toml index e31fb8f..d1625ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ alembic = "^1.13.2" sqlalchemy-utils = "^0.41.2" requests = "^2.32.3" pandas = "^2.2.2" -imap-data-access = "^0.8.0" +imap-data-access = "^0.9.0" cdflib = "^1.3.1" single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} diff --git a/tests/test_main.py b/tests/test_main.py index ed12490..6e36f5f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -120,7 +120,6 @@ def test_fetch_binary_downloads_hk_from_webpoda(wiremock_manager): # noqa: F811 assert output.read() == input.read() -@pytest.mark.skip("Wait for `imap-data-access` to release version > 0.8.0.") @pytest.mark.skipif( os.getenv("GITHUB_ACTIONS") and os.getenv("RUNNER_OS") == "Windows", reason="Wiremock test containers will not work on Windows Github Runner", From 55e0838920d6e7e08503d48599a1b36808f58056 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 30 Aug 2024 13:46:08 +0000 Subject: [PATCH 100/102] fix: partially address Alastair's comments --- .github/workflows/ci.yml | 2 + poetry.lock | 13 +----- pyproject.toml | 1 - ...7_added_version_hash_date_and_software_.py | 22 ++++++++-- src/imap_mag/DB.py | 6 +-- src/imap_mag/__init__.py | 12 ++++-- src/imap_mag/appUtils.py | 11 +++-- src/imap_mag/cli/fetchBinary.py | 4 +- src/imap_mag/outputManager.py | 36 ++++++++-------- tests/test_database.py | 12 +++--- tests/test_fetchScience.py | 2 +- tests/test_outputManager.py | 42 ++++++------------- 12 files changed, 82 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a5d5b..4c6d408 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +# write to checks/pull-request extra permission needed by 5monkeys/cobertura-action to post coverage stats +# write packages needed by docker image step permissions: id-token: write contents: write diff --git a/poetry.lock b/poetry.lock index de6e7c6..d2f7638 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1530,17 +1530,6 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] -[[package]] -name = "single-version" -version = "1.6.0" -description = "Small utility to define version string for Poetry-style Python project." -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "single_version-1.6.0-py3-none-any.whl", hash = "sha256:67a2734e728b9554750e867b33591f3ad9509ccb851bb3047ced7bfe68429ecd"}, - {file = "single_version-1.6.0.tar.gz", hash = "sha256:3b1fb6e9bd2c88268948d9191c78b63ddd3c07554c1f07cd8a85aedf2486e4fc"}, -] - [[package]] name = "six" version = "1.16.0" @@ -1948,4 +1937,4 @@ viz = ["matplotlib", "nc-time-axis", "seaborn"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "5e61cad3f957d9dabe11d074137198dab507d4ef007b072912a21b6239e733f3" +content-hash = "46cf132984999ff68385aae9bfd5317548d568a149d5f15042b9909753929740" diff --git a/pyproject.toml b/pyproject.toml index d1625ab..8ac647c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ requests = "^2.32.3" pandas = "^2.2.2" imap-data-access = "^0.9.0" cdflib = "^1.3.1" -single-version = "^1.6.0" psycopg = {extras = ["binary"], version = "^3.2.1"} [tool.poetry.group.dev.dependencies] diff --git a/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py index 8cdc34d..fbcc999 100644 --- a/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py +++ b/src/imap_db/migrations/versions/2024_08_09-669111c45c37_added_version_hash_date_and_software_.py @@ -6,6 +6,8 @@ """ +from datetime import datetime + import sqlalchemy as sa from alembic import op @@ -18,11 +20,23 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.add_column("files", sa.Column("version", sa.Integer(), nullable=False)) - op.add_column("files", sa.Column("hash", sa.String(length=64), nullable=False)) - op.add_column("files", sa.Column("date", sa.DateTime(), nullable=False)) op.add_column( - "files", sa.Column("software_version", sa.String(length=16), nullable=False) + "files", sa.Column("version", sa.Integer(), nullable=False, default=0) + ) + op.add_column( + "files", sa.Column("hash", sa.String(length=64), nullable=False, default="") + ) + op.add_column( + "files", + sa.Column( + "date", sa.DateTime(), nullable=False, default=datetime.fromtimestamp(0) + ), + ) + op.add_column( + "files", + sa.Column( + "software_version", sa.String(length=16), nullable=False, default="0.0.0" + ), ) op.create_unique_constraint(None, "files", ["path"]) # ### end Alembic commands ### diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 23c5bd4..1a2de86 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import sessionmaker from imap_mag import __version__ -from imap_mag.outputManager import IMetadataProvider, IOutputManager +from imap_mag.outputManager import IFileMetadataProvider, IOutputManager class IDatabase(abc.ABC): @@ -89,8 +89,8 @@ def __init__( self.__database = database def add_file( - self, original_file: Path, metadata_provider: IMetadataProvider - ) -> tuple[Path, IMetadataProvider]: + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: (destination_file, metadata_provider) = self.__output_manager.add_file( original_file, metadata_provider ) diff --git a/src/imap_mag/__init__.py b/src/imap_mag/__init__.py index ad47412..f3ebd37 100644 --- a/src/imap_mag/__init__.py +++ b/src/imap_mag/__init__.py @@ -1,7 +1,13 @@ """The main module for project.""" -from pathlib import Path +from importlib.metadata import PackageNotFoundError, version -from single_version import get_version -__version__ = get_version("imap-mag", Path(__file__).parent.parent) +def get_version() -> str: + try: + return version("imap-mag") + except PackageNotFoundError: + print("IMAP MAG CLI Version unknown, not installed via pip.") + + +__version__ = get_version() diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index c87410c..b6bdf47 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -1,3 +1,4 @@ +import hashlib import logging from pathlib import Path from typing import Optional @@ -8,7 +9,7 @@ from .appConfig import Destination from .DB import DatabaseOutputManager -from .outputManager import IMetadataProvider, IOutputManager, OutputManager +from .outputManager import IFileMetadataProvider, IOutputManager, OutputManager IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") @@ -56,6 +57,10 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() +def generate_hash(file: Path) -> str: + return hashlib.md5(file.read_bytes()).hexdigest() + + def getOutputManager(destination: Destination) -> IOutputManager: """Retrieve output manager based on destination.""" @@ -71,10 +76,10 @@ def copyFileToDestination( file_path: Path, destination: Destination, output_manager: Optional[OutputManager] = None, -) -> tuple[Path, IMetadataProvider]: +) -> tuple[Path, IFileMetadataProvider]: """Copy file to destination folder.""" - class SimpleMetadataProvider(IMetadataProvider): + class SimpleMetadataProvider(IFileMetadataProvider): """Simple metadata provider for compatibility.""" def __init__(self, filename: str) -> None: diff --git a/src/imap_mag/cli/fetchBinary.py b/src/imap_mag/cli/fetchBinary.py index ebf55f3..efbaaf0 100644 --- a/src/imap_mag/cli/fetchBinary.py +++ b/src/imap_mag/cli/fetchBinary.py @@ -22,6 +22,8 @@ class FetchBinaryOptions(typing.TypedDict): class FetchBinary: """Manage WebPODA data.""" + __MAG_PREFIX: str = "mag_" + __web_poda: WebPODA __output_manager: IOutputManager | None @@ -68,7 +70,7 @@ def download_binaries( file, descriptor=options["packet"] .lower() - .strip("mag_") + .strip(self.__MAG_PREFIX) .replace("_", "-"), date=dates[d], extension="pkts", diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index e46a4d0..c40e9a8 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -1,5 +1,4 @@ import abc -import hashlib import logging import shutil import typing @@ -8,8 +7,10 @@ import typer +from .appUtils import generate_hash -class IMetadataProvider(abc.ABC): + +class IFileMetadataProvider(abc.ABC): """Interface for metadata providers.""" version: int = 0 @@ -23,7 +24,7 @@ def get_file_name(self) -> str: """Retireve file name.""" -class DefaultMetadataProvider(IMetadataProvider): +class DatastoreScienceFilepathGenerator(IFileMetadataProvider): """Metadata for output files.""" prefix: str | None = "imap_mag" @@ -66,14 +67,16 @@ class IOutputManager(abc.ABC): @abc.abstractmethod def add_file( - self, original_file: Path, metadata_provider: IMetadataProvider - ) -> tuple[Path, IMetadataProvider]: + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: """Add file to output location.""" def add_default_file( self, original_file: Path, **metadata: typing.Any - ) -> tuple[Path, IMetadataProvider]: - return self.add_file(original_file, DefaultMetadataProvider(**metadata)) + ) -> tuple[Path, IFileMetadataProvider]: + return self.add_file( + original_file, DatastoreScienceFilepathGenerator(**metadata) + ) class OutputManager(IOutputManager): @@ -85,8 +88,8 @@ def __init__(self, location: Path) -> None: self.location = location def add_file( - self, original_file: Path, metadata_provider: IMetadataProvider - ) -> tuple[Path, IMetadataProvider]: + self, original_file: Path, metadata_provider: IFileMetadataProvider + ) -> tuple[Path, IFileMetadataProvider]: """Add file to output location.""" if not self.location.exists(): @@ -102,14 +105,11 @@ def add_file( destination_file.parent.mkdir(parents=True, exist_ok=True) if destination_file.exists(): - if ( - hashlib.md5(destination_file.read_bytes()).hexdigest() - == hashlib.md5(original_file.read_bytes()).hexdigest() - ): + if generate_hash(destination_file) == generate_hash(original_file): logging.info(f"File {destination_file} already exists and is the same.") return (destination_file, metadata_provider) - metadata_provider.version = self.__find_viable_version( + metadata_provider.version = self.__get_next_available_version( destination_file, metadata_provider ) destination_file = self.__assemble_full_path(metadata_provider) @@ -120,7 +120,7 @@ def add_file( return (destination_file, metadata_provider) - def __assemble_full_path(self, metadata_provider: IMetadataProvider) -> Path: + def __assemble_full_path(self, metadata_provider: IFileMetadataProvider) -> Path: """Assemble full path from metadata.""" return ( @@ -129,13 +129,13 @@ def __assemble_full_path(self, metadata_provider: IMetadataProvider) -> Path: / metadata_provider.get_file_name() ) - def __find_viable_version( - self, destination_file: Path, metadata_provider: IMetadataProvider + def __get_next_available_version( + self, destination_file: Path, metadata_provider: IFileMetadataProvider ) -> int: """Find a viable version for a file.""" while destination_file.exists(): - logging.info( + logging.debug( f"File {destination_file} already exists and is different. Increasing version to {metadata_provider.version}." ) metadata_provider.version += 1 diff --git a/tests/test_database.py b/tests/test_database.py index 56d9ad3..acf8ff6 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -11,7 +11,7 @@ from imap_db.model import File from imap_mag import __version__ from imap_mag.DB import DatabaseOutputManager, IDatabase -from imap_mag.outputManager import DefaultMetadataProvider, IOutputManager +from imap_mag.outputManager import DatastoreScienceFilepathGenerator, IOutputManager from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 @@ -37,7 +37,7 @@ def test_database_output_manager_writes_to_database( original_file = create_test_file( Path(tempfile.gettempdir()) / "some_file", "some content" ) - metadata_provider = DefaultMetadataProvider( + metadata_provider = DatastoreScienceFilepathGenerator( version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" ) @@ -72,7 +72,7 @@ def check_inserted_file(file: File): assert actual_metadata_provider == metadata_provider -def test_database_output_manager_errors_destination_file_not_found( +def test_database_output_manager_errors_when_destination_file_is_not_found( mock_output_manager: mock.Mock, mock_database: mock.Mock ) -> None: # Set up. @@ -81,7 +81,7 @@ def test_database_output_manager_errors_destination_file_not_found( original_file = create_test_file( Path(tempfile.gettempdir()) / "some_file", "some content" ) - metadata_provider = DefaultMetadataProvider( + metadata_provider = DatastoreScienceFilepathGenerator( version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" ) @@ -107,7 +107,7 @@ def test_database_output_manager_errors_destination_file_different_hash( original_file = create_test_file( Path(tempfile.gettempdir()) / "some_file", "some content" ) - metadata_provider = DefaultMetadataProvider( + metadata_provider = DatastoreScienceFilepathGenerator( version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" ) @@ -131,7 +131,7 @@ def test_database_output_manager_errors_database_error( original_file = create_test_file( Path(tempfile.gettempdir()) / "some_file", "some content" ) - metadata_provider = DefaultMetadataProvider( + metadata_provider = DatastoreScienceFilepathGenerator( version=1, descriptor="hsk-pw", date=datetime(2025, 5, 2), extension="txt" ) diff --git a/tests/test_fetchScience.py b/tests/test_fetchScience.py index 65d91d8..05f64f7 100644 --- a/tests/test_fetchScience.py +++ b/tests/test_fetchScience.py @@ -33,7 +33,7 @@ def test_fetch_science_no_matching_files( mock_soc, mock_output_manager, modes=[MAGMode.Normal], sensors=[MAGSensor.OBS] ) - mock_soc.get_filename.side_effect = lambda **_: {} + mock_soc.get_filename.side_effect = lambda **_: {} # return empty dictionary # Exercise. actual_downloaded: list[Path] = fetchScience.download_latest_science( diff --git a/tests/test_outputManager.py b/tests/test_outputManager.py index 912ef7b..f235f22 100644 --- a/tests/test_outputManager.py +++ b/tests/test_outputManager.py @@ -3,18 +3,16 @@ from datetime import datetime from pathlib import Path -from imap_mag.outputManager import IMetadataProvider, OutputManager +from imap_mag.outputManager import IFileMetadataProvider, OutputManager -from .testUtils import enableLogging, tidyDataFolders # noqa: F401 +from .testUtils import create_test_file, enableLogging, tidyDataFolders # noqa: F401 def test_copy_new_file(): # Set up. manager = OutputManager(Path("output")) - original_file = Path(".work/some_test_file.txt") - original_file.parent.mkdir(parents=True, exist_ok=True) - original_file.touch() + original_file = create_test_file(Path(".work/some_test_file.txt")) # Exercise. manager.add_default_file( @@ -32,15 +30,10 @@ def test_copy_file_same_content(): # Set up. manager = OutputManager(Path("output")) - original_file = Path(".work/some_test_file.txt") - original_file.parent.mkdir(parents=True, exist_ok=True) - original_file.touch() - original_file.write_bytes(b"some content") - - existing_file = Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt") - existing_file.parent.mkdir(parents=True, exist_ok=True) - existing_file.touch() - existing_file.write_bytes(b"some content") + original_file = create_test_file(Path(".work/some_test_file.txt"), "some content") + existing_file = create_test_file( + Path("output/2025/05/02/imap_mag_pwr_20250502_v000.txt"), "some content" + ) existing_modification_time = existing_file.stat().st_mtime @@ -61,17 +54,12 @@ def test_copy_file_existing_versions(): # Set up. manager = OutputManager(Path("output")) - original_file = Path(".work/some_test_file.txt") - original_file.parent.mkdir(parents=True, exist_ok=True) - original_file.touch() - original_file.write_bytes(b"some content") + original_file = create_test_file(Path(".work/some_test_file.txt"), "some content") for version in range(2): - existing_file = Path( - f"output/2025/05/02/imap_mag_pwr_20250502_v{version:03}.txt" + create_test_file( + Path(f"output/2025/05/02/imap_mag_pwr_20250502_v{version:03}.txt") ) - existing_file.parent.mkdir(parents=True, exist_ok=True) - existing_file.touch() # Exercise. manager.add_default_file( @@ -89,9 +77,7 @@ def test_copy_file_forced_version(): # Set up. manager = OutputManager(Path("output")) - original_file = Path(".work/some_test_file.txt") - original_file.parent.mkdir(parents=True, exist_ok=True) - original_file.touch() + original_file = create_test_file(Path(".work/some_test_file.txt")) # Exercise. manager.add_default_file( @@ -106,7 +92,7 @@ def test_copy_file_forced_version(): assert Path("output/2025/05/02/imap_mag_pwr_20250502_v003.txt").exists() -class TestMetadataProvider(IMetadataProvider): +class TestMetadataProvider(IFileMetadataProvider): def get_folder_structure(self) -> str: return "abc" @@ -118,9 +104,7 @@ def test_copy_file_custom_providers(): # Set up. manager = OutputManager(Path("output")) - original_file = Path(".work/some_test_file.txt") - original_file.parent.mkdir(parents=True, exist_ok=True) - original_file.touch() + original_file = create_test_file(Path(".work/some_test_file.txt")) # Exercise. manager.add_file(original_file, TestMetadataProvider()) From 49a8d22c603efeffd1a759d32cb7edfa02208ad4 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 30 Aug 2024 13:49:38 +0000 Subject: [PATCH 101/102] fix: avoid circular dependency --- src/imap_mag/appUtils.py | 5 ----- src/imap_mag/outputManager.py | 11 ++++++++--- tests/testUtils.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/imap_mag/appUtils.py b/src/imap_mag/appUtils.py index b6bdf47..b15a050 100644 --- a/src/imap_mag/appUtils.py +++ b/src/imap_mag/appUtils.py @@ -1,4 +1,3 @@ -import hashlib import logging from pathlib import Path from typing import Optional @@ -57,10 +56,6 @@ def convertToDatetime(string: str) -> np.datetime64: raise typer.Abort() -def generate_hash(file: Path) -> str: - return hashlib.md5(file.read_bytes()).hexdigest() - - def getOutputManager(destination: Destination) -> IOutputManager: """Retrieve output manager based on destination.""" diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index c40e9a8..5a614bd 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -1,4 +1,5 @@ import abc +import hashlib import logging import shutil import typing @@ -7,8 +8,6 @@ import typer -from .appUtils import generate_hash - class IFileMetadataProvider(abc.ABC): """Interface for metadata providers.""" @@ -105,7 +104,9 @@ def add_file( destination_file.parent.mkdir(parents=True, exist_ok=True) if destination_file.exists(): - if generate_hash(destination_file) == generate_hash(original_file): + if self.__generate_hash(destination_file) == self.__generate_hash( + original_file + ): logging.info(f"File {destination_file} already exists and is the same.") return (destination_file, metadata_provider) @@ -142,3 +143,7 @@ def __get_next_available_version( destination_file = self.__assemble_full_path(metadata_provider) return metadata_provider.version + + @staticmethod + def __generate_hash(file: Path) -> str: + return hashlib.md5(file.read_bytes()).hexdigest() diff --git a/tests/testUtils.py b/tests/testUtils.py index 3f2e56b..bdb846d 100644 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -67,7 +67,7 @@ def create_serialize_config( return (config, config_file) -def create_test_file(file_path: Path, content: str | None) -> Path: +def create_test_file(file_path: Path, content: str | None = None) -> Path: """Create a file with the given content.""" file_path.unlink(missing_ok=True) From 003cf5ace33db9a06efe0dfd0abb1df329cf00f8 Mon Sep 17 00:00:00 2001 From: Michele Facchinelli Date: Fri, 30 Aug 2024 14:05:08 +0000 Subject: [PATCH 102/102] fix: further reduce duplication with hash --- src/imap_mag/DB.py | 10 +++------- src/imap_mag/outputManager.py | 12 +++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/imap_mag/DB.py b/src/imap_mag/DB.py index 1a2de86..a3c4bfd 100644 --- a/src/imap_mag/DB.py +++ b/src/imap_mag/DB.py @@ -1,5 +1,4 @@ import abc -import hashlib import logging import os from pathlib import Path @@ -10,7 +9,7 @@ from sqlalchemy.orm import sessionmaker from imap_mag import __version__ -from imap_mag.outputManager import IFileMetadataProvider, IOutputManager +from imap_mag.outputManager import IFileMetadataProvider, IOutputManager, generate_hash class IDatabase(abc.ABC): @@ -35,8 +34,6 @@ def __init__(self, db_url=None): if db_url is None and env_url is not None: db_url = env_url - # TODO: Check database is available - if db_url is None: raise ValueError( "No database URL provided. Consider setting SQLALCHEMY_URL environment variable." @@ -95,11 +92,10 @@ def add_file( original_file, metadata_provider ) - file_hash: str = hashlib.md5(original_file.read_bytes()).hexdigest() + file_hash: str = generate_hash(original_file) if not ( - destination_file.exists() - and (hashlib.md5(destination_file.read_bytes()).hexdigest() == file_hash) + destination_file.exists() and (generate_hash(destination_file) == file_hash) ): logging.error( f"File {destination_file} does not exist or is not the same as original {original_file}." diff --git a/src/imap_mag/outputManager.py b/src/imap_mag/outputManager.py index 5a614bd..5f667b4 100644 --- a/src/imap_mag/outputManager.py +++ b/src/imap_mag/outputManager.py @@ -9,6 +9,10 @@ import typer +def generate_hash(file: Path) -> str: + return hashlib.md5(file.read_bytes()).hexdigest() + + class IFileMetadataProvider(abc.ABC): """Interface for metadata providers.""" @@ -104,9 +108,7 @@ def add_file( destination_file.parent.mkdir(parents=True, exist_ok=True) if destination_file.exists(): - if self.__generate_hash(destination_file) == self.__generate_hash( - original_file - ): + if generate_hash(destination_file) == generate_hash(original_file): logging.info(f"File {destination_file} already exists and is the same.") return (destination_file, metadata_provider) @@ -143,7 +145,3 @@ def __get_next_available_version( destination_file = self.__assemble_full_path(metadata_provider) return metadata_provider.version - - @staticmethod - def __generate_hash(file: Path) -> str: - return hashlib.md5(file.read_bytes()).hexdigest()