From 7079028eabc0a29d02da7da7aaad7bb95988df76 Mon Sep 17 00:00:00 2001 From: "Yngve S. Kristiansen" Date: Fri, 25 Oct 2024 11:00:50 +0200 Subject: [PATCH] Ensure gen_data is migrated properly --- src/ert/storage/migration/to7.py | 6 ++---- .../test_that_storage_matches/gen_data | 21 +++++++++++++++++++ .../storage/test_storage_migration.py | 14 +++++++++---- 3 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 tests/ert/unit_tests/storage/snapshots/test_storage_migration/test_that_storage_matches/gen_data diff --git a/src/ert/storage/migration/to7.py b/src/ert/storage/migration/to7.py index a7a0b6569d0..d22a387387b 100644 --- a/src/ert/storage/migration/to7.py +++ b/src/ert/storage/migration/to7.py @@ -98,9 +98,7 @@ def _migrate_response_datasets(path: Path) -> None: responses_obj is not None ), f"Failed to load responses.json @ {responses_file}" - gendata_keys = { - k for k, v in responses_obj.items() if v["_ert_kind"] == "GenDataConfig" - } + gendata_keys = responses_obj.get("gen_data", {}).get("keys", []) for ens in ensembles: with open(ens / "index.json", encoding="utf-8") as f: @@ -255,7 +253,7 @@ def _migrate_responses_from_netcdf_to_parquet(path: Path) -> None: def migrate(path: Path) -> None: - _migrate_response_datasets(path) _migrate_response_configs(path) + _migrate_response_datasets(path) _migrate_responses_from_netcdf_to_parquet(path) _migrate_observations_to_grouped_parquet(path) diff --git a/tests/ert/unit_tests/storage/snapshots/test_storage_migration/test_that_storage_matches/gen_data b/tests/ert/unit_tests/storage/snapshots/test_storage_migration/test_that_storage_matches/gen_data new file mode 100644 index 00000000000..1b734d95981 --- /dev/null +++ b/tests/ert/unit_tests/storage/snapshots/test_storage_migration/test_that_storage_matches/gen_data @@ -0,0 +1,21 @@ +realization,response_key,report_step,index,values +0,GEN,1,0,0.0 +0,GEN,1,1,0.1 +1,GEN,1,0,0.0 +1,GEN,1,1,0.1 +2,GEN,1,0,0.0 +2,GEN,1,1,0.1 +3,GEN,1,0,0.0 +3,GEN,1,1,0.1 +4,GEN,1,0,0.0 +4,GEN,1,1,0.1 +5,GEN,1,0,0.0 +5,GEN,1,1,0.1 +6,GEN,1,0,0.0 +6,GEN,1,1,0.1 +7,GEN,1,0,0.0 +7,GEN,1,1,0.1 +8,GEN,1,0,0.0 +8,GEN,1,1,0.1 +9,GEN,1,0,0.0 +9,GEN,1,1,0.1 diff --git a/tests/ert/unit_tests/storage/test_storage_migration.py b/tests/ert/unit_tests/storage/test_storage_migration.py index 71160faebc2..6c7e8b4854c 100644 --- a/tests/ert/unit_tests/storage/test_storage_migration.py +++ b/tests/ert/unit_tests/storage/test_storage_migration.py @@ -184,10 +184,16 @@ def test_that_storage_matches( "observations", ) - try: - ensemble.load_responses("gen_data", tuple(range(ensemble.ensemble_size))) - except KeyError as e: - raise AssertionError("Failed to migrate gen data") from e + gen_data = ensemble.load_responses( + "gen_data", tuple(range(ensemble.ensemble_size)) + ) + snapshot.assert_match( + gen_data.sort(["realization", "response_key", "report_step", "index"]) + .to_pandas() + .set_index(["realization", "response_key", "report_step", "index"]) + .to_csv(), + "gen_data", + ) assert ensemble.get_summary_keyset() == ["FOPR"]