diff --git a/CHANGELOG.md b/CHANGELOG.md index bf9c055b4..845536a91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Removed attachments from csv import. - Updated recommended csv headers for borehole import to camel case e.g. `OriginalName` (snake case e.g. `original_name` is still supported for all properties except for custom identifiers). - Changed order of `Top Bedrock (fresh)` and `Top Bedrock (weathered)` fields in borehole form. +- When importing custom IDs with CSV the headers are now dynamically mapped to the `borehole_identifier` codelists in the database. ### Fixed diff --git a/src/api/BdmsContext.cs b/src/api/BdmsContext.cs index d4f1b5200..d42476298 100644 --- a/src/api/BdmsContext.cs +++ b/src/api/BdmsContext.cs @@ -282,6 +282,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity().Ignore(c => c.Layers); modelBuilder.Entity().Ignore(c => c.Hydrotests); + modelBuilder.Entity().Property(b => b.Conf).HasColumnType("json"); modelBuilder.Entity().ToTable("water_ingress").HasBaseType(); diff --git a/src/api/Controllers/ImportController.cs b/src/api/Controllers/ImportController.cs index 92b8733c3..2d9296ba5 100644 --- a/src/api/Controllers/ImportController.cs +++ b/src/api/Controllers/ImportController.cs @@ -146,7 +146,14 @@ public async Task> UploadFileAsync(int workgroupId, IFormFile // Checks if the provided boreholes file is a CSV file. if (!FileTypeChecker.IsCsv(boreholesFile)) return BadRequest("Invalid file type for borehole csv."); - var boreholeImports = ReadBoreholesFromCsv(boreholesFile); + // The identifier codelists are used to dynamically map imported identifiers to codelists. + var identifierCodelists = await context.Codelists + .Where(c => c.Schema == "borehole_identifier") + .AsNoTracking() + .ToListAsync() + .ConfigureAwait(false); + + var boreholeImports = ReadBoreholesFromCsv(boreholesFile, identifierCodelists); ValidateBoreholeImports(workgroupId, boreholeImports, false); // If any validation error occured, return a bad request. @@ -307,12 +314,12 @@ internal static bool CompareValuesWithTolerance(double? firstValue, double? seco return Math.Abs(firstValue.Value - secondValue.Value) <= tolerance; } - private static List ReadBoreholesFromCsv(IFormFile file) + private static List ReadBoreholesFromCsv(IFormFile file, List identifierCodelists) { using var reader = new StreamReader(file.OpenReadStream()); using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig); - csv.Context.RegisterClassMap(new CsvImportBoreholeMap()); + csv.Context.RegisterClassMap(new CsvImportBoreholeMap(identifierCodelists)); return csv.GetRecords().ToList(); } @@ -348,8 +355,11 @@ private void AddValidationErrorToModelState(int boreholeIndex, string errorMessa private sealed class CsvImportBoreholeMap : ClassMap { - public CsvImportBoreholeMap() + private readonly List codelists; + + public CsvImportBoreholeMap(List codelists) { + this.codelists = codelists; AutoMap(CsvConfigHelper.CsvReadConfig); // Define all optional properties of Borehole (ef navigation properties do not need to be defined as optional). @@ -401,37 +411,31 @@ public CsvImportBoreholeMap() Map(m => m.TopBedrockFreshTvd).Ignore(); Map(m => m.TopBedrockWeatheredTvd).Ignore(); - // Define additional mapping logic Map(m => m.BoreholeCodelists).Convert(args => { var boreholeCodeLists = new List(); - new List<(string Name, int CodeListId)> - { - ("IDGeODin-Shortname", 100000000), - ("IDInfoGeol", 100000003), - ("IDOriginal", 100000004), - ("IDCanton", 100000005), - ("IDGeoQuat", 100000006), - ("IDGeoMol", 100000007), - ("IDGeoTherm", 100000008), - ("IDTopFels", 100000009), - ("IDGeODin", 100000010), - ("IDKernlager", 100000011), - }.ForEach(id => + + foreach (var header in args.Row.HeaderRecord ?? Array.Empty()) { - if (args.Row.HeaderRecord != null && args.Row.HeaderRecord.Any(h => h == id.Name)) + // Find the corresponding codelist by comparing the header with Codelist.En, ignoring whitespace + var codelist = codelists.FirstOrDefault(cl => string.Equals( + cl.En.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase), + header.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase), + StringComparison.OrdinalIgnoreCase)); + + if (codelist != null) { - var value = args.Row.GetField(id.Name); + var value = args.Row.GetField(header); if (!string.IsNullOrEmpty(value)) { boreholeCodeLists.Add(new BoreholeCodelist { - CodelistId = id.CodeListId, + CodelistId = codelist.Id, Value = value, }); } } - }); + } return boreholeCodeLists; }); diff --git a/src/client/docs/import.md b/src/client/docs/import.md index 75f21fdd1..440f1e69c 100644 --- a/src/client/docs/import.md +++ b/src/client/docs/import.md @@ -40,16 +40,6 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v | Feldname | Datentyp | Pflichtfeld | Beschreibung | | --------------------------- | -------------- | ----------- | ------------------------------------------------------------------------------------- | -| IDGeODin-Shortname | Zahl | Nein | ID GeODin-Shortname | -| IDInfoGeol | Zahl | Nein | ID InfoGeol | -| IDOriginal | Zahl | Nein | ID Original | -| IDCanton | Zahl | Nein | ID Kanton | -| IDGeoQuat | Zahl | Nein | ID GeoQuat | -| IDGeoMol | Zahl | Nein | ID GeoMol | -| IDGeoTherm | Zahl | Nein | ID GeoTherm | -| IDTopFels | Zahl | Nein | ID TopFels | -| IDGeODin | Zahl | Nein | ID GeODin | -| IDKernlager | Zahl | Nein | ID Kernlager | | OriginalName | Text | Ja | Originalname | | ProjectName | Text | Nein | Projektname | | Name | Text | Nein | Name | @@ -78,6 +68,10 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v | ChronostratigraphyTopBedrockId| ID (Codeliste) | Nein | Chronostratigraphie Top Fels | | LithostratigraphyTopBedrockId | ID (Codeliste) | Nein | Lithostratigraphie Top Fels | +### Ids +Es können zusätzliche IDs importiert werden. Die dafür zu verwendenden Spaltenüberschriften sind dynamisch und können von Umgebung zu Umgebung variieren. +Um die korrekten Spaltenüberschriften zu erhalten, kann eine Bohrung mit einer entsprechenden ID als CSV-Datei exportiert werden. + ### Koordinaten Koordinaten können in LV95 oder LV03 importiert werden, das räumliche Bezugssystem wird aus den Koordinaten erkannt und abgespeichert. diff --git a/src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx b/src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx index dc3717fd7..00673c480 100644 --- a/src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx +++ b/src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx @@ -64,9 +64,7 @@ const ImportModalContent = ({ setSelectedFile, setFileType, fileType }: ImportMo {t("csvFormatExplanation")} {ExampleHeadings( - "IdOriginal;" + - "IdCanton;IdGeoQuat;IdGeoMol;IdGeoTherm;IdTopFels;" + - "IdGeodin;IdKernlager;OriginalName;ProjectName;Name;" + + "OriginalName;ProjectName;Name;" + "RestrictionId;RestrictionUntil;NationalInterest;LocationX;LocationY;" + "LocationPrecision;ElevationZ;ElevationPrecisionId;" + "ReferenceElevation;ReferenceElevationTypeId;" + diff --git a/tests/api/Controllers/ImportControllerTest.cs b/tests/api/Controllers/ImportControllerTest.cs index a1345d473..e9bddba61 100644 --- a/tests/api/Controllers/ImportControllerTest.cs +++ b/tests/api/Controllers/ImportControllerTest.cs @@ -21,6 +21,7 @@ public class ImportControllerTest private const int MaxBoreholeSeedId = 1002999; private const int MaxStratigraphySeedId = 6002999; private const int MaxLayerSeedId = 7029999; + private const int TestCodelistId = 955253; private BdmsContext context; private ImportController controller; @@ -71,6 +72,7 @@ public async Task TestCleanup() context.Workflows.RemoveRange(addedWorkflows); context.Stratigraphies.RemoveRange(addedStratigraphies); context.Layers.RemoveRange(addedLayers); + context.Codelists.RemoveRange(context.Codelists.Where(c => c.Id == TestCodelistId)); context.SaveChanges(); await context.DisposeAsync(); @@ -511,6 +513,9 @@ public async Task UploadJsonWithDuplicatesExistingBoreholeShouldReturnError() [TestMethod] public async Task UploadShouldSaveDataToDatabaseAsync() { + context.Codelists.Add(new Codelist { Id = TestCodelistId, Schema = "borehole_identifier", Code = "new code", En = "Random New Id", Conf = null }); + await context.SaveChangesAsync(); + httpClientFactoryMock .Setup(cf => cf.CreateClient(It.IsAny())) .Returns(() => new HttpClient()) @@ -532,9 +537,10 @@ public async Task UploadShouldSaveDataToDatabaseAsync() Assert.AreEqual(new DateTime(2024, 06, 15), borehole.RestrictionUntil); Assert.AreEqual(2474.472693, borehole.TotalDepth); Assert.AreEqual("Projekt 6", borehole.ProjectName); - Assert.AreEqual(4, borehole.BoreholeCodelists.Count); + Assert.AreEqual(5, borehole.BoreholeCodelists.Count); Assert.AreEqual("Id_16", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000003).Value); Assert.AreEqual("AUTOSTEED", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000011).Value); + Assert.AreEqual("121314", borehole.BoreholeCodelists.Single(x => x.CodelistId == TestCodelistId).Value); Assert.AreEqual("Bern", borehole.Canton); Assert.AreEqual("Schweiz", borehole.Country); Assert.AreEqual("Thun", borehole.Municipality); diff --git a/tests/api/TestData/testdata.csv b/tests/api/TestData/testdata.csv index c4b601e71..92efe4de8 100644 --- a/tests/api/TestData/testdata.csv +++ b/tests/api/TestData/testdata.csv @@ -1,7 +1,7 @@ -IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id +IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id;RandomNewId Id_1;Id_2;;;;;Id_3;;;kernlager AETHERMAGIC;Unit_Test_1;Projekt 1 ;Unit_Test_1_a;2021-08-06 00:36:21.991827+00;20111002;;20104001;2618962;1144995;;;20113005;640.7726659;20114001;317.9010264;20117002;20114004;20106001;20101001;22103001;22104003;this product is top-notch.;4232.711946;22108003;398.8529283;656.2476436;TRUE;15104669;15001073;15300261 Id_4;;Id_5;Id_6;;;;;;;Unit_Test_2;Projekt 2;Unit_Test_2_a;2021-03-31 12:20:10.341393+00;;;20104001;2631690;1170516;;;20113002;3430.769638;20114005;2016.314814;20117005;20114004;20106001;20101001;22103001;22104008;This product works certainly well. It perfectly improves my tennis by a lot.;794.1547194;22108005;958.2378855;549.9801019;;15104670;15001009;15302009 ;;Id_7;Id_8;;;;Id_9;;;Unit_Test_3;Projekt 3;Unit_Test_3_a;;20111002;01.12.2023;20104001;2614834;1178661;;;20113005;1720.766609;20114003;1829.812475;20117005;20114002;20106001;20101001;;22104002;This is a really good product.;2429.747725;22108002;759.7574008;827.8441205;TRUE;15104671;15001007;15302339 Id_10;;;Id_11;Id_12;;;;;;Unit_Test_4;Projekt 4;Unit_Test_4_a;01.12.2023;;;20104002;2599840;1200560;;;20113004;10.76358115;20114004;1260.544983;20117004;20114001;20106001;20101001;22103001;22104001;;4077.768394;22108004;656.2476436;398.8529283;FALSE;15104672;15001064;15302017 Id_13;;;;Id_14;;;;Id_15;;Unit_Test_5;Projekt 5;Unit_Test_5_a;2021-03-13 23:31:35.390094+00;;;20104001;2631718;1170532;;;20113002;2800.760553;20114004;1928.223082;20117006;20114001;20106001;20101001;22103001;22104008;I tried to maim it but got nectarine all over it.;2971.608569;22108005;549.9801019;958.2378855;FALSE;15104673;15001139; -;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267 +;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267;121314