Skip to content

Commit

Permalink
Make ID Import dynamic
Browse files Browse the repository at this point in the history
  • Loading branch information
@MiraGeowerkstatt
MiraGeowerkstatt committed Dec 20, 2024
1 parent 13b7c8f commit 7c4de4f
Show file tree
Hide file tree
Showing 7 changed files with 42 additions and 38 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
- Removed attachments from csv import.
- Updated recommended csv headers for borehole import to camel case e.g. `OriginalName` (snake case e.g. `original_name` is still supported for all properties except for custom identifiers).
- Changed order of `Top Bedrock (fresh)` and `Top Bedrock (weathered)` fields in borehole form.
- When importing custom IDs with CSV the headers are now dynamically mapped to the `borehole_identifier` codelists in the database.

### Fixed

Expand Down
1 change: 1 addition & 0 deletions src/api/BdmsContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

modelBuilder.Entity<Codelist>().Ignore(c => c.Layers);
modelBuilder.Entity<Codelist>().Ignore(c => c.Hydrotests);
modelBuilder.Entity<Codelist>().Property(b => b.Conf).HasColumnType("json");

modelBuilder.Entity<WaterIngress>().ToTable("water_ingress").HasBaseType<Observation>();

Expand Down
48 changes: 26 additions & 22 deletions src/api/Controllers/ImportController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,14 @@ public async Task<ActionResult<int>> UploadFileAsync(int workgroupId, IFormFile
// Checks if the provided boreholes file is a CSV file.
if (!FileTypeChecker.IsCsv(boreholesFile)) return BadRequest("Invalid file type for borehole csv.");

var boreholeImports = ReadBoreholesFromCsv(boreholesFile);
// The identifier codelists are used to dynamically map imported identifiers to codelists.
var identifierCodelists = await context.Codelists
.Where(c => c.Schema == "borehole_identifier")
.AsNoTracking()
.ToListAsync()
.ConfigureAwait(false);

var boreholeImports = ReadBoreholesFromCsv(boreholesFile, identifierCodelists);
ValidateBoreholeImports(workgroupId, boreholeImports, false);

// If any validation error occured, return a bad request.
Expand Down Expand Up @@ -307,12 +314,12 @@ internal static bool CompareValuesWithTolerance(double? firstValue, double? seco
return Math.Abs(firstValue.Value - secondValue.Value) <= tolerance;
}

private static List<BoreholeImport> ReadBoreholesFromCsv(IFormFile file)
private static List<BoreholeImport> ReadBoreholesFromCsv(IFormFile file, List<Codelist> identifierCodelists)
{
using var reader = new StreamReader(file.OpenReadStream());
using var csv = new CsvReader(reader, CsvConfigHelper.CsvReadConfig);

csv.Context.RegisterClassMap(new CsvImportBoreholeMap());
csv.Context.RegisterClassMap(new CsvImportBoreholeMap(identifierCodelists));

return csv.GetRecords<BoreholeImport>().ToList();
}
Expand Down Expand Up @@ -348,8 +355,11 @@ private void AddValidationErrorToModelState(int boreholeIndex, string errorMessa

private sealed class CsvImportBoreholeMap : ClassMap<BoreholeImport>
{
public CsvImportBoreholeMap()
private readonly List<Codelist> codelists;

public CsvImportBoreholeMap(List<Codelist> codelists)
{
this.codelists = codelists;
AutoMap(CsvConfigHelper.CsvReadConfig);

// Define all optional properties of Borehole (ef navigation properties do not need to be defined as optional).
Expand Down Expand Up @@ -401,37 +411,31 @@ public CsvImportBoreholeMap()
Map(m => m.TopBedrockFreshTvd).Ignore();
Map(m => m.TopBedrockWeatheredTvd).Ignore();

// Define additional mapping logic
Map(m => m.BoreholeCodelists).Convert(args =>
{
var boreholeCodeLists = new List<BoreholeCodelist>();
new List<(string Name, int CodeListId)>
{
("IDGeODin-Shortname", 100000000),
("IDInfoGeol", 100000003),
("IDOriginal", 100000004),
("IDCanton", 100000005),
("IDGeoQuat", 100000006),
("IDGeoMol", 100000007),
("IDGeoTherm", 100000008),
("IDTopFels", 100000009),
("IDGeODin", 100000010),
("IDKernlager", 100000011),
}.ForEach(id =>

foreach (var header in args.Row.HeaderRecord ?? Array.Empty<string>())
{
if (args.Row.HeaderRecord != null && args.Row.HeaderRecord.Any(h => h == id.Name))
// Find the corresponding codelist by comparing the header with Codelist.En, ignoring whitespace
var codelist = codelists.FirstOrDefault(cl => string.Equals(
cl.En.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase),
header.Replace(" ", string.Empty, StringComparison.OrdinalIgnoreCase),
StringComparison.OrdinalIgnoreCase));

if (codelist != null)
{
var value = args.Row.GetField<string?>(id.Name);
var value = args.Row.GetField<string?>(header);
if (!string.IsNullOrEmpty(value))
{
boreholeCodeLists.Add(new BoreholeCodelist
{
CodelistId = id.CodeListId,
CodelistId = codelist.Id,
Value = value,
});
}
}
});
}

return boreholeCodeLists;
});
Expand Down
14 changes: 4 additions & 10 deletions src/client/docs/import.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,16 +40,6 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v

| Feldname | Datentyp | Pflichtfeld | Beschreibung |
| --------------------------- | -------------- | ----------- | ------------------------------------------------------------------------------------- |
| IDGeODin-Shortname | Zahl | Nein | ID GeODin-Shortname |
| IDInfoGeol | Zahl | Nein | ID InfoGeol |
| IDOriginal | Zahl | Nein | ID Original |
| IDCanton | Zahl | Nein | ID Kanton |
| IDGeoQuat | Zahl | Nein | ID GeoQuat |
| IDGeoMol | Zahl | Nein | ID GeoMol |
| IDGeoTherm | Zahl | Nein | ID GeoTherm |
| IDTopFels | Zahl | Nein | ID TopFels |
| IDGeODin | Zahl | Nein | ID GeODin |
| IDKernlager | Zahl | Nein | ID Kernlager |
| OriginalName | Text | Ja | Originalname |
| ProjectName | Text | Nein | Projektname |
| Name | Text | Nein | Name |
Expand Down Expand Up @@ -78,6 +68,10 @@ Die zu importierenden Daten müssen gemäss obigen Anforderungen im CSV-Format v
| ChronostratigraphyTopBedrockId| ID (Codeliste) | Nein | Chronostratigraphie Top Fels |
| LithostratigraphyTopBedrockId | ID (Codeliste) | Nein | Lithostratigraphie Top Fels |

### Ids
Es können zusätzliche IDs importiert werden. Die dafür zu verwendenden Spaltenüberschriften sind dynamisch und können von Umgebung zu Umgebung variieren.
Um die korrekten Spaltenüberschriften zu erhalten, kann eine Bohrung mit einer entsprechenden ID als CSV-Datei exportiert werden.

### Koordinaten

Koordinaten können in LV95 oder LV03 importiert werden, das räumliche Bezugssystem wird aus den Koordinaten erkannt und abgespeichert.
Expand Down
4 changes: 1 addition & 3 deletions src/client/src/pages/overview/sidePanelContent/importer/importModalContent.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,7 @@ const ImportModalContent = ({ setSelectedFile, setFileType, fileType }: ImportMo
<StackHalfWidth direction="column">
{t("csvFormatExplanation")}
{ExampleHeadings(
"IdOriginal;" +
"IdCanton;IdGeoQuat;IdGeoMol;IdGeoTherm;IdTopFels;" +
"IdGeodin;IdKernlager;OriginalName;ProjectName;Name;" +
"OriginalName;ProjectName;Name;" +
"RestrictionId;RestrictionUntil;NationalInterest;LocationX;LocationY;" +
"LocationPrecision;ElevationZ;ElevationPrecisionId;" +
"ReferenceElevation;ReferenceElevationTypeId;" +
Expand Down
8 changes: 7 additions & 1 deletion tests/api/Controllers/ImportControllerTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class ImportControllerTest
private const int MaxBoreholeSeedId = 1002999;
private const int MaxStratigraphySeedId = 6002999;
private const int MaxLayerSeedId = 7029999;
private const int TestCodelistId = 955253;

private BdmsContext context;
private ImportController controller;
Expand Down Expand Up @@ -71,6 +72,7 @@ public async Task TestCleanup()
context.Workflows.RemoveRange(addedWorkflows);
context.Stratigraphies.RemoveRange(addedStratigraphies);
context.Layers.RemoveRange(addedLayers);
context.Codelists.RemoveRange(context.Codelists.Where(c => c.Id == TestCodelistId));
context.SaveChanges();

await context.DisposeAsync();
Expand Down Expand Up @@ -511,6 +513,9 @@ public async Task UploadJsonWithDuplicatesExistingBoreholeShouldReturnError()
[TestMethod]
public async Task UploadShouldSaveDataToDatabaseAsync()
{
context.Codelists.Add(new Codelist { Id = TestCodelistId, Schema = "borehole_identifier", Code = "new code", En = "Random New Id", Conf = null });
await context.SaveChangesAsync();

httpClientFactoryMock
.Setup(cf => cf.CreateClient(It.IsAny<string>()))
.Returns(() => new HttpClient())
Expand All @@ -532,9 +537,10 @@ public async Task UploadShouldSaveDataToDatabaseAsync()
Assert.AreEqual(new DateTime(2024, 06, 15), borehole.RestrictionUntil);
Assert.AreEqual(2474.472693, borehole.TotalDepth);
Assert.AreEqual("Projekt 6", borehole.ProjectName);
Assert.AreEqual(4, borehole.BoreholeCodelists.Count);
Assert.AreEqual(5, borehole.BoreholeCodelists.Count);
Assert.AreEqual("Id_16", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000003).Value);
Assert.AreEqual("AUTOSTEED", borehole.BoreholeCodelists.Single(x => x.CodelistId == 100000011).Value);
Assert.AreEqual("121314", borehole.BoreholeCodelists.Single(x => x.CodelistId == TestCodelistId).Value);
Assert.AreEqual("Bern", borehole.Canton);
Assert.AreEqual("Schweiz", borehole.Country);
Assert.AreEqual("Thun", borehole.Municipality);
Expand Down
4 changes: 2 additions & 2 deletions tests/api/TestData/testdata.csv
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id
IDGeODin-Shortname;IDInfoGeol;IDOriginal;IDCanton;IDGeoQuat;IDGeoMol;IDGeoTherm;IDTopFels;IDGeODin;IDKernlager;original_name;project_name;name;date;restriction_id;restriction_until;original_reference_system;location_x;location_y;location_x_lv_03;location_y_lv_03;location_precision_id;elevation_z;elevation_precision_id;reference_elevation;reference_elevation_type_id;reference_elevation_precision_id;hrs_id;type_id;purpose_id;status_id;remarks;total_depth;depth_precision_id;top_bedrock_fresh_md;top_bedrock_weathered_md;has_groundwater;lithology_top_bedrock_id;chronostratigraphy_top_bedrock_id;lithostratigraphy_top_bedrock_id;RandomNewId
Id_1;Id_2;;;;;Id_3;;;kernlager AETHERMAGIC;Unit_Test_1;Projekt 1 ;Unit_Test_1_a;2021-08-06 00:36:21.991827+00;20111002;;20104001;2618962;1144995;;;20113005;640.7726659;20114001;317.9010264;20117002;20114004;20106001;20101001;22103001;22104003;this product is top-notch.;4232.711946;22108003;398.8529283;656.2476436;TRUE;15104669;15001073;15300261
Id_4;;Id_5;Id_6;;;;;;;Unit_Test_2;Projekt 2;Unit_Test_2_a;2021-03-31 12:20:10.341393+00;;;20104001;2631690;1170516;;;20113002;3430.769638;20114005;2016.314814;20117005;20114004;20106001;20101001;22103001;22104008;This product works certainly well. It perfectly improves my tennis by a lot.;794.1547194;22108005;958.2378855;549.9801019;;15104670;15001009;15302009
;;Id_7;Id_8;;;;Id_9;;;Unit_Test_3;Projekt 3;Unit_Test_3_a;;20111002;01.12.2023;20104001;2614834;1178661;;;20113005;1720.766609;20114003;1829.812475;20117005;20114002;20106001;20101001;;22104002;This is a really good product.;2429.747725;22108002;759.7574008;827.8441205;TRUE;15104671;15001007;15302339
Id_10;;;Id_11;Id_12;;;;;;Unit_Test_4;Projekt 4;Unit_Test_4_a;01.12.2023;;;20104002;2599840;1200560;;;20113004;10.76358115;20114004;1260.544983;20117004;20114001;20106001;20101001;22103001;22104001;;4077.768394;22108004;656.2476436;398.8529283;FALSE;15104672;15001064;15302017
Id_13;;;;Id_14;;;;Id_15;;Unit_Test_5;Projekt 5;Unit_Test_5_a;2021-03-13 23:31:35.390094+00;;;20104001;2631718;1170532;;;20113002;2800.760553;20114004;1928.223082;20117006;20114001;20106001;20101001;22103001;22104008;I tried to maim it but got nectarine all over it.;2971.608569;22108005;549.9801019;958.2378855;FALSE;15104673;15001139;
;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267
;Id_16;;;;Id_17;Id_18;;;AUTOSTEED;Unit_Test_6;Projekt 6;Unit_Test_6_a;2021-11-06 11:15:23.73966+00;20111003;15.06.2024;20104001;2613116;1179127;;;20113004;1090.757525;20114005;1829.812475;20117001;20114001;20106001;20101001;22103004;22104003;talk about fury.;2474.472693;22108005;827.8441205;759.7574008;TRUE;15104674;15001006;15302267;121314

0 comments on commit 7c4de4f

Please sign in to comment.