From d420131a01cff13d53242279f153052ed0d8202f Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 20 Dec 2024 01:55:39 -0600 Subject: [PATCH 1/8] Fix up some codeql/inspection code issues (#2087) * Update OverviewModel.cs Fix up some .Dispose/using issues, make finding most recent load ID more efficient * LINQ tidying * CodeQL fixups * Update Catalogue.cs Add Hashcode, Equals methods. * Update Catalogue.cs Tweak equality semantics for RDMP DB oddities --- ...CommandUpdateCatalogueDataLocationTests.cs | 75 ++--- ...atabaseMSSqlDestinationReExtractionTest.cs | 271 +++++++++--------- .../DataTableUploadDestinationTests.cs | 80 +++--- ...SourceTests_ResolvedAccordingToStrategy.cs | 32 ++- .../DelimitedFileSourceTests_Unresolveable.cs | 25 +- ...utingACohortIdentificationConfiguration.cs | 13 +- ...cuteCommandCloneExtractionConfiguration.cs | 71 ++--- .../AtomicCommands/ExecuteCommandDeprecate.cs | 10 +- ...eCommandRevertToHistoricalCohortVersion.cs | 3 +- ...ecuteCommandUpdateCatalogueDataLocation.cs | 6 +- Rdmp.Core/Curation/Data/Catalogue.cs | 16 +- .../Curation/Data/Overview/OverviewModel.cs | 98 +++---- .../Sources/ExecuteDatasetExtractionSource.cs | 10 +- Rdmp.Core/DataLoad/Engine/Job/JobFactory.cs | 3 +- .../Scheduling/SingleScheduledJobFactory.cs | 4 +- .../DataTableUploadDestination.cs | 19 +- .../Mutilators/RegexRedactionMutilator.cs | 14 +- .../ExecuteCommandReorderFilter.cs | 10 +- .../JoinsAndLookups/LookupConfigurationUI.cs | 21 +- .../LoadExecutionUIs/ExecuteLoadMetadataUI.cs | 10 +- Rdmp.UI/SimpleDialogs/InstanceSettings.cs | 6 +- Rdmp.UI/SimpleDialogs/NewfindUI.cs | 8 +- .../CreateNewDataExtractionProjectUI.cs | 16 +- Tests.Common/UnitTests.cs | 2 +- 24 files changed, 430 insertions(+), 393 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandUpdateCatalogueDataLocationTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandUpdateCatalogueDataLocationTests.cs index ee6b3e2691..5f448b810e 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandUpdateCatalogueDataLocationTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandUpdateCatalogueDataLocationTests.cs @@ -68,22 +68,22 @@ private void CreateCatalogue() if (CatalogueRepository.GetAllObjects().Length == 0) creator.CreatePipelines(new PlatformDatabaseCreationOptions()); - var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(p => p.ID) - .FirstOrDefault(p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); + var pipe = CatalogueRepository.GetAllObjects().OrderByDescending(static p => p.ID) + .FirstOrDefault(static p => p.Name.Contains("BULK INSERT: CSV Import File (automated column-type detection)")); var cmd = new ExecuteCommandCreateNewCatalogueByImportingFile(new ThrowImmediatelyActivator(RepositoryLocator), info, "Column1", db, pipe, null); cmd.Execute(); originalTableInfoID = RepositoryLocator.CatalogueRepository.GetAllObjects().First().ColumnInfo .TableInfo_ID; - var column1 = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column1").First(); - column1.ExtractionInformation.SelectSQL = column1.ExtractionInformation.SelectSQL + " as SOME_ALIAS"; + var column1 = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column1"); + column1.ExtractionInformation.SelectSQL += " as SOME_ALIAS"; column1.ExtractionInformation.SaveToDatabase(); } private void CreateSecondaryCatalogue() { - var fileName = Path.GetTempPath() + Guid.NewGuid() + ".csv"; + var fileName = $"{Path.GetTempPath()}{Guid.NewGuid()}.csv"; using (var outputFile = new StreamWriter(fileName, true)) { outputFile.WriteLine("Column,Other"); @@ -149,8 +149,8 @@ public void UpdateLocationColumnBadType() [Test] public void UpdateLocationCheckOK() { - goodCatalogueID = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2").First().Catalogue_ID; + goodCatalogueID = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column2").Catalogue_ID; var cmd = new ExecuteCommandUpdateCatalogueDataLocation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator.CatalogueRepository.GetAllObjects() .Where(c => c.Catalogue_ID == goodCatalogueID).ToArray(), RemoteTable, null); @@ -171,47 +171,53 @@ public void UpdateLocationExecuteNoCheck_Bad() [Test] public void UpdateLocationExecuteNoCheck_OK() { - goodCatalogueID = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2").First().Catalogue_ID; + goodCatalogueID = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column2").Catalogue_ID; var cmd = new ExecuteCommandUpdateCatalogueDataLocation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator.CatalogueRepository.GetAllObjects() .Where(c => c.Catalogue_ID == goodCatalogueID).ToArray(), RemoteTable, null); Assert.DoesNotThrow(() => cmd.Execute()); - var ci = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2" && c.Catalogue_ID == goodCatalogueID).First(); - Assert.That(ci.ColumnInfo.Name, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column2]")); - Assert.That(ci.ExtractionInformation.SelectSQL, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column2]")); - Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + var ci = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(c => c.Name == "Column2" && c.Catalogue_ID == goodCatalogueID); + Assert.Multiple(() => + { + Assert.That(ci.ColumnInfo.Name, Is.EqualTo($"{RemoteTable.GetFullyQualifiedName()}.[Column2]")); + Assert.That(ci.ExtractionInformation.SelectSQL, Is.EqualTo($"{RemoteTable.GetFullyQualifiedName()}.[Column2]")); + Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + }); } [Test] public void UpdateLocationExecuteNoCheck_AliasCheck() { - goodCatalogueID = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2").First().Catalogue_ID; + goodCatalogueID = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column2").Catalogue_ID; var cmd = new ExecuteCommandUpdateCatalogueDataLocation(new ThrowImmediatelyActivator(RepositoryLocator), RepositoryLocator.CatalogueRepository.GetAllObjects() .Where(c => c.Catalogue_ID == goodCatalogueID).ToArray(), RemoteTable, null); Assert.DoesNotThrow(() => cmd.Execute()); - var ci = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column1" && c.Catalogue_ID == goodCatalogueID).First(); + var ci = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(c => c.Name == "Column1" && c.Catalogue_ID == goodCatalogueID); Assert.That(ci.ColumnInfo.Name, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column1]")); - var ei = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(e => e.ID == ci.ExtractionInformation.ID).First(); - Assert.That(ei.SelectSQL, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column1] as SOME_ALIAS")); - Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + var ei = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(e => e.ID == ci.ExtractionInformation.ID); + Assert.Multiple(() => + { + Assert.That(ei.SelectSQL, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column1] as SOME_ALIAS")); + Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + }); } [Test] public void UpdateLocationWithMultipleExtractionIdentifiers() { - goodCatalogueID = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2").First().Catalogue_ID; - var ci = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column2" && c.Catalogue_ID == goodCatalogueID).First(); + goodCatalogueID = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column2").Catalogue_ID; + var ci = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(c => c.Name == "Column2" && c.Catalogue_ID == goodCatalogueID); - var otherci = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(c => c.Name == "Column").First(); + var otherci = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(static c => c.Name == "Column"); var cmd1 = new ExecuteCommandAddNewCatalogueItem(new ThrowImmediatelyActivator(RepositoryLocator), ci.Catalogue, new List { otherci.ColumnInfo }.ToArray()); Assert.DoesNotThrow(() => cmd1.Execute()); @@ -221,9 +227,12 @@ public void UpdateLocationWithMultipleExtractionIdentifiers() .Where(c => c.Catalogue_ID == goodCatalogueID && c.Name == "Column2").ToArray(), RemoteTable, null); Assert.DoesNotThrow(() => cmd.Execute()); Assert.That(ci.ColumnInfo.Name, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column2]")); - var ei = RepositoryLocator.CatalogueRepository.GetAllObjects() - .Where(e => e.ID == ci.ExtractionInformation.ID).First(); - Assert.That(ei.SelectSQL, Is.EqualTo(RemoteTable.GetFullyQualifiedName() + ".[Column2]")); - Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + var ei = RepositoryLocator.CatalogueRepository + .GetAllObjects().First(e => e.ID == ci.ExtractionInformation.ID); + Assert.Multiple(() => + { + Assert.That(ei.SelectSQL, Is.EqualTo($"{RemoteTable.GetFullyQualifiedName()}.[Column2]")); + Assert.That(ci.ColumnInfo.TableInfo_ID, Is.Not.EqualTo(originalTableInfoID)); + }); } } \ No newline at end of file diff --git a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs index 5edb4639ae..df28b536ba 100644 --- a/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs +++ b/Rdmp.Core.Tests/DataExport/DataExtraction/ExecuteFullExtractionToDatabaseMSSqlDestinationReExtractionTest.cs @@ -82,19 +82,19 @@ public void ReExtractToADatabaseWithNoNewData() null, db, pipe, null); cmd.Execute(); - var catalogue = CatalogueRepository.GetAllObjects().Where(c => c.Name == "bob").FirstOrDefault(); - var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); var ei = chiColumnInfo.ExtractionInformation; ei.IsExtractionIdentifier = true; ei.IsPrimaryKey = true; ei.SaveToDatabase(); var project = new Project(DataExportRepository, "MyProject") { - ProjectNumber = 500 + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() }; - project.ExtractionDirectory = Path.GetTempPath(); project.SaveToDatabase(); - CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic.CreateRootContainerIfNotExists(); var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -106,13 +106,13 @@ public void ReExtractToADatabaseWithNoNewData() dim.SaveToDatabase(); agg1.SaveToDatabase(); - string CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); - string cohortTableName = "Cohort"; - string definitionTableName = "CohortDefinition"; - string ExternalCohortTableNameInCatalogue = "CohortTests"; + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; const string ReleaseIdentifierFieldName = "ReleaseId"; const string DefinitionTableForeignKeyField = "cohortDefinition_id"; - DiscoveredDatabase _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); if (_cohortDatabase.Exists()) DeleteTables(_cohortDatabase); else @@ -166,7 +166,7 @@ public void ReExtractToADatabaseWithNoNewData() }; newExternal.SaveToDatabase(); - var cohortPipeline = CatalogueRepository.GetAllObjects().Where(p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration").First(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( new ThrowImmediatelyActivator(RepositoryLocator), cic, @@ -174,14 +174,14 @@ public void ReExtractToADatabaseWithNoNewData() "MyCohort", project, cohortPipeline - ); + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); var ec = new ExtractionConfiguration(DataExportRepository, project) { Name = "ext1", - Cohort_ID = _extractableCohort.ID + Cohort_ID = extractableCohort.ID }; ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); @@ -197,7 +197,7 @@ public void ReExtractToADatabaseWithNoNewData() var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); - ExternalDatabaseServer _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, @@ -309,19 +309,19 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() null, db, pipe, null); cmd.Execute(); - var catalogue = CatalogueRepository.GetAllObjects().Where(c => c.Name == "bob").FirstOrDefault(); - var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); var ei = chiColumnInfo.ExtractionInformation; ei.IsExtractionIdentifier = true; ei.IsPrimaryKey = true; ei.SaveToDatabase(); var project = new Project(DataExportRepository, "MyProject") { - ProjectNumber = 500 + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() }; - project.ExtractionDirectory = Path.GetTempPath(); project.SaveToDatabase(); - CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic.CreateRootContainerIfNotExists(); var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -333,13 +333,13 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() dim.SaveToDatabase(); agg1.SaveToDatabase(); - string CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); - string cohortTableName = "Cohort"; - string definitionTableName = "CohortDefinition"; - string ExternalCohortTableNameInCatalogue = "CohortTests"; + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; const string ReleaseIdentifierFieldName = "ReleaseId"; const string DefinitionTableForeignKeyField = "cohortDefinition_id"; - DiscoveredDatabase _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); if (_cohortDatabase.Exists()) DeleteTables(_cohortDatabase); else @@ -392,7 +392,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() }; newExternal.SaveToDatabase(); - var cohortPipeline = CatalogueRepository.GetAllObjects().Where(p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration").First(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( new ThrowImmediatelyActivator(RepositoryLocator), cic, @@ -400,14 +400,14 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() "MyCohort", project, cohortPipeline - ); + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); var ec = new ExtractionConfiguration(DataExportRepository, project) { Name = "ext1", - Cohort_ID = _extractableCohort.ID + Cohort_ID = extractableCohort.ID }; ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); @@ -423,7 +423,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); - ExternalDatabaseServer _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, @@ -486,17 +486,17 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() //add new entry here var tbl = db.DiscoverTables(false).First(); tbl.Insert(new Dictionary - { - { "chi","1111111111"}, - {"notes","T"}, - {"dtCreated", new DateTime(2001, 1, 2) }, - {"century",19}, - {"surname","1234"}, - {"forename","yes"}, - {"sex","M"}, - }); + { + { "chi","1111111111"}, + {"notes","T"}, + {"dtCreated", new DateTime(2001, 1, 2) }, + {"century",19}, + {"surname","1234"}, + {"forename","yes"}, + {"sex","M"}, + }); - CohortIdentificationConfiguration cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic2.CreateRootContainerIfNotExists(); var agg12 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); _ = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -509,7 +509,7 @@ public void ReExtractToADatabaseWithNewDataAndNoPKs() agg12.SaveToDatabase(); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); + var _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); ec.Cohort_ID = _extractableCohort2.ID; ec.SaveToDatabase(); @@ -566,24 +566,24 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() null, db, pipe, null); cmd.Execute(); - var catalogue = CatalogueRepository.GetAllObjects().Where(c => c.Name == "bob").FirstOrDefault(); - var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); var ei = chiColumnInfo.ExtractionInformation; ei.IsExtractionIdentifier = true; ei.IsPrimaryKey = true; ei.SaveToDatabase(); - var surnameInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "surname").First(); + var surnameInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "surname"); surnameInfo.ExtractionInformation.IsPrimaryKey = true; surnameInfo.ExtractionInformation.SaveToDatabase(); var project = new Project(DataExportRepository, "MyProject") { - ProjectNumber = 500 + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() }; - project.ExtractionDirectory = Path.GetTempPath(); project.SaveToDatabase(); - CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic.CreateRootContainerIfNotExists(); var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -595,13 +595,13 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() dim.SaveToDatabase(); agg1.SaveToDatabase(); - string CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); - string cohortTableName = "Cohort"; - string definitionTableName = "CohortDefinition"; - string ExternalCohortTableNameInCatalogue = "CohortTests"; + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; const string ReleaseIdentifierFieldName = "ReleaseId"; const string DefinitionTableForeignKeyField = "cohortDefinition_id"; - DiscoveredDatabase _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); if (_cohortDatabase.Exists()) DeleteTables(_cohortDatabase); else @@ -654,7 +654,7 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() }; newExternal.SaveToDatabase(); - var cohortPipeline = CatalogueRepository.GetAllObjects().Where(p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration").First(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( new ThrowImmediatelyActivator(RepositoryLocator), cic, @@ -662,14 +662,14 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() "MyCohort", project, cohortPipeline - ); + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); var ec = new ExtractionConfiguration(DataExportRepository, project) { Name = "ext1", - Cohort_ID = _extractableCohort.ID + Cohort_ID = extractableCohort.ID }; ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); @@ -685,7 +685,7 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); - ExternalDatabaseServer _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, @@ -748,17 +748,17 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() //add new entry here var tbl = db.DiscoverTables(false).First(); tbl.Insert(new Dictionary - { - { "chi","1111111111"}, - {"notes","T"}, - {"dtCreated", new DateTime(2001, 1, 2) }, - {"century",19}, - {"surname","1234"}, - {"forename","yes"}, - {"sex","M"}, - }); + { + { "chi","1111111111"}, + {"notes","T"}, + {"dtCreated", new DateTime(2001, 1, 2) }, + {"century",19}, + {"surname","1234"}, + {"forename","yes"}, + {"sex","M"}, + }); - CohortIdentificationConfiguration cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic2.CreateRootContainerIfNotExists(); var agg12 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); _ = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -770,15 +770,15 @@ public void ReExtractToADatabaseWithNewDataAndSinglePK() dim2.SaveToDatabase(); agg12.SaveToDatabase(); newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( - new ThrowImmediatelyActivator(RepositoryLocator), - cic2, - newExternal, - "MyCohort", - project, - cohortPipeline - ); + new ThrowImmediatelyActivator(RepositoryLocator), + cic2, + newExternal, + "MyCohort", + project, + cohortPipeline + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); + var _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); ec.Cohort_ID = _extractableCohort2.ID; ec.SaveToDatabase(); @@ -835,8 +835,8 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() null, db, pipe, null); cmd.Execute(); - var catalogue = CatalogueRepository.GetAllObjects().Where(c => c.Name == "bob").FirstOrDefault(); - var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); var ei = chiColumnInfo.ExtractionInformation; ei.IsPrimaryKey = true; ei.IsExtractionIdentifier = true; @@ -844,11 +844,11 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() var project = new Project(DataExportRepository, "MyProject") { - ProjectNumber = 500 + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() }; - project.ExtractionDirectory = Path.GetTempPath(); project.SaveToDatabase(); - CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic.CreateRootContainerIfNotExists(); var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -860,13 +860,13 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() dim.SaveToDatabase(); agg1.SaveToDatabase(); - string CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); - string cohortTableName = "Cohort"; - string definitionTableName = "CohortDefinition"; - string ExternalCohortTableNameInCatalogue = "CohortTests"; + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + const string cohortTableName = "Cohort"; + const string definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; const string ReleaseIdentifierFieldName = "ReleaseId"; const string DefinitionTableForeignKeyField = "cohortDefinition_id"; - DiscoveredDatabase _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); if (_cohortDatabase.Exists()) DeleteTables(_cohortDatabase); else @@ -920,7 +920,7 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() }; newExternal.SaveToDatabase(); - var cohortPipeline = CatalogueRepository.GetAllObjects().Where(p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration").First(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( new ThrowImmediatelyActivator(RepositoryLocator), cic, @@ -928,14 +928,14 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() "MyCohort", project, cohortPipeline - ); + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); var ec = new ExtractionConfiguration(DataExportRepository, project) { Name = "ext1", - Cohort_ID = _extractableCohort.ID + Cohort_ID = extractableCohort.ID }; ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); @@ -951,7 +951,7 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); - ExternalDatabaseServer _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + var _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, @@ -1014,17 +1014,17 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() //add new entry here var tbl = db.DiscoverTables(false).First(); tbl.Insert(new Dictionary - { - { "chi","1111111111"}, - {"notes","T"}, - {"dtCreated", new DateTime(2001, 1, 2) }, - {"century",19}, - {"surname","1234"}, - {"forename","yes"}, - {"sex","M"}, - }); + { + { "chi","1111111111"}, + {"notes","T"}, + {"dtCreated", new DateTime(2001, 1, 2) }, + {"century",19}, + {"surname","1234"}, + {"forename","yes"}, + {"sex","M"}, + }); - CohortIdentificationConfiguration cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic2.CreateRootContainerIfNotExists(); var agg12 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); _ = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -1037,7 +1037,7 @@ public void ReExtractToADatabaseWithNewDataAndExtractionIdentifierIsPK() agg12.SaveToDatabase(); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); + var _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); ec.Cohort_ID = _extractableCohort2.ID; ec.SaveToDatabase(); @@ -1094,8 +1094,8 @@ public void ExtractToDatabaseUseTriggers() null, db, pipe, null); cmd.Execute(); - var catalogue = CatalogueRepository.GetAllObjects().Where(c => c.Name == "bob").FirstOrDefault(); - var chiColumnInfo = catalogue.CatalogueItems.Where(ci => ci.Name == "chi").First(); + var catalogue = CatalogueRepository.GetAllObjects().FirstOrDefault(static c => c.Name == "bob"); + var chiColumnInfo = catalogue.CatalogueItems.First(static ci => ci.Name == "chi"); var ei = chiColumnInfo.ExtractionInformation; ei.IsPrimaryKey = true; ei.IsExtractionIdentifier = true; @@ -1103,11 +1103,11 @@ public void ExtractToDatabaseUseTriggers() var project = new Project(DataExportRepository, "MyProject") { - ProjectNumber = 500 + ProjectNumber = 500, + ExtractionDirectory = Path.GetTempPath() }; - project.ExtractionDirectory = Path.GetTempPath(); project.SaveToDatabase(); - CohortIdentificationConfiguration cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic.CreateRootContainerIfNotExists(); var agg1 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); var conf = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -1119,13 +1119,13 @@ public void ExtractToDatabaseUseTriggers() dim.SaveToDatabase(); agg1.SaveToDatabase(); - string CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); - string cohortTableName = "Cohort"; - string definitionTableName = "CohortDefinition"; - string ExternalCohortTableNameInCatalogue = "CohortTests"; + var CohortDatabaseName = TestDatabaseNames.GetConsistentName("CohortDatabase"); + var cohortTableName = "Cohort"; + var definitionTableName = "CohortDefinition"; + var ExternalCohortTableNameInCatalogue = "CohortTests"; const string ReleaseIdentifierFieldName = "ReleaseId"; const string DefinitionTableForeignKeyField = "cohortDefinition_id"; - DiscoveredDatabase _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); + var _cohortDatabase = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExpectDatabase(CohortDatabaseName); if (_cohortDatabase.Exists()) DeleteTables(_cohortDatabase); else @@ -1179,7 +1179,7 @@ public void ExtractToDatabaseUseTriggers() }; newExternal.SaveToDatabase(); - var cohortPipeline = CatalogueRepository.GetAllObjects().Where(p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration").First(); + var cohortPipeline = CatalogueRepository.GetAllObjects().First(static p => p.Name == "CREATE COHORT:By Executing Cohort Identification Configuration"); var newCohortCmd = new ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration( new ThrowImmediatelyActivator(RepositoryLocator), cic, @@ -1187,14 +1187,14 @@ public void ExtractToDatabaseUseTriggers() "MyCohort", project, cohortPipeline - ); + ); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); + var extractableCohort = new ExtractableCohort(DataExportRepository, newExternal, 1); var ec = new ExtractionConfiguration(DataExportRepository, project) { Name = "ext1", - Cohort_ID = _extractableCohort.ID + Cohort_ID = extractableCohort.ID }; ec.AddDatasetToConfiguration(new ExtractableDataSet(DataExportRepository, catalogue)); @@ -1205,20 +1205,20 @@ public void ExtractToDatabaseUseTriggers() typeof(ExecuteFullExtractionToDatabaseMSSql), 0, "MS SQL Destination"); var destinationArguments = component.CreateArgumentsForClassIfNotExists() .ToList(); - var argumentServer = destinationArguments.Single(a => a.Name == "TargetDatabaseServer"); - var argumentDbNamePattern = destinationArguments.Single(a => a.Name == "DatabaseNamingPattern"); - var argumentTblNamePattern = destinationArguments.Single(a => a.Name == "TableNamingPattern"); - var reExtract = destinationArguments.Single(a => a.Name == "AppendDataIfTableExists"); + var argumentServer = destinationArguments.Single(static a => a.Name == "TargetDatabaseServer"); + var argumentDbNamePattern = destinationArguments.Single(static a => a.Name == "DatabaseNamingPattern"); + var argumentTblNamePattern = destinationArguments.Single(static a => a.Name == "TableNamingPattern"); + var reExtract = destinationArguments.Single(static a => a.Name == "AppendDataIfTableExists"); Assert.That(argumentServer.Name, Is.EqualTo("TargetDatabaseServer")); - ExternalDatabaseServer _extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) + var extractionServer = new ExternalDatabaseServer(CatalogueRepository, "myserver", null) { Server = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.Name, Username = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitUsernameIfAny, Password = DiscoveredServerICanCreateRandomDatabasesAndTablesOn.ExplicitPasswordIfAny }; - _extractionServer.SaveToDatabase(); + extractionServer.SaveToDatabase(); - argumentServer.SetValue(_extractionServer); + argumentServer.SetValue(extractionServer); argumentServer.SaveToDatabase(); argumentDbNamePattern.SetValue($"{TestDatabaseNames.Prefix}$p_$n"); argumentDbNamePattern.SaveToDatabase(); @@ -1231,8 +1231,9 @@ public void ExtractToDatabaseUseTriggers() typeof(ExecuteCrossServerDatasetExtractionSource), -1, "Source"); var arguments2 = component2.CreateArgumentsForClassIfNotExists() .ToArray(); - arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SetValue(false); - arguments2.Single(a => a.Name.Equals("AllowEmptyExtractions")).SaveToDatabase(); + var allowEmpty = arguments2.Single(static a => a.Name.Equals("AllowEmptyExtractions")); + allowEmpty.SetValue(false); + allowEmpty.SaveToDatabase(); //configure the component as the destination extractionPipeline.DestinationPipelineComponent_ID = component.ID; @@ -1275,17 +1276,17 @@ public void ExtractToDatabaseUseTriggers() ////add new entry here var tbl = db.DiscoverTables(false).First(); tbl.Insert(new Dictionary - { - { "chi","1111111111"}, - {"notes","T"}, - {"dtCreated", new DateTime(2001, 1, 2) }, - {"century",19}, - {"surname","1234"}, - {"forename","yes"}, - {"sex","M"}, - }); + { + { "chi","1111111111"}, + {"notes","T"}, + {"dtCreated", new DateTime(2001, 1, 2) }, + {"century",19}, + {"surname","1234"}, + {"forename","yes"}, + {"sex","M"}, + }); - CohortIdentificationConfiguration cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); + var cic2 = new CohortIdentificationConfiguration(CatalogueRepository, "Cohort1"); cic2.CreateRootContainerIfNotExists(); var agg12 = new AggregateConfiguration(CatalogueRepository, catalogue, "agg1"); _ = new AggregateConfiguration(CatalogueRepository, catalogue, "UnitTestShortcutAggregate"); @@ -1298,7 +1299,7 @@ public void ExtractToDatabaseUseTriggers() agg12.SaveToDatabase(); newCohortCmd.Execute(); - ExtractableCohort _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); + var _extractableCohort2 = new ExtractableCohort(DataExportRepository, newExternal, 2); ec.Cohort_ID = _extractableCohort2.ID; ec.SaveToDatabase(); diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs index dbf274644d..d80a15c3f1 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/DataTableUploadDestinationTests.cs @@ -1551,10 +1551,11 @@ public void UseTriggerwithoutPrimaryKey() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var t = db.DiscoverTables(false).Select(t => t.GetRuntimeName()).ToList(); + + var t = db.DiscoverTables(false).Select(static t => t.GetRuntimeName()).ToList(); Assert.That(t.Contains("DataTableUploadDestinationTests_Archive"), Is.EqualTo(false)); - var destinationTable = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var columns = destinationTable.DiscoverColumns().Select(c => c.GetRuntimeName()); + var destinationTable = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + var columns = destinationTable.DiscoverColumns().Select(static c => c.GetRuntimeName()); Assert.That(columns.Contains(SpecialFieldNames.DataLoadRunID), Is.EqualTo(false)); } @@ -1586,10 +1587,11 @@ public void UseTriggerwithPrimaryKey() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var t = db.DiscoverTables(false).Select(t => t.GetRuntimeName()).ToList(); + + var t = db.DiscoverTables(false).Select(static t => t.GetRuntimeName()).ToList(); Assert.That(t.Contains("DataTableUploadDestinationTests_Archive"), Is.EqualTo(true)); - var destinationTable = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var columns = destinationTable.DiscoverColumns().Select(c => c.GetRuntimeName()); + var destinationTable = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + var columns = destinationTable.DiscoverColumns().Select(static c => c.GetRuntimeName()); Assert.That(columns.Contains(SpecialFieldNames.DataLoadRunID), Is.EqualTo(true)); } @@ -1633,10 +1635,12 @@ public void DataTableUploadClashSameRow() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); + + var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + using var resultDt = table.GetDataTable(); + Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); } + [Test] public void DataTableUploadClashSameRowWithTrigger() { @@ -1678,10 +1682,12 @@ public void DataTableUploadClashSameRowWithTrigger() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); + + var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + using var resultDt = table.GetDataTable(); + Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); } + [Test] public void DataTableUploadClashUpdateNoTrigger() { @@ -1730,12 +1736,13 @@ public void DataTableUploadClashUpdateNoTrigger() destination.PreInitialize(db, toConsole); Assert.Throws(() => destination.ProcessPipelineData(dt1, toConsole, token)); destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, null); - var table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); - Assert.That(resultDT.Rows[0].ItemArray[0], Is.EqualTo("Fish")); - Assert.That(resultDT.Rows[0].ItemArray[1], Is.EqualTo("Friend")); + var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + var resultDt = table.GetDataTable(); + Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); + Assert.That(resultDt.Rows[0].ItemArray[0], Is.EqualTo("Fish")); + Assert.That(resultDt.Rows[0].ItemArray[1], Is.EqualTo("Friend")); } + [Test] public void DataTableUploadClashUpdateWithTrigger() { @@ -1777,16 +1784,18 @@ public void DataTableUploadClashUpdateWithTrigger() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); - Assert.That(resultDT.Rows[0].ItemArray[0], Is.EqualTo("Fish")); - Assert.That(resultDT.Rows[0].ItemArray[1], Is.EqualTo("Enemy")); - table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive").First(); - resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); - Assert.That(resultDT.Rows[0].ItemArray[0], Is.EqualTo("Fish")); + + var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + using var resultDt = table.GetDataTable(); + Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); + Assert.That(resultDt.Rows[0].ItemArray[0], Is.EqualTo("Fish")); + Assert.That(resultDt.Rows[0].ItemArray[1], Is.EqualTo("Enemy")); + table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive"); + using var resultDt2 = table.GetDataTable(); + Assert.That(resultDt2.Rows, Has.Count.EqualTo(1)); + Assert.That(resultDt2.Rows[0].ItemArray[0], Is.EqualTo("Fish")); } + //need to test rows to modify ln 354 //test when we drop a column [Test] @@ -1830,15 +1839,16 @@ public void DataTableUploadClashUpdateDropColumnWithTrigger() destination.Dispose(ThrowImmediatelyDataLoadEventListener.Quiet, ex); throw; } - var table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests").First(); - var resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); - Assert.That(resultDT.Rows[0].ItemArray[0], Is.EqualTo("Fish")); - Assert.That(resultDT.Rows[0].ItemArray[1], Is.EqualTo(string.Empty)); - table = db.DiscoverTables(false).Where(t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive").First(); - resultDT = table.GetDataTable(); - Assert.That(resultDT.Rows.Count, Is.EqualTo(1)); - Assert.That(resultDT.Rows[0].ItemArray[0], Is.EqualTo("Fish")); + + var table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests"); + using var resultDt = table.GetDataTable(); + Assert.That(resultDt.Rows, Has.Count.EqualTo(1)); + Assert.That(resultDt.Rows[0].ItemArray[0], Is.EqualTo("Fish")); + Assert.That(resultDt.Rows[0].ItemArray[1], Is.EqualTo(string.Empty)); + table = db.DiscoverTables(false).First(static t => t.GetRuntimeName() == "DataTableUploadDestinationTests_Archive"); + using var resultDt2 = table.GetDataTable(); + Assert.That(resultDt2.Rows, Has.Count.EqualTo(1)); + Assert.That(resultDt2.Rows[0].ItemArray[0], Is.EqualTo("Fish")); } diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_ResolvedAccordingToStrategy.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_ResolvedAccordingToStrategy.cs index d4f307b3b4..a0377be2e3 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_ResolvedAccordingToStrategy.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_ResolvedAccordingToStrategy.cs @@ -189,13 +189,6 @@ public void BadCSV_TooFewCellsInRow(BadDataHandlingStrategy strategy, bool tryTo "Other People To Investigate", "Dennis,Hes ok,35"); - void Adjust(DelimitedFlatFileDataFlowSource a) - { - a.BadDataHandlingStrategy = strategy; - a.AttemptToResolveNewLinesInRecords = tryToResolve; - a.ThrowOnEmptyFiles = true; - } - switch (strategy) { case BadDataHandlingStrategy.ThrowException: @@ -216,6 +209,15 @@ void Adjust(DelimitedFlatFileDataFlowSource a) default: throw new ArgumentOutOfRangeException(nameof(strategy)); } + + return; + + void Adjust(DelimitedFlatFileDataFlowSource a) + { + a.BadDataHandlingStrategy = strategy; + a.AttemptToResolveNewLinesInRecords = tryToResolve; + a.ThrowOnEmptyFiles = true; + } } [TestCase(BadDataHandlingStrategy.DivertRows, true)] @@ -229,13 +231,6 @@ public void BadCSV_TooFewColumnsOnLastLine(BadDataHandlingStrategy strategy, boo "Frank,Is the greatest,100", "Bob"); - void Adjust(DelimitedFlatFileDataFlowSource a) - { - a.BadDataHandlingStrategy = strategy; - a.AttemptToResolveNewLinesInRecords = tryToResolve; - a.ThrowOnEmptyFiles = true; - } - switch (strategy) { case BadDataHandlingStrategy.ThrowException: @@ -256,6 +251,15 @@ void Adjust(DelimitedFlatFileDataFlowSource a) default: throw new ArgumentOutOfRangeException(nameof(strategy)); } + + return; + + void Adjust(DelimitedFlatFileDataFlowSource a) + { + a.BadDataHandlingStrategy = strategy; + a.AttemptToResolveNewLinesInRecords = tryToResolve; + a.ThrowOnEmptyFiles = true; + } } [Test] diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_Unresolveable.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_Unresolveable.cs index 84dd2c34bf..645ad405de 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_Unresolveable.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/PipelineTests/Sources/DelimitedFileSourceTests_Unresolveable.cs @@ -28,13 +28,6 @@ public void BadCSV_UnclosedQuote(BadDataHandlingStrategy strategy) "Frank,Is the greatest,100", "Frank,Is the greatest,100"); - void Adjust(DelimitedFlatFileDataFlowSource a) - { - a.BadDataHandlingStrategy = strategy; - a.ThrowOnEmptyFiles = true; - a.IgnoreQuotes = false; - } - switch (strategy) { case BadDataHandlingStrategy.ThrowException: @@ -57,6 +50,15 @@ void Adjust(DelimitedFlatFileDataFlowSource a) default: throw new ArgumentOutOfRangeException(nameof(strategy)); } + + return; + + void Adjust(DelimitedFlatFileDataFlowSource a) + { + a.BadDataHandlingStrategy = strategy; + a.ThrowOnEmptyFiles = true; + a.IgnoreQuotes = false; + } } [Test] @@ -70,15 +72,16 @@ public void BadCSV_UnclosedQuote_IgnoreQuotes() "Frank,Is the greatest,100", "Frank,Is the greatest,100"); + var dt2 = RunGetChunk(file, Adjust); + Assert.That(dt2.Rows, Has.Count.EqualTo(5)); + Assert.That(dt2.Rows[1]["Description"], Is.EqualTo("\"Is the greatest")); + return; + static void Adjust(DelimitedFlatFileDataFlowSource a) { a.BadDataHandlingStrategy = BadDataHandlingStrategy.ThrowException; a.ThrowOnEmptyFiles = true; a.IgnoreQuotes = true; } - - var dt2 = RunGetChunk(file, Adjust); - Assert.That(dt2.Rows, Has.Count.EqualTo(5)); - Assert.That(dt2.Rows[1]["Description"], Is.EqualTo("\"Is the greatest")); } } \ No newline at end of file diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs index 4cb46e7163..7a15a2ab2b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/CohortCreationCommands/ExecuteCommandCreateNewCohortByExecutingACohortIdentificationConfiguration.cs @@ -70,19 +70,20 @@ public override void Execute() if (cic == null) return; - if (BasicActivator.IsInteractive) { - var PromptForVersionOnCohortCommit = false; - var PromptForVersionOnCohortCommitSetting = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(s => s.Key == "PromptForVersionOnCohortCommit").FirstOrDefault(); - if (PromptForVersionOnCohortCommitSetting is not null) PromptForVersionOnCohortCommit = Convert.ToBoolean(PromptForVersionOnCohortCommitSetting.Value); - if (PromptForVersionOnCohortCommit && BasicActivator.YesNo("It is recommended to save your cohort as a new version before committing. Would you like to do this?", "Save cohort as new version before committing?")) + if (BasicActivator.IsInteractive) + { + var promptForVersionOnCohortCommit = false; + var promptForVersionOnCohortCommitSetting = BasicActivator.RepositoryLocator.CatalogueRepository.GetAllObjects().FirstOrDefault(static s => s.Key == "PromptForVersionOnCohortCommit"); + if (promptForVersionOnCohortCommitSetting is not null) promptForVersionOnCohortCommit = Convert.ToBoolean(promptForVersionOnCohortCommitSetting.Value); + if (promptForVersionOnCohortCommit && BasicActivator.YesNo("It is recommended to save your cohort as a new version before committing. Would you like to do this?", "Save cohort as new version before committing?")) { var newVersion = new ExecuteCommandCreateVersionOfCohortConfiguration(BasicActivator, cic); newVersion.Execute(); var versions = cic.GetVersions(); cic = versions.Last(); } - } + if (Project == null && BasicActivator.CoreChildProvider is DataExportChildProvider dx) { var projAssociations = dx.AllProjectAssociatedCics diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneExtractionConfiguration.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneExtractionConfiguration.cs index 2fcb3707f1..998843c30e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneExtractionConfiguration.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandCloneExtractionConfiguration.cs @@ -4,7 +4,6 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . -using System; using System.Collections.Generic; using System.Linq; using Rdmp.Core.Curation.Data; @@ -16,48 +15,47 @@ namespace Rdmp.Core.CommandExecution.AtomicCommands; -public class ExecuteCommandCloneExtractionConfiguration : BasicCommandExecution, IAtomicCommand +public class ExecuteCommandCloneExtractionConfiguration : BasicCommandExecution { private readonly ExtractionConfiguration _extractionConfiguration; private readonly IBasicActivateItems _activeItems; - private readonly List toRemove = []; - private readonly List toAdd = []; + private readonly List _toRemove = []; + private readonly List _toAdd = []; + private void CheckForDeprecatedCatalogues() { - if (_extractionConfiguration.SelectedDataSets.Any(sd => sd.GetCatalogue().IsDeprecated) && _activeItems.IsInteractive) + if (!_extractionConfiguration.SelectedDataSets.Any(static sd => sd.GetCatalogue().IsDeprecated) || + !_activeItems.IsInteractive) return; + if (!YesNo( + "There are Deprecated catalogues in this Extraction Configuration. Would you like to replace them with their replacement (where available)?", + "Replace Deprecated Catalogues")) return; + + var repo = _activeItems.RepositoryLocator.CatalogueRepository; + var deprecatedDatasets = _extractionConfiguration.SelectedDataSets.Where(static sd => sd.GetCatalogue().IsDeprecated).ToList(); + var replacedBy = repo.GetExtendedProperties(ExtendedProperty.ReplacedBy).ToArray(); + foreach (var ds in deprecatedDatasets) { - if (YesNo("There are Deprecated catalogues in this Extraction Configuration. Would you like to replace them with their replacement (where available)?", "Replace Deprecated Catalogues")) + var replacement = replacedBy.FirstOrDefault(rb => rb.ReferencedObjectID == ds.GetCatalogue().ID); + if (replacement is null) continue; + + var replacementCatalogue = repo.GetObjectByID(int.Parse(replacement.Value)); + while (replacementCatalogue.IsDeprecated) { - var repo = _activeItems.RepositoryLocator.CatalogueRepository; - var DeprecatedDatasets = _extractionConfiguration.SelectedDataSets.Where(sd => sd.GetCatalogue().IsDeprecated).ToList(); - var replacedBy = repo.GetExtendedProperties(ExtendedProperty.ReplacedBy); - foreach (ISelectedDataSets ds in DeprecatedDatasets) + var replacementCatalogueIsReplacedBy = replacedBy.FirstOrDefault(rb => rb.ReferencedObjectID == replacementCatalogue.ID); + if (replacementCatalogueIsReplacedBy is not null) { - var replacement = replacedBy.Where(rb => rb.ReferencedObjectID == ds.GetCatalogue().ID).FirstOrDefault(); - if (replacement is not null) - { - var replacementCatalogue = repo.GetObjectByID(Int32.Parse(replacement.Value)); - while (replacementCatalogue.IsDeprecated) - { - var replacementCatalogueIsReplacedBy = replacedBy.Where(rb => rb.ReferencedObjectID == replacementCatalogue.ID).FirstOrDefault(); - if(replacementCatalogueIsReplacedBy is not null) - { - //have found further down the tree - replacementCatalogue = repo.GetObjectByID(Int32.Parse(replacementCatalogueIsReplacedBy.Value)); - } - else - { - //there is no replacement - break; - } - } - toRemove.Add(ds.ExtractableDataSet); - toAdd.Add(replacementCatalogue); - } + //have found further down the tree + replacementCatalogue = repo.GetObjectByID(int.Parse(replacementCatalogueIsReplacedBy.Value)); + } + else + { + //there is no replacement + break; } - - } + + _toRemove.Add(ds.ExtractableDataSet); + _toAdd.Add(replacementCatalogue); } } @@ -82,11 +80,12 @@ public override void Execute() CheckForDeprecatedCatalogues(); var clone = _extractionConfiguration.DeepCloneWithNewIDs(); - foreach (ExtractableDataSet ds in toRemove) + foreach (var ds in _toRemove.Cast()) { clone.RemoveDatasetFromConfiguration(ds); } - foreach (Catalogue c in toAdd) + + foreach (var c in _toAdd) { //check if the eds already exis var eds = _activeItems.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", c.ID).FirstOrDefault(); @@ -95,8 +94,10 @@ public override void Execute() eds = new ExtractableDataSet(_activeItems.RepositoryLocator.DataExportRepository, c); eds.SaveToDatabase(); } + clone.AddDatasetToConfiguration(eds); } + Publish((DatabaseEntity)clone.Project); Emphasise(clone); } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeprecate.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeprecate.cs index 4ef4e502d3..b69f72f548 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeprecate.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandDeprecate.cs @@ -49,20 +49,18 @@ private void ExecuteImpl() { o.IsDeprecated = _desiredState; o.SaveToDatabase(); - if (!_desiredState && o.GetType() == typeof(Catalogue))//false is not-depricated + if (!_desiredState && o is Catalogue catalogue) //false is not-deprecated { - var c = (Catalogue) o; var replacedBy = _activeItems.RepositoryLocator.CatalogueRepository.GetExtendedProperties(ExtendedProperty.ReplacedBy); - var replacement = replacedBy.Where(rb => rb.ReferencedObjectID == c.ID).FirstOrDefault(); - if(replacedBy != null) - replacement.DeleteInDatabase(); + var replacement = replacedBy.FirstOrDefault(rb => rb.ReferencedObjectID == catalogue.ID); + replacement?.DeleteInDatabase(); } } - if (!BasicActivator.IsInteractive || _o.Length != 1 || _o[0] is not Catalogue || !_desiredState || !BasicActivator.YesNo("Do you have a replacement Catalogue you want to link?", "Replacement")) return; + var cmd = new ExecuteCommandReplacedBy(BasicActivator, _o[0], null) { PromptToPickReplacement = true diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertToHistoricalCohortVersion.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertToHistoricalCohortVersion.cs index 3a271eb87a..7457d533a9 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertToHistoricalCohortVersion.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandRevertToHistoricalCohortVersion.cs @@ -35,10 +35,11 @@ public ExecuteCommandRevertToHistoricalCohortVersion(IBasicActivateItems activat public override void Execute() { - if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ClonedFrom_ID", _configuration.ID).Where(cic => cic.Version != null && cic.ID == _historicalConfiguration.ID).Any()) + if (!_activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("ClonedFrom_ID", _configuration.ID).Any(cic => cic.Version != null && cic.ID == _historicalConfiguration.ID)) { throw new System.Exception("Historical configuration is not derived from this cohort configuration"); } + base.Execute(); var clone = _historicalConfiguration.CloneIntoExistingConfiguration(ThrowImmediatelyCheckNotifier.Quiet, _configuration,false); Publish(clone); diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandUpdateCatalogueDataLocation.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandUpdateCatalogueDataLocation.cs index 63e5eed8c4..b4cc3c7b1b 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandUpdateCatalogueDataLocation.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandUpdateCatalogueDataLocation.cs @@ -56,10 +56,10 @@ public string Check() var discoveredColumns = _table.DiscoverColumns(); - var foundColumn = discoveredColumns.AsEnumerable() - .Where(dc => dc.GetFullyQualifiedName().Contains(newColumn)).FirstOrDefault(); + var foundColumn = discoveredColumns + .AsEnumerable().FirstOrDefault(dc => dc.GetFullyQualifiedName().Contains(newColumn)); if (foundColumn is null) return $"Unable to find column '{newColumn}' in selected table"; - if (foundColumn.DataType.ToString() != item.ColumnInfo.Data_type) + if (foundColumn.DataType?.ToString() != item.ColumnInfo.Data_type) return $"The data type of column '{newColumn}' is of type '{foundColumn.DataType}'. This does not match the current type of '{item.ColumnInfo.Data_type}'"; } diff --git a/Rdmp.Core/Curation/Data/Catalogue.cs b/Rdmp.Core/Curation/Data/Catalogue.cs index 48b71b8293..38031316aa 100644 --- a/Rdmp.Core/Curation/Data/Catalogue.cs +++ b/Rdmp.Core/Curation/Data/Catalogue.cs @@ -35,8 +35,8 @@ namespace Rdmp.Core.Curation.Data; /// -public class Catalogue : DatabaseEntity, IComparable, ICatalogue, IInjectKnown, - IInjectKnown +public sealed class Catalogue : DatabaseEntity, IComparable, ICatalogue, IInjectKnown, + IInjectKnown, IEquatable { #region Database Properties @@ -844,9 +844,19 @@ public int CompareTo(object obj) return -string.Compare(obj.ToString(), ToString(), StringComparison.CurrentCulture); //sort alphabetically (reverse) - throw new Exception($"Cannot compare {GetType().Name} to {obj.GetType().Name}"); + throw new Exception($"Cannot compare {GetType().Name} to {obj?.GetType().Name}"); } + public override bool Equals(object obj) => obj is Catalogue c && (ReferenceEquals(this, c) || Equals(c)); + + public bool Equals(Catalogue other) => base.Equals(other); + + public override int GetHashCode() => base.GetHashCode(); + + public static bool operator ==(Catalogue left, Catalogue right) => Equals(left, right); + + public static bool operator !=(Catalogue left, Catalogue right) => !Equals(left, right); + /// /// Checks that the Catalogue has a sensible Name (See ). Then checks that there are no missing ColumnInfos /// diff --git a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs index 609e8384b0..2af6edcaa6 100644 --- a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs +++ b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs @@ -13,7 +13,6 @@ using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; using System; -using System.Collections; using System.Collections.Generic; using System.Data; using System.Linq; @@ -25,7 +24,6 @@ namespace Rdmp.Core.Curation.Data.Overview; /// public class OverviewModel { - private readonly ICatalogue _catalogue; private readonly IBasicActivateItems _activator; @@ -46,20 +44,22 @@ public OverviewModel(IBasicActivateItems activator, ICatalogue catalogue) public void Regen(string whereClause) { - DataTable dt = new(); - bool hasExtractionIdentifier = true; - var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault(); + using var dt = new DataTable(); + var hasExtractionIdentifier = true; + var column = _catalogue.CatalogueItems.FirstOrDefault(static ci => ci.ExtractionInformation.IsExtractionIdentifier); if (column is null) { column = _catalogue.CatalogueItems.FirstOrDefault(); hasExtractionIdentifier = false; } + if (column is null) return; + var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); var server = discoveredColumn.Table.Database.Server; using var con = server.GetConnection(); con.Open(); - string populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : ""; + var populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : ""; var sql = $"SELECT {column.ColumnInfo.GetRuntimeName()} FROM {discoveredColumn.Table.GetRuntimeName()} {populatedWhere}"; using var cmd = server.GetCommand(sql, con); cmd.CommandTimeout = 30000; @@ -67,11 +67,11 @@ public void Regen(string whereClause) dt.BeginLoadData(); da.Fill(dt); dt.EndLoadData(); - con.Dispose(); _numberOfRecords = dt.Rows.Count; - _numberOfPeople = hasExtractionIdentifier?dt.AsEnumerable().Select(r => r[column.ColumnInfo.GetRuntimeName()]).ToList().Distinct().Count():0; + _numberOfPeople = hasExtractionIdentifier + ? dt.AsEnumerable().Select(r => r[column.ColumnInfo.GetRuntimeName()]).Distinct().Count() + : 0; GetDataLoads(); - dt.Dispose(); } public int GetNumberOfRecords() @@ -86,7 +86,7 @@ public int GetNumberOfPeople() public Tuple GetStartEndDates(ColumnInfo dateColumn, string whereClause) { - DataTable dt = new(); + using var dt = new DataTable(); var discoveredColumn = _catalogue.CatalogueItems.First().ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); var server = discoveredColumn.Table.Database.Server; @@ -122,41 +122,37 @@ where occurs >1 dt.BeginLoadData(); da.Fill(dt); var latest = dt.AsEnumerable() - .Max(r => r.Field(dateColumn.Name)); + .Max(r => r.Field(dateColumn.Name)); var earliest = dt.AsEnumerable() - .Min(r => r.Field(dateColumn.Name)); - dt = new(); - dt.Rows.Add([earliest, latest]); + .Min(r => r.Field(dateColumn.Name)); + dt.Rows.Clear(); + dt.Rows.Add(earliest, latest); } - con.Dispose(); + return new Tuple(DateTime.Parse(dt.Rows[0].ItemArray[0].ToString()), DateTime.Parse(dt.Rows[0].ItemArray[1].ToString())); } public static DataTable GetCountsByDatePeriod(ColumnInfo dateColumn, string datePeriod, string optionalWhere = "") { - DataTable dt = new(); + var dt = new DataTable(); if (!(new[] { "Day", "Month", "Year" }).Contains(datePeriod)) { throw new Exception("Invalid Date period"); } + var discoveredColumn = dateColumn.Discover(DataAccessContext.InternalDataProcessing); var server = discoveredColumn.Table.Database.Server; using var con = server.GetConnection(); con.Open(); - var dateString = "yyyy-MM"; - switch (datePeriod) + var dateString = datePeriod switch { - case "Day": - dateString = "yyyy-MM-dd"; - break; - case "Month": - dateString = "yyyy-MM"; - break; - case "Year": - dateString = "yyyy"; - break; - } + "Day" => "yyyy-MM-dd", + "Month" => "yyyy-MM", + "Year" => "yyyy", + _ => "yyyy-MM" + }; + if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer) { var sql = @$" @@ -190,51 +186,51 @@ ORDER BY 1 { counts[key]++; } + dt = new DataTable(); foreach (var item in counts) { - DataRow dr = dt.NewRow(); + var dr = dt.NewRow(); dr["YearMonth"] = item.Key; dr["# Records"] = item.Value; dt.Rows.Add(dr); } - dt.EndLoadData(); + dt.EndLoadData(); } - con.Dispose(); + return dt; } private void GetDataLoads() { - _dataLoads = new(); + _dataLoads = new DataTable(); var repo = new MemoryCatalogueRepository(); var qb = new QueryBuilder(null, null); - var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).FirstOrDefault(); - if (columnInfo != null) - { - qb.AddColumn(new ColumnInfoToIColumn(repo, columnInfo)); - qb.AddCustomLine($"{columnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); - var sql = qb.SQL; - var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server; - using var con = server.GetConnection(); - con.Open(); + var columnInfo = _catalogue.CatalogueItems.Where(static c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).FirstOrDefault(); + if (columnInfo == null) return; - using var cmd = server.GetCommand(sql, con); - cmd.CommandTimeout = 30000; - using var da = server.GetDataAdapter(cmd); - _dataLoads.BeginLoadData(); - da.Fill(_dataLoads); - _dataLoads.EndLoadData(); - } + qb.AddColumn(new ColumnInfoToIColumn(repo, columnInfo)); + qb.AddCustomLine($"{columnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); + var sql = qb.SQL; + var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server; + using var con = server.GetConnection(); + con.Open(); + using var cmd = server.GetCommand(sql, con); + cmd.CommandTimeout = 30000; + using var da = server.GetDataAdapter(cmd); + _dataLoads.BeginLoadData(); + da.Fill(_dataLoads); + _dataLoads.EndLoadData(); } public DataTable GetMostRecentDataLoad() { if (_dataLoads == null) GetDataLoads(); if (_dataLoads.Rows.Count == 0) return null; - var maxDataLoadId = _dataLoads.AsEnumerable().Select(r => int.Parse(r[0].ToString())).Distinct().Max(); + + var maxDataLoadId = _dataLoads.AsEnumerable().Select(static r => int.Parse(r[0].ToString())).Max(); var loggingServers = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("CreatedByAssembly", "Rdmp.Core/Databases.LoggingDatabase"); var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).First(); var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server; @@ -253,12 +249,12 @@ public DataTable GetMostRecentDataLoad() dt.BeginLoadData(); loggingDa.Fill(dt); dt.EndLoadData(); - loggingCon.Dispose(); if (dt.Rows.Count > 0) { break; } } + return dt; } @@ -267,7 +263,5 @@ public List GetExtractions() var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).Select(d => d.ID); var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList(); return results; - } - } diff --git a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs index 7c2ffe9057..d6e602bb4b 100644 --- a/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs +++ b/Rdmp.Core/DataExport/DataExtraction/Pipeline/Sources/ExecuteDatasetExtractionSource.cs @@ -323,11 +323,12 @@ public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCance if (includesReleaseIdentifier) foreach (var idx in _extractionIdentifiersidx.Distinct().ToList()) { - var sub = Request.ReleaseIdentifierSubstitutions.Where(s => s.Alias == chunk.Columns[idx].ColumnName).FirstOrDefault(); - if (sub != null && sub.ColumnInfo.ExtractionInformations.FirstOrDefault() != null && sub.ColumnInfo.ExtractionInformations.FirstOrDefault().IsPrimaryKey) + var sub = Request.ReleaseIdentifierSubstitutions.FirstOrDefault(s => s.Alias == chunk.Columns[idx].ColumnName); + if (sub?.ColumnInfo.ExtractionInformations.FirstOrDefault()?.IsPrimaryKey == true) { pks.Add(chunk.Columns[idx]); } + foreach (DataRow r in chunk.Rows) { if (r[idx] == DBNull.Value) @@ -347,10 +348,7 @@ public virtual DataTable GetChunk(IDataLoadEventListener listener, GracefulCance } _timeSpentCalculatingDISTINCT.Stop(); - foreach (string name in Request.ColumnsToExtract.Where(c => ((ExtractableColumn)(c)).CatalogueExtractionInformation.IsPrimaryKey).Select(column => ((ExtractableColumn)column).CatalogueExtractionInformation.ToString())) - { - pks.Add(chunk.Columns[name]); - } + pks.AddRange(Request.ColumnsToExtract.Where(static c => ((ExtractableColumn)c).CatalogueExtractionInformation.IsPrimaryKey).Select(static column => ((ExtractableColumn)column).CatalogueExtractionInformation.ToString()).Select(name => chunk.Columns[name])); chunk.PrimaryKey = pks.ToArray(); return chunk; diff --git a/Rdmp.Core/DataLoad/Engine/Job/JobFactory.cs b/Rdmp.Core/DataLoad/Engine/Job/JobFactory.cs index 8cd0cb83be..86d2d8ef84 100644 --- a/Rdmp.Core/DataLoad/Engine/Job/JobFactory.cs +++ b/Rdmp.Core/DataLoad/Engine/Job/JobFactory.cs @@ -31,8 +31,7 @@ public IDataLoadJob Create(IRDMPPlatformRepositoryServiceLocator repositoryLocat HICDatabaseConfiguration configuration) { var description = _loadMetadata.Name; - LoadDirectory loadDirectory; - loadDirectory = new LoadDirectory(_loadMetadata.LocationOfForLoadingDirectory, _loadMetadata.LocationOfForArchivingDirectory, _loadMetadata.LocationOfExecutablesDirectory, _loadMetadata.LocationOfCacheDirectory); + var loadDirectory = new LoadDirectory(_loadMetadata.LocationOfForLoadingDirectory, _loadMetadata.LocationOfForArchivingDirectory, _loadMetadata.LocationOfExecutablesDirectory, _loadMetadata.LocationOfCacheDirectory); return new DataLoadJob(repositoryLocator, description, _logManager, _loadMetadata, loadDirectory, listener, configuration); diff --git a/Rdmp.Core/DataLoad/Engine/Job/Scheduling/SingleScheduledJobFactory.cs b/Rdmp.Core/DataLoad/Engine/Job/Scheduling/SingleScheduledJobFactory.cs index b707d24f4f..e3a21a3939 100644 --- a/Rdmp.Core/DataLoad/Engine/Job/Scheduling/SingleScheduledJobFactory.cs +++ b/Rdmp.Core/DataLoad/Engine/Job/Scheduling/SingleScheduledJobFactory.cs @@ -38,9 +38,7 @@ public override bool HasJobs() => protected override ScheduledDataLoadJob CreateImpl(IRDMPPlatformRepositoryServiceLocator repositoryLocator, IDataLoadEventListener listener, HICDatabaseConfiguration configuration) { - LoadDirectory loadDirectory; - - loadDirectory = new LoadDirectory(LoadMetadata.LocationOfForLoadingDirectory, LoadMetadata.LocationOfForArchivingDirectory, LoadMetadata.LocationOfExecutablesDirectory, LoadMetadata.LocationOfCacheDirectory); + var loadDirectory = new LoadDirectory(LoadMetadata.LocationOfForLoadingDirectory, LoadMetadata.LocationOfForArchivingDirectory, LoadMetadata.LocationOfExecutablesDirectory, LoadMetadata.LocationOfCacheDirectory); return new ScheduledDataLoadJob(repositoryLocator, JobDescription, LogManager, LoadMetadata, loadDirectory, listener, configuration) diff --git a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs index a286795be7..c6a61249c7 100644 --- a/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs +++ b/Rdmp.Core/DataLoad/Engine/Pipeline/Destinations/DataTableUploadDestination.cs @@ -279,10 +279,11 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener _firstTime = false; } - if (IncludeTimeStamp && !_discoveredTable.DiscoverColumns().Where(c => c.GetRuntimeName() == _extractionTimeStamp).Any()) + if (IncludeTimeStamp && _discoveredTable.DiscoverColumns().All(c => c.GetRuntimeName() != _extractionTimeStamp)) { _discoveredTable.AddColumn(_extractionTimeStamp, new DatabaseTypeRequest(typeof(DateTime)), true, 30000); } + if (IndexTables) { var indexes = UserDefinedIndexes.Count != 0 ? UserDefinedIndexes : pkColumns.Select(c => c.ColumnName); @@ -353,11 +354,11 @@ public DataTable ProcessPipelineData(DataTable toProcess, IDataLoadEventListener // look up the original value and check we've not already extected the same value under a different release ID var privateIdentifierField = _externalCohortTable.PrivateIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field var releaseIdentifierField = _externalCohortTable.ReleaseIdentifierField.Split('.').Last()[1..^1];//remove the "[]" from the identifier field - DiscoveredTable cohortTable = _externalCohortTable.DiscoverCohortTable(); - var lookupDT = cohortTable.GetDataTable(); + var cohortTable = _externalCohortTable.DiscoverCohortTable(); + using var lookupDT = cohortTable.GetDataTable(); var releaseIdIndex = lookupDT.Columns.IndexOf(releaseIdentifierField); var privateIdIndex = lookupDT.Columns.IndexOf(privateIdentifierField); - var foundRow = lookupDT.Rows.Cast().Where(r => r.ItemArray[releaseIdIndex].ToString() == row[pkCol.ColumnName].ToString()).FirstOrDefault(); + var foundRow = lookupDT.Rows.Cast().FirstOrDefault(r => r.ItemArray[releaseIdIndex].ToString() == row[pkCol.ColumnName].ToString()); if (foundRow is not null) { var originalValue = foundRow.ItemArray[privateIdIndex]; @@ -667,16 +668,16 @@ public void Dispose(IDataLoadEventListener listener, Exception pipelineFailureEx _discoveredTable.CreatePrimaryKey(AlterTimeout, pkColumnsToCreate); } } - if (UseTrigger && _discoveredTable.DiscoverColumns().Where(col => col.IsPrimaryKey).Any()) //can't use triggers without a PK - { + if (UseTrigger && _discoveredTable?.DiscoverColumns().Any(static col => col.IsPrimaryKey) == true) //can't use triggers without a PK + { var factory = new TriggerImplementerFactory(_database.Server.DatabaseType); - var _triggerImplementer = factory.Create(_discoveredTable); - var currentStatus = _triggerImplementer.GetTriggerStatus(); + var triggerImplementer = factory.Create(_discoveredTable); + var currentStatus = triggerImplementer.GetTriggerStatus(); if (currentStatus == TriggerStatus.Missing) try { - _triggerImplementer.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); + triggerImplementer.CreateTrigger(ThrowImmediatelyCheckNotifier.Quiet); } catch (Exception e) { diff --git a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs index abd4bf7b2a..11a571fc29 100644 --- a/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs +++ b/Rdmp.Core/DataLoad/Modules/Mutilators/RegexRedactionMutilator.cs @@ -106,14 +106,16 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di using var da = table.Database.Server.GetDataAdapter(cmd); da.Fill(dt); } + dt.EndLoadData(); var redactionUpates = dt.Clone(); - var columnInfo = relatedCatalogues.SelectMany(c => c.CatalogueItems).ToArray().Select(ci => ci.ColumnInfo).Where(ci => ci.GetRuntimeName() == column.GetRuntimeName()).FirstOrDefault(); + var columnInfo = relatedCatalogues.SelectMany(static c => c.CatalogueItems).ToArray().Select(static ci => ci.ColumnInfo).FirstOrDefault(ci => ci.GetRuntimeName() == column.GetRuntimeName()); if (columnInfo is null) { job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Error, "Unable to find the related column info")); return; } + foreach (DataRow row in dt.Rows) { try @@ -123,21 +125,23 @@ protected override void MutilateTable(IDataLoadJob job, ITableInfo tableInfo, Di catch (Exception e) { job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, $"{e.Message}")); - } } + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Regex Redaction mutilator found {dt.Rows.Count} redactions.")); if (redactionsToSaveTable.Rows.Count == 0) return; - for (int i = 0; i < pksToSave.Rows.Count; i++) + + for (var i = 0; i < pksToSave.Rows.Count; i++) { pksToSave.Rows[i]["ID"] = i + 1; } + job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Creating Temporary tables")); var t1 = table.Database.CreateTable(nameof(RegexRedactionHelper.Constants.pksToSave_Temp), pksToSave); var t2 = table.Database.CreateTable(nameof(RegexRedactionHelper.Constants.redactionsToSaveTable_Temp), redactionsToSaveTable); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Saving Redactions")); - var _server = relatedCatalogues.First().GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); - RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, _server, Timeout * 1000); + var server = relatedCatalogues.First().GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false); + RegexRedactionHelper.SaveRedactions(job.RepositoryLocator.CatalogueRepository, t1, t2, server, Timeout * 1000); t1.Drop(); t2.Drop(); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Performing join update")); diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs index 9e4fe29088..7db441b3a8 100644 --- a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs @@ -37,12 +37,9 @@ public override void Execute() { var order = _target.Order; - var filters = _target.FilterContainer.GetFilters().Where(f => f is ConcreteFilter).Select(f => (ConcreteFilter)f).ToArray(); - Array.Sort( - filters, - delegate (ConcreteFilter a, ConcreteFilter b) { return a.Order.CompareTo(b.Order); } - ); - if (!filters.All(c => c.Order != order)) + var filters = _target.FilterContainer.GetFilters().OfType().OrderBy(static f => f.Order).ToArray(); + // Conflict? Create a space by moving earlier filters down one, later filters up one + if (filters.Any(c => c.Order == order)) { foreach (var orderable in filters) { @@ -55,6 +52,7 @@ public override void Execute() ((ISaveable)orderable).SaveToDatabase(); } } + _source.Order = order; _source.SaveToDatabase(); Publish(_target.FilterContainer); diff --git a/Rdmp.UI/ExtractionUIs/JoinsAndLookups/LookupConfigurationUI.cs b/Rdmp.UI/ExtractionUIs/JoinsAndLookups/LookupConfigurationUI.cs index 0eccd004ea..02ebde98f1 100644 --- a/Rdmp.UI/ExtractionUIs/JoinsAndLookups/LookupConfigurationUI.cs +++ b/Rdmp.UI/ExtractionUIs/JoinsAndLookups/LookupConfigurationUI.cs @@ -259,29 +259,38 @@ private bool ValidateUserInput() HandleError("No Lookup table selected"); return false; } - if (PKRelations.Where(d => d.SelectedItem != null).Count() == 0 || FKRelations.Where(d => d.SelectedItem != null).Count() == 0) + + var pk = PKRelations.Count(static d => d.SelectedItem != null); + var fk = FKRelations.Count(static d => d.SelectedItem != null); + + if (pk == 0 || fk == 0) { HandleError("At least one PK FK mapping must be set"); return false; } - if (PKRelations.Where(d => d.SelectedItem != null).Count() != FKRelations.Where(d => d.SelectedItem != null).Count()) + + if (pk != fk) { HandleError("Must have a 1-to-1 mapping of PK and FK mappings"); return false; } - if (Descriptions.Where(d => d.SelectedItem != null).Count() == 0) + + var descColumnInfos = Descriptions.Where(static d => d.SelectedItem != null).Select(static d => ((ColumnInfo)d.SelectedItem).ID).ToArray(); + if (descColumnInfos.Length == 0) { HandleError("At least one Description column must be set"); return false; } - var descColumnInfos = Descriptions.Where(d => d.SelectedItem != null).Select(d => ((ColumnInfo)d.SelectedItem).ID); - var pkColumnInfos = PKRelations.Where(d => d.SelectedItem != null).Select(d => ((ColumnInfo)d.SelectedItem).ID); - var fkColumnInfos = FKRelations.Where(d => d.SelectedItem != null).Select(d => ((ColumnInfo)d.SelectedItem).ID); + + var pkColumnInfos = PKRelations.Where(static d => d.SelectedItem != null).Select(static d => ((ColumnInfo)d.SelectedItem).ID); + var fkColumnInfos = FKRelations.Where(static d => d.SelectedItem != null).Select(static d => ((ColumnInfo)d.SelectedItem).ID); + if (pkColumnInfos.Intersect(descColumnInfos).Any() || fkColumnInfos.Intersect(descColumnInfos).Any()) { HandleError("A Description Column cannot be used in the PK FK mapping"); return false; } + return true; } diff --git a/Rdmp.UI/LoadExecutionUIs/ExecuteLoadMetadataUI.cs b/Rdmp.UI/LoadExecutionUIs/ExecuteLoadMetadataUI.cs index af64b9cecf..424f03fd44 100644 --- a/Rdmp.UI/LoadExecutionUIs/ExecuteLoadMetadataUI.cs +++ b/Rdmp.UI/LoadExecutionUIs/ExecuteLoadMetadataUI.cs @@ -77,12 +77,12 @@ public override void SetDatabaseObject(IActivateItems activator, LoadMetadata da if (activator.IsInteractive) { - var ShowYestoAllNotoAlldataloadcheck = false; - var ShowYestoAllNotoAlldataloadcheckSetting = activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(s => s.Key == "ToggleYestoAllNotoAlldataloadcheck").FirstOrDefault(); - if (ShowYestoAllNotoAlldataloadcheckSetting is not null) ShowYestoAllNotoAlldataloadcheck = Convert.ToBoolean(ShowYestoAllNotoAlldataloadcheckSetting.Value); - checkAndExecuteUI1.AllowsYesNoToAll = ShowYestoAllNotoAlldataloadcheck; - + var showYestoAllNotoAlldataloadcheck = false; + var showYestoAllNotoAlldataloadcheckSetting = activator.RepositoryLocator.CatalogueRepository.GetAllObjects().FirstOrDefault(static s => s.Key == "ToggleYestoAllNotoAlldataloadcheck"); + if (showYestoAllNotoAlldataloadcheckSetting is not null) showYestoAllNotoAlldataloadcheck = Convert.ToBoolean(showYestoAllNotoAlldataloadcheckSetting.Value); + checkAndExecuteUI1.AllowsYesNoToAll = showYestoAllNotoAlldataloadcheck; } + SetButtonStates(null, null); SetLoadProgressGroupBoxState(); diff --git a/Rdmp.UI/SimpleDialogs/InstanceSettings.cs b/Rdmp.UI/SimpleDialogs/InstanceSettings.cs index cf00011e75..6dfbad2697 100644 --- a/Rdmp.UI/SimpleDialogs/InstanceSettings.cs +++ b/Rdmp.UI/SimpleDialogs/InstanceSettings.cs @@ -58,14 +58,14 @@ private void AddTooltip(Control c, string propertyName) private void RegisterCheckbox(CheckBox cb, string propertyName) { - var prop = _settings.Where(s => s.Key == propertyName).FirstOrDefault(); - var value = false; + var prop = _settings.FirstOrDefault(s => s.Key == propertyName); if (prop is null) { prop = new Setting(_activator.RepositoryLocator.CatalogueRepository, propertyName, Convert.ToString(false)); prop.SaveToDatabase(); } - value = Convert.ToBoolean(prop.Value); + + var value = Convert.ToBoolean(prop.Value); checkboxDictionary.Add(cb, prop); cb.Checked = value; diff --git a/Rdmp.UI/SimpleDialogs/NewfindUI.cs b/Rdmp.UI/SimpleDialogs/NewfindUI.cs index 9d65abf063..a6c9ddac58 100644 --- a/Rdmp.UI/SimpleDialogs/NewfindUI.cs +++ b/Rdmp.UI/SimpleDialogs/NewfindUI.cs @@ -38,12 +38,12 @@ public partial class NewfindUI : Form private readonly bool _showReplaceOptions = false; - private void SimulateClickForAutoFilter() + private void SimulateClickForAutoFilter() { - var item = newFindToolStrip.Items.Find((typeof(T)).Name, false).FirstOrDefault(); - if (item is not null) - item.PerformClick(); + var item = newFindToolStrip.Items.Find(typeof(T2).Name, false).FirstOrDefault(); + item?.PerformClick(); } + private void PresetFiltersBasedOnFocusItem(RDMPUserControl focusItem) { var focusItemType = focusItem.GetType(); diff --git a/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs b/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs index 9021c7b789..5d47ac564f 100644 --- a/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs +++ b/Rdmp.UI/Wizard/CreateNewDataExtractionProjectUI.cs @@ -59,17 +59,15 @@ public partial class CreateNewDataExtractionProjectUI : RDMPForm private void GetNextProjectNumber(IActivateItems activator) { - - var AutoSuggestProjectNumbers = false; - var AutoSuggestProjectNumbersSetting = activator.RepositoryLocator.CatalogueRepository.GetAllObjects().Where(s => s.Key == "AutoSuggestProjectNumbers").FirstOrDefault(); - if (AutoSuggestProjectNumbersSetting is not null) AutoSuggestProjectNumbers = Convert.ToBoolean(AutoSuggestProjectNumbersSetting.Value); + var autoSuggestProjectNumbers = false; + var autoSuggestProjectNumbersSetting = activator.RepositoryLocator.CatalogueRepository.GetAllObjects().FirstOrDefault(static s => s.Key == "AutoSuggestProjectNumbers"); + if (autoSuggestProjectNumbersSetting is not null) autoSuggestProjectNumbers = Convert.ToBoolean(autoSuggestProjectNumbersSetting.Value); _existingProjects = activator.RepositoryLocator.DataExportRepository.GetAllObjects(); - if (AutoSuggestProjectNumbers) - { - var highestNumber = _existingProjects.Max(p => p.ProjectNumber); - tbProjectNumber.Text = highestNumber == null ? "1" : (highestNumber.Value + 1).ToString(); - } + if (!autoSuggestProjectNumbers) return; + + var highestNumber = _existingProjects.Max(static p => p.ProjectNumber); + tbProjectNumber.Text = highestNumber == null ? "1" : (highestNumber.Value + 1).ToString(); } public CreateNewDataExtractionProjectUI(IActivateItems activator) : base(activator) diff --git a/Tests.Common/UnitTests.cs b/Tests.Common/UnitTests.cs index ac85ffa74b..2395d65f5f 100644 --- a/Tests.Common/UnitTests.cs +++ b/Tests.Common/UnitTests.cs @@ -124,7 +124,7 @@ protected virtual void SetUp() public static T WhenIHaveA(MemoryDataExportRepository repository) where T : DatabaseEntity { if (typeof(T) == typeof(Catalogue)) - return (T)(object)Save(new Catalogue(repository, "Mycata")); + return Save(new Catalogue(repository, "Mycata")) as T; if (typeof(T) == typeof(ExtendedProperty)) From 179c5bb522f639b0c2618ffe8e276d5d77e56c0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2024 00:34:26 +0000 Subject: [PATCH 2/8] Bump actions/setup-dotnet from 4.1.0 to 4.2.0 Bumps [actions/setup-dotnet](https://github.com/actions/setup-dotnet) from 4.1.0 to 4.2.0. - [Release notes](https://github.com/actions/setup-dotnet/releases) - [Commits](https://github.com/actions/setup-dotnet/compare/v4.1.0...v4.2.0) --- updated-dependencies: - dependency-name: actions/setup-dotnet dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 10 +++++----- .github/workflows/codeql.yml | 2 +- .github/workflows/docker.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ee9ddb0ec..b26b4f3f09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 - name: Populate Databases.yaml shell: bash run: | @@ -86,7 +86,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 - name: Populate Databases.yaml shell: bash run: | @@ -154,13 +154,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 - name: Determine RDMP build version id: version shell: bash run: perl -ne "print \"rdmpversion=\$1\n\" if /AssemblyInformationalVersion\(\"([0-9a-z.-]+)\"\)/;" SharedAssemblyInfo.cs >> $GITHUB_OUTPUT - name: Setup .NET Core - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 with: dotnet-version: 7.0.x - name: BundleSource @@ -293,7 +293,7 @@ jobs: path: ${{ github.workspace }}/ key: ${{ github.sha }}-your-cache-key-bundled - name: setup .NET - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 - name: Determine RDMP build version id: version shell: bash diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 478de9e5c8..eff78bfcba 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: setup .NET - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5625ce13db..701d02a5bc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: with: submodules: true - name: Setup .NET Core - uses: actions/setup-dotnet@v4.1.0 + uses: actions/setup-dotnet@v4.2.0 with: dotnet-version: 6.0.x - name: Cache Nuget From dbe879812d3ca45e36639c05fe4020f6efb5287a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 00:09:41 +0000 Subject: [PATCH 3/8] Bump NUnit.Analyzers from 4.4.0 to 4.5.0 Bumps [NUnit.Analyzers](https://github.com/nunit/nunit.analyzers) from 4.4.0 to 4.5.0. - [Release notes](https://github.com/nunit/nunit.analyzers/releases) - [Changelog](https://github.com/nunit/nunit.analyzers/blob/master/CHANGES.md) - [Commits](https://github.com/nunit/nunit.analyzers/compare/4.4.0...4.5.0) --- updated-dependencies: - dependency-name: NUnit.Analyzers dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5309eae97e..b7d2f55cff 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -41,7 +41,7 @@ - + From e124f03a0ef006ed85888b9d894cbdf88348b850 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 00:08:01 +0000 Subject: [PATCH 4/8] Bump Minio from 6.0.3 to 6.0.4 Bumps [Minio](https://github.com/minio/minio-dotnet) from 6.0.3 to 6.0.4. - [Release notes](https://github.com/minio/minio-dotnet/releases) - [Commits](https://github.com/minio/minio-dotnet/compare/6.0.3...6.0.4) --- updated-dependencies: - dependency-name: Minio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b7d2f55cff..71bb18b42f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,7 +16,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From 90dfdcfa44faf96eda62e1facbdfd69d7c46973a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:40:41 +0000 Subject: [PATCH 5/8] Bump the aws-sdk group across 1 directory with 4 updates Bumps the aws-sdk group with 4 updates in the / directory: [AWSSDK.S3](https://github.com/aws/aws-sdk-net), [AWSSDK.SecurityToken](https://github.com/aws/aws-sdk-net), [AWSSDK.SSO](https://github.com/aws/aws-sdk-net) and [AWSSDK.SSOOIDC](https://github.com/aws/aws-sdk-net). Updates `AWSSDK.S3` from 3.7.410.6 to 3.7.411 - [Release notes](https://github.com/aws/aws-sdk-net/releases) - [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD) - [Commits](https://github.com/aws/aws-sdk-net/commits/3.7.411.0) Updates `AWSSDK.SecurityToken` from 3.7.401.13 to 3.7.401.21 - [Release notes](https://github.com/aws/aws-sdk-net/releases) - [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD) - [Commits](https://github.com/aws/aws-sdk-net/commits) Updates `AWSSDK.SSO` from 3.7.400.64 to 3.7.400.72 - [Release notes](https://github.com/aws/aws-sdk-net/releases) - [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD) - [Commits](https://github.com/aws/aws-sdk-net/commits) Updates `AWSSDK.SSOOIDC` from 3.7.400.64 to 3.7.400.72 - [Release notes](https://github.com/aws/aws-sdk-net/releases) - [Changelog](https://github.com/aws/aws-sdk-net/blob/main/SDK.CHANGELOG.MD) - [Commits](https://github.com/aws/aws-sdk-net/commits) --- updated-dependencies: - dependency-name: AWSSDK.S3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: aws-sdk - dependency-name: AWSSDK.SecurityToken dependency-type: direct:production update-type: version-update:semver-patch dependency-group: aws-sdk - dependency-name: AWSSDK.SSO dependency-type: direct:production update-type: version-update:semver-patch dependency-group: aws-sdk - dependency-name: AWSSDK.SSOOIDC dependency-type: direct:production update-type: version-update:semver-patch dependency-group: aws-sdk ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 71bb18b42f..b3ffd3d88e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,9 +1,9 @@ - - - - + + + + From 056bfebc35178850bd3189ae8b202f8f49775de9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:39:50 +0000 Subject: [PATCH 6/8] Bump coverlet.collector from 6.0.2 to 6.0.3 Bumps [coverlet.collector](https://github.com/coverlet-coverage/coverlet) from 6.0.2 to 6.0.3. - [Release notes](https://github.com/coverlet-coverage/coverlet/releases) - [Commits](https://github.com/coverlet-coverage/coverlet/compare/v6.0.2...v6.0.3) --- updated-dependencies: - dependency-name: coverlet.collector dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index b3ffd3d88e..796e252fc0 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -44,6 +44,6 @@ - + \ No newline at end of file From 2d60a6ca21798911dcf39ad751663a8d999328ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2024 00:03:15 +0000 Subject: [PATCH 7/8] Bump YamlDotNet from 16.2.1 to 16.3.0 Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 16.2.1 to 16.3.0. - [Release notes](https://github.com/aaubry/YamlDotNet/releases) - [Commits](https://github.com/aaubry/YamlDotNet/compare/v16.2.1...v16.3.0) --- updated-dependencies: - dependency-name: YamlDotNet dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 796e252fc0..1f0a89608d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,7 +27,7 @@ - + From fe07c5e798191504d01d567cc4853f12d6fbbaa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:34:01 +0000 Subject: [PATCH 8/8] Bump NUnit from 4.3.0 to 4.3.2 Bumps [NUnit](https://github.com/nunit/nunit) from 4.3.0 to 4.3.2. - [Release notes](https://github.com/nunit/nunit/releases) - [Changelog](https://github.com/nunit/nunit/blob/main/CHANGES.md) - [Commits](https://github.com/nunit/nunit/compare/4.3.0...4.3.2) --- updated-dependencies: - dependency-name: NUnit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 796e252fc0..d380912c68 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -33,7 +33,7 @@ - +