Task/rdmp 224 dqe update

Application/ResearchDataManagementPlatform/WindowManagement/ActivateItems.cs

@@ -386,7 +386,7 @@ private T Activate<T, T2>(T2 databaseObject, Image<Rgba32> tabImage)
 
         uiInstance.SetDatabaseObject(this, databaseObject);
 
-        if (insertIndex is not null)
+        if (insertIndex is not null && _mainDockPanel.ActivePane is not null)
         {
             _mainDockPanel.ActivePane.SetContentIndex(floatable, (int)insertIndex);
         }

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 - Build on and target .Net 9 rather than 8
+- Add DQE Updater Mutilator for Data Loads see [DQE Post Load runner](./Documentation/DataLoadEngine/DQEPostLoadRunner.md)
+
 ## [8.4.2] - 2024-12-18
 
 - Fix issue with MEF constructing Remote Table Attachers

Documentation/DataLoadEngine/DQEPostLoadRunner.md

@@ -0,0 +1,12 @@
+# DQE Post Load Runner
+
+The DQE post-load runner can be used to automatically perform a DQE update once a data load completes.
+The runner attempts to reuse any existing DQE results that have been unaffected by the data load, however this process can still be slow if the catalogue data is large and/or complex.
+
+## Requirements
+The DQE post-load runner requires an existing DQE result to exist, otherwise it will fail.
+
+## Configuration
+The runner makes a number of queries to the database, the timeout for these commands is configurable via the timeout option.
+
+

Rdmp.Core/CommandLine/Options/DqeOptions.cs

@@ -16,4 +16,10 @@ public class DqeOptions : RDMPCommandLineOptions
 {
     [Option('c', "Catalogue", HelpText = "ID of the Catalogue to run the DQE on", Required = true)]
     public string Catalogue { get; set; }
+
+    [Option('d', "DataLoad", HelpText = "ID of the Data Load to run the DQE on. Adds new data to existing DQE results if they exist", Required = false)]
+    public string DataLoadUpdateID { get; set; }
+
+    [Option('t', "Timeout", HelpText = "How long(in seconds) each internal SQL command should brun for before timing out")]
+    public int CommandTimeout { get; set; }
 }

Rdmp.Core/CommandLine/Runners/DqeRunner.cs

@@ -5,6 +5,7 @@
 // You should have received a copy of the GNU General Public License along with RDMP. If not, see <https://www.gnu.org/licenses/>.
 
 using System;
+using Org.BouncyCastle.Security.Certificates;
 using Rdmp.Core.CommandLine.Options;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.DataFlowPipeline;
@@ -29,12 +30,19 @@ public override int Run(IRDMPPlatformRepositoryServiceLocator repositoryLocator,
         ICheckNotifier checkNotifier, GracefulCancellationToken token)
     {
         var catalogue = GetObjectFromCommandLineString<Catalogue>(repositoryLocator, _options.Catalogue);
+        int? dataLoadID = null;
+        if (_options.DataLoadUpdateID != null)
+            dataLoadID = int.Parse(_options.DataLoadUpdateID);
+
         var report = new CatalogueConstraintReport(catalogue, SpecialFieldNames.DataLoadRunID);
 
         switch (_options.Command)
         {
             case CommandLineActivity.run:
-                report.GenerateReport(catalogue, listener, token.AbortToken);
+                if (dataLoadID is not null)
+                    report.UpdateReport(catalogue, (int)dataLoadID, _options.CommandTimeout, listener, token.AbortToken);
+                else
+                    report.GenerateReport(catalogue, listener, token.AbortToken);
                 return 0;
 
             case CommandLineActivity.check:

Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs

@@ -208,4 +208,4 @@ public AggregateFilter ShallowClone(AggregateFilterContainer into)
         CopyShallowValuesTo(clone);
         return clone;
     }
-}
+}

Rdmp.Core/Curation/Data/ExtractionFilter.cs

@@ -187,4 +187,4 @@ public override void DeleteInDatabase()
 
         base.DeleteInDatabase();
     }
-}
+}

Rdmp.Core/DataLoad/Engine/LoadExecution/Components/Runtime/AttacherRuntimeTask.cs

@@ -11,6 +11,7 @@
 using Rdmp.Core.DataLoad.Engine.Attachers;
 using Rdmp.Core.DataLoad.Engine.Job;
 using Rdmp.Core.DataLoad.Engine.LoadExecution.Components.Arguments;
+using Rdmp.Core.DataLoad.Modules.Attachers;
 using Rdmp.Core.Repositories;
 using Rdmp.Core.ReusableLibraryCode.Checks;
 using Rdmp.Core.ReusableLibraryCode.Progress;
@@ -29,6 +30,8 @@ public class AttacherRuntimeTask : RuntimeTask, IMEFRuntimeTask
     public AttacherRuntimeTask(IProcessTask task, RuntimeArgumentCollection args)
         : base(task, args)
     {
+
+        //RequestsExternalDatabaseCreation
         //All attachers must be marked as mounting stages, and therefore we can pull out the RAW Server and Name
         var mountingStageArgs = args.StageSpecificArguments;
         if (mountingStageArgs.LoadStage != LoadStage.Mounting)

Rdmp.Core/DataLoad/Modules/Mutilators/DQEPostLoadRunner.cs

@@ -27,6 +27,8 @@ namespace Rdmp.Core.DataLoad.Modules.Mutilators;
 public class DQEPostLoadRunner : IMutilateDataTables
 {
 
+    [DemandsInitialization("Timeout length for each query required to run the DQE update",defaultValue:50000)]
+    public int Timeout { get; set; }
     public void Check(ICheckNotifier notifier)
     {
     }
@@ -73,7 +75,9 @@ public ExitCodeType Mutilate(IDataLoadJob job)
             DqeOptions options = new()
             {
                 Catalogue = catalogue.ID.ToString(),
-                Command = CommandLineActivity.run
+                DataLoadUpdateID = job.DataLoadInfo.ID.ToString(),
+                Command = CommandLineActivity.run,
+                CommandTimeout = Timeout
             };
             var runner = RunnerFactory.CreateRunner(new ThrowImmediatelyActivator(job.RepositoryLocator), options);
             runner.Run(job.RepositoryLocator, ThrowImmediatelyDataLoadEventListener.Quiet, new AcceptAllCheckNotifier(),

Rdmp.Core/DataLoad/Triggers/DiffDatabaseDataFetcher.cs

@@ -10,6 +10,7 @@
 using System.Text;
 using FAnsi;
 using FAnsi.Discovery;
+using MongoDB.Driver;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.Curation.Data.Spontaneous;
 using Rdmp.Core.QueryBuilding;
@@ -113,7 +114,7 @@ public void FetchData(ICheckNotifier checkNotifier)
                 CheckResult.Success));
 
             GetInsertData(server, database, checkNotifier);
-            GetUpdatetData(server, database, checkNotifier);
+            GetUpdatedData(server, database, checkNotifier);
         }
         catch (Exception e)
         {
@@ -163,7 +164,7 @@ private void GetInsertData(DiscoveredServer server, DiscoveredDatabase database,
     }
 
 
-    private void GetUpdatetData(DiscoveredServer server, DiscoveredDatabase database, ICheckNotifier checkNotifier)
+    private void GetUpdatedData(DiscoveredServer server, DiscoveredDatabase database, ICheckNotifier checkNotifier)
     {
         const string archive = "archive";
         const string zzArchive = "zzarchivezz";
@@ -191,7 +192,8 @@ private void GetUpdatetData(DiscoveredServer server, DiscoveredDatabase database
 --Records which appear in the archive
 SELECT top {{0}}
 {{6}},
-{{7}}
+{{7}},
+{{8}}
 FROM    {{1}}
 CROSS APPLY
         (
@@ -200,7 +202,7 @@ SELECT  TOP 1 {{2}}.*
         WHERE
          {{3}}
          order by {syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)} desc
-        ) {{8}}
+        ) {{9}}
 where
 {{1}}.{{4}} = {{5}}";
                 break;
@@ -214,13 +216,14 @@ SELECT  TOP 1 {{2}}.*
 /*Records which appear in the archive*/
 SELECT
 {{6}},
-{{7}}
+{{7}},
+{{8}}
 FROM
 {{1}}
 Join
-{{2}} {{8}} on {whereStatement.Replace(archiveTableName, archive)}
+{{2}} {{9}} on {whereStatement.Replace(archiveTableName, archive)}
  AND
- {{8}}.{{9}} = (select max({syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)}) from {{2}} s where {whereStatement.Replace(archiveTableName, archive).Replace(tableName, "s")})
+ {{9}}.{{10}} = (select max({syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)}) from {{2}} s where {whereStatement.Replace(archiveTableName, archive).Replace(tableName, "s")})
  where
   {{1}}.{{4}} = {{5}}
 
@@ -241,8 +244,9 @@ SELECT  TOP 1 {{2}}.*
             _dataLoadRunID, //{5}
             GetSharedColumnsSQL(tableName), //{6}
             GetSharedColumnsSQLWithColumnAliasPrefix(archive, zzArchive), //{7}
-            archive, //{8}
-            syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom)
+            GetHICSpecialColumns(archive, zzArchive),//{8}
+            archive, //{9}
+            syntaxHelper.EnsureWrapped(SpecialFieldNames.ValidFrom) //{10}
         );
 
         var dtComboTable = new DataTable();
@@ -253,11 +257,15 @@ SELECT  TOP 1 {{2}}.*
 
         //add the columns from the combo table to both views
         foreach (DataColumn col in dtComboTable.Columns)
+        {
             if (!col.ColumnName.StartsWith(zzArchive, StringComparison.InvariantCultureIgnoreCase))
             {
                 Updates_New.Columns.Add(col.ColumnName, col.DataType);
                 Updates_Replaced.Columns.Add(col.ColumnName, col.DataType);
             }
+        }
+        Updates_Replaced.Columns.Add(SpecialFieldNames.DataLoadRunID, typeof(int));
+        Updates_Replaced.Columns.Add(SpecialFieldNames.ValidFrom, typeof(DateTime));
 
         foreach (DataRow fromRow in dtComboTable.Rows)
         {
@@ -272,6 +280,13 @@ SELECT  TOP 1 {{2}}.*
         }
     }
 
+    private string GetHICSpecialColumns(string tableName, string columnAliasPrefix = "")
+    {
+        return $@"{tableName}.{SpecialFieldNames.DataLoadRunID} as {columnAliasPrefix}{SpecialFieldNames.DataLoadRunID},
+{tableName}.{SpecialFieldNames.ValidFrom} as {columnAliasPrefix}{SpecialFieldNames.ValidFrom}
+";
+    }
+
     private string GetSharedColumnsSQLWithColumnAliasPrefix(string tableName, string columnAliasPrefix)
     {
         var sb = new StringBuilder();

Rdmp.Core/DataQualityEngine/Data/ColumnState.cs

@@ -145,8 +145,7 @@ public void Commit(Evaluation evaluation, string pivotCategory, DbConnection con
             DatabaseCommandHelper.AddParameterWithValueToCommand("@PivotCategory", cmd, pivotCategory);
             cmd.ExecuteNonQuery();
         }
-
-
+
         IsCommitted = true;
     }
 }

Rdmp.Core/DataQualityEngine/Data/RowState.cs

@@ -25,6 +25,17 @@ public class RowState
     public string PivotCategory { get; private set; }
 
 
+    public RowState(int dataLoadRunID, int correct, int missing, int wrong, int invalid,
+        string validatorXml, string pivotCategory)
+    {
+        Correct = correct;
+        Missing = missing;
+        Wrong = wrong;
+        Invalid = invalid;
+        ValidatorXML = validatorXml;
+        DataLoadRunID = dataLoadRunID;
+    }
+
     public RowState(DbDataReader r)
     {
         Correct = Convert.ToInt32(r["Correct"]);