From 6f02bbb2796886ac373f07170938d5524df3a43f Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 10:06:29 +0000 Subject: [PATCH 1/9] better people generation --- Rdmp.Core/Curation/Data/Overview/OverviewModel.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs index a058df3cf8..cd5a2d446b 100644 --- a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs +++ b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs @@ -69,7 +69,9 @@ public void Regen(string whereClause) dt.EndLoadData(); con.Dispose(); _numberOfRecords = dt.Rows.Count; - _numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0; + _numberOfPeople = hasExtractionIdentifier?dt.AsEnumerable().Select(r => r[column.ColumnInfo.GetRuntimeName()]).ToList().Distinct().Count():0; + // this line here is the issue thart is very slow + //_numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0; GetDataLoads(); dt.Dispose(); } From f9643b2d4d7422935a7bd2ab5d5f5606f1de897c Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 12:31:35 +0000 Subject: [PATCH 2/9] add packages --- Directory.Packages.props | 1 + Documentation/CodeTutorials/Packages.md | 3 ++- Rdmp.Core/Rdmp.Core.csproj | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a615b3a42c..fe5f1a894c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -11,6 +11,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 48521778e1..09f2137fba 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -1,4 +1,4 @@ -# Packages Used By RDMP +packages# Packages Used By RDMP ### Risk Assessment common to all: 1. Packages on NuGet are virus scanned by the NuGet site. @@ -42,4 +42,5 @@ | AWSSDK.SecurityToken | [GitHub](https://github.com/aws/aws-sdk-net) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | | | AWSSDK.SSO | [GitHub](https://github.com/aws/aws-sdk-net) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | | | AWSSDK.SSOOIDC | [GitHub](https://github.com/aws/aws-sdk-net) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | | +| Microsoft.Bcl.AsyncInterfaces | [Nuget](https://www.nuget.org/packages/microsoft.bcl.asyncinterfaces/) | [MIT](https://opensource.org/licenses/MIT) | | [DBMS]: ./Glossary.md#DBMS diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 2de063bb79..44696b6cff 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -327,6 +327,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive From fabd713ea21ca46c14e503243bda43cf417dff33 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 15:05:18 +0000 Subject: [PATCH 3/9] fix typo --- Documentation/CodeTutorials/Packages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 09f2137fba..f6d9aad516 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -1,4 +1,4 @@ -packages# Packages Used By RDMP +# Packages Used By RDMP ### Risk Assessment common to all: 1. Packages on NuGet are virus scanned by the NuGet site. From 61bce70d4c51e38b4c737cbad9e48172b01ebca2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 15:10:54 +0000 Subject: [PATCH 4/9] add changelgo --- CHANGELOG.md | 4 ++++ Rdmp.Core/Curation/Data/Overview/OverviewModel.cs | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d53d8f533..fff491afe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.4.1] - Unreleased + +- Improve Overview Model Geenration Speed + ## [8.4.0] - 2024-12-02 - Add Ordering to Filters diff --git a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs index cd5a2d446b..609e8384b0 100644 --- a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs +++ b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs @@ -70,8 +70,6 @@ public void Regen(string whereClause) con.Dispose(); _numberOfRecords = dt.Rows.Count; _numberOfPeople = hasExtractionIdentifier?dt.AsEnumerable().Select(r => r[column.ColumnInfo.GetRuntimeName()]).ToList().Distinct().Count():0; - // this line here is the issue thart is very slow - //_numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0; GetDataLoads(); dt.Dispose(); } From 52d8378b79cd020828272947bcdd773f253cf84e Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 15:12:08 +0000 Subject: [PATCH 5/9] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fff491afe7..024396ad1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.1] - Unreleased -- Improve Overview Model Geenration Speed +- Improve Overview Model Generation Speed ## [8.4.0] - 2024-12-02 From bd8cc0244fe456358e006c947f9567746227f660 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 6 Dec 2024 15:32:59 +0000 Subject: [PATCH 6/9] revert catalogue ui --- .../Proposals/ProposeExecutionWhenTargetIsCatalogue.cs | 3 ++- SharedAssemblyInfo.cs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCatalogue.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCatalogue.cs index 12a80d2f32..9b2a9a3a6d 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCatalogue.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsCatalogue.cs @@ -10,6 +10,7 @@ using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; using Rdmp.UI.ItemActivation; +using Rdmp.UI.MainFormUITabs; using Rdmp.UI.Overview; namespace Rdmp.UI.CommandExecution.Proposals; @@ -24,7 +25,7 @@ public ProposeExecutionWhenTargetIsCatalogue(IActivateItems itemActivator) : bas public override void Activate(Catalogue c) { - ItemActivator.Activate(c); + ItemActivator.Activate(c); } public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, Catalogue targetCatalogue, diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index a672341220..ec5884a345 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.4.0")] -[assembly: AssemblyFileVersion("8.4.0")] -[assembly: AssemblyInformationalVersion("8.4.0")] +[assembly: AssemblyVersion("8.4.1")] +[assembly: AssemblyFileVersion("8.4.1")] +[assembly: AssemblyInformationalVersion("8.4.1")] From 5fdeaf410b04411e2f0b3be9235e5efcb0087c20 Mon Sep 17 00:00:00 2001 From: James Friel Date: Sun, 8 Dec 2024 20:41:21 +0000 Subject: [PATCH 7/9] Task/rdmp 250 remote table attacher no raw (#2083) * add table attacher without db creation * update docs --- CHANGELOG.md | 4 ++++ .../Modules/Attachers/RemoteAttacher.cs | 2 +- .../Modules/Attachers/RemoteTableAttacher.cs | 2 +- .../RemoteTableWithoutDBCreationAttacher.cs | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableWithoutDBCreationAttacher.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d53d8f533..fb08969ab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.4.1] - Unreleased + +- Add Remote Table Without DB Creation Attacher + ## [8.4.0] - 2024-12-02 - Add Ordering to Filters diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs index 0a86fcc5ed..99e5853c54 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs @@ -24,7 +24,7 @@ namespace Rdmp.Core.DataLoad.Modules.Attachers; public class RemoteAttacher : Attacher, IPluginAttacher { - public RemoteAttacher() : base(true) { } + public RemoteAttacher(bool requestsExternalDatabaseCreation=true) : base(requestsExternalDatabaseCreation) { } [DemandsInitialization("How far back to pull data from")] public AttacherHistoricalDurations HistoricalFetchDuration { get; set; } diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs index 51f231e79f..fa7ff3cd9c 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs @@ -37,7 +37,7 @@ public class RemoteTableAttacher : RemoteAttacher { private const string FutureLoadMessage = "Cannot load data from the future"; - public RemoteTableAttacher() : base() + public RemoteTableAttacher(bool requestsExternalDatabaseCreation=true) : base(requestsExternalDatabaseCreation) { } diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableWithoutDBCreationAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableWithoutDBCreationAttacher.cs new file mode 100644 index 0000000000..f0c86ba337 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableWithoutDBCreationAttacher.cs @@ -0,0 +1,19 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + + +namespace Rdmp.Core.DataLoad.Modules.Attachers; + +/// +/// Data load component for loading tables with records read from a remote database server. Runs the specified query (which can include a date parameter) +/// and inserts the results of the query into RAW. +/// This attcher does not create RAW if it does not exist. Another attacher will be required to generate the initial RAW database +/// +public class RemoteTableWithoutDBCreationAttacher: RemoteTableAttacher +{ + + public RemoteTableWithoutDBCreationAttacher() : base(false) { } +} From c6915acd3d267688c2b0f2c0d9a8b67221257707 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 10 Dec 2024 11:41:24 +0000 Subject: [PATCH 8/9] Update rdmp-client.xml --- rdmp-client.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rdmp-client.xml b/rdmp-client.xml index b8ebc0e723..9157566cec 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 8.4.0.0 - https://github.com/HicServices/RDMP/releases/download/v8.4.0/rdmp-8.4.0-client.zip + 8.4.1.0 + https://github.com/HicServices/RDMP/releases/download/v8.4.1/rdmp-8.4.1-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true From ac0d33a0aa70282cd6ae0195a67eb03440365dfa Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 10 Dec 2024 11:41:59 +0000 Subject: [PATCH 9/9] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28022ff86e..859f5d9afc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [8.4.1] - Unreleased +## [8.4.1] - 2024-12-10 - Improve Overview Model Generation Speed - Add Remote Table Without DB Creation Attacher