Merge branch 'develop' of https://github.com/HicServices/RDMP into fe…

…ature/RDMP-73-cohort-holdouts

.github/workflows/build.yml

@@ -59,6 +59,14 @@ jobs:
         shell: bash
         run: |
             find ./Tools/rdmp/Databases.yaml -type f -exec sed -i 's/RDMP_/TEST_/g' {} \;
+      - name: BundleSource
+        shell: bash
+        run: |
+          mkdir -p Tools/BundleUpSourceIntoZip/output
+          rm -f Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip
+          echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd
+          perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt
+          echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd
       - name: Build
         run: dotnet build --configuration Release --verbosity minimal
       - name: Create MySql Logging, DQE and Cohort Building Cache Db
@@ -108,16 +116,9 @@ jobs:
 
       - name: Package
         run: |
-          dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
-          dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
+          dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo
+          dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo
           dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo
-      - name: BundleSource
-        shell: bash
-        run: |
-          mkdir -p Tools/BundleUpSourceIntoZip/output
-          echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd
-          perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt
-          echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd
 
       - name: Sign
         shell: bash
@@ -130,11 +131,11 @@ jobs:
           mkdir -p dist
           cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }}
           echo '"'$signtool'"' 'Sign  /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} dist/rdmp.msi' | cmd
-          (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd)
+          (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip rdmp.exe NLog.config *.yaml | cmd)
           (cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd)
           mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux
           echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd
-          (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd)
+          (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip ResearchDataManagementPlatform.exe | cmd)
 
       - name: Install Perl dependencies
         uses: shogo82148/[email protected]
@@ -178,6 +179,7 @@ jobs:
           rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar
 
       - name: Build Nuget packages
+        if: contains(github.ref, 'refs/tags/v')
         shell: bash
         run: |
           for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj

Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj

@@ -28,9 +28,6 @@
   <ItemGroup>
     <EmbeddedResource Remove="Updates\UpdaterUI.resx" />
   </ItemGroup>
-  <ItemGroup>
-    <None Remove="NLog.template.config" />
-  </ItemGroup>
   <ItemGroup>
     <PackageReference Include="ConsoleControl" Version="1.3.0" />
     <PackageReference Include="Autoupdater.NET.Official" Version="1.8.4" />
@@ -65,9 +62,6 @@
     </Compile>
     <Content Include="Icon\main.ico" />
     <Content Include="Icon\main.png" />
-    <Content Include="NLog.template.config">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
     <EmbeddedResource Update="RDMPMainForm.resx">
       <DependentUpon>RDMPMainForm.cs</DependentUpon>
     </EmbeddedResource>

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ...
 
+### Changed
+
+- Improved file transfer (FTP/SFTP/FTPS) support
+
 ## [8.1.0] - 2023-09-19
 
 ### Changed

Documentation/CodeTutorials/Coding.md

@@ -94,7 +94,7 @@ Each area of the RDMP codebase has its own documentation.  These include:
   - [Creating Right Click Context Menus](./CreatingANewRightClickMenu.md)
   - [Double Click / Drag and drop](./DoubleClickAndDragDrop.md)
 - [Writing Plugins](./PluginWriting.md)
-- [Cohort Creation](./../../Rdmp.Core/CohortCreation/Readme.md)
+- [Cohort Creation](./../../Rdmp.Core/CohortCreation/CohortCreation.md)
 - [Command Line Interface (CLI)](./../../Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.md)
 
 RDMP has over 20 class diagrams which you can open if you have visual studio.  These files end in the extension `.cd`

Documentation/CodeTutorials/FAQ.md

@@ -334,12 +334,12 @@ Since each section is runnable independently it is trivially easy for RDMP to pr
 
 ![Cohort Builder Tree](./Images/CohortBuilderUI.png)
 
-For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md).
+For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md).
 
 ## Does the Cohort Builder support Excel, CSV files etc?
 <a name="cohort-build-with-file"/>
 
-Yes.  If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/Readme.md).
+Yes.  If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/CohortCommitting.md).
 
 If you need to combine data in the file (or multiple files) with data in your database then the first step is to create a new Catalogue by uploading the data into your database.  This has a number of advantages:
 
@@ -834,7 +834,7 @@ RDMP also supports creating the audit logging database in other [DBMS] types (e.
 ### System Logs
 
 System logs can be enabled for RDMP.  These are low level logs designed to be read by a software developer or IT specialist.  To enable
-this feature, locate the `NLog.template.config` file in your RDMP install directory.  Rename the file `NLog.config` (i.e. remove the word template).
+this feature, download [NLog.config](https://raw.githubusercontent.com/HicServices/RDMP/main/Tools/rdmp/NLog.config) to your RDMP install.
 This template generates file logs to a `./logs/` sub-directory.  You can adjust it to log anywhere including to databases/central server etc
 by [following the NLog targets guide](https://nlog-project.org/config/).  **Logs may contain identifiable or sensitive information so should
 be secured appropriately if enabled**. 
@@ -891,7 +891,7 @@ var catalogues = repository.GetAllObjects<Catalogue>();
 var catalogueItems = repository.GetAllObjects<CatalogueItem>();
 ```
 
-If you think the problem is more widespread then you can also use the [`IInjectKnown<T>`](./../../Reusable/MapsDirectlyToDatabaseTable/Injection/README.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time.
+If you think the problem is more widespread then you can also use the [`IInjectKnown<T>`](./../../Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time.
 
 
 
@@ -904,9 +904,9 @@ Yes there are over 1,000 unit and integration tests, this is covered in [Tests](
 [hic_dataLoadRunID]: #hic_dataLoadRunID
 [Data Load Engine]: #data-load-engine
 [db_executor]: https://www.sqlmatters.com/Articles/Adding%20a%20db_executor%20role.aspx
-[cohort databases]: ../../Rdmp.Core/CohortCommitting/Readme.md
-[cohort database]: ../../Rdmp.Core/CohortCommitting/Readme.md
-[query cache]: ../../Rdmp.Core/CohortCreation/Readme.md
+[cohort databases]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md
+[cohort database]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md
+[query cache]: ../../Rdmp.Core/CohortCreation/CohortCreation.md
 [UNION]: ./Glossary.md#UNION
 [EXCEPT]: ./Glossary.md#EXCEPT
 [INTERSECT]: ./Glossary.md#INTERSECT

Documentation/CodeTutorials/Graphs.md

@@ -41,7 +41,7 @@ This will take you to the 'Edit' page where you can specify which dimensions to
 The RDMP graphing engine supports Bar and Plot graphs.  Not all [DBMS] implement all graph types.  The following
 table shows what is implemented:
 
-|  Graph Type  | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.Oracle/README.md) |
+|  Graph Type  | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/Oracle/README.md) |
 |----|---|----|---|---|
 | [Bar 1 Dimension](#bar-1-dimension) | yes | yes | yes | yes  |
 | [Bar 2 Dimensions](#bar-2-dimensions) | yes | yes | no | yes  |
@@ -119,7 +119,7 @@ _Applying a TOP to PIVOT graphs limits the number of series in the PIVOT_
 
 One of the core strengths of the RDMP graphing system is the ability to run graphs on Filters, Cohorts and/or [ExtractionConfiguration] datasets.  This lets you rapidly confirm that the cohort you are building does not have data holes or missing trends.
 
-To graph a cohort right click it and go to Graph.  Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/Readme.md).
+To graph a cohort right click it and go to Graph.  Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/CohortCreation.md).
 
 ![Right clicking a cohort set in a CohortIdentificationConfiguration and graphing it](Images/Graphs/GraphCohort.png)
 

Documentation/CodeTutorials/Packages.md

@@ -10,6 +10,7 @@
 | Package | Source Code | License | Purpose | Additional Risk Assessment |
 | ------- | ------------| ------- | ------- | -------------------------- |
 | Equ | [GitHub](https://github.com/thedmi/Equ) | [MIT](https://opensource.org/licenses/MIT) | Simplifies object equality implementation | |
+| FluentFTP | [Github](https://github.com/robinrodricks/FluentFTP/) | [MIT](https://opensource.org/licenses/MIT) | FTP(S) client | |
 | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | Database driver for MongoDB | |
 | Microsoft.SourceLink.GitHub | [GitHub](https://github.com/dotnet/sourcelink) | [MIT](https://opensource.org/licenses/MIT) | Enable source linkage from nupkg | Official MS project |
 | Microsoft.XmlSerializer.Generator | [Microsoft](https://learn.microsoft.com/en-us/dotnet/core/additional-tools/xml-serializer-generator) | [MIT](https://opensource.org/licenses/MIT) | XML handling improvements |

Documentation/CodeTutorials/UserManual.md

@@ -120,7 +120,7 @@ There are a couple of other database types which can be created as and when you
 
 |Database|Role|
 |---|---|
-| [Query Caching](./../../Rdmp.Core/CohortCreation/Readme.md)| Improves the performance of complex cohort identification configurations and anonymisation.  Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))|
+| [Query Caching](./../../Rdmp.Core/CohortCreation/CohortCreation.md)| Improves the performance of complex cohort identification configurations and anonymisation.  Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))|
 |Anonymisation|Provides a way of performing identifier dropping / substitution on data load for when you want an entirely anonymous data repository|
 |[Plugin Databases](./FAQ.md#plugins)| RDMP supports plugins which can in some cases have their own database(s)|
 
@@ -585,7 +585,7 @@ New [CohortIdentificationConfigurations] are created with 3 containers:
   - Inclusion Criteria
   - Exclusion Criteria
 
-The first container is an [EXCEPT] while the other two are [UNION].  For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md).
+The first container is an [EXCEPT] while the other two are [UNION].  For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md).
 
 ## Commit Cohort
 

NoteForNewDevelopers.md

@@ -40,16 +40,16 @@ All technical and repo specific documentation are stored in markdown (`.md` form
 
 ** Performance **
 - [Database Change Tracking (A Performance Enhancement)](./Documentation/CodeTutorials/ChangeTracking.md)
-- [Reducing database calls with 'injection'](./Reusable/MapsDirectlyToDatabaseTable/Injection/README.md)
+- [Reducing database calls with 'injection'](./Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md)
 
 **Deep Dives**
 - [How untyped CSV data is parsed by RDMP](./Documentation/CodeTutorials/CSVHandling.md)
 - [How 'Bulk Insert' function works](./Documentation/CodeTutorials/DataTableUpload.md)
 - [How xls / xlsx files are read by RDMP](./Documentation/CodeTutorials/ExcelHandling.md)
 - [Multiple Linkage Columns (e.g. NHS Number or CHI)](./Documentation/CodeTutorials/MultipleExtractionIdentifiers.md)
-- [Storing cohort lists](./Rdmp.Core/CohortCommitting/Readme.md)
-- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/Readme.md)
-- [Tree layout documentation](./Rdmp.Core/Providers/Readme.md)
+- [Storing cohort lists](./Rdmp.Core/CohortCommitting/CohortCommitting.md)
+- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/CohortCreation.md)
+- [Tree layout documentation](./Rdmp.Core/Providers/Providers.md)
 - [Aggregate Graphs](./Documentation/CodeTutorials/Graphs.md)
 - [YamlRepository](./Documentation/CodeTutorials/YamlRepository.md)
 - [Custom Metadata Reports](./Documentation/CodeTutorials/CustomMetadataSubstitutions.md)

Rdmp.Core/CohortCommitting/CohortCommitting.md

@@ -2,7 +2,7 @@
 
 A Cohort is a collection of unique person identifiers which can be linked against datasets during an extraction.  This namespace covers saving a list of identifiers into a cohort database.
 
-See [Cohort Creation](../CohortCreation/Readme.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria.
+See [Cohort Creation](../CohortCreation/CohortCreation.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria.
 
 ## Cohort Storage