diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f6b8f2d76..b02d417376 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,6 +59,14 @@ jobs: shell: bash run: | find ./Tools/rdmp/Databases.yaml -type f -exec sed -i 's/RDMP_/TEST_/g' {} \; + - name: BundleSource + shell: bash + run: | + mkdir -p Tools/BundleUpSourceIntoZip/output + rm -f Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip + echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd + perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt + echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd - name: Build run: dotnet build --configuration Release --verbosity minimal - name: Create MySql Logging, DQE and Cohort Building Cache Db @@ -108,16 +116,9 @@ jobs: - name: Package run: | - dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo - dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo + dotnet publish Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj -r win-x64 --self-contained -c Release -o PublishWinForms -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo + dotnet publish Tools/rdmp/rdmp.csproj -r win-x64 --self-contained -c Release -o PublishWindows -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:IncludeNativeLibrariesForSelfExtract=true --verbosity minimal --nologo dotnet publish Tools/rdmp/rdmp.csproj -r linux-x64 --self-contained -c Release -o PublishLinux -p:GenerateDocumentationFile=false -p:PublishSingleFile=true -p:PublishReadyToRun=true --verbosity minimal --nologo - - name: BundleSource - shell: bash - run: | - mkdir -p Tools/BundleUpSourceIntoZip/output - echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd - perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt - echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd - name: Sign shell: bash @@ -130,11 +131,11 @@ jobs: mkdir -p dist cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }} echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} dist/rdmp.msi' | cmd - (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip . | cmd) + (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-win-x64.zip rdmp.exe NLog.config *.yaml | cmd) (cd PublishLinux ; echo 7z a -mx=0 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.zip . | cmd) mv PublishLinux rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux echo 7z a dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux | cmd - (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip . | cmd) + (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-${{ steps.version.outputs.rdmpversion }}-client.zip ResearchDataManagementPlatform.exe | cmd) - name: Install Perl dependencies uses: shogo82148/actions-setup-perl@v1.24.1 @@ -178,6 +179,7 @@ jobs: rm dist/rdmp-${{ steps.version.outputs.rdmpversion }}-cli-linux-x64.tar - name: Build Nuget packages + if: contains(github.ref, 'refs/tags/v') shell: bash run: | for i in Rdmp.Core/Rdmp.Core.csproj Rdmp.UI/Rdmp.UI.csproj Tests.Common/Tests.Common.csproj diff --git a/Application/ResearchDataManagementPlatform/NLog.template.config b/Application/ResearchDataManagementPlatform/NLog.template.config deleted file mode 100644 index 348ad21979..0000000000 --- a/Application/ResearchDataManagementPlatform/NLog.template.config +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj index 2e4d052b16..e78dd26e59 100644 --- a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj +++ b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj @@ -28,9 +28,6 @@ - - - @@ -65,9 +62,6 @@ - - PreserveNewest - RDMPMainForm.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7100a239d3..83f916faa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ... +### Changed + +- Improved file transfer (FTP/SFTP/FTPS) support + ## [8.1.0] - 2023-09-19 ### Changed diff --git a/Documentation/CodeTutorials/Coding.md b/Documentation/CodeTutorials/Coding.md index f7511d05c7..f4951ee4c2 100644 --- a/Documentation/CodeTutorials/Coding.md +++ b/Documentation/CodeTutorials/Coding.md @@ -94,7 +94,7 @@ Each area of the RDMP codebase has its own documentation. These include: - [Creating Right Click Context Menus](./CreatingANewRightClickMenu.md) - [Double Click / Drag and drop](./DoubleClickAndDragDrop.md) - [Writing Plugins](./PluginWriting.md) -- [Cohort Creation](./../../Rdmp.Core/CohortCreation/Readme.md) +- [Cohort Creation](./../../Rdmp.Core/CohortCreation/CohortCreation.md) - [Command Line Interface (CLI)](./../../Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.md) RDMP has over 20 class diagrams which you can open if you have visual studio. These files end in the extension `.cd` diff --git a/Documentation/CodeTutorials/FAQ.md b/Documentation/CodeTutorials/FAQ.md index 38323d54bd..fda1e96817 100644 --- a/Documentation/CodeTutorials/FAQ.md +++ b/Documentation/CodeTutorials/FAQ.md @@ -334,12 +334,12 @@ Since each section is runnable independently it is trivially easy for RDMP to pr ![Cohort Builder Tree](./Images/CohortBuilderUI.png) -For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md). +For full details on technical implementation see [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md). ## Does the Cohort Builder support Excel, CSV files etc? -Yes. If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/Readme.md). +Yes. If a cohort is solely defined by the contents of such a file it can be [committed directly to the cohort store](../../Rdmp.Core/CohortCommitting/CohortCommitting.md). If you need to combine data in the file (or multiple files) with data in your database then the first step is to create a new Catalogue by uploading the data into your database. This has a number of advantages: @@ -834,7 +834,7 @@ RDMP also supports creating the audit logging database in other [DBMS] types (e. ### System Logs System logs can be enabled for RDMP. These are low level logs designed to be read by a software developer or IT specialist. To enable -this feature, locate the `NLog.template.config` file in your RDMP install directory. Rename the file `NLog.config` (i.e. remove the word template). +this feature, download [NLog.config](https://raw.githubusercontent.com/HicServices/RDMP/main/Tools/rdmp/NLog.config) to your RDMP install. This template generates file logs to a `./logs/` sub-directory. You can adjust it to log anywhere including to databases/central server etc by [following the NLog targets guide](https://nlog-project.org/config/). **Logs may contain identifiable or sensitive information so should be secured appropriately if enabled**. @@ -891,7 +891,7 @@ var catalogues = repository.GetAllObjects(); var catalogueItems = repository.GetAllObjects(); ``` -If you think the problem is more widespread then you can also use the [`IInjectKnown`](./../../Reusable/MapsDirectlyToDatabaseTable/Injection/README.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time. +If you think the problem is more widespread then you can also use the [`IInjectKnown`](./../../Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md) system to perform `Lazy` loads which prevents repeated calls to the same property going back to the database every time. @@ -904,9 +904,9 @@ Yes there are over 1,000 unit and integration tests, this is covered in [Tests]( [hic_dataLoadRunID]: #hic_dataLoadRunID [Data Load Engine]: #data-load-engine [db_executor]: https://www.sqlmatters.com/Articles/Adding%20a%20db_executor%20role.aspx -[cohort databases]: ../../Rdmp.Core/CohortCommitting/Readme.md -[cohort database]: ../../Rdmp.Core/CohortCommitting/Readme.md -[query cache]: ../../Rdmp.Core/CohortCreation/Readme.md +[cohort databases]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md +[cohort database]: ../../Rdmp.Core/CohortCommitting/CohortCommitting.md +[query cache]: ../../Rdmp.Core/CohortCreation/CohortCreation.md [UNION]: ./Glossary.md#UNION [EXCEPT]: ./Glossary.md#EXCEPT [INTERSECT]: ./Glossary.md#INTERSECT diff --git a/Documentation/CodeTutorials/Graphs.md b/Documentation/CodeTutorials/Graphs.md index d1cc516dd4..e1a6336a78 100644 --- a/Documentation/CodeTutorials/Graphs.md +++ b/Documentation/CodeTutorials/Graphs.md @@ -41,7 +41,7 @@ This will take you to the 'Edit' page where you can specify which dimensions to The RDMP graphing engine supports Bar and Plot graphs. Not all [DBMS] implement all graph types. The following table shows what is implemented: -| Graph Type | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/Implementations/FAnsi.Implementations.Oracle/README.md) | +| Graph Type | [Sql Server](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MicrosoftSQL/README.md) | [MySql](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/MySql/README.md) | [Postgres](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/PostgreSql/README.md) | [Oracle](https://github.com/HicServices/FAnsiSql/blob/main/FAnsiSql/Implementations/Oracle/README.md) | |----|---|----|---|---| | [Bar 1 Dimension](#bar-1-dimension) | yes | yes | yes | yes | | [Bar 2 Dimensions](#bar-2-dimensions) | yes | yes | no | yes | @@ -119,7 +119,7 @@ _Applying a TOP to PIVOT graphs limits the number of series in the PIVOT_ One of the core strengths of the RDMP graphing system is the ability to run graphs on Filters, Cohorts and/or [ExtractionConfiguration] datasets. This lets you rapidly confirm that the cohort you are building does not have data holes or missing trends. -To graph a cohort right click it and go to Graph. Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/Readme.md). +To graph a cohort right click it and go to Graph. Make sure you have set up a [cohort caching database](../../Rdmp.Core/CohortCreation/CohortCreation.md). ![Right clicking a cohort set in a CohortIdentificationConfiguration and graphing it](Images/Graphs/GraphCohort.png) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 44d50761dc..d324a70fb6 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -10,6 +10,7 @@ | Package | Source Code | License | Purpose | Additional Risk Assessment | | ------- | ------------| ------- | ------- | -------------------------- | | Equ | [GitHub](https://github.com/thedmi/Equ) | [MIT](https://opensource.org/licenses/MIT) | Simplifies object equality implementation | | +| FluentFTP | [Github](https://github.com/robinrodricks/FluentFTP/) | [MIT](https://opensource.org/licenses/MIT) | FTP(S) client | | | MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) | [Apache 2.0](https://opensource.org/licenses/Apache-2.0) | Database driver for MongoDB | | | Microsoft.SourceLink.GitHub | [GitHub](https://github.com/dotnet/sourcelink) | [MIT](https://opensource.org/licenses/MIT) | Enable source linkage from nupkg | Official MS project | | Microsoft.XmlSerializer.Generator | [Microsoft](https://learn.microsoft.com/en-us/dotnet/core/additional-tools/xml-serializer-generator) | [MIT](https://opensource.org/licenses/MIT) | XML handling improvements | diff --git a/Documentation/CodeTutorials/UserManual.md b/Documentation/CodeTutorials/UserManual.md index f332b24879..224539fccc 100644 --- a/Documentation/CodeTutorials/UserManual.md +++ b/Documentation/CodeTutorials/UserManual.md @@ -120,7 +120,7 @@ There are a couple of other database types which can be created as and when you |Database|Role| |---|---| -| [Query Caching](./../../Rdmp.Core/CohortCreation/Readme.md)| Improves the performance of complex cohort identification configurations and anonymisation. Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))| +| [Query Caching](./../../Rdmp.Core/CohortCreation/CohortCreation.md)| Improves the performance of complex cohort identification configurations and anonymisation. Also allows cross server and plugin Cohort Builder elements (e.g. to [REST APIs](./FAQ.md#apis))| |Anonymisation|Provides a way of performing identifier dropping / substitution on data load for when you want an entirely anonymous data repository| |[Plugin Databases](./FAQ.md#plugins)| RDMP supports plugins which can in some cases have their own database(s)| @@ -585,7 +585,7 @@ New [CohortIdentificationConfigurations] are created with 3 containers: - Inclusion Criteria - Exclusion Criteria -The first container is an [EXCEPT] while the other two are [UNION]. For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/Readme.md). +The first container is an [EXCEPT] while the other two are [UNION]. For more information on how these work see the [Cohort Builder FAQ](./FAQ.md#cohort-builder-overview) or [Cohort Creation](../../Rdmp.Core/CohortCreation/CohortCreation.md). ## Commit Cohort diff --git a/NoteForNewDevelopers.md b/NoteForNewDevelopers.md index 0ac57d0a74..cb9c197f03 100644 --- a/NoteForNewDevelopers.md +++ b/NoteForNewDevelopers.md @@ -40,16 +40,16 @@ All technical and repo specific documentation are stored in markdown (`.md` form ** Performance ** - [Database Change Tracking (A Performance Enhancement)](./Documentation/CodeTutorials/ChangeTracking.md) -- [Reducing database calls with 'injection'](./Reusable/MapsDirectlyToDatabaseTable/Injection/README.md) +- [Reducing database calls with 'injection'](./Rdmp.Core/MapsDirectlyToDatabaseTable/Injection/Injection.md) **Deep Dives** - [How untyped CSV data is parsed by RDMP](./Documentation/CodeTutorials/CSVHandling.md) - [How 'Bulk Insert' function works](./Documentation/CodeTutorials/DataTableUpload.md) - [How xls / xlsx files are read by RDMP](./Documentation/CodeTutorials/ExcelHandling.md) - [Multiple Linkage Columns (e.g. NHS Number or CHI)](./Documentation/CodeTutorials/MultipleExtractionIdentifiers.md) -- [Storing cohort lists](./Rdmp.Core/CohortCommitting/Readme.md) -- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/Readme.md) -- [Tree layout documentation](./Rdmp.Core/Providers/Readme.md) +- [Storing cohort lists](./Rdmp.Core/CohortCommitting/CohortCommitting.md) +- [Cohort Builder docs including info on list caching](./Rdmp.Core/CohortCreation/CohortCreation.md) +- [Tree layout documentation](./Rdmp.Core/Providers/Providers.md) - [Aggregate Graphs](./Documentation/CodeTutorials/Graphs.md) - [YamlRepository](./Documentation/CodeTutorials/YamlRepository.md) - [Custom Metadata Reports](./Documentation/CodeTutorials/CustomMetadataSubstitutions.md) diff --git a/Rdmp.Core/CohortCommitting/CohortCommitting.md b/Rdmp.Core/CohortCommitting/CohortCommitting.md index a6c57cba1d..70b1ce96c2 100644 --- a/Rdmp.Core/CohortCommitting/CohortCommitting.md +++ b/Rdmp.Core/CohortCommitting/CohortCommitting.md @@ -2,7 +2,7 @@ A Cohort is a collection of unique person identifiers which can be linked against datasets during an extraction. This namespace covers saving a list of identifiers into a cohort database. -See [Cohort Creation](../CohortCreation/Readme.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria. +See [Cohort Creation](../CohortCreation/CohortCreation.md) for building queries that identify cohorts from your database based on inclusion/exclusion criteria. ## Cohort Storage diff --git a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs index 06a8fb5a30..49ef7489ca 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs @@ -4,6 +4,8 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +#nullable enable + using System; using System.Collections.Generic; using System.Diagnostics; @@ -14,7 +16,9 @@ using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.RegularExpressions; +using System.Threading; using FAnsi.Discovery; +using FluentFTP; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; @@ -29,18 +33,18 @@ namespace Rdmp.Core.DataLoad.Modules.FTP; /// load component which downloads files from a remote FTP server to the ForLoading directory /// /// Attempts to connect to the FTP server and download all files in the landing folder of the FTP (make sure you really want everything in the -/// root folder - if not then configure redirection on the FTP so you land in the correct directory). Files are downloaded into the ForLoading folder +/// root folder - if not then configure redirection on the FTP, so you land in the correct directory). Files are downloaded into the ForLoading folder /// public class FTPDownloader : IPluginDataProvider { - protected string _host; - protected string _username; - protected string _password; - - private bool _useSSL = false; + private readonly Lazy _connection; + protected readonly List _filesRetrieved = new(); + private ILoadDirectory? _directory; - protected List _filesRetrieved = new(); - private ILoadDirectory _directory; + public FTPDownloader() + { + _connection = new Lazy(SetupFtp, LazyThreadSafetyMode.ExecutionAndPublication); + } [DemandsInitialization( "Determines the behaviour of the system when no files are found on the server. If true the entire data load process immediately stops with exit code LoadNotRequired, if false then the load proceeds as normal (useful if for example if you have multiple Attachers and some files are optional)")] @@ -48,7 +52,7 @@ public class FTPDownloader : IPluginDataProvider [DemandsInitialization( "The Regex expression to validate files on the FTP server against, only files matching the expression will be downloaded")] - public Regex FilePattern { get; set; } + public Regex? FilePattern { get; set; } [DemandsInitialization("The timeout to use when connecting to the FTP server in SECONDS")] public int TimeoutInSeconds { get; set; } @@ -60,10 +64,10 @@ public class FTPDownloader : IPluginDataProvider [DemandsInitialization( "The FTP server to connect to. Server should be specified with only IP:Port e.g. 127.0.0.1:20. You do not have to specify ftp:// at the start", Mandatory = true)] - public ExternalDatabaseServer FTPServer { get; set; } + public ExternalDatabaseServer? FTPServer { get; set; } [DemandsInitialization("The directory on the FTP server that you want to download files from")] - public string RemoteDirectory { get; set; } + public string? RemoteDirectory { get; set; } [DemandsInitialization("True to set keep alive", DefaultValue = true)] public bool KeepAlive { get; set; } @@ -76,38 +80,25 @@ public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken) { - SetupFTP(); - return DownloadFilesOnFTP(_directory, job); - } - - public static string GetDescription() => "See Description attribute of class"; - - public static IDataProvider Clone() => new FTPDownloader(); - - public bool Validate(ILoadDirectory destination) - { - SetupFTP(); - return GetFileList().Any(); + return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job); } - private void SetupFTP() + private FtpClient SetupFtp() { - _host = FTPServer.Server; - _username = FTPServer.Username ?? "anonymous"; - _password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); - - if (string.IsNullOrWhiteSpace(_host)) - throw new NullReferenceException( - $"FTPServer is not set up correctly it must have Server property filled in{FTPServer}"); + var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set"); + var username = FTPServer.Username ?? "anonymous"; + var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); + var c = new FtpClient(host, username, password); + c.AutoConnect(); + return c; } private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener) { - var files = GetFileList(); + var files = GetFileList().ToArray(); - listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, files.Aggregate( - "Identified the following files on the FTP server:", (s, f) => - $"{f},").TrimEnd(','))); + listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + $"Identified the following files on the FTP server:{string.Join(',',files)}")); var forLoadingContainedCachedFiles = false; @@ -117,28 +108,29 @@ private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEve listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"File {file} was evaluated as {action}")); - if (action == SkipReason.DoNotSkip) + + switch (action) { - listener.OnNotify(this, - new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}")); - Download(file, destination, listener); + case SkipReason.DoNotSkip: + listener.OnNotify(this, + new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}")); + Download(file, destination); + break; + case SkipReason.InForLoading: + forLoadingContainedCachedFiles = true; + break; } - - if (action == SkipReason.InForLoading) - forLoadingContainedCachedFiles = true; } - //if no files were downloaded (and there were none skiped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so - if (!forLoadingContainedCachedFiles && !_filesRetrieved.Any() && SendLoadNotRequiredIfFileNotFound) - { - listener.OnNotify(this, - new NotifyEventArgs(ProgressEventType.Information, - "Could not find any files on the remote server worth downloading, so returning LoadNotRequired")); - return ExitCodeType.OperationNotRequired; - } + // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound + if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound) + return ExitCodeType.Success; - //otherwise it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound - return ExitCodeType.Success; + // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so + listener.OnNotify(this, + new NotifyEventArgs(ProgressEventType.Information, + "Could not find any files on the remote server worth downloading, so returning LoadNotRequired")); + return ExitCodeType.OperationNotRequired; } protected enum SkipReason @@ -151,145 +143,43 @@ protected enum SkipReason protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination) { - if (file.StartsWith(".")) + if (file.StartsWith(".",StringComparison.Ordinal)) return SkipReason.IsImaginaryFile; //if there is a regex pattern - if (FilePattern != null) - if (!FilePattern.IsMatch(file)) //and it does not match - return SkipReason.DidNotMatchPattern; //skip because it did not match pattern + if (FilePattern?.IsMatch(file) == false) //and it does not match + return SkipReason.DidNotMatchPattern; //skip because it did not match pattern //if the file on the FTP already exists in the forLoading directory, skip it return destination.ForLoading.GetFiles(file).Any() ? SkipReason.InForLoading : SkipReason.DoNotSkip; } - private bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain, - SslPolicyErrors sslpolicyerrors) => true; //any cert will do! yay + private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3, + SslPolicyErrors _4) => true; //any cert will do! yay - protected virtual string[] GetFileList() + protected virtual IEnumerable GetFileList() { - var result = new StringBuilder(); - WebResponse response = null; - StreamReader reader = null; - try - { - var uri = !string.IsNullOrWhiteSpace(RemoteDirectory) - ? $"ftp://{_host}/{RemoteDirectory}" - : $"ftp://{_host}"; - -#pragma warning disable SYSLIB0014 // Type or member is obsolete - var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(uri)); -#pragma warning restore SYSLIB0014 // Type or member is obsolete - reqFTP.UseBinary = true; - reqFTP.Credentials = new NetworkCredential(_username, _password); - reqFTP.Method = WebRequestMethods.Ftp.ListDirectory; - reqFTP.Timeout = TimeoutInSeconds * 1000; - reqFTP.KeepAlive = KeepAlive; - - reqFTP.Proxy = null; - reqFTP.KeepAlive = false; - reqFTP.UsePassive = true; - reqFTP.EnableSsl = _useSSL; - - //accept any certificates - ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate; - response = reqFTP.GetResponse(); - - reader = new StreamReader(response.GetResponseStream()); - var line = reader.ReadLine(); - while (line != null) - { - result.Append(line); - result.Append('\n'); - line = reader.ReadLine(); - } - - // to remove the trailing '\n' - result.Remove(result.ToString().LastIndexOf('\n'), 1); - return result.ToString().Split('\n'); - } - finally - { - reader?.Close(); - - response?.Close(); - } + return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists); } - protected virtual void Download(string file, ILoadDirectory destination, IDataLoadEventListener job) + protected virtual void Download(string file, ILoadDirectory destination) { - var s = new Stopwatch(); - s.Start(); - - var uri = !string.IsNullOrWhiteSpace(RemoteDirectory) - ? $"ftp://{_host}/{RemoteDirectory}/{file}" - : $"ftp://{_host}/{file}"; - - if (_useSSL) - uri = $"s{uri}"; - - var serverUri = new Uri(uri); - if (serverUri.Scheme != Uri.UriSchemeFtp) return; - -#pragma warning disable SYSLIB0014 // Type or member is obsolete - var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(uri)); -#pragma warning restore SYSLIB0014 // Type or member is obsolete - reqFTP.Credentials = new NetworkCredential(_username, _password); - reqFTP.KeepAlive = false; - reqFTP.Method = WebRequestMethods.Ftp.DownloadFile; - reqFTP.UseBinary = true; - reqFTP.Proxy = null; - reqFTP.UsePassive = true; - reqFTP.EnableSsl = _useSSL; - reqFTP.Timeout = TimeoutInSeconds * 1000; - - var response = (FtpWebResponse)reqFTP.GetResponse(); - var responseStream = response.GetResponseStream(); - var destinationFileName = Path.Combine(destination.ForLoading.FullName, file); + var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory) + ? $"{RemoteDirectory}/{file}" + : file; - using (var writeStream = new FileStream(destinationFileName, FileMode.Create)) - { - responseStream.CopyTo(writeStream); - writeStream.Close(); - } - - response.Close(); - - _filesRetrieved.Add(serverUri.ToString()); - s.Stop(); + var destinationFileName = Path.Combine(destination.ForLoading.FullName, file); + _connection.Value.DownloadFile(destinationFileName, remotePath); + _filesRetrieved.Add(remotePath); } public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) { - if (exitCode == ExitCodeType.Success && DeleteFilesOffFTPServerAfterSuccesfulDataLoad) - foreach (var file in _filesRetrieved) - { -#pragma warning disable SYSLIB0014 - // Type or member is obsolete - var reqFTP = (FtpWebRequest)WebRequest.Create(new Uri(file)); -#pragma warning restore SYSLIB0014 - // Type or member is obsolete - reqFTP.Credentials = new NetworkCredential(_username, _password); - reqFTP.KeepAlive = false; - reqFTP.Method = WebRequestMethods.Ftp.DeleteFile; - reqFTP.UseBinary = true; - reqFTP.Proxy = null; - reqFTP.UsePassive = true; - reqFTP.EnableSsl = _useSSL; - - var response = (FtpWebResponse)reqFTP.GetResponse(); - - if (response.StatusCode != FtpStatusCode.FileActionOK) - postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Warning, - $"Attempt to delete file at URI {file} resulted in response with StatusCode = {response.StatusCode}")); - else - postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, - $"Deleted FTP file at URI {file} status code was {response.StatusCode}")); - - response.Close(); - } + if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return; + + foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file); } @@ -297,7 +187,7 @@ public void Check(ICheckNotifier notifier) { try { - SetupFTP(); + SetupFtp(); } catch (Exception e) { diff --git a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs index 3b507bb673..c08736805e 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs @@ -8,6 +8,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.ReusableLibraryCode.Progress; @@ -18,7 +19,7 @@ namespace Rdmp.Core.DataLoad.Modules.FTP; /// /// load component which downloads files from a remote SFTP (Secure File Transfer Protocol) server to the ForLoading directory /// -/// Operates in the same way as except that it uses SSH. In addition this +/// Operates in the same way as except that it uses SSH. In addition, this /// class will not bother downloading any files that already exist in the forLoading directory (have the same name - file size is NOT checked) /// public class SFTPDownloader : FTPDownloader @@ -26,57 +27,48 @@ public class SFTPDownloader : FTPDownloader [DemandsInitialization("The keep-alive interval. In milliseconds. Requires KeepAlive to be set to take effect.")] public int KeepAliveIntervalMilliseconds { get; set; } - protected override void Download(string file, ILoadDirectory destination, IDataLoadEventListener job) - { - if (file.Contains('/') || file.Contains('\\')) - throw new Exception("Was not expecting a relative path here"); - - var s = new Stopwatch(); - s.Start(); + private readonly Lazy _connection; - using (var sftp = new SftpClient(_host, _username, _password)) - { - if (KeepAlive && KeepAliveIntervalMilliseconds > 0) - sftp.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds); - - sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds); - sftp.Connect(); + public SFTPDownloader(Lazy connection) + { + _connection = new Lazy(SetupSftp,LazyThreadSafetyMode.ExecutionAndPublication); + } - //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList()) - var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file; + private SftpClient SetupSftp() + { + var host = FTPServer?.Server ?? throw new NullReferenceException("FTP server not set"); + var username = FTPServer.Username ?? "anonymous"; + var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); + var c = new SftpClient(host, username, password); + c.Connect(); + return c; + } - var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file); - //register for events - void Callback(ulong totalBytes) => job.OnProgress(this, - new ProgressEventArgs(destinationFilePath, - new ProgressMeasurement((int)(totalBytes * 0.001), ProgressType.Kilobytes), s.Elapsed)); + protected override void Download(string file, ILoadDirectory destination) + { + if (file.Contains('/') || file.Contains('\\')) + throw new Exception("Was not expecting a relative path here"); - using (var fs = new FileStream(destinationFilePath, FileMode.CreateNew)) - { - //download - sftp.DownloadFile(fullFilePath, fs, Callback); - fs.Close(); - } + //if there is a specified remote directory then reference it otherwise reference it locally (or however we were told about it from GetFileList()) + var fullFilePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? Path.Combine(RemoteDirectory, file) : file; - _filesRetrieved.Add(fullFilePath); - } + var destinationFilePath = Path.Combine(destination.ForLoading.FullName, file); - s.Stop(); + using (var dest=File.Create(destinationFilePath)) + _connection.Value.DownloadFile(fullFilePath,dest); + _filesRetrieved.Add(fullFilePath); } public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) { if (exitCode != ExitCodeType.Success) return; - using var sftp = new SftpClient(_host, _username, _password); - sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds); - sftp.Connect(); foreach (var retrievedFiles in _filesRetrieved) try { - sftp.DeleteFile(retrievedFiles); + _connection.Value.DeleteFile(retrievedFiles); postLoadEventListener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Deleted SFTP file {retrievedFiles} from SFTP server")); } @@ -90,15 +82,8 @@ public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEven protected override string[] GetFileList() { - using var sftp = new SftpClient(_host, _username, _password); - sftp.ConnectionInfo.Timeout = new TimeSpan(0, 0, 0, TimeoutInSeconds); - sftp.Connect(); - - var directory = RemoteDirectory; - - if (string.IsNullOrWhiteSpace(directory)) - directory = "."; + var directory = string.IsNullOrWhiteSpace(RemoteDirectory) ? "." : RemoteDirectory; - return sftp.ListDirectory(directory).Select(d => d.Name).ToArray(); + return _connection.Value.ListDirectory(directory).Select(static d => d.Name).ToArray(); } } \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 5d208440b0..2d45fd1a93 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -295,6 +295,7 @@ + @@ -305,23 +306,23 @@ - - + + - + - - PreserveNewest - + + Never + diff --git a/Rdmp.Core/Repositories/Managers/CommentStoreWithKeywords.cs b/Rdmp.Core/Repositories/Managers/CommentStoreWithKeywords.cs index 186235f7cf..fa358f6041 100644 --- a/Rdmp.Core/Repositories/Managers/CommentStoreWithKeywords.cs +++ b/Rdmp.Core/Repositories/Managers/CommentStoreWithKeywords.cs @@ -13,16 +13,16 @@ namespace Rdmp.Core.Repositories.Managers; /// /// Subclass of which also loads KeywordHelp.txt /// -public class CommentStoreWithKeywords : CommentStore +public sealed class CommentStoreWithKeywords : CommentStore { public override void ReadComments(params string[] directoriesToLookInForComments) { base.ReadComments(directoriesToLookInForComments); - var keywords = new FileInfo("./Curation/KeywordHelp.txt"); - - if (keywords.Exists) - AddToHelp(File.ReadAllText(keywords.FullName)); + var assembly=typeof(CommentStoreWithKeywords).Assembly; + using var stream = assembly.GetManifestResourceStream($"{assembly.GetName().Name}.Curation.KeywordHelp.txt"); + using var reader = new StreamReader(stream ?? throw new ApplicationException("Unable to read KeywordHelp.txt resource")); + AddToHelp(reader.ReadToEnd()); } private void AddToHelp(string keywordHelpFileContents) diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index 4c72d5fc9c..931382f5ec 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -41,7 +41,7 @@ - + diff --git a/wix/build.cmd b/wix/build.cmd index 64f2b97912..aa2e5685fe 100644 --- a/wix/build.cmd +++ b/wix/build.cmd @@ -1,9 +1,9 @@ @echo off set RDMPVERSION=%1 set wix="C:\Program Files (x86)\WiX Toolset v3.11\bin" -cd /d wix +cd /d %~dp0 %wix%\candle.exe rdmp.wxs -dVersion=%RDMPVERSION% -arch x64 -ext WixUtilExtension -nologo if errorlevel 1 exit 1 %wix%\light.exe rdmp.wixobj -ext WixUtilExtension -nologo if errorlevel 1 exit 1 -move rdmp.cli ..\dist\ \ No newline at end of file +move rdmp.msi ..\dist\ diff --git a/wix/rdmp.wxs b/wix/rdmp.wxs index 24ee3e3f00..949b738c1f 100644 --- a/wix/rdmp.wxs +++ b/wix/rdmp.wxs @@ -24,39 +24,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -121,30 +58,9 @@ - - - - - - - - - - - - - - - - - - - - -