diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b29c6c22..e9652cf2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,17 +9,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ... -## [1.3.1] - (2020-02-13) +## [1.4.0] - 2020-02-14 -### Changed +### Added -- Pinned fo-dicom to v4.0.1 +- Added in memory caching of the last 1024 values when using Redis wrapper for an IdentifierSwapper +- Added some parallelism and marshalling of backend queries to improve throughput in IdentifierSwapper +- Added temporary flag for RabbitMQAdapter parallelism for the above. Only enabled for the IdentifierMapper for now +- Added new mode to DicomDirectoryProcessor which allows reading in a list of accession directories -## [1.3.0] - (2020-02-06) +## [1.3.1] - 2020-02-13 ### Changed -- Refactor Java exception handling and use of threads +- Pinned fo-dicom to v4.0.1 + +## [1.3.0] - 2020-02-06 ### Added @@ -39,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed - Updated RDMP and Dicom plugins +- Refactor Java exception handling and use of threads ## [1.2.3] - 2020-01-09 @@ -81,15 +87,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improved travis deployment - (Re-)added Smi.NLog.config in builds +- Added better CLI argument descriptions for DicomReprocessor +- Added error logging for RabbitMQ bad Ack responses + - Previously: `BasicReturn for TEST.IdentifiableImageExchange` + - Now : `BasicReturn for Exchange 'TEST.IdentifiableImageExchange' Routing Key 'reprocessed' ReplyCode '312' (NO_ROUTE)` +- Added new swapper `TableLookupWithGuidFallbackSwapper` which performs lookup substitutions but allocates guids for lookup misses +- Added Travis CI build & deploy for all services ### Changed - Make exceptions on startup clearer - -## [1.2.0-rc1] - 2019-12-10 - -### Changed - - Updated to latest RDMP API (4.0.1) - `TableLookupSwapper` now throws consistent error if the provided table does not exist during `Setup` (previously it would error with DBMS specific error message at lookup time) @@ -98,27 +105,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fixed freeze condition when exchanges are not mapped to queues - IdentifierMapper now loads all FAnsi database implementations up front on startup -### Added - -- Added better CLI argument descriptions for DicomReprocessor -- Added error logging for RabbitMQ bad Ack responses - - Previously: `BasicReturn for TEST.IdentifiableImageExchange` - - Now : `BasicReturn for Exchange 'TEST.IdentifiableImageExchange' Routing Key 'reprocessed' ReplyCode '312' (NO_ROUTE)` -- Added new swapper `TableLookupWithGuidFallbackSwapper` which performs lookup substitutions but allocates guids for lookup misses -- Added Travis CI build & deploy for all services - ## [1.1.0] - 2019-11-22 -## Added +### Added - Improvements to unit and integration tests - Documentation fixes - Config file for Dependabot - Test for DicomFile SkipLargeTags option. Closes [#19](https://dev.azure.com/SmiOps/MVP%20Service/_workitems/edit/19) -## Changed +### Changed -### C# +#### C# - Bumped HIC.DicomTypeTranslation from 1.0.0.3 to 2.1.2 - Bumped HIC.RDMP.Plugin from 3.1.1 to 4.0.1-rc2 @@ -127,11 +125,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bumped System.IO.Abstractions from 4.2.17 to 7.0.7 - Bumped MongoDB.Driver from 2.8.0 to 2.9.3 -### Java +#### Java - Bumped jackson-databind from 2.9.6 to 2.9.10.0 - ## [1.0.0] - 2019-11-18 First stable release after importing the repository from the private [SMIPlugin](https://github.com/HicServices/SMIPlugin) repo. @@ -140,23 +137,23 @@ First stable release after importing the repository from the private [SMIPlugin] - ForGuidIdentifierSwapper automatically creates it's mapping database if it does not exist on the server referenced (previously only table was automatically created) -## Changed +### Changed - Updated to [Rdmp.Dicom 2.0.2](https://github.com/HicServices/RdmpDicom/blob/master/CHANGELOG.md#202-2019-11-13) - Updated to [Rdmp.Core 3.2.1](https://github.com/HicServices/RDMP/blob/develop/CHANGELOG.md#321---2019-10-30) -## Removed +### Removed - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported. -[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.3.1...develop +[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.0...develop +[1.4.0]: https://github.com/SMI/SmiServices/compare/v1.3.1...v1.4.0 [1.3.1]: https://github.com/SMI/SmiServices/compare/v1.3.0...v1.3.1 [1.3.0]: https://github.com/SMI/SmiServices/compare/v1.2.3...v1.3.0 [1.2.3]: https://github.com/SMI/SmiServices/compare/v1.2.2...v1.2.3 [1.2.2]: https://github.com/SMI/SmiServices/compare/v1.2.1...v1.2.2 [1.2.1]: https://github.com/SMI/SmiServices/compare/1.2.0...v1.2.1 [1.2.0]: https://github.com/SMI/SmiServices/compare/1.1.0-rc1...1.2.0 -[1.2.0-rc1]: https://github.com/SMI/SmiServices/compare/1.1.0...1.2.0-rc1 [1.1.0]: https://github.com/SMI/SmiServices/compare/1.0.0...1.1.0 [1.0.0]: https://github.com/SMI/SmiServices/releases/tag/1.0.0 diff --git a/PACKAGES.md b/PACKAGES.md index a97224eeb..5e50fc20a 100644 --- a/PACKAGES.md +++ b/PACKAGES.md @@ -9,24 +9,25 @@ | Package | Source Code | Version | License | Purpose | Additional Risk Assessment | | ------- | ------------| --------| ------- | ------- | -------------------------- | -| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | | -| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | | +| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | +| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.1.2](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.1.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | | -| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | | -| [Newtonsoft.Json](https://www.newtonsoft.com/json) | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | -| [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| [RabbitMQ.Client](https://www.rabbitmq.com/) | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | -| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| -| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [7.0.7](https://www.nuget.org/packages/System.IO.Abstractions/7.0.7) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | | -|[MongoDB.Driver](https://docs.mongodb.com/ecosystem/drivers/csharp/)| [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases| +| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | | | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | | -| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing | | -| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | | -| fo-Dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.3](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.3) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | +| Microsoft.Extensions.Caching.Memory | [GitHub]() | [3.1.1](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.1) | | | | +| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)| [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases| +| NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | +| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | +| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | | +| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.0.601](https://www.nuget.org/packages/StackExchange.Redis/2.0.601) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | | | Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | | | System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Drawing.Common/4.6.0) | [MIT](https://opensource.org/licenses/MIT) | Supports reading pixel data | | +| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [7.0.7](https://www.nuget.org/packages/System.IO.Abstractions/7.0.7) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | | | System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | File I/O | | | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT) | File access perimssions| | | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt) | Optical Character Recognition in Dicom Pixel data| | -| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.0.601](https://www.nuget.org/packages/StackExchange.Redis/2.0.601) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | | \ No newline at end of file +| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| +| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | | +| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | | +| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | | diff --git a/README.md b/README.md index d687381f7..ecdd0c917 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ - [![Build Status](https://travis-ci.com/SMI/SmiServices.svg?branch=master)](https://travis-ci.com/SMI/SmiServices) ![GitHub](https://img.shields.io/github/license/SMI/SmiServices) [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/) -Version: `1.3.1` +Version: `1.4.0` # SMI Services @@ -91,10 +90,19 @@ Appart from the Microservices (documented above) the following library classes a Building requires the [.NET Core 2.2 SDK](https://dotnet.microsoft.com/download/dotnet-core/2.2) ```bash -dotnet build [-r RID] +$ dotnet build [-r RID] +``` + +_The RID argument is optional. Use this if you want to build for a different platform e.g. `-r linux-x64` to build for Linux from a Windows machine. See [here](https://docs.microsoft.com/en-us/dotnet/core/rid-catalog) for more info on runtime identifiers._ + +To build an individual sub-project: + +```bash +$ cd src/microservices/Microservices.DicomTagReader/ +$ dotnet build ``` -_To build other OS substitute the respective [runtime identifier](https://docs.microsoft.com/en-us/dotnet/core/rid-catalog) e.g. linux-x64_ +This will automatically rebuild any dependent projects which have changes as well. ### Building the Java Projects diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c712dd422..5875e5b02 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -7,6 +7,6 @@ [assembly: AssemblyCulture("")] // These should be overwritten by release builds -[assembly: AssemblyVersion("1.3.1")] -[assembly: AssemblyFileVersion("1.3.1")] -[assembly: AssemblyInformationalVersion("1.3.1")] // This one can have the extra build info after it +[assembly: AssemblyVersion("1.4.0")] +[assembly: AssemblyFileVersion("1.4.0")] +[assembly: AssemblyInformationalVersion("1.4.0")] // This one can have the extra build info after it diff --git a/src/applications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs b/src/applications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs index 13decff08..7140a9d58 100644 --- a/src/applications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs +++ b/src/applications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs @@ -1,11 +1,11 @@  +using System; +using System.Globalization; +using System.IO; using Applications.DicomDirectoryProcessor.Execution.DirectoryFinders; using Applications.DicomDirectoryProcessor.Options; using Smi.Common.Execution; using Smi.Common.Options; -using System; -using System.Globalization; -using System.IO; namespace Applications.DicomDirectoryProcessor.Execution { @@ -27,14 +27,30 @@ public DicomDirectoryProcessorHost(GlobalOptions globals, DicomDirectoryProcesso { _cliOptions = cliOptions; - if (!Directory.Exists(globals.FileSystemOptions.FileSystemRoot)) - throw new ArgumentException("Cannot find the FileSystemRoot specified in the given MicroservicesOptions (" + globals.FileSystemOptions.FileSystemRoot + ")"); + if (!cliOptions.DirectoryFormat.ToLower().Equals("list")) + { + Logger.Info("This indicates that the list mode is not being recognised"); - if (!cliOptions.ToProcessDir.Exists) - throw new ArgumentException("Could not find directory " + cliOptions.ToProcessDir.FullName); + // TODO(rkm 2020-02-12) I think we want to check this regardless of the mode + // (bp 2020-02-13) By not doing this check on list means that the list of paths is not required to be in PACS and can be imported from anywhere + if (!Directory.Exists(globals.FileSystemOptions.FileSystemRoot)) + throw new ArgumentException("Cannot find the FileSystemRoot specified in the given MicroservicesOptions (" + globals.FileSystemOptions.FileSystemRoot + ")"); - if (!cliOptions.ToProcessDir.FullName.StartsWith(globals.FileSystemOptions.FileSystemRoot, true, CultureInfo.CurrentCulture)) - throw new ArgumentException("Directory parameter (" + cliOptions.ToProcessDir.FullName + ") must be below the FileSystemRoot (" + globals.FileSystemOptions.FileSystemRoot + ")"); + if (!cliOptions.ToProcessDir.Exists) + throw new ArgumentException("Could not find directory " + cliOptions.ToProcessDir.FullName); + + if (!cliOptions.ToProcessDir.FullName.StartsWith(globals.FileSystemOptions.FileSystemRoot, true, CultureInfo.CurrentCulture)) + throw new ArgumentException("Directory parameter (" + cliOptions.ToProcessDir.FullName + ") must be below the FileSystemRoot (" + globals.FileSystemOptions.FileSystemRoot + ")"); + } + else + { + Logger.Info("This indicates that the list mode is being recognised"); + if (!File.Exists(cliOptions.ToProcessDir.FullName)) + throw new ArgumentException("Could not find accession directory list file (" + cliOptions.ToProcessDir.FullName + ")"); + + if (!Path.GetExtension(cliOptions.ToProcessDir.FullName).Equals(".csv")) + throw new ArgumentException("When in 'list' mode, path to accession directory file of format .csv expected (" + cliOptions.ToProcessDir.FullName + ")"); + } if (cliOptions.DirectoryFormat.ToLower().Equals("pacs")) { @@ -43,6 +59,13 @@ public DicomDirectoryProcessorHost(GlobalOptions globals, DicomDirectoryProcesso _ddf = new PacsDirectoryFinder(globals.FileSystemOptions.FileSystemRoot, globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions)); } + else if (cliOptions.DirectoryFormat.ToLower().Equals("list")) + { + Logger.Info("Creating accession directory lister"); + + _ddf = new AccessionDirectoryLister(globals.FileSystemOptions.FileSystemRoot, + globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions)); + } else if (cliOptions.DirectoryFormat.ToLower().Equals("default")) { Logger.Info("Creating basic directory finder"); diff --git a/src/applications/Applications.DicomDirectoryProcessor/Execution/DirectoryFinders/AccessionDirectoryLister.cs b/src/applications/Applications.DicomDirectoryProcessor/Execution/DirectoryFinders/AccessionDirectoryLister.cs new file mode 100644 index 000000000..3fdaeffbe --- /dev/null +++ b/src/applications/Applications.DicomDirectoryProcessor/Execution/DirectoryFinders/AccessionDirectoryLister.cs @@ -0,0 +1,81 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using System.Text.RegularExpressions; +using Smi.Common.Messaging; + +namespace Applications.DicomDirectoryProcessor.Execution.DirectoryFinders +{ + public class AccessionDirectoryLister : DicomDirectoryFinder + { + // Regex that matches when we are at the yyyy\mm\dd\xxxxx directory level + private readonly Regex _accDirectoryRegex = new Regex(@"(20\d{2}[\\\/]\d{2}[\\\/]\d{2})[\\\/]\w+[^.]*?$"); + + public AccessionDirectoryLister(string fileSystemRoot, IFileSystem fileSystem, string dicomSearchPattern, IProducerModel directoriesProducerModel) + : base(fileSystemRoot, fileSystem, dicomSearchPattern, directoriesProducerModel) { } + + public AccessionDirectoryLister(string fileSystemRoot, string dicomSearchPattern, IProducerModel directoriesProducerModel) + : this(fileSystemRoot, new FileSystem(), dicomSearchPattern, directoriesProducerModel) { } + + + public override void SearchForDicomDirectories(string accessionsList) + { + Logger.Info("Starting accession directory path listing from: " + accessionsList); + IsProcessing = true; + TotalSent = 0; + + using (var reader = new StreamReader(accessionsList)) + { + while (!reader.EndOfStream && !TokenSource.IsCancellationRequested) + { + + string accessionDirectory = reader.ReadLine().Replace(",", ""); + + if (!_accDirectoryRegex.IsMatch(accessionDirectory)) + { + Logger.Warn($"This path does not point to an accession directory: ({accessionDirectory}), continuing"); + continue; + } + + if (!FileSystem.Directory.Exists(accessionDirectory)) + { + Logger.Warn($"Can not find {accessionDirectory}, continuing"); + continue; + } + + IDirectoryInfo dirInfo = FileSystem.DirectoryInfo.FromDirectoryName(accessionDirectory); + IEnumerable fileEnumerator; + + try + { + fileEnumerator = dirInfo.EnumerateFiles(SearchPattern); + } + catch (Exception e) + { + Logger.Error($"Could not enumerate files: {e.Message}"); + continue; + } + + if (fileEnumerator.FirstOrDefault() != null) + { + Logger.Debug("Sending message (" + accessionDirectory + ")"); + FoundNewDicomDirectory(accessionDirectory.Remove(0, FileSystemRoot.Length)); + } + else + { + Logger.Warn($"Could not find dicom files in the given accession directory ({accessionDirectory}), skipping"); + } + } + } + + IsProcessing = false; + + Logger.Info("Reading from list finished"); + Logger.Info("Total messages sent: " + TotalSent); + } + } +} + diff --git a/src/common/Smi.Common/Execution/MicroserviceHost.cs b/src/common/Smi.Common/Execution/MicroserviceHost.cs index 866025471..fabad1fa7 100644 --- a/src/common/Smi.Common/Execution/MicroserviceHost.cs +++ b/src/common/Smi.Common/Execution/MicroserviceHost.cs @@ -111,7 +111,10 @@ protected MicroserviceHost(GlobalOptions globals, bool loadSmiLogConfig = true) // throw new ArgumentException("Could not locate the FileSystemRoot \"" + options.FileSystemRoot + "\""); OnFatal += (sender, args) => Fatal(args.Message, args.Exception); - RabbitMqAdapter = new RabbitMqAdapter(globals.RabbitOptions, HostProcessName + HostProcessID, OnFatal); + + // TODO(rkm 2020-02-14) This is only a temporary fix + bool threaded = (HostProcessName == "IdentifierMapper"); + RabbitMqAdapter = new RabbitMqAdapter(globals.RabbitOptions, HostProcessName + HostProcessID, OnFatal, threaded); _controlMessageConsumer = new ControlMessageConsumer(this, globals.RabbitOptions, HostProcessName, HostProcessID); diff --git a/src/common/Smi.Common/Options/GlobalOptions.cs b/src/common/Smi.Common/Options/GlobalOptions.cs index e3e38092a..dcaefb498 100644 --- a/src/common/Smi.Common/Options/GlobalOptions.cs +++ b/src/common/Smi.Common/Options/GlobalOptions.cs @@ -520,6 +520,7 @@ public class RabbitOptions public string RabbitMqPassword { get; set; } public string FatalLoggingExchange { get; set; } public string RabbitMqControlExchangeName { get; set; } + public bool ThreadReceivers { get; set; } public bool Validate() { diff --git a/src/common/Smi.Common/RabbitMQAdapter.cs b/src/common/Smi.Common/RabbitMQAdapter.cs index 502cecaf0..ab7175b76 100644 --- a/src/common/Smi.Common/RabbitMQAdapter.cs +++ b/src/common/Smi.Common/RabbitMQAdapter.cs @@ -56,6 +56,7 @@ public bool HasConsumers private const int MaxSubscriptionAttempts = 5; + private readonly bool _threaded; /// /// @@ -63,8 +64,21 @@ public bool HasConsumers /// Connection parameters to a RabbitMQ server /// Identifier for this host instance /// - public RabbitMqAdapter(RabbitOptions options, string hostId, HostFatalHandler hostFatalHandler = null) + /// + public RabbitMqAdapter(RabbitOptions options, string hostId, HostFatalHandler hostFatalHandler = null, bool threaded = false) { + //_threaded = options.ThreadReceivers; + _threaded = threaded; + + if (_threaded) + { + int minWorker, minIOC; + ThreadPool.GetMinThreads(out minWorker, out minIOC); + int workers = Math.Max(50, minWorker); + ThreadPool.SetMaxThreads(workers, 50); + _logger.Info($"Set Rabbit event concurrency to {workers}"); + } + _factory = new ConnectionFactory { HostName = options.RabbitMqHostName, @@ -387,7 +401,12 @@ private void Consume(ISubscription subscription, IConsumer consumer, Cancellatio BasicDeliverEventArgs e; if (subscription.Next(500, out e)) - consumer.ProcessMessage(e); + { + if (_threaded) + Task.Run(() => consumer.ProcessMessage(e)); + else + consumer.ProcessMessage(e); + } } string reason = "unknown"; diff --git a/src/microservices/Microservices.IdentifierMapper/Execution/Swappers/RedisSwapper.cs b/src/microservices/Microservices.IdentifierMapper/Execution/Swappers/RedisSwapper.cs index 867980e0c..fb31c4e90 100644 --- a/src/microservices/Microservices.IdentifierMapper/Execution/Swappers/RedisSwapper.cs +++ b/src/microservices/Microservices.IdentifierMapper/Execution/Swappers/RedisSwapper.cs @@ -1,7 +1,7 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Collections.Concurrent; +using System.Threading; +using Microsoft.Extensions.Caching.Memory; using NLog; using Smi.Common.Options; using StackExchange.Redis; @@ -19,6 +19,12 @@ public class RedisSwapper : SwapIdentifiers,IDisposable private const string NullString = "NO MATCH"; + private MemoryCache _cache = new MemoryCache(new MemoryCacheOptions() + { + SizeLimit = 1024 + }); + private ConcurrentDictionary _locks = new ConcurrentDictionary(); + public RedisSwapper(string redisHost, ISwapIdentifiers wrappedSwapper) { _redis = ConnectionMultiplexer.Connect(redisHost); @@ -31,45 +37,69 @@ public override void Setup(IMappingTableOptions mappingTableOptions) public override string GetSubstitutionFor(string toSwap, out string reason) { - string output; + string result; reason = null; - IDatabase db = _redis.GetDatabase(); - - //look up Redis for a cached answer - var val = db.StringGet(toSwap); - - //we have a cached answer (which might be null) - if (val.HasValue) + //lookup in memory + if (!_cache.TryGetValue(toSwap, out result)) { - output = val.ToString(); - CacheHit++; - } - else - { + SemaphoreSlim locket = _locks.GetOrAdd(toSwap, k => new SemaphoreSlim(1, 1)); + locket.Wait(); + try + { + if (!_cache.TryGetValue(toSwap, out result)) + { + // Now try Redis cache + IDatabase db = _redis.GetDatabase(); + var val = db.StringGet(toSwap); + //we have a cached answer (which might be null) + if (val.HasValue) + { + result = val.ToString(); + CacheHit++; + } + else + { + //we have no cached answer from Redis + CacheMiss++; - //we have no cached answer from Redis - CacheMiss++; - - //Go to the hosted swapper - output = _hostedSwapper.GetSubstitutionFor(toSwap, out reason); + //Go to the hosted swapper + lock(_hostedSwapper) + { + result = _hostedSwapper.GetSubstitutionFor(toSwap, out reason); + } - //and cache the result (even if it is null - no lookup match found) - db.StringSet(toSwap, output ?? NullString, null, When.NotExists); + //and cache the result (even if it is null - no lookup match found) + db.StringSet(toSwap, result ?? NullString, null, When.NotExists); + } + + _cache.Set(toSwap, result ?? NullString, new MemoryCacheEntryOptions() { + Size=1 + }); + } + } + finally + { + locket.Release(); + } + } + else + { + CacheHit++; } - if (string.Equals(NullString, output)) + if (string.Equals(NullString, result)) { - output = null; + result = null; reason = $"Value '{toSwap}' was cached in Redis as missing (i.e. no mapping was found)"; } - - if (output == null) + + if (result == null) Fail++; else Success++; - return output; + return result; } diff --git a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj index 9f076504f..d446e2243 100644 --- a/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj +++ b/src/microservices/Microservices.IdentifierMapper/Microservices.IdentifierMapper.csproj @@ -16,6 +16,7 @@ + diff --git a/tests/applications/Applications.DicomDirectoryProcessor.Tests/AccessionDirectoryListerTest.cs b/tests/applications/Applications.DicomDirectoryProcessor.Tests/AccessionDirectoryListerTest.cs new file mode 100644 index 000000000..0cf56c321 --- /dev/null +++ b/tests/applications/Applications.DicomDirectoryProcessor.Tests/AccessionDirectoryListerTest.cs @@ -0,0 +1,24 @@ + +using NUnit.Framework; +using Smi.Common.Tests; + + +namespace Applications.DicomDirectoryProcessor.Tests +{ + /// + /// Unit tests for AccessionDirectoryLister + /// + [TestFixture] + public class AccessionDirectoryListerTest + { + [OneTimeSetUp] + public void OneTimeSetUp() + { + TestLogger.Setup(); + } + + // TODO(rkm 2020-02-12) Things to test + // - Valid CSV file + // - CSVs with various invalid data / lines + } +} diff --git a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs index 9c69e504b..c77a0c7e9 100644 --- a/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs +++ b/tests/common/Smi.Common.Tests/NuspecIsCorrectTests.cs @@ -69,6 +69,10 @@ public void TestDependencyCorrect(string csproj, string nuspec, string packagesM string package = p.Groups[1].Value; string version = p.Groups[2].Value; + // NOTE(rkm 2020-02-14) Fix for specifiers which contain lower or upper bounds + if (version.Contains("[") || version.Contains("(")) + version = version.Substring(1,5); + bool found = false; //analyzers do not have to be listed as a dependency in nuspec (but we should document them in packages.md)