Merge pull request #112 from SMI/release/v1.4.0

Release v1.4.0
SMI · Feb 14, 2020 · 3cdffa7 · 3cdffa7
2 parents 8449712 + 7b6e456
commit 3cdffa7
Show file tree

Hide file tree

Showing 13 changed files with 280 additions and 88 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,17 +9,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ...
 
-## [1.3.1] - (2020-02-13)
+## [1.4.0] - 2020-02-14
 
-### Changed
+### Added
 
-- Pinned fo-dicom to v4.0.1
+- Added in memory caching of the last 1024 values when using Redis wrapper for an IdentifierSwapper
+- Added some parallelism and marshalling of backend queries to improve throughput in IdentifierSwapper
+- Added temporary flag for RabbitMQAdapter parallelism for the above. Only enabled for the IdentifierMapper for now
+- Added new mode to DicomDirectoryProcessor which allows reading in a list of accession directories
 
-## [1.3.0] - (2020-02-06)
+## [1.3.1] - 2020-02-13
 
 ### Changed
 
-- Refactor Java exception handling and use of threads
+- Pinned fo-dicom to v4.0.1
+
+## [1.3.0] - 2020-02-06
 
 ### Added
 
@@ -39,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Changed
 
 - Updated RDMP and Dicom plugins
+- Refactor Java exception handling and use of threads
 
 ## [1.2.3] - 2020-01-09
 
@@ -81,15 +87,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 - Improved travis deployment
 - (Re-)added Smi.NLog.config in builds
+- Added better CLI argument descriptions for DicomReprocessor
+- Added error logging for RabbitMQ bad Ack responses 
+  - Previously: `BasicReturn for TEST.IdentifiableImageExchange`
+  - Now : `BasicReturn for Exchange 'TEST.IdentifiableImageExchange' Routing Key 'reprocessed' ReplyCode '312' (NO_ROUTE)`
+- Added new swapper `TableLookupWithGuidFallbackSwapper` which performs lookup substitutions but allocates guids for lookup misses
+- Added Travis CI build & deploy for all services
 
 ### Changed
 
 - Make exceptions on startup clearer
-
-## [1.2.0-rc1] - 2019-12-10
-
-### Changed
-
 - Updated to latest RDMP API (4.0.1)
 - `TableLookupSwapper` now throws consistent error if the provided table does not exist during `Setup` (previously it would error with DBMS specific error message at lookup time)
 
@@ -98,27 +105,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fixed freeze condition when exchanges are not mapped to queues
 - IdentifierMapper now loads all FAnsi database implementations up front on startup
 
-### Added
-
-- Added better CLI argument descriptions for DicomReprocessor
-- Added error logging for RabbitMQ bad Ack responses 
-  - Previously: `BasicReturn for TEST.IdentifiableImageExchange`
-  - Now : `BasicReturn for Exchange 'TEST.IdentifiableImageExchange' Routing Key 'reprocessed' ReplyCode '312' (NO_ROUTE)`
-- Added new swapper `TableLookupWithGuidFallbackSwapper` which performs lookup substitutions but allocates guids for lookup misses
-- Added Travis CI build & deploy for all services
-
 ## [1.1.0] - 2019-11-22
 
-## Added
+### Added
 
 - Improvements to unit and integration tests
 - Documentation fixes
 - Config file for Dependabot
 - Test for DicomFile SkipLargeTags option. Closes [#19](https://dev.azure.com/SmiOps/MVP%20Service/_workitems/edit/19)
 
-## Changed
+### Changed
 
-### C#
+#### C#
 
 - Bumped HIC.DicomTypeTranslation from 1.0.0.3 to 2.1.2
 - Bumped HIC.RDMP.Plugin from 3.1.1 to 4.0.1-rc2
@@ -127,11 +125,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumped System.IO.Abstractions from 4.2.17 to 7.0.7
 - Bumped MongoDB.Driver from 2.8.0 to 2.9.3
 
-### Java
+#### Java
 
 - Bumped jackson-databind from 2.9.6 to 2.9.10.0
 
-
 ## [1.0.0] - 2019-11-18
 
 First stable release after importing the repository from the private [SMIPlugin](https://github.com/HicServices/SMIPlugin) repo.
@@ -140,23 +137,23 @@ First stable release after importing the repository from the private [SMIPlugin]
 
 - ForGuidIdentifierSwapper automatically creates it's mapping database if it does not exist on the server referenced (previously only table was automatically created)
 
-## Changed
+### Changed
 
 - Updated to [Rdmp.Dicom 2.0.2](https://github.com/HicServices/RdmpDicom/blob/master/CHANGELOG.md#202-2019-11-13)
 - Updated to [Rdmp.Core 3.2.1](https://github.com/HicServices/RDMP/blob/develop/CHANGELOG.md#321---2019-10-30)
 
-## Removed
+### Removed
 
 - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`).  Previously skipping database portion was supported.
 
 
-[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.3.1...develop
+[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.0...develop
+[1.4.0]:  https://github.com/SMI/SmiServices/compare/v1.3.1...v1.4.0
 [1.3.1]:  https://github.com/SMI/SmiServices/compare/v1.3.0...v1.3.1
 [1.3.0]:  https://github.com/SMI/SmiServices/compare/v1.2.3...v1.3.0
 [1.2.3]:  https://github.com/SMI/SmiServices/compare/v1.2.2...v1.2.3
 [1.2.2]:  https://github.com/SMI/SmiServices/compare/v1.2.1...v1.2.2
 [1.2.1]:  https://github.com/SMI/SmiServices/compare/1.2.0...v1.2.1
 [1.2.0]:  https://github.com/SMI/SmiServices/compare/1.1.0-rc1...1.2.0
-[1.2.0-rc1]:  https://github.com/SMI/SmiServices/compare/1.1.0...1.2.0-rc1
 [1.1.0]: https://github.com/SMI/SmiServices/compare/1.0.0...1.1.0
 [1.0.0]: https://github.com/SMI/SmiServices/releases/tag/1.0.0
diff --git a/PACKAGES.md b/PACKAGES.md
@@ -9,24 +9,25 @@
 
 | Package | Source Code | Version | License | Purpose | Additional Risk Assessment |
 | ------- | ------------| --------| ------- | ------- | -------------------------- |
-| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | |
-| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | |
+| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing |  |
+| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | |
 | HIC.DicomTypeTranslation | [GitHub](https://github.com/HicServices/DicomTypeTranslation) | [2.1.2](https://www.nuget.org/packages/HIC.DicomTypeTranslation/2.1.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Translate dicom types into C# / database types | |
-| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
 | HIC.RDMP.Dicom | [GitHub](https://github.com/HicServices/RdmpDicom) | [2.0.7](https://www.nuget.org/packages/HIC.RDMP.Dicom/2.0.7) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | RDMP Plugin containing data load / pipeline components for imaging, reading dicom files etc | |
-| [Newtonsoft.Json](https://www.newtonsoft.com/json) | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission |
-| [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
-| [RabbitMQ.Client](https://www.rabbitmq.com/) | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | |
-| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet)  | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files|
-| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [7.0.7](https://www.nuget.org/packages/System.IO.Abstractions/7.0.7) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | |
-|[MongoDB.Driver](https://docs.mongodb.com/ecosystem/drivers/csharp/)| [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)|  [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases|
+| HIC.RDMP.Plugin | [GitHub](https://github.com/HicServices/RDMP) | [4.0.2](https://www.nuget.org/packages/HIC.RDMP.Plugin/4.0.2) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Interact with RDMP objects, base classes for plugin components etc | |
 | JetBrains.Annotations | | [2019.1.3](https://www.nuget.org/packages/JetBrains.Annotations/2019.1.3) |[MIT](https://opensource.org/licenses/MIT) | Static analysis tool | |
-| CommandLineParser | [GitHub](https://github.com/commandlineparser/commandline) | [2.5.0](https://www.nuget.org/packages/CommandLineParser/2.5.0) | [MIT](https://opensource.org/licenses/MIT)| Command line argument parsing |  |
-| CsvHelper | [GitHub](https://github.com/JoshClose/CsvHelper) | [12.1.2](https://www.nuget.org/packages/CsvHelper/12.1.2) | [MS-PL and Apache 2.0](https://github.com/JoshClose/CsvHelper/blob/master/LICENSE.txt)| Writting reports out to CSV reports | |
-| fo-Dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.3](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.3) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | |
+| Microsoft.Extensions.Caching.Memory | [GitHub]() | [3.1.1](https://www.nuget.org/packages/Microsoft.Extensions.Caching.Memory/3.1.1) | | | |
+| MongoDB.Driver | [GitHub](https://github.com/mongodb/mongo-csharp-driver) |[2.9.3](https://www.nuget.org/packages/MongoDB.Driver/2.9.3)|  [Apache 2.0](https://www.nuget.org/packages/MongoDB.Driver/2.8.1/License) | For writting/reading dicom tags into MongoDb databases|
+| NLog | [GitHub](https://github.com/NLog/NLog) | [4.6.4](https://www.nuget.org/packages/NLog/4.6.4) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | |
+| Newtonsoft.Json | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [12.0.3](https://www.nuget.org/packages/Newtonsoft.Json/12.0.3) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission |
+| RabbitMQ.Client | [GitHub](https://github.com/rabbitmq/rabbitmq-dotnet-client) | [5.1.2](https://www.nuget.org/packages/RabbitMQ.Client/5.1.2) | [Apache License v2 / MPL 1.1](https://github.com/rabbitmq/rabbitmq-dotnet-client/blob/master/LICENSE) | Handles messaging between microservices | |
+| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.0.601](https://www.nuget.org/packages/StackExchange.Redis/2.0.601) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | |
 | Stanford.NLP.CoreNLP | [GitHub Pages](https://sergey-tihon.github.io/Stanford.NLP.NET/) | [3.9.2](https://www.nuget.org/packages/Stanford.NLP.CoreNLP/3.9.2) | [GNU v2](https://github.com/sergey-tihon/Stanford.NLP.NET/blob/master/LICENSE.txt)| Name / Organisation detection in text | |
 | System.Drawing.Common | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Drawing.Common/4.6.0) | [MIT](https://opensource.org/licenses/MIT)  | Supports reading pixel data | |
+| System.IO.Abstractions | [GitHub](https://github.com/System-IO-Abstractions/System.IO.Abstractions) | [7.0.7](https://www.nuget.org/packages/System.IO.Abstractions/7.0.7) | [MIT](https://opensource.org/licenses/MIT) | Makes file system injectable in tests | |
 | System.IO.FileSystem | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.IO.FileSystem/4.3.0) |[MIT](https://opensource.org/licenses/MIT)  | File I/O | |
 | System.Security.AccessControl | [GitHub](https://github.com/dotnet/corefx) | [4.7.0](https://www.nuget.org/packages/System.Security.AccessControl/4.7.0) |[MIT](https://opensource.org/licenses/MIT)  | File access perimssions| |
 | Tesseract | [GitHub](https://github.com/charlesw/tesseract/) | [4.1.0-beta1](https://www.nuget.org/packages/Tesseract/4.1.0-beta1) |[Apache License v2](https://github.com/charlesw/tesseract/blob/master/LICENSE.txt)  | Optical Character Recognition in Dicom Pixel data| |
-| StackExchange.Redis | [GitHub](https://github.com/StackExchange/StackExchange.Redis) | [2.0.601](https://www.nuget.org/packages/StackExchange.Redis/2.0.601) |[MIT](https://opensource.org/licenses/MIT) | Required for RedisSwapper | |
+| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet)  | [6.0.0](https://www.nuget.org/packages/YamlDotNet/6.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files|
+| fo-dicom | [GitHub](https://github.com/fo-dicom/fo-dicom) |[4.0.1](https://www.nuget.org/packages/fo-dicom/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL) | Handles reading/writing dicom tags from dicom datasets | |
+| fo-dicom.Drawing | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-Dicom.Drawing/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for reading DICOM pixel data | |
+| fo-dicom.Json | [GitHub](https://github.com/fo-dicom/fo-dicom) | [4.0.1](https://www.nuget.org/packages/fo-dicom.Json/4.0.1) | [MS-PL](https://opensource.org/licenses/MS-PL)| Support library for serializing fo-dicom DICOM datasets to json | |
diff --git a/README.md b/README.md
@@ -1,10 +1,9 @@
 
-
 [![Build Status](https://travis-ci.com/SMI/SmiServices.svg?branch=master)](https://travis-ci.com/SMI/SmiServices)
 ![GitHub](https://img.shields.io/github/license/SMI/SmiServices)
 [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/)
 
-Version: `1.3.1`
+Version: `1.4.0`
 
 # SMI Services
 
@@ -91,10 +90,19 @@ Appart from the Microservices (documented above) the following library classes a
 Building requires the [.NET Core 2.2 SDK](https://dotnet.microsoft.com/download/dotnet-core/2.2)
 
 ```bash
-dotnet build [-r RID]
+$ dotnet build [-r RID]
+```
+
+_The RID argument is optional. Use this if you want to build for a different platform e.g. `-r linux-x64` to build for Linux from a Windows machine. See [here](https://docs.microsoft.com/en-us/dotnet/core/rid-catalog) for more info on runtime identifiers._
+
+To build an individual sub-project:
+
+```bash
+$ cd src/microservices/Microservices.DicomTagReader/
+$ dotnet build
 ```
 
-_To build other OS substitute the respective [runtime identifier](https://docs.microsoft.com/en-us/dotnet/core/rid-catalog) e.g. linux-x64_
+This will automatically rebuild any dependent projects which have changes as well.
 
 ### Building the Java Projects
 

diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs
@@ -7,6 +7,6 @@
 [assembly: AssemblyCulture("")]
 
 // These should be overwritten by release builds
-[assembly: AssemblyVersion("1.3.1")]
-[assembly: AssemblyFileVersion("1.3.1")]
-[assembly: AssemblyInformationalVersion("1.3.1")] // This one can have the extra build info after it
+[assembly: AssemblyVersion("1.4.0")]
+[assembly: AssemblyFileVersion("1.4.0")]
+[assembly: AssemblyInformationalVersion("1.4.0")] // This one can have the extra build info after it
diff --git a/...pplications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs b/...pplications/Applications.DicomDirectoryProcessor/Execution/DicomDirectoryProcessorHost.cs
@@ -1,11 +1,11 @@
 
+using System;
+using System.Globalization;
+using System.IO;
 using Applications.DicomDirectoryProcessor.Execution.DirectoryFinders;
 using Applications.DicomDirectoryProcessor.Options;
 using Smi.Common.Execution;
 using Smi.Common.Options;
-using System;
-using System.Globalization;
-using System.IO;
 
 namespace Applications.DicomDirectoryProcessor.Execution
 {
@@ -27,14 +27,30 @@ public DicomDirectoryProcessorHost(GlobalOptions globals, DicomDirectoryProcesso
         {
             _cliOptions = cliOptions;
 
-            if (!Directory.Exists(globals.FileSystemOptions.FileSystemRoot))
-                throw new ArgumentException("Cannot find the FileSystemRoot specified in the given MicroservicesOptions (" + globals.FileSystemOptions.FileSystemRoot + ")");
+            if (!cliOptions.DirectoryFormat.ToLower().Equals("list"))
+            {
+                Logger.Info("This indicates that the list mode is not being recognised");
 
-            if (!cliOptions.ToProcessDir.Exists)
-                throw new ArgumentException("Could not find directory " + cliOptions.ToProcessDir.FullName);
+                // TODO(rkm 2020-02-12) I think we want to check this regardless of the mode
+                // (bp 2020-02-13) By not doing this check on list means that the list of paths is not required to be in PACS and can be imported from anywhere
+                if (!Directory.Exists(globals.FileSystemOptions.FileSystemRoot))
+                    throw new ArgumentException("Cannot find the FileSystemRoot specified in the given MicroservicesOptions (" + globals.FileSystemOptions.FileSystemRoot + ")");
 
-            if (!cliOptions.ToProcessDir.FullName.StartsWith(globals.FileSystemOptions.FileSystemRoot, true, CultureInfo.CurrentCulture))
-                throw new ArgumentException("Directory parameter (" + cliOptions.ToProcessDir.FullName + ") must be below the FileSystemRoot (" + globals.FileSystemOptions.FileSystemRoot + ")");
+                if (!cliOptions.ToProcessDir.Exists)
+                    throw new ArgumentException("Could not find directory " + cliOptions.ToProcessDir.FullName);
+
+                if (!cliOptions.ToProcessDir.FullName.StartsWith(globals.FileSystemOptions.FileSystemRoot, true, CultureInfo.CurrentCulture))
+                    throw new ArgumentException("Directory parameter (" + cliOptions.ToProcessDir.FullName + ") must be below the FileSystemRoot (" + globals.FileSystemOptions.FileSystemRoot + ")");
+            }
+            else
+            {
+                Logger.Info("This indicates that the list mode is being recognised");
+                if (!File.Exists(cliOptions.ToProcessDir.FullName))
+                    throw new ArgumentException("Could not find accession directory list file (" + cliOptions.ToProcessDir.FullName + ")");
+
+                if (!Path.GetExtension(cliOptions.ToProcessDir.FullName).Equals(".csv"))
+                    throw new ArgumentException("When in 'list' mode, path to accession directory file of format .csv expected (" + cliOptions.ToProcessDir.FullName + ")");
+            }
 
             if (cliOptions.DirectoryFormat.ToLower().Equals("pacs"))
             {
@@ -43,6 +59,13 @@ public DicomDirectoryProcessorHost(GlobalOptions globals, DicomDirectoryProcesso
                 _ddf = new PacsDirectoryFinder(globals.FileSystemOptions.FileSystemRoot,
                     globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions));
             }
+            else if (cliOptions.DirectoryFormat.ToLower().Equals("list"))
+            {
+                Logger.Info("Creating accession directory lister");
+
+                _ddf = new AccessionDirectoryLister(globals.FileSystemOptions.FileSystemRoot,
+                    globals.FileSystemOptions.DicomSearchPattern, RabbitMqAdapter.SetupProducer(globals.ProcessDirectoryOptions.AccessionDirectoryProducerOptions));
+            }
             else if (cliOptions.DirectoryFormat.ToLower().Equals("default"))
             {
                 Logger.Info("Creating basic directory finder");