From af230705c8f2c802998322861cb501db49a25461 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 17 Feb 2020 08:12:30 +0000 Subject: [PATCH 1/5] Fixed state being wrong on `DicomFileMessageToDatasetListWorklist` after failed DLE runs Fixes "The Mounting stage has not populated the RAW database (data_load2_RAW) with any data" (when retrying batches) --- CHANGELOG.md | 4 ++++ .../Execution/DicomFileMessageToDatasetListProvider.cs | 9 +++++++++ .../Messaging/DicomRelationalMapperQueueConsumer.cs | 3 +++ 3 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9652cf2e..5e62f070c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ... +### Fixed + +- Fixed DLE Payload state being wrong when retrying batches (when it is half / completely consumed) + ## [1.4.0] - 2020-02-14 ### Added diff --git a/src/microservices/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs b/src/microservices/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs index 8631d4c2c..3f38833b0 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs +++ b/src/microservices/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs @@ -18,6 +18,15 @@ public DicomFileMessageToDatasetListWorklist(List messages) _messages = messages; } + /// + /// Resets the progress through the work list e.g. if half the list is consumed and you want to + /// start again. + /// + public void ResetProgress() + { + _progress = 0; + } + public DicomDataset GetNextDatasetToProcess(out string filename, out Dictionary otherValuesToStoreInRow) { otherValuesToStoreInRow = new Dictionary(); diff --git a/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs b/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs index 9427c2838..3fe464b27 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs +++ b/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs @@ -220,6 +220,9 @@ private void RunDleIfRequired() // We last ran now! _lastRanDle = DateTime.Now; + + //reset the progress e.g. if we crashed later on in the load + datasetProvider.ResetProgress(); try { From b8b1296562cc721647cb9cdc06371918ba7add50 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 17 Feb 2020 09:49:27 +0000 Subject: [PATCH 2/5] Added random wait duration for DicomRelationalMapper --- CHANGELOG.md | 4 ++++ .../Messaging/DicomRelationalMapperQueueConsumer.cs | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e62f070c..d089af68a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ... +### Added + +- Added randomisation in the retry delay on DicomRelationalMapper (and set minimum wait duration to 10s) + ### Fixed - Fixed DLE Payload state being wrong when retrying batches (when it is half / completely consumed) diff --git a/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs b/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs index 3fe464b27..3c80ef728 100644 --- a/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs +++ b/src/microservices/Microservices.DicomRelationalMapper/Messaging/DicomRelationalMapperQueueConsumer.cs @@ -78,7 +78,7 @@ public DicomRelationalMapperQueueConsumer(IRDMPPlatformRepositoryServiceLocator _minimumBatchSize = options.MinimumBatchSize; _useInsertIntoForRawMigration = options.UseInsertIntoForRAWMigration; _retryOnFailureCount = options.RetryOnFailureCount; - _retryDelayInSeconds = options.RetryDelayInSeconds; + _retryDelayInSeconds = Math.Max(10,options.RetryDelayInSeconds); _maximumRunDelayInSeconds = new TimeSpan(0, 0, 0, options.MaximumRunDelayInSeconds <= 0 ? 15 : 0); StartDleRunnerTask(); @@ -236,8 +236,13 @@ private void RunDleIfRequired() if (remainingRetries > 0) { - Logger.Info("Sleeping " + _retryDelayInSeconds + "s after failure"); - Task.Delay(new TimeSpan(0, 0, 0, _retryDelayInSeconds)).Wait(); + //wait a random length of time averaging the _retryDelayInSeconds to avoid retrying at the same time as other processes + //where there is resource contention that results in simultaneous failures. + var r = new Random(); + var wait = r.Next(_retryDelayInSeconds * 2); + + Logger.Info("Sleeping " + wait + "s after failure"); + Task.Delay(new TimeSpan(0, 0, 0, wait)).Wait(); if (RunChecks) { From 773ee628bec4ab62bf92bea3de5a2abaf7aa80de Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 17 Feb 2020 09:52:28 +0000 Subject: [PATCH 3/5] Added lock on producer send message --- CHANGELOG.md | 1 + .../Messaging/IdentifierMapperQueueConsumer.cs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d089af68a..c66d9155d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Fixed DLE Payload state being wrong when retrying batches (when it is half / completely consumed) +- Added lock on producer sending messages in IdentifierMapper ## [1.4.0] - 2020-02-14 diff --git a/src/microservices/Microservices.IdentifierMapper/Messaging/IdentifierMapperQueueConsumer.cs b/src/microservices/Microservices.IdentifierMapper/Messaging/IdentifierMapperQueueConsumer.cs index 0d122641e..e8730de17 100644 --- a/src/microservices/Microservices.IdentifierMapper/Messaging/IdentifierMapperQueueConsumer.cs +++ b/src/microservices/Microservices.IdentifierMapper/Messaging/IdentifierMapperQueueConsumer.cs @@ -70,8 +70,10 @@ protected override void ProcessMessageImpl(IMessageHeader header, BasicDeliverEv else { // Now ship it to the exchange - _producer.SendMessage(msg, header); - + lock (_producer) + { + _producer.SendMessage(msg, header); + } Ack(header, deliverArgs); } } From df1ebc9ed96b8e293457680c03304d321b2e9431 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 17 Feb 2020 09:58:55 +0000 Subject: [PATCH 4/5] Version bumped to 1.4.1 --- CHANGELOG.md | 3 ++- README.md | 2 +- src/SharedAssemblyInfo.cs | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c66d9155d..84bd0058e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,7 +156,8 @@ First stable release after importing the repository from the private [SMIPlugin] - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`). Previously skipping database portion was supported. -[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.0...develop +[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.1...develop +[1.4.0]: https://github.com/SMI/SmiServices/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/SMI/SmiServices/compare/v1.3.1...v1.4.0 [1.3.1]: https://github.com/SMI/SmiServices/compare/v1.3.0...v1.3.1 [1.3.0]: https://github.com/SMI/SmiServices/compare/v1.2.3...v1.3.0 diff --git a/README.md b/README.md index ecdd0c917..4c3a7983c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![GitHub](https://img.shields.io/github/license/SMI/SmiServices) [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/) -Version: `1.4.0` +Version: `1.4.1` # SMI Services diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 5875e5b02..9b25899ff 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -7,6 +7,6 @@ [assembly: AssemblyCulture("")] // These should be overwritten by release builds -[assembly: AssemblyVersion("1.4.0")] -[assembly: AssemblyFileVersion("1.4.0")] -[assembly: AssemblyInformationalVersion("1.4.0")] // This one can have the extra build info after it +[assembly: AssemblyVersion("1.4.1")] +[assembly: AssemblyFileVersion("1.4.1")] +[assembly: AssemblyInformationalVersion("1.4.1")] // This one can have the extra build info after it From d05218dd11e02839baed1eac9c331e7887c9486a Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 17 Feb 2020 10:21:44 +0000 Subject: [PATCH 5/5] Fixed version number in changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84bd0058e..3eb916f5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ... +## [1.4.1] - 2020-02-17 + ### Added - Added randomisation in the retry delay on DicomRelationalMapper (and set minimum wait duration to 10s) @@ -157,7 +159,7 @@ First stable release after importing the repository from the private [SMIPlugin] [Unreleased]: https://github.com/SMI/SmiServices/compare/v1.4.1...develop -[1.4.0]: https://github.com/SMI/SmiServices/compare/v1.4.0...v1.4.1 +[1.4.1]: https://github.com/SMI/SmiServices/compare/v1.4.0...v1.4.1 [1.4.0]: https://github.com/SMI/SmiServices/compare/v1.3.1...v1.4.0 [1.3.1]: https://github.com/SMI/SmiServices/compare/v1.3.0...v1.3.1 [1.3.0]: https://github.com/SMI/SmiServices/compare/v1.2.3...v1.3.0