Merge branch 'release/1.2.2'

SMI · Jan 8, 2020 · 30cb2f3 · 30cb2f3
2 parents a975bff + 765f619
commit 30cb2f3
Show file tree

Hide file tree

Showing 11 changed files with 52 additions and 61 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+...
+
+## [1.2.2] - 2020-01-08
+
+### Fixed
+
+- RAW to STAGING migration now lists columns explicitly (previously used `SELECT *` which could cause problems if RAW and STAGING column orders somehow differed)
+
 ## [1.2.1] - 2020-01-06
 
 ### Added
@@ -92,7 +100,8 @@ First stable release after importing the repository from the private [SMIPlugin]
 - Anonymous `MappingTableName` must now be fully specified to pass validation (e.g. `mydb.mytbl`).  Previously skipping database portion was supported.
 
 
-[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.2.1...develop
+[Unreleased]: https://github.com/SMI/SmiServices/compare/v1.2.2...develop
+[1.2.2]:  https://github.com/SMI/SmiServices/compare/v1.2.1...v1.2.2
 [1.2.1]:  https://github.com/SMI/SmiServices/compare/1.2.0...v1.2.1
 [1.2.0]:  https://github.com/SMI/SmiServices/compare/1.1.0-rc1...1.2.0
 [1.2.0-rc1]:  https://github.com/SMI/SmiServices/compare/1.1.0...1.2.0-rc1

diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 ![GitHub](https://img.shields.io/github/license/SMI/SmiServices)
 [![Total alerts](https://img.shields.io/lgtm/alerts/g/SMI/SmiServices.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/SMI/SmiServices/alerts/)
 
-Version: `1.2.1`
+Version: `1.2.2`
 
 # SMI Services
 

diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs
@@ -7,6 +7,6 @@
 [assembly: AssemblyCulture("")]
 
 // These should be overwritten by release builds
-[assembly: AssemblyVersion("1.2.1")]
-[assembly: AssemblyFileVersion("1.2.1")]
-[assembly: AssemblyInformationalVersion("1.2.1")] // This one can have the extra build info after it
+[assembly: AssemblyVersion("1.2.2")]
+[assembly: AssemblyFileVersion("1.2.2")]
+[assembly: AssemblyInformationalVersion("1.2.2")] // This one can have the extra build info after it
diff --git a/src/common/Smi.Common/Execution/MicroserviceHost.cs b/src/common/Smi.Common/Execution/MicroserviceHost.cs
@@ -48,10 +48,12 @@ protected MicroserviceHost(GlobalOptions globals, bool loadSmiLogConfig = true)
         {
             HostProcessName = Assembly.GetEntryAssembly()?.GetName().Name ?? throw new ApplicationException("Couldn't get the Assembly name!");
 
+            string logConfigPath = null;
+
             // We may not want to do this during tests, however this should always be true otherwise
             if (loadSmiLogConfig)
             {
-                string logConfigPath = !string.IsNullOrWhiteSpace(globals.FileSystemOptions.LogConfigFile)
+                logConfigPath = !string.IsNullOrWhiteSpace(globals.FileSystemOptions.LogConfigFile)
                     ? globals.FileSystemOptions.LogConfigFile
                     : Path.Combine(globals.CurrentDirectory, "Smi.NLog.config");
 
@@ -75,6 +77,9 @@ protected MicroserviceHost(GlobalOptions globals, bool loadSmiLogConfig = true)
             Logger = LogManager.GetLogger(GetType().Name);
             Logger.Info("Host logger created with " + (loadSmiLogConfig ? "SMI" : "existing") + " logging config");
 
+            if (!string.IsNullOrWhiteSpace(logConfigPath))
+                Logger.Debug($"Logging config loaded from {logConfigPath}");
+
             if (!globals.MicroserviceOptions.TraceLogging)
                 LogManager.GlobalThreshold = LogLevel.Debug;
 
@@ -160,6 +165,8 @@ public virtual void Stop(string reason)
 
             _stopCalled = true;
 
+            Logger.Debug("Shutting down RabbitMQ connections");
+
             // Attempt to destroy the control queue
 
             try

diff --git a/src/common/Smi.Common/Messaging/ControlMessageConsumer.cs b/src/common/Smi.Common/Messaging/ControlMessageConsumer.cs
@@ -64,8 +64,6 @@ public ControlMessageConsumer(MicroserviceHost host, RabbitOptions options, stri
         /// <param name="e"></param>
         public override void ProcessMessage(BasicDeliverEventArgs e)
         {
-            Logger.Info("Control message received");
-
             try
             {
                 ProcessMessageImpl(null, e);
@@ -119,7 +117,7 @@ protected override void ProcessMessageImpl(IMessageHeader header, BasicDeliverEv
                 Logger.Debug("Control command did not match this service");
                 return;
             }
-            
+
             // Handle any general actions - just stop and ping for now
 
             if (action.StartsWith("stop"))

diff --git a/src/common/Smi.Common/RabbitMQAdapter.cs b/src/common/Smi.Common/RabbitMQAdapter.cs
@@ -175,6 +175,7 @@ public Guid StartConsumer(ConsumerOptions consumerOptions, IConsumer consumer, b
             };
 
             consumerTask.Start();
+            _logger.Debug($"Consumer task started [ID={consumerTask.Id}]");
 
             return taskId;
         }
@@ -426,6 +427,13 @@ private class RabbitResources : IDisposable
 
             protected readonly object OResourceLock = new object();
 
+            protected readonly ILogger Logger;
+
+            public RabbitResources()
+            {
+                Logger = LogManager.GetLogger(GetType().Name);
+            }
+
 
             public void Dispose()
             {
@@ -456,19 +464,22 @@ private class ConsumerResources : RabbitResources
 
             public bool Shutdown(int timeout = 5000)
             {
+                bool exitOk;
                 lock (OResourceLock)
                 {
                     TokenSource.Cancel();
 
                     // Consumer task can't directly shut itself down, as it will block here
-                    bool exitOk = ConsumerTask.Wait(timeout);
+                    exitOk = ConsumerTask.Wait(timeout);
 
                     Subscription.Close();
 
                     Dispose();
-
-                    return exitOk;
                 }
+
+                Logger.Debug($"Consumer task shutdown [ID={ConsumerTask.Id}]");
+
+                return exitOk;
             }
         }
 

diff --git a/src/common/Smi.Common/app.config b/src/common/Smi.Common/app.config
@@ -33,17 +33,4 @@
       </dependentAssembly>
     </assemblyBinding>
   </runtime>
-  <system.data>
-    <DbProviderFactories>
-      <remove invariant="Oracle.ManagedDataAccess.Client" />
-      <add name="ODP.NET, Managed Driver" invariant="Oracle.ManagedDataAccess.Client" description="Oracle Data Provider for .NET, Managed Driver" type="Oracle.ManagedDataAccess.Client.OracleClientFactory, Oracle.ManagedDataAccess, Version=4.122.18.3, Culture=neutral, PublicKeyToken=89b483f429c47342" />
-    </DbProviderFactories>
-  </system.data>
-  <oracle.manageddataaccess.client>
-    <version number="*">
-      <dataSources>
-        <dataSource alias="SampleDataSource" descriptor="(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=localhost)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=ORCL))) " />
-      </dataSources>
-    </version>
-  </oracle.manageddataaccess.client>
 </configuration>
diff --git a/src/microservices/Microservices.CohortExtractor/App.config b/src/microservices/Microservices.CohortExtractor/App.config
@@ -41,20 +41,4 @@
             </dependentAssembly>
         </assemblyBinding>
     </runtime>
-    <system.data>
-        <DbProviderFactories>
-            <remove invariant="Oracle.ManagedDataAccess.Client" />
-            <add name="ODP.NET, Managed Driver" invariant="Oracle.ManagedDataAccess.Client" description="Oracle Data Provider for .NET, Managed Driver" type="Oracle.ManagedDataAccess.Client.OracleClientFactory, Oracle.ManagedDataAccess, Version=4.122.18.3, Culture=neutral, PublicKeyToken=89b483f429c47342" />
-        </DbProviderFactories>
-    </system.data>
-    <oracle.manageddataaccess.client>
-        <version number="*">
-            <dataSources>
-                <dataSource alias="SampleDataSource" descriptor="(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=localhost)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=ORCL))) " />
-            </dataSources>
-        </version>
-    </oracle.manageddataaccess.client>
-
-
-
 </configuration>
diff --git a/src/microservices/Microservices.DicomRelationalMapper/App.config b/src/microservices/Microservices.DicomRelationalMapper/App.config
@@ -41,20 +41,4 @@
             </dependentAssembly>
         </assemblyBinding>
     </runtime>
-    <system.data>
-        <DbProviderFactories>
-            <remove invariant="Oracle.ManagedDataAccess.Client" />
-            <add name="ODP.NET, Managed Driver" invariant="Oracle.ManagedDataAccess.Client" description="Oracle Data Provider for .NET, Managed Driver" type="Oracle.ManagedDataAccess.Client.OracleClientFactory, Oracle.ManagedDataAccess, Version=4.122.18.3, Culture=neutral, PublicKeyToken=89b483f429c47342" />
-        </DbProviderFactories>
-    </system.data>
-    <oracle.manageddataaccess.client>
-        <version number="*">
-            <dataSources>
-                <dataSource alias="SampleDataSource" descriptor="(DESCRIPTION=(ADDRESS=(PROTOCOL=tcp)(HOST=localhost)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=ORCL))) " />
-            </dataSources>
-        </version>
-    </oracle.manageddataaccess.client>
-
-
-
 </configuration>
diff --git a/...roservices.DicomRelationalMapper/Execution/MigrateRawToStagingWithSelectIntoStatements.cs b/...roservices.DicomRelationalMapper/Execution/MigrateRawToStagingWithSelectIntoStatements.cs
@@ -12,6 +12,7 @@
 using System.Collections.Generic;
 using System.Data.Common;
 using System.Diagnostics;
+using System.Linq;
 using System.Threading.Tasks;
 using ReusableLibraryCode;
 
@@ -53,9 +54,16 @@ public override ExitCodeType Run(IDataLoadJob job, GracefulCancellationToken can
 
                     IQuerySyntaxHelper syntaxHelper = table.GetQuerySyntaxHelper();
 
-                    string sql = string.Format(@"INSERT INTO {1} SELECT DISTINCT * FROM {0}",
+                    var fromCols = server.ExpectDatabase(fromDb).ExpectTable(fromTable).DiscoverColumns();
+                    var toCols = server.ExpectDatabase(toDb).ExpectTable(toTable).DiscoverColumns();
+
+                    //Migrate only columns that appear in both tables
+                    var commonColumns = fromCols.Select(f => f.GetRuntimeName()).Intersect(toCols.Select(t => t.GetRuntimeName())).ToArray();
+
+                    string sql = string.Format(@"INSERT INTO {1}({2}) SELECT DISTINCT {2} FROM {0}",
                         syntaxHelper.EnsureFullyQualified(fromDb, null, fromTable),
-                        syntaxHelper.EnsureFullyQualified(toDb, null, toTable));
+                        syntaxHelper.EnsureFullyQualified(toDb, null, toTable),
+                        string.Join(",",commonColumns.Select(c=>syntaxHelper.EnsureWrapped(c))));
 
                     job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, "About to send SQL:" + sql));
 

diff --git a/src/microservices/Microservices.DicomReprocessor/Execution/MongoDbReader.cs b/src/microservices/Microservices.DicomReprocessor/Execution/MongoDbReader.cs
@@ -40,6 +40,7 @@ public bool WasCancelled
         private readonly CancellationTokenSource _tokenSource = new CancellationTokenSource();
         private bool _stopping;
 
+        private readonly bool _autoRun;
 
         public MongoDbReader(MongoDbOptions mongoOptions, DicomReprocessorCliOptions reprocessorOptions, string appId)
         {
@@ -57,10 +58,12 @@ public MongoDbReader(MongoDbOptions mongoOptions, DicomReprocessorCliOptions rep
             // https://docs.mongodb.com/manual/reference/method/cursor.batchSize/
             if (reprocessorOptions.MongoDbBatchSize > 1)
                 _findOptionsBase.BatchSize = reprocessorOptions.MongoDbBatchSize;
+
+            _autoRun = reprocessorOptions.AutoRun;
         }
 
 
-        public async Task<TimeSpan> RunQuery(string query, IDocumentProcessor processor, DicomReprocessorOptions options, bool autoRun = false)
+        public async Task<TimeSpan> RunQuery(string query, IDocumentProcessor processor, DicomReprocessorOptions options)
         {
             DateTime start;
 
@@ -72,7 +75,7 @@ public async Task<TimeSpan> RunQuery(string query, IDocumentProcessor processor,
                 _logger.Info($"Batch size is: {(_findOptionsBase.BatchSize.HasValue ? _findOptionsBase.BatchSize.ToString() : "unspecified")}");
                 _logger.Info($"Sleeping for {options.SleepTime.TotalMilliseconds}ms between batches");
 
-                if (!autoRun)
+                if (!_autoRun)
                 {
                     LogManager.Flush();