Skip to content

Commit

Permalink
community support impl and hts fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@koskedk
koskedk committed Jul 9, 2020
1 parent a7880d2 commit 7d59505
Show file tree
Hide file tree
Showing 20 changed files with 132 additions and 108 deletions.
4 changes: 3 additions & 1 deletion src/Dwapi.ExtractsManagement.Core/Extractors/Cbs/MasterPatientIndexSourceExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ public async Task<int> Extract(DbExtract extract, DbProtocol dbProtocol)
// AutoMapper profiles
var extractRecord = Mapper.Map<IDataRecord, TempMasterPatientIndex>(rdr);
extractRecord.Id = LiveGuid.NewGuid();
list.Add(extractRecord);

if(!string.IsNullOrWhiteSpace(extractRecord.sxdmPKValueDoB))
list.Add(extractRecord);

if (count == batch)
{
Expand Down
24 changes: 11 additions & 13 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsClientTestsLoader.cs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AutoMapper;
using Dwapi.ExtractsManagement.Core.Interfaces.Loaders.Hts;
using Dwapi.ExtractsManagement.Core.Interfaces.Repository.Hts;
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Dwh;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -34,24 +37,15 @@ public async Task<int> Load()

try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsClientTestsExtractRepository.GetCleanCount();
var eCount = await _tempHtsClientTestsExtractRepository.GetCleanCount();
var pageCount = _tempHtsClientTestsExtractRepository.PageCount(take, eCount);

int page = 1;
while (page <= pageCount)
{
var tempHtsClientTests =await
_tempHtsClientTestsExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
var tempHtsClientTests = await
_tempHtsClientTestsExtractRepository.GetAll(QueryUtil.Tests, page, take);

var batch = tempHtsClientTests.ToList();
count += batch.Count;
Expand All @@ -61,13 +55,15 @@ public async Task<int> Load()
{
record.Id = LiveGuid.NewGuid();
}

//Batch Insert
var inserted = _htsClientTestsExtractRepository.BatchInsert(extractRecords);
if (!inserted)
{
Log.Error($"Extract {nameof(HtsClientTests)} not Loaded");
return 0;
}

Log.Debug("saved batch");
page++;
/*
Expand All @@ -78,7 +74,9 @@ public async Task<int> Load()
found, count, 0, 0, 0)));
*/
}
DomainEvents.Dispatch(new HtsNotification(new ExtractProgress(nameof(HtsClientTests), "Loading...", Found, 0, 0, 0, 0)));

DomainEvents.Dispatch(new HtsNotification(new ExtractProgress(nameof(HtsClientTests), "Loading...",
Found, 0, 0, 0, 0)));
return count;
}
catch (Exception e)
Expand Down
11 changes: 2 additions & 9 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsClientTracingLoader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -33,14 +34,6 @@ public async Task<int> Load()
int count = 0;
try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsClientTracingExtractRepository.GetCleanCount();
Expand All @@ -50,7 +43,7 @@ public async Task<int> Load()
while (page <= pageCount)
{
var tempHtsClientTracings =await
_tempHtsClientTracingExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
_tempHtsClientTracingExtractRepository.GetAll(QueryUtil.Tracing, page, take);

var batch = tempHtsClientTracings.ToList();
count += batch.Count;
Expand Down
19 changes: 2 additions & 17 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsClientsLinkageLoader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -34,15 +35,6 @@ public async Task<int> Load()

try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsClientsLinkageExtractRepository.GetCleanCount();
var pageCount = _tempHtsClientsLinkageExtractRepository.PageCount(take, eCount);
Expand All @@ -51,7 +43,7 @@ public async Task<int> Load()
while (page <= pageCount)
{
var tempHtsClientLinkages =await
_tempHtsClientsLinkageExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
_tempHtsClientsLinkageExtractRepository.GetAll(QueryUtil.Linkage, page, take);

var batch = tempHtsClientLinkages.ToList();
count += batch.Count;
Expand All @@ -70,13 +62,6 @@ public async Task<int> Load()
}
Log.Debug("saved batch");
page++;
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, count, 0, 0, 0)));
*/
}
DomainEvents.Dispatch(new HtsNotification(new ExtractProgress(nameof(HtsClientLinkage), "Loading...", Found, 0, 0, 0, 0)));
return count;
Expand Down
11 changes: 2 additions & 9 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsPartnerNotificationServicesLoader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -34,14 +35,6 @@ public async Task<int> Load()
int count = 0;
try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsPartnerNotificationServicesExtractRepository.GetCleanCount();
Expand All @@ -51,7 +44,7 @@ public async Task<int> Load()
while (page <= pageCount)
{
var tempHtsPartnerNotifications =await
_tempHtsPartnerNotificationServicesExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
_tempHtsPartnerNotificationServicesExtractRepository.GetAll(QueryUtil.Pns, page, take);

var batch = tempHtsPartnerNotifications.ToList();
count += batch.Count;
Expand Down
11 changes: 2 additions & 9 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsPartnerTracingLoader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -33,14 +34,6 @@ public async Task<int> Load()
int count = 0;
try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsPartnerTracingExtractRepository.GetCleanCount();
Expand All @@ -50,7 +43,7 @@ public async Task<int> Load()
while (page <= pageCount)
{
var tempHtsPartnerTracings =await
_tempHtsPartnerTracingExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
_tempHtsPartnerTracingExtractRepository.GetAll(QueryUtil.PartnerTracing, page, take);

var batch = tempHtsPartnerTracings.ToList();
count += batch.Count;
Expand Down
12 changes: 2 additions & 10 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/HtsTestKitsTracingLoader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Dwapi.ExtractsManagement.Core.Model.Destination.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Core.Notifications;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts;
using Dwapi.SharedKernel.Events;
using Dwapi.SharedKernel.Model;
using Dwapi.SharedKernel.Utility;
Expand All @@ -33,15 +34,6 @@ public async Task<int> Load()
int count = 0;
try
{
/*
DomainEvents.Dispatch(
new ExtractActivityNotification(extractId, new DwhProgress(
nameof(PatientExtract),
nameof(ExtractStatus.Loading),
found, 0, 0, 0, 0)));
*/

const int take = 1000;
var eCount = await _tempHtsTestKitsExtractRepository.GetCleanCount();
var pageCount = _tempHtsTestKitsExtractRepository.PageCount(take, eCount);
Expand All @@ -50,7 +42,7 @@ public async Task<int> Load()
while (page <= pageCount)
{
var tempHtsTestKits =await
_tempHtsTestKitsExtractRepository.GetAll(a => a.ErrorType == 0, page, take);
_tempHtsTestKitsExtractRepository.GetAll(QueryUtil.Kits, page, take);

var batch = tempHtsTestKits.ToList();
count += batch.Count;
Expand Down
33 changes: 33 additions & 0 deletions src/Dwapi.ExtractsManagement.Core/Loader/Hts/QueryUtil.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
namespace Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.TempExtracts
{
public class QueryUtil
{
public static string Linkage => QueryBuilder("TempHtsClientsLinkageExtracts");
public static string Tests => QueryBuilder("TempHtsClientTestsExtracts");
public static string Tracing => QueryBuilder("TempHtsClientTracingExtracts");
public static string Pns => QueryBuilder("TempHtsPartnerNotificationServicesExtracts");
public static string PartnerTracing => QueryBuilder("TempHtsPartnerTracingExtracts");
public static string Kits => QueryBuilder("TempHtsTestKitsExtracts");

public static string LinkageCount => QueryBuilder("TempHtsClientsLinkageExtracts", true);
public static string TestsCount => QueryBuilder("TempHtsClientTestsExtracts", true);
public static string TracingCount => QueryBuilder("TempHtsClientTracingExtracts", true);
public static string PnsCount => QueryBuilder("TempHtsPartnerNotificationServicesExtracts", true);
public static string PartnerTracingCount => QueryBuilder("TempHtsPartnerTracingExtracts", true);
public static string KitsCount => QueryBuilder("TempHtsTestKitsExtracts", true);

public static string QueryBuilder(string extract, bool count = false)
{
var select = count ? "SELECT Count(s.Id) " : "SELECT s.* ";

var from = $@"
FROM {extract} s
INNER JOIN HtsClientsExtracts p ON s.PatientPK = p.PatientPK AND s.SiteCode = p.SiteCode
WHERE ErrorType = 0";

var query = $"{select} {from}";

return query;
}
}
}
9 changes: 7 additions & 2 deletions src/Dwapi.ExtractsManagement.Core/Services/EmrMetricsService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public EmrMetricsService(IEmrMetricRepository emrMetricRepository)
public async Task<EmrMetricSource> Read(AuthProtocol authProtocol, string url)
{
EmrMetricSource metricSoruce = null;
string metricsUrl=string.Empty;

if (authProtocol.HasAuth)
{
Expand All @@ -48,7 +49,8 @@ public async Task<EmrMetricSource> Read(AuthProtocol authProtocol, string url)

try
{
var response = await _httpClient.GetAsync(GetUrl(authProtocol.Url, url));
metricsUrl = GetUrl(authProtocol.Url, url);
var response = await _httpClient.GetAsync(metricsUrl);
if (response.IsSuccessStatusCode)
{
metricSoruce = await response.Content.ReadAsJsonAsync<EmrMetricSource>();
Expand All @@ -60,7 +62,10 @@ public async Task<EmrMetricSource> Read(AuthProtocol authProtocol, string url)
}
catch (Exception e)
{
Log.Error(e, "Error Reading metrics");
Log.Error(new string('-',40));
Log.Error($"Error Reading metrics on [{metricsUrl}]");
Log.Error(e.Message,"Metrics Error");
Log.Error(new string('-',40));
}

if (null != metricSoruce)
Expand Down
12 changes: 6 additions & 6 deletions ...i.ExtractsManagement.Infrastructure/Migrations/20180621082810_MySqlFunctionPermissions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ protected override void Up(MigrationBuilder migrationBuilder)
if (migrationBuilder.ActiveProvider.ToLower().Contains("MySql".ToLower()))
{
migrationBuilder.Sql(@"
GRANT EXECUTE ON FUNCTION fn_calculateJaro TO '*'@'%';
GRANT EXECUTE ON FUNCTION fn_calculateJaroWinkler TO '*'@'%';
GRANT EXECUTE ON FUNCTION fn_calculateMatchWindow TO '*'@'%';
GRANT EXECUTE ON FUNCTION fn_calculatePrefixLength TO '*'@'%';
GRANT EXECUTE ON FUNCTION fn_GetCommonCharacters TO '*'@'%';");
GRANT EXECUTE ON FUNCTION fn_calculateJaro TO 'dwapi'@'%';
GRANT EXECUTE ON FUNCTION fn_calculateJaroWinkler TO 'dwapi'@'%';
GRANT EXECUTE ON FUNCTION fn_calculateMatchWindow TO 'dwapi'@'%';
GRANT EXECUTE ON FUNCTION fn_calculatePrefixLength TO 'dwapi'@'%';
GRANT EXECUTE ON FUNCTION fn_GetCommonCharacters TO 'dwapi'@'%';");
}
}

protected override void Down(MigrationBuilder migrationBuilder)
{

}
}
}
2 changes: 1 addition & 1 deletion src/Dwapi.ExtractsManagement.Infrastructure/Migrations/20180621092045_UpdateFunction.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ IF SUBSTRING( str1, i+1, 1) <> SUBSTRING( str2, i+1, 1) THEN
END;");

migrationBuilder.Sql(@"GRANT EXECUTE ON FUNCTION fn_calculateTranspositions TO '*'@'%';
migrationBuilder.Sql(@"GRANT EXECUTE ON FUNCTION fn_calculateTranspositions TO 'dwapi'@'%';
FLUSH PRIVILEGES;");
}

Expand Down
13 changes: 9 additions & 4 deletions ...agement.Infrastructure/Repository/Hts/TempExtracts/TempHtsClientTestsExtractRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
using System.Data.SqlClient;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Dwapi.ExtractsManagement.Core.Interfaces.Repository.Hts;
using Dwapi.ExtractsManagement.Core.Model.Source.Hts.NewHts;
using Dwapi.ExtractsManagement.Infrastructure.Repository.Hts.Extracts;
using Dwapi.SharedKernel.Infrastructure.Repository;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
Expand Down Expand Up @@ -68,10 +70,13 @@ public bool BatchInsert(IEnumerable<TempHtsClientTests> extracts)

public Task<int> GetCleanCount()
{
return DbSet.AsNoTracking()
.Where(a => a.ErrorType == 0)
.Select(x => x.Id)
.CountAsync();
int count = 0;
using (var cn=GetNewConnection())
{
var query = QueryUtil.TestsCount;
count = cn.ExecuteScalar<int>(query);
}
return Task.FromResult(count);
}
}
}
Loading

0 comments on commit 7d59505

Please sign in to comment.