Merge pull request #109 from varianter/performanceboost

Øker ytelsen til CV-logikken med å splitte queries

src/Infrastructure/ApiClients/DTOs/CvPartnerDTOs.cs

@@ -659,7 +659,7 @@ private static HashSet<string> CreateCompetenciesFromProject(ProjectExperience d
 
         return dto.project_experience_skills
             .Select(skill => skill.tags.no)
-            .Where(tag => tag is not null)
+            .Where(tag => tag != null)
             .Select(tag => tag!.ToUpper()) // Ignore case because tags are unique
             .ToHashSet();
     }

src/Infrastructure/Entities/ProjectExperienceRoleEntity.cs

@@ -1,10 +1,12 @@
 using System.ComponentModel.DataAnnotations;
 
+using ApplicationCore.Models;
+
 namespace Infrastructure.Entities;
 
 public record ProjectExperienceRoleEntity
 {
-    [Key] public string? Id { get; set; }
+    [Key] public required string Id { get; set; }
     public string Title { get; set; }
 
     public string Description { get; set; }
@@ -14,4 +16,15 @@ public record ProjectExperienceRoleEntity
     public ProjectExperienceEntity ProjectExperience { get; set; } = null!;
 
     public DateTime LastSynced { get; set; }
+}
+
+public static class ProjectExperienceRoleEntityExtension
+{
+    public static ProjectExperienceRole ToProjectExperience(this ProjectExperienceRoleEntity pe)
+    {
+        return new ProjectExperienceRole
+        {
+            Description = pe.Description, Id = pe.Id, Title = pe.Title
+        };
+    }
 }

src/Infrastructure/Entities/WorkExperienceEntity.cs

@@ -4,9 +4,9 @@ namespace Infrastructure.Entities;
 
 public record WorkExperienceEntity
 {
-    [Key] public string? Id { get; set; }
+    [Key] public required string Id { get; set; }
 
-    public Guid EmployeeId { get; set; }
+    public required Guid EmployeeId { get; set; }
     public EmployeeEntity Employee { get; set; } = null!;
     public string? MonthFrom { get; set; }
     public string? YearFrom { get; set; }

src/Infrastructure/Repositories/EmployeeRepository.cs

@@ -5,17 +5,23 @@
 
 using Infrastructure.Entities;
 
+using Microsoft.AspNetCore.Authentication.OAuth.Claims;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Internal;
+using Microsoft.Extensions.Logging;
 
 namespace Infrastructure.Repositories;
 
 public class EmployeesRepository : IEmployeesRepository
 {
     private readonly EmployeeContext _db;
+    private readonly ILogger _logger;
 
-    public EmployeesRepository(EmployeeContext db)
+    public EmployeesRepository(EmployeeContext db, ILogger<OrchestratorService> logger)
     {
         _db = db;
+        _logger = logger;
+
     }
 
     public async Task<List<Employee>> GetAllEmployees()
@@ -40,26 +46,26 @@ public async Task<List<Employee>> GetEmployeesByCountry(string country)
             .SingleOrDefaultAsync();
     }
 
-    private async Task<EmployeeEntity?> GetEmployeeEntityWithCv(string email)
+    private async Task<EmployeeEntity?> GetEmployeeWithCv(string email)
     {
         return await _db.Employees
-            .Include(employee => employee.ProjectExperiences).ThenInclude(entity => entity.ProjectExperienceRoles)
-            // For å unngå et stort join produkt så inkluderes ikke competencies da det er ca 90 competencies per ansatt
-            .Include(employee => employee.WorkExperiences)
-            .Include(employee => employee.Presentations)
-            .Include(employee => employee.Certifications)
+            .Include(employee => employee.ProjectExperiences).ThenInclude(pe => pe.ProjectExperienceRoles).AsSplitQuery()
+            .Include(employee => employee.WorkExperiences).AsSplitQuery()
+            .Include(employee => employee.Presentations).AsSplitQuery()
+            .Include(employee => employee.Certifications).AsSplitQuery()
+            .Include(employee => employee.ProjectExperiences).ThenInclude(pe => pe.Competencies).AsSplitQuery()
             .Where(emp => emp.Email == email)
             .SingleOrDefaultAsync();
     }
 
     private async Task<EmployeeEntity?> GetEmployeeEntityWithCv(string alias, string country)
     {
         return await _db.Employees
-            .Include(employee => employee.ProjectExperiences)
-            .ThenInclude(entity => entity.ProjectExperienceRoles)
-            .Include(employee => employee.WorkExperiences)
-            .Include(employee => employee.Presentations)
-            .Include(employee => employee.Certifications)
+            .Include(employee => employee.ProjectExperiences).ThenInclude(pe => pe.ProjectExperienceRoles).AsSplitQuery()
+            .Include(employee => employee.WorkExperiences).AsSplitQuery()
+            .Include(employee => employee.Presentations).AsSplitQuery()
+            .Include(employee => employee.Certifications).AsSplitQuery()
+            .Include(employee => employee.ProjectExperiences).ThenInclude(pe => pe.Competencies).AsSplitQuery()
             .Where(emp => emp.Email.StartsWith($"{alias}@"))
             .Where(emp => emp.CountryCode == country)
             .SingleOrDefaultAsync();
@@ -173,7 +179,8 @@ public async Task AddOrUpdateCvInformation(List<Cv> cvs)
     {
         foreach (Cv cv in cvs)
         {
-            var entity = await GetEmployeeEntityWithCv(cv.Email);
+            _logger.LogInformation("Starting cv processing of " + cv.Email);
+            var entity = await GetEmployeeWithCv(cv.Email);
             if (entity == null)
             {
                 continue;
@@ -183,6 +190,7 @@ public async Task AddOrUpdateCvInformation(List<Cv> cvs)
             await AddWorkExperience(cv.WorkExperiences, entity);
             await AddProjectExperience(cv.ProjectExperiences, entity);
             await AddCertifications(cv.Certifiactions, entity);
+            await _db.SaveChangesAsync();
         }
     }
 
@@ -248,7 +256,6 @@ private async Task AddPresentations(List<Presentation> presentations, EmployeeEn
                 presentationEntity.Title = presentation.Title;
                 presentationEntity.LastSynced = DateTime.Now;
             }
-            await _db.SaveChangesAsync();
         }
     }
 
@@ -262,7 +269,7 @@ private async Task AddWorkExperience(List<WorkExperience> workExperiences, Emplo
                 workExperienceEntity = new WorkExperienceEntity()
                 {
                     Id = workExperience.Id,
-                    Employee = entity,
+                    EmployeeId = entity.Id,
                     Description = workExperience.Description,
                     MonthFrom = workExperience.MonthFrom,
                     MonthTo = workExperience.MonthTo,
@@ -283,14 +290,14 @@ private async Task AddWorkExperience(List<WorkExperience> workExperiences, Emplo
                 workExperienceEntity.YearTo = workExperience.YearTo;
                 workExperienceEntity.LastSynced = DateTime.Now;
             }
-            await _db.SaveChangesAsync();
         }
 
     }
 
     private async Task AddProjectExperience(List<ProjectExperience> projectExperiences,
         EmployeeEntity entity)
     {
+
         foreach (ProjectExperience projectExperience in projectExperiences)
         {
             var projectExperienceEntity =
@@ -321,17 +328,17 @@ private async Task AddProjectExperience(List<ProjectExperience> projectExperienc
                 projectExperienceEntity.YearTo = projectExperience.YearTo;
                 projectExperienceEntity.LastSynced = DateTime.Now;
             }
-            await _db.SaveChangesAsync();
             await AddProjectExperienceRole(projectExperience.Roles, projectExperienceEntity);
             await AddCompetencies(projectExperience.Competencies, projectExperienceEntity);
         }
     }
 
     private async Task AddCompetencies(HashSet<string> competencies, ProjectExperienceEntity projectExperienceEntity)
     {
+
         foreach (string competency in competencies)
         {
-            var competencyEntity = _db.Competencies.Where(c => c.Name == competency && c.ProjectExperienceId == projectExperienceEntity.Id).SingleOrDefault();
+            var competencyEntity = projectExperienceEntity.Competencies.SingleOrDefault(c => c.Name == competency);
             if (competencyEntity == null)
             {
                 competencyEntity = new CompetencyEntity { Name = competency, LastSynced = DateTime.Now, ProjectExperience = projectExperienceEntity };
@@ -341,7 +348,6 @@ private async Task AddCompetencies(HashSet<string> competencies, ProjectExperien
             {
                 competencyEntity.LastSynced = DateTime.Now;
             }
-            await _db.SaveChangesAsync();
         }
     }
 
@@ -350,8 +356,7 @@ private async Task AddProjectExperienceRole(List<ProjectExperienceRole> projectE
     {
         foreach (ProjectExperienceRole projectExperienceRole in projectExperienceRoles)
         {
-            var projectExperienceRoleEntity =
-                projectExperienceEntity.ProjectExperienceRoles.SingleOrDefault(e => e.Id == projectExperienceRole.Id);
+            var projectExperienceRoleEntity = projectExperienceEntity.ProjectExperienceRoles.SingleOrDefault(per => per.Id == projectExperienceRole.Id);
             if (projectExperienceRoleEntity == null)
             {
                 projectExperienceRoleEntity = new ProjectExperienceRoleEntity
@@ -370,7 +375,6 @@ private async Task AddProjectExperienceRole(List<ProjectExperienceRole> projectE
                 projectExperienceRoleEntity.Title = projectExperienceRole.Title;
                 projectExperienceRoleEntity.LastSynced = DateTime.Now;
             }
-            await _db.SaveChangesAsync();
         }
 
     }
@@ -380,17 +384,10 @@ private void AddCompetenciesToProjects(ProjectExperience pe)
         pe.Competencies.UnionWith(_db.Competencies.Where(c => c.ProjectExperienceId == pe.Id).Select(c => c.Name).Distinct().ToHashSet());
     }
 
-
     public async Task<Cv> GetEmployeeWithCv(string alias, string country)
     {
-        var entity = await GetEmployeeEntityWithCv(alias, country);
-        if (entity == null)
-        {
-            throw new HttpRequestException("not found", null, HttpStatusCode.NotFound);
-        }
-        Cv cv = entity.ToCv();
-        cv.ProjectExperiences.ForEach(AddCompetenciesToProjects);
-        return cv;
+        var entity = await GetEmployeeEntityWithCv(alias, country) ?? throw new HttpRequestException("not found", null, HttpStatusCode.NotFound);
+        return entity.ToCv();
     }
 
     private async Task<EmergencyContact?> SetEmergencyContactAsync(Employee employee)