Sync excel with db (#211)

* Fix crawler ids

* Sync excel with db

tools/DanaCrawler/DanaCrawler/AjudaDanaService.cs

@@ -1,14 +1,21 @@
-using Supabase;
+using DanaCrawler.Config;
+using Microsoft.Extensions.Options;
+using Supabase.Postgrest;
+using Client = Supabase.Client;
 
 namespace DanaCrawler;
 
 internal sealed class AjudaDanaService
 {
     private readonly Client _supabaseClient;
+    private readonly IOptions<SupabaseConfig> _config;
+    private readonly ILogger<AjudaDanaService> _logger;
 
-    public AjudaDanaService(Supabase.Client supabaseClient)
+    public AjudaDanaService(Supabase.Client supabaseClient, IOptions<SupabaseConfig> config, ILogger<AjudaDanaService> logger)
     {
         _supabaseClient = supabaseClient ?? throw new ArgumentNullException(nameof(supabaseClient));
+        _config = config ?? throw new ArgumentNullException(nameof(config));
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
     }
 
     public async Task<List<HelpRequest>> GetHelpRequestsWithTownsPaginated(CancellationToken stoppingToken)
@@ -41,4 +48,27 @@ public async Task<List<HelpRequest>> GetHelpRequestsWithTownsPaginated(Cancellat
 
         return allRecords;
     }
+
+    public async Task SyncCrmStatusWithDb(List<HelpRequest> helpRequests)
+    {
+        await _supabaseClient.Auth.SignIn(_config.Value.ServiceAccountEmail, _config.Value.ServiceAccountPassword);
+
+        foreach (var helpRequest in helpRequests)
+        {
+            var statusMapping = helpRequest.CrmStatus switch
+            {
+                "active" => "active",
+                "finished" => "finished",
+                _ => "progress"
+            };
+
+            var update = await _supabaseClient.From<HelpRequest>()
+                .Filter("id", Constants.Operator.Equals, helpRequest.DbId)
+                .Set(x => new KeyValuePair<object, object?>(x.Status, statusMapping)) // For web
+                .Set(x => new KeyValuePair<object, object?>(x.CrmStatus, helpRequest.CrmStatus))
+                .Set(x => new KeyValuePair<object, object?>(x.Notes, helpRequest.Notes))
+                .Set(x => new KeyValuePair<object, object?>(x.Avisos, helpRequest.Avisos))
+                .Update();
+        }
+    }
 }

tools/DanaCrawler/DanaCrawler/Config/SupabaseConfig.cs

@@ -7,4 +7,8 @@ internal sealed class SupabaseConfig
     public required string Url { get; init; }
 
     public required string ApiKey { get; init; }
+
+    public required string ServiceAccountEmail { get; init; }
+
+    public required string ServiceAccountPassword { get; init; }
 }

tools/DanaCrawler/DanaCrawler/GoogleSheetService.cs

@@ -105,28 +105,26 @@ private async Task InsertNeedsRequestsAsync(List<HelpRequest> requests)
 
         var headerRow = new List<object>
         {
-            "Created At", "ID", "Status", "Town","Description", "Help Types",  "Number of People",
-            "Name", "Location", "ContactInfo", "People Needed",
+            "Created At", "ID", "Avisos", "Status", "Town","Description", "Help Types",  "Number of People",
+            "Name", "Location", "ContactInfo", "People Needed", "Notes"
         };
 
         var rows = new List<IList<object>> { headerRow };
         rows.AddRange(requests.OrderBy(x => x.Id).Where(x => x.Type == "necesita").Select(request => new List<object>
         {
             request.CreatedAt.ToString("yyyy-MM-dd HH:mm:ss"),
-            request.Id,
-            //request.Type ?? "",
-            request.Status ?? "",
+            $"=HYPERLINK(\"https://ajudadana.es/solicitudes/{request.Id}\"; \"{request.Id}\")",
+            request.Avisos ?? "",
+            request.CrmStatus ?? "",
             request.Town?.Name ?? "",
             request.Description ?? "",
             string.Join(", ", request.HelpType ?? []),
             request.NumberOfPeople ?? 0,
             request.Name ?? "",
             request.Location ?? "",
             request.ContactInfo ?? "",
-            //request.Urgency ?? "",
-            //request.Latitude ?? "",
-            //request.Longitud ?? "",
             request.PeopleNeeded,
+            request.Notes ?? "",
         }));
 
         var valueRange = new ValueRange
@@ -137,10 +135,65 @@ private async Task InsertNeedsRequestsAsync(List<HelpRequest> requests)
         var updateRequest = _sheetsService.Spreadsheets.Values.Update(
             valueRange,
             _spreadsheetId,
-            $"{sheetName}!A1:K"
+            $"{sheetName}!A1:M"
         );
         updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
 
         await updateRequest.ExecuteAsync();
     }
+
+    public async Task<List<HelpRequest>> GetSheetHelpRequests()
+    {
+        var getRequest = _sheetsService.Spreadsheets.Values.Get(_spreadsheetId, "NecesitaCRMV2 Hoja de Trabajo - 12/11!A1:M");
+
+        var rows = await getRequest.ExecuteAsync();
+
+        var helpRequests = new List<HelpRequest>();
+        for (var i = 0; i < rows.Values.Count; i++)
+        {
+            if (i == 0) continue;
+
+            var rowsValue = rows.Values[i];
+            try
+            {
+                helpRequests.Add(new HelpRequest
+                {
+                    CreatedAt = string.IsNullOrEmpty(rowsValue[0]?.ToString())
+                        ? DateTime.MinValue
+                        : DateTime.Parse(rowsValue[0].ToString()),
+                    DbId = string.IsNullOrEmpty(rowsValue[1]?.ToString())
+                        ? 0
+                        : int.Parse(rowsValue[1].ToString().Split(";")[0]),
+                    Avisos = rowsValue[2]?.ToString() ?? "",
+                    CrmStatus = rowsValue[3]?.ToString() ?? "",
+                    Town = new Town
+                    {
+                        Id = 0,
+                        Name = rowsValue[4]?.ToString() ?? ""
+                    },
+                    Description = rowsValue[5]?.ToString() ?? "",
+                    HelpType = string.IsNullOrEmpty(rowsValue[6]?.ToString())
+                        ? new List<string>()
+                        : rowsValue[6].ToString().Split(",").ToList(),
+                    NumberOfPeople = string.IsNullOrEmpty(rowsValue[7]?.ToString())
+                        ? 0
+                        : int.Parse(rowsValue[7].ToString()),
+                    Name = rowsValue[8]?.ToString() ?? "",
+                    Location = rowsValue[9]?.ToString() ?? "",
+                    ContactInfo = rowsValue[10]?.ToString() ?? "",
+                    PeopleNeeded = string.IsNullOrEmpty(rowsValue[11]?.ToString())
+                        ? 0
+                        : int.Parse(rowsValue[11].ToString()),
+                    Notes = rowsValue.Count >= 13 ? rowsValue[12]?.ToString() ?? "" : "",
+                });
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(e);
+                throw;
+            }
+        }
+
+        return helpRequests;
+    }
 }

tools/DanaCrawler/DanaCrawler/HelpRequest.cs

@@ -46,6 +46,9 @@ internal sealed class HelpRequest : BaseModel
     [Column("status")]
     public string Status { get; init; }
 
+    [Column("crm_status")]
+    public string CrmStatus { get; set; }
+
     [Column("latitude")]
     public string Latitude { get; init; }
 
@@ -73,6 +76,12 @@ internal sealed class HelpRequest : BaseModel
 
     [JsonProperty("towns")]
     public Town Town { get; set; }
+
+    [Column("notes")]
+    public string Notes { get; set; }
+
+    [Column("avisos")]
+    public string Avisos { get; set; }
 }
 
 internal sealed class Resources

tools/DanaCrawler/DanaCrawler/Program.cs

@@ -10,6 +10,8 @@
 
 builder.Services.Configure<SupabaseConfig>(builder.Configuration.GetSection(SupabaseConfig.Supabase));
 builder.Services.Configure<GoogleApiConfig>(builder.Configuration.GetSection(GoogleApiConfig.GoogleApi));
+
+builder.Services.AddSingleton(new GoogleSheetsService(builder.Configuration["GoogleApi:CredentialsPath"]!, builder.Configuration["GoogleApi:SpreadsheetId"]!));
 builder.Services.AddSingleton<AjudaDanaService>();
 
 builder.Services.AddSerilog((services, logConfiguration) =>

tools/DanaCrawler/DanaCrawler/Worker.cs

@@ -1,34 +1,44 @@
+using System.Diagnostics;
 using DanaCrawler.Config;
 using Microsoft.Extensions.Options;
+using Serilog.Context;
 
 namespace DanaCrawler;
 
 internal sealed class Worker : BackgroundService
 {
     private readonly AjudaDanaService _ajudaDanaService;
-    private readonly IOptions<GoogleApiConfig> _googleConfig;
+    private readonly GoogleSheetsService _googleSheetsService;
     private readonly ILogger<Worker> _logger;
 
-    public Worker(AjudaDanaService ajudaDanaService, IOptions<GoogleApiConfig> googleConfig, ILogger<Worker> logger)
+    public Worker(AjudaDanaService ajudaDanaService, GoogleSheetsService googleSheetsService, IOptions<GoogleApiConfig> googleConfig,
+        ILogger<Worker> logger)
     {
         _ajudaDanaService = ajudaDanaService ?? throw new ArgumentNullException(nameof(ajudaDanaService));
-        _googleConfig = googleConfig ?? throw new ArgumentNullException(nameof(googleConfig));
+        _googleSheetsService = googleSheetsService ?? throw new ArgumentNullException(nameof(googleSheetsService));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
         while (!stoppingToken.IsCancellationRequested)
         {
-            var helpRequests = await _ajudaDanaService.GetHelpRequestsWithTownsPaginated(stoppingToken);
-
-            _logger.LogInformation("Got {Count} HelpRequests from AjudaDana.es", helpRequests.Count);
-
-            await ExportToGoogleSheets(helpRequests);
-
-            if (_logger.IsEnabled(LogLevel.Information))
+            using (LogContext.PushProperty("dd_trace_id", Guid.NewGuid()))
             {
-                _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
+                var stopwatch = Stopwatch.StartNew();
+                {
+                    var helpRequests = await _googleSheetsService.GetSheetHelpRequests();
+                    _logger.LogInformation("Syncing {Count} HelpRequests from the google sheet", helpRequests.Count);
+                    await _ajudaDanaService.SyncCrmStatusWithDb(helpRequests);
+                }
+
+                {
+                    var helpRequests = await _ajudaDanaService.GetHelpRequestsWithTownsPaginated(stoppingToken);
+                    _logger.LogInformation("Got {Count} HelpRequests from AjudaDana.es", helpRequests.Count);
+                    await ExportToGoogleSheets(helpRequests);
+                }
+
+                _logger.LogInformation("Worker took: {time}", stopwatch.ToString());
             }
 
             await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
@@ -37,14 +47,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
     private async Task ExportToGoogleSheets(List<HelpRequest> helpRequests)
     {
-        var credentialsPath = _googleConfig.Value.CredentialsPath;
-        var spreadsheetId = _googleConfig.Value.SpreadsheetId;
-
-        var sheetsService = new GoogleSheetsService(credentialsPath, spreadsheetId);
-
         try
         {
-            await sheetsService.InsertHelpRequests(helpRequests);
+            await _googleSheetsService.InsertHelpRequests(helpRequests);
             //await sheetsService.FormatSheet();
             _logger.LogInformation("Data exported successfully!");
         }