Add API Log

app/Console/Commands/ApiLogCommand.php

@@ -0,0 +1,65 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\ApiLog;
+use App\Models\Artwork;
+use Illuminate\Console\Command;
+use Illuminate\Support\Collection;
+
+class ApiLogCommand extends Command
+{
+    protected $signature = 'app:api-log';
+
+    protected $description = 'Track changes to API values.';
+
+    public function handle()
+    {
+        Artwork::cacheArtworkApiData();
+
+        $currentValues = ApiLog::getCurrentValues();
+
+        $this->withProgressBar(Artwork::all(), fn (Artwork $artwork) => $this->getEndpointsForArtwork($artwork)
+            ->each(fn ($endpoint) => collect($endpoint['fields'])
+                ->reject(fn ($field) => $this->ignoring($field))
+                ->reject(fn ($field) => $this->isCurrentValue($artwork, $field, $endpoint['data'], $currentValues))
+                ->each(fn ($field) => $this->log($artwork, $field, $endpoint['data']))
+            )
+        );
+    }
+
+    private function getEndpointsForArtwork(Artwork $artwork): Collection
+    {
+        $artworkApiData = $artwork->getArtworkApiData();
+        $galleryApiData = $artwork->getGalleryApiData($artworkApiData->gallery_id);
+
+        return collect([
+            [
+                'data' => $artworkApiData,
+                'fields' => Artwork::ARTWORK_API_FIELDS,
+            ],
+            [
+                'data' => $galleryApiData,
+                'fields' => Artwork::GALLERY_API_FIELDS,
+            ],
+        ]);
+    }
+
+    private function ignoring(string $field): bool
+    {
+        return in_array($field, ['id', 'main_reference_number', 'image_id', 'thumbnail']);
+    }
+
+    private function isCurrentValue(Artwork $artwork, string $field, object $data, Collection $currentValues): bool
+    {
+        return $currentValues->has(md5($artwork->datahub_id.$field.$data->$field));
+    }
+
+    private function log(Artwork $artwork, string $field, object $data)
+    {
+        tap($artwork->apiLog()->firstOrCreate(
+            ['hash' => md5($artwork->datahub_id.$field.$data->$field)],
+            ['field' => $field, 'value' => $data->$field],
+        ), fn ($log) => $log->wasRecentlyCreated ? null : $log->touch());
+    }
+}

app/Console/Kernel.php

@@ -16,6 +16,10 @@ protected function schedule(Schedule $schedule): void
             ->command('app:cache-json')
             ->everyFiveMinutes()
             ->withoutOverlapping();
+
+        $schedule
+            ->command('app:api-log')
+            ->hourly();
     }
 
     /**

app/Http/Controllers/Twill/ApiLogController.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace App\Http\Controllers\Twill;
+
+use App\Models\ApiLog;
+use Illuminate\Routing\Controller;
+
+class ApiLogController extends Controller
+{
+    public function __invoke()
+    {
+        return view('admin.api-log', [
+            'logs' => ApiLog::getRecentChanges(),
+        ]);
+    }
+}

app/Models/ApiLog.php

@@ -0,0 +1,76 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
+
+class ApiLog extends Model
+{
+    protected $table = 'api_log';
+
+    protected $guarded = [];
+
+    public function artwork(): BelongsTo
+    {
+        return $this->belongsTo(Artwork::class, 'datahub_id', 'datahub_id');
+    }
+
+    public static function getCurrentValues(): Collection
+    {
+        $table = (new static)->getTable();
+
+        return DB::table($table)
+            ->select('*')
+            ->join(
+                DB::raw(
+                    <<<SQL
+                    (
+                        SELECT id, field, MAX(updated_at) as max_updated_at
+                        FROM {$table}
+                        GROUP BY id, field
+                    ) max
+                    SQL
+                ),
+                fn ($join) => $join->on($table.'.id', '=', 'max.id')
+                    ->on($table.'.field', '=', 'max.field')
+                    ->on($table.'.updated_at', '=', 'max.max_updated_at')
+
+            )
+            ->get()
+            ->keyBy('hash');
+    }
+
+    /**
+     * Get the most recent changes to the API data.
+     * Uses the LAG function to get the previous value for each datahub_id/field combination.
+     * LAG is used with the OVER clause, to partition the data by datahub_id/field and orders it by updated_at.
+     * For each datahub_id/field combination, the LAG function will return the value from the previous record in order of updated_at.
+     *
+     * The query is filtered to only include records from the last 6 months.
+     *
+     * @see https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_lag
+     */
+    public static function getRecentChanges(): Collection
+    {
+        return DB::table((new static)->getTable())
+            ->select([
+                'id',
+                'datahub_id',
+                'field',
+                'updated_at',
+                'value as new_value',
+                DB::raw('LAG(value) OVER (PARTITION BY datahub_id, field ORDER BY updated_at) as old_value',
+                )])
+            ->where('updated_at', '>', DB::raw('DATE_SUB(NOW(), INTERVAL 6 MONTH)'))
+            ->orderBy('datahub_id')
+            ->orderBy('field')
+            ->orderBy('updated_at', 'desc')
+            ->get()
+            ->reject(fn ($log) => $log->old_value === null)
+            ->map(fn ($log) => json_decode(json_encode($log), true))
+            ->mapInto(static::class);
+    }
+}

app/Models/Artwork.php

@@ -12,6 +12,7 @@
 use Exception;
 use Illuminate\Contracts\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\HasMany;
 use Illuminate\Database\Eloquent\Relations\HasManyThrough;
 use Illuminate\Support\Arr;
 use Illuminate\Support\Facades\Cache;
@@ -116,6 +117,11 @@ public function themePrompts(): HasManyThrough
         );
     }
 
+    public function apiLog(): HasMany
+    {
+        return $this->hasMany(ApiLog::class, 'datahub_id', 'datahub_id');
+    }
+
     public function scopeActive(Builder $query): void
     {
         $query->published()->onView()->translated();

app/Providers/AppServiceProvider.php

@@ -48,6 +48,10 @@ public function boot(): void
             NavigationLink::make()->title('Directory')->forRoute('twill.directory')
         );
 
+        TwillNavigation::addLink(
+            NavigationLink::make()->title('API Log')->forRoute('twill.api-log')
+        );
+
         $this->app->singleton(ApiQueryBuilder::class, function () {
             $connection = new AicConnection();
 

database/migrations/2024_05_06_204227_create_api_log_table.php

@@ -0,0 +1,20 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        Schema::create('api_log', function (Blueprint $table) {
+            $table->bigIncrements('id');
+            $table->string('datahub_id');
+            $table->string('field');
+            $table->text('value')->nullable();
+            $table->string('hash')->unique();
+            $table->timestamps();
+        });
+    }
+};

resources/views/admin/api-log.blade.php

@@ -0,0 +1,54 @@
+@extends('twill::layouts.free')
+
+@push('extra_css')
+    <link href="/assets/twill/css/custom.css" rel="stylesheet" />
+@endpush
+
+@section('customPageContent')
+    <div class="custom">
+        <div class="mb-8">
+            <h1 class="text-base font-semibold leading-6 text-gray-900">Changes to API Data</h1>
+            <div class="flow-root mt-4">
+                <div class="-mx-4 -my-2 overflow-x-auto sm:-mx-6 lg:-mx-8">
+                    <div class="inline-block min-w-full py-2 align-middle sm:px-6 lg:px-8">
+                        <table class="min-w-full">
+                            <thead class="bg-white">
+                                <tr>
+                                    <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 whitespace-nowrap">Object Id</th>
+                                    <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 whitespace-nowrap">Field</th>
+                                    <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 whitespace-nowrap">Old Value</th>
+                                    <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 whitespace-nowrap">New Value</th>
+                                    <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900 whitespace-nowrap">Updated</th>
+                                </tr>
+                                </thead>
+                            <tbody class="bg-white">
+                                @foreach($logs as $log)
+                                    <tr>
+                                        <td class="px-3 py-4 text-sm text-gray-600 whitespace-nowrap">
+                                            <a href="{{ route('twill.artworks.edit', $log->artwork->id) }}">
+                                                {{ $log->datahub_id }}
+                                                <img class="w-8 h-8 rounded-lg" src="{{ $log->artwork->image('thumbnail') }}" alt="">
+                                            </a>
+                                        </td>
+                                        <td class="px-3 py-4 text-sm text-gray-600 whitespace-nowrap">
+                                            {{ Str::of($log->field)->headline() }}
+                                        </td>
+                                        <td class="px-3 py-4 text-sm text-gray-600">
+                                            {{ $log->old_value }}
+                                        </td>
+                                        <td class="px-3 py-4 text-sm text-gray-600">
+                                            {{ $log->new_value }}
+                                        </td>
+                                        <td class="px-3 py-4 text-sm text-gray-600 whitespace-nowrap">
+                                            {{ Carbon\Carbon::parse($log->updated_at)->diffForHumans() }}
+                                        </td>
+                                    </tr>
+                                @endforeach
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+@endsection

routes/twill.php

@@ -1,6 +1,7 @@
 <?php
 
 use A17\Twill\Facades\TwillRoutes;
+use App\Http\Controllers\Twill\ApiLogController;
 use App\Http\Controllers\Twill\ArtworkController;
 use App\Http\Controllers\Twill\DirectoryController;
 use Illuminate\Support\Facades\Route;
@@ -16,3 +17,6 @@
 
 Route::get('directory', DirectoryController::class)
     ->name('directory');
+
+Route::get('api-log', ApiLogController::class)
+    ->name('api-log');