job: auto-update queryservice allowlist (#870)

app/Console/Kernel.php

@@ -14,6 +14,7 @@
 use App\Jobs\SendEmptyWikiNotificationsJob;
 use App\Jobs\CreateQueryserviceBatchesJob;
 use App\Jobs\FailStalledEntityImportsJob;
+use App\Jobs\UpdateQueryserviceAllowList;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
 
@@ -57,6 +58,8 @@ protected function schedule(Schedule $schedule): void
         $schedule->job(new UpdateWikiSiteStatsJob)->dailyAt('19:00');
 
         $schedule->job(new SendEmptyWikiNotificationsJob)->dailyAt('21:00');
+
+        $schedule->job(new UpdateQueryserviceAllowList)->weeklyOn(Schedule::MONDAY, '01:00');
     }
 
     /**

app/Jobs/UpdateQueryserviceAllowList.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Wiki;
+use Maclof\Kubernetes\Client;
+use Maclof\Kubernetes\Models\ConfigMap;
+
+class UpdateQueryserviceAllowList extends Job
+{
+    public function handle(Client $k8s)
+    {
+        $allowList = implode(
+            PHP_EOL,
+            array_map(
+                fn($domain) => "https://{$domain}/query/sparql",
+                Wiki::all()->pluck('domain')->toArray()
+            )
+        );
+
+        $k8s->setNamespace('default');
+        $configName = 'queryservice-allowlist';
+        $config = $k8s->configMaps()->setFieldSelector([
+            'metadata.name' => $configName
+        ])->first();
+
+        if ($config === null) {
+            $this->fail(
+                new \RuntimeException(
+                    "Queryservice config map '{$configName}' does not exist."
+                )
+            );
+            return;
+        }
+
+        $allowListKey = 'allowlist.txt';
+        $allowListStaticKey = 'allowlist-static.txt';
+
+        $config = $config->toArray();
+        if (array_key_exists($allowListStaticKey, $config['data'])) {
+            $allowList .= PHP_EOL . trim($config['data'][$allowListStaticKey]);
+        }
+        $config['data'][$allowListKey] = trim($allowList);
+        $k8s->configMaps()->update(new ConfigMap($config));
+    }
+}

tests/Jobs/UpdateQueryserviceAllowListTest.php

@@ -0,0 +1,117 @@
+<?php
+
+namespace Tests\Jobs;
+
+use Tests\TestCase;
+use Illuminate\Contracts\Queue\Job;
+use App\Jobs\UpdateQueryserviceAllowList;
+use App\Wiki;
+use Maclof\Kubernetes\Client;
+use Http\Adapter\Guzzle7\Client as GuzzleClient;
+use GuzzleHttp\HandlerStack;
+use GuzzleHttp\Handler\MockHandler;
+use GuzzleHttp\Middleware;
+use GuzzleHttp\Psr7\Response;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+class UpdateQueryserviceAllowListTest extends TestCase
+{
+    use RefreshDatabase;
+
+    public function setUp(): void
+    {
+        parent::setUp();
+        Wiki::factory()->create(['domain' => 'somesite-5.localhost']);
+        Wiki::factory()->create(['domain' => 'somesite-6.localhost']);
+    }
+
+    public function testMissingConfigMapFailure(): void
+    {
+        $mockJob = $this->createMock(Job::class);
+        $mockJob->expects($this->once())
+            ->method('fail')
+            ->with(new \RuntimeException(
+                "Queryservice config map 'queryservice-allowlist' does not exist."
+            ));
+
+        $job = new UpdateQueryserviceAllowList();
+        $job->setJob($mockJob);
+
+        $mock = new MockHandler([
+            new Response(200, [], json_encode(['items' => []])),
+            new Response(200, [], json_encode(['items' => []]))
+        ]);
+
+        $handlerStack = HandlerStack::create($mock);
+        $mockGuzzle = GuzzleClient::createWithConfig([
+            'handler' => $handlerStack,
+            'verify' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+        ]);
+
+        $job->handle(new Client([
+            'master' => 'https://kubernetes.default.svc',
+            'token' => '/var/run/secrets/kubernetes.io/serviceaccount/token'
+        ], null, $mockGuzzle));
+    }
+
+    public function testSuccess(): void
+    {
+        $mockJob = $this->createMock(Job::class);
+        $mockJob->expects($this->never())->method('fail');
+
+        $job = new UpdateQueryserviceAllowList();
+        $job->setJob($mockJob);
+
+        $mock = new MockHandler([
+            new Response(200, [], json_encode(['items' => [[
+                'kind' => 'ConfigMap',
+                'apiVersion' => 'v1',
+                'data' => [
+                    'allowlist-static.txt' => implode(PHP_EOL, [
+                        'https://somesite-1.localhost/query/sparql',
+                        'https://somesite-2.localhost/query/sparql'
+                    ]),
+                    'allowlist.txt' => implode(PHP_EOL, [
+                        'https://somesite-3.localhost/query/sparql',
+                        'https://somesite-4.localhost/query/sparql'
+                    ])
+                ]
+            ]]])),
+            new Response(200, [], json_encode(['items' => []]))
+        ]);
+
+        $requests = [];
+        $handlerStack = HandlerStack::create($mock);
+        $handlerStack->push(Middleware::history($requests));
+        $mockGuzzle = GuzzleClient::createWithConfig([
+            'handler' => $handlerStack,
+            'verify' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+        ]);
+
+        $job->handle(new Client([
+            'master' => 'https://kubernetes.default.svc',
+            'token' => '/var/run/secrets/kubernetes.io/serviceaccount/token'
+        ], null, $mockGuzzle));
+
+        $this->assertCount(2, $requests);
+        $this->assertEquals(
+            [
+                'kind' => 'ConfigMap',
+                'apiVersion' => 'v1',
+                'data' => [
+                    'allowlist-static.txt' => implode(PHP_EOL, [
+                        'https://somesite-1.localhost/query/sparql',
+                        'https://somesite-2.localhost/query/sparql'
+                    ]),
+                    'allowlist.txt' => implode(PHP_EOL, [
+                        'https://somesite-5.localhost/query/sparql',
+                        'https://somesite-6.localhost/query/sparql',
+                        'https://somesite-1.localhost/query/sparql',
+                        'https://somesite-2.localhost/query/sparql'
+                    ])
+                ]
+            ],
+            json_decode($requests[1]['request']->getBody(), true)
+        );
+    }
+}