Release

.env.example

@@ -13,9 +13,17 @@ USER_AGENT=
 # Sentry
 SENTRY_DSN=
 
+# Browserless (for screenshots)
+BROWSERLESS_ENDPOINT=ws://localhost:3000
+SCREENSHOT_HOST=host.docker.internal
+BROWSERLESS_CONCURRENT=2
+
 # S3 parameters
 AWS_S3_ENDPOINT=
+AWS_REGION=
 AWS_S3_BUCKET=
+AWS_S3_ARCHIVE_BUCKET=
+AWS_S3_PRIVATE_BUCKET=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 

.github/workflows/deploy.yml

@@ -0,0 +1,87 @@
+name: Deploy
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+
+jobs:
+  deploy-frontend:
+    runs-on: ubuntu-latest
+
+    environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Build
+      run: npm run build
+
+    - name: Deploy
+      uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read
+      env:
+        AWS_S3_ENDPOINT: ${{ secrets.AWS_S3_ENDPOINT }}
+        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_REGION: ${{ secrets.AWS_REGION }}
+        SOURCE_DIR: 'dist'
+
+  deploy-backend:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: ${{ github.repository }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Extract metadata
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+    - name: Build and push Docker image
+      id: push
+      uses: docker/build-push-action@v6
+      with:
+        context: .
+        file: ./docker/app/Dockerfile
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+
+    - name: Generate artifact attestation
+      uses: actions/attest-build-provenance@v1
+      with:
+        subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+        subject-digest: ${{ steps.push.outputs.digest }}
+        push-to-registry: true

app/data/DataArchiver.mjs

@@ -37,16 +37,16 @@ export default class DataArchiver
 
   get canRun() {
     return process.env.AWS_S3_ENDPOINT
-      && process.env.AWS_S3_REGION
-      && process.env.AWS_S3_BUCKET
+      && process.env.AWS_REGION
+      && process.env.AWS_S3_ARCHIVE_BUCKET
       && process.env.AWS_ACCESS_KEY_ID
       && process.env.AWS_SECRET_ACCESS_KEY;
   }
 
   get s3Client() {
     return this._client ??= new S3Client({
       endpoint: process.env.AWS_S3_ENDPOINT,
-      region: process.env.AWS_S3_REGION,
+      region: process.env.AWS_REGION,
       credentials: {
         accessKeyId: process.env.AWS_ACCESS_KEY_ID,
         secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
@@ -88,7 +88,7 @@ export default class DataArchiver
 
   async uploadViaS3(file, destination) {
     return this.s3Client.send(new PutObjectCommand({
-      Bucket: process.env.AWS_S3_BUCKET,
+      Bucket: process.env.AWS_S3_ARCHIVE_BUCKET,
       Key: destination,
       Body: await fs.readFile(file),
       ACL: 'public-read',

app/data/ImageProcessor.mjs

@@ -1,10 +1,14 @@
 import fs from 'fs/promises';
 import path from 'path';
 import mkdirp from 'mkdirp';
+// eslint-disable-next-line import/no-unresolved
+import PQueue from 'p-queue';
 import prefixedConsole from '../common/prefixedConsole.mjs';
 import { normalizeSplatnetResourcePath } from '../common/util.mjs';
 import { exists } from '../common/fs.mjs';
 
+const queue = new PQueue({ concurrency: 4 });
+
 export default class ImageProcessor
 {
   destinationDirectory = 'dist';
@@ -15,17 +19,27 @@ export default class ImageProcessor
     this.siteUrl = process.env.SITE_URL;
   }
 
-  async process(url) {
+  async process(url, defer = true) {
     // Normalize the path
     let destination = this.normalize(url);
 
     // Download the image if necessary
-    await this.maybeDownload(url, destination);
+    let job = () => this.maybeDownload(url, destination);
+    // defer ? queue.add(job) : await job();
+    if (defer) {
+      queue.add(job);
+    } else {
+      await job();
+    }
 
     // Return the new public URL
     return [destination, this.publicUrl(destination)];
   }
 
+  static onIdle() {
+    return queue.onIdle();
+  }
+
   normalize(url) {
     return normalizeSplatnetResourcePath(url);
   }
@@ -53,6 +67,10 @@ export default class ImageProcessor
     try {
       let result = await fetch(url);
 
+      if (!result.ok) {
+        throw new Error(`Invalid image response code: ${result.status}`);
+      }
+
       await mkdirp(path.dirname(this.localPath(destination)));
       await fs.writeFile(this.localPath(destination), result.body);
     } catch (e) {

app/data/LocalizationProcessor.mjs

@@ -4,6 +4,9 @@ import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import get from 'lodash/get.js';
 import set from 'lodash/set.js';
+import pLimit from 'p-limit';
+
+const limit = pLimit(1);
 
 function makeArray(value) {
   return Array.isArray(value) ? value : [value];
@@ -55,6 +58,12 @@ export class LocalizationProcessor {
   }
 
   async updateLocalizations(data) {
+    // We're reading, modifying, and writing back to the same file,
+    // so we have to make sure the whole operation is atomic.
+    return limit(() => this._updateLocalizations(data));
+  }
+
+  async _updateLocalizations(data) {
     let localizations = await this.readData();
 
     for (let { path, value } of this.dataIterations(data)) {

app/data/index.mjs

@@ -1,10 +1,13 @@
 import * as Sentry from '@sentry/node';
+import S3Syncer from '../sync/S3Syncer.mjs';
+import { canSync } from '../sync/index.mjs';
 import GearUpdater from './updaters/GearUpdater.mjs';
 import StageScheduleUpdater from './updaters/StageScheduleUpdater.mjs';
 import CoopUpdater from './updaters/CoopUpdater.mjs';
 import FestivalUpdater from './updaters/FestivalUpdater.mjs';
 import XRankUpdater from './updaters/XRankUpdater.mjs';
 import StagesUpdater from './updaters/StagesUpdater.mjs';
+import ImageProcessor from './ImageProcessor.mjs';
 
 function updaters() {
   return [
@@ -40,14 +43,19 @@ export async function update(config = 'default') {
 
   let settings = configs[config];
 
-  for (let updater of updaters()) {
+  await Promise.all(updaters().map(async updater => {
     updater.settings = settings;
     try {
       await updater.updateIfNeeded();
     } catch (e) {
       console.error(e);
       Sentry.captureException(e);
     }
+  }));
+
+  if (canSync()) {
+    await ImageProcessor.onIdle();
+    await (new S3Syncer).upload();
   }
 
   console.info(`Done running ${config} updaters`);

app/data/updaters/DataUpdater.mjs

@@ -4,14 +4,14 @@ import { Console } from 'node:console';
 import mkdirp from 'mkdirp';
 import jsonpath from 'jsonpath';
 import ical from 'ical-generator';
+import pFilter from 'p-filter';
 import prefixedConsole from '../../common/prefixedConsole.mjs';
 import SplatNet3Client from '../../splatnet/SplatNet3Client.mjs';
 import ImageProcessor from '../ImageProcessor.mjs';
 import NsoClient from '../../splatnet/NsoClient.mjs';
 import { locales, regionalLocales, defaultLocale } from '../../../src/common/i18n.mjs';
 import { LocalizationProcessor } from '../LocalizationProcessor.mjs';
 import { deriveId, getDateParts, getTopOfCurrentHour } from '../../common/util.mjs';
-
 export default class DataUpdater
 {
   name = null;
@@ -70,7 +70,6 @@ export default class DataUpdater
 
   async updateIfNeeded() {
     if (!(await this.shouldUpdate())) {
-      this.console.info('No need to update data');
       return;
     }
 
@@ -135,16 +134,18 @@ export default class DataUpdater
     }
 
     // Retrieve data for missing languages
-    for (let locale of this.locales.filter(l => l !== initialLocale)) {
-      processor = new LocalizationProcessor(locale, this.localizations);
-
-      if (await processor.hasMissingLocalizations(data)) {
-        this.console.info(`Retrieving localized data for ${locale.code}`);
+    let processors = this.locales.filter(l => l !== initialLocale)
+      .map(l => new LocalizationProcessor(l, this.localizations));
+    let missing = await pFilter(processors, p => p.hasMissingLocalizations(data));
 
-        let regionalData = await this.getData(locale);
+    if (missing.length > 0) {
+      await Promise.all(missing.map(async (processor) => {
+        let regionalData = await this.getData(processor.locale);
         this.deriveIds(regionalData);
         await processor.updateLocalizations(regionalData);
-      }
+
+        this.console.info(`Retrieved localized data for: ${processor.locale.code}`);
+      }));
     }
   }
 
@@ -166,6 +167,8 @@ export default class DataUpdater
       jsonpath.apply(data, expression, url => mapping[url]);
     }
 
+    await ImageProcessor.onIdle();
+
     return images;
   }
 

app/data/updaters/FestivalRankingUpdater.mjs

@@ -43,7 +43,7 @@ export default class FestivalRankingUpdater extends DataUpdater
   async getData(locale) {
     const data = await this.splatnet(locale).getFestRankingData(this.festID);
 
-    for (const team of data.data.fest.teams) {
+    await Promise.all(data.data.fest.teams.map(async (team) => {
       let pageInfo = team.result?.rankingHolders?.pageInfo;
 
       while (pageInfo?.hasNextPage) {
@@ -62,7 +62,7 @@ export default class FestivalRankingUpdater extends DataUpdater
 
         pageInfo = page.data.node.result.rankingHolders.pageInfo;
       }
-    }
+    }));
 
     return data;
   }

app/data/updaters/FestivalUpdater.mjs

@@ -1,11 +1,14 @@
 import fs from 'fs/promises';
 import jsonpath from 'jsonpath';
+import pLimit from 'p-limit';
 import { getFestId } from '../../common/util.mjs';
 import ValueCache from '../../common/ValueCache.mjs';
 import { regionTokens } from '../../splatnet/NsoClient.mjs';
 import FestivalRankingUpdater from './FestivalRankingUpdater.mjs';
 import DataUpdater from './DataUpdater.mjs';
 
+const limit = pLimit(1);
+
 function generateFestUrl(id) {
   return process.env.DEBUG ?
     `https://s.nintendo.com/av5ja-lp1/znca/game/4834290508791808?p=/fest_record/${id}` :
@@ -79,7 +82,7 @@ export default class FestivalUpdater extends DataUpdater
     // Get the detailed data for each Splatfest
     // (unless we're getting localization-specific data)
     if (locale === this.defaultLocale) {
-      for (let node of result.data.festRecords.nodes) {
+      await Promise.all(result.data.festRecords.nodes.map(async node => {
         let detailResult = await this.getFestivalDetails(node);
 
         Object.assign(node, detailResult.data.fest);
@@ -93,7 +96,7 @@ export default class FestivalUpdater extends DataUpdater
             this.console.error(e);
           }
         }
-      }
+      }));
     }
 
     return result;
@@ -128,7 +131,11 @@ export default class FestivalUpdater extends DataUpdater
     return data;
   }
 
-  async formatDataForWrite(data) {
+  formatDataForWrite(data) {
+    return limit(() => this._formatDataForWrite(data));
+  }
+
+  async _formatDataForWrite(data) {
     // Combine this region's data with the other regions' data.
     let result = null;
     try {

app/data/updaters/XRankUpdater.mjs

@@ -46,10 +46,9 @@ export default class XRankUpdater extends DataUpdater
     let result = await this.splatnet(locale).getXRankingData(this.divisionKey);
     let seasons = this.getSeasons(result.data);
 
-    for (let season of seasons) {
-      this.deriveSeasonId(season);
-      await this.updateSeasonDetail(season);
-    }
+    seasons.forEach(s => this.deriveSeasonId(s));
+
+    await Promise.all(seasons.map(season => this.updateSeasonDetail(season)));
 
     return result;
   }
@@ -66,9 +65,9 @@ export default class XRankUpdater extends DataUpdater
   }
 
   async updateSeasonDetail(season) {
-    for (let type of this.splatnet().getXRankingDetailQueryTypes()) {
+    await Promise.all(this.splatnet().getXRankingDetailQueryTypes().map(type => {
       let updater = new XRankDetailUpdater(season.id, season.endTime, type);
-      await updater.updateIfNeeded();
-    }
+      return updater.updateIfNeeded();
+    }));
   }
 }