From ccf400de4e646f30211f48735da6a14afdb05924 Mon Sep 17 00:00:00 2001 From: Gregor Eichelberger Date: Thu, 4 Apr 2024 09:11:58 +0200 Subject: [PATCH 1/3] Add lazy loading of lang files Only the currently used language is loaded instead of all language files from the beginning. A new lazy loading backend loads the lang files as a dynamic import. --- package-lock.json | 80 ++++++++++++++++++++++++ package.json | 2 + src/i18n/LazyLoadingPlugin.ts | 32 ++++++++++ src/i18n/config.tsx | 47 +++++++------- src/i18n/i18next.d.ts | 2 +- src/i18n/locales/{cs-CZ.json => cs.json} | 0 src/i18n/locales/{de-DE.json => de.json} | 0 src/i18n/locales/{el-GR.json => el.json} | 0 src/i18n/locales/{en-US.json => en.json} | 0 src/i18n/locales/{es-ES.json => es.json} | 0 src/i18n/locales/{fr-FR.json => fr.json} | 0 src/i18n/locales/{nl-NL.json => nl.json} | 0 src/main/Header.tsx | 5 +- 13 files changed, 138 insertions(+), 30 deletions(-) create mode 100644 src/i18n/LazyLoadingPlugin.ts rename src/i18n/locales/{cs-CZ.json => cs.json} (100%) rename src/i18n/locales/{de-DE.json => de.json} (100%) rename src/i18n/locales/{el-GR.json => el.json} (100%) rename src/i18n/locales/{en-US.json => en.json} (100%) rename src/i18n/locales/{es-ES.json => es.json} (100%) rename src/i18n/locales/{fr-FR.json => fr.json} (100%) rename src/i18n/locales/{nl-NL.json => nl.json} (100%) diff --git a/package-lock.json b/package-lock.json index 97c780da9..6bcbb9cf0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,8 @@ "final-form": "^4.20.10", "i18next": "^23.7.11", "i18next-browser-languagedetector": "^7.2.0", + "i18next-chained-backend": "^4.6.2", + "i18next-resources-to-backend": "^1.2.0", "lodash": "^4.17.21", "luxon": "^3.4.4", "mui-rff": "^7.3.0", @@ -4462,6 +4464,14 @@ "node": ">=10" } }, + "node_modules/cross-fetch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "devOptional": true, @@ -5843,6 +5853,30 @@ "@babel/runtime": "^7.23.2" } }, + "node_modules/i18next-chained-backend": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/i18next-chained-backend/-/i18next-chained-backend-4.6.2.tgz", + "integrity": "sha512-2P092fR+nAPQlGzPUoIIxbwo7PTBqQYgLxwv1XhSTQUAUoelLo5LkX+FqRxxSDg9WEAsrc8+2WL6mJtMGIa6WQ==", + "dependencies": { + "@babel/runtime": "^7.23.2" + } + }, + "node_modules/i18next-http-backend": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/i18next-http-backend/-/i18next-http-backend-2.5.0.tgz", + "integrity": "sha512-Z/aQsGZk1gSxt2/DztXk92DuDD20J+rNudT7ZCdTrNOiK8uQppfvdjq9+DFQfpAnFPn3VZS+KQIr1S/W1KxhpQ==", + "dependencies": { + "cross-fetch": "4.0.0" + } + }, + "node_modules/i18next-resources-to-backend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/i18next-resources-to-backend/-/i18next-resources-to-backend-1.2.0.tgz", + "integrity": "sha512-8f1l03s+QxDmCfpSXCh9V+AFcxAwIp0UaroWuyOx+hmmv8484GcELHs+lnu54FrNij8cDBEXvEwhzZoXsKcVpg==", + "dependencies": { + "@babel/runtime": "^7.23.2" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "dev": true, @@ -5934,6 +5968,14 @@ "devOptional": true, "license": "ISC" }, + "node_modules/install": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", + "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/internal-slot": { "version": "1.0.5", "dev": true, @@ -9603,6 +9645,44 @@ "tslib": "^2.0.3" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-int64": { "version": "0.4.0", "license": "MIT", diff --git a/package.json b/package.json index 6b86797b1..61a27a23a 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "final-form": "^4.20.10", "i18next": "^23.7.11", "i18next-browser-languagedetector": "^7.2.0", + "i18next-chained-backend": "^4.6.2", + "i18next-resources-to-backend": "^1.2.0", "lodash": "^4.17.21", "luxon": "^3.4.4", "mui-rff": "^7.3.0", diff --git a/src/i18n/LazyLoadingPlugin.ts b/src/i18n/LazyLoadingPlugin.ts new file mode 100644 index 000000000..da0750c21 --- /dev/null +++ b/src/i18n/LazyLoadingPlugin.ts @@ -0,0 +1,32 @@ +import { BackendModule, InitOptions, MultiReadCallback, ReadCallback, ResourceLanguage, Services } from "i18next"; + +export default class LazyLoadingPlugin implements BackendModule { + + type: "backend"; + + constructor(_services: Services, _backendOptions: object, _i18nextOptions: InitOptions) { + this.type = "backend"; + } + + init(_services: Services, _backendOptions: object, _i18nextOptions: InitOptions): void { + // no init needed + } + + read(language: string, _namespace: string, callback: ReadCallback): void { + import(`./locales/${language}.json`).then( + obj => { + callback(null, obj); + } + ); + } + create?(_languages: readonly string[], _namespace: string, _key: string, _fallbackValue: string): void { + throw new Error("Method not implemented."); + } + readMulti?(_languages: readonly string[], _namespaces: readonly string[], _callback: MultiReadCallback): void { + throw new Error("Method not implemented."); + } + save?(_language: string, _namespace: string, _data: ResourceLanguage): void { + throw new Error("Method not implemented."); + } + +} diff --git a/src/i18n/config.tsx b/src/i18n/config.tsx index 7f7349001..d38a01fd2 100644 --- a/src/i18n/config.tsx +++ b/src/i18n/config.tsx @@ -1,41 +1,36 @@ -import i18next, { InitOptions } from "i18next"; +import i18next from "i18next"; import { initReactI18next } from "react-i18next"; import LanguageDetector from "i18next-browser-languagedetector"; +import ChainedBackend, { ChainedBackendOptions } from "i18next-chained-backend"; +import resourcesToBackend from "i18next-resources-to-backend"; -import locales from "./locales.json"; +import LazyLoadingPlugin from "./LazyLoadingPlugin"; const debug = Boolean(new URLSearchParams(window.location.search).get("debug")); -const resources: InitOptions["resources"] = {}; +const bundledResources = { + en: { + translation: import("./locales/en.json"), + }, +}; -const data = import.meta.glob("./locales/*.json"); - -for (const path in data) { - const code = path.replace(/^.*[\\/]/, "").replace(/\..*$/, ""); - if (!locales.some(e => e.includes(code))) { - continue; - } - const short = code.replace(/-.*$/, ""); - const main = locales.filter(l => l.indexOf(short) === 0).length === 1; - - data[path]().then(mod => { - const translation = JSON.parse(JSON.stringify(mod)); - - if (!main) { - resources[code] = { translation: translation }; - } - resources[short] = { translation: translation }; - }); -} +export const locales: string[] = ["de", "el", "en", "es", "fr", "nl", "zh-CN", "zh-TW"]; i18next + .use(ChainedBackend) .use(initReactI18next) .use(LanguageDetector) - .init({ - resources, - fallbackLng: ["en-US", "en"], - nonExplicitSupportedLngs: true, + .init({ + supportedLngs: locales, + fallbackLng: ["en"], + nonExplicitSupportedLngs: false, debug: debug, + backend: { + backends: [ + LazyLoadingPlugin, + resourcesToBackend(bundledResources), + ], + }, }); if (debug) { diff --git a/src/i18n/i18next.d.ts b/src/i18n/i18next.d.ts index e738b1147..aed5136f6 100644 --- a/src/i18n/i18next.d.ts +++ b/src/i18n/i18next.d.ts @@ -2,7 +2,7 @@ import 'i18next'; // import all namespaces (for the default language, only) -import translation from '../i18n/locales/en-US.json'; +import translation from './locales/en.json'; declare module 'i18next' { interface CustomTypeOptions { diff --git a/src/i18n/locales/cs-CZ.json b/src/i18n/locales/cs.json similarity index 100% rename from src/i18n/locales/cs-CZ.json rename to src/i18n/locales/cs.json diff --git a/src/i18n/locales/de-DE.json b/src/i18n/locales/de.json similarity index 100% rename from src/i18n/locales/de-DE.json rename to src/i18n/locales/de.json diff --git a/src/i18n/locales/el-GR.json b/src/i18n/locales/el.json similarity index 100% rename from src/i18n/locales/el-GR.json rename to src/i18n/locales/el.json diff --git a/src/i18n/locales/en-US.json b/src/i18n/locales/en.json similarity index 100% rename from src/i18n/locales/en-US.json rename to src/i18n/locales/en.json diff --git a/src/i18n/locales/es-ES.json b/src/i18n/locales/es.json similarity index 100% rename from src/i18n/locales/es-ES.json rename to src/i18n/locales/es.json diff --git a/src/i18n/locales/fr-FR.json b/src/i18n/locales/fr.json similarity index 100% rename from src/i18n/locales/fr-FR.json rename to src/i18n/locales/fr.json diff --git a/src/i18n/locales/nl-NL.json b/src/i18n/locales/nl.json similarity index 100% rename from src/i18n/locales/nl-NL.json rename to src/i18n/locales/nl.json diff --git a/src/main/Header.tsx b/src/main/Header.tsx index 01650a78f..85713fb5a 100644 --- a/src/main/Header.tsx +++ b/src/main/Header.tsx @@ -17,6 +17,7 @@ import { selectIsEnd } from "../redux/endSlice"; import { checkboxMenuItem, HeaderMenuItemDef, ProtoButton, useColorScheme, WithHeaderMenu } from "@opencast/appkit"; import { IconType } from "react-icons"; import i18next from "i18next"; +import { locales } from "../i18n/config"; function Header() { const theme = useTheme(); @@ -142,9 +143,7 @@ const LanguageButton: React.FC = () => { }).of(language); }; - const resourcesArray: string[] | undefined = i18next.options.resources && Object.keys(i18next.options.resources); - - const languages = resourcesArray?.map(entry => { + const languages = locales?.map(entry => { return { value: entry, label: languageNames(entry) }; }); From 9eb29abead3ffdf65266fd7438cd3fd2caf2ecd4 Mon Sep 17 00:00:00 2001 From: Gregor Eichelberger Date: Thu, 4 Apr 2024 13:20:46 +0200 Subject: [PATCH 2/3] Update crowdin integration --- .github/generate-lngs.sh | 42 +++++++++++++++++++++++ .github/workflows/update-translations.yml | 5 +-- src/i18n/LazyLoadingPlugin.ts | 7 +++- src/i18n/config.tsx | 10 +++--- src/i18n/i18next.d.ts | 2 +- src/i18n/lngs-generated.ts | 11 ++++++ src/i18n/locales/{cs.json => cs-CZ.json} | 0 src/i18n/locales/{de.json => de-DE.json} | 0 src/i18n/locales/{el.json => el-GR.json} | 0 src/i18n/locales/{en.json => en-US.json} | 0 src/i18n/locales/{es.json => es-ES.json} | 0 src/i18n/locales/{fr.json => fr-FR.json} | 0 src/i18n/locales/{nl.json => nl-NL.json} | 0 src/main/Header.tsx | 6 ++-- 14 files changed, 69 insertions(+), 14 deletions(-) create mode 100755 .github/generate-lngs.sh create mode 100644 src/i18n/lngs-generated.ts rename src/i18n/locales/{cs.json => cs-CZ.json} (100%) rename src/i18n/locales/{de.json => de-DE.json} (100%) rename src/i18n/locales/{el.json => el-GR.json} (100%) rename src/i18n/locales/{en.json => en-US.json} (100%) rename src/i18n/locales/{es.json => es-ES.json} (100%) rename src/i18n/locales/{fr.json => fr-FR.json} (100%) rename src/i18n/locales/{nl.json => nl-NL.json} (100%) diff --git a/.github/generate-lngs.sh b/.github/generate-lngs.sh new file mode 100755 index 000000000..f96073886 --- /dev/null +++ b/.github/generate-lngs.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +declare -A country_language_map +declare -a order + +# Loop through each file in the directory +for file in *.json; do + if [ -f "$file" ]; then + # Extract country name and language code from the filename + country=$(basename "$file" .json | cut -d '-' -f 1) + language_code=$(basename "$file" .json) + + if [ ! "${country_language_map[$country]}" ]; then + order+=("$country") + fi + + # Check if the country already exists in the map + if [ -n "${country_language_map[$country]}" ]; then + country_language_map["$country"]+=" $language_code" + else + country_language_map["$country"]="$language_code" + fi + fi +done + +echo "export const languages = new Map([" +# Print the country-language mappings +for i in "${!order[@]}"; do + country=${order[$i]} + languages=() + for language in ${country_language_map[$country]}; do + languages+=("$language") + done + if [ ${#languages[@]} -eq 1 ]; then + echo " [\"$country\", \"${languages[0]}\"]," + else + for language in "${languages[@]}"; do + echo " [\"$language\", \"$language\"]," + done + fi +done +echo "]);" diff --git a/.github/workflows/update-translations.yml b/.github/workflows/update-translations.yml index a9965c837..4a270f5ee 100644 --- a/.github/workflows/update-translations.yml +++ b/.github/workflows/update-translations.yml @@ -34,10 +34,7 @@ jobs: - name: update language list working-directory: src/i18n/locales - run: | - echo -n '[ "' > locales.json - echo -n ??-??.json | sed 's/ */", "/g' >> locales.json - echo '" ]' >> locales.json + run: ./.github/generate-lngs.sh > ../lngs-generated.ts - name: upload translations run: | diff --git a/src/i18n/LazyLoadingPlugin.ts b/src/i18n/LazyLoadingPlugin.ts index da0750c21..1670a870b 100644 --- a/src/i18n/LazyLoadingPlugin.ts +++ b/src/i18n/LazyLoadingPlugin.ts @@ -1,4 +1,5 @@ import { BackendModule, InitOptions, MultiReadCallback, ReadCallback, ResourceLanguage, Services } from "i18next"; +import { languages } from "./lngs-generated"; export default class LazyLoadingPlugin implements BackendModule { @@ -13,18 +14,22 @@ export default class LazyLoadingPlugin implements BackendModule { } read(language: string, _namespace: string, callback: ReadCallback): void { - import(`./locales/${language}.json`).then( + const lng = languages.get(language); + import(`./locales/${lng}.json`).then( obj => { callback(null, obj); } ); } + create?(_languages: readonly string[], _namespace: string, _key: string, _fallbackValue: string): void { throw new Error("Method not implemented."); } + readMulti?(_languages: readonly string[], _namespaces: readonly string[], _callback: MultiReadCallback): void { throw new Error("Method not implemented."); } + save?(_language: string, _namespace: string, _data: ResourceLanguage): void { throw new Error("Method not implemented."); } diff --git a/src/i18n/config.tsx b/src/i18n/config.tsx index d38a01fd2..df3f740cf 100644 --- a/src/i18n/config.tsx +++ b/src/i18n/config.tsx @@ -4,25 +4,25 @@ import LanguageDetector from "i18next-browser-languagedetector"; import ChainedBackend, { ChainedBackendOptions } from "i18next-chained-backend"; import resourcesToBackend from "i18next-resources-to-backend"; +import { languages } from "./lngs-generated"; import LazyLoadingPlugin from "./LazyLoadingPlugin"; + const debug = Boolean(new URLSearchParams(window.location.search).get("debug")); const bundledResources = { en: { - translation: import("./locales/en.json"), + translation: import("./locales/en-US.json"), }, }; -export const locales: string[] = ["de", "el", "en", "es", "fr", "nl", "zh-CN", "zh-TW"]; - i18next .use(ChainedBackend) .use(initReactI18next) .use(LanguageDetector) .init({ - supportedLngs: locales, - fallbackLng: ["en"], + supportedLngs: Array.from(languages.keys()), + fallbackLng: ["en", "en-US"], nonExplicitSupportedLngs: false, debug: debug, backend: { diff --git a/src/i18n/i18next.d.ts b/src/i18n/i18next.d.ts index aed5136f6..c7d5492f2 100644 --- a/src/i18n/i18next.d.ts +++ b/src/i18n/i18next.d.ts @@ -2,7 +2,7 @@ import 'i18next'; // import all namespaces (for the default language, only) -import translation from './locales/en.json'; +import translation from './locales/en-US.json'; declare module 'i18next' { interface CustomTypeOptions { diff --git a/src/i18n/lngs-generated.ts b/src/i18n/lngs-generated.ts new file mode 100644 index 000000000..58a1996cb --- /dev/null +++ b/src/i18n/lngs-generated.ts @@ -0,0 +1,11 @@ +export const languages = new Map([ + ["cs", "cs-CZ"], + ["de", "de-DE"], + ["el", "el-GR"], + ["en", "en-US"], + ["es", "es-ES"], + ["fr", "fr-FR"], + ["nl", "nl-NL"], + ["zh-CN", "zh-CN"], + ["zh-TW", "zh-TW"], +]); diff --git a/src/i18n/locales/cs.json b/src/i18n/locales/cs-CZ.json similarity index 100% rename from src/i18n/locales/cs.json rename to src/i18n/locales/cs-CZ.json diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de-DE.json similarity index 100% rename from src/i18n/locales/de.json rename to src/i18n/locales/de-DE.json diff --git a/src/i18n/locales/el.json b/src/i18n/locales/el-GR.json similarity index 100% rename from src/i18n/locales/el.json rename to src/i18n/locales/el-GR.json diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en-US.json similarity index 100% rename from src/i18n/locales/en.json rename to src/i18n/locales/en-US.json diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es-ES.json similarity index 100% rename from src/i18n/locales/es.json rename to src/i18n/locales/es-ES.json diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr-FR.json similarity index 100% rename from src/i18n/locales/fr.json rename to src/i18n/locales/fr-FR.json diff --git a/src/i18n/locales/nl.json b/src/i18n/locales/nl-NL.json similarity index 100% rename from src/i18n/locales/nl.json rename to src/i18n/locales/nl-NL.json diff --git a/src/main/Header.tsx b/src/main/Header.tsx index 85713fb5a..1250a62e2 100644 --- a/src/main/Header.tsx +++ b/src/main/Header.tsx @@ -17,7 +17,7 @@ import { selectIsEnd } from "../redux/endSlice"; import { checkboxMenuItem, HeaderMenuItemDef, ProtoButton, useColorScheme, WithHeaderMenu } from "@opencast/appkit"; import { IconType } from "react-icons"; import i18next from "i18next"; -import { locales } from "../i18n/config"; +import { languages as lngs } from "../i18n/lngs-generated"; function Header() { const theme = useTheme(); @@ -143,8 +143,8 @@ const LanguageButton: React.FC = () => { }).of(language); }; - const languages = locales?.map(entry => { - return { value: entry, label: languageNames(entry) }; + const languages = Array.from(lngs, ([key, value]) => { + return { value: value, label: languageNames(key) }; }); // menuItems can"t deal with languages being undefined, so we return early From 5e93875871cb5c1373607460f587217ef248c0b9 Mon Sep 17 00:00:00 2001 From: Gregor Eichelberger Date: Thu, 4 Apr 2024 20:19:24 +0200 Subject: [PATCH 3/3] Fix update-translations workflow --- .github/generate-lngs.sh | 2 +- .github/workflows/update-translations.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/generate-lngs.sh b/.github/generate-lngs.sh index f96073886..7bedc83f5 100755 --- a/.github/generate-lngs.sh +++ b/.github/generate-lngs.sh @@ -4,7 +4,7 @@ declare -A country_language_map declare -a order # Loop through each file in the directory -for file in *.json; do +for file in ??-??.json; do if [ -f "$file" ]; then # Extract country name and language code from the filename country=$(basename "$file" .json | cut -d '-' -f 1) diff --git a/.github/workflows/update-translations.yml b/.github/workflows/update-translations.yml index 4a270f5ee..639d136d0 100644 --- a/.github/workflows/update-translations.yml +++ b/.github/workflows/update-translations.yml @@ -34,7 +34,7 @@ jobs: - name: update language list working-directory: src/i18n/locales - run: ./.github/generate-lngs.sh > ../lngs-generated.ts + run: $GITHUB_WORKSPACE/.github/generate-lngs.sh > ../lngs-generated.ts - name: upload translations run: |