Synchronize translations with language context files

packages/ckeditor5-dev-translations/lib/index.js

@@ -6,3 +6,4 @@
 export { default as findMessages } from './findmessages.js';
 export { default as MultipleLanguageTranslationService } from './multiplelanguagetranslationservice.js';
 export { default as CKEditorTranslationsPlugin } from './ckeditortranslationsplugin.js';
+export { default as synchronizeTranslations } from './synchronizetranslations.js';

packages/ckeditor5-dev-translations/lib/synchronizetranslations.js

@@ -0,0 +1,197 @@
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+import upath from 'upath';
+import { logger } from '@ckeditor/ckeditor5-dev-utils';
+import getPackageContexts from './utils/getpackagecontexts.js';
+import { CONTEXT_FILE_PATH } from './utils/constants.js';
+import getSourceMessages from './utils/getsourcemessages.js';
+import updatePackageTranslations from './utils/updatepackagetranslations.js';
+
+/**
+ * Synchronizes translations in provided packages by performing the following steps:
+ * * Collect all i18n messages from all provided packages by finding `t()` calls in source files.
+ * * Detect if translation context is valid, i.e. whether there is no missing, unused or duplicated context.
+ * * If there are no validation errors, update all translation files ("*.po" files) to be in sync with the context file:
+ * * unused translation entries are removed,
+ * * missing translation entries are added with empty string as the message translation,
+ * * missing translation files are created for languages that do not have own "*.po" file yet.
+ *
+ * @param {object} options
+ * @param {Array.<string>} options.sourceFiles An array of source files that contain messages to translate.
+ * @param {Array.<string>} options.packagePaths An array of paths to packages, which will be used to find message contexts.
+ * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package.
+ * @param {boolean} [options.ignoreUnusedCorePackageContexts=false] Whether to skip unused context errors related to
+ * the `@ckeditor/ckeditor5-core` package.
+ * @param {boolean} [options.validateOnly=false] If set, only validates the translations contexts against the source messages without
+ * synchronizing the translations.
+ * @param {boolean} [options.skipLicenseHeader=false] Whether to skip adding the license header to newly created translation files.
+ */
+export default function synchronizeTranslations( options ) {
+ const {
+ sourceFiles,
+ packagePaths,
+ corePackagePath,
+ ignoreUnusedCorePackageContexts = false,
+ validateOnly = false,
+ skipLicenseHeader = false
+ } = options;
+
+ const errors = [];
+ const log = logger();
+
+ log.info( '📍 Loading translations contexts...' );
+ const packageContexts = getPackageContexts( { packagePaths, corePackagePath } );
+
+ log.info( '📍 Loading messages from source files...' );
+ const sourceMessages = getSourceMessages( { packagePaths, sourceFiles, onErrorCallback: error => errors.push( error ) } );
+
+ log.info( '📍 Validating translations contexts against the source messages...' );
+ errors.push(
+ ...assertNoMissingContext( { packageContexts, sourceMessages, corePackagePath } ),
+ ...assertAllContextUsed( { packageContexts, sourceMessages, corePackagePath, ignoreUnusedCorePackageContexts } ),
+ ...assertNoRepeatedContext( { packageContexts } )
+ );
+
+ if ( errors.length ) {
+ log.error( '🔥 The following errors have been found:' );
+
+ for ( const error of errors ) {
+ log.error( ` - ${ error }` );
+ }
+
+ process.exit( 1 );
+ }
+
+ if ( validateOnly ) {
+ log.info( '✨ No errors found.' );
+
+ return;
+ }
+
+ log.info( '📍 Synchronizing translations files...' );
+ updatePackageTranslations( { packageContexts, sourceMessages, skipLicenseHeader } );
+
+ log.info( '✨ Done.' );
+}
+
+/**
+ * @param {object} options
+ * @param {Array.<Context>} options.packageContexts An array of language contexts.
+ * @param {Array.<Message>} options.sourceMessages An array of i18n source messages.
+ * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package.
+ * @returns {Array.<string>}
+ */
+function assertNoMissingContext( { packageContexts, sourceMessages, corePackagePath } ) {
+ const contextMessageIdsGroupedByPackage = packageContexts.reduce( ( result, context ) => {
+ result[ context.packagePath ] = Object.keys( context.contextContent );
+
+ return result;
+ }, {} );
+
+ return sourceMessages
+ .filter( message => {
+ const contextMessageIds = [
+ ...contextMessageIdsGroupedByPackage[ message.packagePath ],
+ ...contextMessageIdsGroupedByPackage[ corePackagePath ]
+ ];
+
+ return !contextMessageIds.includes( message.id );
+ } )
+ .map( message => `Missing context "${ message.id }" in "${ message.filePath }".` );
+}
+
+/**
+ * @param {object} options
+ * @param {Array.<Context>} options.packageContexts An array of language contexts.
+ * @param {Array.<Message>} options.sourceMessages An array of i18n source messages.
+ * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package.
+ * @param {boolean} options.ignoreUnusedCorePackageContexts Whether to skip unused context errors related to the `@ckeditor/ckeditor5-core`
+ * package.
+ * @returns {Array.<string>}
+ */
+function assertAllContextUsed( { packageContexts, sourceMessages, corePackagePath, ignoreUnusedCorePackageContexts } ) {
+ const sourceMessageIds = sourceMessages.map( message => message.id );
+
+ const sourceMessageIdsGroupedByPackage = sourceMessages.reduce( ( result, message ) => {
+ result[ message.packagePath ] = result[ message.packagePath ] || [];
+ result[ message.packagePath ].push( message.id );
+
+ return result;
+ }, {} );
+
+ return packageContexts
+ .flatMap( context => {
+ const { packagePath, contextContent } = context;
+ const messageIds = Object.keys( contextContent );
+
+ return messageIds.map( messageId => ( { messageId, packagePath } ) );
+ } )
+ .filter( ( { messageId, packagePath } ) => {
+ if ( packagePath === corePackagePath ) {
+ return !sourceMessageIds.includes( messageId );
+ }
+
+ if ( !sourceMessageIdsGroupedByPackage[ packagePath ] ) {
+ return true;
+ }
+
+ return !sourceMessageIdsGroupedByPackage[ packagePath ].includes( messageId );
+ } )
+ .filter( ( { packagePath } ) => {
+ if ( ignoreUnusedCorePackageContexts && packagePath === corePackagePath ) {
+ return false;
+ }
+
+ return true;
+ } )
+ .map( ( { messageId, packagePath } ) => `Unused context "${ messageId }" in "${ upath.join( packagePath, CONTEXT_FILE_PATH ) }".` );
+}
+
+/**
+ * @param {object} options
+ * @param {Array.<Context>} options.packageContexts An array of language contexts.
+ * @returns {Array.<string>}
+ */
+function assertNoRepeatedContext( { packageContexts } ) {
+ const contextMessageIds = packageContexts
+ .flatMap( context => {
+ const { contextFilePath, contextContent } = context;
+ const messageIds = Object.keys( contextContent );
+
+ return messageIds.map( messageId => ( { messageId, contextFilePath } ) );
+ } )
+ .reduce( ( result, { messageId, contextFilePath } ) => {
+ result[ messageId ] = result[ messageId ] || [];
+ result[ messageId ].push( contextFilePath );
+
+ return result;
+ }, {} );
+
+ return Object.entries( contextMessageIds )
+ .filter( ( [ , contextFilePaths ] ) => contextFilePaths.length > 1 )
+ .map( ( [ messageId, contextFilePaths ] ) => {
+ return `Duplicated context "${ messageId }" in "${ contextFilePaths.join( '", "' ) }".`;
+ } );
+}
+
+/**
+ * @typedef {object} Message
+ *
+ * @property {string} id
+ * @property {string} string
+ * @property {string} filePath
+ * @property {string} packagePath
+ * @property {string} context
+ * @property {string} [plural]
+ */
+
+/**
+ * @typedef {object} Context
+ *
+ * @property {string} contextFilePath
+ * @property {object} contextContent
+ * @property {string} packagePath
+ */

packages/ckeditor5-dev-translations/lib/templates/translation.po

@@ -0,0 +1,7 @@
+# Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+#
+# Want to contribute to this file? Submit your changes via a GitHub Pull Request.
+#
+# Check out the official contributor's guide:
+# https://ckeditor.com/docs/ckeditor5/latest/framework/guides/contributing/contributing.html
+#

packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js

@@ -0,0 +1,20 @@
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+/**
+ * Removes unused headers from the translation file.
+ *
+ * @param {import('pofile')} translationFileContent Content of the translation file.
+ * @returns {import('pofile')}
+ */
+export default function cleanTranslationFileContent( translationFileContent ) {
+ translationFileContent.headers = {
+ Language: translationFileContent.headers.Language,
+ 'Plural-Forms': translationFileContent.headers[ 'Plural-Forms' ],
+ 'Content-Type': 'text/plain; charset=UTF-8'
+ };
+
+ return translationFileContent;
+}

packages/ckeditor5-dev-translations/lib/utils/constants.js

@@ -0,0 +1,7 @@
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+export const CONTEXT_FILE_PATH = 'lang/contexts.json';
+export const TRANSLATION_FILES_PATH = 'lang/translations';

packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js

@@ -0,0 +1,45 @@
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+import upath from 'upath';
+import fs from 'fs-extra';
+import PO from 'pofile';
+import { fileURLToPath } from 'url';
+import { getNPlurals, getFormula } from 'plural-forms';
+import getLanguages from './getlanguages.js';
+import { TRANSLATION_FILES_PATH } from './constants.js';
+import cleanTranslationFileContent from './cleantranslationfilecontent.js';
+
+const __filename = fileURLToPath( import.meta.url );
+const __dirname = upath.dirname( __filename );
+
+const TRANSLATION_TEMPLATE_PATH = upath.join( __dirname, '../templates/translation.po' );
+
+/**
+ * @param {object} options
+ * @param {string} options.packagePath Path to the package to check for missing translations.
+ * @param {boolean} options.skipLicenseHeader Whether to skip adding the license header to newly created translation files.
+ */
+export default function createMissingPackageTranslations( { packagePath, skipLicenseHeader } ) {
+ const translationsTemplate = skipLicenseHeader ? '' : fs.readFileSync( TRANSLATION_TEMPLATE_PATH, 'utf-8' );
+
+ for ( const { localeCode, languageCode, languageFileName } of getLanguages() ) {
+ const translationFilePath = upath.join( packagePath, TRANSLATION_FILES_PATH, `${ languageFileName }.po` );
+
+ if ( fs.existsSync( translationFilePath ) ) {
+ continue;
+ }
+
+ const translations = PO.parse( translationsTemplate );
+
+ translations.headers.Language = localeCode;
+ translations.headers[ 'Plural-Forms' ] = [
+ `nplurals=${ getNPlurals( languageCode ) };`,
+ `plural=${ getFormula( languageCode ) };`
+ ].join( ' ' );
+
+ fs.outputFileSync( translationFilePath, cleanTranslationFileContent( translations ).toString(), 'utf-8' );
+ }
+}

packages/ckeditor5-dev-translations/lib/utils/getlanguages.js

@@ -0,0 +1,109 @@
+/**
+ * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved.
+ * For licensing, see LICENSE.md.
+ */
+
+const SUPPORTED_LOCALES = [
+ 'en', // English
+ 'af', // Afrikaans
+ 'sq', // Albanian
+ 'ar', // Arabic
+ 'hy', // Armenian
+ 'ast', // Asturian
+ 'az', // Azerbaijani
+ 'eu', // Basque
+ 'bn', // Bengali
+ 'bs', // Bosnian
+ 'bg', // Bulgarian
+ 'ca', // Catalan
+ 'zh_CN', // Chinese (China)
+ 'zh_TW', // Chinese (Taiwan)
+ 'hr', // Croatian
+ 'cs', // Czech
+ 'da', // Danish
+ 'nl', // Dutch
+ 'en_AU', // English (Australia)
+ 'en_GB', // English (United Kingdom)
+ 'eo', // Esperanto
+ 'et', // Estonian
+ 'fi', // Finnish
+ 'fr', // French
+ 'gl', // Galician
+ 'de', // German
+ 'de_CH', // German (Switzerland)
+ 'el', // Greek
+ 'gu', // Gujarati
+ 'he', // Hebrew
+ 'hi', // Hindi
+ 'hu', // Hungarian
+ 'id', // Indonesian
+ 'it', // Italian
+ 'ja', // Japanese
+ 'jv', // Javanese
+ 'kn', // Kannada
+ 'kk', // Kazakh
+ 'km', // Khmer
+ 'ko', // Korean
+ 'ku', // Kurdish
+ 'lv', // Latvian
+ 'lt', // Lithuanian
+ 'ms', // Malay
+ 'ne_NP', // Nepali (Nepal)
+ 'no', // Norwegian
+ 'nb', // Norwegian Bokmål
+ 'oc', // Occitan (post 1500)
+ 'fa', // Persian
+ 'pl', // Polish
+ 'pt', // Portuguese
+ 'pt_BR', // Portuguese (Brazil)
+ 'ro', // Romanian
+ 'ru', // Russian
+ 'sr', // Serbian
+ 'sr@latin', // Serbian (Latin)
+ 'si_LK', // Sinhala (Sri Lanka)
+ 'sk', // Slovak
+ 'sl', // Slovenian
+ 'es', // Spanish
+ 'es_CO', // Spanish (Colombia)
+ 'sv', // Swedish
+ 'tt', // Tatar
+ 'th', // Thai
+ 'ti', // Tigrinya
+ 'tr', // Turkish
+ 'tk', // Turkmen
+ 'uk', // Ukrainian
+ 'ur', // Urdu
+ 'ug', // Uyghur
+ 'uz', // Uzbek
+ 'vi' // Vietnamese
+];
+
+const LOCALES_FILENAME_MAP = {
+ 'ne_NP': 'ne',
+ 'si_LK': 'si',
+ 'zh_TW': 'zh'
+};
+
+/**
+ * @returns {Array.<Language>}
+ */
+export default function getLanguages() {
+ return SUPPORTED_LOCALES.map( localeCode => {
+ const languageCode = localeCode.split( /[-_@]/ )[ 0 ];
+ const languageFileName = LOCALES_FILENAME_MAP[ localeCode ] || localeCode.toLowerCase().replace( /[^a-z0-9]+/, '-' );
+
+ return {
+ localeCode,
+ languageCode,
+ languageFileName
+ };
+ } );
+}
+
+/**
+ * @typedef {object} Language
+ *
+ * @property {string} localeCode
+ * @property {string} languageCode
+ * @property {string} languageFileName
+ */