From c55505dfbabee12a1aeda262a957e6b39b8a168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Mon, 7 Oct 2024 14:46:17 +0200 Subject: [PATCH 01/10] Introduced `synchronizeTranslations()` function to synchronize translations with context files. --- .../ckeditor5-dev-translations/lib/index.js | 1 + .../lib/synchronizetranslations.js | 280 ++++++++++++++++++ .../ckeditor5-dev-translations/package.json | 5 +- 3 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 packages/ckeditor5-dev-translations/lib/synchronizetranslations.js diff --git a/packages/ckeditor5-dev-translations/lib/index.js b/packages/ckeditor5-dev-translations/lib/index.js index db5991323..273fc6250 100644 --- a/packages/ckeditor5-dev-translations/lib/index.js +++ b/packages/ckeditor5-dev-translations/lib/index.js @@ -8,3 +8,4 @@ export { default as cleanPoFileContent } from './cleanpofilecontent.js'; export { default as MultipleLanguageTranslationService } from './multiplelanguagetranslationservice.js'; export { default as createDictionaryFromPoFileContent } from './createdictionaryfrompofilecontent.js'; export { default as CKEditorTranslationsPlugin } from './ckeditortranslationsplugin.js'; +export { default as synchronizeTranslations } from './synchronizetranslations.js'; diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js new file mode 100644 index 000000000..529927e98 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -0,0 +1,280 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import upath from 'upath'; +import fs from 'fs-extra'; +import PO from 'pofile'; +import { glob } from 'glob'; +import { logger } from '@ckeditor/ckeditor5-dev-utils'; +import cleanPoFileContent from './cleanpofilecontent.js'; +import findMessages from './findmessages.js'; + +const CONTEXT_FILE_PATH = upath.join( 'lang', 'contexts.json' ); +const TRANSLATIONS_FILES_PATTERN = upath.join( 'lang', 'translations', '*.po' ); + +/** + * Synchronizes translations in provided packages by performing the following steps: + * * Collect all i18n messages from all provided packages by finding `t()` calls. + * * Detect if translation context is valid, i.e. whether there is no missing, unused or duplicated context. + * * If there are no validation errors, update all translation files (*.po files) to be in sync with the context file. + * + * @param {object} options + * @param {Array.} options.sourceFiles An array of source files that contain messages to translate. + * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. + * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. + * @param {boolean} [options.ignoreUnusedCorePackageContexts=false] Whether to skip unused context errors related to + * the `@ckeditor/ckeditor5-core` package. + */ +export default function synchronizeTranslations( options ) { + const { + sourceFiles, + packagePaths, + corePackagePath, + ignoreUnusedCorePackageContexts = false + } = options; + + const errors = []; + const log = logger(); + + log.info( '📍 Loading translations contexts...' ); + const packageContexts = getPackageContexts( { packagePaths, corePackagePath } ); + + log.info( '📍 Loading messages from source files...' ); + const sourceMessages = getSourceMessages( { packagePaths, sourceFiles, onErrorCallback: error => errors.push( error ) } ); + + log.info( '📍 Validating translations contexts against the source messages...' ); + errors.push( + ...assertNoMissingContext( { packageContexts, sourceMessages, corePackagePath } ), + ...assertAllContextUsed( { packageContexts, sourceMessages, corePackagePath, ignoreUnusedCorePackageContexts } ), + ...assertNoRepeatedContext( { packageContexts } ) + ); + + if ( errors.length ) { + log.error( '🔥 The following errors have been found:' ); + + for ( const error of errors ) { + log.error( ` - ${ error }` ); + } + + process.exit( 1 ); + } + + log.info( '📍 Synchronizing translations files...' ); + updatePackageTranslations( { packageContexts, sourceMessages } ); + + log.info( '✨ Done.' ); +} + +/** + * @param {object} options + * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. + * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. + * @returns {Array.} + */ +function getPackageContexts( { packagePaths, corePackagePath } ) { + // Add path to the core package if not included in the package paths. + // The core package contains common contexts shared between other packages. + if ( !packagePaths.includes( corePackagePath ) ) { + packagePaths.push( corePackagePath ); + } + + return packagePaths.map( packagePath => { + const contextFilePath = upath.join( packagePath, CONTEXT_FILE_PATH ); + const contextContent = fs.existsSync( contextFilePath ) ? fs.readJsonSync( contextFilePath ) : []; + + return { + contextContent, + contextFilePath, + packagePath + }; + } ); +} + +/** + * @param {object} options + * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. + * @param {Array.} options.sourceFiles An array of source files that contain messages to translate. + * @param {Function} options.onErrorCallback Called when there is an error with parsing the source files. + * @returns {Array.} + */ +function getSourceMessages( { packagePaths, sourceFiles, onErrorCallback } ) { + return sourceFiles.flatMap( filePath => { + const fileContent = fs.readFileSync( filePath, 'utf-8' ); + const packagePath = packagePaths.find( packagePath => filePath.includes( packagePath ) ); + const sourceMessages = []; + + const onMessageCallback = message => { + sourceMessages.push( { filePath, packagePath, ...message } ); + }; + + findMessages( fileContent, filePath, onMessageCallback, onErrorCallback ); + + return sourceMessages; + } ); +} + +/** + * @param {object} options + * @param {Array.} options.packageContexts An array of language contexts. + * @param {Array.} options.sourceMessages An array of i18n source messages. + */ +function updatePackageTranslations( { packageContexts, sourceMessages } ) { + // For each package: + for ( const { packagePath, contextContent } of packageContexts ) { + // (1) Find all source messages that are defined in the language context. + const sourceMessagesForPackage = Object.keys( contextContent ) + .map( messageId => sourceMessages.find( message => message.id === messageId ) ) + .filter( Boolean ); + + // (2) Find all translation files (*.po files). + const translationsFiles = glob.sync( upath.join( packagePath, TRANSLATIONS_FILES_PATTERN ) ); + + // Then, for each translation file in a package: + for ( const translationsFile of translationsFiles ) { + const translations = PO.parse( fs.readFileSync( translationsFile, 'utf-8' ) ); + + // (2.1) Remove unused translations. + translations.items = translations.items.filter( item => contextContent[ item.msgid ] ); + + // (2.2) Add missing translations. + translations.items.push( + ...sourceMessagesForPackage + .filter( message => !translations.items.find( item => item.msgid === message.id ) ) + .map( message => { + const numberOfPluralForms = PO.parsePluralForms( translations.headers[ 'Plural-Forms' ] ).nplurals; + const item = new PO.Item( { nplurals: numberOfPluralForms } ); + + item.msgctxt = contextContent[ message.id ]; + item.msgid = message.string; + item.msgstr.push( message.string ); + + if ( message.plural ) { + item.msgid_plural = message.plural; + item.msgstr.push( ...Array( numberOfPluralForms - 1 ).fill( message.plural ) ); + } + + return item; + } ) + ); + + fs.writeFileSync( translationsFile, cleanPoFileContent( translations.toString() ), 'utf-8' ); + } + } +} + +/** + * @param {object} options + * @param {Array.} options.packageContexts An array of language contexts. + * @param {Array.} options.sourceMessages An array of i18n source messages. + * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. + * @returns {Array.} + */ +function assertNoMissingContext( { packageContexts, sourceMessages, corePackagePath } ) { + const contextMessageIdsGroupedByPackage = packageContexts.reduce( ( result, context ) => { + result[ context.packagePath ] = Object.keys( context.contextContent ); + + return result; + }, {} ); + + return sourceMessages + .filter( message => { + const contextMessageIds = [ + ...contextMessageIdsGroupedByPackage[ message.packagePath ], + ...contextMessageIdsGroupedByPackage[ corePackagePath ] + ]; + + return !contextMessageIds.includes( message.id ); + } ) + .map( message => `Missing context "${ message.id }" in "${ message.filePath }".` ); +} + +/** + * @param {object} options + * @param {Array.} options.packageContexts An array of language contexts. + * @param {Array.} options.sourceMessages An array of i18n source messages. + * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. + * @param {boolean} options.ignoreUnusedCorePackageContexts Whether to skip unused context errors related to the `@ckeditor/ckeditor5-core` + * package. + * @returns {Array.} + */ +function assertAllContextUsed( { packageContexts, sourceMessages, corePackagePath, ignoreUnusedCorePackageContexts } ) { + const sourceMessageIds = sourceMessages.map( message => message.id ); + + const sourceMessageIdsGroupedByPackage = sourceMessages.reduce( ( result, message ) => { + result[ message.packagePath ] = result[ message.packagePath ] || []; + result[ message.packagePath ].push( message.id ); + + return result; + }, {} ); + + return packageContexts + .flatMap( context => { + const { packagePath, contextContent } = context; + const messageIds = Object.keys( contextContent ); + + return messageIds.map( messageId => ( { messageId, packagePath } ) ); + } ) + .filter( ( { messageId, packagePath } ) => { + if ( packagePath === corePackagePath ) { + return !sourceMessageIds.includes( messageId ); + } + + return !sourceMessageIdsGroupedByPackage[ packagePath ].includes( messageId ); + } ) + .filter( ( { packagePath } ) => { + if ( ignoreUnusedCorePackageContexts && packagePath === corePackagePath ) { + return false; + } + + return true; + } ) + .map( ( { messageId, packagePath } ) => `Unused context "${ messageId }" in "${ upath.join( packagePath, CONTEXT_FILE_PATH ) }".` ); +} + +/** + * @param {object} options + * @param {Array.} options.packageContexts An array of language contexts. + * @returns {Array.} + */ +function assertNoRepeatedContext( { packageContexts } ) { + const contextMessageIds = packageContexts + .flatMap( context => { + const { contextFilePath, contextContent } = context; + const messageIds = Object.keys( contextContent ); + + return messageIds.map( messageId => ( { messageId, contextFilePath } ) ); + } ) + .reduce( ( result, { messageId, contextFilePath } ) => { + result[ messageId ] = result[ messageId ] || []; + result[ messageId ].push( contextFilePath ); + + return result; + }, {} ); + + return Object.entries( contextMessageIds ) + .filter( ( [ , contextFilePaths ] ) => contextFilePaths.length > 1 ) + .map( ( [ messageId, contextFilePaths ] ) => { + return `Duplicated context "${ messageId }" in "${ contextFilePaths.join( '", "' ) }".`; + } ); +} + +/** + * @typedef {object} Message + * + * @property {string} id + * @property {string} string + * @property {string} filePath + * @property {string} packagePath + * @property {string} context + * @property {string} [plural] + */ + +/** + * @typedef {object} Context + * + * @property {string} contextFilePath + * @property {object} contextContent + * @property {string} packagePath + */ diff --git a/packages/ckeditor5-dev-translations/package.json b/packages/ckeditor5-dev-translations/package.json index fb95af130..57e2b767b 100644 --- a/packages/ckeditor5-dev-translations/package.json +++ b/packages/ckeditor5-dev-translations/package.json @@ -24,11 +24,14 @@ "dependencies": { "@babel/parser": "^7.18.9", "@babel/traverse": "^7.18.9", + "@ckeditor/ckeditor5-dev-utils": "^44.0.0", "chalk": "^5.0.0", "fs-extra": "^11.0.0", + "glob": "^10.0.0", "rimraf": "^5.0.0", "webpack-sources": "^3.0.0", - "pofile": "^1.0.9" + "pofile": "^1.0.9", + "upath": "^2.0.1" }, "devDependencies": { "vitest": "^2.0.5" From 64dcde1b9f5a1e69f4c128e558fa58d63512753c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Tue, 8 Oct 2024 14:58:03 +0200 Subject: [PATCH 02/10] Create missing translation files from empty PO template. --- .../lib/getlanguages.js | 109 ++++++++++++++++++ .../lib/synchronizetranslations.js | 47 ++++++-- .../lib/templates/empty.po | 12 ++ .../ckeditor5-dev-translations/package.json | 1 + 4 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 packages/ckeditor5-dev-translations/lib/getlanguages.js create mode 100644 packages/ckeditor5-dev-translations/lib/templates/empty.po diff --git a/packages/ckeditor5-dev-translations/lib/getlanguages.js b/packages/ckeditor5-dev-translations/lib/getlanguages.js new file mode 100644 index 000000000..364af9afd --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/getlanguages.js @@ -0,0 +1,109 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +const SUPPORTED_LOCALES = [ + 'en', // English + 'af', // Afrikaans + 'sq', // Albanian + 'ar', // Arabic + 'hy', // Armenian + 'ast', // Asturian + 'az', // Azerbaijani + 'eu', // Basque + 'bn', // Bengali + 'bs', // Bosnian + 'bg', // Bulgarian + 'ca', // Catalan + 'zh_CN', // Chinese (China) + 'zh_TW', // Chinese (Taiwan) + 'hr', // Croatian + 'cs', // Czech + 'da', // Danish + 'nl', // Dutch + 'en_AU', // English (Australia) + 'en_GB', // English (United Kingdom) + 'eo', // Esperanto + 'et', // Estonian + 'fi', // Finnish + 'fr', // French + 'gl', // Galician + 'de', // German + 'de_CH', // German (Switzerland) + 'el', // Greek + 'gu', // Gujarati + 'he', // Hebrew + 'hi', // Hindi + 'hu', // Hungarian + 'id', // Indonesian + 'it', // Italian + 'ja', // Japanese + 'jv', // Javanese + 'kn', // Kannada + 'kk', // Kazakh + 'km', // Khmer + 'ko', // Korean + 'ku', // Kurdish + 'lv', // Latvian + 'lt', // Lithuanian + 'ms', // Malay + 'ne_NP', // Nepali (Nepal) + 'no', // Norwegian + 'nb', // Norwegian Bokmål + 'oc', // Occitan (post 1500) + 'fa', // Persian + 'pl', // Polish + 'pt', // Portuguese + 'pt_BR', // Portuguese (Brazil) + 'ro', // Romanian + 'ru', // Russian + 'sr', // Serbian + 'sr@latin', // Serbian (Latin) + 'si_LK', // Sinhala (Sri Lanka) + 'sk', // Slovak + 'sl', // Slovenian + 'es', // Spanish + 'es_CO', // Spanish (Colombia) + 'sv', // Swedish + 'tt', // Tatar + 'th', // Thai + 'ti', // Tigrinya + 'tr', // Turkish + 'tk', // Turkmen + 'uk', // Ukrainian + 'ur', // Urdu + 'ug', // Uyghur + 'uz', // Uzbek + 'vi' // Vietnamese +]; + +const LOCALES_FILENAME_MAP = { + 'ne_NP': 'ne', + 'si_LK': 'si', + 'zh_TW': 'zh' +}; + +/** + * @returns {Array.} + */ +export function getLanguages() { + return SUPPORTED_LOCALES.map( localeCode => { + const languageCode = localeCode.split( /[-_@]/ )[ 0 ]; + const languageFileName = LOCALES_FILENAME_MAP[ localeCode ] || localeCode.toLowerCase().replace( /[^a-z0-9]+/, '-' ); + + return { + localeCode, + languageCode, + languageFileName + }; + } ); +} + +/** + * @typedef {object} Language + * + * @property {string} localeCode + * @property {string} languageCode + * @property {string} languageFileName + */ diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js index 529927e98..3edccb774 100644 --- a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -7,12 +7,19 @@ import upath from 'upath'; import fs from 'fs-extra'; import PO from 'pofile'; import { glob } from 'glob'; +import { fileURLToPath } from 'url'; +import { getNPlurals, getFormula } from 'plural-forms'; import { logger } from '@ckeditor/ckeditor5-dev-utils'; import cleanPoFileContent from './cleanpofilecontent.js'; import findMessages from './findmessages.js'; +import { getLanguages } from './getlanguages.js'; +const __filename = fileURLToPath( import.meta.url ); +const __dirname = upath.dirname( __filename ); + +const EMPTY_TRANSLATION_TEMPLATE = upath.join( __dirname, 'templates', 'empty.po' ); const CONTEXT_FILE_PATH = upath.join( 'lang', 'contexts.json' ); -const TRANSLATIONS_FILES_PATTERN = upath.join( 'lang', 'translations', '*.po' ); +const TRANSLATION_FILES_PATH = upath.join( 'lang', 'translations' ); /** * Synchronizes translations in provided packages by performing the following steps: @@ -128,12 +135,14 @@ function updatePackageTranslations( { packageContexts, sourceMessages } ) { .map( messageId => sourceMessages.find( message => message.id === messageId ) ) .filter( Boolean ); + createMissingPackageTranslations( { packagePath } ); + // (2) Find all translation files (*.po files). - const translationsFiles = glob.sync( upath.join( packagePath, TRANSLATIONS_FILES_PATTERN ) ); + const translationFilePaths = glob.sync( upath.join( packagePath, TRANSLATION_FILES_PATH, '*.po' ) ); // Then, for each translation file in a package: - for ( const translationsFile of translationsFiles ) { - const translations = PO.parse( fs.readFileSync( translationsFile, 'utf-8' ) ); + for ( const translationFilePath of translationFilePaths ) { + const translations = PO.parse( fs.readFileSync( translationFilePath, 'utf-8' ) ); // (2.1) Remove unused translations. translations.items = translations.items.filter( item => contextContent[ item.msgid ] ); @@ -148,22 +157,46 @@ function updatePackageTranslations( { packageContexts, sourceMessages } ) { item.msgctxt = contextContent[ message.id ]; item.msgid = message.string; - item.msgstr.push( message.string ); + item.msgstr.push( '' ); if ( message.plural ) { item.msgid_plural = message.plural; - item.msgstr.push( ...Array( numberOfPluralForms - 1 ).fill( message.plural ) ); + item.msgstr.push( ...Array( numberOfPluralForms - 1 ).fill( '' ) ); } return item; } ) ); - fs.writeFileSync( translationsFile, cleanPoFileContent( translations.toString() ), 'utf-8' ); + fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); } } } +/** + * @param {object} options + * @param {string} options.packagePath Path to the package to check for missing translations. + */ +function createMissingPackageTranslations( { packagePath } ) { + for ( const { localeCode, languageCode, languageFileName } of getLanguages() ) { + const translationFilePath = upath.join( packagePath, TRANSLATION_FILES_PATH, `${ languageFileName }.po` ); + + if ( fs.existsSync( translationFilePath ) ) { + continue; + } + + const translations = PO.parse( fs.readFileSync( EMPTY_TRANSLATION_TEMPLATE, 'utf-8' ) ); + + translations.headers.Language = localeCode; + translations.headers[ 'Plural-Forms' ] = [ + `nplurals=${ getNPlurals( languageCode ) };`, + `plural=${ getFormula( languageCode ) };` + ].join( ' ' ); + + fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + } +} + /** * @param {object} options * @param {Array.} options.packageContexts An array of language contexts. diff --git a/packages/ckeditor5-dev-translations/lib/templates/empty.po b/packages/ckeditor5-dev-translations/lib/templates/empty.po new file mode 100644 index 000000000..57d096967 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/templates/empty.po @@ -0,0 +1,12 @@ +# Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. +# +# !!! IMPORTANT !!! +# +# Before you edit this file, please keep in mind that contributing to the project +# translations is possible ONLY via the Transifex online service. +# +# To submit your translations, visit https://www.transifex.com/ckeditor/ckeditor5. +# +# To learn more, check out the official contributor's guide: +# https://ckeditor.com/docs/ckeditor5/latest/framework/guides/contributing/contributing.html +# diff --git a/packages/ckeditor5-dev-translations/package.json b/packages/ckeditor5-dev-translations/package.json index 57e2b767b..b9642ef98 100644 --- a/packages/ckeditor5-dev-translations/package.json +++ b/packages/ckeditor5-dev-translations/package.json @@ -30,6 +30,7 @@ "glob": "^10.0.0", "rimraf": "^5.0.0", "webpack-sources": "^3.0.0", + "plural-forms": "^0.5.5", "pofile": "^1.0.9", "upath": "^2.0.1" }, From d037454018e0a5cd488e0e1c98746ab5fda1f61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Thu, 10 Oct 2024 07:25:02 +0200 Subject: [PATCH 03/10] Split utils to own modules. Added tests. --- .../lib/synchronizetranslations.js | 153 ++--------------- .../templates/{empty.po => translation.po} | 0 .../lib/utils/constants.js | 7 + .../utils/createmissingpackagetranslations.js | 44 +++++ .../lib/{ => utils}/getlanguages.js | 2 +- .../lib/utils/getpackagecontexts.js | 33 ++++ .../lib/utils/getsourcemessages.js | 32 ++++ .../lib/utils/updatepackagetranslations.js | 64 +++++++ .../tests/utils/constants.js | 17 ++ .../utils/createmissingpackagetranslations.js | 98 +++++++++++ .../tests/utils/getlanguages.js | 57 +++++++ .../tests/utils/getpackagecontexts.js | 116 +++++++++++++ .../tests/utils/getsourcemessages.js | 92 ++++++++++ .../tests/utils/updatepackagetranslations.js | 158 ++++++++++++++++++ 14 files changed, 732 insertions(+), 141 deletions(-) rename packages/ckeditor5-dev-translations/lib/templates/{empty.po => translation.po} (100%) create mode 100644 packages/ckeditor5-dev-translations/lib/utils/constants.js create mode 100644 packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js rename packages/ckeditor5-dev-translations/lib/{ => utils}/getlanguages.js (98%) create mode 100644 packages/ckeditor5-dev-translations/lib/utils/getpackagecontexts.js create mode 100644 packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js create mode 100644 packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/constants.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/getlanguages.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/getpackagecontexts.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js index 3edccb774..f3d129990 100644 --- a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -4,28 +4,20 @@ */ import upath from 'upath'; -import fs from 'fs-extra'; -import PO from 'pofile'; -import { glob } from 'glob'; -import { fileURLToPath } from 'url'; -import { getNPlurals, getFormula } from 'plural-forms'; import { logger } from '@ckeditor/ckeditor5-dev-utils'; -import cleanPoFileContent from './cleanpofilecontent.js'; -import findMessages from './findmessages.js'; -import { getLanguages } from './getlanguages.js'; - -const __filename = fileURLToPath( import.meta.url ); -const __dirname = upath.dirname( __filename ); - -const EMPTY_TRANSLATION_TEMPLATE = upath.join( __dirname, 'templates', 'empty.po' ); -const CONTEXT_FILE_PATH = upath.join( 'lang', 'contexts.json' ); -const TRANSLATION_FILES_PATH = upath.join( 'lang', 'translations' ); +import getPackageContexts from './utils/getpackagecontexts.js'; +import { CONTEXT_FILE_PATH } from './utils/constants.js'; +import getSourceMessages from './utils/getsourcemessages.js'; +import updatePackageTranslations from './utils/updatepackagetranslations.js'; /** * Synchronizes translations in provided packages by performing the following steps: - * * Collect all i18n messages from all provided packages by finding `t()` calls. + * * Collect all i18n messages from all provided packages by finding `t()` calls in source files. * * Detect if translation context is valid, i.e. whether there is no missing, unused or duplicated context. - * * If there are no validation errors, update all translation files (*.po files) to be in sync with the context file. + * * If there are no validation errors, update all translation files ("*.po" files) to be in sync with the context file: + * * unused translation entries are removed, + * * missing translation entries are added with empty string, + * * missing translation files are created for languages that do not have own "*.po" file yet. * * @param {object} options * @param {Array.} options.sourceFiles An array of source files that contain messages to translate. @@ -74,129 +66,6 @@ export default function synchronizeTranslations( options ) { log.info( '✨ Done.' ); } -/** - * @param {object} options - * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. - * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. - * @returns {Array.} - */ -function getPackageContexts( { packagePaths, corePackagePath } ) { - // Add path to the core package if not included in the package paths. - // The core package contains common contexts shared between other packages. - if ( !packagePaths.includes( corePackagePath ) ) { - packagePaths.push( corePackagePath ); - } - - return packagePaths.map( packagePath => { - const contextFilePath = upath.join( packagePath, CONTEXT_FILE_PATH ); - const contextContent = fs.existsSync( contextFilePath ) ? fs.readJsonSync( contextFilePath ) : []; - - return { - contextContent, - contextFilePath, - packagePath - }; - } ); -} - -/** - * @param {object} options - * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. - * @param {Array.} options.sourceFiles An array of source files that contain messages to translate. - * @param {Function} options.onErrorCallback Called when there is an error with parsing the source files. - * @returns {Array.} - */ -function getSourceMessages( { packagePaths, sourceFiles, onErrorCallback } ) { - return sourceFiles.flatMap( filePath => { - const fileContent = fs.readFileSync( filePath, 'utf-8' ); - const packagePath = packagePaths.find( packagePath => filePath.includes( packagePath ) ); - const sourceMessages = []; - - const onMessageCallback = message => { - sourceMessages.push( { filePath, packagePath, ...message } ); - }; - - findMessages( fileContent, filePath, onMessageCallback, onErrorCallback ); - - return sourceMessages; - } ); -} - -/** - * @param {object} options - * @param {Array.} options.packageContexts An array of language contexts. - * @param {Array.} options.sourceMessages An array of i18n source messages. - */ -function updatePackageTranslations( { packageContexts, sourceMessages } ) { - // For each package: - for ( const { packagePath, contextContent } of packageContexts ) { - // (1) Find all source messages that are defined in the language context. - const sourceMessagesForPackage = Object.keys( contextContent ) - .map( messageId => sourceMessages.find( message => message.id === messageId ) ) - .filter( Boolean ); - - createMissingPackageTranslations( { packagePath } ); - - // (2) Find all translation files (*.po files). - const translationFilePaths = glob.sync( upath.join( packagePath, TRANSLATION_FILES_PATH, '*.po' ) ); - - // Then, for each translation file in a package: - for ( const translationFilePath of translationFilePaths ) { - const translations = PO.parse( fs.readFileSync( translationFilePath, 'utf-8' ) ); - - // (2.1) Remove unused translations. - translations.items = translations.items.filter( item => contextContent[ item.msgid ] ); - - // (2.2) Add missing translations. - translations.items.push( - ...sourceMessagesForPackage - .filter( message => !translations.items.find( item => item.msgid === message.id ) ) - .map( message => { - const numberOfPluralForms = PO.parsePluralForms( translations.headers[ 'Plural-Forms' ] ).nplurals; - const item = new PO.Item( { nplurals: numberOfPluralForms } ); - - item.msgctxt = contextContent[ message.id ]; - item.msgid = message.string; - item.msgstr.push( '' ); - - if ( message.plural ) { - item.msgid_plural = message.plural; - item.msgstr.push( ...Array( numberOfPluralForms - 1 ).fill( '' ) ); - } - - return item; - } ) - ); - - fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); - } - } -} - -/** - * @param {object} options - * @param {string} options.packagePath Path to the package to check for missing translations. - */ -function createMissingPackageTranslations( { packagePath } ) { - for ( const { localeCode, languageCode, languageFileName } of getLanguages() ) { - const translationFilePath = upath.join( packagePath, TRANSLATION_FILES_PATH, `${ languageFileName }.po` ); - - if ( fs.existsSync( translationFilePath ) ) { - continue; - } - - const translations = PO.parse( fs.readFileSync( EMPTY_TRANSLATION_TEMPLATE, 'utf-8' ) ); - - translations.headers.Language = localeCode; - translations.headers[ 'Plural-Forms' ] = [ - `nplurals=${ getNPlurals( languageCode ) };`, - `plural=${ getFormula( languageCode ) };` - ].join( ' ' ); - - fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); - } -} - /** * @param {object} options * @param {Array.} options.packageContexts An array of language contexts. @@ -254,6 +123,10 @@ function assertAllContextUsed( { packageContexts, sourceMessages, corePackagePat return !sourceMessageIds.includes( messageId ); } + if ( !sourceMessageIdsGroupedByPackage[ packagePath ] ) { + return true; + } + return !sourceMessageIdsGroupedByPackage[ packagePath ].includes( messageId ); } ) .filter( ( { packagePath } ) => { diff --git a/packages/ckeditor5-dev-translations/lib/templates/empty.po b/packages/ckeditor5-dev-translations/lib/templates/translation.po similarity index 100% rename from packages/ckeditor5-dev-translations/lib/templates/empty.po rename to packages/ckeditor5-dev-translations/lib/templates/translation.po diff --git a/packages/ckeditor5-dev-translations/lib/utils/constants.js b/packages/ckeditor5-dev-translations/lib/utils/constants.js new file mode 100644 index 000000000..806583893 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/constants.js @@ -0,0 +1,7 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +export const CONTEXT_FILE_PATH = 'lang/contexts.json'; +export const TRANSLATION_FILES_PATH = 'lang/translations'; diff --git a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js new file mode 100644 index 000000000..cd668759f --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js @@ -0,0 +1,44 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import upath from 'upath'; +import fs from 'fs-extra'; +import PO from 'pofile'; +import { fileURLToPath } from 'url'; +import { getNPlurals, getFormula } from 'plural-forms'; +import cleanPoFileContent from '../cleanpofilecontent.js'; +import getLanguages from './getlanguages.js'; +import { TRANSLATION_FILES_PATH } from './constants.js'; + +const __filename = fileURLToPath( import.meta.url ); +const __dirname = upath.dirname( __filename ); + +const TRANSLATION_TEMPLATE_PATH = upath.join( __dirname, '../templates/translation.po' ); + +/** + * @param {object} options + * @param {string} options.packagePath Path to the package to check for missing translations. + */ +export default function createMissingPackageTranslations( { packagePath } ) { + const translationsTemplate = fs.readFileSync( TRANSLATION_TEMPLATE_PATH, 'utf-8' ); + + for ( const { localeCode, languageCode, languageFileName } of getLanguages() ) { + const translationFilePath = upath.join( packagePath, TRANSLATION_FILES_PATH, `${ languageFileName }.po` ); + + if ( fs.existsSync( translationFilePath ) ) { + continue; + } + + const translations = PO.parse( translationsTemplate ); + + translations.headers.Language = localeCode; + translations.headers[ 'Plural-Forms' ] = [ + `nplurals=${ getNPlurals( languageCode ) };`, + `plural=${ getFormula( languageCode ) };` + ].join( ' ' ); + + fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + } +} diff --git a/packages/ckeditor5-dev-translations/lib/getlanguages.js b/packages/ckeditor5-dev-translations/lib/utils/getlanguages.js similarity index 98% rename from packages/ckeditor5-dev-translations/lib/getlanguages.js rename to packages/ckeditor5-dev-translations/lib/utils/getlanguages.js index 364af9afd..dc6b27211 100644 --- a/packages/ckeditor5-dev-translations/lib/getlanguages.js +++ b/packages/ckeditor5-dev-translations/lib/utils/getlanguages.js @@ -87,7 +87,7 @@ const LOCALES_FILENAME_MAP = { /** * @returns {Array.} */ -export function getLanguages() { +export default function getLanguages() { return SUPPORTED_LOCALES.map( localeCode => { const languageCode = localeCode.split( /[-_@]/ )[ 0 ]; const languageFileName = LOCALES_FILENAME_MAP[ localeCode ] || localeCode.toLowerCase().replace( /[^a-z0-9]+/, '-' ); diff --git a/packages/ckeditor5-dev-translations/lib/utils/getpackagecontexts.js b/packages/ckeditor5-dev-translations/lib/utils/getpackagecontexts.js new file mode 100644 index 000000000..fca663fc0 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/getpackagecontexts.js @@ -0,0 +1,33 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import upath from 'upath'; +import fs from 'fs-extra'; +import { CONTEXT_FILE_PATH } from './constants.js'; + +/** + * @param {object} options + * @param {Array.} options.packagePaths An array of paths to packages, which will be used to find message contexts. + * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. + * @returns {Array.} + */ +export default function getPackageContexts( { packagePaths, corePackagePath } ) { + // Add path to the core package if not included in the package paths. + // The core package contains common contexts shared between other packages. + if ( !packagePaths.includes( corePackagePath ) ) { + packagePaths.push( corePackagePath ); + } + + return packagePaths.map( packagePath => { + const contextFilePath = upath.join( packagePath, CONTEXT_FILE_PATH ); + const contextContent = fs.existsSync( contextFilePath ) ? fs.readJsonSync( contextFilePath ) : {}; + + return { + contextContent, + contextFilePath, + packagePath + }; + } ); +} diff --git a/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js b/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js new file mode 100644 index 000000000..ebe7fd7ec --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js @@ -0,0 +1,32 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import fs from 'fs-extra'; +import findMessages from '../findmessages.js'; + +/** + * @param {object} options + * @param {Array.} options.packagePaths An array of paths to packages that contain source files with messages to translate. + * @param {Array.} options.sourceFiles An array of source files that contain messages to translate. + * @param {Function} options.onErrorCallback Called when there is an error with parsing the source files. + * @returns {Array.} + */ +export default function getSourceMessages( { packagePaths, sourceFiles, onErrorCallback } ) { + return sourceFiles + .filter( filePath => packagePaths.some( packagePath => filePath.includes( packagePath ) ) ) + .flatMap( filePath => { + const fileContent = fs.readFileSync( filePath, 'utf-8' ); + const packagePath = packagePaths.find( packagePath => filePath.includes( packagePath ) ); + const sourceMessages = []; + + const onMessageCallback = message => { + sourceMessages.push( { filePath, packagePath, ...message } ); + }; + + findMessages( fileContent, filePath, onMessageCallback, onErrorCallback ); + + return sourceMessages; + } ); +} diff --git a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js new file mode 100644 index 000000000..299fccb79 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js @@ -0,0 +1,64 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import upath from 'upath'; +import fs from 'fs-extra'; +import PO from 'pofile'; +import { glob } from 'glob'; +import cleanPoFileContent from '../cleanpofilecontent.js'; +import createMissingPackageTranslations from './createmissingpackagetranslations.js'; +import { TRANSLATION_FILES_PATH } from './constants.js'; + +/** + * @param {object} options + * @param {Array.} options.packageContexts An array of language contexts. + * @param {Array.} options.sourceMessages An array of i18n source messages. + */ +export default function updatePackageTranslations( { packageContexts, sourceMessages } ) { + // For each package: + for ( const { packagePath, contextContent } of packageContexts ) { + // (1) Create missing translation files for languages that do not have own "*.po" file yet. + createMissingPackageTranslations( { packagePath } ); + + // (2) Find all source messages that are defined in the language context. + const sourceMessagesForPackage = Object.keys( contextContent ) + .map( messageId => sourceMessages.find( message => message.id === messageId ) ) + .filter( Boolean ); + + // (3) Find all translation files ("*.po" files). + const translationFilePaths = glob.sync( upath.join( packagePath, TRANSLATION_FILES_PATH, '*.po' ) ); + + // Then, for each translation file in a package: + for ( const translationFilePath of translationFilePaths ) { + const translations = PO.parse( fs.readFileSync( translationFilePath, 'utf-8' ) ); + + // (3.1) Remove unused translations. + translations.items = translations.items.filter( item => contextContent[ item.msgid ] ); + + // (3.2) Add missing translations. + translations.items.push( + ...sourceMessagesForPackage + .filter( message => !translations.items.find( item => item.msgid === message.id ) ) + .map( message => { + const numberOfPluralForms = PO.parsePluralForms( translations.headers[ 'Plural-Forms' ] ).nplurals; + const item = new PO.Item( { nplurals: numberOfPluralForms } ); + + item.msgctxt = contextContent[ message.id ]; + item.msgid = message.string; + item.msgstr.push( '' ); + + if ( message.plural ) { + item.msgid_plural = message.plural; + item.msgstr.push( ...Array( numberOfPluralForms - 1 ).fill( '' ) ); + } + + return item; + } ) + ); + + fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + } + } +} diff --git a/packages/ckeditor5-dev-translations/tests/utils/constants.js b/packages/ckeditor5-dev-translations/tests/utils/constants.js new file mode 100644 index 000000000..48a43457f --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/constants.js @@ -0,0 +1,17 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { describe, expect, it } from 'vitest'; +import * as constants from '../../lib/utils/constants.js'; + +describe( 'constants', () => { + it( '#CONTEXT_FILE_PATH', () => { + expect( constants.CONTEXT_FILE_PATH ).toBeTypeOf( 'string' ); + } ); + + it( '#TRANSLATION_FILES_PATH', () => { + expect( constants.TRANSLATION_FILES_PATH ).toBeTypeOf( 'string' ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js new file mode 100644 index 000000000..5806b883b --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js @@ -0,0 +1,98 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs-extra'; +import PO from 'pofile'; +import { getNPlurals, getFormula } from 'plural-forms'; +import cleanPoFileContent from '../../lib/cleanpofilecontent.js'; +import getLanguages from '../../lib/utils/getlanguages.js'; +import createMissingPackageTranslations from '../../lib/utils/createmissingpackagetranslations.js'; + +vi.mock( 'fs-extra' ); +vi.mock( 'pofile' ); +vi.mock( 'plural-forms' ); +vi.mock( '../../lib/cleanpofilecontent.js' ); +vi.mock( '../../lib/utils/getlanguages.js' ); + +describe( 'createMissingPackageTranslations()', () => { + let translations; + + beforeEach( () => { + translations = { + headers: {}, + toString: () => 'Raw PO file content.' + }; + + vi.mocked( PO.parse ).mockReturnValue( translations ); + + vi.mocked( getNPlurals ).mockReturnValue( 4 ); + vi.mocked( getFormula ).mockReturnValue( 'example plural formula' ); + + vi.mocked( getLanguages ).mockReturnValue( [ + { localeCode: 'en', languageCode: 'en', languageFileName: 'en' }, + { localeCode: 'zh_TW', languageCode: 'zh', languageFileName: 'zh-tw' } + ] ); + + vi.mocked( cleanPoFileContent ).mockReturnValue( 'Clean PO file content.' ); + + vi.mocked( fs.existsSync ).mockImplementation( path => { + if ( path === 'packages/ckeditor5-foo/lang/translations/en.po' ) { + return true; + } + + return false; + } ); + + vi.mocked( fs.readFileSync ).mockReturnValue( [ + `# Copyright (c) 2003-${ new Date().getFullYear() }, CKSource Holding sp. z o.o. All rights reserved.`, + '', + '# Example translation file header.', + '' + ].join( '\n' ) ); + } ); + + it( 'should be a function', () => { + expect( createMissingPackageTranslations ).toBeInstanceOf( Function ); + } ); + + it( 'should check if translation files exist for each language', () => { + createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + + expect( fs.existsSync ).toHaveBeenCalledTimes( 2 ); + expect( fs.existsSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/en.po' ); + expect( fs.existsSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/zh-tw.po' ); + } ); + + it( 'should create missing translation files from the template', () => { + createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + + expect( fs.readFileSync ).toHaveBeenCalledTimes( 1 ); + expect( fs.readFileSync ).toHaveBeenCalledWith( + expect.stringMatching( 'ckeditor5-dev/packages/ckeditor5-dev-translations/lib/templates/translation.po' ), + 'utf-8' + ); + + expect( getNPlurals ).toHaveBeenCalledWith( 'zh' ); + expect( getFormula ).toHaveBeenCalledWith( 'zh' ); + + expect( translations.headers.Language ).toEqual( 'zh_TW' ); + expect( translations.headers[ 'Plural-Forms' ] ).toEqual( 'nplurals=4; plural=example plural formula;' ); + } ); + + it( 'should save missing translation files on filesystem after cleaning the content', () => { + createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + + expect( cleanPoFileContent ).toHaveBeenCalledTimes( 1 ); + expect( cleanPoFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); + + expect( fs.writeFileSync ).toHaveBeenCalledTimes( 1 ); + expect( fs.writeFileSync ).toHaveBeenCalledWith( + 'packages/ckeditor5-foo/lang/translations/zh-tw.po', + 'Clean PO file content.', + 'utf-8' + ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/getlanguages.js b/packages/ckeditor5-dev-translations/tests/utils/getlanguages.js new file mode 100644 index 000000000..fbede17a3 --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/getlanguages.js @@ -0,0 +1,57 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { describe, expect, it } from 'vitest'; +import getLanguages from '../../lib/utils/getlanguages.js'; + +describe( 'getLanguages()', () => { + it( 'should be a function', () => { + expect( getLanguages ).toBeInstanceOf( Function ); + } ); + + it( 'should return an array of languages', () => { + const languages = getLanguages(); + + expect( languages ).toBeInstanceOf( Array ); + expect( languages[ 0 ] ).toEqual( expect.objectContaining( { + localeCode: expect.any( String ), + languageCode: expect.any( String ), + languageFileName: expect.any( String ) + } ) ); + } ); + + it( 'should return Polish language', () => { + const languages = getLanguages(); + const languagePolish = languages.find( item => item.localeCode === 'pl' ); + + expect( languagePolish ).toEqual( { + localeCode: 'pl', + languageCode: 'pl', + languageFileName: 'pl' + } ); + } ); + + it( 'should normalize language if it contains special characters', () => { + const languages = getLanguages(); + const languageSerbianLatin = languages.find( l => l.localeCode === 'sr@latin' ); + + expect( languageSerbianLatin ).toEqual( { + localeCode: 'sr@latin', + languageCode: 'sr', + languageFileName: 'sr-latin' + } ); + } ); + + it( 'should use predefined filename if defined', () => { + const languages = getLanguages(); + const languageChineseTaiwan = languages.find( l => l.localeCode === 'zh_TW' ); + + expect( languageChineseTaiwan ).toEqual( { + localeCode: 'zh_TW', + languageCode: 'zh', + languageFileName: 'zh' + } ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/getpackagecontexts.js b/packages/ckeditor5-dev-translations/tests/utils/getpackagecontexts.js new file mode 100644 index 000000000..341340371 --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/getpackagecontexts.js @@ -0,0 +1,116 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs-extra'; +import getPackageContexts from '../../lib/utils/getpackagecontexts.js'; + +vi.mock( 'fs-extra' ); + +describe( 'getPackageContexts()', () => { + let defaultOptions; + + beforeEach( () => { + defaultOptions = { + packagePaths: [ 'packages/ckeditor5-foo' ], + corePackagePath: 'packages/ckeditor5-core' + }; + + vi.mocked( fs.existsSync ).mockImplementation( path => { + if ( path === 'packages/ckeditor5-foo/lang/contexts.json' ) { + return true; + } + + if ( path === 'packages/ckeditor5-core/lang/contexts.json' ) { + return true; + } + + return false; + } ); + + vi.mocked( fs.readJsonSync ).mockImplementation( path => { + if ( path === 'packages/ckeditor5-foo/lang/contexts.json' ) { + return { + 'Text ID in "ckeditor5-foo"': 'Example context for text in "ckeditor5-foo".' + }; + } + + if ( path === 'packages/ckeditor5-core/lang/contexts.json' ) { + return { + 'Text ID in "ckeditor5-core"': 'Example context for text in "ckeditor5-core".' + }; + } + + throw new Error( `ENOENT: no such file or directory, open ${ path }` ); + } ); + } ); + + it( 'should be a function', () => { + expect( getPackageContexts ).toBeInstanceOf( Function ); + } ); + + it( 'should read existing context files from packages (including core package)', () => { + getPackageContexts( defaultOptions ); + + expect( defaultOptions.packagePaths ).toEqual( expect.arrayContaining( [ 'packages/ckeditor5-core' ] ) ); + + expect( fs.existsSync ).toHaveBeenCalledTimes( 2 ); + expect( fs.existsSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/contexts.json' ); + expect( fs.existsSync ).toHaveBeenCalledWith( 'packages/ckeditor5-core/lang/contexts.json' ); + + expect( fs.readJsonSync ).toHaveBeenCalledTimes( 2 ); + expect( fs.readJsonSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/contexts.json' ); + expect( fs.readJsonSync ).toHaveBeenCalledWith( 'packages/ckeditor5-core/lang/contexts.json' ); + } ); + + it( 'should not duplicate core package if it is already included in the packages', () => { + defaultOptions.packagePaths.push( 'packages/ckeditor5-core' ); + + getPackageContexts( defaultOptions ); + + expect( defaultOptions.packagePaths ).toHaveLength( 2 ); + } ); + + it( 'should return package contexts', () => { + const result = getPackageContexts( defaultOptions ); + + expect( result ).toBeInstanceOf( Array ); + expect( result ).toHaveLength( 2 ); + expect( result ).toEqual( expect.arrayContaining( [ + { + contextContent: { + 'Text ID in "ckeditor5-foo"': 'Example context for text in "ckeditor5-foo".' + }, + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + packagePath: 'packages/ckeditor5-foo' + } + ] ) ); + expect( result ).toEqual( expect.arrayContaining( [ + { + contextContent: { + 'Text ID in "ckeditor5-core"': 'Example context for text in "ckeditor5-core".' + }, + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + packagePath: 'packages/ckeditor5-core' + } + ] ) ); + } ); + + it( 'should return empty context if package does not have context file', () => { + defaultOptions.packagePaths.push( 'packages/ckeditor5-bar' ); + + const result = getPackageContexts( defaultOptions ); + + expect( result ).toBeInstanceOf( Array ); + expect( result ).toHaveLength( 3 ); + expect( result ).toEqual( expect.arrayContaining( [ + { + contextContent: {}, + contextFilePath: 'packages/ckeditor5-bar/lang/contexts.json', + packagePath: 'packages/ckeditor5-bar' + } + ] ) ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js b/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js new file mode 100644 index 000000000..47ba5ffc5 --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js @@ -0,0 +1,92 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs-extra'; +import findMessages from '../../lib/findmessages.js'; +import getSourceMessages from '../../lib/utils/getsourcemessages.js'; + +vi.mock( 'fs-extra' ); +vi.mock( '../../lib/findmessages.js' ); + +describe( 'getSourceMessages()', () => { + let defaultOptions; + + beforeEach( () => { + defaultOptions = { + packagePaths: [ 'packages/ckeditor5-foo' ], + sourceFiles: [ + '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + '/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts' + ], + onErrorCallback: vi.fn() + }; + + vi.mocked( fs.readFileSync ).mockImplementation( path => { + if ( path === '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' ) { + return 'Content from file.ts.'; + } + + throw new Error( `ENOENT: no such file or directory, open ${ path }` ); + } ); + } ); + + it( 'should be a function', () => { + expect( getSourceMessages ).toBeInstanceOf( Function ); + } ); + + it( 'should read source files only from provided packages', () => { + getSourceMessages( defaultOptions ); + + expect( fs.readFileSync ).toHaveBeenCalledTimes( 1 ); + expect( fs.readFileSync ).toHaveBeenCalledWith( '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', 'utf-8' ); + } ); + + it( 'should find messages from source files', () => { + getSourceMessages( defaultOptions ); + + expect( findMessages ).toHaveBeenCalledTimes( 1 ); + expect( findMessages ).toHaveBeenCalledWith( + 'Content from file.ts.', + '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + expect.any( Function ), + defaultOptions.onErrorCallback + ); + } ); + + it( 'should return found messages from source files', () => { + vi.mocked( findMessages ).mockImplementation( ( fileContent, filePath, onMessageCallback ) => { + onMessageCallback( { id: 'id1', string: 'Example message 1.' } ); + onMessageCallback( { id: 'id2', string: 'Example message 2.' } ); + } ); + + const result = getSourceMessages( defaultOptions ); + + expect( result ).toEqual( [ + { + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + packagePath: 'packages/ckeditor5-foo', + id: 'id1', + string: 'Example message 1.' + }, + { + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + packagePath: 'packages/ckeditor5-foo', + id: 'id2', + string: 'Example message 2.' + } + ] ); + } ); + + it( 'should call error callback in case of an error', () => { + vi.mocked( findMessages ).mockImplementation( ( fileContent, filePath, onMessageCallback, onErrorCallback ) => { + onErrorCallback( 'Example problem has been detected.' ); + } ); + + getSourceMessages( defaultOptions ); + + expect( defaultOptions.onErrorCallback ).toHaveBeenCalledWith( 'Example problem has been detected.' ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js new file mode 100644 index 000000000..c0cbf7aea --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js @@ -0,0 +1,158 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'fs-extra'; +import PO from 'pofile'; +import { glob } from 'glob'; +import cleanPoFileContent from '../../lib/cleanpofilecontent.js'; +import createMissingPackageTranslations from '../../lib/utils/createmissingpackagetranslations.js'; +import updatePackageTranslations from '../../lib/utils/updatepackagetranslations.js'; + +vi.mock( 'fs-extra' ); +vi.mock( 'pofile' ); +vi.mock( 'glob' ); +vi.mock( '../../lib/utils/createmissingpackagetranslations.js' ); +vi.mock( '../../lib/cleanpofilecontent.js' ); + +describe( 'updatePackageTranslations()', () => { + let defaultOptions, translations, stubs; + + beforeEach( () => { + defaultOptions = { + packageContexts: [ + { + packagePath: 'packages/ckeditor5-foo', + contextContent: { + 'id1': 'Context for example message 1', + 'id2': 'Context for example message 2' + } + } + ], + sourceMessages: [ + { + id: 'id1', + string: 'Example message 1' + }, + { + id: 'id2', + string: 'Example message 2', + plural: 'Example message 2 - plural form' + } + ] + }; + + translations = { + headers: {}, + items: [ + { msgid: 'id1' }, + { msgid: 'id2' } + ], + toString: () => 'Raw PO file content.' + }; + + stubs = { + poItemConstructor: vi.fn() + }; + + vi.mocked( PO.parse ).mockReturnValue( translations ); + + vi.mocked( PO.parsePluralForms ).mockReturnValue( { nplurals: 4 } ); + + vi.mocked( PO.Item ).mockImplementation( () => new class { + constructor( ...args ) { + stubs.poItemConstructor( ...args ); + + this.msgid = ''; + this.msgctxt = ''; + this.msgstr = []; + this.msgid_plural = ''; + } + }() ); + + vi.mocked( glob.sync ).mockImplementation( pattern => [ pattern.replace( '*', 'en' ) ] ); + + vi.mocked( cleanPoFileContent ).mockReturnValue( 'Clean PO file content.' ); + } ); + + it( 'should be a function', () => { + expect( updatePackageTranslations ).toBeInstanceOf( Function ); + } ); + + it( 'should create missing translations', () => { + updatePackageTranslations( defaultOptions ); + + expect( createMissingPackageTranslations ).toHaveBeenCalledTimes( 1 ); + expect( createMissingPackageTranslations ).toHaveBeenCalledWith( { packagePath: 'packages/ckeditor5-foo' } ); + } ); + + it( 'should search for translation files', () => { + updatePackageTranslations( defaultOptions ); + + expect( glob.sync ).toHaveBeenCalledTimes( 1 ); + expect( glob.sync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/*.po' ); + } ); + + it( 'should parse each translation file', () => { + vi.mocked( glob.sync ).mockImplementation( pattern => { + return [ 'en', 'pl' ].map( language => pattern.replace( '*', language ) ); + } ); + + updatePackageTranslations( defaultOptions ); + + expect( fs.readFileSync ).toHaveBeenCalledTimes( 2 ); + expect( fs.readFileSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/en.po', 'utf-8' ); + expect( fs.readFileSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/pl.po', 'utf-8' ); + } ); + + it( 'should remove unused translations', () => { + translations.items.push( + { msgid: 'id3' }, + { msgid: 'id4' } + ); + + updatePackageTranslations( defaultOptions ); + + expect( translations.items ).toEqual( [ + { msgid: 'id1' }, + { msgid: 'id2' } + ] ); + } ); + + it( 'should add missing translations', () => { + translations.items = []; + + updatePackageTranslations( defaultOptions ); + + expect( translations.items ).toEqual( [ + { + msgid: 'Example message 1', + msgctxt: 'Context for example message 1', + msgid_plural: '', + msgstr: [ '' ] + }, + { + msgid: 'Example message 2', + msgctxt: 'Context for example message 2', + msgid_plural: 'Example message 2 - plural form', + msgstr: [ '', '', '', '' ] + } + ] ); + } ); + + it( 'should save updated translation files on filesystem after cleaning the content', () => { + updatePackageTranslations( defaultOptions ); + + expect( cleanPoFileContent ).toHaveBeenCalledTimes( 1 ); + expect( cleanPoFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); + + expect( fs.writeFileSync ).toHaveBeenCalledTimes( 1 ); + expect( fs.writeFileSync ).toHaveBeenCalledWith( + 'packages/ckeditor5-foo/lang/translations/en.po', + 'Clean PO file content.', + 'utf-8' + ); + } ); +} ); From 2a4486b8968357c91c433918e60acfd69350c5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Thu, 10 Oct 2024 07:52:56 +0200 Subject: [PATCH 04/10] Fixed path in test to match the template PO file. --- .../tests/utils/createmissingpackagetranslations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js index 5806b883b..15b81af92 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js @@ -71,7 +71,7 @@ describe( 'createMissingPackageTranslations()', () => { expect( fs.readFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.readFileSync ).toHaveBeenCalledWith( - expect.stringMatching( 'ckeditor5-dev/packages/ckeditor5-dev-translations/lib/templates/translation.po' ), + expect.stringMatching( 'ckeditor5-dev-translations/lib/templates/translation.po' ), 'utf-8' ); From 13c42fa4f7e131d4b763c05c9d874e6f866268bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Thu, 10 Oct 2024 13:42:38 +0200 Subject: [PATCH 05/10] Tests. --- .../lib/synchronizetranslations.js | 2 +- .../tests/synchronizetranslations.js | 781 ++++++++++++++++++ 2 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 packages/ckeditor5-dev-translations/tests/synchronizetranslations.js diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js index f3d129990..a79de3e58 100644 --- a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -16,7 +16,7 @@ import updatePackageTranslations from './utils/updatepackagetranslations.js'; * * Detect if translation context is valid, i.e. whether there is no missing, unused or duplicated context. * * If there are no validation errors, update all translation files ("*.po" files) to be in sync with the context file: * * unused translation entries are removed, - * * missing translation entries are added with empty string, + * * missing translation entries are added with empty string as the message translation, * * missing translation files are created for languages that do not have own "*.po" file yet. * * @param {object} options diff --git a/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js new file mode 100644 index 000000000..9d8bab3c6 --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js @@ -0,0 +1,781 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import getPackageContexts from '../lib/utils/getpackagecontexts.js'; +import getSourceMessages from '../lib/utils/getsourcemessages.js'; +import updatePackageTranslations from '../lib/utils/updatepackagetranslations.js'; +import synchronizeTranslations from '../lib/synchronizetranslations.js'; + +const stubs = vi.hoisted( () => { + return { + logger: { + info: vi.fn(), + error: vi.fn() + } + }; +} ); + +vi.mock( '@ckeditor/ckeditor5-dev-utils', () => ( { + logger: vi.fn( () => stubs.logger ) +} ) ); +vi.mock( '../lib/utils/getpackagecontexts.js' ); +vi.mock( '../lib/utils/getsourcemessages.js' ); +vi.mock( '../lib/utils/updatepackagetranslations.js' ); + +describe( 'synchronizeTranslations()', () => { + let defaultOptions; + + beforeEach( () => { + defaultOptions = { + sourceFiles: [ + '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + '/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts' + ], + packagePaths: [ + 'packages/ckeditor5-foo', + 'packages/ckeditor5-bar' + ], + corePackagePath: 'packages/ckeditor5-core', + ignoreUnusedCorePackageContexts: false + }; + + vi.mocked( getPackageContexts ).mockReturnValue( [] ); + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + vi.spyOn( process, 'exit' ).mockImplementation( () => {} ); + } ); + + it( 'should be a function', () => { + expect( synchronizeTranslations ).toBeInstanceOf( Function ); + } ); + + it( 'should load translations contexts', () => { + synchronizeTranslations( defaultOptions ); + + expect( getPackageContexts ).toHaveBeenCalledTimes( 1 ); + expect( getPackageContexts ).toHaveBeenCalledWith( { + packagePaths: [ + 'packages/ckeditor5-foo', + 'packages/ckeditor5-bar' + ], + corePackagePath: 'packages/ckeditor5-core' + } ); + + expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Loading translations contexts...' ); + } ); + + it( 'should load messages from source files', () => { + synchronizeTranslations( defaultOptions ); + + expect( getSourceMessages ).toHaveBeenCalledTimes( 1 ); + expect( getSourceMessages ).toHaveBeenCalledWith( { + packagePaths: [ + 'packages/ckeditor5-foo', + 'packages/ckeditor5-bar' + ], + sourceFiles: [ + '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts', + '/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts' + ], + onErrorCallback: expect.any( Function ) + } ); + + expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Loading messages from source files...' ); + } ); + + it( 'should collect errors when loading messages from source files failed', () => { + vi.mocked( getSourceMessages ).mockImplementation( ( { onErrorCallback } ) => { + onErrorCallback( 'Example error when loading messages from source files.' ); + + return []; + } ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( '🔥 The following errors have been found:' ); + expect( stubs.logger.error ).toHaveBeenCalledWith( ' - Example error when loading messages from source files.' ); + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should synchronize translations files', () => { + synchronizeTranslations( defaultOptions ); + + expect( updatePackageTranslations ).toHaveBeenCalledTimes( 1 ); + expect( updatePackageTranslations ).toHaveBeenCalledWith( { + packageContexts: [], + sourceMessages: [] + } ); + + expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Synchronizing translations files...' ); + } ); + + describe( 'validation', () => { + describe( 'missing context', () => { + it( 'should return no error if there is no missing context (no context, no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "foo", no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "core", no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "foo", message in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "core", message in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "core", message in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-core', + filePath: '/absolute/path/to/packages/ckeditor5-core/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return no error if there is no missing context (context in "foo" and "core", messages in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id2: 'Example message 2.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + }, + { + id: 'id2', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Missing context' ) ); + } ); + + it( 'should return error if there is missing context (no context, message in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Missing context "id1" in "/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should return error if there is missing context (context in "foo", message in "bar")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-bar', + contextFilePath: 'packages/ckeditor5-bar/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-bar', + filePath: '/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Missing context "id1" in "/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should return error if there is missing context (context in "foo", message in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-core', + filePath: '/absolute/path/to/packages/ckeditor5-core/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Missing context "id1" in "/absolute/path/to/packages/ckeditor5-core/src/utils/file.ts".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + } ); + + describe( 'all context used', () => { + it( 'should return no error if all context is used (no context, no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return no error if all context is used (context in "foo", message in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return no error if all context is used (context in "core", message in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return no error if all context is used (context in "core", message in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-core', + filePath: '/absolute/path/to/packages/ckeditor5-core/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return no error if all context is used (context in "foo" and "core", messages in "foo")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id2: 'Example message 2.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + }, + { + id: 'id2', + packagePath: 'packages/ckeditor5-foo', + filePath: '/absolute/path/to/packages/ckeditor5-foo/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return no error if all context is used (context in "core", no message, ignore core)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + defaultOptions.ignoreUnusedCorePackageContexts = true; + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Unused context' ) ); + } ); + + it( 'should return error if there is unused context (context in "foo", no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Unused context "id1" in "packages/ckeditor5-foo/lang/contexts.json".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should return error if there is unused context (context in "foo", message in "bar")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-bar', + contextFilePath: 'packages/ckeditor5-bar/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-bar', + filePath: '/absolute/path/to/packages/ckeditor5-bar/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Unused context "id1" in "packages/ckeditor5-foo/lang/contexts.json".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should return error if there is unused context (context in "foo", message in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [ + { + id: 'id1', + packagePath: 'packages/ckeditor5-core', + filePath: '/absolute/path/to/packages/ckeditor5-core/src/utils/file.ts' + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Unused context "id1" in "packages/ckeditor5-foo/lang/contexts.json".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + + it( 'should return error if there is unused context (context in "core", no message)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Unused context "id1" in "packages/ckeditor5-core/lang/contexts.json".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + } ); + + describe( 'duplicated context', () => { + it( 'should return no error if there is no duplicated context (no context)', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: {} + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Duplicated context' ) ); + } ); + + it( 'should return no error if there is no duplicated context (no context in "foo", context in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: {} + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Duplicated context' ) ); + } ); + + it( 'should return no error if there is no duplicated context (context in "foo", another context in "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id2: 'Example message 2.' + } + } + ] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).not.toHaveBeenCalledWith( expect.stringContaining( 'Duplicated context' ) ); + } ); + + it( 'should return error if there is duplicated context (the same context in "foo" and "core")', () => { + vi.mocked( getPackageContexts ).mockReturnValue( [ + { + packagePath: 'packages/ckeditor5-foo', + contextFilePath: 'packages/ckeditor5-foo/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + }, + { + packagePath: 'packages/ckeditor5-core', + contextFilePath: 'packages/ckeditor5-core/lang/contexts.json', + contextContent: { + id1: 'Example message 1.' + } + } + ] ); + + vi.mocked( getSourceMessages ).mockReturnValue( [] ); + + synchronizeTranslations( defaultOptions ); + + expect( stubs.logger.error ).toHaveBeenCalledWith( + ' - Duplicated context "id1" in ' + + '"packages/ckeditor5-foo/lang/contexts.json", "packages/ckeditor5-core/lang/contexts.json".' + ); + + expect( process.exit ).toHaveBeenCalledWith( 1 ); + } ); + } ); + } ); +} ); From 81dd32b782ca4c1cf08461d0a44709d14a6ce638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Fri, 11 Oct 2024 07:38:52 +0200 Subject: [PATCH 06/10] Added support for "validate only" mode. --- .../lib/synchronizetranslations.js | 11 ++++++- .../utils/createmissingpackagetranslations.js | 2 +- .../lib/utils/updatepackagetranslations.js | 29 ++++++++++++++----- .../tests/synchronizetranslations.js | 11 ++++++- .../utils/createmissingpackagetranslations.js | 4 +-- .../tests/utils/updatepackagetranslations.js | 24 +++++++++++++++ 6 files changed, 69 insertions(+), 12 deletions(-) diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js index a79de3e58..6ed3de197 100644 --- a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -25,13 +25,16 @@ import updatePackageTranslations from './utils/updatepackagetranslations.js'; * @param {string} options.corePackagePath A relative to `process.cwd()` path to the `@ckeditor/ckeditor5-core` package. * @param {boolean} [options.ignoreUnusedCorePackageContexts=false] Whether to skip unused context errors related to * the `@ckeditor/ckeditor5-core` package. + * @param {boolean} [options.validateOnly=false] If set, only validates the translations contexts against the source messages without + * synchronizing the translations. */ export default function synchronizeTranslations( options ) { const { sourceFiles, packagePaths, corePackagePath, - ignoreUnusedCorePackageContexts = false + ignoreUnusedCorePackageContexts = false, + validateOnly = false } = options; const errors = []; @@ -60,6 +63,12 @@ export default function synchronizeTranslations( options ) { process.exit( 1 ); } + if ( validateOnly ) { + log.info( '✨ No errors found.' ); + + return; + } + log.info( '📍 Synchronizing translations files...' ); updatePackageTranslations( { packageContexts, sourceMessages } ); diff --git a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js index cd668759f..aebdb6a59 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js @@ -39,6 +39,6 @@ export default function createMissingPackageTranslations( { packagePath } ) { `plural=${ getFormula( languageCode ) };` ].join( ' ' ); - fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + fs.outputFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); } } diff --git a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js index 299fccb79..f4da79967 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js @@ -19,25 +19,33 @@ import { TRANSLATION_FILES_PATH } from './constants.js'; export default function updatePackageTranslations( { packageContexts, sourceMessages } ) { // For each package: for ( const { packagePath, contextContent } of packageContexts ) { - // (1) Create missing translation files for languages that do not have own "*.po" file yet. + // (1) Skip packages that do not contain language context. + const hasContext = Object.keys( contextContent ).length > 0; + + if ( !hasContext ) { + continue; + } + + // (2) Create missing translation files for languages that do not have own "*.po" file yet. createMissingPackageTranslations( { packagePath } ); - // (2) Find all source messages that are defined in the language context. + // (3) Find all source messages that are defined in the language context. const sourceMessagesForPackage = Object.keys( contextContent ) .map( messageId => sourceMessages.find( message => message.id === messageId ) ) .filter( Boolean ); - // (3) Find all translation files ("*.po" files). + // (4) Find all translation files ("*.po" files). const translationFilePaths = glob.sync( upath.join( packagePath, TRANSLATION_FILES_PATH, '*.po' ) ); // Then, for each translation file in a package: for ( const translationFilePath of translationFilePaths ) { - const translations = PO.parse( fs.readFileSync( translationFilePath, 'utf-8' ) ); + const translationFile = fs.readFileSync( translationFilePath, 'utf-8' ); + const translations = PO.parse( translationFile ); - // (3.1) Remove unused translations. + // (4.1) Remove unused translations. translations.items = translations.items.filter( item => contextContent[ item.msgid ] ); - // (3.2) Add missing translations. + // (4.2) Add missing translations. translations.items.push( ...sourceMessagesForPackage .filter( message => !translations.items.find( item => item.msgid === message.id ) ) @@ -58,7 +66,14 @@ export default function updatePackageTranslations( { packageContexts, sourceMess } ) ); - fs.writeFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + const translationFileUpdated = cleanPoFileContent( translations.toString() ); + + // (4.3) Save translation file only if it has been updated. + if ( translationFile === translationFileUpdated ) { + continue; + } + + fs.writeFileSync( translationFilePath, translationFileUpdated, 'utf-8' ); } } } diff --git a/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js index 9d8bab3c6..9d94442eb 100644 --- a/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js @@ -39,7 +39,8 @@ describe( 'synchronizeTranslations()', () => { 'packages/ckeditor5-bar' ], corePackagePath: 'packages/ckeditor5-core', - ignoreUnusedCorePackageContexts: false + ignoreUnusedCorePackageContexts: false, + validateOnly: false }; vi.mocked( getPackageContexts ).mockReturnValue( [] ); @@ -112,6 +113,14 @@ describe( 'synchronizeTranslations()', () => { expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Synchronizing translations files...' ); } ); + it( 'should not synchronize translations files when validation mode is enabled', () => { + defaultOptions.validateOnly = true; + synchronizeTranslations( defaultOptions ); + + expect( updatePackageTranslations ).not.toHaveBeenCalled(); + expect( stubs.logger.info ).toHaveBeenCalledWith( '✨ No errors found.' ); + } ); + describe( 'validation', () => { describe( 'missing context', () => { it( 'should return no error if there is no missing context (no context, no message)', () => { diff --git a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js index 15b81af92..0a4e77cc1 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js @@ -88,8 +88,8 @@ describe( 'createMissingPackageTranslations()', () => { expect( cleanPoFileContent ).toHaveBeenCalledTimes( 1 ); expect( cleanPoFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); - expect( fs.writeFileSync ).toHaveBeenCalledTimes( 1 ); - expect( fs.writeFileSync ).toHaveBeenCalledWith( + expect( fs.outputFileSync ).toHaveBeenCalledTimes( 1 ); + expect( fs.outputFileSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/zh-tw.po', 'Clean PO file content.', 'utf-8' diff --git a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js index c0cbf7aea..2579bb89c 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js @@ -74,6 +74,8 @@ describe( 'updatePackageTranslations()', () => { vi.mocked( glob.sync ).mockImplementation( pattern => [ pattern.replace( '*', 'en' ) ] ); + vi.mocked( fs.readFileSync ).mockReturnValue( 'Raw PO file content.' ); + vi.mocked( cleanPoFileContent ).mockReturnValue( 'Clean PO file content.' ); } ); @@ -88,6 +90,20 @@ describe( 'updatePackageTranslations()', () => { expect( createMissingPackageTranslations ).toHaveBeenCalledWith( { packagePath: 'packages/ckeditor5-foo' } ); } ); + it( 'should not update any files when package does not contain translation context', () => { + defaultOptions.packageContexts = [ + { + packagePath: 'packages/ckeditor5-foo', + contextContent: {} + } + ]; + + updatePackageTranslations( defaultOptions ); + + expect( createMissingPackageTranslations ).not.toHaveBeenCalled(); + expect( fs.writeFileSync ).not.toHaveBeenCalled(); + } ); + it( 'should search for translation files', () => { updatePackageTranslations( defaultOptions ); @@ -155,4 +171,12 @@ describe( 'updatePackageTranslations()', () => { 'utf-8' ); } ); + + it( 'should not save translation files on filesystem if their content is not updated', () => { + vi.mocked( cleanPoFileContent ).mockImplementation( input => input ); + + updatePackageTranslations( defaultOptions ); + + expect( fs.writeFileSync ).not.toHaveBeenCalled(); + } ); } ); From eb102f33e86d9f9defe1959fa92f428c73eeb691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Fri, 11 Oct 2024 07:54:15 +0200 Subject: [PATCH 07/10] Updated translation file template. --- .../lib/templates/translation.po | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/ckeditor5-dev-translations/lib/templates/translation.po b/packages/ckeditor5-dev-translations/lib/templates/translation.po index 57d096967..5ead76f68 100644 --- a/packages/ckeditor5-dev-translations/lib/templates/translation.po +++ b/packages/ckeditor5-dev-translations/lib/templates/translation.po @@ -1,12 +1,7 @@ # Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. # -# !!! IMPORTANT !!! +# Want to contribute to this file? Submit your changes via a GitHub Pull Request. # -# Before you edit this file, please keep in mind that contributing to the project -# translations is possible ONLY via the Transifex online service. -# -# To submit your translations, visit https://www.transifex.com/ckeditor/ckeditor5. -# -# To learn more, check out the official contributor's guide: -# https://ckeditor.com/docs/ckeditor5/latest/framework/guides/contributing/contributing.html +# Check out the official contributor's guide: +# https://ckeditor.com/docs/ckeditor5/latest/framework/guides/contributing/contributing.html # From be42ba0d61bd589b278fdcd23f165658cd08c6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Fri, 11 Oct 2024 09:08:48 +0200 Subject: [PATCH 08/10] Added support for skipping adding the license header to the newly created translation files. --- .../lib/synchronizetranslations.js | 6 ++- .../lib/utils/cleantranslationfilecontent.js | 24 +++++++++ .../utils/createmissingpackagetranslations.js | 9 ++-- .../lib/utils/updatepackagetranslations.js | 9 ++-- .../tests/synchronizetranslations.js | 21 +++++++- .../utils/cleantranslationfilecontent.js | 51 +++++++++++++++++++ .../utils/createmissingpackagetranslations.js | 37 ++++++++++---- .../tests/utils/updatepackagetranslations.js | 32 +++++++++--- 8 files changed, 160 insertions(+), 29 deletions(-) create mode 100644 packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js create mode 100644 packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js diff --git a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js index 6ed3de197..f0283148c 100644 --- a/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/synchronizetranslations.js @@ -27,6 +27,7 @@ import updatePackageTranslations from './utils/updatepackagetranslations.js'; * the `@ckeditor/ckeditor5-core` package. * @param {boolean} [options.validateOnly=false] If set, only validates the translations contexts against the source messages without * synchronizing the translations. + * @param {boolean} [options.skipLicenseHeader=false] Whether to skip adding the license header to newly created translation files. */ export default function synchronizeTranslations( options ) { const { @@ -34,7 +35,8 @@ export default function synchronizeTranslations( options ) { packagePaths, corePackagePath, ignoreUnusedCorePackageContexts = false, - validateOnly = false + validateOnly = false, + skipLicenseHeader = false } = options; const errors = []; @@ -70,7 +72,7 @@ export default function synchronizeTranslations( options ) { } log.info( '📍 Synchronizing translations files...' ); - updatePackageTranslations( { packageContexts, sourceMessages } ); + updatePackageTranslations( { packageContexts, sourceMessages, skipLicenseHeader } ); log.info( '✨ Done.' ); } diff --git a/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js b/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js new file mode 100644 index 000000000..33907c453 --- /dev/null +++ b/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js @@ -0,0 +1,24 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import PO from 'pofile'; + +/** + * Removes unused headers from the translation file. + * + * @param {string} translationFileContent Content of the translation file. + * @returns {string} + */ +export default function cleanTranslationFileContent( translationFileContent ) { + const translations = PO.parse( translationFileContent ); + + translations.headers = { + Language: translations.headers.Language, + 'Plural-Forms': translations.headers[ 'Plural-Forms' ], + 'Content-Type': 'text/plain; charset=UTF-8' + }; + + return translations.toString(); +} diff --git a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js index aebdb6a59..e90c99015 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js @@ -8,9 +8,9 @@ import fs from 'fs-extra'; import PO from 'pofile'; import { fileURLToPath } from 'url'; import { getNPlurals, getFormula } from 'plural-forms'; -import cleanPoFileContent from '../cleanpofilecontent.js'; import getLanguages from './getlanguages.js'; import { TRANSLATION_FILES_PATH } from './constants.js'; +import cleanTranslationFileContent from './cleantranslationfilecontent.js'; const __filename = fileURLToPath( import.meta.url ); const __dirname = upath.dirname( __filename ); @@ -20,9 +20,10 @@ const TRANSLATION_TEMPLATE_PATH = upath.join( __dirname, '../templates/translati /** * @param {object} options * @param {string} options.packagePath Path to the package to check for missing translations. + * @param {boolean} options.skipLicenseHeader Whether to skip adding the license header to newly created translation files. */ -export default function createMissingPackageTranslations( { packagePath } ) { - const translationsTemplate = fs.readFileSync( TRANSLATION_TEMPLATE_PATH, 'utf-8' ); +export default function createMissingPackageTranslations( { packagePath, skipLicenseHeader } ) { + const translationsTemplate = skipLicenseHeader ? '' : fs.readFileSync( TRANSLATION_TEMPLATE_PATH, 'utf-8' ); for ( const { localeCode, languageCode, languageFileName } of getLanguages() ) { const translationFilePath = upath.join( packagePath, TRANSLATION_FILES_PATH, `${ languageFileName }.po` ); @@ -39,6 +40,6 @@ export default function createMissingPackageTranslations( { packagePath } ) { `plural=${ getFormula( languageCode ) };` ].join( ' ' ); - fs.outputFileSync( translationFilePath, cleanPoFileContent( translations.toString() ), 'utf-8' ); + fs.outputFileSync( translationFilePath, cleanTranslationFileContent( translations.toString() ), 'utf-8' ); } } diff --git a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js index f4da79967..357abffbd 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js @@ -7,16 +7,17 @@ import upath from 'upath'; import fs from 'fs-extra'; import PO from 'pofile'; import { glob } from 'glob'; -import cleanPoFileContent from '../cleanpofilecontent.js'; import createMissingPackageTranslations from './createmissingpackagetranslations.js'; import { TRANSLATION_FILES_PATH } from './constants.js'; +import cleanTranslationFileContent from './cleantranslationfilecontent.js'; /** * @param {object} options * @param {Array.} options.packageContexts An array of language contexts. * @param {Array.} options.sourceMessages An array of i18n source messages. + * @param {boolean} options.skipLicenseHeader Whether to skip adding the license header to newly created translation files. */ -export default function updatePackageTranslations( { packageContexts, sourceMessages } ) { +export default function updatePackageTranslations( { packageContexts, sourceMessages, skipLicenseHeader } ) { // For each package: for ( const { packagePath, contextContent } of packageContexts ) { // (1) Skip packages that do not contain language context. @@ -27,7 +28,7 @@ export default function updatePackageTranslations( { packageContexts, sourceMess } // (2) Create missing translation files for languages that do not have own "*.po" file yet. - createMissingPackageTranslations( { packagePath } ); + createMissingPackageTranslations( { packagePath, skipLicenseHeader } ); // (3) Find all source messages that are defined in the language context. const sourceMessagesForPackage = Object.keys( contextContent ) @@ -66,7 +67,7 @@ export default function updatePackageTranslations( { packageContexts, sourceMess } ) ); - const translationFileUpdated = cleanPoFileContent( translations.toString() ); + const translationFileUpdated = cleanTranslationFileContent( translations.toString() ); // (4.3) Save translation file only if it has been updated. if ( translationFile === translationFileUpdated ) { diff --git a/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js index 9d94442eb..a2693310d 100644 --- a/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/synchronizetranslations.js @@ -40,7 +40,8 @@ describe( 'synchronizeTranslations()', () => { ], corePackagePath: 'packages/ckeditor5-core', ignoreUnusedCorePackageContexts: false, - validateOnly: false + validateOnly: false, + skipLicenseHeader: false }; vi.mocked( getPackageContexts ).mockReturnValue( [] ); @@ -107,7 +108,23 @@ describe( 'synchronizeTranslations()', () => { expect( updatePackageTranslations ).toHaveBeenCalledTimes( 1 ); expect( updatePackageTranslations ).toHaveBeenCalledWith( { packageContexts: [], - sourceMessages: [] + sourceMessages: [], + skipLicenseHeader: false + } ); + + expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Synchronizing translations files...' ); + } ); + + it( 'should synchronize translations files with skipping the license header', () => { + defaultOptions.skipLicenseHeader = true; + + synchronizeTranslations( defaultOptions ); + + expect( updatePackageTranslations ).toHaveBeenCalledTimes( 1 ); + expect( updatePackageTranslations ).toHaveBeenCalledWith( { + packageContexts: [], + sourceMessages: [], + skipLicenseHeader: true } ); expect( stubs.logger.info ).toHaveBeenCalledWith( '📍 Synchronizing translations files...' ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js b/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js new file mode 100644 index 000000000..cbd65c0c2 --- /dev/null +++ b/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js @@ -0,0 +1,51 @@ +/** + * @license Copyright (c) 2003-2024, CKSource Holding sp. z o.o. All rights reserved. + * For licensing, see LICENSE.md. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import PO from 'pofile'; +import cleanTranslationFileContent from '../../lib/utils/cleantranslationfilecontent.js'; + +vi.mock( 'pofile' ); + +describe( 'cleanTranslationFileContent()', () => { + let translations; + + beforeEach( () => { + translations = { + headers: { + 'Project-Id-Version': 'Value from Project-Id-Version', + 'Report-Msgid-Bugs-To': 'Value from Report-Msgid-Bugs-To', + 'POT-Creation-Date': 'Value from POT-Creation-Date', + 'PO-Revision-Date': 'Value from PO-Revision-Date', + 'Last-Translator': 'Value from Last-Translator', + 'Language': 'Value from Language', + 'Language-Team': 'Value from Language-Team', + 'Content-Type': 'Value from Content-Type', + 'Content-Transfer-Encoding': 'Value from Content-Transfer-Encoding', + 'Plural-Forms': 'Value from Plural-Forms' + }, + toString: () => JSON.stringify( translations ) + }; + + vi.mocked( PO.parse ).mockReturnValue( translations ); + } ); + + it( 'should be a function', () => { + expect( cleanTranslationFileContent ).toBeInstanceOf( Function ); + } ); + + it( 'should return translation file without unneeded headers', () => { + const result = cleanTranslationFileContent( 'Example content.' ); + + expect( PO.parse ).toHaveBeenCalledWith( 'Example content.' ); + expect( JSON.parse( result ) ).toEqual( { + headers: { + 'Language': 'Value from Language', + 'Content-Type': 'text/plain; charset=UTF-8', + 'Plural-Forms': 'Value from Plural-Forms' + } + } ); + } ); +} ); diff --git a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js index 0a4e77cc1..00c79d3eb 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js @@ -7,18 +7,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'fs-extra'; import PO from 'pofile'; import { getNPlurals, getFormula } from 'plural-forms'; -import cleanPoFileContent from '../../lib/cleanpofilecontent.js'; +import cleanTranslationFileContent from '../../lib/utils/cleantranslationfilecontent.js'; import getLanguages from '../../lib/utils/getlanguages.js'; import createMissingPackageTranslations from '../../lib/utils/createmissingpackagetranslations.js'; vi.mock( 'fs-extra' ); vi.mock( 'pofile' ); vi.mock( 'plural-forms' ); -vi.mock( '../../lib/cleanpofilecontent.js' ); +vi.mock( '../../lib/utils/cleantranslationfilecontent.js' ); vi.mock( '../../lib/utils/getlanguages.js' ); describe( 'createMissingPackageTranslations()', () => { - let translations; + let translations, defaultOptions; beforeEach( () => { translations = { @@ -26,6 +26,11 @@ describe( 'createMissingPackageTranslations()', () => { toString: () => 'Raw PO file content.' }; + defaultOptions = { + packagePath: 'packages/ckeditor5-foo', + skipLicenseHeader: false + }; + vi.mocked( PO.parse ).mockReturnValue( translations ); vi.mocked( getNPlurals ).mockReturnValue( 4 ); @@ -36,7 +41,7 @@ describe( 'createMissingPackageTranslations()', () => { { localeCode: 'zh_TW', languageCode: 'zh', languageFileName: 'zh-tw' } ] ); - vi.mocked( cleanPoFileContent ).mockReturnValue( 'Clean PO file content.' ); + vi.mocked( cleanTranslationFileContent ).mockReturnValue( 'Clean PO file content.' ); vi.mocked( fs.existsSync ).mockImplementation( path => { if ( path === 'packages/ckeditor5-foo/lang/translations/en.po' ) { @@ -59,7 +64,7 @@ describe( 'createMissingPackageTranslations()', () => { } ); it( 'should check if translation files exist for each language', () => { - createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + createMissingPackageTranslations( defaultOptions ); expect( fs.existsSync ).toHaveBeenCalledTimes( 2 ); expect( fs.existsSync ).toHaveBeenCalledWith( 'packages/ckeditor5-foo/lang/translations/en.po' ); @@ -67,7 +72,7 @@ describe( 'createMissingPackageTranslations()', () => { } ); it( 'should create missing translation files from the template', () => { - createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + createMissingPackageTranslations( defaultOptions ); expect( fs.readFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.readFileSync ).toHaveBeenCalledWith( @@ -82,11 +87,25 @@ describe( 'createMissingPackageTranslations()', () => { expect( translations.headers[ 'Plural-Forms' ] ).toEqual( 'nplurals=4; plural=example plural formula;' ); } ); + it( 'should not read the template if `skipLicenseHeader` flag is set', () => { + defaultOptions.skipLicenseHeader = true; + + createMissingPackageTranslations( defaultOptions ); + + expect( fs.readFileSync ).not.toHaveBeenCalled(); + + expect( getNPlurals ).toHaveBeenCalledWith( 'zh' ); + expect( getFormula ).toHaveBeenCalledWith( 'zh' ); + + expect( translations.headers.Language ).toEqual( 'zh_TW' ); + expect( translations.headers[ 'Plural-Forms' ] ).toEqual( 'nplurals=4; plural=example plural formula;' ); + } ); + it( 'should save missing translation files on filesystem after cleaning the content', () => { - createMissingPackageTranslations( { packagePath: 'packages/ckeditor5-foo' } ); + createMissingPackageTranslations( defaultOptions ); - expect( cleanPoFileContent ).toHaveBeenCalledTimes( 1 ); - expect( cleanPoFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); + expect( cleanTranslationFileContent ).toHaveBeenCalledTimes( 1 ); + expect( cleanTranslationFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); expect( fs.outputFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.outputFileSync ).toHaveBeenCalledWith( diff --git a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js index 2579bb89c..7ba4bba05 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js @@ -7,7 +7,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'fs-extra'; import PO from 'pofile'; import { glob } from 'glob'; -import cleanPoFileContent from '../../lib/cleanpofilecontent.js'; +import cleanTranslationFileContent from '../../lib/utils/cleantranslationfilecontent.js'; import createMissingPackageTranslations from '../../lib/utils/createmissingpackagetranslations.js'; import updatePackageTranslations from '../../lib/utils/updatepackagetranslations.js'; @@ -15,7 +15,7 @@ vi.mock( 'fs-extra' ); vi.mock( 'pofile' ); vi.mock( 'glob' ); vi.mock( '../../lib/utils/createmissingpackagetranslations.js' ); -vi.mock( '../../lib/cleanpofilecontent.js' ); +vi.mock( '../../lib/utils/cleantranslationfilecontent.js' ); describe( 'updatePackageTranslations()', () => { let defaultOptions, translations, stubs; @@ -41,7 +41,8 @@ describe( 'updatePackageTranslations()', () => { string: 'Example message 2', plural: 'Example message 2 - plural form' } - ] + ], + skipLicenseHeader: false }; translations = { @@ -76,7 +77,7 @@ describe( 'updatePackageTranslations()', () => { vi.mocked( fs.readFileSync ).mockReturnValue( 'Raw PO file content.' ); - vi.mocked( cleanPoFileContent ).mockReturnValue( 'Clean PO file content.' ); + vi.mocked( cleanTranslationFileContent ).mockReturnValue( 'Clean PO file content.' ); } ); it( 'should be a function', () => { @@ -87,7 +88,22 @@ describe( 'updatePackageTranslations()', () => { updatePackageTranslations( defaultOptions ); expect( createMissingPackageTranslations ).toHaveBeenCalledTimes( 1 ); - expect( createMissingPackageTranslations ).toHaveBeenCalledWith( { packagePath: 'packages/ckeditor5-foo' } ); + expect( createMissingPackageTranslations ).toHaveBeenCalledWith( { + packagePath: 'packages/ckeditor5-foo', + skipLicenseHeader: false + } ); + } ); + + it( 'should create missing translations with skipping the license header', () => { + defaultOptions.skipLicenseHeader = true; + + updatePackageTranslations( defaultOptions ); + + expect( createMissingPackageTranslations ).toHaveBeenCalledTimes( 1 ); + expect( createMissingPackageTranslations ).toHaveBeenCalledWith( { + packagePath: 'packages/ckeditor5-foo', + skipLicenseHeader: true + } ); } ); it( 'should not update any files when package does not contain translation context', () => { @@ -161,8 +177,8 @@ describe( 'updatePackageTranslations()', () => { it( 'should save updated translation files on filesystem after cleaning the content', () => { updatePackageTranslations( defaultOptions ); - expect( cleanPoFileContent ).toHaveBeenCalledTimes( 1 ); - expect( cleanPoFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); + expect( cleanTranslationFileContent ).toHaveBeenCalledTimes( 1 ); + expect( cleanTranslationFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); expect( fs.writeFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.writeFileSync ).toHaveBeenCalledWith( @@ -173,7 +189,7 @@ describe( 'updatePackageTranslations()', () => { } ); it( 'should not save translation files on filesystem if their content is not updated', () => { - vi.mocked( cleanPoFileContent ).mockImplementation( input => input ); + vi.mocked( cleanTranslationFileContent ).mockImplementation( input => input ); updatePackageTranslations( defaultOptions ); From 0b4c709d4cd143ec3614d9c984c54d85e5e0f7a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Fri, 11 Oct 2024 10:37:35 +0200 Subject: [PATCH 09/10] Simplified `cleanTranslationFileContent()` util. --- .../lib/utils/cleantranslationfilecontent.js | 16 ++++++---------- .../utils/createmissingpackagetranslations.js | 2 +- .../lib/utils/updatepackagetranslations.js | 2 +- .../tests/utils/cleantranslationfilecontent.js | 15 ++++----------- .../utils/createmissingpackagetranslations.js | 8 ++++---- .../tests/utils/updatepackagetranslations.js | 5 +++-- 6 files changed, 19 insertions(+), 29 deletions(-) diff --git a/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js b/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js index 33907c453..2943a1836 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js +++ b/packages/ckeditor5-dev-translations/lib/utils/cleantranslationfilecontent.js @@ -3,22 +3,18 @@ * For licensing, see LICENSE.md. */ -import PO from 'pofile'; - /** * Removes unused headers from the translation file. * - * @param {string} translationFileContent Content of the translation file. - * @returns {string} + * @param {import('pofile')} translationFileContent Content of the translation file. + * @returns {import('pofile')} */ export default function cleanTranslationFileContent( translationFileContent ) { - const translations = PO.parse( translationFileContent ); - - translations.headers = { - Language: translations.headers.Language, - 'Plural-Forms': translations.headers[ 'Plural-Forms' ], + translationFileContent.headers = { + Language: translationFileContent.headers.Language, + 'Plural-Forms': translationFileContent.headers[ 'Plural-Forms' ], 'Content-Type': 'text/plain; charset=UTF-8' }; - return translations.toString(); + return translationFileContent; } diff --git a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js index e90c99015..6097ff6f2 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/createmissingpackagetranslations.js @@ -40,6 +40,6 @@ export default function createMissingPackageTranslations( { packagePath, skipLic `plural=${ getFormula( languageCode ) };` ].join( ' ' ); - fs.outputFileSync( translationFilePath, cleanTranslationFileContent( translations.toString() ), 'utf-8' ); + fs.outputFileSync( translationFilePath, cleanTranslationFileContent( translations ).toString(), 'utf-8' ); } } diff --git a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js index 357abffbd..8e4293c57 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/lib/utils/updatepackagetranslations.js @@ -67,7 +67,7 @@ export default function updatePackageTranslations( { packageContexts, sourceMess } ) ); - const translationFileUpdated = cleanTranslationFileContent( translations.toString() ); + const translationFileUpdated = cleanTranslationFileContent( translations ).toString(); // (4.3) Save translation file only if it has been updated. if ( translationFile === translationFileUpdated ) { diff --git a/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js b/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js index cbd65c0c2..7b774d278 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js +++ b/packages/ckeditor5-dev-translations/tests/utils/cleantranslationfilecontent.js @@ -3,12 +3,9 @@ * For licensing, see LICENSE.md. */ -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import PO from 'pofile'; +import { beforeEach, describe, expect, it } from 'vitest'; import cleanTranslationFileContent from '../../lib/utils/cleantranslationfilecontent.js'; -vi.mock( 'pofile' ); - describe( 'cleanTranslationFileContent()', () => { let translations; @@ -25,11 +22,8 @@ describe( 'cleanTranslationFileContent()', () => { 'Content-Type': 'Value from Content-Type', 'Content-Transfer-Encoding': 'Value from Content-Transfer-Encoding', 'Plural-Forms': 'Value from Plural-Forms' - }, - toString: () => JSON.stringify( translations ) + } }; - - vi.mocked( PO.parse ).mockReturnValue( translations ); } ); it( 'should be a function', () => { @@ -37,10 +31,9 @@ describe( 'cleanTranslationFileContent()', () => { } ); it( 'should return translation file without unneeded headers', () => { - const result = cleanTranslationFileContent( 'Example content.' ); + const result = cleanTranslationFileContent( translations ); - expect( PO.parse ).toHaveBeenCalledWith( 'Example content.' ); - expect( JSON.parse( result ) ).toEqual( { + expect( result ).toEqual( { headers: { 'Language': 'Value from Language', 'Content-Type': 'text/plain; charset=UTF-8', diff --git a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js index 00c79d3eb..1524cd9b0 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/createmissingpackagetranslations.js @@ -22,8 +22,7 @@ describe( 'createMissingPackageTranslations()', () => { beforeEach( () => { translations = { - headers: {}, - toString: () => 'Raw PO file content.' + headers: {} }; defaultOptions = { @@ -41,7 +40,9 @@ describe( 'createMissingPackageTranslations()', () => { { localeCode: 'zh_TW', languageCode: 'zh', languageFileName: 'zh-tw' } ] ); - vi.mocked( cleanTranslationFileContent ).mockReturnValue( 'Clean PO file content.' ); + vi.mocked( cleanTranslationFileContent ).mockReturnValue( { + toString: () => 'Clean PO file content.' + } ); vi.mocked( fs.existsSync ).mockImplementation( path => { if ( path === 'packages/ckeditor5-foo/lang/translations/en.po' ) { @@ -105,7 +106,6 @@ describe( 'createMissingPackageTranslations()', () => { createMissingPackageTranslations( defaultOptions ); expect( cleanTranslationFileContent ).toHaveBeenCalledTimes( 1 ); - expect( cleanTranslationFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); expect( fs.outputFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.outputFileSync ).toHaveBeenCalledWith( diff --git a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js index 7ba4bba05..6c9ff8882 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js +++ b/packages/ckeditor5-dev-translations/tests/utils/updatepackagetranslations.js @@ -77,7 +77,9 @@ describe( 'updatePackageTranslations()', () => { vi.mocked( fs.readFileSync ).mockReturnValue( 'Raw PO file content.' ); - vi.mocked( cleanTranslationFileContent ).mockReturnValue( 'Clean PO file content.' ); + vi.mocked( cleanTranslationFileContent ).mockReturnValue( { + toString: () => 'Clean PO file content.' + } ); } ); it( 'should be a function', () => { @@ -178,7 +180,6 @@ describe( 'updatePackageTranslations()', () => { updatePackageTranslations( defaultOptions ); expect( cleanTranslationFileContent ).toHaveBeenCalledTimes( 1 ); - expect( cleanTranslationFileContent ).toHaveBeenCalledWith( 'Raw PO file content.' ); expect( fs.writeFileSync ).toHaveBeenCalledTimes( 1 ); expect( fs.writeFileSync ).toHaveBeenCalledWith( From 8b1f68d98b2d2aef1507d2798b7b19d173c4a804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Smyrek?= Date: Mon, 14 Oct 2024 11:03:00 +0200 Subject: [PATCH 10/10] Do not find messsages if package path does not match exactly the file path. --- .../lib/utils/getsourcemessages.js | 4 ++-- .../tests/utils/getsourcemessages.js | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js b/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js index ebe7fd7ec..1b92d9f2b 100644 --- a/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js +++ b/packages/ckeditor5-dev-translations/lib/utils/getsourcemessages.js @@ -15,10 +15,10 @@ import findMessages from '../findmessages.js'; */ export default function getSourceMessages( { packagePaths, sourceFiles, onErrorCallback } ) { return sourceFiles - .filter( filePath => packagePaths.some( packagePath => filePath.includes( packagePath ) ) ) + .filter( filePath => packagePaths.some( packagePath => filePath.includes( `/${ packagePath }/` ) ) ) .flatMap( filePath => { const fileContent = fs.readFileSync( filePath, 'utf-8' ); - const packagePath = packagePaths.find( packagePath => filePath.includes( packagePath ) ); + const packagePath = packagePaths.find( packagePath => filePath.includes( `/${ packagePath }/` ) ); const sourceMessages = []; const onMessageCallback = message => { diff --git a/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js b/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js index 47ba5ffc5..58ec47070 100644 --- a/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js +++ b/packages/ckeditor5-dev-translations/tests/utils/getsourcemessages.js @@ -80,6 +80,16 @@ describe( 'getSourceMessages()', () => { ] ); } ); + it( 'should not find messages if package paths do not match exactly the file path', () => { + defaultOptions.sourceFiles = [ + '/absolute/path/to/packages/ckeditor5-foo-bar/src/utils/file.ts' + ]; + + getSourceMessages( defaultOptions ); + + expect( findMessages ).not.toHaveBeenCalled(); + } ); + it( 'should call error callback in case of an error', () => { vi.mocked( findMessages ).mockImplementation( ( fileContent, filePath, onMessageCallback, onErrorCallback ) => { onErrorCallback( 'Example problem has been detected.' );