From 0d69c23e6006423bbf4d8d496b15a5e52184feaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Droz?= Date: Sun, 28 Oct 2018 18:22:36 -0300 Subject: [PATCH] implement an AST-based Twig function scanner Current Twig parser operate by 1. Transforming Twig as PHP code = tokenize() + parse() + render() 2. Using a PhpCode extractor (and its specific configuration about function name) Disadvantage: * Twig rendered PHP code can be transformed/wrapped in call_user_func() making that gettext functions undetectable by PhpCode extractor. (See for example timber/timber#1753). * Can't handle templates making use of custom Twig extensions (very common) This patch offer an extractor that: * Parse Twig generated AST tree (= tokenize()+parse()) * Recursively iterate over node three to find function gettext calls. Advantages: * Operating sooner, at the AST level, Twig expressions like `{{ __("foo", "domain") }}` aren't yet wrapped. * More robust because it directly iterates over the AST from Twig parser instead of looking at PHP source-code. * Supports initialized `$twig` environment, thus supporting user-defined extensions? * Possibly more efficient. Ref: https://github.com/wp-cli/i18n-command/pull/59 --- composer.json | 3 +- src/Extractors/Twig.php | 27 +++++ src/Utils/TwigFunctionsScanner.php | 165 +++++++++++++++++++++++++++++ tests/AssetsTest.php | 20 +++- tests/assets/twig-timber/Po.po | 58 ++++++++++ tests/assets/twig-timber/input.php | 23 ++++ 6 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 src/Utils/TwigFunctionsScanner.php create mode 100644 tests/assets/twig-timber/Po.po create mode 100644 tests/assets/twig-timber/input.php diff --git a/composer.json b/composer.json index 24b5083d..1166f80d 100644 --- a/composer.json +++ b/composer.json @@ -27,7 +27,8 @@ "twig/extensions": "*", "symfony/yaml": "~2", "phpunit/phpunit": "^4.8|^5.7|^6.5", - "squizlabs/php_codesniffer": "^3.0" + "squizlabs/php_codesniffer": "^3.0", + "timber/timber": "^1.8" }, "suggest": { "illuminate/view": "Is necessary if you want to use the Blade extractor", diff --git a/src/Extractors/Twig.php b/src/Extractors/Twig.php index 03ff4209..ec87a39f 100644 --- a/src/Extractors/Twig.php +++ b/src/Extractors/Twig.php @@ -16,6 +16,20 @@ class Twig extends Extractor implements ExtractorInterface public static $options = [ 'extractComments' => 'notes:', 'twig' => null, + 'ast' => [ + 'constants' => [], + 'functions' => [ + // WordPress defaults + '__' => 'text_domain', + '_e' => 'text_domain', + '_x' => 'text_context_domain', + '_ex' => 'text_context_domain', + '_n' => 'single_plural_number_domain', + '_nx' => 'single_plural_number_context_domain', + '_n_noop' => 'single_plural_domain', + '_nx_noop' => 'single_plural_context_domain' + ] + ] ]; /** @@ -42,4 +56,17 @@ private static function createTwig() return static::$options['twig'] = $twig; } + + /** + * Register intoa Twig instance additional functions recognized by Timber, + * the Twig for WordPress library. + * + * @return NULL + */ + private static function enableTimber() { + $timber = new \Timber\Twig(); + $timber->add_timber_functions(static::$options['twig']); + $timber->add_timber_filters(static::$options['twig']); + } + } diff --git a/src/Utils/TwigFunctionsScanner.php b/src/Utils/TwigFunctionsScanner.php new file mode 100644 index 00000000..55b54238 --- /dev/null +++ b/src/Utils/TwigFunctionsScanner.php @@ -0,0 +1,165 @@ +filename = $filename = $file['filename']; + $this->content = $content = file_get_contents($filename); + } + elseif (isset($file['content'])) { + $this->filename = $filename = ''; + $this->content = $content = $file['content']; + } + else { + return NULL; + } + $this->function_names = $func_names; + $this->twig = $twig; + $this->tokens = $twig->parse($twig->tokenize(new \Twig_Source($content, $filename))); + } + + /** + * A pseudo-generator to extract twig nodes corresponding to i18n function calls. + * @param array $constants Unused yet. + * @return array List of functions arguments/line-number compatible with PhpFunctionsScanner. + */ + public function getFunctions(array $constants = []) { + return self::_get_gettext_functions($this->tokens); + } + + private function is_gettext_function($obj ) { + return ($obj instanceof \Twig_Node_Expression_Function && in_array($obj->getAttribute('name'), $this->function_names, TRUE)); + } + + private function _get_gettext_functions($tokens) { + if (is_array($tokens)) { + $functions = []; + foreach($tokens as $v) { + $functions = array_merge($functions, self::_get_gettext_functions($v)); + } + return $functions; + } + + $value = $tokens; + if ($this->is_gettext_function($value)) { + $arguments_obj = (array)$value->getNode('arguments')->getIterator(); + $name = $value->getAttribute('name'); + $line = $value->getTemplateLine(); + + // basic verification of node arguments + if (count($arguments_obj) < 2) { + // can't have a text domain... ToDo + } + if (! ($arguments_obj[0] instanceof \Twig_Node_Expression_Constant)) { + printf(STDERR, "Translation expression does not contains constant expression" . PHP_EOL); + printf(STDERR, print_r($arguments_obj, TRUE)); + return []; + } + if (FALSE && ! ($arguments_obj[1] instanceof \Twig_Node_Expression_Constant)) { + printf(STDERR, "Translation expression does not contains constant text domain" . PHP_EOL); + printf(STDERR, print_r($arguments_obj, TRUE)); + return []; + } + + $arguments = array_map(function($obj) use($name) { + if ($name == '_n' && $obj instanceof \Twig_Node_Expression_GetAttr) { + return "count"; + } else { + return $obj->getAttribute('value'); + } + }, $arguments_obj); + + return [ [ $name, $line, $arguments ] ]; + } + + $functions = []; + foreach($tokens->getIterator() as $v) { + $functions = array_merge($functions, self::_get_gettext_functions($v)); + } + return $functions; + } + + // This is bundled as-is from + // https://github.com/wp-cli/i18n-command/blob/master/src/PhpFunctionsScanner.php#L12 + public function saveGettextFunctions($translations, $options) { + $functions = $options['functions']; + $file = $options['file']; + + $f = $this->getFunctions($options['constants']); + foreach ($f as $function) { + list($name, $line, $args) = $function; + + if (! isset($functions[ $name ])) { + continue; + } + + $context = $plural = null; + + switch ($functions[ $name ]) { + case 'text_domain': + case 'gettext': + list($original, $domain) = array_pad($args, 2, null); + break; + + case 'text_context_domain': + list($original, $context, $domain) = array_pad($args, 3, null); + break; + + case 'single_plural_number_domain': + list($original, $plural, $number, $domain) = array_pad($args, 4, null); + break; + + case 'single_plural_number_context_domain': + list($original, $plural, $number, $context, $domain) = array_pad($args, 5, null); + break; + + case 'single_plural_domain': + list($original, $plural, $domain) = array_pad($args, 3, null); + break; + + case 'single_plural_context_domain': + list($original, $plural, $context, $domain) = array_pad($args, 4, null); + break; + + default: + // Should never happen. + \WP_CLI::error(sprintf("Internal error: unknown function map '%s' for '%s'.", $functions[ $name ], $name)); + } + + if ((string) $original !== '' && ($domain === $translations->getDomain() || null === $translations->getDomain())) { + $translation = $translations->insert($context, $original, $plural); + $translation = $translation->addReference($file, $line); + + if (isset($function[3])) { + foreach ($function[3] as $extractedComment) { + $translation = $translation->addExtractedComment($extractedComment); + } + } + } + } + } +} diff --git a/tests/AssetsTest.php b/tests/AssetsTest.php index e90968b8..027b8e1d 100644 --- a/tests/AssetsTest.php +++ b/tests/AssetsTest.php @@ -381,7 +381,7 @@ public function testPhpCode3() public function testTwig() { - $translations = static::get('twig/input', 'Twig'); + $translations = static::get('twig/input', 'Twig', ['ast' => FALSE]); $countTranslations = 10; $countTranslated = 0; $countHeaders = 8; @@ -419,6 +419,22 @@ public function testTwig() $this->runTestFormat('twig/YamlDictionary', $countTranslations, $countTranslated); } + public function testTimberTwig() + { + require_once('vendor/timber/timber/lib/Timber.php'); + $translations = Translations::fromTwigFile('./tests/assets/twig-timber/input.php'); + $countTranslations = 11; + $countTranslated = 0; + $countHeaders = 8; + + $this->assertCount($countTranslations, $translations); + $this->assertCount($countHeaders, $translations->getHeaders()); + $this->assertEquals(0, $translations->countTranslated()); + + $this->assertContent($translations, 'twig-timber/Po'); + $this->runTestFormat('twig-timber/Po', $countTranslations, $countTranslated, $countHeaders); + } + public function testVueJs() { $translations = static::get('vuejs/input', 'VueJs'); @@ -500,4 +516,4 @@ public function testPhpCode4() $this->runTestFormat('phpcode4/YamlDictionary', $countTranslations, $countTranslated); } -} \ No newline at end of file +} diff --git a/tests/assets/twig-timber/Po.po b/tests/assets/twig-timber/Po.po new file mode 100644 index 00000000..652c1006 --- /dev/null +++ b/tests/assets/twig-timber/Po.po @@ -0,0 +1,58 @@ +msgid "" +msgstr "" +"Content-Transfer-Encoding: 8bit\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Language: \n" +"Language-Team: \n" +"Last-Translator: \n" +"MIME-Version: 1.0\n" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" + +#: ./tests/assets/twig-timber/input.php:2 +msgid "text 1" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:6 +msgid "text 2" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:7 +msgid "text 3 (with parenthesis)" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:8 +msgid "text 4 \"with double quotes\"" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:9 +msgid "text 5 'with escaped single quotes'" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:13 +msgid "text 6" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:14 +msgid "text 7 (with parenthesis)" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:15 +msgid "text 8 \"with escaped double quotes\"" +msgstr "" + +#: ./tests/assets/twig-timber/input.php:16 +msgid "text 9 'with single quotes'" +msgstr "" + +#. notes: This is an actual note for translators. +#: ./tests/assets/twig-timber/input.php:19 +msgid "text 10 with plural" +msgid_plural "The plural form" +msgstr[0] "" + +#: ./tests/assets/twig-timber/input.php:22 +msgctxt "This gives some context." +msgid "test" +msgid_plural "tests" +msgstr[0] "" diff --git a/tests/assets/twig-timber/input.php b/tests/assets/twig-timber/input.php new file mode 100644 index 00000000..ce39521e --- /dev/null +++ b/tests/assets/twig-timber/input.php @@ -0,0 +1,23 @@ +
+

{{ __('text 1') }}

+
+ +
+

{{ __('text 2') }}

+

{{ __('text 3 (with parenthesis)') }}

+

{{ __('text 4 "with double quotes"') }}

+

{{ __('text 5 \'with escaped single quotes\'') }}

+
+ +
+

{{ __("text 6") }}

+

{{ __("text 7 (with parenthesis)") }}

+

{{ __("text 8 \"with escaped double quotes\"") }}

+

{{ __("text 9 'with single quotes'") }}

+

+ {# This is an actual note for translators. #} + {{ _n("text 10 with plural", "The plural form", 5) }} +

+

+ {{ _nx("test", "tests", 2, "This gives some context.") }} +