implement an AST-based Twig function scanner

Current Twig parser operate by
1. Transforming Twig as PHP code = tokenize() + parse() + render()
2. Using a PhpCode extractor (and its specific configuration about function name)

Disadvantage:
* Twig rendered PHP code can be transformed/wrapped in call_user_func() making that gettext functions undetectable by PhpCode extractor.
  (See for example timber/timber#1753).
* Can't handle templates making use of custom Twig extensions (very common)

This patch offer an extractor that:
* Parse Twig generated AST tree (= tokenize()+parse())
* Recursively iterate over node three to find function gettext calls.

Advantages:
* Operating sooner, at the AST level, Twig expressions like `{{ __("foo", "domain") }}` aren't yet wrapped.
* More robust because it directly iterates over the AST from Twig parser instead of looking at PHP source-code.
* Supports initialized `$twig` environment, thus supporting user-defined extensions?
* Possibly more efficient.

Ref: wp-cli/i18n-command#59

composer.json

@@ -27,7 +27,8 @@
         "twig/extensions": "*",
         "symfony/yaml": "~2",
         "phpunit/phpunit": "^4.8|^5.7|^6.5",
-        "squizlabs/php_codesniffer": "^3.0"
+        "squizlabs/php_codesniffer": "^3.0",
+        "timber/timber": "^1.8"
     },
     "suggest": {
         "illuminate/view": "Is necessary if you want to use the Blade extractor",

src/Extractors/Twig.php

@@ -16,6 +16,20 @@ class Twig extends Extractor implements ExtractorInterface
     public static $options = [
         'extractComments' => 'notes:',
         'twig' => null,
+        'ast' => [
+            'constants' => [],
+            'functions' => [
+                // WordPress defaults
+                '__'              => 'text_domain',
+                '_e'              => 'text_domain',
+                '_x'              => 'text_context_domain',
+                '_ex'             => 'text_context_domain',
+                '_n'              => 'single_plural_number_domain',
+                '_nx'             => 'single_plural_number_context_domain',
+                '_n_noop'         => 'single_plural_domain',
+                '_nx_noop'        => 'single_plural_context_domain'
+            ]
+        ]
     ];
 
     /**
@@ -42,4 +56,17 @@ private static function createTwig()
 
         return static::$options['twig'] = $twig;
     }
+
+    /**
+     * Register intoa Twig instance additional functions recognized by Timber,
+     * the Twig for WordPress library.
+     *
+     * @return NULL
+     */
+    private static function enableTimber() {
+        $timber = new \Timber\Twig();
+        $timber->add_timber_functions(static::$options['twig']);
+        $timber->add_timber_filters(static::$options['twig']);
+    }
+
 }

src/Utils/TwigFunctionsScanner.php

@@ -0,0 +1,165 @@
+<?php
+
+/**
+ * Copyright (C) 2018, [email protected]
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ */
+
+namespace Gettext\Utils;
+
+use Gettext\Utils\PhpFunctionsScanner;
+
+/**
+ * This is a function scanner for Twig templates. The initial string (whether template text or filename) is parsed.
+ * Then getFunctions() recursively traverses nodes.
+ * Nodes expressing a function call to one of the known i18n functions are extracted.
+ * Their list is passed to constructor $func_names argument.
+ */
+class TwigFunctionsScanner extends PhpFunctionsScanner {
+
+    private $filename = '';
+    private $content = null;
+    private $twig = null;
+    private $function_names = [];
+
+    public function __construct(array $file, $twig, $func_names) {
+        if (isset($file['filename'])) {
+            $this->filename = $filename = $file['filename'];
+            $this->content = $content = file_get_contents($filename);
+        }
+        elseif (isset($file['content'])) {
+            $this->filename = $filename = '';
+            $this->content = $content = $file['content'];
+        }
+        else {
+            return NULL;
+        }
+        $this->function_names = $func_names;
+        $this->twig = $twig;
+        $this->tokens = $twig->parse($twig->tokenize(new \Twig_Source($content, $filename)));
+    }
+
+    /**
+     * A pseudo-generator to extract twig nodes corresponding to i18n function calls.
+     * @param array $constants Unused yet.
+     * @return array List of functions arguments/line-number compatible with PhpFunctionsScanner.
+     */
+    public function getFunctions(array $constants = []) {
+        return self::_get_gettext_functions($this->tokens);
+    }
+
+    private function is_gettext_function($obj ) {
+        return ($obj instanceof \Twig_Node_Expression_Function && in_array($obj->getAttribute('name'), $this->function_names, TRUE));
+    }
+
+    private function _get_gettext_functions($tokens) {
+        if (is_array($tokens)) {
+            $functions = [];
+            foreach($tokens as $v) {
+                $functions = array_merge($functions, self::_get_gettext_functions($v));
+            }
+            return $functions;
+        }
+
+        $value = $tokens;
+        if ($this->is_gettext_function($value)) {
+            $arguments_obj = (array)$value->getNode('arguments')->getIterator();
+            $name = $value->getAttribute('name');
+            $line = $value->getTemplateLine();
+
+            // basic verification of node arguments
+            if (count($arguments_obj) < 2) {
+                // can't have a text domain... ToDo
+            }
+            if (! ($arguments_obj[0] instanceof \Twig_Node_Expression_Constant)) {
+                printf(STDERR, "Translation expression does not contains constant expression" . PHP_EOL);
+                printf(STDERR, print_r($arguments_obj, TRUE));
+                return [];
+            }
+            if (FALSE && ! ($arguments_obj[1] instanceof \Twig_Node_Expression_Constant)) {
+                printf(STDERR, "Translation expression does not contains constant text domain" . PHP_EOL);
+                printf(STDERR, print_r($arguments_obj, TRUE));
+                return [];
+            }
+
+            $arguments = array_map(function($obj) use($name) {
+                if ($name == '_n' && $obj instanceof \Twig_Node_Expression_GetAttr) {
+                    return "count";
+                } else {
+                    return $obj->getAttribute('value');
+                }
+            }, $arguments_obj);
+
+            return [ [ $name, $line, $arguments ] ];
+        }
+
+        $functions = [];
+        foreach($tokens->getIterator() as $v) {
+            $functions = array_merge($functions, self::_get_gettext_functions($v));
+        }
+        return $functions;
+    }
+
+    // This is bundled as-is from
+    // https://github.com/wp-cli/i18n-command/blob/master/src/PhpFunctionsScanner.php#L12
+    public function saveGettextFunctions($translations, $options) {
+        $functions = $options['functions'];
+        $file      = $options['file'];
+
+        $f = $this->getFunctions($options['constants']);
+        foreach ($f as $function) {
+            list($name, $line, $args) = $function;
+
+            if (! isset($functions[ $name ])) {
+                continue;
+            }
+
+            $context = $plural = null;
+
+            switch ($functions[ $name ]) {
+            case 'text_domain':
+            case 'gettext':
+                list($original, $domain) = array_pad($args, 2, null);
+                break;
+
+            case 'text_context_domain':
+                list($original, $context, $domain) = array_pad($args, 3, null);
+                break;
+
+            case 'single_plural_number_domain':
+                list($original, $plural, $number, $domain) = array_pad($args, 4, null);
+                break;
+
+            case 'single_plural_number_context_domain':
+                list($original, $plural, $number, $context, $domain) = array_pad($args, 5, null);
+                break;
+
+            case 'single_plural_domain':
+                list($original, $plural, $domain) = array_pad($args, 3, null);
+                break;
+
+            case 'single_plural_context_domain':
+                list($original, $plural, $context, $domain) = array_pad($args, 4, null);
+                break;
+
+            default:
+                // Should never happen.
+                \WP_CLI::error(sprintf("Internal error: unknown function map '%s' for '%s'.", $functions[ $name ], $name));
+            }
+
+            if ((string) $original !== '' && ($domain === $translations->getDomain() || null === $translations->getDomain())) {
+                $translation = $translations->insert($context, $original, $plural);
+                $translation = $translation->addReference($file, $line);
+
+                if (isset($function[3])) {
+                    foreach ($function[3] as $extractedComment) {
+                        $translation = $translation->addExtractedComment($extractedComment);
+                    }
+                }
+            }
+        }
+    }
+}

tests/AssetsTest.php

@@ -381,7 +381,7 @@ public function testPhpCode3()
 
     public function testTwig()
     {
-        $translations = static::get('twig/input', 'Twig');
+        $translations = static::get('twig/input', 'Twig', ['ast' => FALSE]);
         $countTranslations = 10;
         $countTranslated = 0;
         $countHeaders = 8;
@@ -419,6 +419,22 @@ public function testTwig()
         $this->runTestFormat('twig/YamlDictionary', $countTranslations, $countTranslated);
     }
 
+    public function testTimberTwig()
+    {
+        require_once('vendor/timber/timber/lib/Timber.php');
+        $translations = Translations::fromTwigFile('./tests/assets/twig-timber/input.php');
+        $countTranslations = 11;
+        $countTranslated = 0;
+        $countHeaders = 8;
+
+        $this->assertCount($countTranslations, $translations);
+        $this->assertCount($countHeaders, $translations->getHeaders());
+        $this->assertEquals(0, $translations->countTranslated());
+
+        $this->assertContent($translations, 'twig-timber/Po');
+        $this->runTestFormat('twig-timber/Po', $countTranslations, $countTranslated, $countHeaders);
+    }
+
     public function testVueJs()
     {
         $translations = static::get('vuejs/input', 'VueJs');
@@ -500,4 +516,4 @@ public function testPhpCode4()
         $this->runTestFormat('phpcode4/YamlDictionary', $countTranslations, $countTranslated);
     }
 
-}
+}

tests/assets/twig-timber/Po.po

@@ -0,0 +1,58 @@
+msgid ""
+msgstr ""
+"Content-Transfer-Encoding: 8bit\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Language: \n"
+"Language-Team: \n"
+"Last-Translator: \n"
+"MIME-Version: 1.0\n"
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+
+#: ./tests/assets/twig-timber/input.php:2
+msgid "text 1"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:6
+msgid "text 2"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:7
+msgid "text 3 (with parenthesis)"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:8
+msgid "text 4 \"with double quotes\""
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:9
+msgid "text 5 'with escaped single quotes'"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:13
+msgid "text 6"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:14
+msgid "text 7 (with parenthesis)"
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:15
+msgid "text 8 \"with escaped double quotes\""
+msgstr ""
+
+#: ./tests/assets/twig-timber/input.php:16
+msgid "text 9 'with single quotes'"
+msgstr ""
+
+#. notes: This is an actual note for translators.
+#: ./tests/assets/twig-timber/input.php:19
+msgid "text 10 with plural"
+msgid_plural "The plural form"
+msgstr[0] ""
+
+#: ./tests/assets/twig-timber/input.php:22
+msgctxt "This gives some context."
+msgid "test"
+msgid_plural "tests"
+msgstr[0] ""

tests/assets/twig-timber/input.php

@@ -0,0 +1,23 @@
+<header>
+    <h1>{{ __('text 1') }}</h1>
+</header>
+
+<div>
+    <p>{{ __('text 2') }}</p>
+    <p>{{ __('text 3 (with parenthesis)') }}</p>
+    <p>{{ __('text 4 "with double quotes"') }}</p>
+    <p>{{ __('text 5 \'with escaped single quotes\'') }}</p>
+</div>
+
+<div>
+    <p>{{ __("text 6") }}</p>
+    <p>{{ __("text 7 (with parenthesis)") }}</p>
+    <p>{{ __("text 8 \"with escaped double quotes\"") }}</p>
+    <p>{{ __("text 9 'with single quotes'") }}</p>
+    <p>
+    {# This is an actual note for translators. #}
+    {{ _n("text 10 with plural", "The plural form", 5) }}
+    </p>
+    <p>
+    {{ _nx("test", "tests", 2, "This gives some context.") }}
+</div>