From c6331b2324471620831c54b79c9228cea6a512f8 Mon Sep 17 00:00:00 2001
From: Zach Laine <whatwasthataddress@gmail.com>
Date: Sun, 10 Nov 2024 02:40:44 -0600
Subject: [PATCH] Change token<> to include the position of the start of the
 token in the underlying sequence, and change the way that the error handler
 is invoked, so that it detects token iterators, and passes iterators into the
 underlying range to the error handler, instead of the token iterators.

See #202.
---
 include/boost/parser/config.hpp |   10 +
 include/boost/parser/lexer.hpp  |  108 ++--
 include/boost/parser/parser.hpp |   59 +-
 test/lexer.cpp                  |   46 +-
 test/lexer_adobe_files.cpp      | 1003 ++++++++++++++++---------------
 5 files changed, 682 insertions(+), 544 deletions(-)

diff --git a/include/boost/parser/config.hpp b/include/boost/parser/config.hpp
index 512bbd92..da9f214e 100644
--- a/include/boost/parser/config.hpp
+++ b/include/boost/parser/config.hpp
@@ -59,6 +59,12 @@
     also defined. */
 #    define BOOST_PARSER_TRACE_TO_VS_OUTPUT
 
+/** When lexing is enabled, each token contains its position within the
+    underlying range.  To save a bit of space, an `unsiged int` is used for
+    this.  If you parse input sequences longer than 2^32-1 characters, define
+    `BOOST_PARSER_TOKEN_POSITION_TYPE` to be a larger integral type. */
+#    define BOOST_PARSER_TOKEN_POSITION_TYPE unsigned int
+
 #else
 
 #    ifdef BOOST_PARSER_NO_RUNTIME_ASSERTIONS
@@ -103,6 +109,10 @@
 #    define BOOST_PARSER_MAX_AGGREGATE_SIZE 25
 #endif
 
+#if !defined(BOOST_PARSER_TOKEN_POSITION_TYPE)
+#    define BOOST_PARSER_TOKEN_POSITION_TYPE unsigned int
+#endif
+
 // VS2019 and VS2017 need conditional constexpr in some places, even in C++17 mode.
 #if !defined(_MSC_VER) || 1930 <= _MSC_VER
 #    define BOOST_PARSER_CONSTEXPR constexpr
diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp
index d88112b4..e2a515a9 100644
--- a/include/boost/parser/lexer.hpp
+++ b/include/boost/parser/lexer.hpp
@@ -151,27 +151,47 @@ namespace boost { namespace parser {
     {
         using char_type = CharType;
         using string_view = std::basic_string_view<CharType>;
+        using position_type = BOOST_PARSER_TOKEN_POSITION_TYPE;
 
         constexpr token() :
-            value_(0ll), id_(), kind_(detail::token_kind::string_view)
+            value_(0ll),
+            underlying_position_(),
+            id_(),
+            kind_(detail::token_kind::string_view)
         {}
-        constexpr token(int id, string_view value) :
-            value_(0ll), id_(id), kind_(detail::token_kind::string_view)
+        constexpr token(
+            int id, position_type underlying_position, string_view value) :
+            value_(0ll),
+            underlying_position_(),
+            id_(id),
+            kind_(detail::token_kind::string_view)
         {
             value_.sv_ = value;
         }
-        constexpr token(int id, long long value) :
-            value_(0ll), id_(id), kind_(detail::token_kind::long_long)
+        constexpr token(
+            int id, position_type underlying_position, long long value) :
+            value_(0ll),
+            underlying_position_(underlying_position),
+            id_(id),
+            kind_(detail::token_kind::long_long)
         {
             value_.ll_ = value;
         }
-        constexpr token(int id, long double value) :
-            value_(0ll), id_(id), kind_(detail::token_kind::long_double)
+        constexpr token(
+            int id, position_type underlying_position, long double value) :
+            value_(0ll),
+            underlying_position_(underlying_position),
+            id_(id),
+            kind_(detail::token_kind::long_double)
         {
             value_.d_ = value;
         }
 
         constexpr int id() const { return id_; }
+        constexpr position_type underlying_position() const
+        {
+            return underlying_position_;
+        }
 
         constexpr bool has_string_view() const
         {
@@ -238,6 +258,7 @@ namespace boost { namespace parser {
             long double d_;
             string_view sv_;
         } value_;
+        position_type underlying_position_ = 0;
         // TODO: Document the 22-bit size limitation on id_ (values must be
         // positive).
         int id_ : 24;
@@ -539,8 +560,10 @@ namespace boost { namespace parser {
         };
 
         template<parse_spec Spec, typename CharType>
-        token<CharType>
-        make_token(int id, std::basic_string_view<CharType> ctre_token)
+        token<CharType> make_token(
+            int id,
+            std::basic_string_view<CharType> ctre_token,
+            BOOST_PARSER_TOKEN_POSITION_TYPE underlying_position)
         {
             auto f = ctre_token.data();
             auto const l = f + ctre_token.size();
@@ -553,16 +576,20 @@ namespace boost { namespace parser {
 
             switch (Spec.type) {
             case token_parsed_type::character:
-                return {character_id, (long long)ctre_token[0]};
+                return {
+                    character_id,
+                    underlying_position,
+                    (long long)ctre_token[0]};
 
-            case token_parsed_type::string_view: return {id, ctre_token};
+            case token_parsed_type::string_view:
+                return {id, underlying_position, ctre_token};
 
             case token_parsed_type::bool_:
                 using namespace std::literals;
                 if (std::ranges::equal(ctre_token, "true"sv)) {
-                    return {id, 1ll};
+                    return {id, underlying_position, 1ll};
                 } else if (std::ranges::equal(ctre_token, "false"sv)) {
-                    return {id, 0ll};
+                    return {id, underlying_position, 0ll};
                 } else {
                     // TODO: report error.
                 }
@@ -573,7 +600,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<true, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::unsigned_char: {
                 unsigned char value;
@@ -581,7 +608,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::short_: {
                 short value;
@@ -589,7 +616,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<true, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::unsigned_short: {
                 unsigned short value;
@@ -597,7 +624,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::int_: {
                 int value;
@@ -605,7 +632,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<true, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::unsigned_int: {
                 unsigned int value;
@@ -613,7 +640,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::long_: {
                 long value;
@@ -621,7 +648,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<true, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::unsigned_long: {
                 unsigned long value;
@@ -629,7 +656,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::long_long: {
                 long long value;
@@ -637,7 +664,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<true, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::unsigned_long_long: {
                 unsigned long long value;
@@ -645,7 +672,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::wchar_t_: {
                 unsigned int value;
@@ -653,7 +680,7 @@ namespace boost { namespace parser {
                     type_wrapper<wchar_t>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::char8_t_: {
                 unsigned int value;
@@ -661,7 +688,7 @@ namespace boost { namespace parser {
                     type_wrapper<char8_t>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::char16_t_: {
                 unsigned int value;
@@ -669,7 +696,7 @@ namespace boost { namespace parser {
                     type_wrapper<char16_t>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
             case token_parsed_type::char32_t_: {
                 unsigned int value;
@@ -677,7 +704,7 @@ namespace boost { namespace parser {
                     type_wrapper<char32_t>{},
                     Spec.radix,
                     numeric::parse_int<false, Spec.radix, 1, -1>(f, l, value));
-                return {id, (long long)value};
+                return {id, underlying_position, (long long)value};
             }
 
             case token_parsed_type::float_: {
@@ -686,7 +713,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     0,
                     numeric::parse_real(f, l, value));
-                return {id, (long double)value};
+                return {id, underlying_position, (long double)value};
             }
             case token_parsed_type::double_: {
                 double value;
@@ -694,7 +721,7 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     0,
                     numeric::parse_real(f, l, value));
-                return {id, (long double)value};
+                return {id, underlying_position, (long double)value};
             }
             case token_parsed_type::long_double: {
                 long double value;
@@ -702,14 +729,14 @@ namespace boost { namespace parser {
                     type_wrapper<decltype(value)>{},
                     0,
                     numeric::parse_real(f, l, value));
-                return {id, value};
+                return {id, underlying_position, value};
             }
             case token_parsed_type::ws:
             default:
 #if defined(__cpp_lib_unreachable)
                 std::unreachable();
 #endif
-                return {id, 0ll};
+                return {id, underlying_position, 0ll};
             }
         }
     }
@@ -721,7 +748,7 @@ namespace boost { namespace parser {
 
     /** TODO */
     template<
-        std::ranges::forward_range V,
+        std::ranges::contiguous_range V,
         typename Lexer,
         typename TokenCache = std::vector<typename Lexer::token_type>>
         requires std::ranges::view<V>
@@ -754,7 +781,7 @@ namespace boost { namespace parser {
             lexer_(std::move(lexer)),
             tokens_(owned_cache_)
         {
-            latest_ = base_.begin();
+            latest_ = std::ranges::begin(base_);
         }
         constexpr explicit tokens_view(
             V base,
@@ -764,7 +791,7 @@ namespace boost { namespace parser {
             lexer_(std::move(lexer)),
             tokens_(external_cache.get())
         {
-            latest_ = base_.begin();
+            latest_ = std::ranges::begin(base_);
         }
 
         constexpr V base() const &
@@ -830,7 +857,7 @@ namespace boost { namespace parser {
                 parent_->tokens_.reserve(new_size);
 
                 auto r = std::ranges::subrange(
-                    parent_->latest_, parent_->base_.end());
+                    parent_->latest_, std::ranges::end(parent_->base_));
                 auto ctre_range = Lexer::regex_range(r);
                 auto ctre_first = ctre_range.begin();
                 auto const ctre_last = ctre_range.end();
@@ -864,7 +891,11 @@ namespace boost { namespace parser {
                                 constexpr detail::parse_spec parse_spec =
                                     parent_->lexer_.specs()[i.value];
                                 parent_->tokens_.push_back(
-                                    detail::make_token<parse_spec>(id, sv));
+                                    detail::make_token<parse_spec>(
+                                        id,
+                                        sv,
+                                        ctre_first.current -
+                                            ctre_first.orig_begin));
                                 return sv;
                             } else {
                                 return state;
@@ -902,6 +933,9 @@ namespace boost { namespace parser {
                 BOOST_PARSER_DEBUG_ASSERT(parent_ == rhs.parent_);
                 return token_offset_ == rhs.token_offset_;
             }
+
+            auto range_begin() const { std::ranges::begin(parent_->base_); }
+            auto range_end() const { std::ranges::end(parent_->base_); }
         };
 
         template<bool Const>
@@ -936,7 +970,7 @@ namespace boost { namespace parser {
                 if (x.token_offset_ != x.parent_->tokens_.size())
                     return false;
                 auto r = std::ranges::subrange(
-                    x.parent_->latest_, x.parent_->base_.end());
+                    x.parent_->latest_, std::ranges::end(x.parent_->base_));
                 auto ctre_range = Lexer::regex_range(r);
                 return !ctre_range.begin().current_match;
             }
diff --git a/include/boost/parser/parser.hpp b/include/boost/parser/parser.hpp
index f78bbd8d..cd49fc0b 100644
--- a/include/boost/parser/parser.hpp
+++ b/include/boost/parser/parser.hpp
@@ -2273,7 +2273,12 @@ namespace boost { namespace parser {
             scoped_base_assign(BaseIter & base, Iter & it) :
                 base_(base), it_(it)
             {}
-            ~scoped_base_assign() { base_ = it_.base(); }
+            ~scoped_base_assign()
+            {
+                if constexpr (!std::is_same_v<BaseIter, Iter>) {
+                    base_ = it_.base();
+                }
+            }
 
             BaseIter & base_;
             Iter & it_;
@@ -2310,6 +2315,31 @@ namespace boost { namespace parser {
             pending_symbol_table_operations_t & pending_ops_;
         };
 
+        template<typename I>
+        constexpr bool is_token_iter_v = is_token_v<iter_value_t<I>>;
+
+        template<typename Iter, typename Sentinel, typename ErrorHandler>
+        bool handle_parse_exception(
+            ErrorHandler const & error_handler,
+            Iter initial_first,
+            Sentinel last,
+            parse_error<Iter> const & e)
+        {
+            if constexpr (is_token_iter_v<Iter>) {
+                auto const underlying_first = e.iter.range_begin();
+                auto const underlying_curr = e.iter->underlying_position();
+                auto const underlying_last = e.iter.range_end();
+                parse_error underlying_error{underlying_curr, e.what()};
+                return error_handler(
+                           underlying_first,
+                           underlying_last,
+                           underlying_error) == error_handler_result::rethrow;
+            } else {
+                return error_handler(initial_first, last, e) ==
+                       error_handler_result::rethrow;
+            }
+        }
+
         template<
             bool Debug,
             typename Iter,
@@ -2356,8 +2386,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, attr);
                 return success;
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 return false;
@@ -2410,8 +2440,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, nope{});
                 return detail::make_parse_result(attr_, success);
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 attr_t attr_{};
@@ -2465,8 +2495,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, nope{});
                 return success;
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 return false;
@@ -2516,8 +2546,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, attr);
                 return success;
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 return false;
@@ -2569,8 +2599,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, nope{});
                 return detail::make_parse_result(attr_, success);
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 attr_t attr_{};
@@ -2622,8 +2652,8 @@ namespace boost { namespace parser {
                     detail::final_trace(context, flags, nope{});
                 return success;
             } catch (parse_error<Iter> const & e) {
-                if (error_handler(initial_first, last, e) ==
-                    error_handler_result::rethrow) {
+                if (detail::handle_parse_exception(
+                        error_handler, initial_first, last, e)) {
                     throw;
                 }
                 return false;
@@ -2633,8 +2663,7 @@ namespace boost { namespace parser {
         template<typename I, typename S>
         constexpr auto make_input_subrange(I first, S last) noexcept
         {
-            using value_type = iter_value_t<I>;
-            if constexpr (is_token_v<value_type>) {
+            if constexpr (is_token_iter_v<I>) {
                 return BOOST_PARSER_SUBRANGE(first, last);
             } else {
                 return BOOST_PARSER_SUBRANGE(first, last) |
diff --git a/test/lexer.cpp b/test/lexer.cpp
index d383e13b..077161cb 100644
--- a/test/lexer.cpp
+++ b/test/lexer.cpp
@@ -315,9 +315,9 @@ int main()
 
         using tok_t = bp::token<char>;
         tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, "foo"),
-            tok_t(bp::character_id, (long long)'='),
-            tok_t((int)my_tokens::bar, "bar")};
+            tok_t((int)my_tokens::foo, 0, "foo"),
+            tok_t(bp::character_id, 0, (long long)'='),
+            tok_t((int)my_tokens::bar, 0, "bar")};
 
         int position = 0;
 
@@ -360,9 +360,9 @@ int main()
         std::string s = "foo = bar";
         using tok_t = bp::token<char>;
         tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, "foo"),
-            tok_t(bp::character_id, (long long)'='),
-            tok_t((int)my_tokens::bar, "bar")};
+            tok_t((int)my_tokens::foo, 0, "foo"),
+            tok_t(bp::character_id, 0, (long long)'='),
+            tok_t((int)my_tokens::bar, 0, "bar")};
 
         auto const lexer8 = bp::lexer<char8_t, my_tokens> |
                             bp::token_spec<"foo", my_tokens::foo> |
@@ -373,9 +373,9 @@ int main()
         std::u8string u8s = u8"foo = bar";
         using tok8_t = bp::token<char8_t>;
         tok8_t const expected8[] = {
-            tok8_t((int)my_tokens::foo, u8"foo"),
-            tok8_t(bp::character_id, (long long)'='),
-            tok8_t((int)my_tokens::bar, u8"bar")};
+            tok8_t((int)my_tokens::foo, 0, u8"foo"),
+            tok8_t(bp::character_id, 0, (long long)'='),
+            tok8_t((int)my_tokens::bar, 0, u8"bar")};
 
         auto const lexer16 = bp::lexer<char16_t, my_tokens> |
                              bp::token_spec<"foo", my_tokens::foo> |
@@ -386,9 +386,9 @@ int main()
         std::u16string u16s = u"foo = bar";
         using tok16_t = bp::token<char16_t>;
         tok16_t const expected16[] = {
-            tok16_t((int)my_tokens::foo, u"foo"),
-            tok16_t(bp::character_id, (long long)'='),
-            tok16_t((int)my_tokens::bar, u"bar")};
+            tok16_t((int)my_tokens::foo, 0, u"foo"),
+            tok16_t(bp::character_id, 0, (long long)'='),
+            tok16_t((int)my_tokens::bar, 0, u"bar")};
 
         auto const lexer32 = bp::lexer<char32_t, my_tokens> |
                              bp::token_spec<"foo", my_tokens::foo> |
@@ -399,9 +399,9 @@ int main()
         std::u32string u32s = U"foo = bar";
         using tok32_t = bp::token<char32_t>;
         tok32_t const expected32[] = {
-            tok32_t((int)my_tokens::foo, U"foo"),
-            tok32_t(bp::character_id, (long long)'='),
-            tok32_t((int)my_tokens::bar, U"bar")};
+            tok32_t((int)my_tokens::foo, 0, U"foo"),
+            tok32_t(bp::character_id, 0, (long long)'='),
+            tok32_t((int)my_tokens::bar, 0, U"bar")};
 
 
         int position = 0;
@@ -458,9 +458,9 @@ int main()
         std::string s = "foo=bar";
         using tok_t = bp::token<char>;
         tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, "foo"),
-            tok_t(bp::character_id, (long long)'='),
-            tok_t((int)my_tokens::bar, "bar")};
+            tok_t((int)my_tokens::foo, 0, "foo"),
+            tok_t(bp::character_id, 0, (long long)'='),
+            tok_t((int)my_tokens::bar, 0, "bar")};
 
         int position = 0;
         for (auto tok : s | bp::to_tokens(lexer)) {
@@ -482,11 +482,11 @@ int main()
         std::string s = "foo = bar";
         using tok_t = bp::token<char>;
         tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, "foo"),
-            tok_t((int)my_tokens::ws, " "),
-            tok_t(bp::character_id, (long long)'='),
-            tok_t((int)my_tokens::ws, " "),
-            tok_t((int)my_tokens::bar, "bar")};
+            tok_t((int)my_tokens::foo, 0, "foo"),
+            tok_t((int)my_tokens::ws, 0, " "),
+            tok_t(bp::character_id, 0, (long long)'='),
+            tok_t((int)my_tokens::ws, 0, " "),
+            tok_t((int)my_tokens::bar, 0, "bar")};
 
         int position = 0;
         for (auto tok : s | bp::to_tokens(lexer)) {
diff --git a/test/lexer_adobe_files.cpp b/test/lexer_adobe_files.cpp
index 9a7290e0..5ac17b16 100644
--- a/test/lexer_adobe_files.cpp
+++ b/test/lexer_adobe_files.cpp
@@ -80,25 +80,25 @@ sheet alert_dialog
 
             using tok_t = bp::token<char>;
             tok_t const expected[] = {
-                tok_t((int)adobe_tokens::lead_comment, R"(/*
+                tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
     Copyright 2005-2007 Adobe Systems Incorporated
     Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
     or a copy at http://stlab.adobe.com/licenses.html)
 */)"),
-                tok_t((int)adobe_tokens::identifier, "sheet"),
-                tok_t((int)adobe_tokens::identifier, "alert_dialog"),
-                tok_t(bp::character_id, (long long)'{'),
-                tok_t((int)adobe_tokens::identifier, "output"),
-                tok_t(bp::character_id, (long long)':'),
-                tok_t((int)adobe_tokens::identifier, "result"),
-                tok_t((int)adobe_tokens::define, "<=="),
-                tok_t(bp::character_id, (long long)'{'),
-                tok_t((int)adobe_tokens::identifier, "dummy_value"),
-                tok_t(bp::character_id, (long long)':'),
-                tok_t((int)adobe_tokens::number, (long double)42.0),
-                tok_t(bp::character_id, (long long)'}'),
-                tok_t(bp::character_id, (long long)';'),
-                tok_t(bp::character_id, (long long)'}')};
+                tok_t((int)adobe_tokens::identifier, 0, "sheet"),
+                tok_t((int)adobe_tokens::identifier, 0, "alert_dialog"),
+                tok_t(bp::character_id, 0, (long long)'{'),
+                tok_t((int)adobe_tokens::identifier, 0, "output"),
+                tok_t(bp::character_id, 0, (long long)':'),
+                tok_t((int)adobe_tokens::identifier, 0, "result"),
+                tok_t((int)adobe_tokens::define, 0, "<=="),
+                tok_t(bp::character_id, 0, (long long)'{'),
+                tok_t((int)adobe_tokens::identifier, 0, "dummy_value"),
+                tok_t(bp::character_id, 0, (long long)':'),
+                tok_t((int)adobe_tokens::number, 0, (long double)42.0),
+                tok_t(bp::character_id, 0, (long long)'}'),
+                tok_t(bp::character_id, 0, (long long)';'),
+                tok_t(bp::character_id, 0, (long long)'}')};
 
             // make a tokens_view
             {
@@ -283,470 +283,535 @@ sheet image_size
 )";
 
                 tok_t const expected[] = {
-                    tok_t((int)adobe_tokens::lead_comment, R"(/*
+                    tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
     Copyright 2005-2007 Adobe Systems Incorporated
     Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
     or a copy at http://stlab.adobe.com/licenses.html)
 */)"),
-                    tok_t((int)adobe_tokens::identifier, "sheet"),
-                    tok_t((int)adobe_tokens::identifier, "image_size"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "input"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)1600.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)1200.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "original_resolution"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)300.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "constant"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_width"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_height"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "interface"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 1ll),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "unlink"),
-                    tok_t((int)adobe_tokens::identifier, "constrain"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 1ll),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "constrain"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 1ll),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "unlink"),
-                    tok_t((int)adobe_tokens::identifier, "scale_styles"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 1ll),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)adobe_tokens::and_, "&&"),
-                    tok_t((int)adobe_tokens::identifier, "constrain"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "scale_styles"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0ll),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "resample_method"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "bicubic"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "round"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "round"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_width"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::lead_comment, R"(/*
+                    tok_t((int)adobe_tokens::identifier, 0, "sheet"),
+                    tok_t((int)adobe_tokens::identifier, 0, "image_size"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "input"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1600.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1200.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)300.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "constant"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_doc_height"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "interface"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "unlink"),
+                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "unlink"),
+                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)adobe_tokens::and_, 0, "&&"),
+                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::keyword_true_false, 0, 0ll),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "bicubic"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::identifier, 0, "round"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::identifier, 0, "round"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
         Resolution must be initialized before width and height inches to allow proportions
         to be constrained.
     */)"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_height"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_percent"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "auto_quality"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "draft"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "screen_lpi"),
-                    tok_t((int)bp::character_id, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_doc_height"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
                     tok_t(
                         (int)adobe_tokens::trail_comment,
+                        0,
                         "// initialized from doc_resolution"),
-                    tok_t((int)adobe_tokens::identifier, "logic"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_percent"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_width"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_width"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_percent"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_height"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_doc_height"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "screen_lpi"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, "=="),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "draft"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, (long double)1.0),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, "=="),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "good"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, (long double)1.5),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)2.0),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "screen_lpi"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, "=="),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "draft"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, (long double)1.0),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, "=="),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "good"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, (long double)1.5),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::number, (long double)2.0),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::number, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)bp::character_id, (long long)'!'),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)bp::character_id, (long long)'!'),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, "/"),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "when"),
-                    tok_t((int)bp::character_id, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, "constrain"),
-                    tok_t((int)adobe_tokens::and_, "&&"),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, "relate"),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_percent"),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, "output"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "byte_count"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, "*"),
-                    tok_t((int)adobe_tokens::number, (long double)32.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "result"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "resample"),
-                    tok_t((int)bp::character_id, (long long)'?'),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "command"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "resize_image"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "width"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "height"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "resolution"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "scale_styles"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "scale_styles"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "resample_method"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "resample_method"),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, "command"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)bp::character_id, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, "set_resolution"),
-                    tok_t((int)bp::character_id, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, "resolution"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "doc_resolution"),
-                    tok_t((int)bp::character_id, (long long)'}'),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "invariant"),
-                    tok_t((int)bp::character_id, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, "width_max"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::rel_op, "<"),
-                    tok_t((int)bp::character_id, (long long)'='),
-                    tok_t((int)adobe_tokens::number, (long double)300000.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, "height_max"),
-                    tok_t((int)adobe_tokens::define, "<=="),
-                    tok_t((int)adobe_tokens::identifier, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::rel_op, "<"),
-                    tok_t((int)bp::character_id, (long long)'='),
-                    tok_t((int)adobe_tokens::number, (long double)300000.0),
-                    tok_t((int)bp::character_id, (long long)';'),
-                    tok_t((int)bp::character_id, (long long)'}')};
+                    tok_t((int)adobe_tokens::identifier, 0, "logic"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_doc_height"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier,
+                        0,
+                        "original_doc_height"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
+                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1.0),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
+                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "good"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1.5),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)2.0),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
+                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1.0),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
+                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "good"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)1.5),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::number, 0, (long double)2.0),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)bp::character_id, 0, (long long)'!'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)bp::character_id, 0, (long long)'!'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "when"),
+                    tok_t((int)bp::character_id, 0, (long long)'('),
+                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
+                    tok_t((int)adobe_tokens::and_, 0, "&&"),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)')'),
+                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)adobe_tokens::identifier, 0, "output"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "byte_count"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
+                    tok_t((int)adobe_tokens::number, 0, (long double)32.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "result"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
+                    tok_t((int)bp::character_id, 0, (long long)'?'),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "command"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resize_image"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "width"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "height"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'{'),
+                    tok_t((int)adobe_tokens::identifier, 0, "command"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)bp::character_id, 0, (long long)'@'),
+                    tok_t((int)adobe_tokens::identifier, 0, "set_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)','),
+                    tok_t((int)adobe_tokens::identifier, 0, "resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
+                    tok_t((int)bp::character_id, 0, (long long)'}'),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "invariant"),
+                    tok_t((int)bp::character_id, 0, (long long)':'),
+                    tok_t((int)adobe_tokens::identifier, 0, "width_max"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
+                    tok_t((int)adobe_tokens::rel_op, 0, "<"),
+                    tok_t((int)bp::character_id, 0, (long long)'='),
+                    tok_t((int)adobe_tokens::number, 0, (long double)300000.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)adobe_tokens::identifier, 0, "height_max"),
+                    tok_t((int)adobe_tokens::define, 0, "<=="),
+                    tok_t(
+                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
+                    tok_t((int)adobe_tokens::rel_op, 0, "<"),
+                    tok_t((int)bp::character_id, 0, (long long)'='),
+                    tok_t((int)adobe_tokens::number, 0, (long double)300000.0),
+                    tok_t((int)bp::character_id, 0, (long long)';'),
+                    tok_t((int)bp::character_id, 0, (long long)'}')};
 
                 int position = 0;
                 for (auto tok :