Sketch in support for matching tokens against expected values in toke…

…n_parser.

See #202.

include/boost/parser/lexer.hpp

@@ -140,12 +140,6 @@ namespace boost { namespace parser {
         };
     }
 
-    /** TODO */
-    inline constexpr int ws_id = -1000000;
-
-    /** TODO */
-    inline constexpr int character_id = -2000000;
-
     /** TODO */
     inline constexpr ctll::fixed_string no_ws = "";
 
@@ -248,22 +242,6 @@ namespace boost { namespace parser {
         detail::token_kind kind_ : 8;
     };
 
-#if BOOST_PARSER_DOXYGEN
-
-    /** TODO */
-    template<typename T>
-    constexpr bool is_token_v = detail::foo;
-
-#else
-
-    template<typename T>
-    constexpr bool is_token_v = false;
-
-    template<typename CharType>
-    constexpr bool is_token_v<token<CharType>> = true;
-
-#endif
-
 #if defined(BOOST_PARSER_TESTING)
     template<typename CharType>
     std::ostream & operator<<(std::ostream & os, token<CharType> const & token)

include/boost/parser/parser.hpp

@@ -2755,7 +2755,23 @@ namespace boost { namespace parser {
         };
     }
 
-#ifndef BOOST_PARSER_DOXYGEN
+
+#if BOOST_PARSER_DOXYGEN
+
+    /** TODO */
+    template<typename T>
+    constexpr bool is_token_v = detail::foo;
+
+#else
+
+    template<typename CharType>
+    struct token;
+
+    template<typename T>
+    constexpr bool is_token_v = false;
+
+    template<typename CharType>
+    constexpr bool is_token_v<token<CharType>> = true;
 
     // This constraint is only here to allow the alternate-call semantic
     // action metaprogramming logic to function on MSVC.
@@ -6573,11 +6589,29 @@ namespace boost { namespace parser {
                 success = false;
                 return;
             }
-            attribute_type<decltype(*first)> const x = *first;
-            if (detail::unequal(context, x, expected_)) {
-                success = false;
-                return;
+
+            using attribute_t = attribute_type<decltype(*first)>;
+            attribute_t x = 0;
+
+            if constexpr (is_token_v<std::decay_t<decltype(*first)>>) {
+                if (first->id() != character_id || !first->has_long_long()) {
+                    success = false;
+                    return;
+                }
+                char32_t c = (char32_t)first->has_long_long();
+                if (detail::unequal(context, c, expected_)) {
+                    success = false;
+                    return;
+                }
+                x = (attribute_t)c;
+            } else {
+                x = *first;
+                if (detail::unequal(context, x, expected_)) {
+                    success = false;
+                    return;
+                }
             }
+
             detail::assign(retval, x);
             ++first;
         }

include/boost/parser/parser_fwd.hpp

@@ -68,6 +68,12 @@ namespace boost { namespace parser {
         return BOOST_PARSER_SUBRANGE(ptr, detail::text::null_sentinel);
     }
 
+    /** The token ID used for whitespace tokens. */
+    inline constexpr int ws_id = -1000000;
+
+    /** The token ID used for single-character tokens. */
+    inline constexpr int character_id = -2000000;
+
     namespace detail {
         template<typename T>
         constexpr bool is_optional_v = enable_optional<T>;
@@ -147,6 +153,17 @@ namespace boost { namespace parser {
         {};
         struct upper_case_chars
         {};
+
+        struct any_token
+        {
+            bool matches_id(int) const { return true; }
+
+            template<typename T>
+            bool matches_value(T) const
+            {
+                return true;
+            }
+        };
     }
 
     /** Repeats the application of another parser `p` of type `Parser`,
@@ -428,8 +445,15 @@ namespace boost { namespace parser {
     template<typename T>
     struct float_parser;
 
+    /** A tag type used to stand in for any specialization of
+        `boost::parser::token<>`. */
+    struct token_tag
+    {};
+
     /** TODO */
-    template<typename AttributeType = void>
+    template<
+        typename AttributeType = token_tag,
+        typename Expected = detail::any_token>
     struct token_parser;
 
     /** Applies at most one of the parsers in `OrParser`.  If `switch_value_`

include/boost/parser/token_parser.hpp

@@ -42,20 +42,57 @@ namespace boost { namespace parser {
                     "integral values (including charater types).");
             }
         }
+
+        struct token_with_id
+        {
+            explicit token_with_id(int id) : id_(id) {}
+
+            bool matches(int id) const { return id == id_; }
+
+            template<typename T>
+            bool matches_value(T) const
+            {
+                return true;
+            }
+
+            int id_;
+        };
+
+        template<typename T>
+        struct token_with_id_and_value
+        {
+            explicit token_with_id_and_value(int id, T value) :
+                id_(id), value_(value)
+            {}
+
+            bool matches(int id) const { return id == id_; }
+            bool matches_value(T value) const { return value == value_; }
+
+            int id_;
+            T value_;
+        };
     }
 
 #ifndef BOOST_PARSER_DOXYGEN
 
     // TODO: Needs a printer.
-    template<typename AttributeType>
+    // TODO: Constrain the AttributeType to something that detail::token_as()
+    // can handle.
+    template<typename AttributeType, typename Expected>
     struct token_parser
     {
         using attribute_type = std::conditional_t<
             std::is_same_v<AttributeType, void>,
             detail::nope,
             AttributeType>;
 
-        constexpr token_parser() {}
+        using expected_value_type = std::conditional_t<
+            std::is_same_v<attribute_type, token_tag>,
+            detail::nope,
+            attribute_type>;
+
+        constexpr token_parser() = default;
+        constexpr token_parser(Expected expected) : expected_(expected) {}
 
         template<
             typename Iter,
@@ -103,22 +140,68 @@ namespace boost { namespace parser {
                 success = false;
                 return;
             }
+
             value_type const x = *first;
-            // TODO: Test for equality with some expectation, if any.
-            auto opt_attr = detail::token_as<attribute_type>(x);
-            if (!opt_attr) {
+            if (!expected_.matches_id(x.id())) {
                 success = false;
                 return;
             }
-            detail::assign(retval, *opt_attr);
+
+            if constexpr (std::same_as<AttributeType, token_tag>) {
+                detail::assign(retval, x);
+            } else {
+                auto opt_attr = detail::token_as<attribute_type>(x);
+                if (!opt_attr || !expected_.matches_value(*opt_attr)) {
+                    success = false;
+                    return;
+                }
+                detail::assign(retval, *opt_attr);
+            }
+
             ++first;
         }
+
+        // TODO: token_spec_t needs these same operator() overloads, each of
+        // which will return a token_parser.
+
+        // TODO: Constrain both ID params below only to accept type
+        // convertible to int.
+
+        /** TODO */
+        template<typename ID>
+        constexpr auto operator()(ID id) const noexcept
+        {
+            BOOST_PARSER_ASSERT(
+                (detail::is_nope_v<Expected> &&
+                 "If you're seeing this, you tried to chain calls on tok or "
+                 "tok_t, like 'tok(id1)(id2)'.  Quit it!'"));
+            return parser_interface(detail::token_with_id((int)id));
+        }
+
+        /** TODO */
+        template<typename ID>
+        constexpr auto
+        operator()(ID id, expected_value_type value) const noexcept
+        {
+            BOOST_PARSER_ASSERT(
+                (detail::is_nope_v<Expected> &&
+                 "If you're seeing this, you tried to chain calls on tok or "
+                 "tok_t, like 'tok(id1)(id2)'.  Quit it!'"));
+            return parser_interface(
+                detail::token_with_id_and_value((int)id, value));
+        }
+
+        Expected expected_;
     };
 
 #endif
 
     /** TODO */
-    constexpr parser_interface<token_parser<detail::nope>> tok;
+    constexpr parser_interface<token_parser<>> tok;
+
+    /** TODO */
+    template<typename AttributeType>
+    constexpr parser_interface<token_parser<AttributeType>> tok_t;
 
 }}