From 93e0aa2ab788016e60e95d29e7fd2e014d6c7776 Mon Sep 17 00:00:00 2001
From: Zach Laine <whatwasthataddress@gmail.com>
Date: Sat, 9 Nov 2024 16:44:57 -0600
Subject: [PATCH] Add token_parser::operator()(range) to support matchign
 against tokens that producestring_views.

See #202.
---
 include/boost/parser/lexer.hpp        |  8 ++---
 include/boost/parser/parser_fwd.hpp   |  2 +-
 include/boost/parser/token_parser.hpp | 46 ++++++++++++++++++++++++---
 test/lexer_and_parser.cpp             |  3 +-
 4 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp
index 65e02b7b..0fd39b7f 100644
--- a/include/boost/parser/lexer.hpp
+++ b/include/boost/parser/lexer.hpp
@@ -476,10 +476,10 @@ namespace boost { namespace parser {
             auto ID2,
             typename ValueType,
             int Base>
-        constexpr auto operator|(
-            parser_interface<token_parser<
-                token_spec_t<RegexStr2, ID2, ValueType, Base>>> const & rhs)
-            const;
+        constexpr auto
+        operator|(parser_interface<token_parser<
+                      token_spec_t<RegexStr2, ID2, ValueType, Base>,
+                      detail::nope>> const &) const;
 
         template<auto Ch, auto... Chs>
         constexpr auto
diff --git a/include/boost/parser/parser_fwd.hpp b/include/boost/parser/parser_fwd.hpp
index 8ca0db27..d3ad0f59 100644
--- a/include/boost/parser/parser_fwd.hpp
+++ b/include/boost/parser/parser_fwd.hpp
@@ -450,7 +450,7 @@ namespace boost { namespace parser {
     {};
 
     /** TODO */
-    template<typename TokenSpec, typename Expected = detail::nope>
+    template<typename TokenSpec, typename Expected>
     struct token_parser;
 
     /** Applies at most one of the parsers in `OrParser`.  If `switch_value_`
diff --git a/include/boost/parser/token_parser.hpp b/include/boost/parser/token_parser.hpp
index f4784dc8..6ad2cb7f 100644
--- a/include/boost/parser/token_parser.hpp
+++ b/include/boost/parser/token_parser.hpp
@@ -11,6 +11,8 @@
 #include <boost/parser/concepts.hpp>
 #include <boost/parser/error_handling.hpp>
 
+#include <algorithm>
+
 
 namespace boost { namespace parser {
 
@@ -42,10 +44,24 @@ namespace boost { namespace parser {
         template<typename T>
         struct token_with_value
         {
-            explicit token_with_value(T value) : value_(std::move(value)) {}
-            bool matches(T const & value) const { return value == value_; }
+            explicit token_with_value(T value) : value_(value) {}
+            bool matches(T value) const { return value == value_; }
             T value_;
         };
+
+        template<typename Subrange>
+        struct token_with_string_view
+        {
+            explicit token_with_string_view(Subrange value) : value_(value) {}
+
+            template<typename CharType>
+            bool matches(std::basic_string_view<CharType> value) const
+            {
+                return std::ranges::equal(value, value_);
+            }
+
+            Subrange value_;
+        };
     }
 
 #ifndef BOOST_PARSER_DOXYGEN
@@ -149,6 +165,27 @@ namespace boost { namespace parser {
                 detail::token_with_value(std::move(value))));
         }
 
+        template<parsable_range_like R>
+        constexpr auto operator()(R && r) const noexcept
+        {
+            BOOST_PARSER_ASSERT(
+                ((!std::is_rvalue_reference_v<R &&> ||
+                  !detail::is_range<detail::remove_cv_ref_t<R>>) &&
+                 "It looks like you tried to pass an rvalue range to "
+                 "token_spec().  Don't do that, or you'll end up with dangling "
+                 "references."));
+            BOOST_PARSER_ASSERT(
+                (detail::is_nope_v<Expected> &&
+                 "If you're seeing this, you tried to chain calls on "
+                 "token_spec, like 'token_spec(char-set)(char-set)'.  Quit "
+                 "it!'"));
+            auto expected =
+                detail::token_with_string_view{BOOST_PARSER_SUBRANGE(
+                    std::ranges::begin(r), std::ranges::end(r))};
+            return parser_interface(
+                token_parser<token_spec, decltype(expected)>(expected));
+        }
+
         Expected expected_;
     };
 
@@ -161,7 +198,7 @@ namespace boost { namespace parser {
         typename ValueType = string_view_tag,
         int Base = 10>
     constexpr parser_interface token_spec{
-        token_parser<token_spec_t<Regex, ID, ValueType, Base>>()};
+        token_parser<token_spec_t<Regex, ID, ValueType, Base>, detail::nope>()};
 
 #ifndef BOOST_PARSER_DOXYGEN
 
@@ -180,7 +217,8 @@ namespace boost { namespace parser {
     constexpr auto
     lexer_t<CharType, ID, WsStr, RegexStr, IDs, Specs>::operator|(
         parser_interface<token_parser<
-            token_spec_t<RegexStr2, ID2, ValueType, Base>>> const &) const
+            token_spec_t<RegexStr2, ID2, ValueType, Base>,
+            detail::nope>> const &) const
     {
         static_assert(
             std::same_as<ID, decltype(ID2)>,
diff --git a/test/lexer_and_parser.cpp b/test/lexer_and_parser.cpp
index ff9136ff..09e26ebc 100644
--- a/test/lexer_and_parser.cpp
+++ b/test/lexer_and_parser.cpp
@@ -61,8 +61,7 @@ int main()
         (void)parser2;
     }
     {
-        // TODO: identifier("foo") does not work!
-        auto parser1 = identifier >> '=' >> true_false >> ';';
+        auto parser1 = identifier("foo") >> '=' >> true_false >> ';';
         (void)parser1;
     }