Add token_parser::operator()(range) to support matchign against token…

…s that producestring_views. See #202.
boostorg · Nov 9, 2024 · 43b6a2f · 43b6a2f
1 parent c624681
commit 43b6a2f
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 12 deletions.
diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp
@@ -476,10 +476,10 @@ namespace boost { namespace parser {
             auto ID2,
             typename ValueType,
             int Base>
-        constexpr auto operator|(
-            parser_interface<token_parser<
-                token_spec_t<RegexStr2, ID2, ValueType, Base>>> const & rhs)
-            const;
+        constexpr auto
+        operator|(parser_interface<token_parser<
+                      token_spec_t<RegexStr2, ID2, ValueType, Base>,
+                      detail::nope>> const &) const;
 
         template<auto Ch, auto... Chs>
         constexpr auto

diff --git a/include/boost/parser/parser_fwd.hpp b/include/boost/parser/parser_fwd.hpp
@@ -450,7 +450,7 @@ namespace boost { namespace parser {
     {};
 
     /** TODO */
-    template<typename TokenSpec, typename Expected = detail::nope>
+    template<typename TokenSpec, typename Expected>
     struct token_parser;
 
     /** Applies at most one of the parsers in `OrParser`.  If `switch_value_`

diff --git a/include/boost/parser/token_parser.hpp b/include/boost/parser/token_parser.hpp
@@ -11,6 +11,8 @@
 #include <boost/parser/concepts.hpp>
 #include <boost/parser/error_handling.hpp>
 
+#include <algorithm>
+
 
 namespace boost { namespace parser {
 
@@ -42,10 +44,24 @@ namespace boost { namespace parser {
         template<typename T>
         struct token_with_value
         {
-            explicit token_with_value(T value) : value_(std::move(value)) {}
-            bool matches(T const & value) const { return value == value_; }
+            explicit token_with_value(T value) : value_(value) {}
+            bool matches(T value) const { return value == value_; }
             T value_;
         };
+
+        template<typename Subrange>
+        struct token_with_string_view
+        {
+            explicit token_with_string_view(Subrange value) : value_(value) {}
+
+            template<typename CharType>
+            bool matches(std::basic_string_view<CharType> value) const
+            {
+                return std::ranges::equal(value, value_);
+            }
+
+            Subrange value_;
+        };
     }
 
 #ifndef BOOST_PARSER_DOXYGEN
@@ -149,6 +165,27 @@ namespace boost { namespace parser {
                 detail::token_with_value(std::move(value))));
         }
 
+        template<parsable_range_like R>
+        constexpr auto operator()(R && r) const noexcept
+        {
+            BOOST_PARSER_ASSERT(
+                ((!std::is_rvalue_reference_v<R &&> ||
+                  !detail::is_range<detail::remove_cv_ref_t<R>>) &&
+                 "It looks like you tried to pass an rvalue range to "
+                 "token_spec().  Don't do that, or you'll end up with dangling "
+                 "references."));
+            BOOST_PARSER_ASSERT(
+                (detail::is_nope_v<Expected> &&
+                 "If you're seeing this, you tried to chain calls on "
+                 "token_spec, like 'token_spec(char-set)(char-set)'.  Quit "
+                 "it!'"));
+            auto expected =
+                detail::token_with_string_view{BOOST_PARSER_SUBRANGE(
+                    std::ranges::begin(r), std::ranges::end(r))};
+            return parser_interface(
+                token_parser<token_spec, decltype(expected)>(expected));
+        }
+
         Expected expected_;
     };
 
@@ -161,7 +198,7 @@ namespace boost { namespace parser {
         typename ValueType = string_view_tag,
         int Base = 10>
     constexpr parser_interface token_spec{
-        token_parser<token_spec_t<Regex, ID, ValueType, Base>>()};
+        token_parser<token_spec_t<Regex, ID, ValueType, Base>, detail::nope>()};
 
 #ifndef BOOST_PARSER_DOXYGEN
 
@@ -180,7 +217,8 @@ namespace boost { namespace parser {
     constexpr auto
     lexer_t<CharType, ID, WsStr, RegexStr, IDs, Specs>::operator|(
         parser_interface<token_parser<
-            token_spec_t<RegexStr2, ID2, ValueType, Base>>> const &) const
+            token_spec_t<RegexStr2, ID2, ValueType, Base>,
+            detail::nope>> const &) const
     {
         static_assert(
             std::same_as<ID, decltype(ID2)>,

diff --git a/test/lexer_and_parser.cpp b/test/lexer_and_parser.cpp
@@ -61,9 +61,8 @@ int main()
         (void)parser2;
     }
     {
-        // TODO: identifier("foo") does not work!
-        auto parser1 = identifier >> '=' >> true_false >> ';';
-        (void)parser1;
+        auto parser = identifier("foo") >> '=' >> true_false >> ';';
+        (void)parser;
     }
 
     // TODO    {