From 8830dfed0210e2057de5a3be4b8ef91a8bcf757f Mon Sep 17 00:00:00 2001
From: Zach Laine <whatwasthataddress@gmail.com>
Date: Sun, 10 Nov 2024 01:05:34 -0600
Subject: [PATCH] Alter detail::make_input_subrange() to return its input
 unchanged when the range's value type is a specializtion of token.

See #202.
---
 include/boost/parser/parser.hpp | 48 ++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/include/boost/parser/parser.hpp b/include/boost/parser/parser.hpp
index ef0c8438..75764eec 100644
--- a/include/boost/parser/parser.hpp
+++ b/include/boost/parser/parser.hpp
@@ -324,6 +324,25 @@ namespace boost { namespace parser {
     };
 #endif
 
+#ifdef BOOST_PARSER_DOXYGEN
+
+    /** TODO */
+    template<typename T>
+    constexpr bool is_token_v = detail::foo;
+
+#else
+
+    template<typename CharType>
+    struct token;
+
+    template<typename T>
+    constexpr bool is_token_v = false;
+
+    template<typename CharType>
+    constexpr bool is_token_v<token<CharType>> = true;
+
+#endif
+
     namespace detail {
         // Follows boost/mpl/print.hpp.
 #if defined(_MSC_VER)
@@ -2614,7 +2633,13 @@ namespace boost { namespace parser {
         template<typename I, typename S>
         constexpr auto make_input_subrange(I first, S last) noexcept
         {
-            return BOOST_PARSER_SUBRANGE(first, last) | detail::text::as_utf32;
+            using value_type = iter_value_t<I>;
+            if constexpr (is_token_v<value_type>) {
+                return BOOST_PARSER_SUBRANGE(first, last);
+            } else {
+                return BOOST_PARSER_SUBRANGE(first, last) |
+                       detail::text::as_utf32;
+            }
         }
 
         template<typename R>
@@ -2622,7 +2647,9 @@ namespace boost { namespace parser {
         {
             using r_t = remove_cv_ref_t<R>;
             using value_type = range_value_t<r_t>;
-            if constexpr (text::detail::is_bounded_array_v<r_t>) {
+            if constexpr (is_token_v<value_type>) {
+                return (R &&)r;
+            } else if constexpr (text::detail::is_bounded_array_v<r_t>) {
                 if constexpr (std::is_same_v<value_type, char>) {
                     auto first = detail::text::detail::begin(r);
                     auto last = detail::text::detail::end(r);
@@ -2849,22 +2876,7 @@ namespace boost { namespace parser {
     }
 
 
-#if BOOST_PARSER_DOXYGEN
-
-    /** TODO */
-    template<typename T>
-    constexpr bool is_token_v = detail::foo;
-
-#else
-
-    template<typename CharType>
-    struct token;
-
-    template<typename T>
-    constexpr bool is_token_v = false;
-
-    template<typename CharType>
-    constexpr bool is_token_v<token<CharType>> = true;
+#ifndef BOOST_PARSER_DOXYGEN
 
     // This constraint is only here to allow the alternate-call semantic
     // action metaprogramming logic to function on MSVC.