Unify the implementations of the non-callback parse() API, to make it…

… posible

for the new token parsingcode path to use that one unified implementation too.
The previous implementation dispatched like parse() -> prefix_parse(), the
latter of which is incompatible with token parsing.

include/boost/parser/concepts.hpp

@@ -12,6 +12,8 @@
 
 #if defined(BOOST_PARSER_DOXYGEN) || BOOST_PARSER_USE_CONCEPTS
 
+#include <boost/parser/lexer_fwd.hpp>
+
 #include <ranges>
 
 
@@ -33,8 +35,9 @@ namespace boost { namespace parser {
     //[ parsable_range_like_concept
     //[ parsable_range_concept
     template<typename T>
-    concept parsable_range = std::ranges::forward_range<T> &&
-        code_unit<std::ranges::range_value_t<T>>;
+    concept parsable_range = (std::ranges::forward_range<T> &&
+                              code_unit<std::ranges::range_value_t<T>>) ||
+                             detail::is_tokens_view_v<T>;
     //]
 
     template<typename T>

include/boost/parser/lexer.hpp

@@ -17,6 +17,7 @@
 #error "lexer.hpp must be included before parser.hpp."
 #endif
 
+#include <boost/parser/lexer_fwd.hpp>
 #include <boost/parser/parser_fwd.hpp>
 #include <boost/parser/concepts.hpp>
 #include <boost/parser/detail/debug_assert.hpp>
@@ -750,7 +751,7 @@ namespace boost { namespace parser {
     template<
         std::ranges::contiguous_range V,
         typename Lexer,
-        typename TokenCache = std::vector<typename Lexer::token_type>>
+        typename TokenCache>
         requires std::ranges::view<V>
     struct tokens_view
         : public std::ranges::view_interface<tokens_view<V, Lexer, TokenCache>>

include/boost/parser/lexer_fwd.hpp

@@ -0,0 +1,32 @@
+// Copyright (C) 2024 T. Zachary Laine
+//
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+#ifndef BOOST_PARSER_LEXER_FWD_HPP
+#define BOOST_PARSER_LEXER_FWD_HPP
+
+#include <ranges>
+#include <vector>
+
+namespace boost { namespace parser {
+
+    /** TODO */
+    template<
+        std::ranges::contiguous_range V,
+        typename Lexer,
+        typename TokenCache = std::vector<typename Lexer::token_type>>
+        requires std::ranges::view<V>
+    struct tokens_view;
+
+    namespace detail {
+        template<typename T>
+        constexpr bool is_tokens_view_v = false;
+        template<typename V, typename Lexer, typename TokenCache>
+        constexpr bool is_tokens_view_v<tokens_view<V, Lexer, TokenCache>> =
+            true;
+    }
+
+}}
+
+#endif