diff --git a/include/boost/parser/concepts.hpp b/include/boost/parser/concepts.hpp index 5f7295c0..b4049899 100644 --- a/include/boost/parser/concepts.hpp +++ b/include/boost/parser/concepts.hpp @@ -12,6 +12,8 @@ #if defined(BOOST_PARSER_DOXYGEN) || BOOST_PARSER_USE_CONCEPTS +#include + #include @@ -33,8 +35,9 @@ namespace boost { namespace parser { //[ parsable_range_like_concept //[ parsable_range_concept template - concept parsable_range = std::ranges::forward_range && - code_unit>; + concept parsable_range = (std::ranges::forward_range && + code_unit>) || + detail::is_tokens_view_v; //] template diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp index cb693828..68e8bc9c 100644 --- a/include/boost/parser/lexer.hpp +++ b/include/boost/parser/lexer.hpp @@ -17,6 +17,7 @@ #error "lexer.hpp must be included before parser.hpp." #endif +#include #include #include #include @@ -750,7 +751,7 @@ namespace boost { namespace parser { template< std::ranges::contiguous_range V, typename Lexer, - typename TokenCache = std::vector> + typename TokenCache> requires std::ranges::view struct tokens_view : public std::ranges::view_interface> diff --git a/include/boost/parser/lexer_fwd.hpp b/include/boost/parser/lexer_fwd.hpp new file mode 100644 index 00000000..fc0ea59b --- /dev/null +++ b/include/boost/parser/lexer_fwd.hpp @@ -0,0 +1,32 @@ +// Copyright (C) 2024 T. Zachary Laine +// +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +#ifndef BOOST_PARSER_LEXER_FWD_HPP +#define BOOST_PARSER_LEXER_FWD_HPP + +#include +#include + +namespace boost { namespace parser { + + /** TODO */ + template< + std::ranges::contiguous_range V, + typename Lexer, + typename TokenCache = std::vector> + requires std::ranges::view + struct tokens_view; + + namespace detail { + template + constexpr bool is_tokens_view_v = false; + template + constexpr bool is_tokens_view_v> = + true; + } + +}} + +#endif diff --git a/include/boost/parser/parser.hpp b/include/boost/parser/parser.hpp index cbc02e11..be141828 100644 --- a/include/boost/parser/parser.hpp +++ b/include/boost/parser/parser.hpp @@ -8492,6 +8492,135 @@ namespace boost { namespace parser { `*parse()` functions. */ enum class trace { off, on }; + namespace detail { + + template + auto get_error_handler_iterator(R & r) + { + if constexpr (is_token_iter_v>) { + return detail::text::detail::begin(r).range_begin(); + } else { + return detail::text::detail::begin(r); + } + } + + template + auto get_error_handler_sentinel(R & r) + { + if constexpr (is_token_iter_v>) { + return detail::text::detail::begin(r).range_end(); + } else { + return detail::text::detail::begin(r); + } + } + + template< + typename Attr, + typename I, + typename S, + typename Parser, + typename SkipParser> + auto prefix_parse_impl( + I & first, + S last, + Parser const & parser, + SkipParser const & skip, + Attr attr, + trace trace_mode) + { + // TODO: shouldn't this be detail::is_char_iter_v instead? + if constexpr (!detail::is_char8_iter_v) { + if constexpr (std::is_reference_v) { + detail::attr_reset reset(attr); + static_assert( + decltype(detail::has_attribute(first, last, parser)){}, + "If you're seeing this error, you're trying to get " + "parse() to fill in attr above, using the attribute " + "generated by parser. However, parser does not " + "generate an attribute."); + if (trace_mode == trace::on) { + return !!( + reset = detail::parse_impl( + first, + last, + parser, + skip, + parser.error_handler_, + attr)); + } else { + return !!( + reset = detail::parse_impl( + first, + last, + parser, + skip, + parser.error_handler_, + attr)); + } + } else { + if (trace_mode == trace::on) { + return detail::parse_impl( + first, + last, + parser, + skip, + parser.error_handler_, + attr); + } else { + return detail::parse_impl( + first, + last, + parser, + skip, + parser.error_handler_, + attr); + } + } + } else { + auto r = detail::make_input_subrange(first, last); + auto f = r.begin(); + auto const l = r.end(); + auto _ = detail::scoped_base_assign(first, f); + if constexpr (std::is_reference_v) { + detail::attr_reset reset(attr); + static_assert( + decltype(detail::has_attribute(f, l, parser)){}, + "If you're seeing this error, you're trying to get " + "parse() to fill in attr above, using the attribute " + "generated by parser. However, parser does not " + "generate an attribute."); + if (trace_mode == trace::on) { + return !!( + reset = detail::parse_impl( + f, + l, + parser, + skip, + parser.error_handler_, + attr)); + } else { + return !!( + reset = detail::parse_impl( + f, + l, + parser, + skip, + parser.error_handler_, + attr)); + } + } else { + if (trace_mode == trace::on) { + return detail::parse_impl( + f, l, parser, skip, parser.error_handler_, attr); + } else { + return detail::parse_impl( + f, l, parser, skip, parser.error_handler_, attr); + } + } + } + } + } + // Parse API. /** Parses `[first, last)` using `parser`, and returns whether the parse @@ -8536,58 +8665,8 @@ namespace boost { namespace parser { // clang-format on #endif { - detail::attr_reset reset(attr); - if constexpr (!detail::is_char8_iter_v) { - static_assert( - decltype(detail::has_attribute(first, last, parser)){}, - "If you're seeing this error, you're trying to get parse() to " - "fill in attr above, using the attribute generated by parser. " - "However, parser does not generate an attribute."); - if (trace_mode == trace::on) { - return reset = detail::parse_impl( - first, - last, - parser, - detail::null_parser{}, - parser.error_handler_, - attr); - } else { - return reset = detail::parse_impl( - first, - last, - parser, - detail::null_parser{}, - parser.error_handler_, - attr); - } - } else { - auto r = detail::make_input_subrange(first, last); - auto f = r.begin(); - auto const l = r.end(); - auto _ = detail::scoped_base_assign(first, f); - static_assert( - decltype(detail::has_attribute(f, l, parser)){}, - "If you're seeing this error, you're trying to get parse() to " - "fill in attr above, using the attribute generated by parser. " - "However, parser does not generate an attribute."); - if (trace_mode == trace::on) { - return reset = detail::parse_impl( - f, - l, - parser, - detail::null_parser{}, - parser.error_handler_, - attr); - } else { - return reset = detail::parse_impl( - f, - l, - parser, - detail::null_parser{}, - parser.error_handler_, - attr); - } - } + return detail::prefix_parse_impl( + first, last, parser, detail::null_parser{}, attr, trace_mode); } /** Parses `r` using `parser`, and returns whether the parse was @@ -8632,8 +8711,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> && (!detail::derived_from_parser_interface_v>) // clang-format on @@ -8646,7 +8725,13 @@ namespace boost { namespace parser { return reset = detail::if_full_parse( first, last, - parser::prefix_parse(first, last, parser, attr, trace_mode)); + detail::prefix_parse_impl( + first, + last, + parser, + detail::null_parser{}, + attr, + trace_mode)); } /** Parses `[first, last)` using `parser`. Returns a `std::optional` @@ -8680,47 +8765,13 @@ namespace boost { namespace parser { parser_interface const & parser, trace trace_mode = trace::off) { - if constexpr (!detail::is_char8_iter_v) { - if (trace_mode == trace::on) { - return detail::parse_impl( - first, - last, - parser, - detail::null_parser{}, - parser.error_handler_, - detail::nope{}); - } else { - return detail::parse_impl( - first, - last, - parser, - detail::null_parser{}, - parser.error_handler_, - detail::nope{}); - } - } else { - auto r = detail::make_input_subrange(first, last); - auto f = r.begin(); - auto const l = r.end(); - auto _ = detail::scoped_base_assign(first, f); - if (trace_mode == trace::on) { - return detail::parse_impl( - f, - l, - parser, - detail::null_parser{}, - parser.error_handler_, - detail::nope{}); - } else { - return detail::parse_impl( - f, - l, - parser, - detail::null_parser{}, - parser.error_handler_, - detail::nope{}); - } - } + return detail::prefix_parse_impl( + first, + last, + parser, + detail::null_parser{}, + detail::nope{}, + trace_mode); } /** Parses `r` using `parser`. Returns a `std::optional` containing the @@ -8758,8 +8809,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> // clang-format on #endif @@ -8768,7 +8819,15 @@ namespace boost { namespace parser { auto first = r_.begin(); auto const last = r_.end(); return detail::if_full_parse( - first, last, parser::prefix_parse(first, last, parser, trace_mode)); + first, + last, + detail::prefix_parse_impl( + first, + last, + parser, + detail::null_parser{}, + detail::nope{}, + trace_mode)); } /** Parses `[first, last)` using `parser`, skipping all input recognized @@ -8807,48 +8866,8 @@ namespace boost { namespace parser { Attr & attr, trace trace_mode = trace::off) { - detail::attr_reset reset(attr); - if constexpr (!detail::is_char8_iter_v) { - static_assert( - decltype(detail::has_attribute(first, last, parser)){}, - "If you're seeing this error, you're trying to get parse() to " - "fill in attr above, using the attribute generated by parser. " - "However, parser does not generate an attribute."); - if (trace_mode == trace::on) { - return reset = detail::parse_impl( - first, - last, - parser, - skip, - parser.error_handler_, - attr); - } else { - return reset = detail::parse_impl( - first, - last, - parser, - skip, - parser.error_handler_, - attr); - } - } else { - auto r = detail::make_input_subrange(first, last); - auto f = r.begin(); - auto const l = r.end(); - auto _ = detail::scoped_base_assign(first, f); - static_assert( - decltype(detail::has_attribute(f, l, parser)){}, - "If you're seeing this error, you're trying to get parse() to " - "fill in attr above, using the attribute generated by parser. " - "However, parser does not generate an attribute."); - if (trace_mode == trace::on) { - return reset = detail::parse_impl( - f, l, parser, skip, parser.error_handler_, attr); - } else { - return reset = detail::parse_impl( - f, l, parser, skip, parser.error_handler_, attr); - } - } + return detail::prefix_parse_impl( + first, last, parser, skip, attr, trace_mode); } /** Parses `r` using `parser`, skipping all input recognized by `skip` @@ -8892,8 +8911,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> // clang-format on #endif @@ -8905,7 +8924,7 @@ namespace boost { namespace parser { return reset = detail::if_full_parse( first, last, - parser::prefix_parse( + detail::prefix_parse_impl( first, last, parser, skip, attr, trace_mode)); } @@ -8942,37 +8961,8 @@ namespace boost { namespace parser { parser_interface const & skip, trace trace_mode = trace::off) { - if constexpr (!detail::is_char8_iter_v) { - if (trace_mode == trace::on) { - return detail::parse_impl( - first, - last, - parser, - skip, - parser.error_handler_, - detail::nope{}); - } else { - return detail::parse_impl( - first, - last, - parser, - skip, - parser.error_handler_, - detail::nope{}); - } - } else { - auto r = detail::make_input_subrange(first, last); - auto f = r.begin(); - auto const l = r.end(); - auto _ = detail::scoped_base_assign(first, f); - if (trace_mode == trace::on) { - return detail::parse_impl( - f, l, parser, skip, parser.error_handler_, detail::nope{}); - } else { - return detail::parse_impl( - f, l, parser, skip, parser.error_handler_, detail::nope{}); - } - } + return detail::prefix_parse_impl( + first, last, parser, skip, detail::nope{}, trace_mode); } /** Parses `r` using `parser`, skipping all input recognized by `skip` @@ -9013,8 +9003,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> // clang-format on #endif @@ -9025,7 +9015,8 @@ namespace boost { namespace parser { return detail::if_full_parse( first, last, - parser::prefix_parse(first, last, parser, skip, trace_mode)); + detail::prefix_parse_impl( + first, last, parser, skip, detail::nope{}, trace_mode)); } /** Parses `[first, last)` using `parser`, and returns whether the parse @@ -9150,8 +9141,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> // clang-format on #endif @@ -9285,8 +9276,8 @@ namespace boost { namespace parser { // clang-format off requires error_handler< ErrorHandler, - std::ranges::iterator_t, - std::ranges::sentinel_t, + decltype(detail::get_error_handler_iterator(r)), + decltype(detail::get_error_handler_sentinel(r)), GlobalState> // clang-format on #endif diff --git a/test/lexer_and_parser.cpp b/test/lexer_and_parser.cpp index 50a34ced..ad291428 100644 --- a/test/lexer_and_parser.cpp +++ b/test/lexer_and_parser.cpp @@ -65,6 +65,13 @@ int main() (void)parser; } + // Minimal tests of using a lexer and parser together. + { + auto parser = identifier("foo") >> '=' >> true_false >> ';'; + auto r = "some input" | bp::to_tokens(adobe_lexer); + // TODO auto result = bp::parse(r, parser); + } + // TODO { // TODO std::string str = "a"; // TODO BOOST_TEST(parse(str, char_));