diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp index 8bd8a6b1..4b5e6470 100644 --- a/include/boost/parser/lexer.hpp +++ b/include/boost/parser/lexer.hpp @@ -140,12 +140,6 @@ namespace boost { namespace parser { }; } - /** TODO */ - inline constexpr int ws_id = -1000000; - - /** TODO */ - inline constexpr int character_id = -2000000; - /** TODO */ inline constexpr ctll::fixed_string no_ws = ""; @@ -248,22 +242,6 @@ namespace boost { namespace parser { detail::token_kind kind_ : 8; }; -#if BOOST_PARSER_DOXYGEN - - /** TODO */ - template - constexpr bool is_token_v = detail::foo; - -#else - - template - constexpr bool is_token_v = false; - - template - constexpr bool is_token_v> = true; - -#endif - #if defined(BOOST_PARSER_TESTING) template std::ostream & operator<<(std::ostream & os, token const & token) diff --git a/include/boost/parser/parser.hpp b/include/boost/parser/parser.hpp index 641f84e4..8b5faa1d 100644 --- a/include/boost/parser/parser.hpp +++ b/include/boost/parser/parser.hpp @@ -2755,7 +2755,23 @@ namespace boost { namespace parser { }; } -#ifndef BOOST_PARSER_DOXYGEN + +#if BOOST_PARSER_DOXYGEN + + /** TODO */ + template + constexpr bool is_token_v = detail::foo; + +#else + + template + struct token; + + template + constexpr bool is_token_v = false; + + template + constexpr bool is_token_v> = true; // This constraint is only here to allow the alternate-call semantic // action metaprogramming logic to function on MSVC. @@ -6570,11 +6586,29 @@ namespace boost { namespace parser { success = false; return; } - attribute_type const x = *first; - if (detail::unequal(context, x, expected_)) { - success = false; - return; + + using attribute_t = attribute_type; + attribute_t x = 0; + + if constexpr (is_token_v>) { + if (first->id() != character_id || !first->has_long_long()) { + success = false; + return; + } + char32_t c = (char32_t)first->has_long_long(); + if (detail::unequal(context, c, expected_)) { + success = false; + return; + } + x = (attribute_t)c; + } else { + x = *first; + if (detail::unequal(context, x, expected_)) { + success = false; + return; + } } + detail::assign(retval, x); ++first; } diff --git a/include/boost/parser/parser_fwd.hpp b/include/boost/parser/parser_fwd.hpp index ce1558d4..eb8321d7 100644 --- a/include/boost/parser/parser_fwd.hpp +++ b/include/boost/parser/parser_fwd.hpp @@ -68,6 +68,12 @@ namespace boost { namespace parser { return BOOST_PARSER_SUBRANGE(ptr, detail::text::null_sentinel); } + /** The token ID used for whitespace tokens. */ + inline constexpr int ws_id = -1000000; + + /** The token ID used for single-character tokens. */ + inline constexpr int character_id = -2000000; + namespace detail { template constexpr bool is_optional_v = enable_optional; @@ -147,6 +153,17 @@ namespace boost { namespace parser { {}; struct upper_case_chars {}; + + struct any_token + { + bool matches_id(int) const { return true; } + + template + bool matches_value(T) const + { + return true; + } + }; } /** Repeats the application of another parser `p` of type `Parser`, @@ -428,8 +445,15 @@ namespace boost { namespace parser { template struct float_parser; + /** A tag type used to stand in for any specialization of + `boost::parser::token<>`. */ + struct token_tag + {}; + /** TODO */ - template + template< + typename AttributeType = token_tag, + typename Expected = detail::any_token> struct token_parser; /** Applies at most one of the parsers in `OrParser`. If `switch_value_` diff --git a/include/boost/parser/token_parser.hpp b/include/boost/parser/token_parser.hpp index e6ddd959..173230f4 100644 --- a/include/boost/parser/token_parser.hpp +++ b/include/boost/parser/token_parser.hpp @@ -42,12 +42,43 @@ namespace boost { namespace parser { "integral values (including charater types)."); } } + + struct token_with_id + { + explicit token_with_id(int id) : id_(id) {} + + bool matches(int id) const { return id == id_; } + + template + bool matches_value(T) const + { + return true; + } + + int id_; + }; + + template + struct token_with_id_and_value + { + explicit token_with_id_and_value(int id, T value) : + id_(id), value_(value) + {} + + bool matches(int id) const { return id == id_; } + bool matches_value(T value) const { return value == value_; } + + int id_; + T value_; + }; } #ifndef BOOST_PARSER_DOXYGEN // TODO: Needs a printer. - template + // TODO: Constrain the AttributeType to something that detail::token_as() + // can handle. + template struct token_parser { using attribute_type = std::conditional_t< @@ -55,7 +86,13 @@ namespace boost { namespace parser { detail::nope, AttributeType>; - constexpr token_parser() {} + using expected_value_type = std::conditional_t< + std::is_same_v, + detail::nope, + attribute_type>; + + constexpr token_parser() = default; + constexpr token_parser(Expected expected) : expected_(expected) {} template< typename Iter, @@ -103,22 +140,68 @@ namespace boost { namespace parser { success = false; return; } + value_type const x = *first; - // TODO: Test for equality with some expectation, if any. - auto opt_attr = detail::token_as(x); - if (!opt_attr) { + if (!expected_.matches_id(x.id())) { success = false; return; } - detail::assign(retval, *opt_attr); + + if constexpr (std::same_as) { + detail::assign(retval, x); + } else { + auto opt_attr = detail::token_as(x); + if (!opt_attr || !expected_.matches_value(*opt_attr)) { + success = false; + return; + } + detail::assign(retval, *opt_attr); + } + ++first; } + + // TODO: token_spec_t needs these same operator() overloads, each of + // which will return a token_parser. + + // TODO: Constrain both ID params below only to accept type + // convertible to int. + + /** TODO */ + template + constexpr auto operator()(ID id) const noexcept + { + BOOST_PARSER_ASSERT( + (detail::is_nope_v && + "If you're seeing this, you tried to chain calls on tok or " + "tok_t, like 'tok(id1)(id2)'. Quit it!'")); + return parser_interface(detail::token_with_id((int)id)); + } + + /** TODO */ + template + constexpr auto + operator()(ID id, expected_value_type value) const noexcept + { + BOOST_PARSER_ASSERT( + (detail::is_nope_v && + "If you're seeing this, you tried to chain calls on tok or " + "tok_t, like 'tok(id1)(id2)'. Quit it!'")); + return parser_interface( + detail::token_with_id_and_value((int)id, value)); + } + + Expected expected_; }; #endif /** TODO */ - constexpr parser_interface> tok; + constexpr parser_interface> tok; + + /** TODO */ + template + constexpr parser_interface> tok_t; }}