From c6246816235ec622b3df14ef7e8ecda7c253a36c Mon Sep 17 00:00:00 2001 From: Zach Laine Date: Sat, 9 Nov 2024 01:49:39 -0600 Subject: [PATCH] Completely rethink the relationship among token_parser, token_spec_t, and parser_interface. token_spec is now a variable template that generates a parser_interface wrapping a token_parser, which parameterized on the token_spec_t. This way, a single token_spec use can be used to specify how to lex, and how to parse. See #202. --- include/boost/parser/lexer.hpp | 66 +++------- include/boost/parser/parser_fwd.hpp | 15 +-- include/boost/parser/token_parser.hpp | 167 ++++++++++---------------- test/adobe_lexer.hpp | 76 ++++++++++++ test/lexer.cpp | 66 +++++----- test/lexer_adobe_files.cpp | 141 +++++----------------- test/lexer_and_parser.cpp | 20 ++- 7 files changed, 250 insertions(+), 301 deletions(-) create mode 100644 test/adobe_lexer.hpp diff --git a/include/boost/parser/lexer.hpp b/include/boost/parser/lexer.hpp index 1c92d646..65e02b7b 100644 --- a/include/boost/parser/lexer.hpp +++ b/include/boost/parser/lexer.hpp @@ -13,6 +13,7 @@ "In order to work, the Boost.Parser lexer requires C++20 and CTRE's ctre-unicode.hpp single-header file in the #include path. CTRE can be found at https://github.com/hanickadot/compile-time-regular-expressions . The required header is at https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/refs/heads/main/single-header/ctre-unicode.hpp ." #endif +#include #include #include #include @@ -46,8 +47,6 @@ std::ostream & operator<<(std::ostream & os, std::array const & arr) namespace boost { namespace parser { - struct none; - namespace detail { enum class token_kind { no_value, string_view, long_long, long_double }; @@ -281,7 +280,7 @@ namespace boost { namespace parser { constexpr auto base = TokenSpec::base; if constexpr (TokenSpec::is_character_token) { return parse_spec{token_parsed_type::character, base}; - } else if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { return parse_spec{token_parsed_type::string_view, base}; } else if constexpr (std::is_same_v) { return parse_spec{token_parsed_type::bool_, base}; @@ -321,11 +320,15 @@ namespace boost { namespace parser { return parse_spec{token_parsed_type::long_double, base}; } else { static_assert( - !std::is_same_v, + !std::is_same_v, "The only valid types for the 'Value' template parameter " - "to 'lexer_token_spec' are 'none', integral types, and " - "floating-point types."); + "to 'token_spec' are 'string_view_tag', integral types, " + "and floating-point types."); } +#if defined(__cpp_lib_unreachable) + std::unreachable(); + return parse_spec{token_parsed_type::string_view, base}; +#endif } template @@ -417,11 +420,7 @@ namespace boost { namespace parser { } /** TODO */ - template< - ctll::fixed_string Regex, - auto ID, - typename ValueType = none, - int Base = 10> + template struct token_spec_t { using id_type = decltype(ID); @@ -438,26 +437,8 @@ namespace boost { namespace parser { static constexpr id_type id = ID; static constexpr int base = Base < 0 ? 10 : Base; static constexpr bool is_character_token = Base < 0; - - /** TODO */ - template - constexpr auto operator()(ID2 id) const noexcept; - - /** TODO */ - template - constexpr auto operator()(ID2 id, Value value) const noexcept; - - // implementations in token_parser.hpp }; - /** TODO */ - template< - ctll::fixed_string Regex, - auto ID, - typename ValueType = none, - int Base = 10> - constexpr auto token_spec = token_spec_t{}; - // TODO: Document that this takes a pack of char -- and nothing else. Also // note that for anything more complicated, including a short UTF-8 sequence // that encodes a code point, you must use the token_spec form. @@ -489,33 +470,20 @@ namespace boost { namespace parser { static constexpr auto ids() { return IDs.as_array(); } static constexpr auto specs() { return Specs.as_array(); } + // implementation in token_parser.hpp template< ctll::fixed_string RegexStr2, auto ID2, typename ValueType, int Base> - auto operator|(token_spec_t rhs) const - { - static_assert( - std::same_as, - "All id_types must be the same for all token_specs."); - constexpr auto new_regex = - detail::wrap_escape_concat(); - constexpr auto new_ids = IDs.template append<(int)ID2>(); - constexpr auto new_specs = - Specs - .template append()>(); - return lexer_t< - CharType, - ID, - WsStr, - new_regex, - new_ids, - new_specs>{}; - } + constexpr auto operator|( + parser_interface>> const & rhs) + const; template - auto operator|(detail::token_chars_spec rhs) const + constexpr auto + operator|(detail::token_chars_spec const & rhs) const { constexpr auto new_regex = detail::wrap_escape_concat(); diff --git a/include/boost/parser/parser_fwd.hpp b/include/boost/parser/parser_fwd.hpp index eb8321d7..8ca0db27 100644 --- a/include/boost/parser/parser_fwd.hpp +++ b/include/boost/parser/parser_fwd.hpp @@ -154,10 +154,8 @@ namespace boost { namespace parser { struct upper_case_chars {}; - struct any_token + struct any_token_value { - bool matches_id(int) const { return true; } - template bool matches_value(T) const { @@ -445,15 +443,14 @@ namespace boost { namespace parser { template struct float_parser; - /** A tag type used to stand in for any specialization of - `boost::parser::token<>`. */ - struct token_tag + /** A tag type used to represent a value type that is any specialization + of `std::basic_string_view`. Which specialization is used depends on + the input. */ + struct string_view_tag {}; /** TODO */ - template< - typename AttributeType = token_tag, - typename Expected = detail::any_token> + template struct token_parser; /** Applies at most one of the parsers in `OrParser`. If `switch_value_` diff --git a/include/boost/parser/token_parser.hpp b/include/boost/parser/token_parser.hpp index 23f51486..f4784dc8 100644 --- a/include/boost/parser/token_parser.hpp +++ b/include/boost/parser/token_parser.hpp @@ -18,11 +18,7 @@ namespace boost { namespace parser { template std::optional token_as(token tok) { - if constexpr (std::same_as) { - return nope{}; - } else if constexpr (std::same_as< - AttributeType, - std::basic_string_view>) { + if constexpr (std::same_as) { if (tok.has_string_view()) return tok.get_string_view(); return std::nullopt; @@ -43,32 +39,11 @@ namespace boost { namespace parser { } } - struct token_with_id - { - explicit token_with_id(int id) : id_(id) {} - - bool matches(int id) const { return id == id_; } - - template - bool matches_value(T) const - { - return true; - } - - int id_; - }; - template - struct token_with_id_and_value + struct token_with_value { - explicit token_with_id_and_value(int id, T value) : - id_(id), value_(value) - {} - - bool matches(int id) const { return id == id_; } - bool matches_value(T value) const { return value == value_; } - - int id_; + explicit token_with_value(T value) : value_(std::move(value)) {} + bool matches(T const & value) const { return value == value_; } T value_; }; } @@ -78,18 +53,15 @@ namespace boost { namespace parser { // TODO: Needs a printer. // TODO: Constrain the AttributeType to something that detail::token_as() // can handle. - template + template struct token_parser { - using attribute_type = std::conditional_t< - std::is_same_v, - detail::nope, - AttributeType>; + using token_spec = TokenSpec; - using expected_value_type = std::conditional_t< - std::is_same_v, - detail::nope, - attribute_type>; + using attribute_type = std::conditional_t< + std::is_same_v, + string_view_tag, + typename token_spec::value_type>; constexpr token_parser() = default; constexpr token_parser(Expected expected) : expected_(expected) {} @@ -142,97 +114,88 @@ namespace boost { namespace parser { } value_type const x = *first; - if (!expected_.matches_id(x.id())) { + if (x.id() != token_spec::id) { success = false; return; } - if constexpr (std::same_as) { - detail::assign(retval, x); - } else { + constexpr bool use_expected = !std::same_as; + if (use_expected || detail::gen_attrs(flags)) { auto opt_attr = detail::token_as(x); - if (!opt_attr || !expected_.matches_value(*opt_attr)) { - success = false; - return; + if constexpr (use_expected) { + if (!opt_attr || !expected_.matches_value(*opt_attr)) { + success = false; + return; + } } - detail::assign(retval, *opt_attr); + if (detail::gen_attrs(flags)) + detail::assign(retval, *opt_attr); } ++first; } - // TODO: Constrain all ID params below (incl. the ones from - // token_spec_t) only to accept type convertible to int. - - /** TODO */ - template - constexpr auto operator()(ID id) const noexcept - { - BOOST_PARSER_ASSERT( - (detail::is_nope_v && - "If you're seeing this, you tried to chain calls on tok, " - "tok_t, or one of your token_spec_t's, like 'tok(id1)(id2)'. " - "Quit it!'")); - return parser_interface( - token_parser( - detail::token_with_id((int)id))); - } - /** TODO */ - template - constexpr auto - operator()(ID id, expected_value_type value) const noexcept + constexpr auto operator()(attribute_type value) const noexcept { BOOST_PARSER_ASSERT( (detail::is_nope_v && - "If you're seeing this, you tried to chain calls on tok, " - "tok_t, or one of your token_spec_t's, like 'tok(id1)(id2)'. " - "Quit it!'")); - return parser_interface( - token_parser< - AttributeType, - detail::token_with_id_and_value>( - detail::token_with_id_and_value((int)id, value))); + "If you're seeing this, you tried to chain calls on one of " + "your token_spec's, like 'my_token_spec(id1)(id2)'. Quit " + "it!'")); + return parser_interface(token_parser< + TokenSpec, + detail::token_with_value>( + detail::token_with_value(std::move(value)))); } Expected expected_; }; - template - template - constexpr auto - token_spec_t::operator()(ID2 id) const noexcept - { - using attribute_type = std:: - conditional_t, token_tag, ValueType>; - return parser_interface( - token_parser( - detail::token_with_id((int)id))); - } +#endif + + /** TODO */ + template< + ctll::fixed_string Regex, + auto ID, + typename ValueType = string_view_tag, + int Base = 10> + constexpr parser_interface token_spec{ + token_parser>()}; + +#ifndef BOOST_PARSER_DOXYGEN - template - template - constexpr auto token_spec_t::operator()( - ID2 id, Value value) const noexcept + template< + typename CharType, + typename ID, + ctll::fixed_string WsStr, + ctll::fixed_string RegexStr, + detail::nttp_array IDs, + detail::nttp_array Specs> + template< + ctll::fixed_string RegexStr2, + auto ID2, + typename ValueType, + int Base> + constexpr auto + lexer_t::operator|( + parser_interface>> const &) const { - using attribute_type = std:: - conditional_t, token_tag, ValueType>; - return parser_interface(token_parser< - attribute_type, - detail::token_with_id_and_value>( - detail::token_with_id_and_value((int)id, value))); + static_assert( + std::same_as, + "All id_types must be the same for all token_specs."); + constexpr auto new_regex = + detail::wrap_escape_concat(); + constexpr auto new_ids = IDs.template append<(int)ID2>(); + constexpr auto new_specs = Specs.template append>()>(); + return lexer_t{}; } #endif - /** TODO */ - constexpr parser_interface> tok; - - /** TODO */ - template - constexpr parser_interface> tok_t; - }} #endif diff --git a/test/adobe_lexer.hpp b/test/adobe_lexer.hpp new file mode 100644 index 00000000..2f66578f --- /dev/null +++ b/test/adobe_lexer.hpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2024 T. Zachary Laine + * + * Distributed under the Boost Software License, Version 1.0. (See + * accompanying file LICENSE_1_0.txt or copy at + * http://www.boost.org/LICENSE_1_0.txt) + */ +#ifndef BOOST_PARSER_TEST_ADOBE_LEXER +#define BOOST_PARSER_TEST_ADOBE_LEXER + +#include +#include + + +namespace bp = boost::parser; + +enum class adobe_tokens { + keyword_true_false, + keyword_empty, + identifier, + lead_comment, + trail_comment, + quoted_string, + number, + eq_op, + rel_op, + mul_op, + define, + or_, + and_ +}; + +constexpr auto true_false = + bp::token_spec<"true|false", adobe_tokens::keyword_true_false, bool>; +constexpr auto empty = bp::token_spec<"empty", adobe_tokens::keyword_empty>; +constexpr auto identifier = + bp::token_spec<"[a-zA-Z]\\w*", adobe_tokens::identifier>; +constexpr auto lead_comment = bp::token_spec< + "\\/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*\\/", + adobe_tokens::lead_comment>; +constexpr auto trail_comment = + bp::token_spec<"\\/\\/.*$", adobe_tokens::trail_comment>; +constexpr auto quoted_string = + bp::token_spec<"\\\"[^\\\"]*\\\"|'[^']*'", adobe_tokens::quoted_string>; +constexpr auto number = + bp::token_spec<"\\d+(?:\\.\\d*)?", adobe_tokens::number, double>; +constexpr auto eq_op = bp::token_spec<"==|!=", adobe_tokens::eq_op>; +constexpr auto define = bp::token_spec<"<==", adobe_tokens::define>; +constexpr auto rel_op = bp::token_spec<"<|>|<=|>=", adobe_tokens::rel_op>; +constexpr auto mul_op = bp::token_spec<"\\*|\\/|%", adobe_tokens::mul_op>; +constexpr auto or_ = bp::token_spec<"\\|\\|", adobe_tokens::or_>; +constexpr auto and_ = bp::token_spec<"&&", adobe_tokens::and_>; + +constexpr auto adobe_lexer = bp::lexer | true_false | + empty | identifier | lead_comment | trail_comment | + quoted_string | number | eq_op | define | rel_op | + mul_op | or_ | and_ | + bp::token_chars< + '=', + '+', + '-', + '!', + '?', + ':', + '.', + ',', + '(', + ')', + '[', + ']', + '{', + '}', + '@', + ';'>; + +#endif diff --git a/test/lexer.cpp b/test/lexer.cpp index 8a2b445b..d383e13b 100644 --- a/test/lexer.cpp +++ b/test/lexer.cpp @@ -7,6 +7,7 @@ */ #define BOOST_PARSER_TESTING #include +#include #include @@ -28,67 +29,78 @@ int main() { auto const token_spec = bp::token_spec<"foo", 12>; - const bp::token_spec_t<"foo", 12, bp::none, 10> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, bp::string_view_tag, 10> + token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", my_tokens::foo>; - const bp::token_spec_t<"foo", my_tokens::foo, bp::none, 10> + bp::token_spec_t<"foo", my_tokens::foo, bp::string_view_tag, 10> token_spec_explicit; - static_assert( - std::same_as); + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"bar", my_tokens::bar>; - const bp::token_spec_t<"bar", my_tokens::bar, bp::none, 10> + bp::token_spec_t<"bar", my_tokens::bar, bp::string_view_tag, 10> token_spec_explicit; - static_assert( - std::same_as); + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12, int, 2>; - const bp::token_spec_t<"foo", 12, int, 2> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, int, 2> token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12>; - const bp::token_spec_t<"foo", 12, bp::none, 10> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, bp::string_view_tag, 10> + token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12, unsigned int, 8>; - const bp::token_spec_t<"foo", 12, unsigned int, 8> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, unsigned int, 8> token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12, short>; - const bp::token_spec_t<"foo", 12, short, 10> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, short, 10> token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12, float>; - const bp::token_spec_t<"foo", 12, float, 10> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, float, 10> token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } { auto const token_spec = bp::token_spec<"foo", 12, double>; - const bp::token_spec_t<"foo", 12, double, 10> token_spec_explicit; - static_assert( - std::same_as); + bp::token_spec_t<"foo", 12, double, 10> token_spec_explicit; + static_assert(std::same_as< + decltype(token_spec.parser_)::token_spec, + decltype(token_spec_explicit)>); } // making lexers diff --git a/test/lexer_adobe_files.cpp b/test/lexer_adobe_files.cpp index c27cc201..9a7290e0 100644 --- a/test/lexer_adobe_files.cpp +++ b/test/lexer_adobe_files.cpp @@ -11,6 +11,7 @@ #include #include "ill_formed.hpp" +#include "adobe_lexer.hpp" #include #include @@ -20,86 +21,18 @@ namespace bp = boost::parser; -enum class adobe_tokens { - keyword_true_false, - keyword_empty, - identifier, - lead_comment, - trail_comment, - quoted_string, - number, - eq_op, - rel_op, - mul_op, - define, - or_, - and_ -}; - int main() { { - // Document that maximum munch does not appear to apply -- putting "<==" - // after "<|>|<=|>=" causes input "<==" to be tokenized as "<", "==". - auto const lexer = - bp::lexer | - - bp::token_spec< - "true|false", - adobe_tokens::keyword_true_false, - bool> | - bp::token_spec<"empty", adobe_tokens::keyword_empty> | - bp::token_spec<"[a-zA-Z]\\w*", adobe_tokens::identifier> | - bp::token_spec< - "\\/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*\\/", - adobe_tokens::lead_comment> | - bp::token_spec<"\\/\\/.*$", adobe_tokens::trail_comment> | - bp::token_spec< - "\\\"[^\\\"]*\\\"|'[^']*'", - adobe_tokens::quoted_string> | - bp::token_spec<"\\d+(?:\\.\\d*)?", adobe_tokens::number, double> | - bp::token_spec<"==|!=", adobe_tokens::eq_op> | - bp::token_spec<"<==", adobe_tokens::define> | - bp::token_spec<"<|>|<=|>=", adobe_tokens::rel_op> | - bp::token_spec<"\\*|\\/|%", adobe_tokens::mul_op> | - bp::token_spec<"\\|\\|", adobe_tokens::or_> | - bp::token_spec<"&&", adobe_tokens::and_> | - bp::token_chars< - '=', - '+', - '-', - '!', - '?', - ':', - '.', - ',', - '(', - ')', - '[', - ']', - '{', - '}', - '@', - ';'>; - -#if 0 - std::cout << "lexer.regex_str =\n" - << (lexer.regex_str.content | bp::as_utf8) << "\n"; - std::cout << "lexer.ws_str =\n" - << (lexer.ws_str.content | bp::as_utf8) << "\n"; - std::cout << "lexer::size()=" << lexer.size() << "\n"; - constexpr auto combined = - bp::detail::wrap_escape_concat(); - std::cout << "lexer combined regex str =\n" - << (combined.content | bp::as_utf8) << "\n"; - std::cout << "lexer IDs =\n" << lexer.ids() << "\n"; - std::cout << "lexer parse_specs =\n" << lexer.specs() << "\n"; -#endif - - static_assert(decltype(lexer)::size() == 29 + 1); - static_assert(std::same_as); - - // tokens_view from lexer + // TODO: Document that maximum munch does not appear to apply in CTRE + // regexes -- putting "<==" after "<|>|<=|>=" causes input "<==" to be + // tokenized as "<", "==". + + static_assert(decltype(adobe_lexer)::size() == 29 + 1); + static_assert( + std::same_as); + + // tokens_view from adobe_lexer { char const input[] = R"(/* Copyright 2005-2007 Adobe Systems Incorporated @@ -122,36 +55,19 @@ sheet alert_dialog */)", R"( -)", - R"(sheet)", - R"( )", - R"(alert_dialog)", +)", R"(sheet)", R"( )", R"(alert_dialog)", R"( -)", - R"({)", +)", R"({)", R"( -)", - R"(output)", - R"(:)", +)", R"(output)", R"(:)", R"( - )", - R"(result)", - R"( )", - R"(<==)", - R"( )", - R"({)", - R"( )", - R"(dummy_value)", - R"(:)", - R"( )", - R"(42)", - R"( )", - R"(})", - R"(;)", + )", R"(result)", R"( )", R"(<==)", + R"( )", R"({)", R"( )", R"(dummy_value)", + R"(:)", R"( )", R"(42)", R"( )", + R"(})", R"(;)", R"( -)", - R"(})"}; - auto r = lexer.regex_range(input); +)", R"(})"}; + auto r = adobe_lexer.regex_range(input); int position = 0; for (auto subrange : r) { std::string_view sv = subrange; @@ -186,7 +102,7 @@ sheet alert_dialog // make a tokens_view { - auto r = bp::tokens_view(input, lexer); + auto r = bp::tokens_view(input, adobe_lexer); int position = 0; for (auto tok : r) { BOOST_TEST(tok == expected[position]); @@ -198,7 +114,7 @@ sheet alert_dialog // to_tokens range adaptor { int position = 0; - for (auto tok: bp::to_tokens(input, lexer)) { + for (auto tok : bp::to_tokens(input, adobe_lexer)) { BOOST_TEST(tok == expected[position]); ++position; } @@ -207,7 +123,7 @@ sheet alert_dialog { std::string const input_str = input; int position = 0; - for (auto tok: bp::to_tokens(input_str, lexer)) { + for (auto tok : bp::to_tokens(input_str, adobe_lexer)) { BOOST_TEST(tok == expected[position]); ++position; } @@ -215,7 +131,8 @@ sheet alert_dialog } { int position = 0; - for (auto tok : std::string(input) | bp::to_tokens(lexer)) { + for (auto tok : + std::string(input) | bp::to_tokens(adobe_lexer)) { BOOST_TEST(tok == expected[position]); ++position; } @@ -226,7 +143,8 @@ sheet alert_dialog { std::vector> cache; int position = 0; - for (auto tok : bp::to_tokens(input, lexer, std::ref(cache))) { + for (auto tok : + bp::to_tokens(input, adobe_lexer, std::ref(cache))) { BOOST_TEST(tok == expected[position]); ++position; } @@ -235,7 +153,8 @@ sheet alert_dialog { boost::container::small_vector, 10> cache; int position = 0; - for (auto tok : input | bp::to_tokens(lexer, std::ref(cache))) { + for (auto tok : + input | bp::to_tokens(adobe_lexer, std::ref(cache))) { BOOST_TEST(tok == expected[position]); ++position; } @@ -831,7 +750,7 @@ sheet image_size int position = 0; for (auto tok : - std::string(large_input) | bp::to_tokens(lexer)) { + std::string(large_input) | bp::to_tokens(adobe_lexer)) { BOOST_TEST(tok == expected[position]); if (tok != expected[position]) { std::cout << "At pos=" << position << ": got " << tok @@ -841,7 +760,7 @@ sheet image_size } BOOST_TEST(position == (int)std::size(expected)); } - } + } } return boost::report_errors(); diff --git a/test/lexer_and_parser.cpp b/test/lexer_and_parser.cpp index 6ef10187..ff9136ff 100644 --- a/test/lexer_and_parser.cpp +++ b/test/lexer_and_parser.cpp @@ -11,9 +11,10 @@ #include +#include "adobe_lexer.hpp" -namespace bp = boost::parser; +namespace bp = boost::parser; int main() { @@ -21,7 +22,7 @@ int main() // the parse() API. { bp::token tokens[1] = {}; - bp::token_parser p; + auto p = bp::token_spec<"12", 12, int>; auto first = std::begin(tokens); auto const last = std::end(tokens); @@ -48,10 +49,23 @@ int main() auto const flags = bp::detail::flags::gen_attrs; std::optional result = - p.call(first, last, context, bp::ws, flags, success); + p(first, last, context, bp::ws, flags, success); (void)result; } + // Minimal tests of building parsers from token_parser and token_spec. + { + auto parser1 = true_false(true); + auto parser2 = true_false(false); + (void)parser1; + (void)parser2; + } + { + // TODO: identifier("foo") does not work! + auto parser1 = identifier >> '=' >> true_false >> ';'; + (void)parser1; + } + // TODO { // TODO std::string str = "a"; // TODO BOOST_TEST(parse(str, char_));