Skip to content

Commit

Permalink
Add a smoke test for using token views as input to parse(); fix errors.
Browse files Browse the repository at this point in the history 
See #202.
  • Loading branch information
@tzlaine
tzlaine committed Nov 29, 2024
1 parent 8658f8c commit 67c3ec1
Show file tree
Hide file tree
Showing 7 changed files with 128 additions and 68 deletions.
24 changes: 7 additions & 17 deletions include/boost/parser/detail/printing.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,29 +609,19 @@ namespace boost { namespace parser { namespace detail {
Context const & context,
flags f,
Attribute const & attr,
std::string name) :
os_(os),
initial_first_(first),
first_(first),
last_(last),
context_(context),
flags_(f),
attr_(attr),
name_(std::move(name))
{
if (!detail::do_trace(flags_))
return;
detail::trace_prefix(os, first_, last_, context_, name_);
}
std::string name);
~scoped_trace_t();
// implemented in printing_impl.hpp

~scoped_trace_t()
template<typename I, typename S>
void impl(I initial_first, I first, S last)
{
if (!detail::do_trace(flags_))
return;
detail::trace_indent(os_, detail::_indent(context_));
if (*context_.pass_) {
os_ << "matched ";
detail::trace_input(os_, initial_first_, first_);
detail::trace_input(os_, initial_first, first);
os_ << "\n";
detail::print_attribute(
os_,
Expand All @@ -640,7 +630,7 @@ namespace boost { namespace parser { namespace detail {
} else {
os_ << "no match\n";
}
detail::trace_suffix(os_, first_, last_, context_, name_);
detail::trace_suffix(os_, first, last, context_, name_);
}

std::ostream & os_;
Expand Down
54 changes: 54 additions & 0 deletions include/boost/parser/detail/printing_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -977,6 +977,60 @@ namespace boost { namespace parser { namespace detail {

#endif

template<
bool DoTrace,
typename Iter,
typename Sentinel,
typename Context,
typename Attribute>
scoped_trace_t<DoTrace, Iter, Sentinel, Context, Attribute>::scoped_trace_t(
std::ostream & os,
Iter & first,
Sentinel last,
Context const & context,
flags f,
Attribute const & attr,
std::string name) :
os_(os),
initial_first_(first),
first_(first),
last_(last),
context_(context),
flags_(f),
attr_(attr),
name_(std::move(name))
{
if (!detail::do_trace(flags_))
return;
if constexpr (is_token_iter_v<Iter>) {
auto const initial_first = first_.range_begin();
auto const first = initial_first + (*first_).underlying_position();
auto const last = first_.range_end();
detail::trace_prefix(os, first, last, context_, name_);
} else {
detail::trace_prefix(os, first_, last_, context_, name_);
}
}

template<
bool DoTrace,
typename Iter,
typename Sentinel,
typename Context,
typename Attribute>
scoped_trace_t<DoTrace, Iter, Sentinel, Context, Attribute>::
~scoped_trace_t()
{
if constexpr (is_token_iter_v<Iter>) {
auto const initial_first = first_.range_begin();
auto const first = initial_first + (*first_).underlying_position();
auto const last = first_.range_end();
impl(initial_first, first, last);
} else {
impl(initial_first_, first_, last_);
}
}

}}}

#endif
6 changes: 4 additions & 2 deletions include/boost/parser/detail/text/transcode_view.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -693,12 +693,14 @@ namespace boost::parser::detail { namespace text {
using T = detail::remove_cv_ref_t<R>;
if constexpr (forward_range_v<T>) {
auto unpacked =
boost::parser::detail::text::unpack_iterator_and_sentinel(detail::begin(r), detail::end(r));
boost::parser::detail::text::unpack_iterator_and_sentinel(
detail::begin(r), detail::end(r));
if constexpr (is_bounded_array_v<T>) {
constexpr auto n = std::extent_v<T>;
if (n && !r[n - 1])
--unpacked.last;
return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(unpacked.first, unpacked.last);
return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(
unpacked.first, unpacked.last);
} else if constexpr (
!std::is_same_v<decltype(unpacked.first), iterator_t<R>> ||
!std::is_same_v<decltype(unpacked.last), sentinel_t<R>>) {
Expand Down
5 changes: 3 additions & 2 deletions include/boost/parser/lexer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -895,8 +895,9 @@ namespace boost { namespace parser {
detail::make_token<parse_spec>(
id,
sv,
ctre_first.current -
ctre_first.orig_begin));
(ctre_first.current -
ctre_first.orig_begin) -
sv.size()));
return sv;
} else {
return state;
Expand Down
48 changes: 30 additions & 18 deletions include/boost/parser/parser.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2339,9 +2339,10 @@ namespace boost { namespace parser {
{
if constexpr (is_token_iter_v<Iter>) {
auto const underlying_first = e.iter.range_begin();
auto const underlying_curr = e.iter->underlying_position();
auto const underlying_last = e.iter.range_end();
parse_error underlying_error{underlying_curr, e.message()};
parse_error underlying_error(
underlying_first + (*e.iter).underlying_position(),
e.message);
return error_handler(
underlying_first,
underlying_last,
Expand Down Expand Up @@ -6494,11 +6495,21 @@ namespace boost { namespace parser {
constexpr char_parser() {}
constexpr char_parser(Expected expected) : expected_(expected) {}

template<typename T>
using attribute_type = std::conditional_t<
std::is_same_v<AttributeType, void>,
std::decay_t<T>,
AttributeType>;
template<typename Iter>
static constexpr auto attribute_type_()
{
if constexpr (!std::is_same_v<AttributeType, void>) {
return detail::wrapper<AttributeType>{};
} else if constexpr (is_token_v<detail::iter_value_t<Iter>>) {
return detail::wrapper<
typename detail::iter_value_t<Iter>::char_type>{};
} else {
return detail::wrapper<detail::iter_value_t<Iter>>{};
}
}

template<typename Iter>
using attribute_type = typename decltype(attribute_type_<Iter>())::type;

template<
typename Iter,
Expand All @@ -6511,9 +6522,9 @@ namespace boost { namespace parser {
Context const & context,
SkipParser const & skip,
detail::flags flags,
bool & success) const -> attribute_type<decltype(*first)>
bool & success) const -> attribute_type<Iter>
{
attribute_type<decltype(*first)> retval{};
attribute_type<Iter> retval{};
call(first, last, context, skip, flags, success, retval);
return retval;
}
Expand Down Expand Up @@ -6541,29 +6552,30 @@ namespace boost { namespace parser {
return;
}

using attribute_t = attribute_type<decltype(*first)>;
attribute_t x = 0;

if constexpr (is_token_v<std::decay_t<decltype(*first)>>) {
if (first->id() != character_id || !first->has_long_long()) {
if constexpr (is_token_v<detail::iter_value_t<Iter>>) {
using value_type = detail::iter_value_t<Iter>;
using attribute_t = typename value_type::char_type;
if ((*first).id() != character_id ||
!(*first).has_long_long()) {
success = false;
return;
}
char32_t c = (char32_t)first->has_long_long();
char32_t c = (char32_t)(*first).get_long_long();
if (detail::unequal(context, c, expected_)) {
success = false;
return;
}
x = (attribute_t)c;
detail::assign(retval, (attribute_t)c);
} else {
using attribute_t = attribute_type<Iter>;
attribute_t x = 0;
x = *first;
if (detail::unequal(context, x, expected_)) {
success = false;
return;
}
detail::assign(retval, x);
}

detail::assign(retval, x);
++first;
}

Expand Down
41 changes: 19 additions & 22 deletions include/boost/parser/token_parser.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,7 @@ namespace boost { namespace parser {
template<typename AttributeType, typename CharType>
std::optional<AttributeType> token_as(token<CharType> tok)
{
if constexpr (std::same_as<AttributeType, string_view_tag>) {
if (tok.has_string_view())
return tok.get_string_view();
return std::nullopt;
} else if constexpr (std::is_floating_point_v<AttributeType>) {
if constexpr (std::is_floating_point_v<AttributeType>) {
if (tok.has_long_double())
return tok.get_long_double();
return std::nullopt;
Expand All @@ -36,11 +32,9 @@ namespace boost { namespace parser {
return AttributeType(tok.get_long_long());
return std::nullopt;
} else {
static_assert(
!std::same_as<CharType, CharType>,
"The only attribute types that can be pulled out of a "
"token are no-attribute, floating-point values, or "
"integral values (including charater types).");
if (tok.has_string_view())
return tok.get_string_view();
return std::nullopt;
}
}

Expand Down Expand Up @@ -76,9 +70,11 @@ namespace boost { namespace parser {
{
using token_spec = TokenSpec;

template<typename Iter>
using attribute_type = std::conditional_t<
std::is_same_v<typename token_spec::value_type, none>,
string_view_tag,
std::same_as<typename token_spec::value_type, string_view_tag>,
std::basic_string_view<
typename detail::iter_value_t<Iter>::char_type>,
typename token_spec::value_type>;

constexpr token_parser() = default;
Expand All @@ -95,9 +91,9 @@ namespace boost { namespace parser {
Context const & context,
SkipParser const & skip,
detail::flags flags,
bool & success) const -> attribute_type
bool & success) const -> attribute_type<Iter>
{
attribute_type retval;
attribute_type<Iter> retval;
call(first, last, context, skip, flags, success, retval);
return retval;
}
Expand Down Expand Up @@ -132,16 +128,16 @@ namespace boost { namespace parser {
}

value_type const x = *first;
if (x.id() != token_spec::id) {
if (x.id() != (int)token_spec::id) {
success = false;
return;
}

constexpr bool use_expected = !std::same_as<Expected, detail::nope>;
if (use_expected || detail::gen_attrs(flags)) {
auto opt_attr = detail::token_as<attribute_type>(x);
auto opt_attr = detail::token_as<attribute_type<Iter>>(x);
if constexpr (use_expected) {
if (!opt_attr || !expected_.matches_value(*opt_attr)) {
if (!opt_attr || !expected_.matches(*opt_attr)) {
success = false;
return;
}
Expand All @@ -154,17 +150,18 @@ namespace boost { namespace parser {
}

/** TODO */
constexpr auto operator()(attribute_type value) const noexcept
template<typename T>
requires std::is_integral_v<T> || std::is_floating_point_v<T>
constexpr auto operator()(T value) const noexcept
{
BOOST_PARSER_ASSERT(
(detail::is_nope_v<Expected> &&
"If you're seeing this, you tried to chain calls on one of "
"your token_spec's, like 'my_token_spec(id1)(id2)'. Quit "
"it!'"));
return parser_interface(token_parser<
TokenSpec,
detail::token_with_value<attribute_type>>(
detail::token_with_value(std::move(value))));
return parser_interface(
token_parser<TokenSpec, detail::token_with_value<T>>(
detail::token_with_value(std::move(value))));
}

template<parsable_range_like R>
Expand Down
18 changes: 11 additions & 7 deletions test/lexer_and_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
* http://www.boost.org/LICENSE_1_0.txt)
*/

#define BOOST_PARSER_TESTING
#include <boost/parser/lexer.hpp>
#include <boost/parser/parser.hpp>

Expand Down Expand Up @@ -69,14 +70,17 @@ int main()
{
auto parser = identifier("foo") >> '=' >> true_false >> ';';
auto r = "some input" | bp::to_tokens(adobe_lexer);
// TODO auto result = bp::parse(r, parser);
auto result = bp::parse(r, parser);
BOOST_TEST(!result);
}
{
auto parser = identifier >> '=' >> true_false >> ';';
auto r = "foo = false;" | bp::to_tokens(adobe_lexer);
auto result = bp::parse(r, parser);
BOOST_TEST(result);
BOOST_TEST(std::get<0>(*result) == "foo");
BOOST_TEST(std::get<1>(*result) == false);
}

// TODO {
// TODO std::string str = "a";
// TODO BOOST_TEST(parse(str, char_));
// TODO BOOST_TEST(!parse(str, char_('b')));
// TODO }

return boost::report_errors();
}

0 comments on commit 67c3ec1

Please sign in to comment.