parser stopping early with different line endings. Is this a memory issue?

[rowlesmr]

I am writing a parser for a text data format, and it appears to work, but (at least appears to) successfully finishes before parsing the entire file. My test files are a few hundred lines, but "real" ones would be many 10s of megabytes.

In the example below, I am parsing the exact same string, except for different line endings. In this example, each block of data is 17 lines of two values with a blockheading. The actual blocks would be up to 20000 lines of 8 columns, with 10s to 100s of blocks per file.

There should be 546 lines in the output. This is successful in the first case, but not the second.

What am I doing wrong?

Thanks

Matthew

#include <iostream>
#include <string>
#include <vector>

#include <tao/pegtl.hpp>
#include <tao/pegtl/contrib/trace.hpp>
#include <tao/pegtl/contrib/analyze.hpp>

namespace pegtl = tao::pegtl;

struct DATA : TAO_PEGTL_ISTRING("data_") {};

struct EOL       : pegtl::eol {};
struct SP        : pegtl::one<' '> {};
struct HT        : pegtl::one<'\t'> {};

struct nonblankchar : pegtl::range< '!', '~' > {};  //not: SP HT
struct whitespace   : pegtl::plus< pegtl::sor< SP, HT, EOL > > {};
struct tag          : pegtl::seq< pegtl::one< '_' >, pegtl::plus< nonblankchar > > {};
struct value        : pegtl::seq < pegtl::plus< nonblankchar >, pegtl::at < pegtl::sor<  pegtl::one<' ', '\n', '\r', '\t' >, pegtl::eof> > > {};

// tag whitespace value
struct dataitem  : pegtl::if_must< tag, whitespace, value, pegtl::discard > {};
// DATA_ nonblankchar+
struct blockheading : pegtl::seq< DATA, pegtl::plus< nonblankchar > > {};
// blockheading (whitespace dataitem)*
struct block : pegtl::seq< blockheading, pegtl::star< pegtl::seq< whitespace, dataitem > > > {};
// whitespace? (block (whitespace block)*)? whitespace?
struct data : pegtl::seq< pegtl::opt< whitespace >, pegtl::opt< pegtl::seq< block, pegtl::star< pegtl::seq< whitespace, block > >, pegtl::opt< whitespace > > > > {};

template<typename Rule> struct Action : pegtl::nothing<Rule> {};

template<> struct Action< blockheading > {
   template<typename Input> static void apply(const Input& in, std::vector<std::string>& out) {
      out.emplace_back(in.string());
   }
};
  
template<> struct Action< tag > {
   template<typename Input> static void apply(const Input& in, std::vector<std::string>& out) {
      out.emplace_back(in.string());
   }
};

template<> struct Action< value > {
   template<typename Input> static void apply(const Input& in, std::vector<std::string>& out) {
      out.emplace_back(in.string());
   }
};

template<typename Input> 
void parse_input(std::vector<std::string>& d, Input&& in) {
   pegtl::parse< data, Action >(in, d);
}

template<typename Input> 
std::vector<std::string> read_input(Input&& in) {
   std::vector<std::string> data;
   parse_input(data, in);
   return data;
}

int main(int argc, char** argv)
{
   const std::size_t issues = tao::pegtl::analyze< data > (1);
   std::cout << issues << std::endl;
   
   std::string source_unix = "data_NH4NO3_1\n_4.03	154.0\n_4.09	140.0\n_4.15	134.0\n_4.21	171.0\n_4.27	158.0\n_4.33	142.0\n_4.39	159.0\n_4.45	189.0\n_4.51	173.0\n_4.57	150.0\n_4.63	149.0\n_4.69	146.0\n_4.75	158.0\n_4.81	153.0\n_4.87	175.0\n_4.93	162.0\n_4.99	185.0\ndata_NH4NO3_2\n_4.03	134.0\n_4.09	167.0\n_4.15	147.0\n_4.21	175.0\n_4.27	157.0\n_4.33	141.0\n_4.39	131.0\n_4.45	160.0\n_4.51	148.0\n_4.57	149.0\n_4.63	179.0\n_4.69	149.0\n_4.75	171.0\n_4.81	164.0\n_4.87	135.0\n_4.93	155.0\n_4.99	175.0\ndata_NH4NO3_3\n_4.03	144.0\n_4.09	174.0\n_4.15	158.0\n_4.21	159.0\n_4.27	161.0\n_4.33	171.0\n_4.39	146.0\n_4.45	154.0\n_4.93	175.0\n_4.99	169.0\ndata_NH4NO3_4\n_4.03	173.0\n_4.09	145.0\n_4.15	151.0\n_4.21	147.0\n_4.27	153.0\n_4.33	160.0\n_4.39	179.0\n_4.45	167.0\n_4.51	152.0\n_4.57	161.0\n_4.63	165.0\n_4.69	165.0\n_4.75	163.0\n_4.81	160.0\n_4.87	147.0\n_4.93	155.0\n_4.99	171.0\ndata_NH4NO3_5\n_4.03	167.0\n_4.09	158.0\n_4.15	146.0\n_4.21	152.0\n_4.27	152.0\n_4.33	169.0\n_4.39	139.0\n_4.45	132.0\n_4.51	168.0\n_4.57	147.0\n_4.63	147.0\n_4.69	163.0\n_4.75	140.0\n_4.81	142.0\n_4.87	177.0\n_4.93	169.0\n_4.99	166.0\ndata_NH4NO3_6\n_4.03	149.0\n_4.09	142.0\n_4.15	163.0\n_4.21	171.0\n_4.27	164.0\n_4.33	141.0\n_4.39	181.0\n_4.45	151.0\n_4.51	162.0\n_4.57	169.0\n_4.63	154.0\n_4.69	161.0\n_4.75	188.0\n_4.81	160.0\n_4.87	181.0\n_4.93	181.0\n_4.99	146.0\ndata_NH4NO3_7\n_4.03	141.0\n_4.09	130.0\n_4.15	139.0\n_4.21	157.0\n_4.27	169.0\n_4.33	185.0\n_4.39	155.0\n_4.45	155.0\n_4.51	142.0\n_4.57	178.0\n_4.63	158.0\n_4.69	170.0\n_4.75	161.0\n_4.81	157.0\n_4.87	165.0\n_4.93	201.0\n_4.99	168.0\ndata_NH4NO3_8\n_4.03	161.0\n_4.09	136.0\n_4.15	154.0\n_4.21	166.0\n_4.27	148.0\n_4.33	165.0\n_4.39	158.0\n_4.45	161.0\n_4.51	158.0\n_4.57	152.0\n_4.63	164.0\n_4.69	166.0\n_4.75	169.0\n_4.81	167.0\n_4.87	168.0\n_4.93	144.0\n_4.99	166.0\ndata_NH4NO3_9\n_4.03	170.0\n_4.09	142.0\n_4.15	144.0\n_4.21	143.0\n_4.27	158.0\n_4.33	147.0\n_4.39	164.0\n_4.45	151.0\n_4.51	163.0\n_4.57	167.0\n_4.63	158.0\n_4.69	170.0\n_4.75	171.0\n_4.81	160.0\n_4.87	172.0\n_4.93	163.0\n_4.99	164.0\ndata_NH4NO3_10\n_4.03	135.0\n_4.09	136.0\n_4.15	162.0\n_4.21	150.0\n_4.27	142.0\n_4.33	159.0\n_4.39	162.0\n_4.45	149.0\n_4.51	141.0\n_4.57	132.0\n_4.63	165.0\n_4.69	166.0\n_4.75	149.0\n_4.81	145.0\n_4.87	157.0\n_4.93	153.0\n_4.99	171.0\ndata_NH4NO3_11\n_4.03	155.0\n_4.09	153.0\n_4.15	140.0\n_4.21	156.0\n_4.27	177.0\n_4.33	142.0\n_4.39	155.0\n_4.45	157.0\n_4.51	143.0\n_4.57	150.0\n_4.63	154.0\n_4.69	178.0\n_4.75	162.0\n_4.81	144.0\n_4.87	167.0\n_4.93	160.0\n_4.99	142.0\ndata_NH4NO3_12\n_4.03	153.0\n_4.09	154.0\n_4.15	142.0\n_4.21	146.0\n_4.27	158.0\n_4.33	166.0\n_4.39	159.0\n_4.45	159.0\n_4.51	160.0\n_4.57	153.0\n_4.63	164.0\n_4.69	143.0\n_4.75	156.0\n_4.81	173.0\n_4.87	155.0\n_4.93	171.0\n_4.99	170.0\ndata_NH4NO3_13\n_4.03	143.0\n_4.09	164.0\n_4.15	159.0\n_4.21	144.0\n_4.27	122.0\n_4.33	173.0\n_4.39	147.0\n_4.45	147.0\n_4.51	141.0\n_4.57	166.0\n_4.63	177.0\n_4.69	145.0\n_4.75	161.0\n_4.81	151.0\n_4.87	137.0\n_4.93	182.0\n_4.99	163.0\ndata_NH4NO3_14\n_4.03	148.0\n_4.09	146.0\n_4.15	154.0\n_4.21	155.0\n_4.27	149.0\n_4.33	141.0\n_4.39	145.0\n_4.45	162.0\n_4.51	172.0\n_4.57	145.0\n_4.63	157.0\n_4.69	172.0\n_4.75	174.0\n_4.81	160.0\n_4.87	171.0\n_4.93	165.0\n_4.99	158.0\ndata_NH4NO3_15\n_4.03	159.0\n_4.09	142.0\n_4.15	152.0\n_4.21	135.0\n_4.27	171.0\n_4.33	172.0\n_4.39	164.0\n_4.45	161.0\n_4.51	141.0\n_4.57	164.0\n_4.63	162.0\n_4.69	147.0\n_4.75	156.0\n_4.81	174.0\n_4.87	175.0\n_4.93	163.0\n_4.99	157.0\ndata_NH4NO3_16\n_4.03	131.0\n_4.09	136.0\n_4.15	159.0\n_4.21	160.0\n_4.27	170.0\n_4.33	144.0\n_4.39	165.0\n_4.45	142.0\n_4.51	138.0\n_4.57	164.0\n_4.63	178.0\n_4.69	140.0\n_4.75	156.0\n_4.81	161.0\n_4.87	168.0\n_4.93	181.0\n_4.99	160.0";
   std::string source_win  = "data_NH4NO3_1\r\n_4.03	154.0\r\n_4.09	140.0\r\n_4.15	134.0\r\n_4.21	171.0\r\n_4.27	158.0\r\n_4.33	142.0\r\n_4.39	159.0\r\n_4.45	189.0\r\n_4.51	173.0\r\n_4.57	150.0\r\n_4.63	149.0\r\n_4.69	146.0\r\n_4.75	158.0\r\n_4.81	153.0\r\n_4.87	175.0\r\n_4.93	162.0\r\n_4.99	185.0\r\ndata_NH4NO3_2\r\n_4.03	134.0\r\n_4.09	167.0\r\n_4.15	147.0\r\n_4.21	175.0\r\n_4.27	157.0\r\n_4.33	141.0\r\n_4.39	131.0\r\n_4.45	160.0\r\n_4.51	148.0\r\n_4.57	149.0\r\n_4.63	179.0\r\n_4.69	149.0\r\n_4.75	171.0\r\n_4.81	164.0\r\n_4.87	135.0\r\n_4.93	155.0\r\n_4.99	175.0\r\ndata_NH4NO3_3\r\n_4.03	144.0\r\n_4.09	174.0\r\n_4.15	158.0\r\n_4.21	159.0\r\n_4.27	161.0\r\n_4.33	171.0\r\n_4.39	146.0\r\n_4.45	154.0\r\n_4.93	175.0\r\n_4.99	169.0\r\ndata_NH4NO3_4\r\n_4.03	173.0\r\n_4.09	145.0\r\n_4.15	151.0\r\n_4.21	147.0\r\n_4.27	153.0\r\n_4.33	160.0\r\n_4.39	179.0\r\n_4.45	167.0\r\n_4.51	152.0\r\n_4.57	161.0\r\n_4.63	165.0\r\n_4.69	165.0\r\n_4.75	163.0\r\n_4.81	160.0\r\n_4.87	147.0\r\n_4.93	155.0\r\n_4.99	171.0\r\ndata_NH4NO3_5\r\n_4.03	167.0\r\n_4.09	158.0\r\n_4.15	146.0\r\n_4.21	152.0\r\n_4.27	152.0\r\n_4.33	169.0\r\n_4.39	139.0\r\n_4.45	132.0\r\n_4.51	168.0\r\n_4.57	147.0\r\n_4.63	147.0\r\n_4.69	163.0\r\n_4.75	140.0\r\n_4.81	142.0\r\n_4.87	177.0\r\n_4.93	169.0\r\n_4.99	166.0\r\ndata_NH4NO3_6\r\n_4.03	149.0\r\n_4.09	142.0\r\n_4.15	163.0\r\n_4.21	171.0\r\n_4.27	164.0\r\n_4.33	141.0\r\n_4.39	181.0\r\n_4.45	151.0\r\n_4.51	162.0\r\n_4.57	169.0\r\n_4.63	154.0\r\n_4.69	161.0\r\n_4.75	188.0\r\n_4.81	160.0\r\n_4.87	181.0\r\n_4.93	181.0\r\n_4.99	146.0\r\ndata_NH4NO3_7\r\n_4.03	141.0\r\n_4.09	130.0\r\n_4.15	139.0\r\n_4.21	157.0\r\n_4.27	169.0\r\n_4.33	185.0\r\n_4.39	155.0\r\n_4.45	155.0\r\n_4.51	142.0\r\n_4.57	178.0\r\n_4.63	158.0\r\n_4.69	170.0\r\n_4.75	161.0\r\n_4.81	157.0\r\n_4.87	165.0\r\n_4.93	201.0\r\n_4.99	168.0\r\ndata_NH4NO3_8\r\n_4.03	161.0\r\n_4.09	136.0\r\n_4.15	154.0\r\n_4.21	166.0\r\n_4.27	148.0\r\n_4.33	165.0\r\n_4.39	158.0\r\n_4.45	161.0\r\n_4.51	158.0\r\n_4.57	152.0\r\n_4.63	164.0\r\n_4.69	166.0\r\n_4.75	169.0\r\n_4.81	167.0\r\n_4.87	168.0\r\n_4.93	144.0\r\n_4.99	166.0\r\ndata_NH4NO3_9\r\n_4.03	170.0\r\n_4.09	142.0\r\n_4.15	144.0\r\n_4.21	143.0\r\n_4.27	158.0\r\n_4.33	147.0\r\n_4.39	164.0\r\n_4.45	151.0\r\n_4.51	163.0\r\n_4.57	167.0\r\n_4.63	158.0\r\n_4.69	170.0\r\n_4.75	171.0\r\n_4.81	160.0\r\n_4.87	172.0\r\n_4.93	163.0\r\n_4.99	164.0\r\ndata_NH4NO3_10\r\n_4.03	135.0\r\n_4.09	136.0\r\n_4.15	162.0\r\n_4.21	150.0\r\n_4.27	142.0\r\n_4.33	159.0\r\n_4.39	162.0\r\n_4.45	149.0\r\n_4.51	141.0\r\n_4.57	132.0\r\n_4.63	165.0\r\n_4.69	166.0\r\n_4.75	149.0\r\n_4.81	145.0\r\n_4.87	157.0\r\n_4.93	153.0\r\n_4.99	171.0\r\ndata_NH4NO3_11\r\n_4.03	155.0\r\n_4.09	153.0\r\n_4.15	140.0\r\n_4.21	156.0\r\n_4.27	177.0\r\n_4.33	142.0\r\n_4.39	155.0\r\n_4.45	157.0\r\n_4.51	143.0\r\n_4.57	150.0\r\n_4.63	154.0\r\n_4.69	178.0\r\n_4.75	162.0\r\n_4.81	144.0\r\n_4.87	167.0\r\n_4.93	160.0\r\n_4.99	142.0\r\ndata_NH4NO3_12\r\n_4.03	153.0\r\n_4.09	154.0\r\n_4.15	142.0\r\n_4.21	146.0\r\n_4.27	158.0\r\n_4.33	166.0\r\n_4.39	159.0\r\n_4.45	159.0\r\n_4.51	160.0\r\n_4.57	153.0\r\n_4.63	164.0\r\n_4.69	143.0\r\n_4.75	156.0\r\n_4.81	173.0\r\n_4.87	155.0\r\n_4.93	171.0\r\n_4.99	170.0\r\ndata_NH4NO3_13\r\n_4.03	143.0\r\n_4.09	164.0\r\n_4.15	159.0\r\n_4.21	144.0\r\n_4.27	122.0\r\n_4.33	173.0\r\n_4.39	147.0\r\n_4.45	147.0\r\n_4.51	141.0\r\n_4.57	166.0\r\n_4.63	177.0\r\n_4.69	145.0\r\n_4.75	161.0\r\n_4.81	151.0\r\n_4.87	137.0\r\n_4.93	182.0\r\n_4.99	163.0\r\ndata_NH4NO3_14\r\n_4.03	148.0\r\n_4.09	146.0\r\n_4.15	154.0\r\n_4.21	155.0\r\n_4.27	149.0\r\n_4.33	141.0\r\n_4.39	145.0\r\n_4.45	162.0\r\n_4.51	172.0\r\n_4.57	145.0\r\n_4.63	157.0\r\n_4.69	172.0\r\n_4.75	174.0\r\n_4.81	160.0\r\n_4.87	171.0\r\n_4.93	165.0\r\n_4.99	158.0\r\ndata_NH4NO3_15\r\n_4.03	159.0\r\n_4.09	142.0\r\n_4.15	152.0\r\n_4.21	135.0\r\n_4.27	171.0\r\n_4.33	172.0\r\n_4.39	164.0\r\n_4.45	161.0\r\n_4.51	141.0\r\n_4.57	164.0\r\n_4.63	162.0\r\n_4.69	147.0\r\n_4.75	156.0\r\n_4.81	174.0\r\n_4.87	175.0\r\n_4.93	163.0\r\n_4.99	157.0\r\ndata_NH4NO3_16\r\n_4.03	131.0\r\n_4.09	136.0\r\n_4.15	159.0\r\n_4.21	160.0\r\n_4.27	170.0\r\n_4.33	144.0\r\n_4.39	165.0\r\n_4.45	142.0\r\n_4.51	138.0\r\n_4.57	164.0\r\n_4.63	178.0\r\n_4.69	140.0\r\n_4.75	156.0\r\n_4.81	161.0\r\n_4.87	168.0\r\n_4.93	181.0\r\n_4.99	160.0";

   pegtl::memory_input in_unix(source_unix, "string");
   pegtl::memory_input in_win(source_win, "string");


   std::vector<std::string> match_unix = read_input(in_unix);
   std::vector<std::string> match_win = read_input(in_win);


   std::cout << match_unix.size()<<std::endl; //should be 546
   std::cout << match_win.size() <<std::endl;

   return 0;
}

[d-frey]

It seems to be a bug in the main branch, it works as expected when you use the latest release or the 3.x branch.

[ColinH]

FWIW it also works correctly in the evolution branch. I'll try to track it down in main.

[ColinH]

It was a bug that I introduced temporarily at the beginning of the input refactoring, it's now fixed in main in 3567b2d.

[rowlesmr]

Ta a lot. Somehow I'd managed to check out dev, rather than the latest release. At least it was of some help!