diff --git a/src/c4/yml/filter_processor.hpp b/src/c4/yml/filter_processor.hpp index f5380a1f..f08ee79c 100644 --- a/src/c4/yml/filter_processor.hpp +++ b/src/c4/yml/filter_processor.hpp @@ -181,7 +181,7 @@ struct FilterProcessorInplace { if(wpos < wcap) // respect write-capacity { - if(wpos <= rpos) + if((wpos <= rpos) && !unfiltered_chars) src.str[wpos] = c; } else @@ -197,7 +197,7 @@ struct FilterProcessorInplace RYML_ASSERT(num); if(wpos + num <= wcap) // respect write-capacity { - if(wpos <= rpos) + if((wpos <= rpos) && !unfiltered_chars) memset(src.str + wpos, c, num); } else @@ -214,7 +214,7 @@ struct FilterProcessorInplace RYML_ASSERT(rpos < src.len); if(wpos < wcap) // respect write-capacity { - if(wpos < rpos) // write only if wpos is behind rpos + if((wpos < rpos) && !unfiltered_chars) // write only if wpos is behind rpos src.str[wpos] = src.str[rpos]; } else @@ -232,7 +232,7 @@ struct FilterProcessorInplace RYML_ASSERT(rpos+num <= src.len); if(wpos + num <= wcap) // respect write-capacity { - if(wpos < rpos) // write only if wpos is behind rpos + if((wpos < rpos) && !unfiltered_chars) // write only if wpos is behind rpos { if(wpos + num <= rpos) // there is no overlap memcpy(src.str + wpos, src.str + rpos, num); @@ -255,7 +255,7 @@ struct FilterProcessorInplace RYML_ASSERT(rpos + 2 <= src.len); if(wpos < wcap) // respect write-capacity { - if(wpos <= rpos) + if((wpos <= rpos) && !unfiltered_chars) src.str[wpos] = c; } else @@ -277,7 +277,7 @@ struct FilterProcessorInplace const size_t rpos_next = rpos + nr + 1u; // add 1u to account for the escape character if(wpos_next <= rpos_next) // read and write do not overlap. just do a vanilla copy. { - if(wpos_next <= wcap) + if((wpos_next <= wcap) && !unfiltered_chars) memcpy(src.str + wpos, s, nw); rpos = rpos_next; wpos = wpos_next; @@ -292,8 +292,11 @@ struct FilterProcessorInplace RYML_ASSERT(rpos+nr+excess <= src.len); if(wpos_next <= wcap) { - memmove(src.str + wpos_next, src.str + rpos_next, src.len - rpos_next); - memcpy(src.str + wpos, s, nw); + if(!unfiltered_chars) + { + memmove(src.str + wpos_next, src.str + rpos_next, src.len - rpos_next); + memcpy(src.str + wpos, s, nw); + } rpos = wpos_next; // wpos, not rpos } else @@ -313,7 +316,7 @@ struct FilterProcessorInplace //const size_t unw = nw > (nr + 1u) ? nw - (nr + 1u) : 0; RYML_ASSERT(rpos_next <= src.len); const size_t required_size = wpos_next + (src.len - rpos_next); - _c4dbgip("inplace: add unfiltered {}->{} maxcap={}->{}!", unfiltered_chars, true, maxcap, required_size); + _c4dbgip("inplace: add unfiltered {}->{} maxcap={}->{}!", unfiltered_chars, true, maxcap, required_size > maxcap ? required_size : maxcap); RYML_ASSERT(required_size > wcap); unfiltered_chars = true; maxcap = required_size > maxcap ? required_size : maxcap; diff --git a/src/c4/yml/parse.cpp b/src/c4/yml/parse.cpp index d5bf37e5..3dc36884 100644 --- a/src/c4/yml/parse.cpp +++ b/src/c4/yml/parse.cpp @@ -4787,7 +4787,7 @@ FilterResultInPlace Parser::filter_scalar_squoted_in_place(substr dst, size_t ca /* double quoted */ // a debugging scaffold: -#if 0 +#if 1 #define _c4dbgfdq(fmt, ...) _c4dbgpf("filt_dquo[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__) #else #define _c4dbgfdq(...) @@ -4992,6 +4992,10 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) { proc.skip(); } + else + { + _c4err("unknown character '{}' after '\\' pos={}", _c4prc(next), proc.rpos); + } _c4dbgfdq("backslash...sofar=[{}]~~~{}~~~", proc.wpos, proc.sofar()); } diff --git a/test/test_filter.cpp b/test/test_filter.cpp index 1bc27a75..e7d28b5b 100644 --- a/test/test_filter.cpp +++ b/test/test_filter.cpp @@ -444,14 +444,14 @@ TEST(FilterProcessorInplace, translate_esc_single) EXPECT_EQ(t.subject, "\t\b\n\r\tn\\r\\t"); EXPECT_FALSE(t.proc.unfiltered_chars); // can write this one - t.proc.translate_esc('+'); - EXPECT_EQ(t.proc.rpos, 12); // this is fine, no read is done + t.proc.set('+'); + EXPECT_EQ(t.proc.rpos, 10); // this is fine, no read is done EXPECT_EQ(t.proc.wpos, 10); EXPECT_EQ(t.subject, "\t\b\n\r\tn\\r\\+"); EXPECT_FALSE(t.proc.unfiltered_chars); // but this one will set to unfiltered - t.proc.translate_esc('x'); - EXPECT_EQ(t.proc.rpos, 14); // this is fine, no read is done + t.proc.set('x'); + EXPECT_EQ(t.proc.rpos, 10); // this is fine, no read is done EXPECT_EQ(t.proc.wpos, 11); EXPECT_EQ(t.subject, "\t\b\n\r\tn\\r\\+"); EXPECT_TRUE(t.proc.unfiltered_chars); @@ -870,11 +870,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 7); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122d"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "0011223"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 7); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122d3445566"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -884,11 +884,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 8); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122de"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233"); // can set because now wpos < rpos EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 8); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122de445566"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -898,11 +898,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 12); EXPECT_EQ(t.proc.wpos, 9); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122def"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "001122334"); // can set because now wpos < rpos EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 9); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122def45566"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -912,11 +912,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 10); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defg"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "0011223344"); // can set because now wpos < rpos EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 10); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defg5566"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -926,11 +926,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 11); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defgh"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233445"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 11); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defgh566"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -940,11 +940,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 12); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghi"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "001122334455"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 12); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defghi66"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -954,11 +954,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 13); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghij"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "0011223344556"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 13); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defghij6"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -968,11 +968,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 14); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghijk"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233445566"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 14); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defghijk"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -982,11 +982,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 15); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghijk"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233445566"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 15); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defghijk"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -996,11 +996,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 16); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghijk"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233445566"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 16); EXPECT_EQ(t.proc.result().required_len(), 16); - EXPECT_EQ(t.subject, "001122defghijk"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 16); EXPECT_TRUE(t.proc.unfiltered_chars); // 00112233445566 @@ -1010,11 +1010,11 @@ TEST(FilterProcessorInplace, translate_esc_with_temporary_excess_requirement__tr EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 17); EXPECT_EQ(t.proc.src.len, 14); - EXPECT_EQ(t.proc.sofar(), "001122defghijk"); // can set because now wpos < rpos + EXPECT_EQ(t.proc.sofar(), "00112233445566"); EXPECT_EQ(t.proc.result().str.str, nullptr); EXPECT_EQ(t.proc.result().str.len, 17); EXPECT_EQ(t.proc.result().required_len(), 17); // increased! - EXPECT_EQ(t.subject, "001122defghijk"); + EXPECT_EQ(t.subject, "00112233445566"); EXPECT_EQ(t.proc.maxcap, 17); EXPECT_TRUE(t.proc.unfiltered_chars); }