From 76d031cd99c84d70d8dc45cc32b228956d78f29a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 10 Jun 2024 15:33:23 +0100 Subject: [PATCH] Add variable size row test Signed-off-by: Gavin Halliday --- testing/unittests/jstreamtests.cpp | 86 ++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/testing/unittests/jstreamtests.cpp b/testing/unittests/jstreamtests.cpp index c5a96e2b8a5..d89dfd24ee5 100644 --- a/testing/unittests/jstreamtests.cpp +++ b/testing/unittests/jstreamtests.cpp @@ -212,6 +212,76 @@ class RandomDataProvider : public CDataProvider std::mt19937 generator; }; +//Output (int8 , string, dataset({unsigned3 }) +class VariableDataProvider : public CDataProvider +{ +public: + VariableDataProvider(bool _useCount) : useCount(_useCount) + { + name.append("Var_").append(useCount ? 'C' : 'S'); + } + + virtual size32_t create(IBufferedSerialOutputStream * target, unsigned row) + { + //Output (row, (string)row, (row % 7)items of (row, row*2, row*3)) + __uint64 id = row; + StringBuffer name; + name.append(row); + + target->put(8, &id); + size32_t len = name.length(); + target->put(4, &len); + target->put(len, name.str()); + size32_t childCount = (row % 7); + size32_t childSize = 3 * childCount; + if (useCount) + target->put(4, &childCount); + else + target->suspend(sizeof(size32_t)); + for (unsigned i=0; i < childCount; i++) + { + size32_t value = row * (i+1); + target->put(3, &value); + } + if (!useCount) + target->resume(sizeof(childSize), &childSize); + return 8 + 4 + len + 4 + childSize; + } + + virtual size32_t check(IBufferedSerialInputStream * source, unsigned row) + { + //Output (row, (string)row, (row % 7)items of (row, row*2, row*3)) + __uint64 id = row; + source->read(8, &id); + assertex(id == row); + + size32_t len; + source->read(4, &len); + StringBuffer name; + source->read(len, name.reserve(len)); + assertex(atoi(name) == row); + + size32_t size; + source->read(sizeof(size), &size); + if (useCount) + assertex(size == (row % 7)); + else + assertex(size == (row % 7) * 3); + for (unsigned i=0; i < (row % 7); i++) + { + size32_t value = 0; + source->read(3, &value); + size32_t expected = ((row * (i+1)) & 0xFFFFFF); + assertex(value == expected); + } + return 8 + 4 + len + 4 + (row % 7) * 3; + } + +protected: + bool useCount; +}; + + class NullOuputStream : public CInterfaceOf { virtual size32_t write(size32_t len, const void * ptr) { return len; } @@ -251,16 +321,16 @@ class JlibStreamStressTest : public CppUnit::TestFixture static constexpr bool testSkip = true; static constexpr bool testHighCompression = true; - static constexpr bool timeGeneration = true; + static constexpr bool timeGeneration = false; static constexpr bool testSimple = true; static constexpr bool testIncSequential = false; static constexpr bool testEvenSequential = true; static constexpr bool testParallel = true; static constexpr bool testThreadedWrite = true; - __uint64 timeSeq; - __uint64 timeSkip; - __uint64 timeRand; + __uint64 timeSeq = 0; + __uint64 timeSkip = 0; + __uint64 timeRand = 0; __uint64 testGeneration(CDataProvider & dataProvider, unsigned numRows) { @@ -531,6 +601,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture SequenceDataProvider seqProviderRC(40, true, false); SequenceDataProvider seqProviderPC(40, false, false); Sequence2DataProvider seq2Provider(40); + VariableDataProvider varProvider(false); SkipSequenceDataProvider skipProvider(17); ReservedDataProvider resProvider(40); RandomDataProvider randProvider(37); @@ -541,6 +612,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture { (this->*testFunction)(nullptr, seqProvider, 0x100000, 0x100000, numTestRows); (this->*testFunction)(lz4, seqProvider, 0x100000, 0x100000, numTestRows); + (this->*testFunction)(lz4, varProvider, 0x100000, 0x100000, numTestRows); (this->*testFunction)(lz4, resProvider, 0x100000, 0x100000, numTestRows); } @@ -613,6 +685,8 @@ class JlibStreamStressTest : public CppUnit::TestFixture SequenceDataProvider seqProviderRW(40, true, true); SequenceDataProvider seqProviderRC(40, true, false); SequenceDataProvider seqProviderPC(40, false, false); + VariableDataProvider varcProvider(true); + VariableDataProvider varsProvider(false); Sequence2DataProvider seq2Provider(40); ReservedDataProvider resProvider(40); ICompressHandler * lz4 = queryCompressHandler(COMPRESS_METHOD_LZ4); @@ -622,6 +696,8 @@ class JlibStreamStressTest : public CppUnit::TestFixture runSimpleStream(nullptr, seqProviderRC, 0x100000, 0x100000, numTestRows); runSimpleStream(nullptr, seqProviderPC, 0x100000, 0x100000, numTestRows); runSimpleStream(nullptr, seq2Provider, 0x100000, 0x100000, numTestRows); + runSimpleStream(nullptr, varsProvider, 0x100000, 0x100000, numTestRows); + runSimpleStream(nullptr, varcProvider, 0x100000, 0x100000, numTestRows); runSimpleStream(nullptr, resProvider, 0x100000, 0x100000, numTestRows); runSimpleStream(lz4, seqProviderPW, 0x100000, 0x100000, numTestRows); @@ -629,6 +705,8 @@ class JlibStreamStressTest : public CppUnit::TestFixture runSimpleStream(lz4, seqProviderRC, 0x100000, 0x100000, numTestRows); runSimpleStream(lz4, seqProviderPC, 0x100000, 0x100000, numTestRows); runSimpleStream(lz4, seq2Provider, 0x100000, 0x100000, numTestRows); + runSimpleStream(lz4, varsProvider, 0x100000, 0x100000, numTestRows); + runSimpleStream(lz4, varcProvider, 0x100000, 0x100000, numTestRows); runSimpleStream(lz4, resProvider, 0x100000, 0x100000, numTestRows); runSimpleStream(nullptr, seqProviderPW, 7, 0, numTestRows/10);