Skip to content

Commit

Permalink
Add variable size row test
Browse files Browse the repository at this point in the history 
Signed-off-by: Gavin Halliday <[email protected]>
  • Loading branch information
@ghalliday
ghalliday committed Jun 10, 2024
1 parent 2dffeca commit 76d031c
Showing 1 changed file with 82 additions and 4 deletions.
86 changes: 82 additions & 4 deletions testing/unittests/jstreamtests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,76 @@ class RandomDataProvider : public CDataProvider
std::mt19937 generator;
};

//Output (int8 , string, dataset({unsigned3 })
class VariableDataProvider : public CDataProvider
{
public:
VariableDataProvider(bool _useCount) : useCount(_useCount)
{
name.append("Var_").append(useCount ? 'C' : 'S');
}

virtual size32_t create(IBufferedSerialOutputStream * target, unsigned row)
{
//Output (row, (string)row, (row % 7)items of (row, row*2, row*3))
__uint64 id = row;
StringBuffer name;
name.append(row);

target->put(8, &id);
size32_t len = name.length();
target->put(4, &len);
target->put(len, name.str());
size32_t childCount = (row % 7);
size32_t childSize = 3 * childCount;
if (useCount)
target->put(4, &childCount);
else
target->suspend(sizeof(size32_t));
for (unsigned i=0; i < childCount; i++)
{
size32_t value = row * (i+1);
target->put(3, &value);
}
if (!useCount)
target->resume(sizeof(childSize), &childSize);
return 8 + 4 + len + 4 + childSize;
}

virtual size32_t check(IBufferedSerialInputStream * source, unsigned row)
{
//Output (row, (string)row, (row % 7)items of (row, row*2, row*3))
__uint64 id = row;
source->read(8, &id);
assertex(id == row);

size32_t len;
source->read(4, &len);
StringBuffer name;
source->read(len, name.reserve(len));
assertex(atoi(name) == row);

size32_t size;
source->read(sizeof(size), &size);
if (useCount)
assertex(size == (row % 7));
else
assertex(size == (row % 7) * 3);
for (unsigned i=0; i < (row % 7); i++)
{
size32_t value = 0;
source->read(3, &value);
size32_t expected = ((row * (i+1)) & 0xFFFFFF);
assertex(value == expected);
}
return 8 + 4 + len + 4 + (row % 7) * 3;
}

protected:
bool useCount;
};


class NullOuputStream : public CInterfaceOf<IBufferedSerialOutputStream>
{
virtual size32_t write(size32_t len, const void * ptr) { return len; }
Expand Down Expand Up @@ -251,16 +321,16 @@ class JlibStreamStressTest : public CppUnit::TestFixture
static constexpr bool testSkip = true;
static constexpr bool testHighCompression = true;

static constexpr bool timeGeneration = true;
static constexpr bool timeGeneration = false;
static constexpr bool testSimple = true;
static constexpr bool testIncSequential = false;
static constexpr bool testEvenSequential = true;
static constexpr bool testParallel = true;
static constexpr bool testThreadedWrite = true;

__uint64 timeSeq;
__uint64 timeSkip;
__uint64 timeRand;
__uint64 timeSeq = 0;
__uint64 timeSkip = 0;
__uint64 timeRand = 0;

__uint64 testGeneration(CDataProvider & dataProvider, unsigned numRows)
{
Expand Down Expand Up @@ -531,6 +601,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture
SequenceDataProvider seqProviderRC(40, true, false);
SequenceDataProvider seqProviderPC(40, false, false);
Sequence2DataProvider seq2Provider(40);
VariableDataProvider varProvider(false);
SkipSequenceDataProvider skipProvider(17);
ReservedDataProvider resProvider(40);
RandomDataProvider randProvider(37);
Expand All @@ -541,6 +612,7 @@ class JlibStreamStressTest : public CppUnit::TestFixture
{
(this->*testFunction)(nullptr, seqProvider, 0x100000, 0x100000, numTestRows);
(this->*testFunction)(lz4, seqProvider, 0x100000, 0x100000, numTestRows);
(this->*testFunction)(lz4, varProvider, 0x100000, 0x100000, numTestRows);
(this->*testFunction)(lz4, resProvider, 0x100000, 0x100000, numTestRows);
}

Expand Down Expand Up @@ -613,6 +685,8 @@ class JlibStreamStressTest : public CppUnit::TestFixture
SequenceDataProvider seqProviderRW(40, true, true);
SequenceDataProvider seqProviderRC(40, true, false);
SequenceDataProvider seqProviderPC(40, false, false);
VariableDataProvider varcProvider(true);
VariableDataProvider varsProvider(false);
Sequence2DataProvider seq2Provider(40);
ReservedDataProvider resProvider(40);
ICompressHandler * lz4 = queryCompressHandler(COMPRESS_METHOD_LZ4);
Expand All @@ -622,13 +696,17 @@ class JlibStreamStressTest : public CppUnit::TestFixture
runSimpleStream(nullptr, seqProviderRC, 0x100000, 0x100000, numTestRows);
runSimpleStream(nullptr, seqProviderPC, 0x100000, 0x100000, numTestRows);
runSimpleStream(nullptr, seq2Provider, 0x100000, 0x100000, numTestRows);
runSimpleStream(nullptr, varsProvider, 0x100000, 0x100000, numTestRows);
runSimpleStream(nullptr, varcProvider, 0x100000, 0x100000, numTestRows);
runSimpleStream(nullptr, resProvider, 0x100000, 0x100000, numTestRows);

runSimpleStream(lz4, seqProviderPW, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, seqProviderRW, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, seqProviderRC, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, seqProviderPC, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, seq2Provider, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, varsProvider, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, varcProvider, 0x100000, 0x100000, numTestRows);
runSimpleStream(lz4, resProvider, 0x100000, 0x100000, numTestRows);

runSimpleStream(nullptr, seqProviderPW, 7, 0, numTestRows/10);
Expand Down

0 comments on commit 76d031c

Please sign in to comment.