diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/ParentQueryIteratorTest.java b/warehouse/query-core/src/test/java/datawave/query/iterator/ParentQueryIteratorTest.java index 83105188030..e58922346a0 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/ParentQueryIteratorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/ParentQueryIteratorTest.java @@ -4,18 +4,18 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.SortedMap; -import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; -import org.junit.Before; import org.junit.Test; import com.google.common.collect.Maps; @@ -34,288 +34,90 @@ public class ParentQueryIteratorTest { private static final String ID_PREFIX = "idpart1.idpart2."; - DocumentDeserializer deserializer = null; + private final DocumentDeserializer deserializer = new KryoDocumentDeserializer(); - @Before - public void setup() { - this.deserializer = new KryoDocumentDeserializer(); + @Test + public void testGetRangeProvider() { + ParentQueryIterator iterator = new ParentQueryIterator(); + RangeProvider provider = iterator.getRangeProvider(); + assertEquals(ParentRangeProvider.class.getSimpleName(), provider.getClass().getSimpleName()); } @Test - public void test() throws Throwable { - ParentQueryIterator qitr = new ParentQueryIterator(); - Map options = Maps.newHashMap(); - + public void testNormalIteration() throws Throwable { SortedMap data = QueryIteratorTest.createTestData(ID_PREFIX + "idpart3"); - createChildren(data); - options.put(QueryOptions.DISABLE_EVALUATION, "false"); + Map options = getOptions(); options.put(QueryOptions.QUERY, "FOO=='bars'"); - options.put(QueryOptions.TYPE_METADATA, "FOO:[test:datawave.data.type.LcNoDiacriticsType]"); - options.put(QueryOptions.REDUCED_RESPONSE, "false"); - options.put(Constants.RETURN_TYPE, "kryo"); - options.put(QueryOptions.FULL_TABLE_SCAN_ONLY, "false"); - options.put(QueryOptions.FILTER_MASKED_VALUES, "true"); - options.put(QueryOptions.TERM_FREQUENCY_FIELDS, "FOO"); - options.put(QueryOptions.INCLUDE_DATATYPE, "true"); - options.put(QueryOptions.INDEX_ONLY_FIELDS, "FOO"); - options.put(QueryOptions.START_TIME, "0"); - options.put(QueryOptions.END_TIME, Long.toString(Long.MAX_VALUE)); - options.put(QueryOptions.POSTPROCESSING_CLASSES, ""); - options.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "false"); - options.put(QueryOptions.NON_INDEXED_DATATYPES, ""); options.put(QueryOptions.CONTAINS_INDEX_ONLY_TERMS, "true"); - // the iterator will npe if these guys aren't set - qitr.setTimeFilter(TimeFilter.alwaysTrue()); - - qitr.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - qitr.seek(new Range(new Key("20121126_0", "foobar\u0000idpart1.idpart2.idpart31"), true, new Key("2121126_0", "foobar\u0000idpart1.idpart2" + "\0"), - false), Collections. emptySet(), false); - - assertTrue(qitr.hasTop()); - Key topKey = qitr.getTopKey(); - Key expectedKey = new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - assertEquals(expectedKey, topKey); - - Entry doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); - - Attribute recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_0/foobar/idpart1.idpart2.idpart31", recordId.getData()); - - assertTrue(qitr.hasTop()); - qitr.next(); - expectedKey = new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - topKey = qitr.getTopKey(); - assertEquals(expectedKey, topKey); - - doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); - - recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_0/foobar/idpart1.idpart2.idpart31", recordId.getData()); - - qitr.next(); + ParentQueryIterator iter = getIterator(); + iter.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - assertTrue(qitr.hasTop()); + Key start = new Key("20121126_0", "foobar\u0000idpart1.idpart2.idpart31"); + Key stop = new Key("2121126_0", "foobar\u0000idpart1.idpart2" + "\0"); + Range range = new Range(start, true, stop, false); - expectedKey = new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.3", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - topKey = qitr.getTopKey(); - assertEquals(expectedKey, topKey); + String expectedRecordId = "20121126_0/foobar/idpart1.idpart2.idpart31"; - doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); + Set expectedTKs = new HashSet<>(); + expectedTKs.add(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); + expectedTKs.add(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); + expectedTKs.add(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31.3", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); - recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_0/foobar/idpart1.idpart2.idpart31", recordId.getData()); - - qitr.next(); - - assertFalse(qitr.hasTop()); - } - - @Test - public void testGetRangeProvider() { - ParentQueryIterator iterator = new ParentQueryIterator(); - RangeProvider provider = iterator.getRangeProvider(); - assertEquals(ParentRangeProvider.class.getSimpleName(), provider.getClass().getSimpleName()); - } - - private void createChildren(SortedMap map) { - long ts = QueryIteratorTest.getTimeStamp(); - - long ts2 = ts + 10000; - long ts3 = ts + 200123; - - map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 1, ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 2, ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 3, ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 1, "FOO\0bars", ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 1, "BAR\0foo", ts2), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 2, "FOO\0bars", ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 2, "BAR\0foo", ts2), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 3, "FOO\0bars", ts), new Value(new byte[0])); - map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 3, "BAR\0foo", ts2), new Value(new byte[0])); - - map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 4, ts), new Value(new byte[0])); - map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 5, ts), new Value(new byte[0])); - map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 6, ts), new Value(new byte[0])); - map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 4, "FOO\0bar", ts), new Value(new byte[0])); - map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 5, "FOO\0bar", ts), new Value(new byte[0])); - map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 5, "BAR\0foo", ts2), new Value(new byte[0])); - map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 6, "FOO\0bar", ts2), new Value(new byte[0])); - - map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 7, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 8, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 9, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 7, "FOO\0bar", ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 7, "BAR\0foo", ts3), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 8, "FOO\0bar", ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 8, "BAR\0foo", ts3), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 9, "FOO\0bar", ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 9, "BAR\0foo", ts3), new Value(new byte[0])); + driveIterator(iter, range, expectedRecordId, expectedTKs); } @Test public void testParentFiOnlyDocsAllowed() throws Throwable { - ParentQueryIterator qitr = new ParentQueryIterator(); - Map options = Maps.newHashMap(); - SortedMap data = QueryIteratorTest.createTestData(ID_PREFIX + "idpart3"); - createOrphanedChildren(data); - options.put(QueryOptions.DISABLE_EVALUATION, "false"); + Map options = getOptions(); options.put(QueryOptions.QUERY, "FOO=='baz'"); - options.put(QueryOptions.TYPE_METADATA, "FOO:[test:datawave.data.type.LcNoDiacriticsType]"); - options.put(QueryOptions.REDUCED_RESPONSE, "false"); - options.put(Constants.RETURN_TYPE, "kryo"); - options.put(QueryOptions.FULL_TABLE_SCAN_ONLY, "false"); - options.put(QueryOptions.FILTER_MASKED_VALUES, "true"); - options.put(QueryOptions.TERM_FREQUENCY_FIELDS, "FOO"); - options.put(QueryOptions.INCLUDE_DATATYPE, "true"); - options.put(QueryOptions.INDEX_ONLY_FIELDS, "FOO"); - options.put(QueryOptions.START_TIME, "0"); - options.put(QueryOptions.END_TIME, Long.toString(Long.MAX_VALUE)); - options.put(QueryOptions.POSTPROCESSING_CLASSES, ""); - options.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "false"); - options.put(QueryOptions.NON_INDEXED_DATATYPES, ""); options.put(QueryOptions.CONTAINS_INDEX_ONLY_TERMS, "true"); - // the iterator will npe if these guys aren't set - qitr.setTimeFilter(TimeFilter.alwaysTrue()); + ParentQueryIterator iter = getIterator(); + iter.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - qitr.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - qitr.seek(new Range(new Key("20121126_2", "foobar\u0000idpart1.idpart2.idpart34"), true, new Key("2121126_3", "foobar\u0000idpart1.idpart2.idpart35"), - false), Collections. emptySet(), false); + Key start = new Key("20121126_2", "foobar\u0000idpart1.idpart2.idpart34"); + Key stop = new Key("2121126_3", "foobar\u0000idpart1.idpart2.idpart35"); + Range range = new Range(start, true, stop, false); - assertTrue(qitr.hasTop()); - Key topKey = qitr.getTopKey(); - Key expectedKey = new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - assertEquals(expectedKey, topKey); + String expectedRecordId = "20121126_2/foobar/idpart1.idpart2.idpart36"; - Entry doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); + Set expectedTKs = new HashSet<>(); + expectedTKs.add(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); + expectedTKs.add(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); - Attribute recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_2/foobar/idpart1.idpart2.idpart36", recordId.getData()); - - assertTrue(qitr.hasTop()); - qitr.next(); - expectedKey = new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - topKey = qitr.getTopKey(); - assertEquals(expectedKey, topKey); - - doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); - - recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_2/foobar/idpart1.idpart2.idpart36", recordId.getData()); - - qitr.next(); - - assertFalse(qitr.hasTop()); + driveIterator(iter, range, expectedRecordId, expectedTKs); } @Test public void testParentNoFiOnlyDocs() throws Throwable { - ParentQueryIterator qitr = new ParentQueryIterator(); - Map options = Maps.newHashMap(); - SortedMap data = QueryIteratorTest.createTestData(ID_PREFIX + "idpart3"); - createOrphanedChildren(data); - options.put(QueryOptions.DISABLE_EVALUATION, "false"); + Map options = getOptions(); + options.put(QueryOptions.QUERY, "FOO=='baz'"); - options.put(QueryOptions.TYPE_METADATA, "FOO:[test:datawave.data.type.LcNoDiacriticsType]"); - options.put(QueryOptions.REDUCED_RESPONSE, "false"); - options.put(Constants.RETURN_TYPE, "kryo"); - options.put(QueryOptions.FULL_TABLE_SCAN_ONLY, "false"); - options.put(QueryOptions.FILTER_MASKED_VALUES, "true"); - options.put(QueryOptions.TERM_FREQUENCY_FIELDS, "FOO"); - options.put(QueryOptions.INCLUDE_DATATYPE, "true"); - options.put(QueryOptions.INDEX_ONLY_FIELDS, "FOO"); - options.put(QueryOptions.START_TIME, "0"); - options.put(QueryOptions.DISABLE_DOCUMENTS_WITHOUT_EVENTS, "true"); - options.put(QueryOptions.END_TIME, Long.toString(Long.MAX_VALUE)); - options.put(QueryOptions.POSTPROCESSING_CLASSES, ""); - options.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "false"); - options.put(QueryOptions.NON_INDEXED_DATATYPES, ""); options.put(QueryOptions.CONTAINS_INDEX_ONLY_TERMS, "true"); - // the iterator will npe if these guys aren't set - qitr.setTimeFilter(TimeFilter.alwaysTrue()); + ParentQueryIterator iter = getIterator(); + iter.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - qitr.init(new SortedMapIterator(data), options, new SourceManagerTest.MockIteratorEnvironment()); - qitr.seek(new Range(new Key("20121126_2", "foobar\u0000idpart1.idpart2.idpart34"), true, new Key("2121126_3", "foobar\u0000idpart1.idpart2.idpart35"), - false), Collections. emptySet(), false); + Key start = new Key("20121126_2", "foobar\u0000idpart1.idpart2.idpart34"); + Key stop = new Key("2121126_3", "foobar\u0000idpart1.idpart2.idpart35"); + Range range = new Range(start, true, stop, false); - assertTrue(qitr.hasTop()); - Key topKey = qitr.getTopKey(); - Key expectedKey = new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - assertEquals(expectedKey, topKey); - - Entry doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); - - Attribute recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_2/foobar/idpart1.idpart2.idpart36", recordId.getData()); - - assertTrue(qitr.hasTop()); - qitr.next(); - expectedKey = new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()); - topKey = qitr.getTopKey(); - assertEquals(expectedKey, topKey); - - doc = deserializer.apply(Maps.immutableEntry(topKey, qitr.getTopValue())); - - recordId = doc.getValue().get(Document.DOCKEY_FIELD_NAME); - if (recordId instanceof Attributes) { - recordId = ((Attributes) recordId).getAttributes().iterator().next(); - } - assertEquals("20121126_2/foobar/idpart1.idpart2.idpart36", recordId.getData()); + String expectedRecordId = "20121126_2/foobar/idpart1.idpart2.idpart36"; - qitr.next(); + Set expectedTKs = new HashSet<>(); + expectedTKs.add(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.1", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); + expectedTKs.add(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36.2", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); - assertFalse(qitr.hasTop()); - } - - private void createOrphanedChildren(SortedMap map) { - long ts = QueryIteratorTest.getTimeStamp(); - - long ts3 = ts + 200123; - - // scenario 1, fi keys for child docs, but on cihldren or parent - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 1, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 2, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 3, ts), new Value(new byte[0])); - - // scenario 2, fi keys for child docs, children exist but no parent - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart35." + 1, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart35." + 2, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart35." + 1, "FOO\0baz", ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart35." + 2, "BAR\0foo", ts3), new Value(new byte[0])); - - // scenario 3, fi keys child docs, no children and parent exists - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart36." + 1, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart36." + 2, ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36", "FOO\0baz", ts), new Value(new byte[0])); - map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36", "BAR\0foo", ts3), new Value(new byte[0])); + driveIterator(iter, range, expectedRecordId, expectedTKs); } @Test @@ -330,21 +132,9 @@ public void testTearDown() throws Exception { new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart3" + 8, QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp()), new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart39", QueryIteratorTest.DEFAULT_CQ, "", QueryIteratorTest.getTimeStamp())); - Map options = Maps.newHashMap(); - - options.put(QueryOptions.DISABLE_EVALUATION, "false"); + Map options = getOptions(); options.put(QueryOptions.QUERY, "FOO == 'bar' && BAR == 'foo'"); - options.put(QueryOptions.TYPE_METADATA, "FOO:[test:datawave.data.type.LcNoDiacriticsType]"); - options.put(QueryOptions.REDUCED_RESPONSE, "true"); - options.put(Constants.RETURN_TYPE, "kryo"); - options.put(QueryOptions.FULL_TABLE_SCAN_ONLY, "false"); - options.put(QueryOptions.FILTER_MASKED_VALUES, "true"); - options.put(QueryOptions.INCLUDE_DATATYPE, "true"); options.put(QueryOptions.INDEX_ONLY_FIELDS, ""); - options.put(QueryOptions.START_TIME, "0"); - options.put(QueryOptions.END_TIME, Long.toString(Long.MAX_VALUE)); - options.put(QueryOptions.POSTPROCESSING_CLASSES, ""); - options.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "false"); options.put(QueryOptions.NON_INDEXED_DATATYPES, ""); options.put(QueryOptions.CONTAINS_INDEX_ONLY_TERMS, "false"); @@ -352,7 +142,7 @@ public void testTearDown() throws Exception { qi.init(iter, options, new SourceManagerTest.MockIteratorEnvironment()); - qi.seek(new Range(new Key("20121126"), false, new Key("20121127"), false), Collections. emptyList(), false); + qi.seek(new Range(new Key("20121126"), false, new Key("20121127"), false), Collections.emptyList(), false); while (qi.hasTop()) { System.out.println("begin loop1: " + expectation); @@ -375,8 +165,7 @@ public void testTearDown() throws Exception { qi.init(iter, options, new SourceManagerTest.MockIteratorEnvironment()); - qi.seek(new Range(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart35"), false, new Key("20121127"), false), - Collections. emptyList(), false); + qi.seek(new Range(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart35"), false, new Key("20121127"), false), Collections.emptyList(), false); while (qi.hasTop()) { Key tk = qi.getTopKey(); @@ -386,4 +175,114 @@ public void testTearDown() throws Exception { assertTrue("Still had expected keys: " + expectation, expectation.isEmpty()); } + + private void driveIterator(ParentQueryIterator iter, Range range, String expectedRecordId, Set expectedTopKeys) throws IOException { + + iter.seek(range, Collections.emptySet(), false); + + while (iter.hasTop()) { + Key tk = iter.getTopKey(); + assertTrue("unexpected top key was found: " + tk.toStringNoTime(), expectedTopKeys.remove(tk)); + + Document d = deserializer.apply(Maps.immutableEntry(tk, iter.getTopValue())).getValue(); + assertEquals(expectedRecordId, getRecordId(d)); + + iter.next(); + } + + assertFalse(iter.hasTop()); + assertTrue("expected top keys remain: " + expectedTopKeys, expectedTopKeys.isEmpty()); + } + + private String getRecordId(Document d) { + Attribute attr = d.get(Document.DOCKEY_FIELD_NAME); + if (attr instanceof Attributes) { + attr = ((Attributes) attr).getAttributes().iterator().next(); + } + return String.valueOf(attr.getData()); + } + + private ParentQueryIterator getIterator() { + ParentQueryIterator iter = new ParentQueryIterator(); + iter.setTimeFilter(TimeFilter.alwaysTrue()); + return iter; + } + + private Map getOptions() { + Map options = new HashMap<>(); + options.put(QueryOptions.DISABLE_EVALUATION, "false"); + options.put(QueryOptions.TYPE_METADATA, "FOO:[test:datawave.data.type.LcNoDiacriticsType]"); + options.put(QueryOptions.REDUCED_RESPONSE, "false"); + options.put(Constants.RETURN_TYPE, "kryo"); + options.put(QueryOptions.FULL_TABLE_SCAN_ONLY, "false"); + options.put(QueryOptions.FILTER_MASKED_VALUES, "true"); + options.put(QueryOptions.TERM_FREQUENCY_FIELDS, "FOO"); + options.put(QueryOptions.INCLUDE_DATATYPE, "true"); + options.put(QueryOptions.INDEX_ONLY_FIELDS, "FOO"); + options.put(QueryOptions.START_TIME, "0"); + options.put(QueryOptions.END_TIME, Long.toString(Long.MAX_VALUE)); + options.put(QueryOptions.POSTPROCESSING_CLASSES, ""); + options.put(QueryOptions.INCLUDE_GROUPING_CONTEXT, "false"); + options.put(QueryOptions.NON_INDEXED_DATATYPES, ""); + + return options; + } + + private void createChildren(SortedMap map) { + long ts = QueryIteratorTest.getTimeStamp(); + + long ts2 = ts + 10000; + long ts3 = ts + 200123; + + map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 1, ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 2, ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "fi\0" + "FOO", "bars\0" + "foobar\0" + ID_PREFIX + "idpart31." + 3, ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 1, "FOO\0bars", ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 1, "BAR\0foo", ts2), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 2, "FOO\0bars", ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 2, "BAR\0foo", ts2), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 3, "FOO\0bars", ts), new Value(new byte[0])); + map.put(new Key("20121126_0", "foobar\0" + ID_PREFIX + "idpart31." + 3, "BAR\0foo", ts2), new Value(new byte[0])); + + map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 4, ts), new Value(new byte[0])); + map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 5, ts), new Value(new byte[0])); + map.put(new Key("20121126_1", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart32." + 6, ts), new Value(new byte[0])); + map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 4, "FOO\0bar", ts), new Value(new byte[0])); + map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 5, "FOO\0bar", ts), new Value(new byte[0])); + map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 5, "BAR\0foo", ts2), new Value(new byte[0])); + map.put(new Key("20121126_1", "foobar\0" + ID_PREFIX + "idpart32." + 6, "FOO\0bar", ts2), new Value(new byte[0])); + + map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 7, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 8, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "bar\0" + "foobar\0" + ID_PREFIX + "idpart33." + 9, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 7, "FOO\0bar", ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 7, "BAR\0foo", ts3), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 8, "FOO\0bar", ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 8, "BAR\0foo", ts3), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 9, "FOO\0bar", ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart33." + 9, "BAR\0foo", ts3), new Value(new byte[0])); + } + + private void createOrphanedChildren(SortedMap map) { + long ts = QueryIteratorTest.getTimeStamp(); + + long ts3 = ts + 200123; + + // scenario 1, fi keys for child docs, but on cihldren or parent + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 1, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 2, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart34." + 3, ts), new Value(new byte[0])); + + // scenario 2, fi keys for child docs, children exist but no parent + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart35." + 1, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart35." + 2, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart35." + 1, "FOO\0baz", ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart35." + 2, "BAR\0foo", ts3), new Value(new byte[0])); + + // scenario 3, fi keys child docs, no children and parent exists + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart36." + 1, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "fi\0" + "FOO", "baz\0" + "foobar\0" + ID_PREFIX + "idpart36." + 2, ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36", "FOO\0baz", ts), new Value(new byte[0])); + map.put(new Key("20121126_2", "foobar\0" + ID_PREFIX + "idpart36", "BAR\0foo", ts3), new Value(new byte[0])); + } }