Change definedView to be a map definedViews (NationalSecurityAgency#2227

)
cawaring · Feb 9, 2024 · 655b71d · 655b71d
1 parent c5287da
commit 655b71d
Show file tree

Hide file tree

Showing 22 changed files with 255 additions and 51 deletions.
diff --git a/properties/default.properties b/properties/default.properties
@@ -394,7 +394,7 @@ lookup.uuid.mappings=
 # Default uuidTypes
 lookup.uuid.uuidTypes=
 # Default lookup.uuid.beginDate
-lookup.uuid.beginDate=20100101
+lookup.uuid.beginDate=19700101
 
 ############################
 #

diff --git a/properties/dev.properties b/properties/dev.properties
@@ -211,28 +211,36 @@ event.query.data.decorators= \
 lookup.uuid.uuidTypes= \
           <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="ID" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
+\n        </bean> \
+\n        <bean class="datawave.query.data.UUIDType"> \
+\n            <property name="fieldName" value="EMBEDDED_CAST_PERSON_ID" /> \
+\n            <property name="definedViews">\
+\n              <util:map>\
+\n                <entry key="default" value="LuceneUUIDEventQuery"/> \
+\n              </util:map> \
+\n            </property> \
 \n        </bean> \
 \n        <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="EVENT_ID" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
 \n            <property name="allowWildcardAfter" value="28" /> \
 \n        </bean> \
 \n        <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="UUID" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
 \n        </bean> \
 \n        <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="PARENT_UUID" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
 \n        </bean> \
 \n        <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="PAGE_ID" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
 \n        </bean> \
 \n        <bean class="datawave.query.data.UUIDType"> \
 \n            <property name="fieldName" value="PAGE_TITLE" /> \
-\n            <property name="definedView" value="LuceneUUIDEventQuery" /> \
+\n            <property name="definedViews" ref="DefaultUUIDViews" /> \
 \n        </bean>
 
 query.metrics.marking=(PUBLIC)

diff --git a/warehouse/core/src/main/java/datawave/query/data/UUIDType.java b/warehouse/core/src/main/java/datawave/query/data/UUIDType.java
@@ -1,18 +1,34 @@
 package datawave.query.data;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+
 public class UUIDType {
 
+    public static final String DEFAULT_VIEW = "default";
+
     private String fieldName = null;
-    private String definedView = null;
     private Integer allowWildcardAfter = null;
 
+    private final Map<String,String> definedViews = new HashMap<>();
+
     public UUIDType() {}
 
     public UUIDType(String field, String view, Integer allowWildcardAfter) {
 
         this.fieldName = field;
-        this.definedView = view;
         this.allowWildcardAfter = allowWildcardAfter;
+
+        this.definedViews.put(DEFAULT_VIEW, view);
+    }
+
+    public UUIDType(String field, Map<String,String> views, Integer allowWildcardAfter) {
+        this.fieldName = field;
+        this.allowWildcardAfter = allowWildcardAfter;
+
+        this.definedViews.putAll(views);
     }
 
     public Integer getAllowWildcardAfter() {
@@ -31,11 +47,26 @@ public void setFieldName(String fieldName) {
         this.fieldName = fieldName;
     }
 
-    public String getDefinedView() {
-        return definedView;
+    public String getDefinedView(String context) {
+        if (StringUtils.isEmpty(context)) {
+            context = DEFAULT_VIEW;
+        }
+        return getDefinedViews().get(context);
+    }
+
+    public Map<String,String> getDefinedViews() {
+        return definedViews;
+    }
+
+    public void setDefinedViews(Map<String,String> views) {
+        this.definedViews.clear();
+        this.definedViews.putAll(views);
     }
 
-    public void setDefinedView(String definedView) {
-        this.definedView = definedView;
+    public void setDefinedView(String context, String view) {
+        if (StringUtils.isEmpty(context)) {
+            context = DEFAULT_VIEW;
+        }
+        getDefinedViews().put(context, view);
     }
 }
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java
@@ -91,6 +91,15 @@ else if (helper.getSeparator().equals("\\t"))
         String[] dataFields = tokenizer.getTokenArray();
         processFields(fields, dataFields);
 
+        // do the same for the aux data if it exists
+        if (event.getAuxData() instanceof String) {
+            String auxData = (String) event.getAuxData();
+            tokenizer.reset(auxData);
+            for (String dataField : tokenizer.getTokenArray()) {
+                processExtraField(fields, dataField);
+            }
+        }
+
         // and return the normalized fields
         return normalize(fields);
     }

diff --git a/...re/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/...re/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java
@@ -343,7 +343,7 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) {
      * @return the visibility
      */
     @Override
-    protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) {
+    public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) {
         byte[] visibility;
         if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) {
             try {

diff --git a/...st-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/...st-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -783,7 +783,7 @@ protected Multimap<BulkIngestKey,Value> createTermIndexColumn(RawRecordContainer
      *            the entry value
      * @return the visibility
      */
-    protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) {
+    public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) {
         ColumnVisibility visibility = event.getVisibility();
         if (value.getMarkings() != null && !value.getMarkings().isEmpty()) {
             try {

diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java
@@ -37,7 +37,7 @@ public class CSVRecordReader extends CSVReaderBase implements EventFixer {
     private static final IngestConfiguration ingestConfig = IngestConfigurationFactory.getIngestConfiguration();
     private static final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions();
 
-    private String csvEventId;
+    protected String csvEventId;
     private final Multimap<String,Object> metadataForValidation = ArrayListMultimap.create(100, 1);
     private String rawData = null;
 

diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -599,7 +599,7 @@ public QueryLogicTransformer getTransformer(Query settings) {
             reducedInSettings = Boolean.parseBoolean(reducedResponseStr);
         }
         boolean reduced = (this.isReducedResponse() || reducedInSettings);
-        DocumentTransformer transformer = new DocumentTransformer(this, settings, markingFunctions, responseObjectFactory, reduced);
+        DocumentTransformer transformer = createDocumentTransformer(this, settings, markingFunctions, responseObjectFactory, reduced);
         transformer.setEventQueryDataDecoratorTransformer(eventQueryDataDecoratorTransformer);
         transformer.setContentFieldNames(getConfig().getContentFieldNames());
         transformer.setLogTimingDetails(this.getLogTimingDetails());
@@ -612,6 +612,11 @@ public QueryLogicTransformer getTransformer(Query settings) {
         return this.transformerInstance;
     }
 
+    protected DocumentTransformer createDocumentTransformer(BaseQueryLogic<Entry<Key,Value>> logic, Query settings, MarkingFunctions markingFunctions,
+                    ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) {
+        return new DocumentTransformer(logic, settings, markingFunctions, responseObjectFactory, reducedResponse);
+    }
+
     public boolean isLongRunningQuery() {
         return getConfig().getGroupFields().hasGroupByFields();
     }

diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java
@@ -24,6 +24,7 @@
 import datawave.webservice.result.BaseQueryResponse;
 import datawave.webservice.result.EventQueryResponseBase;
 
+@SuppressWarnings("rawtypes")
 public class ContentQueryTransformer extends BaseQueryLogicTransformer<Entry<Key,Value>,EventBase> {
 
     private static final Logger log = Logger.getLogger(ContentQueryTransformer.class);

diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java
@@ -139,6 +139,8 @@ public String getFieldName(int number) {
                     return "fields";
                 case 3:
                     return "markings";
+                case 4:
+                    return "payload";
                 default:
                     return null;
             }

diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java b/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java
@@ -50,7 +50,7 @@ public abstract class AbstractDataTypeConfig implements DataTypeHadoopConfig {
     protected static final String DATE_FIELD_FORMAT = "yyyyMMdd";
     public static final SimpleDateFormat YMD_DateFormat = new SimpleDateFormat(DATE_FIELD_FORMAT);
     private static final String TEST_VISIBILITY = "public";
-    private static final String[] AUTH_VALUES = new String[] {"public", "Euro", "NA"};
+    private static final String[] AUTH_VALUES = new String[] {"public", "private", "Euro", "NA"};
     private static final Authorizations TEST_AUTHS = new Authorizations(AUTH_VALUES);
 
     /**

diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -15,9 +16,11 @@
 
 import datawave.data.normalizer.Normalizer;
 import datawave.data.type.NumberType;
+import datawave.ingest.csv.config.helper.ExtendedCSVHelper;
 import datawave.ingest.data.config.CSVHelper;
 import datawave.ingest.data.config.ingest.BaseIngestHelper;
 import datawave.ingest.input.reader.EventRecordReader;
+import datawave.marking.MarkingFunctions;
 import datawave.query.testframework.AbstractDataTypeConfig;
 import datawave.query.testframework.FieldConfig;
 import datawave.query.testframework.RawDataManager;
@@ -74,7 +77,8 @@ public enum CarField {
         COLOR(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true),
         DOORS(Normalizer.NUMBER_NORMALIZER),
         WHEELS(Normalizer.NUMBER_NORMALIZER),
-        DESC(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true);
+        DESC(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true),
+        VISIBILITY(Normalizer.NOOP_NORMALIZER, false);
 
         private static final List<String> Headers;
 
@@ -122,6 +126,14 @@ public static List<String> headers() {
             return Headers;
         }
 
+        public static List<String> eventSecurityFieldNames() {
+            return Collections.singletonList(VISIBILITY.name());
+        }
+
+        public static List<String> eventSecurityFieldDomains() {
+            return Collections.singletonList(MarkingFunctions.Default.COLUMN_VISIBILITY);
+        }
+
         /**
          * Returns a random set of fields, with o without {@link #EVENT_ID}.
          *
@@ -221,6 +233,9 @@ public CarsDataType(final String car, final String ingestFile, final FieldConfig
 
         // fields
         this.hConf.set(this.dataType + CSVHelper.DATA_HEADER, String.join(",", CarField.headers()));
+        this.hConf.set(this.dataType + ExtendedCSVHelper.Properties.EVENT_SECURITY_MARKING_FIELD_NAMES, String.join(",", CarField.eventSecurityFieldNames()));
+        this.hConf.set(this.dataType + ExtendedCSVHelper.Properties.EVENT_SECURITY_MARKING_FIELD_DOMAINS,
+                        String.join(",", CarField.eventSecurityFieldDomains()));
 
         log.debug(this.toString());
     }

diff --git a/warehouse/query-core/src/test/resources/input/ford-cars.csv b/warehouse/query-core/src/test/resources/input/ford-cars.csv
@@ -1,3 +1,3 @@
-20150501,ford-eventid-001,ford,explorer,blue,4,4,common description
-20150601,ford-eventid-002,ford,mustang,silver,2,4,common description
-20150701,ford-eventid-003,ford,fusion,magenta,4,4,common description
+20150501,ford-eventid-001,ford,explorer,blue,4,4,common description,public
+20150601,ford-eventid-002,ford,mustang,silver,2,4,common description,public
+20150701,ford-eventid-003,ford,fusion,magenta,4,4,common description,private
diff --git a/warehouse/query-core/src/test/resources/input/tesla-cars.csv b/warehouse/query-core/src/test/resources/input/tesla-cars.csv
@@ -1,3 +1,3 @@
-20150501,tesla-eventid-001,tesla,models,red,4,4,common description
-20150601,tesla-eventid-002,tesla,modelx,blue,4,4,common description
-20150701,tesla-eventid-003,tesla,model3,yellow,4,4,common description
+20150501,tesla-eventid-001,tesla,models,red,4,4,common description,public
+20150601,tesla-eventid-002,tesla,modelx,blue,4,4,common description,public
+20150701,tesla-eventid-003,tesla,model3,yellow,4,4,common description,private
diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml
@@ -344,12 +344,17 @@
         </property>
     </bean>
 
+    <util:map id="DefaultUUIDViews">
+        <entry key="default" value="LuceneUUIDEventQuery"/>
+    </util:map>
+
     <util:list id="UUIDTypeList" value-type="datawave.query.data.UUIDType">
         ${lookup.uuid.uuidTypes}
     </util:list>
 
     <bean id="LookupUUIDConfiguration" class="datawave.webservice.query.configuration.LookupUUIDConfiguration">
         <property name="uuidTypes" ref="UUIDTypeList" />
+        <property name="contentLookupTypes" ref="contentLookupTypes" />
         <property name="columnVisibility" value="" />
         <property name="beginDate" value="${lookup.uuid.beginDate}" />
     </bean>
@@ -398,7 +403,12 @@
             </list>
         </property>
     </bean>
-
+
+    <util:map id="contentLookupTypes">
+        <entry key="default" value="ContentQuery"/>
+    </util:map>
+
+
     <!-- Query Logic that returns document content -->
     <bean id="ContentQuery" parent="baseQueryLogic" scope="prototype"  class="datawave.query.tables.content.ContentQueryTable">
         <property name="tableName" value="${shard.table.name}" />

diff --git a/...uery/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java b/...uery/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java
@@ -42,7 +42,7 @@ public void setUuidTypes(List<UUIDType> uuidTypes) {
         List<UUIDType> goodTypes = new ArrayList<>();
         if (uuidTypes != null) {
             for (UUIDType uuidType : uuidTypes) {
-                if (uuidType.getDefinedView().equalsIgnoreCase("LuceneUUIDEventQuery")) {
+                if ("LuceneUUIDEventQuery".equalsIgnoreCase(uuidType.getDefinedView("default"))) {
                     goodTypes.add(uuidType);
                 }
             }

diff --git a/.../query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java b/.../query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java
@@ -1,6 +1,7 @@
 package datawave.webservice.query.configuration;
 
 import java.util.List;
+import java.util.Map;
 
 import javax.ws.rs.core.MultivaluedMap;
 
@@ -16,6 +17,8 @@
 public class LookupUUIDConfiguration {
 
     protected List<UUIDType> uuidTypes = null;
+    protected Map<String,String> contentLookupTypes = null;
+
     protected int batchLookupUpperLimit = LookupUUIDConstants.DEFAULT_BATCH_LOOKUP_UPPER_LIMIT;
     protected String beginDate = null;
     protected String columnVisibility;
@@ -37,6 +40,10 @@ public String getColumnVisibility() {
         return this.columnVisibility;
     }
 
+    public Map<String,String> getContentLookupTypes() {
+        return this.contentLookupTypes;
+    }
+
     public List<UUIDType> getUuidTypes() {
         return this.uuidTypes;
     }
@@ -63,6 +70,10 @@ public void setUuidTypes(List<UUIDType> uuidTypes) {
         this.uuidTypes = uuidTypes;
     }
 
+    public void setContentLookupTypes(Map<String,String> contentLookupTypes) {
+        this.contentLookupTypes = contentLookupTypes;
+    }
+
     public MultivaluedMap<String,String> optionalParamsToMap() {
         MultivaluedMap<String,String> p = new MultivaluedMapImpl<>();
         if (this.columnVisibility != null) {