diff --git a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java index a85ddfb3f29..988792fed8c 100644 --- a/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java +++ b/core/query/src/main/java/datawave/core/query/logic/QueryLogic.java @@ -463,6 +463,7 @@ default void preInitialize(Query settings, Set userAuthorization * Set a client configuration for scanner hints and consistency. * * @param config + * accumuloConfiguration */ void setClientConfig(AccumuloClientConfiguration config); diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java index 407d6baaf4e..169164716e8 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java @@ -35,6 +35,8 @@ public void parseHeader(String[] headerTokens) { case DURATION_OVERRIDE_COLUMN_HEADER: this.overrideColumnNumber = columnNumber; break; + default: + break; } columnNumber++; } diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java index 152f23e7bdc..74630654de9 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.StringWriter; import java.io.Writer; +import java.util.concurrent.atomic.AtomicInteger; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; @@ -50,14 +51,14 @@ void format(Writer writer) throws IOException { writer.write(transformToXmlString(ruleConfig)); } - private AgeOffRuleLoader.RuleConfig createRuleConfig(AgeOffRuleConfiguration configuration) throws IOException { - AgeOffRuleLoader.RuleConfig ruleConfig = new AgeOffRuleLoader.RuleConfig(this.configuration.getFilterClass().getName(), index++); + private static AgeOffRuleLoader.RuleConfig createRuleConfig(AgeOffRuleConfiguration configuration) throws IOException { + AgeOffRuleLoader.RuleConfig ruleConfig = new AgeOffRuleLoader.RuleConfig(configuration.getFilterClass().getName(), index++); ruleConfig.label(configuration.getRuleLabel()); - ruleConfig.setIsMerge(this.configuration.shouldMerge()); - ruleConfig.ttlValue(this.configuration.getTtlDuration()); - ruleConfig.ttlUnits(this.configuration.getTtlUnits()); - ruleConfig.matchPattern(buildMatchPattern()); - ruleConfig.customElements(this.configuration.getCustomElements()); + ruleConfig.setIsMerge(configuration.shouldMerge()); + ruleConfig.ttlValue(configuration.getTtlDuration()); + ruleConfig.ttlUnits(configuration.getTtlUnits()); + ruleConfig.matchPattern(buildMatchPattern(configuration, configuration.getIndentation())); + ruleConfig.customElements(configuration.getCustomElements()); return ruleConfig; } @@ -93,7 +94,7 @@ private String calculateIndentAmount() { return Integer.toString(length); } - private String buildMatchPattern() throws IOException { + private static String buildMatchPattern(AgeOffRuleConfiguration configuration, String indent) throws IOException { if (configuration.getPatternConfiguration() == null) { return ""; } @@ -104,14 +105,14 @@ private String buildMatchPattern() throws IOException { AgeOffCsvToMatchPatternFormatter patternFormatter = new AgeOffCsvToMatchPatternFormatter(configuration.getPatternConfiguration()); // add two indentations: one for items under the rule element and another for items under the matchPattern element - String extraIndentation = this.indent + this.indent; + String extraIndentation = indent + indent; patternFormatter.write(new IndentingDelegatingWriter(extraIndentation, writer)); String result = writer.toString(); // final indentation to precede the closing of matchPattern if (result.endsWith("\n")) { - return result + this.indent; + return result + indent; } return result; } diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java index 9af89959e21..e29b63255c4 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java @@ -21,7 +21,7 @@ public void write(String line) throws IOException { this.shouldIndentNextWrite = false; } - String indentedLine = line.replaceAll("\n", "\n" + indentation); + String indentedLine = line.replace("\n", "\n" + indentation); // withhold indentation until later if (indentedLine.endsWith("\n" + indentation)) { diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java index a46c0893109..84339875c21 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java @@ -8,6 +8,9 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; public abstract class TokenSpecParser { + + private static final String UNEXPECTED_EOS_ERROR_MSG = "Unexpected end of string literal parsing escape code"; + /** * Add a new token with its TTL to the structure. * @@ -221,7 +224,7 @@ protected String parseStrliteral() { } else { charPos++; if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } c = literalContent.charAt(charPos); switch (c) { @@ -250,7 +253,7 @@ protected String parseStrliteral() { String ordTxt = literalContent.substring(charPos + 1, charPos + 5); if (ordTxt.length() != 4) { if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } } try { @@ -265,7 +268,7 @@ protected String parseStrliteral() { String ordTxt = literalContent.substring(charPos + 1, charPos + 3); if (ordTxt.length() != 2) { if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } } try { diff --git a/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java b/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java index 81dc61c4dab..a7cd47ba1c7 100644 --- a/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java +++ b/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java @@ -62,7 +62,7 @@ protected static HashUID newId(final HashUID template, final String... extras) { if (null != template) { // Get the existing and new extras, if any final String extra1 = template.getExtra(); - final String extra2 = HashUID.mergeExtras(extras); + final String extra2 = UID.mergeExtras(extras); // Create a new UID based on existing and new extras if ((null != extra1) && (null != extra2)) { diff --git a/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java b/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java index 5d823f9caba..a523ee07521 100644 --- a/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java +++ b/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java @@ -215,7 +215,7 @@ protected static SnowflakeUID newId(final SnowflakeUID template, final String... if (null != template) { // Get the existing and new extras, if any final String extra1 = template.getExtra(); - final String extra2 = SnowflakeUID.mergeExtras(extras); + final String extra2 = UID.mergeExtras(extras); // Create a new UID based on existing and new extras if ((null != extra1) && (null != extra2)) { diff --git a/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java b/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java index c354aa8c834..6f80f923bb3 100644 --- a/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java +++ b/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java @@ -328,12 +328,6 @@ private EdgeValue(com.google.protobuf.CodedInputStream input, com.google.protobu case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; count_ = input.readInt64(); @@ -428,6 +422,12 @@ private EdgeValue(com.google.protobuf.CodedInputStream input, com.google.protobu uuidString_ = bs; break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -526,7 +526,7 @@ public UUID getDefaultInstanceForType() { return defaultInstance; } - private final com.google.protobuf.UnknownFieldSet unknownFields; + private final transient com.google.protobuf.UnknownFieldSet unknownFields; @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -546,12 +546,6 @@ private UUID(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; leastSignificantBits_ = input.readSInt64(); @@ -562,6 +556,12 @@ private UUID(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext mostSignificantBits_ = input.readSInt64(); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -1213,7 +1213,7 @@ public com.google.protobuf.ByteString getSourceValueBytes() { } public static final int SINKVALUE_FIELD_NUMBER = 6; - private java.lang.Object sinkValue_; + private transient java.lang.Object sinkValue_; /** * optional string sinkValue = 6; @@ -1260,7 +1260,7 @@ public com.google.protobuf.ByteString getSinkValueBytes() { } public static final int LOADDATE_FIELD_NUMBER = 7; - private java.lang.Object loadDate_; + private transient java.lang.Object loadDate_; /** * optional string loadDate = 7; @@ -1370,7 +1370,7 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder( } public static final int UUID_STRING_FIELD_NUMBER = 10; - private java.lang.Object uuidString_; + private transient java.lang.Object uuidString_; /** * optional string uuid_string = 10; diff --git a/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java b/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java index 3e9d83becb6..1fa74354c60 100644 --- a/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java +++ b/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java @@ -44,6 +44,8 @@ public class EdgeKey { private static final Logger log = Logger.getLogger(EdgeKey.class); + private static final String UNKNOWN_EDGE_KEY_FORMAT_MSG = "Can't encode unknown edge key format."; + // use the builder, not this nightmare constructor private EdgeKey(EDGE_FORMAT format, STATS_TYPE statsType, String sourceData, String sinkData, String family, String sourceRelationship, String sinkRelationship, String sourceAttribute1, String sinkAttribute1, String yyyymmdd, String attribute3, String attribute2, Text colvis, @@ -925,7 +927,7 @@ public Key encode() { return encode(EDGE_VERSION.DATE_PROTOBUF); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(UNKNOWN_EDGE_KEY_FORMAT_MSG + this); } } @@ -938,7 +940,7 @@ public Key encodeLegacyProtobufKey() { return encode(EDGE_VERSION.PROTOBUF); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(UNKNOWN_EDGE_KEY_FORMAT_MSG + this); } } @@ -949,7 +951,7 @@ public Key encodeLegacyAttribute2Key() { return encode(EDGE_VERSION.BASE_ATTRIBUTE2); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(UNKNOWN_EDGE_KEY_FORMAT_MSG + this); } } @@ -960,7 +962,7 @@ public Key encodeLegacyKey() { return encode(EDGE_VERSION.BASE); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(UNKNOWN_EDGE_KEY_FORMAT_MSG + this); } } diff --git a/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java b/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java index 42d6dc28912..3df9a7d5230 100644 --- a/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java +++ b/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java @@ -36,7 +36,7 @@ public void initialize(InputSplit split, TaskAttemptContext context) throws IOEx end = fileSplit.getLength() - start; pos = start; - FileOperations ops = RFileOperations.getInstance(); + FileOperations ops = FileOperations.getInstance(); String file = fileSplit.getPath().toString(); FileSystem fs = fileSplit.getPath().getFileSystem(context.getConfiguration()); CryptoService cs = CryptoFactoryLoader.getServiceForClient(CryptoEnvironment.Scope.TABLE, diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java index 5313274d3e1..a4a02ef29f2 100644 --- a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java +++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java @@ -145,12 +145,6 @@ private Info(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 24: { if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { termOffset_ = new java.util.ArrayList<>(); @@ -219,6 +213,12 @@ private Info(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext zeroOffsetMatch_ = input.readBool(); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java index d1c3e42f2be..df0a89e776c 100644 --- a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java +++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java @@ -155,12 +155,6 @@ private List(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; iGNORE_ = input.readBool(); @@ -187,6 +181,12 @@ private List(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext rEMOVEDUID_.add(input.readBytes()); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -273,7 +273,7 @@ public long getCOUNT() { // repeated string UID = 3; public static final int UID_FIELD_NUMBER = 3; - private com.google.protobuf.LazyStringList uID_; + private transient com.google.protobuf.LazyStringList uID_; /** * repeated string UID = 3; diff --git a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java index d51bcfdfca0..256d44750a4 100644 --- a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java +++ b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java @@ -21,7 +21,7 @@ public class MarkingFunctionsFactory { @Inject @SpringBean(refreshable = true) - private MarkingFunctions applicationMarkingFunctions; + private static MarkingFunctions applicationMarkingFunctions; public static final Logger log = LoggerFactory.getLogger(MarkingFunctionsFactory.class); @@ -59,7 +59,7 @@ public void init(@SuppressWarnings("UnusedParameters") @Observes @Initialized(Ap } @PostConstruct - public void postContruct() { + public static void postContruct() { markingFunctions = applicationMarkingFunctions; } } diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index fd55a67bba5..09130be2e30 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -117,6 +117,8 @@ public class BulkInputFormat extends InputFormat { protected static final String RANGESPLITSTRATEGY = PREFIX + ".split.strategy.class"; protected static final String MOCK = ".useInMemoryInstance"; + protected static final String UTF8 = "UTF-8"; + protected static final String RANGES = PREFIX + ".ranges"; protected static final String AUTO_ADJUST_RANGES = PREFIX + ".ranges.autoAdjust"; @@ -1296,8 +1298,8 @@ public AccumuloIteratorOption(String iteratorOption) { StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP); this.iteratorName = tokenizer.nextToken(); try { - this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8"); - this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8"); + this.key = URLDecoder.decode(tokenizer.nextToken(), UTF8); + this.value = URLDecoder.decode(tokenizer.nextToken(), UTF8); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -1318,7 +1320,7 @@ public String getValue() { @Override public String toString() { try { - return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8"); + return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF8") + FIELD_SEP + URLEncoder.encode(value, "UTF8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java index 5787bdf15f9..3b9a6a4546b 100644 --- a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java +++ b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java @@ -28,6 +28,9 @@ public class OperationEvaluator { public static final String GREATER_THAN_EQUALS = ">="; public static final String LESS_THAN = "<"; public static final String LESS_THAN_EQUALS = "<="; + private static final String OPERATOR_MUST_NOT_BE_BLANK = "operator must not be blank"; + private static final String NOT_A_VALID_CALC_OPERATOR = " is not a valid calculation operator"; + private static final String NOT_A_VALID_COMPARISON_OPERATOR = " is not a valid comparison operator"; /** * Return the result of the calculation for the provided integers using the provided calculation operator. Supported operators: @@ -48,7 +51,7 @@ public class OperationEvaluator { * @return the calculation result */ public static int calculate(int left, int right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -62,7 +65,7 @@ public static int calculate(int left, int right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -85,7 +88,7 @@ public static int calculate(int left, int right, String operator) { * @return the calculation result */ public static long calculate(long left, long right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -99,7 +102,7 @@ public static long calculate(long left, long right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -122,7 +125,7 @@ public static long calculate(long left, long right, String operator) { * @return the calculation result */ public static float calculate(float left, float right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -136,7 +139,7 @@ public static float calculate(float left, float right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -159,7 +162,7 @@ public static float calculate(float left, float right, String operator) { * @return the calculation result */ public static double calculate(double left, double right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -173,7 +176,7 @@ public static double calculate(double left, double right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -196,7 +199,7 @@ public static double calculate(double left, double right, String operator) { * @return the calculation result */ public static long calculate(Date left, Date right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); return calculate(left.getTime(), right.getTime(), operator); } @@ -221,7 +224,7 @@ public static long calculate(Date left, Date right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(int left, int right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -238,7 +241,7 @@ public static boolean compare(int left, int right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -262,7 +265,7 @@ public static boolean compare(int left, int right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(long left, long right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -279,7 +282,7 @@ public static boolean compare(long left, long right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -303,7 +306,7 @@ public static boolean compare(long left, long right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(float left, float right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -320,7 +323,7 @@ public static boolean compare(float left, float right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -344,7 +347,7 @@ public static boolean compare(float left, float right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(double left, double right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -361,7 +364,7 @@ public static boolean compare(double left, double right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -385,7 +388,7 @@ public static boolean compare(double left, double right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(Date left, Date right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); return compare(left.getTime(), right.getTime(), operator); } @@ -412,7 +415,7 @@ public static boolean compare(Date left, Date right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static > boolean compare(T left, T right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -429,7 +432,7 @@ public static > boolean compare(T left, T right, String case GREATER_THAN_EQUALS: return left.compareTo(right) >= 0; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -457,7 +460,7 @@ public static > boolean compare(T left, T right, String * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(T left, T right, String operator, Comparator comparator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); Preconditions.checkNotNull(comparator, "comparator must not be null"); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { @@ -475,7 +478,7 @@ public static boolean compare(T left, T right, String operator, Comparator= 0; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java b/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java index f51ab4b7a9a..2444ebd3c74 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java @@ -14,46 +14,46 @@ public class OptionsParser { // This is a first step towards standardizing these flags. // The intent is to expand upon this class later, but limiting to this set for now to avoid scope creep. Must.... resist... - public static final String instanceFlag = "-instance"; - public static final String instanceFlag2 = "-i"; - public static final String zookeepersFlag = "-zookeepers"; - public static final String zookeepersFlag2 = "-zk"; - public static final String userFlag = "-user"; - public static final String userFlag2 = "-u"; - public static final String passwordFlag = "-pass"; - public static final String passwordFlag2 = "-p"; - public static final String accCacheDirFlag = "-accCacheDir"; - public static final String configDirFlag = "-cd"; - public static final String additionalResourceFlag = "-"; - public static final String configDirSuffix = "config.xml"; + public static final String INSTANCE_FLAG = "-instance"; + public static final String INSTANCE_FLAG_2 = "-i"; + public static final String ZOOKEEPERS_FLAG = "-zookeepers"; + public static final String ZOOKEEPERS_FLAG_2 = "-zk"; + public static final String USER_FLAG = "-user"; + public static final String USER_FLAG_2 = "-u"; + public static final String PASSWORD_FLAG = "-pass"; + public static final String PASSWORD_FLAG_2 = "-p"; + public static final String ACC_CACHE_DIR_FLAG = "-accCacheDir"; + public static final String CONFIG_DIR_FLAG = "-cd"; + public static final String ADDITIONAL_RESOURCE_FLAG = "-"; + public static final String CONFIG_DIR_SUFFIX = "config.xml"; // todo: ingest options and more public static Configuration parseArguments(String[] args, Configuration conf) { String configDir = null; for (int i = 0; i < args.length; i++) { - if (args[i].equals(instanceFlag) || args[i].equals(instanceFlag2)) { + if (args[i].equals(INSTANCE_FLAG) || args[i].equals(INSTANCE_FLAG_2)) { AccumuloHelper.setInstanceName(conf, args[++i]); - } else if (args[i].equals(zookeepersFlag) || args[i].equals(zookeepersFlag2)) { + } else if (args[i].equals(ZOOKEEPERS_FLAG) || args[i].equals(ZOOKEEPERS_FLAG_2)) { AccumuloHelper.setZooKeepers(conf, args[++i]); - } else if (args[i].equals(userFlag) || args[i].equals(userFlag2)) { + } else if (args[i].equals(USER_FLAG) || args[i].equals(USER_FLAG_2)) { AccumuloHelper.setUsername(conf, args[++i]); - } else if (args[i].equals(passwordFlag) || args[i].equals(passwordFlag2)) { + } else if (args[i].equals(PASSWORD_FLAG) || args[i].equals(PASSWORD_FLAG_2)) { AccumuloHelper.setPassword(conf, PasswordConverter.parseArg(args[++i]).getBytes()); - } else if (args[i].equals(accCacheDirFlag)) { + } else if (args[i].equals(ACC_CACHE_DIR_FLAG)) { conf.set(TableConfigCache.ACCUMULO_CONFIG_CACHE_PATH_PROPERTY, args[++i]); - } else if (args[i].equals(configDirFlag)) { + } else if (args[i].equals(CONFIG_DIR_FLAG)) { configDir = args[++i]; if (null != configDir) { - ConfigurationFileHelper.setConfigurationFromFiles(conf, configDir, configDirSuffix); + ConfigurationFileHelper.setConfigurationFromFiles(conf, configDir, CONFIG_DIR_SUFFIX); } - } else if (args[i].startsWith(additionalResourceFlag)) { - String configName = args[i].substring(args[i].indexOf(additionalResourceFlag) + 1); + } else if (args[i].startsWith(ADDITIONAL_RESOURCE_FLAG)) { + String configName = args[i].substring(args[i].indexOf(ADDITIONAL_RESOURCE_FLAG) + 1); String configValue = args[++i]; conf.set(configName, configValue); } - else if (!args[i].startsWith(additionalResourceFlag)) { + else if (!args[i].startsWith(ADDITIONAL_RESOURCE_FLAG)) { conf.addResource(args[i]); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java index 6d09758cc4e..99254977691 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java @@ -31,7 +31,7 @@ public final class XMLFieldConfigHelper implements FieldConfigHelper { private static final Logger log = Logger.getLogger(XMLFieldConfigHelper.class); /** be explicit and use Apache Xerces-J here instead of relying on java to plug in the proper parser */ - private static final SAXParserFactory parserFactory = SAXParserFactoryImpl.newInstance(); + private static final SAXParserFactory parserFactory = SAXParserFactory.newInstance(); private boolean noMatchStored = true; private boolean noMatchIndexed = false; @@ -43,6 +43,8 @@ public final class XMLFieldConfigHelper implements FieldConfigHelper { private final Map knownFields = new HashMap<>(); private TreeMap patterns = new TreeMap<>(new BaseIngestHelper.MatcherComparator()); + private static final String UNEXPECTED_ATTRIBUTE = "Unexpected attribute encountered in: "; + public static class FieldInfo { boolean stored; boolean indexed; @@ -339,7 +341,7 @@ void startDefault(String uri, String localName, String qName, Attributes attribu this.defaultFieldType = lv; seenAttr.remove(INDEX_TYPE); } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'default' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'default' tag: '" + qn + "'"); } } @@ -381,7 +383,7 @@ void startNoMatch(String uri, String localName, String qName, Attributes attribu } seenAttr.remove(INDEX_TYPE); } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'nomatch' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'nomatch' tag: '" + qn + "'"); } } @@ -426,7 +428,7 @@ void startField(String uri, String localName, String qName, Attributes attribute } else if (INDEX_TYPE.equals(qn)) { fieldType = lv; } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'field' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'field' tag: '" + qn + "'"); } } @@ -481,7 +483,7 @@ void startFieldPattern(String uri, String localName, String qName, Attributes at } else if (INDEX_TYPE.equals(qn)) { fieldType = lv; } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'field' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'field' tag: '" + qn + "'"); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java index c3d28d3a2d8..c3d3e386825 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java @@ -139,6 +139,8 @@ public abstract class BaseIngestHelper extends AbstractIngestHelper implements C public static final String FIELD_CONFIG_FILE = ".data.category.field.config.file"; + private static final String PROPERTY_MALFORMED = " property malformed: "; + private static final Logger log = ThreadConfigurableLogger.getLogger(BaseIngestHelper.class); private Multimap> typeFieldMap = null; @@ -521,8 +523,8 @@ public static String getFieldName(Type dataType, String property, String propert // if this type already has a '.', then we have a malformed property // name if (dataType.typeName().indexOf('.') >= 0) { - log.error(propertyPattern + " property malformed: " + property); - throw new IllegalArgumentException(propertyPattern + " property malformed: " + property); + log.error(propertyPattern + PROPERTY_MALFORMED + property); + throw new IllegalArgumentException(propertyPattern + PROPERTY_MALFORMED + property); } String fieldName = property.substring(dataType.typeName().length() + 1, property.length() - propertyPattern.length()); @@ -544,8 +546,8 @@ public static String getFieldType(Type dataType, String property, String propert // if this type already has a '.', then we have a malformed property // name if (dataType.typeName().indexOf('.') >= 0) { - log.error(propertyPattern + " property malformed: " + property); - throw new IllegalArgumentException(propertyPattern + " property malformed: " + property); + log.error(propertyPattern + PROPERTY_MALFORMED + property); + throw new IllegalArgumentException(propertyPattern + PROPERTY_MALFORMED + property); } String fieldName = property.substring(dataType.typeName().length() + 1, property.length() - propertyPattern.length()); @@ -746,9 +748,7 @@ public HashSet normalizeFieldValue(String fieldName, value.setEventFieldValue(null); } values.add(value); - if (log.isDebugEnabled()) { - log.debug("added normalized field " + value + " to values set."); - } + logNormalizedField(normalizedContent, values); } return values; } @@ -796,9 +796,7 @@ protected Set normalize(NormalizedContentInterface n } else { values.add(normalize(normalizedContent, dataType)); } - if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); - } + logNormalizedField(normalizedContent, values); } return values; } @@ -813,9 +811,7 @@ protected Set normalize(NormalizedContentInterface n HashSet values = new HashSet<>(dataTypes.size()); for (datawave.data.type.Type dataType : dataTypes) { values.add(normalizeFieldValue(normalizedContent, dataType)); - if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); - } + logNormalizedField(normalizedContent, values); } return values; } else { @@ -827,14 +823,18 @@ protected Set normalize(NormalizedContentInterface n HashSet values = new HashSet<>(dataTypes.size()); for (datawave.data.type.Type dataType : dataTypes) { values.add(normalize(normalizedContent, dataType)); - if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); - } + logNormalizedField(normalizedContent, values); } return values; } } + private void logNormalizedField(NormalizedContentInterface normalizedContent, HashSet values) { + if (log.isDebugEnabled()) { + log.debug("added normalized field " + normalizedContent + " to values " + values); + } + } + @Override public boolean isNormalizedField(String fieldName) { if (this.normalizedFields.contains(fieldName)) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 925a1bea6a1..b24bb798f9f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -118,15 +118,15 @@ class VirtualFieldNormalizer extends NoOpType { public static final String VIRTUAL_FIELD_IGNORE_NORMALIZATION_ON_FIELD = ".data.combine.ignore.normalization.on.fields"; protected Map virtualFieldDefinitions = new HashMap<>(); - protected Map compiledFieldPatterns = null; + protected transient Map compiledFieldPatterns = null; protected String defaultSeparator = null; protected String defaultStartSeparator = null; protected String defaultEndSeparator = null; - protected Map grouping = new HashMap<>(); - protected Map allowMissing = new HashMap<>(); + protected transient Map grouping = new HashMap<>(); + protected transient Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - private MarkingFunctions markingFunctions; + private transient MarkingFunctions markingFunctions; public void setup(Type type, String instance, Configuration config) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java index 186055207d7..6dd57fff501 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java @@ -17,6 +17,7 @@ import datawave.ingest.data.config.NormalizedFieldAndValue; public abstract class AbstractNormalizer implements TextNormalizer { + private static final String FAILED_TO_NORMALIZE = "Failed to normalize "; private static final Logger log = Logger.getLogger(AbstractNormalizer.class); @Override @@ -223,7 +224,7 @@ public NormalizedContentInterface normalize(NormalizedContentInterface field) { if (field.getEventFieldName().equals("IP_GEO_FM_COORDINATES") && field.getEventFieldValue().equals("-99.999/-999.999")) { log.warn("Found know bad default value: IP_GEO_FM_COORDINATES=-99.999/-999.999"); } else { - log.error("Failed to normalize " + field.getEventFieldName() + '=' + field.getEventFieldValue(), e); + log.error(FAILED_TO_NORMALIZE + field.getEventFieldName() + '=' + field.getEventFieldValue(), e); } n.setError(e); } @@ -240,7 +241,7 @@ public Multimap normalize(Multimap normalizeMap(Multimap(), false, false); - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (IOException ioException) { + Assert.fail(ioException.getMessage()); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); @@ -1669,8 +1674,11 @@ public void testCleanUpJobDirectoryUnableToMakeDirectory() throws Exception { BulkIngestMapFileLoaderTest.WrappedLocalFileSystem fs = new BulkIngestMapFileLoaderTest.WrappedLocalFileSystem( createMockInputStream(new String[] {"/dummy/entry"}), null, false, false, false, false, null, false, false); - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); @@ -1713,9 +1721,11 @@ public void testCleanUpJobDirectoryJobSuccess() throws Exception { BulkIngestMapFileLoaderTest.WrappedLocalFileSystem fs = new BulkIngestMapFileLoaderTest.WrappedLocalFileSystem(createMockInputStream(), new FileStatus[0], false, false, false, false, null, false, false); - - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/ExcerptFields.java b/warehouse/query-core/src/main/java/datawave/query/attributes/ExcerptFields.java index a2f763fb222..7db096ab904 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/ExcerptFields.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/ExcerptFields.java @@ -162,7 +162,9 @@ public void put(String field, Integer offset, String direction) { * Replace a field mapping with another field * * @param field + * the field mapping * @param replacement + * the replacement */ public void replace(String field, String replacement) { SortedMap value = fieldMap.remove(field); diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java b/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java index 0fab5c560ce..a748e943d3c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java +++ b/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java @@ -185,7 +185,9 @@ public void putAll(Multimap fieldMap) { * Replace a field mapping with another field * * @param field + * the field * @param replacement + * the replacement */ public void replace(String field, String replacement) { Collection value = fieldMap.removeAll(field); diff --git a/warehouse/query-core/src/main/java/datawave/query/config/EdgeExtendedSummaryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/EdgeExtendedSummaryConfiguration.java index 41deac5f782..70ab0d7eef4 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/EdgeExtendedSummaryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/EdgeExtendedSummaryConfiguration.java @@ -64,7 +64,9 @@ public EdgeExtendedSummaryConfiguration(EdgeExtendedSummaryConfiguration other) * needs to be copied over here. * * @param other + * the other configuration * @param queries + * the collection of queries */ public EdgeExtendedSummaryConfiguration(EdgeExtendedSummaryConfiguration other, Collection queries) { this(other); diff --git a/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java index 6fab3abc41c..18728a22a94 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java @@ -109,7 +109,9 @@ public EdgeQueryConfiguration(EdgeQueryConfiguration other) { * needs to be copied over here. * * @param other + * the other configuration * @param queries + * the collection of queries */ public EdgeQueryConfiguration(EdgeQueryConfiguration other, Collection queries) { this(other); diff --git a/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java index 88bb1166b2b..bab3c67ce35 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java @@ -54,7 +54,9 @@ public RemoteQueryConfiguration(RemoteQueryConfiguration other) { * needs to be copied over here. * * @param other + * the other configuration * @param queries + * the collection of queries */ public RemoteQueryConfiguration(RemoteQueryConfiguration other, Collection queries) { this(other); diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java index a28866f6a47..234cdd964d7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java @@ -766,7 +766,9 @@ public void copyFrom(ShardQueryConfiguration other) { * needs to be copied over here. * * @param other + * the other configuration * @param queries + * the collection of queries */ public ShardQueryConfiguration(ShardQueryConfiguration other, Collection queries) { super(other); diff --git a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java index 315a0b5343e..ed676bef61e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/discovery/DiscoveryLogic.java @@ -622,6 +622,8 @@ public ShardIndexQueryTable clone() { * * @param scanner * a batch scanner + * @param queryData + * data from the query * @param indexedFields * set of currently indexed fields * @return iterator for discoveredthings diff --git a/warehouse/query-core/src/main/java/datawave/query/index/lookup/TupleToRange.java b/warehouse/query-core/src/main/java/datawave/query/index/lookup/TupleToRange.java index db8fd9a2beb..3d4492811c2 100644 --- a/warehouse/query-core/src/main/java/datawave/query/index/lookup/TupleToRange.java +++ b/warehouse/query-core/src/main/java/datawave/query/index/lookup/TupleToRange.java @@ -30,6 +30,10 @@ public class TupleToRange implements Function,Iterator< protected ShardQueryConfiguration config; /** + * The constructor + * + * @param tableName + * the table name * @param currentNode * the jexl node * @param config @@ -93,8 +97,10 @@ public static boolean isShardRange(String shard) { } /** + * Create document ranges * - * + * @param tableName + * the table name * @param queryNode * a query node * @param shard diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java index 27d8461dcac..ccfe79f76e3 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/DatawaveInterpreter.java @@ -148,7 +148,7 @@ public Object visit(ASTFunctionNode node, Object data) { * the variable name * @param undef * whether the variable is undefined or null - * @return + * @return null unless the strict option is set */ @Override protected Object unsolvableVariable(JexlNode node, String var, boolean undef) { @@ -168,7 +168,7 @@ protected Object unsolvableVariable(JexlNode node, String var, boolean undef) { * the method/function name * @param xany * the cause - * @return + * @return never returns, only throws exceptions */ @Override protected JexlException invocationException(JexlNode node, String methodName, Throwable xany) { diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/PrintingVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/PrintingVisitor.java index d043c39f94c..e8a44278fac 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/PrintingVisitor.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/PrintingVisitor.java @@ -255,6 +255,8 @@ public static String formattedQueryString(String query) throws ParseException { * a query node * @param maxChildNodes * maximum number of child nodes + * @param maxTermsToPrint + * maximum number of terms to print * @return formatted string * @throws ParseException * for parsing issues @@ -289,6 +291,8 @@ public static String formattedQueryString(JexlNode query) { * a query node * @param maxChildNodes * maximum number of child nodes + * @param maxTermsToPrint + * maximum number of terms to print * @return a formatted string */ public static String formattedQueryString(JexlNode query, int maxChildNodes, int maxTermsToPrint) { @@ -323,6 +327,8 @@ public static List formattedQueryStringList(JexlNode query) { * a query node * @param maxChildNodes * maximum number of child nodes + * @param maxTermsToPrint + * maximum number of terms to print * @return list of the formatted strings */ public static List formattedQueryStringList(JexlNode query, int maxChildNodes, int maxTermsToPrint) { diff --git a/warehouse/query-core/src/main/java/datawave/query/model/edge/EdgeQueryModel.java b/warehouse/query-core/src/main/java/datawave/query/model/edge/EdgeQueryModel.java index 98ea410b2c8..36f559d47e7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/model/edge/EdgeQueryModel.java +++ b/warehouse/query-core/src/main/java/datawave/query/model/edge/EdgeQueryModel.java @@ -37,6 +37,8 @@ public class EdgeQueryModel extends QueryModel { * * @param other * the other model + * @param fields + * the edge model fields * @throws InvalidModelException * if the model is invalid */ @@ -55,6 +57,8 @@ public EdgeModelFields getFields() { * * @param queryModelXml * the model xml + * @param fields + * the fields * @return EdgeQueryModel instance * @throws Exception * if there is an issue @@ -66,6 +70,10 @@ public static EdgeQueryModel loadModel(String queryModelXml, EdgeModelFields fie /** * Simple factory method to load a query model from the specified classpath resource. * + * @param queryModelXml + * the query model as xml + * @param fieldsFactory + * the edge model fields factory * @return EdgeQueryModel instance */ public static EdgeQueryModel loadModel(String queryModelXml, EdgeModelFieldsFactory fieldsFactory) throws Exception { @@ -75,7 +83,11 @@ public static EdgeQueryModel loadModel(String queryModelXml, EdgeModelFieldsFact /** * Simple factory method to load a query model from the specified classpath resource. * + * @param queryModelXml + * the query model as an xml String * @return EdgeQueryModel instance + * @throws Exception + * if unable to load the model */ public static EdgeQueryModel loadModel(String queryModelXml) throws Exception { return loadModel(queryModelXml, new DefaultEdgeModelFieldsFactory()); diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java b/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java index 2484f4e600d..c406442f948 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java @@ -282,7 +282,9 @@ public void setTableName(String tableName) { } /** - * @return + * Get the settings + * + * @return the settings */ public List getSettings() { return settings; diff --git a/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java b/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java index 4fd5e59e004..1c57e6bfe9c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java +++ b/warehouse/query-core/src/main/java/datawave/query/scheduler/PushdownScheduler.java @@ -136,11 +136,17 @@ public Iterator iterator() { } /** - * @return + * Concatenate Iterators + * + * @return the concatenated iterators * @throws ParseException + * if there is an error parsing * @throws TableNotFoundException + * if the table is not found * @throws AccumuloSecurityException + * if there is a security issue with Accumulo * @throws AccumuloException + * if there is a general Accumulo error */ protected Iterator concatIterators() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, ParseException { boolean hasNext = config.getQueriesIter().hasNext(); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardIndexQueryTable.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardIndexQueryTable.java index dc17a735467..2220d1b6e8d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardIndexQueryTable.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardIndexQueryTable.java @@ -420,6 +420,7 @@ public Iterator apply(Entry from) { * @param baseConfig * The shard query configuration * @param checkpoint + * The query checkpoint */ @Override public void setupQuery(AccumuloClient client, GenericQueryConfiguration baseConfig, QueryCheckpoint checkpoint) throws Exception { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 6182524dd66..9b292a8e5e7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -2779,6 +2779,7 @@ public Set getEvaluationOnlyFields() { * @param baseConfig * The shard query configuration * @param checkpoint + * The query checkpoint */ @Override public void setupQuery(AccumuloClient client, GenericQueryConfiguration baseConfig, QueryCheckpoint checkpoint) throws Exception { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/edge/EdgeQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/edge/EdgeQueryLogic.java index 432d4eff4e7..31fc9855b2b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/edge/EdgeQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/edge/EdgeQueryLogic.java @@ -713,7 +713,10 @@ public void setupQuery(GenericQueryConfiguration configuration) throws Exception * Takes in a batch scanner and returns an iterator over the DiscoveredThing objects contained in the value. * * @param scanner - * @return + * the batch scanner + * @param queryData + * the query data + * @return the transform iterator */ public static Iterator> transformScanner(final BatchScanner scanner, final QueryData queryData) { return transform(scanner.iterator(), new Function,Entry>() { @@ -795,6 +798,9 @@ public void close() { * the QueryData for the query logic to be configured * @param priority * the priority for the first of iterator filters + * + * @throws Exception + * when unable to add custom filter */ protected void addCustomFilters(QueryData data, int priority) throws Exception {} diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java index 7b7c03aa8dd..4b492ceb69e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java @@ -48,7 +48,11 @@ public Set getOverlappingNgrams() { return overlappingNgrams; } - /** Return a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep */ + /** + * Returns a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep + * + * @return a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep + **/ public String getOverlapsAsString() { return String.join(", ", getOverlappingNgrams().stream().map(NGramTuple::toString).collect(Collectors.toCollection(TreeSet::new))); } diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java index eb86e9629c3..f1efe3c7215 100644 --- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java +++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java @@ -4,6 +4,7 @@ * An interface for things that take two hashes and compare them to produce some sort of result. * * @param + * Hash type to score */ public interface SSDeepHashScorer { public T apply(SSDeepHash signature1, SSDeepHash signature2); diff --git a/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java b/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java index d0943b3d0d4..7bda9e160db 100644 --- a/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java +++ b/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java @@ -267,6 +267,7 @@ public GenericResponse ooziesubmit(MultivaluedMap queryPa OozieClient oozieClient = null; Properties oozieConf = null; + BadRequestException exception = null; try { oozieClient = new OozieClient((String) job.getJobConfigurationProperties().get(OozieJobConstants.OOZIE_CLIENT_PROP)); oozieConf = oozieClient.createConfiguration(); @@ -304,14 +305,17 @@ public GenericResponse ooziesubmit(MultivaluedMap queryPa log.error("Error validating audit parameters", e); BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); response.addException(qe); - throw new BadRequestException(qe, response); + exception = new BadRequestException(qe, response); } catch (Exception e) { log.error("Error auditing query", e); response.addMessage("Error auditing query - " + e.getMessage()); - throw new BadRequestException(e, response); + exception = new BadRequestException(e, response); } } } + if (null != exception) { + throw exception; + } // Submit the Oozie workflow. try { String jobID = null; diff --git a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java index 94d6da61b78..cdfc7facdda 100644 --- a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java +++ b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java @@ -421,6 +421,7 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP AccumuloClient client = null; BatchWriter writer = null; String tableName = this.checkModelTableName(modelTableName); + DatawaveWebApplicationException exception = null; try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); @@ -443,7 +444,7 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); log.error(qe); response.addException(qe); - throw new DatawaveWebApplicationException(qe, response); + exception = new DatawaveWebApplicationException(qe, response); } } if (null != client) { @@ -454,6 +455,9 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP } } } + if (null != exception) { + throw exception; + } cache.reloadTableCache(tableName); return response; } @@ -494,6 +498,7 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String AccumuloClient client = null; BatchWriter writer = null; String tableName = this.checkModelTableName(modelTableName); + DatawaveWebApplicationException exception = null; try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(getCurrentUserDN(), getCurrentProxyServers(), AccumuloConnectionFactory.Priority.LOW, trackingMap); @@ -516,7 +521,7 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); log.error(qe); response.addException(qe); - throw new DatawaveWebApplicationException(qe, response); + exception = new DatawaveWebApplicationException(qe, response); } } if (null != client) { @@ -527,6 +532,10 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String } } } + if (null != exception) { + throw exception; + } + if (reloadCache) cache.reloadTableCache(tableName); return response; diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java index ddb81dc9a7d..38628877ad1 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java +++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java @@ -872,6 +872,7 @@ public GenericResponse planQuery(@Required("logicName") @PathParam("logi } AuditType auditType = qd.logic.getAuditType(null); + Exception exception = null; try { Map> optionalQueryParameters = qp.getUnknownParameters(MapUtils.toMultiValueMap(queryParameters)); q = persister.create(qd.userDn, qd.dnList, marking, queryLogicName, qp, MapUtils.toMultivaluedMap(optionalQueryParameters)); @@ -900,16 +901,20 @@ public GenericResponse planQuery(@Required("logicName") @PathParam("logi log.error("Error validating audit parameters", e); BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); response.addException(qe); - throw new BadRequestException(qe, response); + exception = new BadRequestException(qe, response); } catch (Exception e) { log.error("Error auditing query", e); QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e); response.addException(qe); - throw qe; + exception = qe; } } } + if (null != exception) { + throw exception; + } + priority = qd.logic.getConnectionPriority(); Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); q.populateTrackingMap(trackingMap);