From 09bf83bd2ca7bba2c60b1f554b539c729c14b960 Mon Sep 17 00:00:00 2001 From: Christopher Gross cogross Date: Mon, 22 Apr 2024 18:25:48 +0000 Subject: [PATCH] fix critical sonarqube issues --- .../off/util/AgeOffCsvColumnInformation.java | 2 + .../age/off/util/AgeOffRuleFormatter.java | 7 ++- .../off/util/IndentingDelegatingWriter.java | 2 +- .../iterators/filter/TokenSpecParser.java | 9 ++-- .../datawave/data/hash/HashUIDBuilder.java | 2 +- .../data/hash/SnowflakeUIDBuilder.java | 2 +- .../java/datawave/edge/protobuf/EdgeData.java | 34 +++++++------- .../main/java/datawave/edge/util/EdgeKey.java | 10 ++-- .../java/datawave/edge/util/EdgeKeyUtil.java | 10 ++-- .../mapreduce/job/RFileRecordReader.java | 2 +- .../datawave/ingest/protobuf/TermWeight.java | 14 +++--- .../java/datawave/ingest/protobuf/Uid.java | 16 +++---- .../marking/MarkingFunctionsFactory.java | 4 +- .../datawave/mr/bulk/BulkInputFormat.java | 8 ++-- .../datawave/util/OperationEvaluator.java | 47 ++++++++++--------- .../java/datawave/ingest/OptionsParser.java | 44 ++++++++--------- .../data/config/XMLFieldConfigHelper.java | 12 +++-- .../data/config/ingest/BaseIngestHelper.java | 20 ++++---- .../data/config/ingest/VirtualIngest.java | 8 ++-- .../data/normalizer/AbstractNormalizer.java | 7 +-- .../job/BulkIngestMapFileLoaderTest.java | 22 ++++++--- .../query/tables/ShardQueryLogic.java | 3 ++ .../query/tables/ssdeep/ScoredSSDeepPair.java | 6 ++- .../query/transformer/UniqueTransform.java | 3 ++ .../util/sortedset/RewritableSortedSet.java | 2 + .../sortedset/RewritableSortedSetImpl.java | 4 ++ .../util/ssdeep/SSDeepHashScorer.java | 1 + .../datawave/webservice/mr/MapReduceBean.java | 8 +++- .../webservice/query/model/ModelBean.java | 13 ++++- .../webservice/query/logic/QueryLogic.java | 1 + .../query/runner/QueryExecutorBean.java | 9 +++- 31 files changed, 200 insertions(+), 132 deletions(-) diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java index 407d6baaf4e..169164716e8 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffCsvColumnInformation.java @@ -35,6 +35,8 @@ public void parseHeader(String[] headerTokens) { case DURATION_OVERRIDE_COLUMN_HEADER: this.overrideColumnNumber = columnNumber; break; + default: + break; } columnNumber++; } diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java index 93e2e145ae8..d0776e0fbc6 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/AgeOffRuleFormatter.java @@ -51,8 +51,13 @@ void format(Writer writer) throws IOException { writer.close(); } + private static void increment() { + index++; + } + private AgeOffRuleLoader.RuleConfig createRuleConfig(AgeOffRuleConfiguration configuration) throws IOException { - AgeOffRuleLoader.RuleConfig ruleConfig = new AgeOffRuleLoader.RuleConfig(this.configuration.getFilterClass().getName(), index++); + AgeOffRuleLoader.RuleConfig ruleConfig = new AgeOffRuleLoader.RuleConfig(this.configuration.getFilterClass().getName(), index); + increment(); ruleConfig.label(configuration.getRuleLabel()); ruleConfig.setIsMerge(this.configuration.shouldMerge()); ruleConfig.ttlValue(this.configuration.getTtlDuration()); diff --git a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java index 9af89959e21..e29b63255c4 100644 --- a/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java +++ b/warehouse/age-off-utils/src/main/java/datawave/age/off/util/IndentingDelegatingWriter.java @@ -21,7 +21,7 @@ public void write(String line) throws IOException { this.shouldIndentNextWrite = false; } - String indentedLine = line.replaceAll("\n", "\n" + indentation); + String indentedLine = line.replace("\n", "\n" + indentation); // withhold indentation until later if (indentedLine.endsWith("\n" + indentation)) { diff --git a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java index a46c0893109..84339875c21 100644 --- a/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java +++ b/warehouse/age-off/src/main/java/datawave/iterators/filter/TokenSpecParser.java @@ -8,6 +8,9 @@ import datawave.iterators.filter.ageoff.AgeOffPeriod; public abstract class TokenSpecParser { + + private static final String UNEXPECTED_EOS_ERROR_MSG = "Unexpected end of string literal parsing escape code"; + /** * Add a new token with its TTL to the structure. * @@ -221,7 +224,7 @@ protected String parseStrliteral() { } else { charPos++; if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } c = literalContent.charAt(charPos); switch (c) { @@ -250,7 +253,7 @@ protected String parseStrliteral() { String ordTxt = literalContent.substring(charPos + 1, charPos + 5); if (ordTxt.length() != 4) { if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } } try { @@ -265,7 +268,7 @@ protected String parseStrliteral() { String ordTxt = literalContent.substring(charPos + 1, charPos + 3); if (ordTxt.length() != 2) { if (charPos >= literalContent.length()) { - throw error("Unexpected end of string literal parsing escape code", token.offset + charPos - 1); + throw error(UNEXPECTED_EOS_ERROR_MSG, token.offset + charPos - 1); } } try { diff --git a/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java b/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java index 81dc61c4dab..a7cd47ba1c7 100644 --- a/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java +++ b/warehouse/core/src/main/java/datawave/data/hash/HashUIDBuilder.java @@ -62,7 +62,7 @@ protected static HashUID newId(final HashUID template, final String... extras) { if (null != template) { // Get the existing and new extras, if any final String extra1 = template.getExtra(); - final String extra2 = HashUID.mergeExtras(extras); + final String extra2 = UID.mergeExtras(extras); // Create a new UID based on existing and new extras if ((null != extra1) && (null != extra2)) { diff --git a/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java b/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java index 5d823f9caba..a523ee07521 100644 --- a/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java +++ b/warehouse/core/src/main/java/datawave/data/hash/SnowflakeUIDBuilder.java @@ -215,7 +215,7 @@ protected static SnowflakeUID newId(final SnowflakeUID template, final String... if (null != template) { // Get the existing and new extras, if any final String extra1 = template.getExtra(); - final String extra2 = SnowflakeUID.mergeExtras(extras); + final String extra2 = UID.mergeExtras(extras); // Create a new UID based on existing and new extras if ((null != extra1) && (null != extra2)) { diff --git a/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java b/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java index c354aa8c834..b9503f9a664 100644 --- a/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java +++ b/warehouse/core/src/main/java/datawave/edge/protobuf/EdgeData.java @@ -308,7 +308,7 @@ public EdgeValue getDefaultInstanceForType() { return defaultInstance; } - private final com.google.protobuf.UnknownFieldSet unknownFields; + private transient final com.google.protobuf.UnknownFieldSet unknownFields; @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -328,12 +328,6 @@ private EdgeValue(com.google.protobuf.CodedInputStream input, com.google.protobu case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; count_ = input.readInt64(); @@ -428,6 +422,12 @@ private EdgeValue(com.google.protobuf.CodedInputStream input, com.google.protobu uuidString_ = bs; break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -526,7 +526,7 @@ public UUID getDefaultInstanceForType() { return defaultInstance; } - private final com.google.protobuf.UnknownFieldSet unknownFields; + private final transient com.google.protobuf.UnknownFieldSet unknownFields; @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -546,12 +546,6 @@ private UUID(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; leastSignificantBits_ = input.readSInt64(); @@ -562,6 +556,12 @@ private UUID(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext mostSignificantBits_ = input.readSInt64(); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -1213,7 +1213,7 @@ public com.google.protobuf.ByteString getSourceValueBytes() { } public static final int SINKVALUE_FIELD_NUMBER = 6; - private java.lang.Object sinkValue_; + private transient java.lang.Object sinkValue_; /** * optional string sinkValue = 6; @@ -1260,7 +1260,7 @@ public com.google.protobuf.ByteString getSinkValueBytes() { } public static final int LOADDATE_FIELD_NUMBER = 7; - private java.lang.Object loadDate_; + private transient java.lang.Object loadDate_; /** * optional string loadDate = 7; @@ -1370,7 +1370,7 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder( } public static final int UUID_STRING_FIELD_NUMBER = 10; - private java.lang.Object uuidString_; + private transient java.lang.Object uuidString_; /** * optional string uuid_string = 10; diff --git a/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java b/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java index 1d9e5de6b58..79fd8cfbeac 100644 --- a/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java +++ b/warehouse/core/src/main/java/datawave/edge/util/EdgeKey.java @@ -44,6 +44,8 @@ public class EdgeKey { private static final Logger log = Logger.getLogger(EdgeKey.class); + private static final String CANT_ENCODE_UNKNOWN_ERROR_MSG = "Can't encode unknown edge key format."; + // use the builder, not this nightmare constructor private EdgeKey(EDGE_FORMAT format, STATS_TYPE statsType, String sourceData, String sinkData, String family, String sourceRelationship, String sinkRelationship, String sourceAttribute1, String sinkAttribute1, String yyyymmdd, String attribute3, String attribute2, Text colvis, @@ -926,7 +928,7 @@ public Key encode() { return encode(EDGE_VERSION.DATE_PROTOBUF); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(CANT_ENCODE_UNKNOWN_ERROR_MSG + this); } } @@ -939,7 +941,7 @@ public Key encodeLegacyProtobufKey() { return encode(EDGE_VERSION.PROTOBUF); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(CANT_ENCODE_UNKNOWN_ERROR_MSG + this); } } @@ -950,7 +952,7 @@ public Key encodeLegacyAttribute2Key() { return encode(EDGE_VERSION.BASE_ATTRIBUTE2); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(CANT_ENCODE_UNKNOWN_ERROR_MSG + this); } } @@ -961,7 +963,7 @@ public Key encodeLegacyKey() { return encode(EDGE_VERSION.BASE); } else { // EDGE_FORMAT.UNKNOWN - throw new IllegalStateException("Can't encode unknown edge key format." + this); + throw new IllegalStateException(CANT_ENCODE_UNKNOWN_ERROR_MSG + this); } } diff --git a/warehouse/core/src/main/java/datawave/edge/util/EdgeKeyUtil.java b/warehouse/core/src/main/java/datawave/edge/util/EdgeKeyUtil.java index 637d9499aec..3b343e09919 100644 --- a/warehouse/core/src/main/java/datawave/edge/util/EdgeKeyUtil.java +++ b/warehouse/core/src/main/java/datawave/edge/util/EdgeKeyUtil.java @@ -21,7 +21,7 @@ * Utility class for generating regular expressions to scan various formats of the edge table. */ public class EdgeKeyUtil { - protected static final String edgeTypePrefix = "(?:^|STATS/[^/]+/)"; + protected static final String EDGE_TYPE_PREFIX = "(?:^|STATS/[^/]+/)"; public static final String MAX_UNICODE_STRING = new String(Character.toChars(Character.MAX_CODE_POINT)); public static Set normalizeSource(String source, List> dataTypes, boolean protobuffEdgeFormat) { @@ -183,7 +183,7 @@ public static Map dissasembleKey(Key key, boolean protobuffEdge public static String getEdgeColumnFamilyRegex(String edgeType, String edgeRelationship, String edgeAttribute1) { StringBuilder cfsb = new StringBuilder(); if (edgeType != null) { - cfsb.append(edgeTypePrefix).append(edgeType).append("/"); + cfsb.append(EDGE_TYPE_PREFIX).append(edgeType).append("/"); if (edgeRelationship != null && edgeAttribute1 == null) { cfsb.append(edgeRelationship); } else if (edgeRelationship != null && edgeAttribute1 != null) { @@ -194,11 +194,11 @@ public static String getEdgeColumnFamilyRegex(String edgeType, String edgeRelati cfsb.append(".*"); } } else if (edgeRelationship != null && edgeAttribute1 == null) { - cfsb.append(edgeTypePrefix + "[^/]+/").append(edgeRelationship).append(".*"); + cfsb.append(EDGE_TYPE_PREFIX + "[^/]+/").append(edgeRelationship).append(".*"); } else if (edgeAttribute1 != null && edgeRelationship == null) { - cfsb.append(edgeTypePrefix + "[^/]+/[^/]+/").append(edgeAttribute1).append(".*"); + cfsb.append(EDGE_TYPE_PREFIX + "[^/]+/[^/]+/").append(edgeAttribute1).append(".*"); } else if (edgeAttribute1 != null && edgeRelationship != null) { - cfsb.append(edgeTypePrefix + "[^/]+/").append(edgeRelationship).append("/").append(edgeAttribute1).append(".*"); + cfsb.append(EDGE_TYPE_PREFIX + "[^/]+/").append(edgeRelationship).append("/").append(edgeAttribute1).append(".*"); } return cfsb.toString(); diff --git a/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java b/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java index 42d6dc28912..3df9a7d5230 100644 --- a/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java +++ b/warehouse/core/src/main/java/datawave/ingest/mapreduce/job/RFileRecordReader.java @@ -36,7 +36,7 @@ public void initialize(InputSplit split, TaskAttemptContext context) throws IOEx end = fileSplit.getLength() - start; pos = start; - FileOperations ops = RFileOperations.getInstance(); + FileOperations ops = FileOperations.getInstance(); String file = fileSplit.getPath().toString(); FileSystem fs = fileSplit.getPath().getFileSystem(context.getConfiguration()); CryptoService cs = CryptoFactoryLoader.getServiceForClient(CryptoEnvironment.Scope.TABLE, diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java index 5313274d3e1..c2438422b19 100644 --- a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java +++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java @@ -125,7 +125,7 @@ public Info getDefaultInstanceForType() { return defaultInstance; } - private final com.google.protobuf.UnknownFieldSet unknownFields; + private final transient com.google.protobuf.UnknownFieldSet unknownFields; @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -145,12 +145,6 @@ private Info(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 24: { if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { termOffset_ = new java.util.ArrayList<>(); @@ -219,6 +213,12 @@ private Info(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext zeroOffsetMatch_ = input.readBool(); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java index d1c3e42f2be..a02d23a00bb 100644 --- a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java +++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java @@ -135,7 +135,7 @@ public List getDefaultInstanceForType() { return defaultInstance; } - private final com.google.protobuf.UnknownFieldSet unknownFields; + private final transient com.google.protobuf.UnknownFieldSet unknownFields; @java.lang.Override public final com.google.protobuf.UnknownFieldSet getUnknownFields() { @@ -155,12 +155,6 @@ private List(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext case 0: done = true; break; - default: { - if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { - done = true; - } - break; - } case 8: { bitField0_ |= 0x00000001; iGNORE_ = input.readBool(); @@ -187,6 +181,12 @@ private List(com.google.protobuf.CodedInputStream input, com.google.protobuf.Ext rEMOVEDUID_.add(input.readBytes()); break; } + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -273,7 +273,7 @@ public long getCOUNT() { // repeated string UID = 3; public static final int UID_FIELD_NUMBER = 3; - private com.google.protobuf.LazyStringList uID_; + private transient com.google.protobuf.LazyStringList uID_; /** * repeated string UID = 3; diff --git a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java index d51bcfdfca0..256d44750a4 100644 --- a/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java +++ b/warehouse/core/src/main/java/datawave/marking/MarkingFunctionsFactory.java @@ -21,7 +21,7 @@ public class MarkingFunctionsFactory { @Inject @SpringBean(refreshable = true) - private MarkingFunctions applicationMarkingFunctions; + private static MarkingFunctions applicationMarkingFunctions; public static final Logger log = LoggerFactory.getLogger(MarkingFunctionsFactory.class); @@ -59,7 +59,7 @@ public void init(@SuppressWarnings("UnusedParameters") @Observes @Initialized(Ap } @PostConstruct - public void postContruct() { + public static void postContruct() { markingFunctions = applicationMarkingFunctions; } } diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java index 4370cd6099d..f65a7040fdf 100644 --- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java +++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java @@ -110,6 +110,8 @@ public class BulkInputFormat extends InputFormat { protected static final String RANGESPLITSTRATEGY = PREFIX + ".split.strategy.class"; protected static final String MOCK = ".useInMemoryInstance"; + protected static final String UTF8 = "UTF-8"; + protected static final String RANGES = PREFIX + ".ranges"; protected static final String AUTO_ADJUST_RANGES = PREFIX + ".ranges.autoAdjust"; @@ -1289,8 +1291,8 @@ public AccumuloIteratorOption(String iteratorOption) { StringTokenizer tokenizer = new StringTokenizer(iteratorOption, FIELD_SEP); this.iteratorName = tokenizer.nextToken(); try { - this.key = URLDecoder.decode(tokenizer.nextToken(), "UTF-8"); - this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF-8"); + this.key = URLDecoder.decode(tokenizer.nextToken(), UTF8); + this.value = URLDecoder.decode(tokenizer.nextToken(), "UTF8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } @@ -1311,7 +1313,7 @@ public String getValue() { @Override public String toString() { try { - return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF-8") + FIELD_SEP + URLEncoder.encode(value, "UTF-8"); + return iteratorName + FIELD_SEP + URLEncoder.encode(key, "UTF8") + FIELD_SEP + URLEncoder.encode(value, "UTF8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } diff --git a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java index 4cb164c8e5c..b4c3e97ccc5 100644 --- a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java +++ b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java @@ -28,6 +28,9 @@ public class OperationEvaluator { public static final String GREATER_THAN_EQUALS = ">="; public static final String LESS_THAN = "<"; public static final String LESS_THAN_EQUALS = "<="; + private static final String OPERATOR_MUST_NOT_BE_BLANK = "operator must not be blank"; + private static final String NOT_A_VALID_CALC_OPERATOR = " is not a valid calculation operator"; + private static final String NOT_A_VALID_COMPARISON_OPERATOR = " is not a valid comparison operator"; /** * Return the result of the calculation for the provided integers using the provided calculation operator. Supported operators: @@ -48,7 +51,7 @@ public class OperationEvaluator { * @return the calculation result */ public static int calculate(int left, int right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -62,7 +65,7 @@ public static int calculate(int left, int right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -85,7 +88,7 @@ public static int calculate(int left, int right, String operator) { * @return the calculation result */ public static long calculate(long left, long right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -99,7 +102,7 @@ public static long calculate(long left, long right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -122,7 +125,7 @@ public static long calculate(long left, long right, String operator) { * @return the calculation result */ public static float calculate(float left, float right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -136,7 +139,7 @@ public static float calculate(float left, float right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -159,7 +162,7 @@ public static float calculate(float left, float right, String operator) { * @return the calculation result */ public static double calculate(double left, double right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case ADD: @@ -173,7 +176,7 @@ public static double calculate(double left, double right, String operator) { case MODULO: return left % right; default: - throw new IllegalArgumentException(operator + " is not a valid calculation operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_CALC_OPERATOR); } } @@ -196,7 +199,7 @@ public static double calculate(double left, double right, String operator) { * @return the calculation result */ public static long calculate(Date left, Date right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); return calculate(left.getTime(), right.getTime(), operator); } @@ -221,7 +224,7 @@ public static long calculate(Date left, Date right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(int left, int right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -238,7 +241,7 @@ public static boolean compare(int left, int right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -262,7 +265,7 @@ public static boolean compare(int left, int right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(long left, long right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -279,7 +282,7 @@ public static boolean compare(long left, long right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -303,7 +306,7 @@ public static boolean compare(long left, long right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(float left, float right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -320,7 +323,7 @@ public static boolean compare(float left, float right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -344,7 +347,7 @@ public static boolean compare(float left, float right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(double left, double right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (operator) { case EQUALS: @@ -361,7 +364,7 @@ public static boolean compare(double left, double right, String operator) { case GREATER_THAN_EQUALS: return left >= right; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -385,7 +388,7 @@ public static boolean compare(double left, double right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(Date left, Date right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); return compare(left.getTime(), right.getTime(), operator); } @@ -412,7 +415,7 @@ public static boolean compare(Date left, Date right, String operator) { * @return true if the logical expression evaluates to true, or false otherwise */ public static > boolean compare(T left, T right, String operator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); operator = CharMatcher.whitespace().removeFrom(operator); switch (CharMatcher.whitespace().removeFrom(operator)) { case EQUALS: @@ -429,7 +432,7 @@ public static > boolean compare(T left, T right, String case GREATER_THAN_EQUALS: return left.compareTo(right) >= 0; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } @@ -457,7 +460,7 @@ public static > boolean compare(T left, T right, String * @return true if the logical expression evaluates to true, or false otherwise */ public static boolean compare(T left, T right, String operator, Comparator comparator) { - Preconditions.checkArgument(!StringUtils.isBlank(operator), "operator must not be blank"); + Preconditions.checkArgument(!StringUtils.isBlank(operator), OPERATOR_MUST_NOT_BE_BLANK); Preconditions.checkNotNull(comparator, "comparator must not be null"); operator = CharMatcher.whitespace().removeFrom(operator); switch (CharMatcher.whitespace().removeFrom(operator)) { @@ -475,7 +478,7 @@ public static boolean compare(T left, T right, String operator, Comparator= 0; default: - throw new IllegalArgumentException(operator + " is not a valid comparison operator"); + throw new IllegalArgumentException(operator + NOT_A_VALID_COMPARISON_OPERATOR); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java b/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java index f51ab4b7a9a..2444ebd3c74 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/OptionsParser.java @@ -14,46 +14,46 @@ public class OptionsParser { // This is a first step towards standardizing these flags. // The intent is to expand upon this class later, but limiting to this set for now to avoid scope creep. Must.... resist... - public static final String instanceFlag = "-instance"; - public static final String instanceFlag2 = "-i"; - public static final String zookeepersFlag = "-zookeepers"; - public static final String zookeepersFlag2 = "-zk"; - public static final String userFlag = "-user"; - public static final String userFlag2 = "-u"; - public static final String passwordFlag = "-pass"; - public static final String passwordFlag2 = "-p"; - public static final String accCacheDirFlag = "-accCacheDir"; - public static final String configDirFlag = "-cd"; - public static final String additionalResourceFlag = "-"; - public static final String configDirSuffix = "config.xml"; + public static final String INSTANCE_FLAG = "-instance"; + public static final String INSTANCE_FLAG_2 = "-i"; + public static final String ZOOKEEPERS_FLAG = "-zookeepers"; + public static final String ZOOKEEPERS_FLAG_2 = "-zk"; + public static final String USER_FLAG = "-user"; + public static final String USER_FLAG_2 = "-u"; + public static final String PASSWORD_FLAG = "-pass"; + public static final String PASSWORD_FLAG_2 = "-p"; + public static final String ACC_CACHE_DIR_FLAG = "-accCacheDir"; + public static final String CONFIG_DIR_FLAG = "-cd"; + public static final String ADDITIONAL_RESOURCE_FLAG = "-"; + public static final String CONFIG_DIR_SUFFIX = "config.xml"; // todo: ingest options and more public static Configuration parseArguments(String[] args, Configuration conf) { String configDir = null; for (int i = 0; i < args.length; i++) { - if (args[i].equals(instanceFlag) || args[i].equals(instanceFlag2)) { + if (args[i].equals(INSTANCE_FLAG) || args[i].equals(INSTANCE_FLAG_2)) { AccumuloHelper.setInstanceName(conf, args[++i]); - } else if (args[i].equals(zookeepersFlag) || args[i].equals(zookeepersFlag2)) { + } else if (args[i].equals(ZOOKEEPERS_FLAG) || args[i].equals(ZOOKEEPERS_FLAG_2)) { AccumuloHelper.setZooKeepers(conf, args[++i]); - } else if (args[i].equals(userFlag) || args[i].equals(userFlag2)) { + } else if (args[i].equals(USER_FLAG) || args[i].equals(USER_FLAG_2)) { AccumuloHelper.setUsername(conf, args[++i]); - } else if (args[i].equals(passwordFlag) || args[i].equals(passwordFlag2)) { + } else if (args[i].equals(PASSWORD_FLAG) || args[i].equals(PASSWORD_FLAG_2)) { AccumuloHelper.setPassword(conf, PasswordConverter.parseArg(args[++i]).getBytes()); - } else if (args[i].equals(accCacheDirFlag)) { + } else if (args[i].equals(ACC_CACHE_DIR_FLAG)) { conf.set(TableConfigCache.ACCUMULO_CONFIG_CACHE_PATH_PROPERTY, args[++i]); - } else if (args[i].equals(configDirFlag)) { + } else if (args[i].equals(CONFIG_DIR_FLAG)) { configDir = args[++i]; if (null != configDir) { - ConfigurationFileHelper.setConfigurationFromFiles(conf, configDir, configDirSuffix); + ConfigurationFileHelper.setConfigurationFromFiles(conf, configDir, CONFIG_DIR_SUFFIX); } - } else if (args[i].startsWith(additionalResourceFlag)) { - String configName = args[i].substring(args[i].indexOf(additionalResourceFlag) + 1); + } else if (args[i].startsWith(ADDITIONAL_RESOURCE_FLAG)) { + String configName = args[i].substring(args[i].indexOf(ADDITIONAL_RESOURCE_FLAG) + 1); String configValue = args[++i]; conf.set(configName, configValue); } - else if (!args[i].startsWith(additionalResourceFlag)) { + else if (!args[i].startsWith(ADDITIONAL_RESOURCE_FLAG)) { conf.addResource(args[i]); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java index 6d09758cc4e..99254977691 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/XMLFieldConfigHelper.java @@ -31,7 +31,7 @@ public final class XMLFieldConfigHelper implements FieldConfigHelper { private static final Logger log = Logger.getLogger(XMLFieldConfigHelper.class); /** be explicit and use Apache Xerces-J here instead of relying on java to plug in the proper parser */ - private static final SAXParserFactory parserFactory = SAXParserFactoryImpl.newInstance(); + private static final SAXParserFactory parserFactory = SAXParserFactory.newInstance(); private boolean noMatchStored = true; private boolean noMatchIndexed = false; @@ -43,6 +43,8 @@ public final class XMLFieldConfigHelper implements FieldConfigHelper { private final Map knownFields = new HashMap<>(); private TreeMap patterns = new TreeMap<>(new BaseIngestHelper.MatcherComparator()); + private static final String UNEXPECTED_ATTRIBUTE = "Unexpected attribute encountered in: "; + public static class FieldInfo { boolean stored; boolean indexed; @@ -339,7 +341,7 @@ void startDefault(String uri, String localName, String qName, Attributes attribu this.defaultFieldType = lv; seenAttr.remove(INDEX_TYPE); } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'default' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'default' tag: '" + qn + "'"); } } @@ -381,7 +383,7 @@ void startNoMatch(String uri, String localName, String qName, Attributes attribu } seenAttr.remove(INDEX_TYPE); } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'nomatch' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'nomatch' tag: '" + qn + "'"); } } @@ -426,7 +428,7 @@ void startField(String uri, String localName, String qName, Attributes attribute } else if (INDEX_TYPE.equals(qn)) { fieldType = lv; } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'field' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'field' tag: '" + qn + "'"); } } @@ -481,7 +483,7 @@ void startFieldPattern(String uri, String localName, String qName, Attributes at } else if (INDEX_TYPE.equals(qn)) { fieldType = lv; } else { - throw new IllegalArgumentException("Unexpected attribute encounteded in: " + uri + " in 'field' tag: '" + qn + "'"); + throw new IllegalArgumentException(UNEXPECTED_ATTRIBUTE + uri + " in 'field' tag: '" + qn + "'"); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java index 8784be6ecd9..56d8285e6a4 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java @@ -138,6 +138,10 @@ public abstract class BaseIngestHelper extends AbstractIngestHelper implements C public static final String FIELD_CONFIG_FILE = ".data.category.field.config.file"; + private static final String PROPERTY_MALFORMED = " property malformed: "; + private static final String ADDED_NORMALIZED_FIELD = "added normalized field "; + private static final String TO_VALUES = " to values "; + private static final Logger log = ThreadConfigurableLogger.getLogger(BaseIngestHelper.class); private Multimap> typeFieldMap = null; @@ -520,8 +524,8 @@ public static String getFieldName(Type dataType, String property, String propert // if this type already has a '.', then we have a malformed property // name if (dataType.typeName().indexOf('.') >= 0) { - log.error(propertyPattern + " property malformed: " + property); - throw new IllegalArgumentException(propertyPattern + " property malformed: " + property); + log.error(propertyPattern + PROPERTY_MALFORMED + property); + throw new IllegalArgumentException(propertyPattern + PROPERTY_MALFORMED + property); } String fieldName = property.substring(dataType.typeName().length() + 1, property.length() - propertyPattern.length()); @@ -543,8 +547,8 @@ public static String getFieldType(Type dataType, String property, String propert // if this type already has a '.', then we have a malformed property // name if (dataType.typeName().indexOf('.') >= 0) { - log.error(propertyPattern + " property malformed: " + property); - throw new IllegalArgumentException(propertyPattern + " property malformed: " + property); + log.error(propertyPattern + PROPERTY_MALFORMED + property); + throw new IllegalArgumentException(propertyPattern + PROPERTY_MALFORMED + property); } String fieldName = property.substring(dataType.typeName().length() + 1, property.length() - propertyPattern.length()); @@ -746,7 +750,7 @@ public HashSet normalizeFieldValue(String fieldName, } values.add(value); if (log.isDebugEnabled()) { - log.debug("added normalized field " + value + " to values set."); + log.debug(ADDED_NORMALIZED_FIELD + value + " to values set."); } } return values; @@ -796,7 +800,7 @@ protected Set normalize(NormalizedContentInterface n values.add(normalize(normalizedContent, dataType)); } if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); + log.debug(ADDED_NORMALIZED_FIELD + normalizedContent + TO_VALUES + values); } } return values; @@ -813,7 +817,7 @@ protected Set normalize(NormalizedContentInterface n for (datawave.data.type.Type dataType : dataTypes) { values.add(normalizeFieldValue(normalizedContent, dataType)); if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); + log.debug(ADDED_NORMALIZED_FIELD + normalizedContent + TO_VALUES + values); } } return values; @@ -827,7 +831,7 @@ protected Set normalize(NormalizedContentInterface n for (datawave.data.type.Type dataType : dataTypes) { values.add(normalize(normalizedContent, dataType)); if (log.isDebugEnabled()) { - log.debug("added normalized field " + normalizedContent + " to values " + values); + log.debug(ADDED_NORMALIZED_FIELD + normalizedContent + TO_VALUES + values); } } return values; diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java index 925a1bea6a1..b24bb798f9f 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/VirtualIngest.java @@ -118,15 +118,15 @@ class VirtualFieldNormalizer extends NoOpType { public static final String VIRTUAL_FIELD_IGNORE_NORMALIZATION_ON_FIELD = ".data.combine.ignore.normalization.on.fields"; protected Map virtualFieldDefinitions = new HashMap<>(); - protected Map compiledFieldPatterns = null; + protected transient Map compiledFieldPatterns = null; protected String defaultSeparator = null; protected String defaultStartSeparator = null; protected String defaultEndSeparator = null; - protected Map grouping = new HashMap<>(); - protected Map allowMissing = new HashMap<>(); + protected transient Map grouping = new HashMap<>(); + protected transient Map allowMissing = new HashMap<>(); private Set ignoreNormalizationForFields = new HashSet<>(); - private MarkingFunctions markingFunctions; + private transient MarkingFunctions markingFunctions; public void setup(Type type, String instance, Configuration config) { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java index 186055207d7..6dd57fff501 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/normalizer/AbstractNormalizer.java @@ -17,6 +17,7 @@ import datawave.ingest.data.config.NormalizedFieldAndValue; public abstract class AbstractNormalizer implements TextNormalizer { + private static final String FAILED_TO_NORMALIZE = "Failed to normalize "; private static final Logger log = Logger.getLogger(AbstractNormalizer.class); @Override @@ -223,7 +224,7 @@ public NormalizedContentInterface normalize(NormalizedContentInterface field) { if (field.getEventFieldName().equals("IP_GEO_FM_COORDINATES") && field.getEventFieldValue().equals("-99.999/-999.999")) { log.warn("Found know bad default value: IP_GEO_FM_COORDINATES=-99.999/-999.999"); } else { - log.error("Failed to normalize " + field.getEventFieldName() + '=' + field.getEventFieldValue(), e); + log.error(FAILED_TO_NORMALIZE + field.getEventFieldName() + '=' + field.getEventFieldValue(), e); } n.setError(e); } @@ -240,7 +241,7 @@ public Multimap normalize(Multimap normalizeMap(Multimap(), false, false); - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); @@ -1671,8 +1674,11 @@ public void testCleanUpJobDirectoryUnableToMakeDirectory() throws Exception { BulkIngestMapFileLoaderTest.WrappedLocalFileSystem fs = new BulkIngestMapFileLoaderTest.WrappedLocalFileSystem( createMockInputStream(new String[] {"/dummy/entry"}), null, false, false, false, false, null, false, false); - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); @@ -1715,9 +1721,11 @@ public void testCleanUpJobDirectoryJobSuccess() throws Exception { BulkIngestMapFileLoaderTest.WrappedLocalFileSystem fs = new BulkIngestMapFileLoaderTest.WrappedLocalFileSystem(createMockInputStream(), new FileStatus[0], false, false, false, false, null, false, false); - - Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); - + try { + Whitebox.invokeMethod(FileSystem.class, "addFileSystemForTesting", BulkIngestMapFileLoaderTest.FILE_SYSTEM_URI, conf, fs); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } Path mapFilesDir = createNewPath(url); uut.cleanUpJobDirectory(mapFilesDir); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 62eba5f42c6..3e77b19d16b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -635,6 +635,9 @@ public boolean isLongRunningQuery() { /** * If the configuration didn't exist, OR IT CHANGED, we need to create or update the transformers that have been added. + * + * @throws QueryException + * when there is a problem creating a query */ private void addConfigBasedTransformers() throws QueryException { if (getConfig() != null) { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java index 7b7c03aa8dd..4b492ceb69e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/ScoredSSDeepPair.java @@ -48,7 +48,11 @@ public Set getOverlappingNgrams() { return overlappingNgrams; } - /** Return a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep */ + /** + * Returns a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep + * + * @return a String representing the sorted list of ngrams that are shared by both the query and the matching ssdeep + **/ public String getOverlapsAsString() { return String.join(", ", getOverlappingNgrams().stream().map(NGramTuple::toString).collect(Collectors.toCollection(TreeSet::new))); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java index d68ae473dbc..b7199c195b8 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/UniqueTransform.java @@ -444,8 +444,11 @@ public int compare(Map.Entry o1, Map.Entry o2) * Build a list of potential hdfs directories based on each ivarator cache dir configs. * * @param ivaratorCacheDirConfigs + * list of cache dir configs * @param hdfsSiteConfigURLs + * hdfs site configuration urls * @param subdirectory + * the subdirectory * @return A path * @throws IOException * for issues with read/write diff --git a/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSet.java b/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSet.java index 68b09eece2d..74f89b89583 100644 --- a/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSet.java +++ b/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSet.java @@ -16,7 +16,9 @@ interface RewriteStrategy { * Determine if the object should be rewritten * * @param original + * the original * @param update + * the update * @return true of the original should be replaced with the update */ boolean rewrite(E original, E update); diff --git a/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSetImpl.java b/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSetImpl.java index 4fc42e16eba..77201148b3b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSetImpl.java +++ b/warehouse/query-core/src/main/java/datawave/query/util/sortedset/RewritableSortedSetImpl.java @@ -34,7 +34,9 @@ public class RewritableSortedSetImpl implements RewritableSortedSet, Clone * Create the rewritable sorted set * * @param comparator + * the comparator * @param rewriteStrategy + * the rewrite strategy */ public RewritableSortedSetImpl(Comparator comparator, RewriteStrategy rewriteStrategy) { this.set = new TreeMap<>(comparator); @@ -45,6 +47,7 @@ public RewritableSortedSetImpl(Comparator comparator, RewriteStrategy rewrite * Create the rewritable sorted set * * @param rewriteStrategy + * the rewrite strategy */ public RewritableSortedSetImpl(RewriteStrategy rewriteStrategy) { this(null, rewriteStrategy); @@ -54,6 +57,7 @@ public RewritableSortedSetImpl(RewriteStrategy rewriteStrategy) { * Create the rewritable sorted set * * @param comparator + * the comparator */ public RewritableSortedSetImpl(Comparator comparator) { this(comparator, null); diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java index eb86e9629c3..f1efe3c7215 100644 --- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java +++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java @@ -4,6 +4,7 @@ * An interface for things that take two hashes and compare them to produce some sort of result. * * @param + * Hash type to score */ public interface SSDeepHashScorer { public T apply(SSDeepHash signature1, SSDeepHash signature2); diff --git a/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java b/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java index c5c0d21f288..791d462384f 100644 --- a/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java +++ b/web-services/map-reduce/src/main/java/datawave/webservice/mr/MapReduceBean.java @@ -265,6 +265,7 @@ public GenericResponse ooziesubmit(MultivaluedMap queryPa OozieClient oozieClient = null; Properties oozieConf = null; + BadRequestException exception = null; try { oozieClient = new OozieClient((String) job.getJobConfigurationProperties().get(OozieJobConstants.OOZIE_CLIENT_PROP)); oozieConf = oozieClient.createConfiguration(); @@ -302,14 +303,17 @@ public GenericResponse ooziesubmit(MultivaluedMap queryPa log.error("Error validating audit parameters", e); BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); response.addException(qe); - throw new BadRequestException(qe, response); + exception = new BadRequestException(qe, response); } catch (Exception e) { log.error("Error auditing query", e); response.addMessage("Error auditing query - " + e.getMessage()); - throw new BadRequestException(e, response); + exception = new BadRequestException(e, response); } } } + if (null != exception) { + throw exception; + } // Submit the Oozie workflow. try { String jobID = null; diff --git a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java index c597d0e79fd..9d635db0e07 100644 --- a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java +++ b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java @@ -420,6 +420,7 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP AccumuloClient client = null; BatchWriter writer = null; String tableName = this.checkModelTableName(modelTableName); + DatawaveWebApplicationException exception = null; try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); @@ -442,7 +443,7 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); log.error(qe); response.addException(qe); - throw new DatawaveWebApplicationException(qe, response); + exception = new DatawaveWebApplicationException(qe, response); } } if (null != client) { @@ -453,6 +454,9 @@ public VoidResponse insertMapping(datawave.webservice.model.Model model, @QueryP } } } + if (null != exception) { + throw exception; + } cache.reloadCache(tableName); return response; } @@ -493,6 +497,7 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String AccumuloClient client = null; BatchWriter writer = null; String tableName = this.checkModelTableName(modelTableName); + DatawaveWebApplicationException exception = null; try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); @@ -515,7 +520,7 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String QueryException qe = new QueryException(DatawaveErrorCode.WRITER_CLOSE_ERROR, e1); log.error(qe); response.addException(qe); - throw new DatawaveWebApplicationException(qe, response); + exception = new DatawaveWebApplicationException(qe, response); } } if (null != client) { @@ -526,6 +531,10 @@ private VoidResponse deleteMapping(datawave.webservice.model.Model model, String } } } + if (null != exception) { + throw exception; + } + if (reloadCache) cache.reloadCache(tableName); return response; diff --git a/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java b/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java index a6b2e353366..cd2a27b9c74 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java +++ b/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java @@ -452,6 +452,7 @@ default void preInitialize(Query settings, Set userAuthorization * Set a client configuration for scanner hints and consistency. * * @param config + * accumuloConfiguration */ void setClientConfig(AccumuloClientConfiguration config); diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java index 0e35675483e..77c1d7605a3 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java +++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java @@ -856,6 +856,7 @@ public GenericResponse planQuery(@Required("logicName") @PathParam("logi } AuditType auditType = qd.logic.getAuditType(null); + Exception exception = null; try { MultivaluedMap optionalQueryParameters = new MultivaluedMapImpl<>(); optionalQueryParameters.putAll(qp.getUnknownParameters(queryParameters)); @@ -885,16 +886,20 @@ public GenericResponse planQuery(@Required("logicName") @PathParam("logi log.error("Error validating audit parameters", e); BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); response.addException(qe); - throw new BadRequestException(qe, response); + exception = new BadRequestException(qe, response); } catch (Exception e) { log.error("Error auditing query", e); QueryException qe = new QueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e); response.addException(qe); - throw qe; + exception = qe; } } } + if (null != exception) { + throw exception; + } + priority = qd.logic.getConnectionPriority(); Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); q.populateTrackingMap(trackingMap);