Skip to content

Commit

Permalink
Merge pull request #27 from NationalSecurityAgency/feature/addDocumen…
Browse files Browse the repository at this point in the history 
…tSizeMetric

Adding support for document size in metrics
  • Loading branch information
@mineralntl
mineralntl authored Oct 9, 2024
2 parents 8281bfc + 73d4132 commit 7bdd001
Show file tree
Hide file tree
Showing 12 changed files with 84 additions and 34 deletions.
10 changes: 10 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,8 @@ public int getFieldNumber(String name) {
@XmlJavaTypeAdapter(StringMapAdapter.class)
protected Map<String,String> versionMap = new TreeMap<>();
@XmlElement
protected long docSize = 0;
@XmlElement
protected long docRanges = 0;
@XmlElement
protected long fiRanges = 0;
Expand Down Expand Up @@ -899,6 +901,14 @@ public void setYieldCount(long yieldCount) {
this.yieldCount = yieldCount;
}

public long getDocSize() {
return docSize;
}

public void setDocSize(long docSize) {
this.docSize = docSize;
}

public long getDocRanges() {
return docRanges;
}
Expand Down
25 changes: 18 additions & 7 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetric.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public QueryMetric(QueryMetric other) {
this.seekCount = other.seekCount;
this.yieldCount = other.yieldCount;
this.versionMap = other.versionMap;
this.docSize = other.docSize;
this.docRanges = other.docRanges;
this.fiRanges = other.fiRanges;
this.plan = other.plan;
Expand Down Expand Up @@ -140,8 +141,9 @@ public int hashCode() {
.append(this.getHost()).append(this.getPageTimes()).append(this.getProxyServers()).append(this.getLifecycle())
.append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName())
.append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount())
.append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime())
.append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
.append(this.getYieldCount()).append(this.getDocSize()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan())
.append(this.getLoginTime()).append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates())
.append(this.getVersionMap()).toHashCode();
}

@Override
Expand All @@ -166,11 +168,11 @@ public boolean equals(Object o) {
.append(this.getLifecycle(), other.getLifecycle()).append(this.getErrorMessage(), other.getErrorMessage())
.append(this.getErrorCode(), other.getErrorCode()).append(this.getSourceCount(), other.getSourceCount())
.append(this.getNextCount(), other.getNextCount()).append(this.getSeekCount(), other.getSeekCount())
.append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges())
.append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan())
.append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions())
.append(this.getMarkings(), other.getMarkings()).append(this.getNumUpdates(), other.getNumUpdates())
.append(this.getVersionMap(), other.getVersionMap()).isEquals();
.append(this.getYieldCount(), other.getYieldCount()).append(this.getDocSize(), other.getDocSize())
.append(this.getDocRanges(), other.getDocRanges()).append(this.getFiRanges(), other.getFiRanges())
.append(this.getPlan(), other.getPlan()).append(this.getLoginTime(), other.getLoginTime())
.append(this.getPredictions(), other.getPredictions()).append(this.getMarkings(), other.getMarkings())
.append(this.getNumUpdates(), other.getNumUpdates()).append(this.getVersionMap(), other.getVersionMap()).isEquals();
} else {
return false;
}
Expand Down Expand Up @@ -208,6 +210,7 @@ public String toString() {
buf.append(" NextCount: ").append(this.getNextCount());
buf.append(" Seek Count: ").append(this.getSeekCount());
buf.append(" Yield Count: ").append(this.getYieldCount());
buf.append(" Doc Size: ").append(this.getDocSize());
buf.append(" Doc Ranges: ").append(this.getDocRanges());
buf.append(" FI Ranges: ").append(this.getFiRanges());
buf.append(" Login Time: ").append(this.getLoginTime());
Expand Down Expand Up @@ -403,6 +406,8 @@ public void writeTo(Output output, QueryMetric message) throws IOException {
output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true);
}
}

output.writeInt64(39, message.docSize, false);
}

public void mergeFrom(Input input, QueryMetric message) throws IOException {
Expand Down Expand Up @@ -551,6 +556,9 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException {
message.versionMap.put(split[0], split[1]);
}
break;
case 39:
message.docSize = input.readInt64();
break;
default:
input.handleUnknownField(number, this);
break;
Expand Down Expand Up @@ -637,6 +645,8 @@ public String getFieldName(int number) {
return "version";
case 38:
return "versionMap";
case 39:
return "docSize";
default:
return null;
}
Expand Down Expand Up @@ -688,6 +698,7 @@ public int getFieldNumber(String name) {
fieldMap.put("predictions", 36);
fieldMap.put("version", 37);
fieldMap.put("versionMap", 38);
fieldMap.put("docSize", 39);
}
};

Expand Down
4 changes: 4 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ public String getNumResultsStr() {
return nf.format(numResults);
}

public String getDocSizeStr() {
return nf.format(docSize);
}

public String getDocRangesStr() {
return nf.format(docRanges);
}
Expand Down
2 changes: 2 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public interface QueryMetricModelFormat {

String getNumResultsStr();

String getDocSizeStr();

String getDocRangesStr();

String getFiRangesStr();
Expand Down
1 change: 1 addition & 0 deletions api/src/main/resources/templates/querymetric-horizontal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ <h1>Query Metrics</h1>
<th>Query Setup Call Time (ms)</th>
<th>Number Pages</th>
<th>Number Results</th>
<th>Doc Size</th>
<th>Doc Ranges</th>
<th>FI Ranges</th>
<th>Sources</th>
Expand Down
2 changes: 1 addition & 1 deletion service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
<version.microservice.accumulo-api>4.0.0</version.microservice.accumulo-api>
<version.microservice.hazelcast>4.0.2</version.microservice.hazelcast>
<version.microservice.query-metric-api>4.0.7</version.microservice.query-metric-api>
<version.microservice.query-metric-api>4.0.8-SNAPSHOT</version.microservice.query-metric-api>
<version.microservice.starter>4.0.2</version.microservice.starter>
<version.microservice.starter-datawave-query-metric>3.0.3</version.microservice.starter-datawave-query-metric>
<version.microservice.starter-metadata>3.0.2</version.microservice.starter-metadata>
Expand Down
1 change: 1 addition & 0 deletions service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public Long process(Map.Entry<String,QueryMetricUpdateHolder> entry) {
storedHolder.addValue("nextCount", updatedMetric.getNextCount());
storedHolder.addValue("seekCount", updatedMetric.getSeekCount());
storedHolder.addValue("yieldCount", updatedMetric.getYieldCount());
storedHolder.addValue("docSize", updatedMetric.getDocSize());
storedHolder.addValue("docRanges", updatedMetric.getDocRanges());
storedHolder.addValue("fiRanges", updatedMetric.getFiRanges());
}
Expand Down
2 changes: 2 additions & 0 deletions .../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public class QueryMetricHandlerProperties {
"CREATE_CALL_TIME",
"CREATE_DATE",
"DOC_RANGES",
"DOC_SIZE",
"ELAPSED_TIME",
"END_DATE",
"ERROR_CODE",
Expand Down Expand Up @@ -141,6 +142,7 @@ public class QueryMetricHandlerProperties {
protected List<String> numericFields = Arrays.asList(
"CREATE_CALL_TIME",
"DOC_RANGES",
"DOC_SIZE",
"ELAPSED_TIME",
"FI_RANGES",
"LOGIN_TIME",
Expand Down
6 changes: 6 additions & 0 deletions .../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,9 @@ public Multimap<String,String> getEventFieldsToWrite(T updated, T stored) {
if (isChanged(updated.getDocRanges(), stored == null ? -1 : stored.getDocRanges())) {
fields.put("DOC_RANGES", Long.toString(updated.getDocRanges()));
}
if (isChanged(updated.getDocSize(), stored == null ? -1 : stored.getDocSize())) {
fields.put("DOC_SIZE", Long.toString(updated.getDocSize()));
}
if (isChanged(updated.getElapsedTime(), stored == null ? -1 : stored.getElapsedTime())) {
fields.put("ELAPSED_TIME", Long.toString(updated.getElapsedTime()));
}
Expand Down Expand Up @@ -383,6 +386,9 @@ public Multimap<String,String> getEventFieldsToDelete(T updated, T stored) {
if (isChanged(updated.getDocRanges(), stored.getDocRanges())) {
fields.put("DOC_RANGES", Long.toString(stored.getDocRanges()));
}
if (isChanged(updated.getDocSize(), stored.getDocSize())) {
fields.put("DOC_SIZE", Long.toString(stored.getDocSize()));
}
if (isChanged(updated.getElapsedTime(), stored.getElapsedTime())) {
fields.put("ELAPSED_TIME", Long.toString(stored.getElapsedTime()));
}
Expand Down
2 changes: 2 additions & 0 deletions service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,15 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy
combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount());
combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount());
combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount());
combinedMetric.setDocSize(combinedMetric.getDocSize() + updatedQueryMetric.getDocSize());
combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges());
combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges());
} else {
combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount());
combinedMetric.setNextCount(updatedQueryMetric.getNextCount());
combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount());
combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount());
combinedMetric.setDocSize(updatedQueryMetric.getDocSize());
combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges());
combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges());
}
Expand Down
41 changes: 25 additions & 16 deletions ...src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ public T toMetric(EventBase event) {
Date d = sdf_date_time1.parse(fieldValue);
m.setBeginDate(d);
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("CREATE_CALL_TIME")) {
m.setCreateCallTime(Long.parseLong(fieldValue));
Expand All @@ -562,7 +562,7 @@ public T toMetric(EventBase event) {
m.setCreateDate(d);
createDateSet = true;
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("DOC_RANGES")) {
try {
Expand All @@ -571,14 +571,23 @@ public T toMetric(EventBase event) {
m.setDocRanges(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("DOC_SIZE")) {
try {
long l = Long.parseLong(fieldValue);
if (l > m.getDocSize()) {
m.setDocSize(l);
}
} catch (Exception e) {
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("END_DATE")) {
try {
Date d = sdf_date_time1.parse(fieldValue);
m.setEndDate(d);
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("ERROR_CODE")) {
m.setErrorCode(fieldValue);
Expand All @@ -591,7 +600,7 @@ public T toMetric(EventBase event) {
m.setFiRanges(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("HOST")) {
m.setHost(fieldValue);
Expand All @@ -603,7 +612,7 @@ public T toMetric(EventBase event) {
m.setLastUpdated(d);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("LIFECYCLE")) {
Lifecycle l = Lifecycle.valueOf(fieldValue);
Expand All @@ -627,7 +636,7 @@ public T toMetric(EventBase event) {
m.setNextCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("NUM_UPDATES")) {
try {
Expand All @@ -636,14 +645,14 @@ public T toMetric(EventBase event) {
m.setNumUpdates(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.startsWith("PAGE_METRICS")) {
int index = fieldName.indexOf(".");
if (-1 == index) {
log.error("Could not parse field name to extract repetition count: " + fieldName);
log.error("Could not parse field name to extract repetition count: {}", fieldName);
} else {
Long pageNum = Long.parseLong(fieldName.substring(index + 1));
long pageNum = Long.parseLong(fieldName.substring(index + 1));
PageMetric pageMetric = PageMetric.parse(fieldValue);
if (pageMetric != null) {
pageMetric.setPageNumber(pageNum);
Expand Down Expand Up @@ -673,7 +682,7 @@ public T toMetric(EventBase event) {
int x = fieldValue.indexOf(":");
if (x > -1) {
String predictionName = fieldValue.substring(0, x);
Double predictionValue = Double.parseDouble(fieldValue.substring(x + 1));
double predictionValue = Double.parseDouble(fieldValue.substring(x + 1));
m.addPrediction(new Prediction(predictionName, predictionValue));
}
} catch (Exception e) {
Expand All @@ -699,7 +708,7 @@ public T toMetric(EventBase event) {
m.setSeekCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("SETUP_TIME")) {
m.setSetupTime(Long.parseLong(fieldValue));
Expand All @@ -710,7 +719,7 @@ public T toMetric(EventBase event) {
m.setSourceCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("USER")) {
m.setUser(fieldValue);
Expand All @@ -727,7 +736,7 @@ public T toMetric(EventBase event) {
m.setYieldCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
}
}
Expand Down Expand Up @@ -773,12 +782,12 @@ protected void createAndConfigureTablesIfNecessary(String[] tableNames, Accumulo
if (tableHelper != null) {
tableHelper.configure(accumuloClient.tableOperations());
} else {
log.info("No configuration supplied for table: " + table);
log.info("No configuration supplied for table: {}", table);
}
}
} catch (TableExistsException te) {
// in this case, somebody else must have created the table after our existence check
log.debug("Tried to create " + table + " but somebody beat us to the punch");
log.debug("Tried to create {} but somebody beat us to the punch", table);
}
}
}
Expand Down
Loading

0 comments on commit 7bdd001

Please sign in to comment.