Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding support for document size in metrics #27

Merged
merged 1 commit into from
Oct 9, 2024
Merged

Adding support for document size in metrics #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/BaseQueryMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,8 @@ public int getFieldNumber(String name) {
@XmlJavaTypeAdapter(StringMapAdapter.class)
protected Map<String,String> versionMap = new TreeMap<>();
@XmlElement
protected long docSize = 0;
@XmlElement
protected long docRanges = 0;
@XmlElement
protected long fiRanges = 0;
Expand Down Expand Up @@ -899,6 +901,14 @@ public void setYieldCount(long yieldCount) {
this.yieldCount = yieldCount;
}

public long getDocSize() {
return docSize;
}

public void setDocSize(long docSize) {
this.docSize = docSize;
}

public long getDocRanges() {
return docRanges;
}
Expand Down
25 changes: 18 additions & 7 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetric.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public QueryMetric(QueryMetric other) {
this.seekCount = other.seekCount;
this.yieldCount = other.yieldCount;
this.versionMap = other.versionMap;
this.docSize = other.docSize;
this.docRanges = other.docRanges;
this.fiRanges = other.fiRanges;
this.plan = other.plan;
Expand Down Expand Up @@ -140,8 +141,9 @@ public int hashCode() {
.append(this.getHost()).append(this.getPageTimes()).append(this.getProxyServers()).append(this.getLifecycle())
.append(this.getErrorMessage()).append(this.getCreateCallTime()).append(this.getErrorCode()).append(this.getQueryName())
.append(this.getParameters()).append(this.getSourceCount()).append(this.getNextCount()).append(this.getSeekCount())
.append(this.getYieldCount()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan()).append(this.getLoginTime())
.append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates()).append(this.getVersionMap()).toHashCode();
.append(this.getYieldCount()).append(this.getDocSize()).append(this.getDocRanges()).append(this.getFiRanges()).append(this.getPlan())
.append(this.getLoginTime()).append(this.getPredictions()).append(this.getMarkings()).append(this.getNumUpdates())
.append(this.getVersionMap()).toHashCode();
}

@Override
Expand All @@ -166,11 +168,11 @@ public boolean equals(Object o) {
.append(this.getLifecycle(), other.getLifecycle()).append(this.getErrorMessage(), other.getErrorMessage())
.append(this.getErrorCode(), other.getErrorCode()).append(this.getSourceCount(), other.getSourceCount())
.append(this.getNextCount(), other.getNextCount()).append(this.getSeekCount(), other.getSeekCount())
.append(this.getYieldCount(), other.getYieldCount()).append(this.getDocRanges(), other.getDocRanges())
.append(this.getFiRanges(), other.getFiRanges()).append(this.getPlan(), other.getPlan())
.append(this.getLoginTime(), other.getLoginTime()).append(this.getPredictions(), other.getPredictions())
.append(this.getMarkings(), other.getMarkings()).append(this.getNumUpdates(), other.getNumUpdates())
.append(this.getVersionMap(), other.getVersionMap()).isEquals();
.append(this.getYieldCount(), other.getYieldCount()).append(this.getDocSize(), other.getDocSize())
.append(this.getDocRanges(), other.getDocRanges()).append(this.getFiRanges(), other.getFiRanges())
.append(this.getPlan(), other.getPlan()).append(this.getLoginTime(), other.getLoginTime())
.append(this.getPredictions(), other.getPredictions()).append(this.getMarkings(), other.getMarkings())
.append(this.getNumUpdates(), other.getNumUpdates()).append(this.getVersionMap(), other.getVersionMap()).isEquals();
} else {
return false;
}
Expand Down Expand Up @@ -208,6 +210,7 @@ public String toString() {
buf.append(" NextCount: ").append(this.getNextCount());
buf.append(" Seek Count: ").append(this.getSeekCount());
buf.append(" Yield Count: ").append(this.getYieldCount());
buf.append(" Doc Size: ").append(this.getDocSize());
buf.append(" Doc Ranges: ").append(this.getDocRanges());
buf.append(" FI Ranges: ").append(this.getFiRanges());
buf.append(" Login Time: ").append(this.getLoginTime());
Expand Down Expand Up @@ -403,6 +406,8 @@ public void writeTo(Output output, QueryMetric message) throws IOException {
output.writeString(38, StringUtils.join(Arrays.asList(entry.getKey(), entry.getValue()), "\0"), true);
}
}

output.writeInt64(39, message.docSize, false);
}

public void mergeFrom(Input input, QueryMetric message) throws IOException {
Expand Down Expand Up @@ -551,6 +556,9 @@ public void mergeFrom(Input input, QueryMetric message) throws IOException {
message.versionMap.put(split[0], split[1]);
}
break;
case 39:
message.docSize = input.readInt64();
break;
default:
input.handleUnknownField(number, this);
break;
Expand Down Expand Up @@ -637,6 +645,8 @@ public String getFieldName(int number) {
return "version";
case 38:
return "versionMap";
case 39:
return "docSize";
default:
return null;
}
Expand Down Expand Up @@ -688,6 +698,7 @@ public int getFieldNumber(String name) {
fieldMap.put("predictions", 36);
fieldMap.put("version", 37);
fieldMap.put("versionMap", 38);
fieldMap.put("docSize", 39);
}
};

Expand Down
4 changes: 4 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetricModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,10 @@ public String getNumResultsStr() {
return nf.format(numResults);
}

public String getDocSizeStr() {
return nf.format(docSize);
}

public String getDocRangesStr() {
return nf.format(docRanges);
}
Expand Down
2 changes: 2 additions & 0 deletions api/src/main/java/datawave/microservice/querymetric/QueryMetricModelFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ public interface QueryMetricModelFormat {

String getNumResultsStr();

String getDocSizeStr();

String getDocRangesStr();

String getFiRangesStr();
Expand Down
1 change: 1 addition & 0 deletions api/src/main/resources/templates/querymetric-horizontal.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ <h1>Query Metrics</h1>
<th>Query Setup Call Time (ms)</th>
<th>Number Pages</th>
<th>Number Results</th>
<th>Doc Size</th>
<th>Doc Ranges</th>
<th>FI Ranges</th>
<th>Sources</th>
Expand Down
2 changes: 1 addition & 1 deletion service/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<version.in-memory-accumulo>4.0.0</version.in-memory-accumulo>
<version.microservice.accumulo-api>4.0.0</version.microservice.accumulo-api>
<version.microservice.hazelcast>4.0.2</version.microservice.hazelcast>
<version.microservice.query-metric-api>4.0.7</version.microservice.query-metric-api>
<version.microservice.query-metric-api>4.0.8-SNAPSHOT</version.microservice.query-metric-api>
<version.microservice.starter>4.0.2</version.microservice.starter>
<version.microservice.starter-datawave-query-metric>3.0.3</version.microservice.starter-datawave-query-metric>
<version.microservice.starter-metadata>3.0.2</version.microservice.starter-metadata>
Expand Down
1 change: 1 addition & 0 deletions service/src/main/java/datawave/microservice/querymetric/MetricUpdateEntryProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public Long process(Map.Entry<String,QueryMetricUpdateHolder> entry) {
storedHolder.addValue("nextCount", updatedMetric.getNextCount());
storedHolder.addValue("seekCount", updatedMetric.getSeekCount());
storedHolder.addValue("yieldCount", updatedMetric.getYieldCount());
storedHolder.addValue("docSize", updatedMetric.getDocSize());
storedHolder.addValue("docRanges", updatedMetric.getDocRanges());
storedHolder.addValue("fiRanges", updatedMetric.getFiRanges());
}
Expand Down
2 changes: 2 additions & 0 deletions .../src/main/java/datawave/microservice/querymetric/config/QueryMetricHandlerProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ public class QueryMetricHandlerProperties {
"CREATE_CALL_TIME",
"CREATE_DATE",
"DOC_RANGES",
"DOC_SIZE",
"ELAPSED_TIME",
"END_DATE",
"ERROR_CODE",
Expand Down Expand Up @@ -141,6 +142,7 @@ public class QueryMetricHandlerProperties {
protected List<String> numericFields = Arrays.asList(
"CREATE_CALL_TIME",
"DOC_RANGES",
"DOC_SIZE",
"ELAPSED_TIME",
"FI_RANGES",
"LOGIN_TIME",
Expand Down
6 changes: 6 additions & 0 deletions .../main/java/datawave/microservice/querymetric/handler/ContentQueryMetricsIngestHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,9 @@ public Multimap<String,String> getEventFieldsToWrite(T updated, T stored) {
if (isChanged(updated.getDocRanges(), stored == null ? -1 : stored.getDocRanges())) {
fields.put("DOC_RANGES", Long.toString(updated.getDocRanges()));
}
if (isChanged(updated.getDocSize(), stored == null ? -1 : stored.getDocSize())) {
fields.put("DOC_SIZE", Long.toString(updated.getDocSize()));
}
if (isChanged(updated.getElapsedTime(), stored == null ? -1 : stored.getElapsedTime())) {
fields.put("ELAPSED_TIME", Long.toString(updated.getElapsedTime()));
}
Expand Down Expand Up @@ -383,6 +386,9 @@ public Multimap<String,String> getEventFieldsToDelete(T updated, T stored) {
if (isChanged(updated.getDocRanges(), stored.getDocRanges())) {
fields.put("DOC_RANGES", Long.toString(stored.getDocRanges()));
}
if (isChanged(updated.getDocSize(), stored.getDocSize())) {
fields.put("DOC_SIZE", Long.toString(stored.getDocSize()));
}
if (isChanged(updated.getElapsedTime(), stored.getElapsedTime())) {
fields.put("ELAPSED_TIME", Long.toString(stored.getElapsedTime()));
}
Expand Down
2 changes: 2 additions & 0 deletions service/src/main/java/datawave/microservice/querymetric/handler/QueryMetricCombiner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,15 @@ public T combineMetrics(T updatedQueryMetric, T cachedQueryMetric, QueryMetricTy
combinedMetric.setNextCount(combinedMetric.getNextCount() + updatedQueryMetric.getNextCount());
combinedMetric.setSeekCount(combinedMetric.getSeekCount() + updatedQueryMetric.getSeekCount());
combinedMetric.setYieldCount(combinedMetric.getYieldCount() + updatedQueryMetric.getYieldCount());
combinedMetric.setDocSize(combinedMetric.getDocSize() + updatedQueryMetric.getDocSize());
combinedMetric.setDocRanges(combinedMetric.getDocRanges() + updatedQueryMetric.getDocRanges());
combinedMetric.setFiRanges(combinedMetric.getFiRanges() + updatedQueryMetric.getFiRanges());
} else {
combinedMetric.setSourceCount(updatedQueryMetric.getSourceCount());
combinedMetric.setNextCount(updatedQueryMetric.getNextCount());
combinedMetric.setSeekCount(updatedQueryMetric.getSeekCount());
combinedMetric.setYieldCount(updatedQueryMetric.getYieldCount());
combinedMetric.setDocSize(updatedQueryMetric.getDocSize());
combinedMetric.setDocRanges(updatedQueryMetric.getDocRanges());
combinedMetric.setFiRanges(updatedQueryMetric.getFiRanges());
}
Expand Down
41 changes: 25 additions & 16 deletions ...src/main/java/datawave/microservice/querymetric/handler/ShardTableQueryMetricHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,7 @@ public T toMetric(EventBase event) {
Date d = sdf_date_time1.parse(fieldValue);
m.setBeginDate(d);
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("CREATE_CALL_TIME")) {
m.setCreateCallTime(Long.parseLong(fieldValue));
Expand All @@ -562,7 +562,7 @@ public T toMetric(EventBase event) {
m.setCreateDate(d);
createDateSet = true;
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("DOC_RANGES")) {
try {
Expand All @@ -571,14 +571,23 @@ public T toMetric(EventBase event) {
m.setDocRanges(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("DOC_SIZE")) {
try {
long l = Long.parseLong(fieldValue);
if (l > m.getDocSize()) {
m.setDocSize(l);
}
} catch (Exception e) {
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("END_DATE")) {
try {
Date d = sdf_date_time1.parse(fieldValue);
m.setEndDate(d);
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("ERROR_CODE")) {
m.setErrorCode(fieldValue);
Expand All @@ -591,7 +600,7 @@ public T toMetric(EventBase event) {
m.setFiRanges(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("HOST")) {
m.setHost(fieldValue);
Expand All @@ -603,7 +612,7 @@ public T toMetric(EventBase event) {
m.setLastUpdated(d);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("LIFECYCLE")) {
Lifecycle l = Lifecycle.valueOf(fieldValue);
Expand All @@ -627,7 +636,7 @@ public T toMetric(EventBase event) {
m.setNextCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("NUM_UPDATES")) {
try {
Expand All @@ -636,14 +645,14 @@ public T toMetric(EventBase event) {
m.setNumUpdates(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.startsWith("PAGE_METRICS")) {
int index = fieldName.indexOf(".");
if (-1 == index) {
log.error("Could not parse field name to extract repetition count: " + fieldName);
log.error("Could not parse field name to extract repetition count: {}", fieldName);
} else {
Long pageNum = Long.parseLong(fieldName.substring(index + 1));
long pageNum = Long.parseLong(fieldName.substring(index + 1));
PageMetric pageMetric = PageMetric.parse(fieldValue);
if (pageMetric != null) {
pageMetric.setPageNumber(pageNum);
Expand Down Expand Up @@ -673,7 +682,7 @@ public T toMetric(EventBase event) {
int x = fieldValue.indexOf(":");
if (x > -1) {
String predictionName = fieldValue.substring(0, x);
Double predictionValue = Double.parseDouble(fieldValue.substring(x + 1));
double predictionValue = Double.parseDouble(fieldValue.substring(x + 1));
m.addPrediction(new Prediction(predictionName, predictionValue));
}
} catch (Exception e) {
Expand All @@ -699,7 +708,7 @@ public T toMetric(EventBase event) {
m.setSeekCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("SETUP_TIME")) {
m.setSetupTime(Long.parseLong(fieldValue));
Expand All @@ -710,7 +719,7 @@ public T toMetric(EventBase event) {
m.setSourceCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
} else if (fieldName.equals("USER")) {
m.setUser(fieldValue);
Expand All @@ -727,7 +736,7 @@ public T toMetric(EventBase event) {
m.setYieldCount(l);
}
} catch (Exception e) {
log.error(fieldName + ":" + fieldValue + ":" + e.getMessage());
log.error("{}:{}:{}", fieldName, fieldValue, e.getMessage());
}
}
}
Expand Down Expand Up @@ -773,12 +782,12 @@ protected void createAndConfigureTablesIfNecessary(String[] tableNames, Accumulo
if (tableHelper != null) {
tableHelper.configure(accumuloClient.tableOperations());
} else {
log.info("No configuration supplied for table: " + table);
log.info("No configuration supplied for table: {}", table);
}
}
} catch (TableExistsException te) {
// in this case, somebody else must have created the table after our existence check
log.debug("Tried to create " + table + " but somebody beat us to the punch");
log.debug("Tried to create {} but somebody beat us to the punch", table);
}
}
}
Expand Down
Loading
Loading