Skip to content

Commit

Permalink
Adding Document size to metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@mineralntl
mineralntl committed Sep 18, 2024
1 parent 6421fee commit 1ac72e4
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 24 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
<version.microservice.metadata-utils>4.0.6</version.microservice.metadata-utils>
<version.microservice.metrics-reporter>3.0.0</version.microservice.metrics-reporter>
<version.microservice.query-api>1.0.0</version.microservice.query-api>
<version.microservice.query-metric-api>4.0.7</version.microservice.query-metric-api>
<version.microservice.query-metric-api>4.0.8-SNAPSHOT</version.microservice.query-metric-api>
<version.microservice.type-utils>3.0.3</version.microservice.type-utils>
<version.minlog>1.2</version.minlog>
<version.mockito>2.23.0</version.mockito>
Expand Down
18 changes: 4 additions & 14 deletions warehouse/query-core/src/main/java/datawave/query/attributes/Document.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ public class Document extends AttributeBag<Document> implements Serializable {
long _bytes = 0;
private static final long ONE_HUNDRED_M = 1024L * 1000 * 100;
private static final long ONE_M = 1024L * 1000;
private static final long FIVE_HUNDRED_K = 1024L * 500;
TreeMap<String,Attribute<? extends Comparable<?>>> dict;

/**
Expand All @@ -59,7 +58,7 @@ public class Document extends AttributeBag<Document> implements Serializable {
private boolean trackSizes;

/**
* Whether or not this document represents an intermediate result. If true, then the document fields should also be empty.
* Whether this document represents an intermediate result. If true, then the document fields should also be empty.
*/
private boolean intermediateResult;

Expand Down Expand Up @@ -211,21 +210,12 @@ public Attribute<?> toDocKeyAttributes(Set<Key> docKeys, boolean keepRecordId) {

public void debugDocumentSize(Key docKey) {
long bytes = sizeInBytes();
// if more than 100M, then error
// if more than 100M, then info
if (bytes > (ONE_HUNDRED_M)) {
log.error("Document " + docKey + "; size = " + size() + "; bytes = " + bytes);
}
// if more than 10M, then warn
// else if (bytes > (1024l * 1000 * 10)) {
// log.warn("Document " + docKey + "; size = " + size() + "; bytes = " + bytes);
// }

// if more than 1M, then info
else if (bytes > (ONE_M)) {
log.info("Document " + docKey + "; size = " + size() + "; bytes = " + bytes);
}
// if more than 500K, then debug
else if (bytes > (FIVE_HUNDRED_K) && log.isDebugEnabled()) {
// if more than 1M, then debug
else if (bytes > (ONE_M)) {
log.debug("Document " + docKey + "; size = " + size() + "; bytes = " + bytes);
}
// trace everything
Expand Down
8 changes: 5 additions & 3 deletions ...ouse/query-core/src/main/java/datawave/query/metrics/ContentQueryMetricsIngestHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,12 @@ public Multimap<String,String> getEventFieldsToWrite(T updatedQueryMetric) {
fields.put("NEXT_COUNT", Long.toString(updatedQueryMetric.getNextCount()));
fields.put("SEEK_COUNT", Long.toString(updatedQueryMetric.getSeekCount()));
fields.put("YIELD_COUNT", Long.toString(updatedQueryMetric.getYieldCount()));
fields.put("DOC_SIZE", Long.toString(updatedQueryMetric.getDocSize()));
fields.put("DOC_RANGES", Long.toString(updatedQueryMetric.getDocRanges()));
fields.put("FI_RANGES", Long.toString(updatedQueryMetric.getFiRanges()));
Map<String,String> versionMap = updatedQueryMetric.getVersionMap();
if (versionMap != null) {
versionMap.entrySet().stream().forEach(e -> {
fields.put("VERSION." + e.getKey().toUpperCase(), e.getValue());
});
versionMap.forEach((key, value) -> fields.put("VERSION." + key.toUpperCase(), value));
}
Set<Prediction> predictions = updatedQueryMetric.getPredictions();
if (predictions != null && !predictions.isEmpty()) {
Expand Down Expand Up @@ -289,6 +288,9 @@ public Multimap<String,String> getEventFieldsToDelete(T updatedQueryMetric, T st
if (updatedQueryMetric.getYieldCount() != storedQueryMetric.getYieldCount()) {
fields.put("YIELD_COUNT", Long.toString(storedQueryMetric.getYieldCount()));
}
if (updatedQueryMetric.getDocSize() != storedQueryMetric.getDocSize()) {
fields.put("DOC_SIZE", Long.toString(storedQueryMetric.getDocSize()));
}
if (updatedQueryMetric.getDocRanges() != storedQueryMetric.getDocRanges()) {
fields.put("DOC_RANGES", Long.toString(storedQueryMetric.getDocRanges()));
}
Expand Down
16 changes: 10 additions & 6 deletions ...house/query-core/src/main/java/datawave/query/transformer/DocumentTransformerSupport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,15 @@ public abstract class DocumentTransformerSupport<I,O> extends EventQueryTransfor
private long nextCount = 0;
private long seekCount = 0;
private long yieldCount = 0L;
private long docSize = 0;
private long docRanges = 0;
private long fiRanges = 0;
private boolean logTimingDetails = false;
private CardinalityRecord resultCardinalityDocumentDate = null;
private CardinalityRecord resultCardinalityQueryDate = null;
protected CardinalityConfiguration cardinalityConfiguration = null;
private int objectsTransformed = 0;
private long logicCreated = System.currentTimeMillis();
private final long logicCreated = System.currentTimeMillis();
private Set<String> projectFields = Collections.emptySet();
private Set<String> disallowlistedFields = Collections.emptySet();

Expand Down Expand Up @@ -151,8 +152,8 @@ protected Map<String,String> getAdditionalCardinalityValues(Key documentKey, Doc
return additionalValues;
}

// When a single Ivarator is used during a query on the teserver, we save time by not sorting the UIDs (not necessary for further comparisons).
// To ensure that returned keys appear to be in sorted order on the way back we prpend a one-up number to the colFam.
// When a single Ivarator is used during a query on the tserver, we save time by not sorting the UIDs (not necessary for further comparisons).
// To ensure that returned keys appear to be in sorted order on the way back we prepend a one-up number to the colFam.
// In this edge case, the prepended number needs to be removed.
protected static Key correctKey(Key origKey) {
Key key = origKey;
Expand Down Expand Up @@ -239,6 +240,7 @@ protected void extractMetrics(Document document, Key documentKey) {
nextCount += currentNextCount;
seekCount += currentSeekCount;
yieldCount += currentYieldCount;
docSize = document.sizeInBytes();
Map<String,Long> stageTimers = timingMetadata.getStageTimers();
if (stageTimers.containsKey(QuerySpan.Stage.DocumentSpecificTree.toString())) {
docRanges++;
Expand Down Expand Up @@ -309,6 +311,7 @@ public void writeQueryMetrics(BaseQueryMetric metric) {
metric.setNextCount(nextCount);
metric.setSeekCount(seekCount);
metric.setYieldCount(yieldCount);
metric.setDocSize(docSize);
metric.setDocRanges(docRanges);
metric.setFiRanges(fiRanges);
}
Expand All @@ -320,6 +323,7 @@ public void resetMetrics() {
nextCount = 0;
seekCount = 0;
yieldCount = 0;
docSize = 0;
docRanges = 0;
fiRanges = 0;
}
Expand All @@ -332,7 +336,7 @@ protected List<String> getFieldValues(Document document, String field, boolean s
for (Entry<String,Attribute<? extends Comparable<?>>> e : document.getDictionary().entrySet()) {
String docField = e.getKey();
String baseDocField = JexlASTHelper.removeGroupingContext(docField);
String reverseMappedField = reverseModel.containsKey(baseDocField) ? reverseModel.get(baseDocField) : "";
String reverseMappedField = reverseModel.getOrDefault(baseDocField, "");
if (baseDocField.equals(field) || reverseMappedField.equals(field)) {
Attribute<?> a = e.getValue();
if (a instanceof Attributes) {
Expand All @@ -359,7 +363,7 @@ protected void collectCardinalities(Document document, Key documentKey, String u
String uidField = cardinalityConfiguration.getCardinalityUidField();
if (org.apache.commons.lang.StringUtils.isNotBlank(uidField)) {
List<String> documentUidValues = getFieldValues(document, uidField, true);
if (documentUidValues.isEmpty() == false) {
if (!documentUidValues.isEmpty()) {
eventId = documentUidValues.get(0);
}
}
Expand Down Expand Up @@ -451,7 +455,7 @@ protected Collection<FieldBase<?>> buildDocumentFields(Key documentKey, String f
Set<FieldBase<?>> myFields = new HashSet<>();

if (attr instanceof Attributes) {
Attributes attributeList = Attributes.class.cast(attr);
Attributes attributeList = (Attributes) attr;

for (Attribute<? extends Comparable<?>> embeddedAttr : attributeList.getAttributes())
myFields.addAll(buildDocumentFields(documentKey, fieldName, embeddedAttr, topLevelColumnVisibility, markingFunctions));
Expand Down

0 comments on commit 1ac72e4

Please sign in to comment.