Skip to content

Commit

Permalink
chore(server): use bodybuilder for opensearch searches
Browse files Browse the repository at this point in the history
  • Loading branch information
@arielweinberger
arielweinberger committed Oct 14, 2023
1 parent f472202 commit a46ac0c
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 165 deletions.
88 changes: 7 additions & 81 deletions apps/server/src/app/metrics/metrics.utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { MsearchBody } from "@opensearch-project/opensearch/api/types";
import { ProjectMetricType } from "./inputs/get-project-metrics.input";
import bodybuilder from "bodybuilder";

export function getPercentageChange(
currentValue: number,
Expand All @@ -15,37 +14,12 @@ export function getPercentageChange(
export function buildBaseProjectMetricQuery(
projectId: string,
startDate: string,
endDate: string,
options: Partial<MsearchBody> = {}
) {
const body: MsearchBody = {
query: {
bool: {
filter: [
{
term: {
"ownership.projectId": projectId,
},
},
{
range: {
timestamp: {
gte: startDate,
lte: endDate,
},
},
},
...((options?.query?.bool?.filter as any[]) || []),
],
must_not: [...((options?.query?.bool?.must_not as any[]) || [])],
},
},
size: 0,
};

if (options.aggs) {
body.aggs = options.aggs;
}
endDate: string
): bodybuilder.Bodybuilder {
const body = bodybuilder()
.filter("term", "ownership.projectId", projectId)
.filter("range", "timestamp", { gte: startDate, lte: endDate })
.size(0);

return body;
}
Expand Down Expand Up @@ -81,51 +55,3 @@ export function getStartAndEndDates(
},
};
}

export function getMetricHistogramParams(metric: ProjectMetricType): {
aggregation: any;
filters?: any[];
} {
switch (metric) {
case ProjectMetricType.requests:
return {
aggregation: {
value_count: {
field: "timestamp",
},
},
};
case ProjectMetricType.duration:
return {
aggregation: {
avg: {
field: "calculated.duration",
},
},
};
case ProjectMetricType.erroneousRequests:
return {
aggregation: {
value_count: {
field: "timestamp",
},
},
filters: [
{
bool: {
must_not: [
{
term: {
"response.status": 200,
},
},
],
},
},
],
};

default:
throw new Error("Invalid metric");
}
}
147 changes: 63 additions & 84 deletions apps/server/src/app/metrics/project-metrics.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import { OpenSearchService } from "../opensearch/opensearch.service";
import { HistogramMetric, ProjectMetric } from "./models/project-metric.model";
import {
buildBaseProjectMetricQuery,
getMetricHistogramParams,
getStartAndEndDates,
} from "./metrics.utils";
import { OpenSearchIndex } from "../opensearch/types";
import bodybuilder from "bodybuilder";

@Injectable()
export class ProjectMetricsService {
Expand Down Expand Up @@ -47,17 +47,13 @@ export class ProjectMetricsService {
const { current, previous } = getStartAndEndDates(startDate, endDate);

const buildAndExecuteQuery = async (startDate: string, endDate: string) => {
const body = buildBaseProjectMetricQuery(projectId, startDate, endDate)
.aggregation("sum", "calculated.totalCost", "total_cost")
.build();

const query = {
index: OpenSearchIndex.Requests,
body: buildBaseProjectMetricQuery(projectId, startDate, endDate, {
aggs: {
total_cost: {
sum: {
field: "calculated.totalCost",
},
},
},
}),
body,
};

const result = await this.openSearchService.client.search(query);
Expand All @@ -84,17 +80,12 @@ export class ProjectMetricsService {
const { current, previous } = getStartAndEndDates(startDate, endDate);

const buildAndExecuteQuery = async (startDate: string, endDate: string) => {
const body = buildBaseProjectMetricQuery(projectId, startDate, endDate)
.aggregation("avg", "calculated.duration", "avg_duration")
.build();
const result = await this.openSearchService.client.search({
index: OpenSearchIndex.Requests,
body: buildBaseProjectMetricQuery(projectId, startDate, endDate, {
aggs: {
avg_duration: {
avg: {
field: "calculated.duration",
},
},
},
}),
body,
});

return result.body.aggregations.avg_duration.value || 0;
Expand All @@ -119,21 +110,13 @@ export class ProjectMetricsService {
const { current, previous } = getStartAndEndDates(startDate, endDate);

const buildAndExecuteQuery = async (startDate: string, endDate: string) => {
const body = buildBaseProjectMetricQuery(projectId, startDate, endDate)
.filter("term", "response.status", 200)
.build();

const result = await this.openSearchService.client.search({
index: OpenSearchIndex.Requests,
body: buildBaseProjectMetricQuery(projectId, startDate, endDate, {
query: {
bool: {
filter: [
{
term: {
"response.status": 200,
},
},
],
},
},
}),
body,
});

return result.body.hits.total.value || 0;
Expand All @@ -158,21 +141,13 @@ export class ProjectMetricsService {
const { current, previous } = getStartAndEndDates(startDate, endDate);

const buildAndExecuteQuery = async (startDate: string, endDate: string) => {
const body = buildBaseProjectMetricQuery(projectId, startDate, endDate)
.notFilter("term", "response.status", 200)
.build();

const result = await this.openSearchService.client.search({
index: OpenSearchIndex.Requests,
body: buildBaseProjectMetricQuery(projectId, startDate, endDate, {
query: {
bool: {
must_not: [
{
term: {
"response.status": 200,
},
},
],
},
},
}),
body,
});

return result.body.hits.total.value || 0;
Expand All @@ -196,48 +171,52 @@ export class ProjectMetricsService {
endDate: Date,
bucketSize: string
): Promise<HistogramMetric[]> {
const { aggregation, filters = [] } = getMetricHistogramParams(metricType);
let body = bodybuilder();

const result = await this.openSearchService.client.search({
index: OpenSearchIndex.Requests,
body: {
query: {
bool: {
filter: [
{
term: {
"ownership.projectId": projectId,
},
},
{
range: {
timestamp: {
gte: startDate.toISOString(),
lte: endDate.toISOString(),
},
},
},
...filters,
],
},
},
aggs: {
metrics_over_time: {
date_histogram: {
field: "timestamp",
interval: bucketSize,
extended_bounds: {
min: startDate.toISOString(),
max: endDate.toISOString(),
},
},
aggs: {
metric_value: aggregation,
},
},
body = body
.addFilter("term", "ownership.projectId", projectId)
.andFilter("range", "timestamp", {
gte: startDate.toISOString(),
lte: endDate.toISOString(),
});

let aggType, aggField;

switch (metricType) {
case ProjectMetricType.requests:
aggType = "value_count";
aggField = "timestamp";
break;
case ProjectMetricType.duration:
aggType = "avg";
aggField = "calculated.duration";
break;
case ProjectMetricType.erroneousRequests:
aggType = "value_count";
aggField = "timestamp";
body = body.notFilter("term", "response.status", 200);
break;
}

body = body.aggregation(
"date_histogram",
"timestamp",
{
interval: bucketSize,
extended_bounds: {
min: startDate.toISOString(),
max: endDate.toISOString(),
},
size: 0,
},
"metrics_over_time",
(agg) => agg.aggregation(aggType, aggField, "metric_value")
);

body = body.size(0);

const result = await this.openSearchService.client.search({
index: OpenSearchIndex.Requests,
body: body.build(),
});

// Convert the response to the HistogramMetric format
Expand Down

0 comments on commit a46ac0c

Please sign in to comment.