Skip to content

Commit

Permalink
kibana_usage_collection: optimize SO PIT searches (elastic#168677)
Browse files Browse the repository at this point in the history 
## Summary

Properly loop though PIT search pages instead of fetching all documents
at once

---------

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
@pgayvallet @kibanamachine
pgayvallet and kibanamachine authored Oct 15, 2023
1 parent 81544b7 commit 0217724
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 262 deletions.
28 changes: 0 additions & 28 deletions ...ns/kibana_usage_collection/server/collectors/application_usage/fetch_all_saved_objects.ts
View file

This file was deleted.

19 changes: 6 additions & 13 deletions ...plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,18 +177,11 @@ describe('rollTotals', () => {
],
{ overwrite: true }
);
expect(savedObjectClient.delete).toHaveBeenCalledTimes(3);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-2:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01'
);
expect(savedObjectClient.delete).toHaveBeenCalledWith(
SAVED_OBJECTS_DAILY_TYPE,
'appId-1:2020-01-01:viewId-1'
);
expect(savedObjectClient.bulkDelete).toHaveBeenCalledTimes(1);
expect(savedObjectClient.bulkDelete).toHaveBeenCalledWith([
{ type: SAVED_OBJECTS_DAILY_TYPE, id: 'appId-2:2020-01-01' },
{ type: SAVED_OBJECTS_DAILY_TYPE, id: 'appId-1:2020-01-01' },
{ type: SAVED_OBJECTS_DAILY_TYPE, id: 'appId-1:2020-01-01:viewId-1' },
]);
});
});
73 changes: 36 additions & 37 deletions src/plugins/kibana_usage_collection/server/collectors/application_usage/rollups/total.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import {
SAVED_OBJECTS_TOTAL_TYPE,
} from '../saved_objects_types';
import { serializeKey } from './utils';
import { fetchAllSavedObjects } from '../fetch_all_saved_objects';

/**
* Moves all the daily documents into aggregated "total" documents as we don't care about any granularity after 90 days
Expand All @@ -29,56 +28,56 @@ export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObje
}

try {
const [rawApplicationUsageTotals, rawApplicationUsageDaily] = await Promise.all([
fetchAllSavedObjects<ApplicationUsageTotal>(savedObjectsClient, {
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
fetchAllSavedObjects<ApplicationUsageDaily>(savedObjectsClient, {
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
}),
]);
const usageTotalsFinder = savedObjectsClient.createPointInTimeFinder<ApplicationUsageTotal>({
type: SAVED_OBJECTS_TOTAL_TYPE,
perPage: 200,
});
const existingTotals: Record<
string,
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
> = {};
for await (const { saved_objects: savedObjects } of usageTotalsFinder.find()) {
for (const savedObject of savedObjects) {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
numberOfClicks,
minutesOnScreen,
} = savedObject.attributes;

const existingTotals = rawApplicationUsageTotals.reduce(
(
acc,
{
attributes: { appId, viewId = MAIN_APP_DEFAULT_VIEW_ID, numberOfClicks, minutesOnScreen },
}
) => {
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);

// No need to sum because there should be 1 document per appId only
acc[key] = { appId, viewId, numberOfClicks, minutesOnScreen };
return acc;
},
{} as Record<
string,
{ appId: string; viewId: string; minutesOnScreen: number; numberOfClicks: number }
>
);
existingTotals[key] = { appId, viewId, numberOfClicks, minutesOnScreen };
}
}

const totals = rawApplicationUsageDaily.reduce(
(acc, { attributes }) => {
const usageDailyFinder = savedObjectsClient.createPointInTimeFinder<ApplicationUsageDaily>({
type: SAVED_OBJECTS_DAILY_TYPE,
filter: `${SAVED_OBJECTS_DAILY_TYPE}.attributes.timestamp < now-90d`,
perPage: 200,
});
const totals = { ...existingTotals };
const usageDailyIdsToDelete: string[] = [];
for await (const { saved_objects: savedObjects } of usageDailyFinder.find()) {
for (const savedObject of savedObjects) {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
numberOfClicks,
minutesOnScreen,
} = attributes;
} = savedObject.attributes;
const key = viewId === MAIN_APP_DEFAULT_VIEW_ID ? appId : serializeKey(appId, viewId);
const existing = acc[key] || { minutesOnScreen: 0, numberOfClicks: 0 };
const existing = totals[key] || { minutesOnScreen: 0, numberOfClicks: 0 };

acc[key] = {
totals[key] = {
appId,
viewId,
numberOfClicks: numberOfClicks + existing.numberOfClicks,
minutesOnScreen: minutesOnScreen + existing.minutesOnScreen,
};
return acc;
},
{ ...existingTotals }
);
usageDailyIdsToDelete.push(savedObject.id);
}
}

await Promise.all([
Object.entries(totals).length &&
Expand All @@ -90,8 +89,8 @@ export async function rollTotals(logger: Logger, savedObjectsClient?: ISavedObje
})),
{ overwrite: true }
),
...rawApplicationUsageDaily.map(
({ id }) => savedObjectsClient.delete(SAVED_OBJECTS_DAILY_TYPE, id) // There is no bulkDelete :(
savedObjectsClient.bulkDelete(
usageDailyIdsToDelete.map((id) => ({ id, type: SAVED_OBJECTS_DAILY_TYPE }))
),
]);
} catch (err) {
Expand Down
87 changes: 41 additions & 46 deletions ...e_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import { applicationUsageSchema } from './schema';
import { rollTotals, serializeKey } from './rollups';
import { ROLL_TOTAL_INDICES_INTERVAL, ROLL_INDICES_START } from './constants';
import type { ApplicationUsageTelemetryReport, ApplicationUsageViews } from './types';
import { fetchAllSavedObjects } from './fetch_all_saved_objects';

export const transformByApplicationViews = (
report: ApplicationUsageViews
Expand Down Expand Up @@ -68,29 +67,27 @@ export function registerApplicationUsageCollector(
if (typeof savedObjectsClient === 'undefined') {
return;
}
const [rawApplicationUsageTotals, rawApplicationUsageDaily] = await Promise.all([
fetchAllSavedObjects<ApplicationUsageTotal>(savedObjectsClient, {

const usageTotalsFinder = savedObjectsClient.createPointInTimeFinder<ApplicationUsageTotal>(
{
type: SAVED_OBJECTS_TOTAL_TYPE,
}),
fetchAllSavedObjects<ApplicationUsageDaily>(savedObjectsClient, {
type: SAVED_OBJECTS_DAILY_TYPE,
}),
]);

const applicationUsageFromTotals = rawApplicationUsageTotals.reduce(
(
acc,
{
attributes: {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen,
numberOfClicks,
},
}
) => {
const existing = acc[appId] || { clicks_total: 0, minutes_on_screen_total: 0 };
acc[serializeKey(appId, viewId)] = {
perPage: 200,
}
);
const applicationUsageFromTotals: ApplicationUsageTelemetryReport = {};
for await (const { saved_objects: savedObjects } of usageTotalsFinder.find()) {
for (const savedObject of savedObjects) {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen,
numberOfClicks,
} = savedObject.attributes;
const existing = applicationUsageFromTotals[appId] || {
clicks_total: 0,
minutes_on_screen_total: 0,
};
applicationUsageFromTotals[serializeKey(appId, viewId)] = {
appId,
viewId,
clicks_total: numberOfClicks + existing.clicks_total,
Expand All @@ -102,28 +99,28 @@ export function registerApplicationUsageCollector(
minutes_on_screen_30_days: 0,
minutes_on_screen_90_days: 0,
};
return acc;
},
{} as ApplicationUsageTelemetryReport
);
}
}

const nowMinus7 = moment().subtract(7, 'days');
const nowMinus30 = moment().subtract(30, 'days');
const nowMinus90 = moment().subtract(90, 'days');

const applicationUsage = rawApplicationUsageDaily.reduce(
(
acc,
{
attributes: {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen,
numberOfClicks,
timestamp,
},
}
) => {
const existing = acc[serializeKey(appId, viewId)] || {
const usageDailyFinder = savedObjectsClient.createPointInTimeFinder<ApplicationUsageDaily>({
type: SAVED_OBJECTS_DAILY_TYPE,
perPage: 200,
});
const applicationUsage = { ...applicationUsageFromTotals };
for await (const { saved_objects: savedObjects } of usageDailyFinder.find()) {
for (const savedObject of savedObjects) {
const {
appId,
viewId = MAIN_APP_DEFAULT_VIEW_ID,
minutesOnScreen,
numberOfClicks,
timestamp,
} = savedObject.attributes;
const existing = applicationUsage[serializeKey(appId, viewId)] || {
appId,
viewId,
clicks_total: 0,
Expand Down Expand Up @@ -154,18 +151,16 @@ export function registerApplicationUsageCollector(
minutes_on_screen_90_days: existing.minutes_on_screen_90_days + minutesOnScreen,
};

acc[serializeKey(appId, viewId)] = {
applicationUsage[serializeKey(appId, viewId)] = {
...existing,
clicks_total: existing.clicks_total + numberOfClicks,
minutes_on_screen_total: existing.minutes_on_screen_total + minutesOnScreen,
...(isInLast7Days ? last7Days : {}),
...(isInLast30Days ? last30Days : {}),
...(isInLast90Days ? last90Days : {}),
};
return acc;
},
applicationUsageFromTotals
);
}
}

return transformByApplicationViews(applicationUsage);
},
Expand Down
99 changes: 0 additions & 99 deletions src/plugins/saved_objects_management/server/lib/find_all.test.ts
View file

This file was deleted.

Loading

0 comments on commit 0217724

Please sign in to comment.