Skip to content

Commit

Permalink
Merge pull request #161 from DataCloud-project/b-156
Browse files Browse the repository at this point in the history 
fix: overall metrics fixed
  • Loading branch information
@goranbs
goranbs authored Jun 10, 2024
2 parents 161cdd6 + 0503fc4 commit 3b810b6
Showing 1 changed file with 129 additions and 47 deletions.
176 changes: 129 additions & 47 deletions frontend/src/routes/projects/dryruns/[dry_run]/[resource]/+page.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,7 @@
import { colors } from './Config.js';
import { stepsList, selectedProjectName, selectedDryRunName } from '$stores/stores';
import Legend from './legend.svelte';
import {
getMetricsUsageUtils,
getMetricsAnalyticsUtils,
displayStepDuration
} from '$utils/resource-utils';
import type { MetricsAnalytics } from '$utils/resource-utils';
import { getMetricsUsageUtils, displayStepDuration } from '$utils/resource-utils';
import { displayAlert } from '$utils/alerts-utils';
import type { DryRunMetrics, DryRun, MetricsWithTimeStamps } from '$typesdefinitions';
Expand All @@ -46,8 +41,6 @@
[key: string]: MetricsWithTimeStamps[];
} = {};
let pipelineMetricsAnalytics: MetricsAnalytics = {};
let allStepNames: string[] = [];
$: selectedStep = 'Total';
Expand All @@ -57,6 +50,58 @@
selectedStep = 'Total';
}
let maxCpu = 'N/A';
let maxMemory = 'N/A';
function findMax(input: number[]): number {
if (input?.length === 0) return 0;
return Math.max(...input);
}
let averageCpu = 'N/A';
let averageMemory = 'N/A';
function computeAverage(input: number[]): number {
if (input?.length === 0) return 0;
return input.reduce((a, b) => a + b) / input.length;
}
let networkTotalReceived = 'N/A';
let networkTotalTransmitted = 'N/A';
let pipelineDuration = 'N/A';
function formatDuration(seconds: number): string {
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const remainingSeconds = seconds % 60;
let durationString = '';
if (hours > 0) {
durationString += `${hours}h `;
}
if (minutes > 0) {
durationString += `${minutes}m `;
}
if (remainingSeconds > 0) {
durationString += `${remainingSeconds}s`;
}
return durationString.trim();
}
function computePipelineDuration(): void {
let totalDuration = 0;
if (reactiveStepsList) {
if (selectedStep === 'Total') {
reactiveStepsList?.forEach((step) => {
totalDuration += step.duration;
});
} else {
totalDuration = reactiveStepsList.find((step) => step.displayName === selectedStep)
?.duration as number;
}
}
pipelineDuration = formatDuration(totalDuration);
}
let dryRunPhaseMessage: string | null;
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type, consistent-return
const getMetricsResponse = async () => {
Expand Down Expand Up @@ -129,13 +174,6 @@
cumulativeNetworkData = result.cumulativeNetworkData;
currentNetworkData = result.currentNetworkData;
logs = result.logs;
pipelineMetricsAnalytics = await getMetricsAnalyticsUtils(
allStepNames,
metricsResponse as unknown as DryRunMetrics[],
cumulativeCpuData,
memoryData,
cumulativeNetworkData
);
const responses = {
workflow: workflowResponse.project,
dryrun: dryrunResponse,
Expand Down Expand Up @@ -255,6 +293,7 @@
function stepOnClick(stepName: string): void {
selectedStep = stepName;
computePipelineDuration();
}
// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
Expand Down Expand Up @@ -283,22 +322,41 @@
case 'cpu-current': {
if (Object.keys(currentCpuData).length > 0)
if (selectedStep === 'Total') {
const maxCpuList: number[] = [];
const avgCpulist: number[] = [];
allStepNames.forEach((step) => {
if (currentCpuData[step]) data.push(currentCpuData[step]);
if (currentCpuData[step]) {
data.push(currentCpuData[step]);
maxCpuList.push(findMax(currentCpuData[step].y));
avgCpulist.push(computeAverage(currentCpuData[step].y));
}
maxCpu = findMax(maxCpuList).toFixed(2).toString();
averageCpu = computeAverage(avgCpulist).toFixed(2).toString();
});
} else {
data.push(currentCpuData[selectedStep]);
maxCpu = findMax(currentCpuData[selectedStep].y).toFixed(2).toString();
averageCpu = computeAverage(currentCpuData[selectedStep].y).toFixed(2).toString();
}
break;
}
case 'memory': {
if (Object.keys(memoryData).length > 0)
if (selectedStep === 'Total') {
const maxMemoryList: number[] = [];
const avgMemoryList: number[] = [];
allStepNames.forEach((step) => {
if (memoryData[step]) data.push(memoryData[step]);
if (memoryData[step]) {
data.push(memoryData[step]);
maxMemoryList.push(findMax(memoryData[step].y));
avgMemoryList.push(computeAverage(memoryData[step].y));
}
maxMemory = filesize(findMax(maxMemoryList), { round: 2 });
averageMemory = filesize(computeAverage(avgMemoryList), { round: 2 });
});
} else {
data.push(memoryData[selectedStep]);
maxMemory = filesize(findMax(memoryData[selectedStep].y), { round: 2 });
}
break;
}
Expand All @@ -318,18 +376,45 @@
break;
}
case 'network-current': {
if (Object.keys(currentNetworkData).length > 0)
if (Object.keys(currentNetworkData).length > 0) {
let networkReceived = 0;
let networkTransmitted = 0;
if (selectedStep === 'Total') {
allStepNames.forEach((step) => {
currentNetworkData[step].forEach((networkData) => {
data.push(networkData);
const networkSendReceiveType = networkData.name.split('-')[1];
if (networkSendReceiveType === 'received') {
networkData.y.forEach((value) => {
networkReceived += value;
});
}
if (networkSendReceiveType === 'transmitted') {
networkData.y.forEach((value) => {
networkTransmitted += value;
});
}
});
});
} else {
currentNetworkData[selectedStep].forEach((networkData) => {
data.push(networkData);
const networkSendReceiveType = networkData.name.split('-')[1];
if (networkSendReceiveType === 'received') {
networkData.y.forEach((value) => {
networkReceived += value;
});
}
if (networkSendReceiveType === 'transmitted') {
networkData.y.forEach((value) => {
networkTransmitted += value;
});
}
});
}
networkTotalReceived = filesize(networkReceived, { round: 2 });
networkTotalTransmitted = filesize(networkTransmitted, { round: 2 });
}
break;
}
default: {
Expand All @@ -346,13 +431,14 @@
workflow = getDataResponse.workflow;
allStepNames = getDataResponse.allstepnames;
await buildDiagram();
computePipelineDuration();
loadingFinished = true;
});
function getPartLogs(stepName: string, nmaxlinelength: number): string {
const steplogs = logs[stepName];
console.log('stepName:', stepName);
console.log('steplogs:', steplogs);
// console.log('stepName:', stepName);
// console.log('steplogs:', steplogs);
// eslint-disable-next-line unicorn/prefer-ternary
if (steplogs.length < nmaxlinelength) return steplogs;
// eslint-disable-next-line no-else-return
Expand Down Expand Up @@ -483,35 +569,31 @@
<thead>
<tr>
<th>Resource</th>
<th>Average, Maximum usage</th>
<th>Metrics</th>
</tr>
</thead>
<tbody>
{#each Object.keys(pipelineMetricsAnalytics[selectedStep]) as key}
<tr>
<td>{key}</td>
{#if key === 'CPU'}
<td>
{pipelineMetricsAnalytics[selectedStep][key].avg.toFixed(3)} %,
{pipelineMetricsAnalytics[selectedStep][key].max.toFixed(3)}
%
</td>
<!-- for eslint -->
{:else if key === 'Memory' || key === 'Network_received' || key === 'Network_transferred'}
<td
>{filesize(pipelineMetricsAnalytics[selectedStep][key].avg)},
{filesize(pipelineMetricsAnalytics[selectedStep][key].max)}</td
>
<!-- <td
>{pipelineMetricsAnalytics[selectedStep][key].avg},
{pipelineMetricsAnalytics[selectedStep][key].max}</td
> -->
{:else if key === 'Duration'}
<td> {pipelineMetricsAnalytics[selectedStep][key]}</td>
{/if}
</tr>
{/each}
</tbody>
<tr>
<td>CPU</td>
<td>{maxCpu} % [ max ] {averageCpu} % [ avg ]</td>
</tr>
<tr>
<td>Memory</td>
<td>{maxMemory} [ max ] {averageMemory} [ avg ]</td>
</tr>
<tr>
<td>Network received</td>
<td> {networkTotalReceived} </td>
</tr>
<tr>
<td>Network transmitted</td>
<td> {networkTotalTransmitted} </td>
</tr>
<tr>
<td>Duration</td>
<td> {pipelineDuration} </td>
</tr></tbody
>
</table>
</div>

Expand Down Expand Up @@ -561,7 +643,7 @@
}
.logbox {
overflow-y: scroll;
max-height: 100vh;
max-height: 50vh;
}
ul {
max-height: 75vh;
Expand Down

0 comments on commit 3b810b6

Please sign in to comment.