Merge pull request #18 from deepeshmittal/master

Updated sample tests log files and added other new files

ExecScripts/SampleDaytonaFramework/sample_execscript.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Sample execution script to run a simple performance test within Daytona
+# Demonstrates the steps of gathering arguments from the UI and runs a performance test
+# Command line arguments:
+
+echo "--------------------execscript--------------------"
+echo $@
+echo "--------------------------------------------------"
+
+
+iterations=`echo $1 | sed "s/\"//g"`
+delay=`echo $2 | sed "s/\"//g"`
+
+# Run your performance/benchmark/workload here
+x=1
+while [ $x -le $iterations ]
+do
+ dd if=/dev/zero of=/dev/null count=20000000
+ sleep $delay
+ x=$(( $x + 1 ))
+done
+
+echo "Iterations : " $iterations
+echo "Delay : " $delay
+
+echo "Benchmark Test Completed"
+
+# These KPI's would be computed by your benchmark 
+avglatency=10.5
+maxlatency=50
+minlatency=6
+failurerate=0.1
+
+# Create your results.csv with all KPI's in name, value format
+echo Key, Value > results.csv
+echo Iterations, $iterations >> results.csv
+echo AvgLatency-ms, $avglatency >> results.csv
+echo MaxLatency-ms, $maxlatency >> results.csv
+echo MinLatency-ms, $minlatency >> results.csv
+echo FailureRate, $failurerate >> results.csv
+echo Delay-secs, $delay >> results.csv
+
+# Create a multi-column ( > 2) csv file with multiple rows
+echo Col1, Col2, Col3, Col4 > multicol.csv
+echo 1, 2, 3, 4 >> multicol.csv
+echo 5, 6, 7, 8 >> multicol.csv
+echo 9, 10, 11, 12 >> multicol.csv
+echo 13, 14, 15, 16 >> multicol.csv
+
+# Collect /proc/cpuinfo and /proc/meminfo data
+cat /proc/cpuinfo > cpuinfo.txt
+cat /proc/meminfo > meminfo.txt

Scheduler+Agent/process_files.py

@@ -0,0 +1,185 @@
+import os
+import datetime
+import csv
+import re
+
+
+class ProcessOutputFiles:
+ def __init__(self, logctx):
+ self.lctx = logctx
+
+ def process_output_files(self, path):
+ # Process top output files
+ self.process_top_output(path)
+ self.process_docker_output(path)
+ return "Files processing completed"
+
+ def process_top_output(self, path):
+ output_file = path + "top_output.txt"
+ self.lctx.debug("Top output processing started for file : " + output_file)
+
+ cpu_map = {}
+ mem_map = {}
+ res_mem_map = {}
+ cpu_array = []
+ mem_array = []
+ res_mem_array = []
+ cpu_array.append(["Time"])
+ mem_array.append(["Time"])
+ res_mem_array.append(["Time"])
+ timestamp = 0
+
+ if not os.path.exists(output_file):
+ return "File Not Found Exception"
+
+ cpu_file = open(path + "cpu_usage.plt", 'w+')
+ if not cpu_file:
+ return "Not able to create cpu_usage.plt"
+
+ mem_file = open(path + "memory_usage.plt", 'w+')
+ if not mem_file:
+ return "Not able to create memory_usage.plt"
+
+ res_mem_file = open(path + "res_memory_usage.plt", 'w+')
+ if not res_mem_file:
+ return "Not able to create res_memory_usage.plt"
+
+ process_data = True
+ with open(output_file) as f:
+ for line in f:
+ line = line.strip('\n')
+ line = line.strip()
+ line = re.sub(' +', ' ', line)
+ line = line.replace(' ', ',')
+
+ if line.startswith("top"):
+ process_data = False
+ continue
+
+ if line.startswith("PID"):
+ process_data = True
+ continue
+
+ if process_data:
+ if len(line) > 0:
+ try:
+ datetime.datetime.strptime(line, "%Y-%m-%dT%H:%M:%S%fz")
+ cpu_array = self.update_array_from_map(timestamp, cpu_array, cpu_map)
+ mem_array = self.update_array_from_map(timestamp, mem_array, mem_map)
+ res_mem_array = self.update_array_from_map(timestamp, res_mem_array, res_mem_map)
+ timestamp = line
+ cpu_map.clear()
+ mem_map.clear()
+ res_mem_map.clear()
+ except:
+ line_array = line.split(",")
+ cpu_map[line_array[11] + " - " + line_array[0]] = line_array[8]
+ mem_map[line_array[11] + " - " + line_array[0]] = line_array[9]
+ res_mem_map[line_array[11] + " - " + line_array[0]] = line_array[5]
+
+ cpu_array = self.update_array_from_map(timestamp, cpu_array, cpu_map)
+ mem_array = self.update_array_from_map(timestamp, mem_array, mem_map)
+ res_mem_array = self.update_array_from_map(timestamp, res_mem_array, res_mem_map)
+
+ self.create_plt_from_array(cpu_file, cpu_array)
+ self.create_plt_from_array(mem_file, mem_array)
+ self.create_plt_from_array(res_mem_file, res_mem_array)
+
+ def process_docker_output(self, path):
+ output_file = path + "docker_stat.txt"
+ self.lctx.debug("Docker stat output processing started for file : " + output_file)
+
+ cpu_map = {}
+ mem_map = {}
+
+ cpu_array = []
+ mem_array = []
+
+ cpu_array.append(["Time"])
+ mem_array.append(["Time"])
+
+ timestamp = 0
+
+ if not os.path.exists(output_file):
+ return "File Not Found Exception"
+
+ cpu_file = open(path + "docker_cpu.plt", 'w+')
+ if not cpu_file:
+ return "Not able to create docker_cpu.plt"
+
+ mem_file = open(path + "docker_memory.plt", 'w+')
+ if not mem_file:
+ return "Not able to create docker_memory.plt"
+
+ with open(output_file) as f:
+ for line in f:
+ line = line.strip('\n')
+ line = line.strip()
+ line = re.sub(' +', ' ', line)
+ line = line.replace(' ', ',')
+
+ if line.startswith("NAME"):
+ continue
+
+ if len(line) > 0:
+ try:
+ datetime.datetime.strptime(line, "%Y-%m-%dT%H:%M:%S%fz")
+ cpu_array = self.update_array_from_map(timestamp, cpu_array, cpu_map)
+ mem_array = self.update_array_from_map(timestamp, mem_array, mem_map)
+
+ timestamp = line
+ cpu_map.clear()
+ mem_map.clear()
+ except:
+ line_array = line.split(",")
+ cpu_map[line_array[0] + " - " + line_array[1]] = line_array[2]
+ mem_map[line_array[0] + " - " + line_array[1]] = line_array[3]
+
+ cpu_array = self.update_array_from_map(timestamp, cpu_array, cpu_map)
+ mem_array = self.update_array_from_map(timestamp, mem_array, mem_map)
+
+ self.create_plt_from_array(cpu_file, cpu_array)
+ self.create_plt_from_array(mem_file, mem_array)
+
+ def create_plt_from_array(self, fh, array):
+ with fh as f:
+ writer = csv.writer(f)
+ writer.writerows(array)
+ fh.close()
+
+ def update_array_from_map(self, ts, input_array, input_map):
+ row_count = len(input_array)
+ col_count = len(input_array[0])
+ if len(input_map) == 0 and ts != 0:
+ temp_list = list()
+ temp_list.append(ts)
+ for i in range(1, col_count):
+ temp_list.append(0.0)
+
+ input_array.append(temp_list)
+
+ elif len(input_map) > 0 and ts != 0:
+ temp_list = list()
+ temp_list.append(ts)
+ for i in range(1, col_count):
+ if input_array[0][i] in input_map:
+ temp_list.append(input_map.get(input_array[0][i]))
+ input_map.pop(input_array[0][i], None)
+ else:
+ temp_list.append(0.0)
+
+ input_array.append(temp_list)
+ row_count += 1
+
+ if len(input_map) > 0:
+ for x in input_map:
+ input_array[0].append(x)
+ col_count += 1
+ for i in range(1, row_count):
+ if input_array[i][0] == ts:
+ input_array[i].append(input_map[x])
+ else:
+ input_array[i].append(0.0)
+
+ return input_array
+