Skip to content

Commit

Permalink
added two new distributions, mixed termination and fixed bug in logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtpaulo
jtpaulo committed May 19, 2020
1 parent 39527a4 commit 0956b9d
Show file tree
Hide file tree
Showing 7 changed files with 314 additions and 118 deletions.
162 changes: 62 additions & 100 deletions DEDISbench.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,33 +144,31 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co
stat.snap_ops=malloc(sizeof(double)*1000);
stat.snap_time=malloc(sizeof(unsigned long long int)*1000);

//check if terminationis time or not
int termination_type;
uint64_t begin;
uint64_t end;
uint64_t begin_time, begin_size;
uint64_t end_time, end_size;
uint64_t ru_begin;
struct timeval tim;
int duration = conf->time_to_run;
if(duration > 0 ){


gettimeofday(&tim, NULL);
begin_time=tim.tv_sec;
ru_begin = begin_time + conf->start;

if(conf->time_to_run > 0 ){
//Get current time to mark the beggining of the benchmark and check
//when it should end
end_time = begin_time + conf->start + conf->time_to_run - conf->finish;
}else{
end_time=UINT64_MAX;
}

gettimeofday(&tim, NULL);
begin=tim.tv_sec;
ru_begin = begin + conf->start*30;
end = begin+duration - conf->finish*30;

termination_type=TIME;

if(conf->number_ops > 0 ){
begin_size=0;
end_size=conf->number_ops/nproc;
}
//SIZE termination
else{
begin=0;
ru_begin = 0;
end=conf->number_ops/nproc;

//printf("end is %lu\n", end);
termination_type=SIZE;
end_size=UINT64_MAX;
}

//global timeval structure for nominal tests
Expand All @@ -186,7 +184,7 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co
double time_elapsed=1;

//while bench time has not ended or amount of data is not written
while(begin<end){
while( begin_time < end_time && begin_size < end_size){
//for nominal testes only
//number of operations performed for all processes
//since we are running N processes concurrently at the same I/O rate
Expand Down Expand Up @@ -291,13 +289,13 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co
perror("Error writing block ");

if(stat.beginio==-1){
if(begin >= ru_begin){
if(begin_time >= ru_begin){
stat.beginio=t1;
stat.last_snap_time=stat.t1snap;
}
}

if(begin >= ru_begin){
if(begin_time >= ru_begin){
stat.latency+=(t2-t1);
stat.snap_lat+=(t2-t1);
}
Expand Down Expand Up @@ -344,13 +342,13 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co
}

if(stat.beginio==-1){
if(begin >= ru_begin){
if(begin_time >= ru_begin){
stat.beginio=t1;
stat.last_snap_time=stat.t1snap;
}
}

if(begin >= ru_begin){
if(begin_time >= ru_begin){
stat.latency+=(t2-t1);
stat.snap_lat+=(t2-t1);
}
Expand All @@ -365,28 +363,28 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co

free(buf);

//One more operation was performed
if(begin>=ru_begin){
//One more operation was performed
if(begin_time>=ru_begin){
stat.tot_ops++;
stat.snap_totops++;
}

if(stat.t1snap>=stat.last_snap_time+30*1e6){

if(begin >= ru_begin){
stat.snap_throughput[stat.iter_snap]=(stat.snap_totops/((stat.t1snap-stat.last_snap_time)/1.0e6));
stat.snap_latency[stat.iter_snap]=(stat.snap_lat/stat.snap_totops)/1000;
stat.snap_ops[stat.iter_snap]=(stat.snap_totops);
stat.snap_time[stat.iter_snap]=stat.t1snap;
}
stat.iter_snap++;
stat.snap_lat=0;
stat.snap_totops=0;
stat.last_snap_time=stat.t1snap;
}

if(conf->termination_type==SIZE){
begin++;
if(stat.t1snap>=stat.last_snap_time+30*1e6){

if(begin_time >= ru_begin){
stat.snap_throughput[stat.iter_snap]=(stat.snap_totops/((stat.t1snap-stat.last_snap_time)/1.0e6));
stat.snap_latency[stat.iter_snap]=(stat.snap_lat/stat.snap_totops)/1000;
stat.snap_ops[stat.iter_snap]=(stat.snap_totops);
stat.snap_time[stat.iter_snap]=stat.t1snap;
}
stat.iter_snap++;
stat.snap_lat=0;
stat.snap_totops=0;
stat.last_snap_time=stat.t1snap;
}

if(conf->number_ops>0){
begin_size++;
}
}

}
Expand All @@ -411,10 +409,8 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co

//update current time
gettimeofday(&tim, NULL);
if(termination_type==TIME){
begin=tim.tv_sec;
}

begin_time=tim.tv_sec;

}


Expand All @@ -426,7 +422,7 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co

if(stat.t1snap>stat.last_snap_time){
//Write last snap because ther may be some operations missing
if(begin>=ru_begin){
if(begin_time>=ru_begin){
stat.snap_throughput[stat.iter_snap]=(stat.snap_totops/((stat.t1snap-stat.last_snap_time)/1.0e6));
stat.snap_latency[stat.iter_snap]=(stat.snap_lat/stat.snap_totops)/1000;
stat.snap_ops[stat.iter_snap]=(stat.snap_totops);
Expand All @@ -440,7 +436,7 @@ void process_run(int idproc, int nproc, double ratio, int iotype, struct user_co
}

//calculate average latency milisseconds
if(begin >= ru_begin){
if(begin_time >= ru_begin){
stat.latency=(stat.latency/stat.tot_ops)/1000.0;
stat.throughput=(stat.tot_ops/((stat.endio-stat.beginio)/1.0e6));
}
Expand Down Expand Up @@ -663,24 +659,18 @@ static int config_handler(void* config, const char* section, const char* name, c
strcpy(conf->printfile, token);
}

if(conf->termination_type == TIME){
token = strtok(NULL, ":");
if(token)
conf->start = atoi(token);

token = strtok(NULL, ":");
if(token)
conf->finish = atoi(token);

}
token = strtok(NULL, ":");
if(token)
conf->start = atoi(token);

token = strtok(NULL, ":");
if(token)
conf->finish = atoi(token);

free(val);
if(conf->termination_type == TIME){
printf("Ramp up time: %d sec\n", conf->start*30);
printf("Cool down time: %d sec\n", conf->finish*30);
}

printf("Ramp up time: %d sec\n", conf->start);
printf("Cool down time: %d sec\n", conf->finish);

printf("Output of DEDISbench will be printed to './results/%s'\n", conf->printfile);
}
else if(MATCH("results","access_results")){
Expand Down Expand Up @@ -798,16 +788,15 @@ int main(int argc, char *argv[]){
.zero_copy_blocks=0, .u_count =0};

struct user_confs conf = {.destroypfile = 1, .start=0, .finish=0, .accesstype = TPCC, .iotype = -1, .testtype = -1,
.ratio = -1, .ratiow = -1, .ratior = -1, .termination_type = -1, .nprocs = 4, .filesize = 2048LLU,
.block_size = 4096LL, .populate=-1};
.ratio = -1, .ratiow = -1, .ratior = -1, .nprocs = 4, .filesize = 2048LLU,
.block_size = 4096LL, .populate=-1, .time_to_run=0, .number_ops=0};
conf.seed=tim.tv_sec*1000000+(tim.tv_usec);
bzero(conf.tempfilespath,PATH_SIZE);
bzero(conf.printfile,PATH_SIZE);
bzero(conf.accessfilelog,PATH_SIZE);
bzero(conf.rawpath,PATH_SIZE);
bzero(conf.distfile,PATH_SIZE);
bzero(conf.outputfile,PATH_SIZE);


while ((argc > 1) && (argv[1][0] == '-'))
{
Expand Down Expand Up @@ -861,29 +850,14 @@ int main(int argc, char *argv[]){
printf("Cannot use both -p and -n\n\n");
usage();}
break;

case 'm':
conf.mixedIO=1;
break;
case 't':
if(conf.termination_type!=SIZE){
conf.termination_type=TIME;
conf.time_to_run=atoi(&argv[1][2]);
}
else{
printf("Cannot use both -t and -s\n\n");
usage();
}
conf.time_to_run=atoi(&argv[1][2]);
break;
case 's':
if(conf.termination_type!=TIME){
conf.termination_type=SIZE;
conf.number_ops=atoll(&argv[1][2]);
}
else{
printf("Cannot use both -t and -s\n\n");
usage();
}
conf.number_ops=atoll(&argv[1][2]);
break;
case 'f':
if(!confarg){
Expand Down Expand Up @@ -922,23 +896,11 @@ int main(int argc, char *argv[]){
exit(0);
}
//test if testype is defined
if(conf.termination_type!=TIME && conf.termination_type!=SIZE){
printf("missing -t or -s<value>\n\n");
if(conf.number_ops<=0 && conf.time_to_run<=0){
printf("missing -t or -s<value> with value higher than 0 \n\n");
usage();
exit(0);
}
//test if time_to_run is defined and > 0
if(conf.termination_type==TIME && conf.time_to_run<0){
printf("missing -t<value> with value higher than 0\n\n");
usage();
exit(0);
}
//test if numbe_ops is defined and > 0
if(conf.termination_type==SIZE && conf.number_ops<0){
printf("missing -s<value> with value higher than 0\n\n");
usage();
exit(0);
}

//test if filesize > 0
if(conf.filesize<=0){
Expand Down Expand Up @@ -979,10 +941,10 @@ int main(int argc, char *argv[]){
conf.totblocks = conf.filesize/conf.block_size;

//convert time_to_run to seconds
if(conf.termination_type==TIME)
if(conf.time_to_run > 0)
conf.time_to_run=conf.time_to_run*60;

if(conf.termination_type==SIZE)
if(conf.number_ops > 0)
conf.number_ops=(conf.number_ops*1024*1024)/conf.block_size;


Expand Down
15 changes: 9 additions & 6 deletions README.mkd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ DEDISbench comes with three distinct distributions extracted from real storage s
- An Archival storage where most files have a write-once policy, with non-significant updates, but with sporadic data deletion. This distribution is called dist_archival.
- A Personal Files storage where some files are updated and deleted frequently and the I/O requests latency is expected to be lower than the one found in archival storages. This distributions is called dist_personalfiles.
- A High Performance storage where most files are updated and deleted frequently and I/O latency is expected to be as minimal as possible. This distribution is called dist_highperf.

- Dataset with containing 40 versions of the linux kernel (2.6.0 - 2.6.39) downloaded from https://mirrors.edge.kernel.org/pub/linux/kernel/v2.6/. This distribution is called dist_kernels.
- Dataset containing 33 Ubuntu images downloaded from https://cloud-images.ubuntu.com/. This dataset This distribution is called dist_ubuntuamd.

All these distributions are available and can be simulated by DEDISbench.
NOTE: By default DEDISbench uses the Personal Files Storage distribution.

Expand Down Expand Up @@ -61,7 +63,7 @@ Required flags

-w or -r or -m Write or Read Benchmark or a Mix of write and read operations.

-t`value` or -s`value` Benchmark duration (-t) in Minutes or amount of data to write (-s) in MB
-t`value` or -s`value` Benchmark duration (-t) in Minutes or amount of data to write (-s) in MB. If both are chosen, the first condition to be reached will end the benchmark.

# Configuration file options

Expand All @@ -70,8 +72,9 @@ Required flags
## [execution] section

distfile=`value` Input File with duplicate distribution default:internal file conf/dist_personalfiles
DEDISbench can simulate three real distributions extracted respectively from an Archival, Personal Files and High Performance Storage;
For choosing these distributions the `value` must be dist_archival, dist_personalfiles or dist_highperf respectively.
DEDISbench can simulate five real distributions.
For choosing these distributions the `value` must be dist_archival, dist_personalfiles, dist_highperf,
dist_kernels or dist_ubuntuamd respectively.
The input file details the amount of blocks with a certain number of duplicates
and the format is: <number_duplicates> <number_blocks>
See below for more info for customizing distribution files and above for info on the default distributions.
Expand Down Expand Up @@ -126,8 +129,8 @@ Required flags
general_results=`file`:RU:CD Write to file path the output of DEDISbench. This feature also writes two additional files with the same name
as given in `file` and a snaplat and snapthr suffix that shows the throughput and latency average values
for 30 seconds intervals.
RU is the ramp up time in half minutes.
CD is the cool down time also half minutes.
RU is the ramp up time in seconds.
CD is the cool down time in seconds.
It also writes the necessary files to plot a graph of both throughput and latency, with gnuplot.

## [structure] section
Expand Down
6 changes: 3 additions & 3 deletions conf/defconf.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@

[execution]
# Input File with duplicate distribution default:internal file called dist_personalfiles.
# DEDISbench can simulate three real distributions extracted respectively from an Archival, Personal Files and High Performance Storage; For choosing these distributions the `value` must be dist_archival, dist_personalfiles or dist_highperf respectively.
# DEDISbench can simulate five real distributions (see documentation for more info). For choosing these distributions the `value` must be dist_archival, dist_personalfiles, dist_highperf, dist_ubuntuamd or dist_kernels respectively.
#The input file details the amount of blocks with a certain number of duplicates and the format is: <number_duplicates> <number_blocks>
distfile=conf/dist_personalfiles

# Access pattern for I/O operations: 0-sequential | 1-uniform | 2-TPCC(default)
access_type=2

# Number of concurrent processes default:4. Each process has an independent file associated or a common device if -i flag is used
nprocs=2
nprocs=1

# Size of the file of each process in MB. If rawdevice option is used, this parameter defines the size of the raw device. default:2048 MB
filesize=2048
Expand Down Expand Up @@ -47,7 +47,7 @@ blocksize=4096
# I/O latency results are written to a log file to extract additional statistics. Each process writes these values in a file called result<processid> and each line, corresponds to a single I/O operations and presents: (latency of I/O operation in microseconds) (current time in seconds).
#logging=0

# Write to file path the output of DEDISbench. This feature also writes two additional files with the same name as given in 'out' and a snaplat and snapthr suffix that shows the throughput and latency average values for 30 seconds intervals. It also writes the necessary files to plot a graph of both throughput and latency, with gnuplot. Ramp up e cool down values should only be used when the flag -t<value> is used
# Write to file path the output of DEDISbench. This feature also writes two additional files with the same name as given in 'out' and a snaplat and snapthr suffix that shows the throughput and latency average values for 30 seconds intervals. It also writes the necessary files to plot a graph of both throughput and latency, with gnuplot.
general_results=out:0:0
#general_results=out:0:5
#general_results=out:5:5
Expand Down
Loading

0 comments on commit 0956b9d

Please sign in to comment.