paper.bib

@INPROCEEDINGS{6546066,
  author={J. M. {Wozniak} and T. G. {Armstrong} and M. {Wilde} and D. S. {Katz} and E. {Lusk} and I. T. {Foster}},
  booktitle={2013 13th IEEE/ACM International Symposium on Cluster, Cloud, and Grid Computing},
  title={{Swift/T: Large-Scale Application Composition via Distributed-Memory Dataflow Processing}},
  year={2013},
  volume={},
  number={},
  pages={95-102},
}

  @inproceedings{slawinska2013maya,
  title={{A Maya use case: adaptable scientific workflows with ADIOS for general relativistic astrophysics}},
  author={Slawinska, Magdalena and Clark, Michael and Wolf, Matthew and Bode, Tanja and Zou, Hongbo and Laguna, Pablo and Logan, Jeremy and Kinsey, Matthew and Klasky, Scott},
  booktitle={Proceedings of the Conference on Extreme Science and Engineering Discovery Environment: Gateway to Discovery},
  pages={1--8},
  year={2013}
}

@misc{2019_3361225,
  title        = {{Final implementation of the Earth System Data Middleware (ESDM) (Deliverable 4.3 -- ESiWACE)}},
  month        = jun,
  year         = 2019,
  publisher    = {Zenodo},
  doi          = {10.5281/zenodo.3361225},
  url          = {https://doi.org/10.5281/zenodo.3361225}
}

@article{rajasekar2010irods,
  title={{iRODS primer: integrated rule-oriented data system}},
  author={Rajasekar, Arcot and Moore, Reagan and Hou, Chien-yi and Lee, Christopher A and Marciano, Richard and de Torcy, Antoine and Wan, Michael and Schroeder, Wayne and Chen, Sheau-Yen and Gilbert, Lucas and others},
  journal={Synthesis Lectures on Information Concepts, Retrieval, and Services},
  volume={2},
  number={1},
  pages={1--143},
  year={2010},
  publisher={Morgan \& Claypool Publishers}
}

@inproceedings{miranda2019norns,
  title={{NORNS: extending Slurm to support data-driven workflows through asynchronous data staging}},
  author={Miranda, Alberto and Jackson, Adrian and Tocci, Tommaso and Panourgias, Iakovos and Nou, Ramon},
  booktitle={2019 IEEE International Conference on Cluster Computing (CLUSTER)},
  pages={1--12},
  year={2019},
  organization={IEEE}
}

@inproceedings{subedi2019leveraging,
  title={Leveraging machine learning for anticipatory data delivery in extreme scale in-situ workflows},
  author={Subedi, Pradeep and Davis, Philip E and Parashar, Manish},
  booktitle={2019 IEEE International Conference on Cluster Computing (CLUSTER)},
  pages={1--11},
  year={2019},
  organization={IEEE}
}

@article{kougkas2020acceleration,
  title={{I/O acceleration via multi-tiered data buffering and prefetching}},
  author={Kougkas, Anthony and Devarajan, Hariharan and Sun, Xian-He},
  journal={Journal of Computer Science and Technology},
  volume={35},
  number={1},
  pages={92--120},
  year={2020},
  publisher={Springer}
}

@article{deelman2019role,
  title={The role of machine learning in scientific workflows},
  author={Deelman, Ewa and Mandal, Anirban and Jiang, Ming and Sakellariou, Rizos},
  journal={The International Journal of High Performance Computing Applications},
  volume={33},
  number={6},
  pages={1128--1139},
  year={2019},
  publisher={SAGE Publications Sage UK: London, England}
}

@inproceedings{subedi2018stacker,
  title={Stacker: an autonomic data movement engine for extreme-scale data staging-based in-situ workflows},
  author={Subedi, Pradeep and Davis, Philip and Duan, Shaohua and Klasky, Scott and Kolla, Hemanth and Parashar, Manish},
  booktitle={SC18: International Conference for High Performance Computing, Networking, Storage and Analysis},
  pages={920--930},
  year={2018},
  organization={IEEE}
}

@article{alkhanak2016cost,
  title={Cost optimization approaches for scientific workflow scheduling in cloud and grid computing: a review, classifications, and open issues},
  author={Alkhanak, Ehab Nabiel and Lee, Sai Peck and Rezaei, Reza and Parizi, Reza Meimandi},
  journal={Journal of Systems and Software},
  volume={113},
  pages={1--26},
  year={2016},
  publisher={Elsevier}
}

@inproceedings{ozik2016desktop,
  title={{From desktop to large-scale model exploration with Swift/T}},
  author={Ozik, Jonathan and Collier, Nicholson T and Wozniak, Justin M and Spagnuolo, Carmine},
  booktitle={2016 Winter Simulation Conference (WSC)},
  pages={206--220},
  year={2016},
  organization={IEEE}
}

@inproceedings{isard2007dryad,
  title={Dryad: distributed data-parallel programs from sequential building blocks},
  author={Isard, Michael and Budiu, Mihai and Yu, Yuan and Birrell, Andrew and Fetterly, Dennis},
  booktitle={Proceedings of the 2nd ACM SIGOPS/EuroSys European Conference on Computer Systems 2007},
  pages={59--72},
  year={2007}
}

@article{dai2018cross,
  author = {Dai, Dong and Ross, Robert and Khaldi, Dounia and Yan, Yonghong and Dorier, Matthieu and Tavakoli, Neda and Chen, Yong},
  year = {2018},
  month = {05},
  title={{A cross-layer solution in scientific workflow system for tackling data movement challenge}},
}

@article{liu2015survey,
  title={A survey of data-intensive scientific workflow management},
  author={Liu, Ji and Pacitti, Esther and Valduriez, Patrick and Mattoso, Marta},
  journal={Journal of Grid Computing},
  volume={13},
  number={4},
  pages={457--493},
  year={2015},
  publisher={Springer}
}

@article{pandey2011workflow,
  title={Workflow engine for clouds},
  author={Pandey, Suraj and Karunamoorthy, Dileban and Buyya, Rajkumar},
  journal={Cloud computing: principles and paradigms},
  volume={87},
  pages={321--344},
  year={2011},
  publisher={Wiley Online Library}
}

@article{8675433,
  author={H. {Oliver} and M. {Shin} and D. {Matthews} and O. {Sanders} and S. {Bartholomew} and A. {Clark} and B. {Fitzpatrick} and R. {van Haren} and R. {Hut} and N. {Drost}},
  journal={Computing in Science Engineering},
  title={{Workflow automation for cycling systems: the Cylc workflow engine}},
  year={2019},
  volume={21},
  number={4},
  pages={7--21},
  keywords={automation;geophysics computing;weather forecasting;workflow management software;workflow automation;cycling systems;complex cycling workflows;numerical weather prediction;environmental forecasting systems;regular intervals;forecast cycles;NWP workflow schedulers;simpler nonoverlapping sequence;single-cycle workflows;Cylc;infinite cycling workflows;historical runs;open source development;Task analysis;Weather forecasting;Computational modeling;Predictive models;Workflow management software;Real-time systems},
  doi={10.1109/MCSE.2019.2906593},
  ISSN={1558-366X},
  month={July},
}

@inproceedings{Vladimirov2014FileIO,
  title={{File I/O on Intel Xeon Phi Coprocessors: RAM disks, VirtIO, NFS and Lustre}},
  author={Andrey E. Vladimirov and V. Karpusenko and Tony Yoo},
  year={2014}
}

@inproceedings{Jette02slurm:simple,
  author = {Morris A. Jette and Andy B. Yoo and Mark Grondona},
  title = {{SLURM: Simple Linux Utility for Resource Management}},
  booktitle = {In Lecture Notes in Computer Science: Proceedings of Job Scheduling Strategies for Parallel Processing (JSSPP) 2003},
  year = {2002},
  pages = {44--60},
  publisher = {Springer-Verlag}
}

@inproceedings{esdm,
  author = {Bryan N. Lawrence and Julian M. Kunkel and Jonathan Churchill and Neil Massey and Philip Kershaw and Matt Pritchard},
  title = {Beating data bottlenecks in weather and climate science},
  booktitle = {Extreme Data Workshop -- Forschungszentrum Jülich, Proceedings, IAS series, volume 40},
  year = {2018},
  pages = {31--36}
}

@misc{xios,
  author = {Yann Meurdesoif and A Caubel and R Lacroix and J D'erouillat and M H Nguyen},
  title = {{XIOS Tutorial}},
  year = {2016},
  url = {http://forge.ipsl.jussieu.fr/ioserver/raw-attachment/wiki/WikiStart/XIOS-tutorial.pdf}
}

@MISC{netcdf,
  author = {UCAR/Unidata Program Center},
  title = {{Network Common Data Form (NetCDF)}},
  doi	 = {http://doi.org/10.5065/D6H70CW6}
}

@inproceedings{BODIAIFSFI19,
	author	 = {Eugen Betke and Julian Kunkel},
	title	 = {{Benefit of DDN's IME-Fuse and IME-Lustre file systems for I/O intensive HPC applications}},
	year	 = {2019},
	month	 = {01},
	booktitle	 = {{High Performance Computing: ISC High Performance 2018 International Workshops, Frankfurt/Main, Germany, June 28, 2018, Revised Selected Papers}},
	editor	 = {Rio Yokota and Michele Weiland and John Shalf and Sadaf Alam},
	publisher	 = {Springer},
	series	 = {Lecture Notes in Computer Science},
	number	 = {11203},
	pages	 = {131--144},
	conference	 = {WOPSSS workshop, ISC HPC},
	organization	 = {ISC Team},
	location	 = {Frankfurt, Germany},
	isbn	 = {978-3-030-02465-9},
	issn	 = {1611-3349},
	doi	 = {{https://doi.org/10.1007/978-3-030-02465-9\_9}},
	abstract	 = {Many scientific applications are limited by I/O performance offered by parallel file systems on conventional storage systems. Flash- based burst buffers provide significant better performance than HDD backed storage, but at the expense of capacity. Burst buffers are consid- ered as the next step towards achieving wire-speed of interconnect and providing more predictable low latency I/O, which are the holy grail of storage. A critical evaluation of storage technology is mandatory as there is no long-term experience with performance behavior for particular applica- tions scenarios. The evaluation enables data centers choosing the right products and system architects the integration in HPC architectures. This paper investigates the native performance of DDN-IME, a flash- based burst buffer solution. Then, it takes a closer look at the IME-FUSE file systems, which uses IMEs as burst buffer and a Lustre file system as back-end. Finally, by utilizing a NetCDF benchmark, it estimates the performance benefit for climate applications.},
}

@inproceedings{TUIBIHWLSC19,
	author	 = {Jakob Lüttgau and Shane Snyder and Philip Carns and Justin M. Wozniak and Julian Kunkel and Thomas Ludwig},
	title	 = {{Toward understanding I/O behavior in HPC workflows}},
	year	 = {2019},
	month	 = {02},
	booktitle	 = {{IEEE/ACM 3rd International Workshop on Parallel Data Storage \& Data Intensive Scalable Computing Systems (PDSW-DISCS)}},
	editor	 = {},
	publisher	 = {IEEE Computer Society},
	address	 = {Washington, DC, USA},
	pages	 = {64--75},
	conference	 = {PDSW-DISCS},
	location	 = {Dallas, Texas},
	isbn	 = {978-1-7281-0192-7},
	doi	 = {https://doi.org/10.1109/PDSW-DISCS.2018.00012},
	abstract	 = {Scientific discovery increasingly depends on complex workflows consisting of multiple phases and sometimes millions of parallelizable tasks or pipelines. These workflows access storage resources for a variety of purposes, including preprocessing, simulation output, and postprocessing steps. Unfortunately, most workflow models focus on the scheduling and allocation of computational resources for tasks while the impact on storage systems remains a secondary objective and an open research question. I/O performance is not usually accounted for in workflow telemetry reported to users. In this paper, we present an approach to augment the I/O efficiency of the individual tasks of workflows by combining workflow description frameworks with system I/O telemetry data. A conceptual architecture and a prototype implementation for HPC data center deployments are introduced. We also identify and discuss challenges that will need to be addressed by workflow management and monitoring systems for HPC in the future. We demonstrate how real-world applications and workflows could benefit from the approach, and we show how the approach helps communicate performance-tuning guidance to users.},
}

@article{TDTSOCAFQC17,
	author	 = {Julian Kunkel and Anastasiia Novikova and Eugen Betke},
	title	 = {{Towards decoupling the selection of compression algorithms from quality constraints -- an investigation of lossy compression efficiency}},
	year	 = {2017},
	month	 = {12},
	editor	 = {Jack Dongarra and Vladimir Voevodin},
	journal	 = {Supercomputing Frontiers and Innovations},
	series	 = {Volume 4, Number 4},
	pages	 = {17--33},
	doi	 = {https://doi.org/10.14529/jsfi170402},
	abstract	 = {Data intense scientific domains use data compression to reduce the storage space needed. Lossless data compression preserves information accurately but lossy data compression can achieve much higher compression rates depending on the tolerable error margins. There are many ways of defining precision and to exploit this knowledge, therefore, the field of lossy compression is subject to active research. From the perspective of a scientist, the qualitative definition about the implied loss of data precision should only matter.With the Scientific Compression Library (SCIL), we are developing a meta-compressor that allows users to define various quantities for acceptable error and expected performance behavior. The library then picks a suitable chain of algorithms yielding the user's requirements, the ongoing work is a preliminary stage for the design of an adaptive selector. This approach is a crucial step towards a scientifically safe use of much-needed lossy data compression, because it disentangles the tasks of determining scientific characteristics of tolerable noise, from the task of determining an optimal compression strategy. Future algorithms can be used without changing application code. In this paper, we evaluate various lossy compression algorithms for compressing different scientific datasets (Isabel, ECHAM6), and focus on the analysis of synthetically created data that serves as blueprint for many observed datasets. We also briefly describe the available quantities of SCIL to define data precision and introduce two efficient compression algorithms for individual data points. This shows that the best algorithm depends on user settings and data properties.},
	url	 = {http://superfri.org/superfri/article/view/149},
}

@article{1056220,
  author={M. {Hellman}},
  journal={IEEE Transactions on Information Theory},
  title={A cryptanalytic time-memory trade-off},
  year={1980},
  volume={26},
  number={4},
  pages={401--406},
  keywords={Cryptography},
  doi={10.1109/TIT.1980.1056220},
  ISSN={1557-9654},
  month={July},
}

@inproceedings{jimenez2017popper,
  title={The popper convention: making reproducible systems evaluation practical},
  author={Jimenez, Ivo and Sevilla, Michael and Watkins, Noah and Maltzahn, Carlos and Lofstead, Jay and Mohror, Kathryn and Arpaci-Dusseau, Andrea and Arpaci-Dusseau, Remzi},
  booktitle={Parallel and Distributed Processing Symposium Workshops (IPDPSW), 2017 IEEE International},
  pages={1561--1570},
  year={2017},
  organization={IEEE}
}

@article{bts480,
  author = {Köster, Johannes and Rahmann, Sven},
  title = {Snakemake: a scalable bioinformatics workflow engine},
  journal = {Bioinformatics},
  volume = {28},
  number = {19},
  pages = {2520--2522},
  year = {2012},
  month = {08},
  abstract = "{Summary: Snakemake is a workflow engine that provides a readable Python-based workflow definition language and a powerful execution environment that scales from single-core workstations to compute clusters without modifying the workflow. It is the first system to support the use of automatically inferred multiple named wildcards (or variables) in input and output filenames.Availability:http://snakemake.googlecode.com.Contact:johannes.koester@uni-due.de}",
  issn = {1367-4803},
  doi = {10.1093/bioinformatics/bts480},
  url = {https://doi.org/10.1093/bioinformatics/bts480},
  eprint = {https://academic.oup.com/bioinformatics/article-pdf/28/19/2520/819790/bts480.pdf},
}

@article{Nextflow,
  author = {Di Tommaso, Paolo and Chatzou, Maria and Floden, Evan W. and Barja, Pablo and Palumbo, Emilio and Notredame, Cedric},
  year = {2017},
  month = {04},
  pages = {316--319},
  title = {Nextflow enables reproducible computational workflows},
  volume = {35},
  journal = {Nature Biotechnology},
  doi = {10.1038/nbt.3820}
}

@article{RomanusRP15,
  author    = {Melissa Romanus and
               Robert B. Ross and
               Manish Parashar},
  title     = {Challenges and considerations for utilizing burst buffers in high-performance
               computing},
  journal   = {CoRR},
  volume    = {abs/1509.05492},
  year      = {2015},
  url       = {http://arxiv.org/abs/1509.05492},
  archivePrefix = {arXiv},
  eprint    = {1509.05492},
  timestamp = {Mon, 13 Aug 2018 16:47:05 +0200},
  biburl    = {https://dblp.org/rec/journals/corr/RomanusRP15.bib},
  bibsource = {dblp computer science bibliography, https://dblp.org}
}

@inproceedings{SchmuckH02,
  added-at = {2002-08-28T00:00:00.000+0200},
  author = {Schmuck, Frank B. and Haskin, Roger L.},
  biburl = {https://www.bibsonomy.org/bibtex/23ecf747f0ed8355badee2be26e55a241/dblp},
  booktitle = {FAST},
  editor = {Long, Darrell D. E.},
  ee = {http://www.usenix.org/publications/library/proceedings/fast02/schmuck.html},
  interhash = {385458bd477bfce0faecf737b124ab25},
  intrahash = {3ecf747f0ed8355badee2be26e55a241},
  isbn = {1-880446-03-0},
  keywords = {dblp},
  pages = {231--244},
  publisher = {USENIX},
  timestamp = {2018-07-05T11:57:15.000+0200},
  title = {{GPFS: a shared-disk file system for large computing clusters}},
  url = {http://dblp.uni-trier.de/db/conf/fast/fast2002.html#SchmuckH02},
  year = {2002}
}

@inproceedings{528214,
  author={R. W. {Watson} and R. A. {Coyne}},
  booktitle={Proceedings of IEEE 14th Symposium on Mass Storage Systems},
  title={{The parallel I/O architecture of the high-performance storage system (HPSS)}},
  year={1995},
  volume={},
  number={},
  pages={27--44},
  keywords={application program interfaces;digital storage;transport protocols;file organisation;parallel I/O architecture;high-performance storage system;terabyte size;petabyte total capacities;storage-system performance;system functionality;peripheral-to-peripheral transfers;remote file transfers;parallel transport protocol;parallel client application programming interface;local parallel file system;Laboratories;Network servers;Government;System performance;Computer architecture;Large-scale systems;US Department of Energy;Educational institutions;Asynchronous transfer mode;Access protocols},
  doi={10.1109/MASS.1995.528214},
  ISSN={1051-9173},
  month={Sep.},
}

@inproceedings{beegfs,
  author={Fahim Chowdhury and Yue Zhu and Todd Heer and Saul Paredes and Adam T Moody and Robin Goldstone and Kathryn M Mohror and Weikuan Yu},
  booktitle={ICPP 2019: Proceedings of the 48th International Conference on Parallel Processing},
  title={{The parallel I/O architecture of the high-performance storage system (HPSS)}},
  year={2019},
  pages={1--10},
  doi={10.1145/3337821.3337902},
  month={August},
}

@article{abs-1903-01955,
  author    = {Peter Braam},
  title     = {{The Lustre storage architecture}},
  journal   = {CoRR},
  volume    = {abs/1903.01955},
  year      = {2019},
  url       = {http://arxiv.org/abs/1903.01955},
  archivePrefix = {arXiv},
  eprint    = {1903.01955},
  timestamp = {Sat, 30 Mar 2019 19:27:21 +0100},
  biburl    = {https://dblp.org/rec/journals/corr/abs-1903-01955.bib},
}