Add test-workload for testing setup

perfmetrics/scripts/testing_on_gke/examples/workloads_test.json

@@ -0,0 +1,59 @@
+{
+ "_comment": "_ in the starting of element name indicates comment.",
+ "TestConfig": {
+ "workloadConfig": {
+ "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+ "runOnSSD": false,
+ "workloads": [
+ {
+ "_description": "This is a dummy fio workload (missing the 'fioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a fio workload, it must have a valid 'fioWorkload', a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.",
+ "_fioWorkload": {
+ "_description": "Every fioWorkload must have fileSize, filesPerThread, numThreads, and blockSize fields. readTypes is an array of string values 'read' and 'randread'. If readTypes is missing, then it defaults to [\"read\",\"randread\"].",
+ "fileSize": "64K",
+ "filesPerThread": 20000,
+ "numThreads": 50,
+ "blockSize": "64K",
+ "readTypes": ["read","randread"]
+ },
+ "gcsfuseMountOptions": "GCSFuse mount-options, in a compact stringified format, to be used for the test scenario gcsfuse-generic. The individual config/cli flag values should be separated by comma. Each cli flag should be of the form <flag>[=<value>], while each config-file flag should be of form <config>[:<subconfig>[:<subsubconfig>[...]]]:<value>. For example, a legal value would be: implicit-dirs,file_mode=777,file-cache:enable-parallel-downloads:true,metadata-cache:ttl-secs:-1 .",
+ "bucket":"The bucket must have objects with name Workload.{i}/{j} for every i,j where i:0-{numThreads}-1, j:0-{filesPerThread}-1, and each of these objects must be of size {fileSize}. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration."
+ },
+ {
+ "fioWorkload": {
+ "fileSize": "64K",
+ "filesPerThread": 100,
+ "numThreads": 20,
+ "blockSize": "64K",
+ "readTypes": ["randread"]
+ },
+ "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+ "bucket":"fio-64k-1m-us-west1",
+ "_bucket_alt2":"fio-64k-1m-us-central1",
+ "_bucket_alt3":"gke-fio-64k-1m"
+ },
+ {
+ "_description": "This is a dummy dlio workload (missing the 'dlioWorkload' field), purely standing as a header and does not execute any workload. For it to execute a dlio workload, it must have a valid 'dlioWorkload' object and a valid 'bucket' attribute, and a valid gcsfuseMountOption attribute.",
+ "_dlioWorkload": {
+ "_description": "Every dlioWorkload must have numFilesTrain, recordLength, and batchSizes fields. batchSizes is an array of integer values",
+ "numFilesTrain": 500000,
+ "recordLength": 102400,
+ "batchSizes": [800,128]
+ },
+ "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+ "bucket":"The bucket must have objects with name 'train/', 'valid/', and train/img_{i}_of_{numFilesTrain}.npz for every i where i:0-{numFilesTrain}-1 and each train/img_{i}_of_{numFilesTrain}.npz must be of size {recordLength} bytes. The buckets gke-* are all in us-central1, are owned by GKE team and are in their GCP project(s). For best performance, please ensure that the bucket is in the same google-cloud region and GCP project as that of the GKE cluster used for running this test configuration."
+ },
+ {
+ "dlioWorkload": {
+ "numFilesTrain": 1000,
+ "recordLength": 3145728,
+ "batchSizes": [200]
+ },
+ "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+ "bucket":"dlio-unet3d-3mb-100k-us-west1",
+ "_bucket_alt2":"dlio-unet3d-3mb-100k-us-central1",
+ "_bucket_alt3":"gke-dlio-unet3d-3mb-100k"
+ }
+ ]
+ }
+ }
+}