dev <- Mlperf inference

.github/workflows/test-mlperf-inference-abtf-poc.yml

@@ -37,7 +37,7 @@ jobs:
  cm pull repo mlcommons@cm4abtf --branch=poc
  - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker
  run: |
- cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=5 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v
+ cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v
 
  build2:
  runs-on: ${{ matrix.os }}
@@ -62,7 +62,7 @@ jobs:
  cm pull repo mlcommons@cm4abtf --branch=poc
  - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
  run: |
- cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v
+ cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --adr.compiler.tags=gcc --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 --quiet -v
 
  build3:
  runs-on: ${{ matrix.os }}
@@ -89,4 +89,4 @@ jobs:
  cm pull repo mlcommons@cm4abtf --branch=poc
  - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }}
  run: |
- cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 -v
+ cm run script --tags=run-abtf,inference,_poc-demo --test_query_count=2 --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off --adr.cocoeval.version_max=1.5.7 --adr.cocoeval.version_max_usable=1.5.7 -v

automation/script/module.py

@@ -351,15 +351,16 @@ def _run(self, i):
 
  debug_uid = i.get('debug_uid', '')
  if debug_uid!='':
- env['CM_TMP_DEBUG_UID'] = debug_uid
+ r = _update_env(env, 'CM_TMP_DEBUG_UID', debug_uid)
+ if r['return']>0: return r
 
  fake_deps = i.get('fake_deps', False)
  if fake_deps: env['CM_TMP_FAKE_DEPS']='yes'
 
  if str(i.get('skip_sys_utils', '')).lower() in ['true', 'yes']:
- env['CM_SKIP_SYS_UTILS']='yes' 
+ env['CM_SKIP_SYS_UTILS']='yes'
  if str(i.get('skip_sudo', '')).lower() in ['true', 'yes']:
- env['CM_TMP_SKIP_SUDO']='yes' 
+ env['CM_TMP_SKIP_SUDO']='yes'
 
  run_state = i.get('run_state', self.run_state)
  if not run_state.get('version_info', []):
@@ -387,9 +388,9 @@ def _run(self, i):
  elif 'v' in i: verbose=i['v']
 
  if verbose:
- env['CM_VERBOSE']='yes'
- run_state['tmp_verbose']=True
- logging.getLogger().setLevel(logging.DEBUG)
+  env['CM_VERBOSE']='yes'
+  run_state['tmp_verbose']=True
+  logging.getLogger().setLevel(logging.DEBUG)
 
 
  print_deps = i.get('print_deps', False)
@@ -418,7 +419,8 @@ def _run(self, i):
 
  # Detect current path and record in env for further use in native scripts
  current_path = os.path.abspath(os.getcwd())
- env['CM_TMP_CURRENT_PATH'] = current_path
+ r = _update_env(env, 'CM_TMP_CURRENT_PATH', current_path)
+ if r['return']>0: return r
 
  # Check if quiet mode
  quiet = i.get('quiet', False) if 'quiet' in i else (env.get('CM_QUIET','').lower() == 'yes')
@@ -472,6 +474,9 @@ def _run(self, i):
  if value != '':
  env['CM_' + key.upper()] = value
 
+ r = update_env_with_values(env)
+ if r['return']>0: return r 
+
 
  ############################################################################################################
  # Check if we want to skip cache (either by skip_cache or by fake_run)
@@ -1317,7 +1322,8 @@ def _run(self, i):
 
  logging.debug(recursion_spaces+' - Version is not specified - use either default_version from meta or min/max/usable: {}'.format(version))
 
- env['CM_VERSION'] = version
+ r = _update_env(env, 'CM_VERSION', version)
+ if r['return']>0: return r
 
  if 'version-'+version not in cached_tags: cached_tags.append('version-'+version)
 
@@ -1329,8 +1335,9 @@ def _run(self, i):
  if "add_deps_recursive" in versions_meta:
  self._merge_dicts_with_tags(add_deps_recursive, versions_meta['add_deps_recursive'])
 
- env['CM_TMP_CURRENT_SCRIPT_PATH'] = path
-
+ r = _update_env(env, 'CM_TMP_CURRENT_SCRIPT_PATH', path)
+ if r['return']>0: return r
+
  # Run chain of docker dependencies if current run cmd is from inside a docker container
  docker_deps = []
  if i.get('docker_run_deps'):
@@ -1444,7 +1451,8 @@ def _run(self, i):
  elif pip_version_max != '':
  pip_version_string = '<='+pip_version_max
 
- env['CM_TMP_PIP_VERSION_STRING'] = pip_version_string
+ r = _update_env(env, 'CM_TMP_PIP_VERSION_STRING', pip_version_string)
+ if r['return']>0: return r
  if pip_version_string != '':
  logging.debug(recursion_spaces+' # potential PIP version string (if needed): '+pip_version_string)
 
@@ -4359,6 +4367,20 @@ def any_enable_or_skip_script(meta, env):
 
  return False
 
+############################################################################################################
+def _update_env(env, key=None, value=None):
+ if key == None or value == None:
+ return {'return': 1, 'error': 'None value not expected in key and value arguments in _update_env.'}
+ if not isinstance(key, str):
+ return {'return': 1, 'error': 'String value expected inside key argument.'}
+
+ env[key] = value
+
+ r = update_env_with_values(env)
+ if r['return']>0: return r 
+
+ return {'return': 0}
+
 ############################################################################################################
 def update_env_with_values(env, fail_on_not_found=False, extra_env={}):
  """
@@ -4532,9 +4554,12 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
  path = '"' + path + '"'
 
  cur_dir = os.getcwd()
-
- env['CM_TMP_CURRENT_SCRIPT_PATH'] = path
- env['CM_TMP_CURRENT_SCRIPT_WORK_PATH'] = cur_dir
+
+ r = _update_env(env, 'CM_TMP_CURRENT_SCRIPT_PATH', path)
+ if r['return']>0: return r
+
+ r = _update_env(env, 'CM_TMP_CURRENT_SCRIPT_WORK_PATH', cur_dir)
+ if r['return']>0: return r
 
  # Record state
  if tmp_file_state != '':

script/app-mlperf-inference-nvidia/_cm.yaml

@@ -783,7 +783,7 @@ variations:
  CM_MLPERF_NVIDIA_HARNESS_MAXN: yes
 
  preprocess-data:
- alias: preprocess-data
+ alias: preprocess_data
 
  preprocess_data:
  group: run-mode

script/download-file/_cm.json

@@ -83,6 +83,19 @@
  "deps": [
  {
  "tags": "get,rclone"
+ },
+ {
+ "tags": "get,rclone-config",
+ "update_tags_from_env_with_prefix": {
+ "_": [
+ "CM_RCLONE_CONFIG_NAME"
+ ]
+ },
+ "enable_if_env": {
+ "CM_RCLONE_CONFIG_NAME": [
+ "on"
+ ]
+ }
  }
  ],
  "env": {

script/download-file/customize.py

@@ -159,7 +159,7 @@ def preprocess(i):
  env['CM_DOWNLOAD_CMD'] += f" || ((rm -f {env['CM_DOWNLOAD_FILENAME']} || true) && gdown {extra_download_options} {url})"
 
  elif tool == "rclone":
- if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
+ if env.get('CM_RCLONE_CONFIG_CMD', '') != '': #keeping this for backward compatibility. Ideally should be done via get,rclone-config script
  env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD']
  rclone_copy_using = env.get('CM_RCLONE_COPY_USING', 'sync')
  if rclone_copy_using == "sync":
@@ -168,19 +168,9 @@ def preprocess(i):
  # have to modify the variable from url to temp_url if it is going to be used anywhere after this point
  url = url.replace("%", "%%")
  temp_download_file = env['CM_DOWNLOAD_FILENAME'].replace("%", "%%")
- env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P"
+ env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P --error-on-no-transfer"
  else:
- env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P"
- for i in range(1,5):
- url = env.get('CM_DOWNLOAD_URL'+str(i),'')
- if url == '':
- break
- if env["CM_HOST_OS_TYPE"] == "windows":
- url = url.replace("%", "%%")
- temp_download_file = env['CM_DOWNLOAD_FILENAME'].replace("%", "%%")
- env['CM_DOWNLOAD_CMD'] = f" || ((rm -f {env['CM_DOWNLOAD_FILENAME']} || true) && rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P)"
- else:
- env['CM_DOWNLOAD_CMD'] = f" || ((rm -f {env['CM_DOWNLOAD_FILENAME']} || true) && rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P"
+ env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P --error-on-no-transfer"
 
  filename = env['CM_DOWNLOAD_FILENAME']
  env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename

script/download-file/run.sh

@@ -35,8 +35,10 @@ fi
 
 if [[ ${require_download} == "1" ]]; then
  echo ""
- echo ${CM_PRE_DOWNLOAD_CLEAN_CMD}
- ${CM_PRE_DOWNLOAD_CLEAN_CMD}
+ if [ -e "${CM_PRE_DOWNLOAD_CLEAN}" ]; then
+ echo ${CM_PRE_DOWNLOAD_CLEAN_CMD}
+ ${CM_PRE_DOWNLOAD_CLEAN_CMD}
+ fi
 
  echo ""
  echo "${CM_DOWNLOAD_CMD}"

script/get-ml-model-dlrm-terabyte/_cm.json

@@ -101,7 +101,7 @@
  },
  "pytorch,fp32,weight_sharded,rclone": {
  "env": {
- "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
+ "CM_RCLONE_CONFIG_NAME": "mlc-inference",
  "CM_PACKAGE_URL": "mlc-inference:mlcommons-inference-wg-public/model_weights"
  }
  },

script/get-ml-model-gptj/_cm.json

@@ -87,7 +87,7 @@
  "CM_UNZIP": "yes",
  "CM_DOWNLOAD_CHECKSUM_NOT_USED": "e677e28aaf03da84584bb3073b7ee315",
  "CM_PACKAGE_URL": "https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download",
- "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
+ "CM_RCLONE_CONFIG_NAME": "mlc-inference",
  "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-public/gpt-j"
  },
  "required_disk_space": 22700

script/get-ml-model-stable-diffusion/_cm.json

@@ -160,7 +160,7 @@
  "rclone": {
  "group": "download-tool",
  "env": {
- "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
+ "CM_RCLONE_CONFIG_NAME": "mlc-inference",
  "CM_DOWNLOAD_TOOL": "rclone"
  },
  "adr": {

script/get-preprocessed-dataset-criteo/_cm.json

@@ -191,7 +191,7 @@
  ],
  "extra_cache_tags": "criteo,preprocessed,dataset",
  "env": {
- "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
+ "CM_RCLONE_CONFIG_NAME": "mlc-inference",
  "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-public/dlrm_preprocessed",
  "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_DATASET_PREPROCESSED_PATH",
  "CM_EXTRACT_FINAL_ENV_NAME": "CM_DATASET_PREPROCESSED_PATH",

script/get-preprocessed-dataset-openorca/_cm.json

@@ -143,15 +143,15 @@
  "mlcommons": {
  "env": {
  "CM_DATASET_PREPROCESSED_BY_MLC": "yes",
- "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com",
  "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-public/open_orca"
  },
  "deps": [
  {
  "env": {
  "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_OPENORCA_PREPROCESSED_ROOT",
  "CM_EXTRACT_FINAL_ENV_NAME": "CM_OPENORCA_PREPROCESSED_ROOT",
- "CM_EXTRACT_TO_FOLDER": "openorca-preprocessed"
+ "CM_EXTRACT_TO_FOLDER": "openorca-preprocessed",
+ "CM_RCLONE_CONFIG_NAME": "mlc-inference"
  },
  "tags": "download-and-extract,_rclone",
  "update_tags_from_env_with_prefix": {

script/get-rclone-config/_cm.yaml

@@ -0,0 +1,13 @@
+alias: get-rclone-config
+automation_alias: script
+automation_uid: 5b4e0237da074764
+cache: false #keeping cache off as rerunning the command is safe
+can_force_cache: true
+tags:
+- get
+- rclone-config
+uid: 6c59ddbc6cd046e3
+variations:
+ mlc-inference:
+ env:
+ CM_RCLONE_CONFIG_CMD: 'rclone config create mlc-inference s3 provider=Cloudflare access_key_id=f65ba5eef400db161ea49967de89f47b secret_access_key=fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b endpoint=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com'

script/get-rclone-config/customize.py

@@ -0,0 +1,25 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+ os_info = i['os_info']
+
+ env = i['env']
+
+ meta = i['meta']
+
+ automation = i['automation']
+
+ quiet = (env.get('CM_QUIET', False) == 'yes')
+
+ if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
+ env['CM_RUN_CMD'] = env['CM_RCLONE_CONFIG_CMD']
+
+ return {'return':0}
+
+def postprocess(i):
+
+ env = i['env']
+
+ return {'return':0}

script/get-rclone-config/run.bat

@@ -0,0 +1 @@
+rem native script

script/get-rclone-config/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#CM Script location: ${CM_TMP_CURRENT_SCRIPT_PATH}
+
+#To export any variable
+#echo "VARIABLE_NAME=VARIABLE_VALUE" >>tmp-run-env.out
+
+#${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
+
+echo "Running: "
+echo "${CM_RUN_CMD}"
+echo ""
+
+if [[ ${CM_FAKE_RUN} != "yes" ]]; then
+ eval "${CM_RUN_CMD}"
+ test $? -eq 0 || exit 1
+fi