Merge pull request #345 from GATEOverflow/mlperf-inference

Merge from go

.github/workflows/test-mlperf-inference-resnet50.yml

@@ -42,6 +42,7 @@ jobs:
  - name: Test MLPerf Inference ResNet50 (Windows)
  if: matrix.os == 'windows-latest'
  run: |
+ git config --system core.longpaths true
  cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_windows --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
  - name: Test MLPerf Inference ResNet50 (Linux/macOS)
  if: matrix.os != 'windows-latest'
@@ -53,12 +54,12 @@ jobs:
  USER: "GitHub Action"
  EMAIL: "[email protected]"
  run: |
- git config --global user.name "$USER"
- git config --global user.email "$EMAIL"
+ git config --global user.name "${{ env.USER }}"
+ git config --global user.email "${{ env.EMAIL }}"
  git config --global credential.https://github.com.helper ""
  git config --global credential.https://github.com.helper "!gh auth git-credential"
  git config --global credential.https://gist.github.com.helper ""
  git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
 
  cm run script --tags=auth,gh,cli --with_token="${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}"
- cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action" --quiet
+ cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet

script/authenticate-github-cli/run.sh

@@ -7,8 +7,8 @@
 
 #${CM_PYTHON_BIN_WITH_PATH} contains the path to python binary if "get,python" is added as a dependency
 
-echo "Running: "
-echo "${CM_RUN_CMD}"
+echo "Running gh auth: " #Not printing as it can contain secret
+#echo "${CM_RUN_CMD}"
 echo ""
 
 if [[ ${CM_FAKE_RUN} != "yes" ]]; then

script/benchmark-program-mlperf/customize.py

@@ -16,13 +16,40 @@ def postprocess(i):
 
 
  if env.get('CM_MLPERF_POWER', '') == "yes":
- if os_info['platform'] == 'windows':
- return {'return':1, 'error':'TBD: this script is not yet supported on Windows'}
-
+
  if env.get('CM_MLPERF_SHORT_RANGING_RUN', '') != 'no':
- os.system("echo '0' > "+env.get('CM_RUN_DIR','')+ "/count.txt")
- env['CM_MLPERF_RUN_CMD'] = "CM_MLPERF_RUN_COUNT=\$(cat \${CM_RUN_DIR}/count.txt); echo \${CM_MLPERF_RUN_COUNT}; CM_MLPERF_RUN_COUNT=\$((CM_MLPERF_RUN_COUNT+1)); echo \${CM_MLPERF_RUN_COUNT} > \${CM_RUN_DIR}/count.txt && if [ \${CM_MLPERF_RUN_COUNT} -eq \'1\' ]; then export CM_MLPERF_USER_CONF=\${CM_MLPERF_RANGING_USER_CONF}; else export CM_MLPERF_USER_CONF=\${CM_MLPERF_TESTING_USER_CONF}; fi && "+env.get('CM_RUN_CMD','').strip()
+ # Write '0' to the count.txt file in CM_RUN_DIR
+ count_file = os.path.join(env.get('CM_RUN_DIR', ''), 'count.txt')
+ with open(count_file, 'w') as f:
+ f.write('0')
+
+ if os_info['platform'] != 'windows':
+ # Construct the shell command with proper escaping
+ env['CM_MLPERF_RUN_CMD'] = (
+ "CM_MLPERF_RUN_COUNT=$(cat ${CM_RUN_DIR}/count.txt); "
+ "echo ${CM_MLPERF_RUN_COUNT}; "
+ "CM_MLPERF_RUN_COUNT=$((CM_MLPERF_RUN_COUNT+1)); "
+ "echo ${CM_MLPERF_RUN_COUNT} > ${CM_RUN_DIR}/count.txt && "
+ "if [ ${CM_MLPERF_RUN_COUNT} -eq 1 ]; then "
+ "export CM_MLPERF_USER_CONF=${CM_MLPERF_RANGING_USER_CONF}; "
+ "else "
+ "export CM_MLPERF_USER_CONF=${CM_MLPERF_TESTING_USER_CONF}; "
+ "fi && " + env.get('CM_RUN_CMD', '').strip()
+ )
+ else:
+ env['CM_MLPERF_RUN_CMD'] = (
+ "set /P CM_MLPERF_RUN_COUNT=<\"%CM_RUN_DIR%\\count.txt\" & "
+ "echo %CM_MLPERF_RUN_COUNT% & "
+ "set /A CM_MLPERF_RUN_COUNT=!CM_MLPERF_RUN_COUNT!+1 & "
+ "echo !CM_MLPERF_RUN_COUNT! > \"%CM_RUN_DIR%\\count.txt\" & "
+ "if !CM_MLPERF_RUN_COUNT! EQU 1 ( "
+ "set CM_MLPERF_USER_CONF=%CM_MLPERF_RANGING_USER_CONF% "
+ ") else ( "
+ "set CM_MLPERF_USER_CONF=%CM_MLPERF_TESTING_USER_CONF% "
+ ") & " + env.get('CM_RUN_CMD', '').strip()
+ )
  else:
- env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD','').strip()
+ # Just use the existing CM_RUN_CMD if no ranging run is needed
+ env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD', '').strip()
 
  return {'return':0}

script/benchmark-program/customize.py

@@ -45,7 +45,7 @@ def preprocess(i):
 
  if os_info['platform'] != 'windows' and str(env.get('CM_SAVE_CONSOLE_LOG', True)).lower() not in [ "no", "false", "0"]:
  logs_dir = env.get('CM_LOGS_DIR', env['CM_RUN_DIR'])
- env['CM_RUN_CMD'] += " 2>&1 ; echo \$? > exitstatus | tee " + q+ os.path.join(logs_dir, "console.out") + q
+ env['CM_RUN_CMD'] += r" 2>&1 ; echo \$? > exitstatus | tee " + q+ os.path.join(logs_dir, "console.out") + q
 
  # additional arguments and tags for measuring system informations(only if 'CM_PROFILE_NVIDIA_POWER' is 'on')
  if env.get('CM_PROFILE_NVIDIA_POWER', '') == "on":
@@ -73,15 +73,15 @@ def preprocess(i):
  # running the script as a process in background
  pre_run_cmd = pre_run_cmd + 'cm run script --tags=runtime,system,utilisation' + env['CM_SYS_UTILISATION_SCRIPT_TAGS'] + ' --quiet & '
  # obtain the command if of the background process
- pre_run_cmd += ' cmd_pid=\$!' + ' && ' + 'echo CMD_PID=\$cmd_pid'
+ pre_run_cmd += r" cmd_pid=\$! && echo CMD_PID=\$cmd_pid"
  print(f"Pre run command for recording the runtime system information: {pre_run_cmd}")
 
  env['CM_PRE_RUN_CMD'] = pre_run_cmd
 
  # generate the post run cmd - for killing the process that records runtime system infos
  post_run_cmd = ""
  if env.get('CM_PROFILE_NVIDIA_POWER', '') == "on":
- post_run_cmd += "echo killing process \$cmd_pid && kill -TERM \${cmd_pid}"
+ post_run_cmd += r"echo killing process \$cmd_pid && kill -TERM \${cmd_pid}"
  print(f"Post run command for killing the process that measures the runtime system information: {post_run_cmd}")
 
  env['CM_POST_RUN_CMD'] = post_run_cmd

script/detect-os/_cm.json

@@ -25,6 +25,11 @@
  "windows"
  ]
  },
+ "skip_if_env": {
+ "CM_WINDOWS_SYS_UTILS_MIN_INSTALL": [
+ "yes"
+ ]
+ },
  "tags": "get,sys-utils-min"
  }
  ],

script/detect-os/customize.py

@@ -55,6 +55,9 @@ def postprocess(i):
 
  env['CM_HOST_OS_MACHINE'] = state['os_uname_machine']
 
+ else:
+ env['CM_HOST_OS_PACKAGE_MANAGER'] = "choco"
+
  import platform
 
  env['CM_HOST_SYSTEM_NAME'] = platform.node()
@@ -93,6 +96,9 @@ def postprocess(i):
  elif env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "zypper":
  env['CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD'] = "zypper install -y"
  env['CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD'] = "zypper update -y"
+ elif env.get('CM_HOST_OS_PACKAGE_MANAGER', '') == "choco":
+ env['CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD'] = "choco install -y"
+ env['CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD'] = "choco upgrade -y"
 
  if os.path.exists("/.dockerenv"):
  env['CM_RUN_INSIDE_DOCKER'] = "yes"

script/get-dataset-imagenet-val/_cm.yaml

@@ -93,8 +93,8 @@ variations:
  default: true
  env:
  CM_DAE_FILENAME: ILSVRC2012_img_val_500.tar
-# CM_DAE_URL: http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar
- CM_DAE_URL: https://www.dropbox.com/scl/fi/a7fhjnzxi6x3ceapxh5bm/ILSVRC2012_img_val_500.tar?rlkey=hz4rabo9ve43co3c303y9r6l7&st=ulcgb3av&dl=1
+ CM_DAE_URL: http://cKnowledge.org/ai/data/ILSVRC2012_img_val_500.tar
+ CM_DOWNLOAD_URL1: https://www.dropbox.com/scl/fi/a7fhjnzxi6x3ceapxh5bm/ILSVRC2012_img_val_500.tar?rlkey=hz4rabo9ve43co3c303y9r6l7&st=ulcgb3av&dl=1
  CM_DATASET_SIZE: '500'
  CM_DOWNLOAD_CHECKSUM: 8627befdd8c2bcf305729020e9db354e
  CM_DOWNLOAD_FILENAME: ILSVRC2012_img_val_500.tar

script/get-generic-sys-util/_cm.json

@@ -676,7 +676,8 @@
  "apt": "xz-utils",
  "brew": "xz",
  "dnf": "xz",
- "yum": "xz"
+ "yum": "xz",
+ "choco": "xz"
  }
  }
  },
@@ -686,7 +687,8 @@
  },
  "state": {
  "zlib": {
- "apt": "zlib1g"
+ "apt": "zlib1g",
+ "choco": "zlib"
  }
  }
  },

script/get-generic-sys-util/customize.py

@@ -29,8 +29,16 @@ def preprocess(i):
 
  # Only "install" mode reaches here
  pm = env.get('CM_HOST_OS_PACKAGE_MANAGER')
+ util = env.get('CM_SYS_UTIL_NAME', '')
+ if util == '':
+ return {'return': 1, 'error': 'Please select a variation specifying the sys util name'}
 
- if os_info['platform'] == 'windows':
+ package = state.get(util)
+ package_name = None
+ if package and pm:
+ package_name = package.get(pm)
+
+ if os_info['platform'] == 'windows' and not package_name:
  print ('')
  print ('WARNING: for now skipping get-generic-sys-util on Windows ...')
  print ('')
@@ -40,17 +48,10 @@ def preprocess(i):
  if not pm:
  return {'return': 1, 'error': 'Package manager not detected for the given OS'}
 
- util = env.get('CM_SYS_UTIL_NAME', '')
- if util == '':
- return {'return': 1, 'error': 'Please select a variation specifying the sys util name'}
-
-
- package = state.get(util)
 
  if not package:
  return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)}
 
- package_name = package.get(pm)
  if not package_name:
  return {'return': 1, 'error': 'No package name specified for {} and util name {}'.format(pm, util)}
 

script/get-git-repo/run.bat

@@ -6,7 +6,7 @@ rem echo Cloning MLCommons from %CM_GIT_URL% with branch %CM_GIT_CHECKOUT% %CM_G
 rem git clone %CM_GIT_RECURSE_SUBMODULES% %CM_GIT_URL% %CM_GIT_DEPTH% inference
 rem cd inference
 rem git checkout -b "%CM_GIT_CHECKOUT%"
-rem 
+rem
 
 rem Next line allows ERRORLEVEL inside if statements!
 setlocal enabledelayedexpansion
@@ -18,19 +18,23 @@ set folder=%CM_GIT_CHECKOUT_FOLDER%
 
 if not exist "%CM_TMP_GIT_PATH%" (
 
- if exist %folder% (
- deltree %folder%
+ if exist "%folder%" (
+ rmdir /S /Q "%folder%" rem Use rmdir instead of deltree
  )
+
  echo ******************************************************
  echo Current directory: %CUR_DIR%
  echo.
  echo Cloning %CM_GIT_REPO_NAME% from %CM_GIT_URL%
  echo.
  echo "%CM_GIT_CLONE_CMD%"
  echo.
+
  %CM_GIT_CLONE_CMD%
  IF !ERRORLEVEL! NEQ 0 EXIT !ERRORLEVEL!
- cd %folder%
+
+ cd "%folder%"
+
  if not "%CM_GIT_SHA%" == "" (
  echo.
  echo.
@@ -39,9 +43,7 @@ if not exist "%CM_TMP_GIT_PATH%" (
  )
 
 ) else (
-
- cd %folder%
-
+ cd "%folder%"
 )
 
 if not "%CM_GIT_SUBMODULES%" == "" (
@@ -62,6 +64,7 @@ if "%CM_GIT_PATCH%" == "yes" (
  )
 )
 
-cd %CUR_DIR%
+cd "%CUR_DIR%"
 
 exit /b 0
+

script/get-sys-utils-min/_cm.yaml

@@ -8,13 +8,21 @@ cache: true
 
 category: Detection or installation of tools and artifacts
 
-deps: []
+deps:
+ - tags: detect,os
+ - tags: get,generic,sys-util,_xz
+ enable_if_env:
+ CM_HOST_OS_TYPE:
+ - windows
+ - tags: get,generic,sys-util,_zlib
+ enable_if_env:
+ CM_HOST_OS_TYPE:
+ - windows
 
 env:
  CM_CLEAN_DIRS: bin
+ CM_WINDOWS_SYS_UTILS_MIN_INSTALL: yes
  CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1
- ; https://www.dropbox.com/scl/fi/cj7xmgmk4wu1r0iuoedcq/zlib123dllx64-bin.zip?rlkey=sqmldotiste2zhk6nhmijg09e&st=i5w7esjx&dl=1
- ; https://www.dropbox.com/scl/fi/x94qvrmac55ht03257139/xz-5.2.9-win64.zip?rlkey=adzixlsqrapncrusy9ndd2qji&st=tqobzlfo&dl=1
  CM_SUDO: sudo
 
 new_env_keys:

script/install-github-cli/run.bat

@@ -0,0 +1 @@
+choco install gh

script/push-mlperf-inference-results-to-github/_cm.json

@@ -31,7 +31,7 @@
  }
  ],
  "default_env": {
- "CM_MLPERF_RESULTS_GIT_REPO_URL": "https://github.com/ctuning/mlperf_inference_submissions_v4.0"
+ "CM_MLPERF_RESULTS_GIT_REPO_URL": "https://github.com/mlcommons/mlperf_inference_submissions_v4.0"
  },
  "input_mapping": {
  "repo_url": "CM_MLPERF_RESULTS_GIT_REPO_URL",

script/push-mlperf-inference-results-to-github/run.bat

@@ -0,0 +1,34 @@
+@echo off
+
+REM Check if CM_GIT_REPO_CHECKOUT_PATH is set
+if not defined CM_GIT_REPO_CHECKOUT_PATH (
+ echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set."
+ exit /b 1
+)
+
+echo %cd%
+echo "Doing cd"
+REM Change directory
+cd "%CM_GIT_REPO_CHECKOUT_PATH%" || (
+ echo "Error: Failed to change directory to %CM_GIT_REPO_CHECKOUT_PATH%"
+ exit /b 1
+)
+echo %cd%
+echo "%CM_GIT_REPO_CHECKOUT_PATH%"
+git pull
+git add *
+
+REM Check if the CM_MLPERF_INFERENCE_SUBMISSION_DIR variable is set
+if defined CM_MLPERF_INFERENCE_SUBMISSION_DIR (
+ robocopy "%CM_MLPERF_INFERENCE_SUBMISSION_DIR%" "%CM_GIT_REPO_CHECKOUT_PATH%" /MIR
+ git add *
+)
+
+REM Check if the previous command was successful
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+git commit -a -m "%CM_MLPERF_RESULTS_REPO_COMMIT_MESSAGE%"
+REM git push
+
+REM Check if the previous command was successful
+if %errorlevel% neq 0 exit /b %errorlevel%

script/push-mlperf-inference-results-to-github/run.sh

@@ -1,10 +1,16 @@
 #!/bin/bash
 
-cd "${CM_GIT_CHECKOUT_PATH}"
+# Check if CM_GIT_REPO_CHECKOUT_PATH is set
+if [ -z "${CM_GIT_REPO_CHECKOUT_PATH}" ]; then
+ echo "Error: CM_GIT_REPO_CHECKOUT_PATH is not set."
+ exit 1
+fi
+
+cd "${CM_GIT_REPO_CHECKOUT_PATH}"
 git pull
 git add *
 if [[ -n ${CM_MLPERF_INFERENCE_SUBMISSION_DIR} ]]; then
- rsync -avz "${CM_MLPERF_INFERENCE_SUBMISSION_DIR}/" "${CM_GIT_CHECKOUT_PATH}/"
+ rsync -avz "${CM_MLPERF_INFERENCE_SUBMISSION_DIR}/" "${CM_GIT_REPO_CHECKOUT_PATH}/"
  git add *
 fi
 test $? -eq 0 || exit $?