Merge branch 'GATEOverflow:mlperf-inference' into changesmixtral

.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml

@@ -4,7 +4,7 @@
 name: MLPerf inference MLCommons C++ ResNet50
 
 on:
- pull_request:
+ pull_request_target:
  branches: [ "main", "dev", "mlperf-inference" ]
  paths:
  - '.github/workflows/test-mlperf-inference-mlcommons-cpp-resnet50.yml'
@@ -13,16 +13,18 @@ on:
 
 jobs:
  build:
-
- runs-on: ubuntu-latest
+ runs-on: ${{ matrix.os }}
  strategy:
  fail-fast: false
  matrix:
  python-version: [ "3.12" ]
  llvm-version: [ "15.0.6", "16.0.4", "17.0.6" ]
+ os: [ubuntu-latest, windows-latest, macos-latest]
  exclude:
- - llvm-version: "15.0.6"
- - llvm-version: "16.0.4"
+ - llvm-version: "15.0.6"
+ - llvm-version: "16.0.4"
+ - os: windows-latest
+ - os: macos-latest
 
  steps:
  - uses: actions/checkout@v3
@@ -36,6 +38,25 @@ jobs:
  cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
  cm run script --quiet --tags=get,sys-utils-cm
  cm run script --quiet --tags=install,prebuilt,llvm --version=${{ matrix.llvm-version }}
- - name: Test MLPerf Inference MLCommons C++ ResNet50
+ - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
+ if: matrix.os == 'windows-latest'
+ run: |
+ cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --adr.loadgen.tags=_from-pip --pip_loadgen=yes -v --quiet
+ - name: Test MLPerf Inference MLCommons C++ ResNet50 on ${{ matrix.os }}
+ if: matrix.os != 'windows-latest'
+ run: |
+ cmr "app mlperf inference mlcommons cpp" --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} -v --quiet
+ - name: Push Results
+ if: github.repository_owner == 'gateoverflow'
+ env:
+ USER: "GitHub Action"
+ EMAIL: "[email protected]"
+ GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
  run: |
- cmr "app mlperf inference mlcommons cpp" -v --quiet
+ git config --global user.name "${{ env.USER }}"
+ git config --global user.email "${{ env.EMAIL }}"
+ git config --global credential.https://github.com.helper ""
+ git config --global credential.https://github.com.helper "!gh auth git-credential"
+ git config --global credential.https://gist.github.com.helper ""
+ git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+ cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/mlperf_inference_test_submissions_v5.0 --repo_branch=main --commit_message="Results from MLCommons C++ ResNet50 GH action on ${{ matrix.os }}" --quiet

.github/workflows/test-mlperf-inference-sdxl.yaml

@@ -6,7 +6,7 @@ on:
 
 jobs:
  build_reference:
- if: github.repository_owner == 'gateoverflow_off'
+ if: github.repository_owner == 'gateoverflow'
  runs-on: [ self-hosted, linux, x64 ]
  strategy:
  fail-fast: false
@@ -22,30 +22,5 @@ jobs:
  export CM_REPOS=$HOME/GH_CM
  python3 -m pip install cm4mlops
  cm pull repo
- cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
- cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
- cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results
+ cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
  cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
-
- build_nvidia:
- if: github.repository_owner == 'gateoverflow_off'
- runs-on: [ self-hosted, linux, x64 ]
- strategy:
- fail-fast: false
- matrix:
- python-version: [ "3.12" ]
- backend: [ "tensorrt" ]
- precision: [ "float16" ]
- implementation: [ "nvidia" ]
- steps:
- - name: Test MLPerf Inference SDXL Nvidia
- run: |
- source gh_action/bin/deactivate || python3 -m venv gh_action
- source gh_action/bin/activate
- export CM_REPOS=$HOME/GH_CM
- cm pull repo
- cm run script --tags=run-mlperf,inference,_performance-only,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
- cm run script --tags=run-mlperf,inference,_short --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --category=edge --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --env.CM_MLPERF_MODEL_SDXL_DOWNLOAD_TO_HOST=yes --clean
- cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=edge --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons --submission_dir=$HOME/gh_action_submissions --results_dir=$HOME/gh_action_results/test_results
- cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet --submission_dir=$HOME/gh_action_submissions
-

.github/workflows/test-qaic-software-kit.yml

@@ -38,4 +38,3 @@ jobs:
  - name: Test Software Kit for compilation on Ubuntu 20.04
  run: |
  cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet
- cm run script --tags=get,qaic,software,kit --adr.compiler.tags=${{ matrix.compiler }} --adr.compiler.version=${{ matrix.llvm-version }} --quiet

automation/script/module.py

@@ -413,10 +413,6 @@ def _run(self, i):
 
  ignore_script_error = i.get('ignore_script_error', False)
 
- # Get constant env and state
- const = i.get('const',{})
- const_state = i.get('const_state',{})
-
  # Detect current path and record in env for further use in native scripts
  current_path = os.path.abspath(os.getcwd())
  r = _update_env(env, 'CM_TMP_CURRENT_PATH', current_path)
@@ -838,8 +834,8 @@ def _run(self, i):
  script_artifact_env = meta.get('env',{})
  env.update(script_artifact_env)
 
-
-
+ script_artifact_state = meta.get('state',{})
+ utils.merge_dicts({'dict1':state, 'dict2':script_artifact_state, 'append_lists':True, 'append_unique':True})
 
 
 
@@ -853,7 +849,7 @@ def _run(self, i):
 
 
  # STEP 700: Overwrite env with keys from the script input (to allow user friendly CLI)
- # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env']
+ # IT HAS THE PRIORITY OVER meta['default_env'] and meta['env'] but not over the meta from versions/variations
  # (env OVERWRITE - user enforces it from CLI)
  # (it becomes const)
  if input_mapping:
@@ -866,7 +862,9 @@ def _run(self, i):
  # update_env_from_input_mapping(const, i, docker_input_mapping)
 
 
-
+ #Update env/state with cost
+ env.update(const)
+ utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
 
 
 
@@ -882,7 +880,7 @@ def _run(self, i):
  variations = script_artifact.meta.get('variations', {})
  state['docker'] = meta.get('docker', {})
 
- r = self._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose)
+ r = self._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose)
  if r['return'] > 0:
  return r
 
@@ -952,7 +950,7 @@ def _run(self, i):
 
  if version!='' and version in versions:
  versions_meta = versions[version]
- r = update_state_from_meta(versions_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
+ r = update_state_from_meta(versions_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
  if r['return']>0: return r
  adr=get_adr(versions_meta)
  if adr:
@@ -1328,7 +1326,7 @@ def _run(self, i):
 
  if default_version in versions:
  versions_meta = versions[default_version]
- r = update_state_from_meta(versions_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
+ r = update_state_from_meta(versions_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
  if r['return']>0: return r
 
  if "add_deps_recursive" in versions_meta:
@@ -1374,7 +1372,6 @@ def _run(self, i):
  r = update_env_with_values(env)
  if r['return']>0: return r
 
-
  # Clean some output files
  clean_tmp_files(clean_files, recursion_spaces)
 
@@ -1451,8 +1448,12 @@ def _run(self, i):
  elif pip_version_max != '':
  pip_version_string = '<='+pip_version_max
 
+ env.update(const)
+ utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
+
  r = _update_env(env, 'CM_TMP_PIP_VERSION_STRING', pip_version_string)
  if r['return']>0: return r
+
  if pip_version_string != '':
  logging.debug(recursion_spaces+' # potential PIP version string (if needed): '+pip_version_string)
 
@@ -1462,10 +1463,6 @@ def _run(self, i):
 
  logging.debug(recursion_spaces+' - Running preprocess ...')
 
- # Update env and state with const
- utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True})
- utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
-
  run_script_input['run_state'] = run_state
 
  ii = copy.deepcopy(customize_common_input)
@@ -1916,7 +1913,7 @@ def _dump_version_info_for_script(self, output_dir = os.getcwd(), quiet = False,
  return {'return': 0}
 
  ######################################################################################
- def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose):
+ def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose):
 
  # Save current explicit variations
  import copy
@@ -2019,7 +2016,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env
  if variation_tag_dynamic_suffix:
  self._update_variation_meta_with_dynamic_suffix(variation_meta, variation_tag_dynamic_suffix)
 
- r = update_state_from_meta(variation_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
+ r = update_state_from_meta(variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
  if r['return']>0: return r
 
  if variation_meta.get('script_name', '')!='':
@@ -2050,7 +2047,7 @@ def _update_state_from_variations(self, i, meta, variation_tags, variations, env
 
  combined_variation_meta = variations[combined_variation]
 
- r = update_state_from_meta(combined_variation_meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
+ r = update_state_from_meta(combined_variation_meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, i)
  if r['return']>0: return r
 
  adr=get_adr(combined_variation_meta)
@@ -3012,8 +3009,8 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
  'remembered_selections': remembered_selections,
  'env':env,
  'state':state,
- 'const':const,
- 'const_state':const_state,
+ 'const':copy.deepcopy(const),
+ 'const_state':copy.deepcopy(const_state),
  'add_deps_recursive':add_deps_recursive,
  'debug_script_tags':debug_script_tags,
  'verbose':verbose,
@@ -3040,6 +3037,11 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
  r = update_env_with_values(env)
  if r['return']>0: return r
 
+ #Update env/state with cost
+ env.update(const)
+ utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
+
+
  return {'return': 0}
 
  ##############################################################################
@@ -4418,7 +4420,7 @@ def update_env_with_values(env, fail_on_not_found=False, extra_env={}):
 
  # Check cases such as --env.CM_SKIP_COMPILE
  if type(value)==bool:
- env[key] = str(value)
+ env[key] = value
  continue
 
  tmp_values = re.findall(r'<<<(.*?)>>>', str(value))
@@ -5110,20 +5112,31 @@ def update_env_from_input_mapping(env, inp, input_mapping):
  env[input_mapping[key]] = inp[key]
 
 ##############################################################################
-def update_state_from_meta(meta, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i):
+def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys, new_state_keys, i):
  """
  Internal: update env and state from meta
  """
 
  default_env = meta.get('default_env',{})
  for key in default_env:
  env.setdefault(key, default_env[key])
+
  update_env = meta.get('env', {})
  env.update(update_env)
 
+ update_const = meta.get('const', {})
+ if update_const:
+ const.update(update_const)
+ env.update(const)
+
  update_state = meta.get('state', {})
  utils.merge_dicts({'dict1':state, 'dict2':update_state, 'append_lists':True, 'append_unique':True})
 
+ update_const_state = meta.get('const_state', {})
+ if const_state:
+ utils.merge_dicts({'dict1':const_state, 'dict2':update_const_state, 'append_lists':True, 'append_unique':True})
+ utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
+
  new_deps = meta.get('deps', [])
  if len(new_deps)>0:
  append_deps(deps, new_deps)

automation/script/module_misc.py

@@ -1393,6 +1393,8 @@ def dockerfile(i):
 
  env=i.get('env', {})
  state = i.get('state', {})
+ const=i.get('const', {})
+ const_state = i.get('const_state', {})
  script_automation = i['self_module']
 
  dockerfile_env=i.get('dockerfile_env', {})
@@ -1420,7 +1422,7 @@ def dockerfile(i):
  state['docker'] = docker_settings
  add_deps_recursive = i.get('add_deps_recursive', {})
 
- r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False)
+ r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False)
  if r['return'] > 0:
  return r
 
@@ -1741,6 +1743,8 @@ def docker(i):
  env['CM_RUN_STATE_DOCKER'] = False
  script_automation = i['self_module']
  state = i.get('state', {})
+ const = i.get('const', {})
+ const_state = i.get('const_state', {})
 
  tags_split = i.get('tags', '').split(",")
  variation_tags = [ t[1:] for t in tags_split if t.startswith("_") ]
@@ -1793,7 +1797,7 @@ def docker(i):
  state['docker'] = docker_settings
  add_deps_recursive = i.get('add_deps_recursive', {})
 
- r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False)
+ r = script_automation._update_state_from_variations(i, meta, variation_tags, variations, env, state, const, const_state, deps = [], post_deps = [], prehook_deps = [], posthook_deps = [], new_env_keys_from_meta = [], new_state_keys_from_meta = [], add_deps_recursive = add_deps_recursive, run_state = {}, recursion_spaces='', verbose = False)
  if r['return'] > 0:
  return r