Merge pull request #141 from GATEOverflow/mlperf-inference

llama2 fixes

script/app-mlperf-inference/customize.py

@@ -33,6 +33,9 @@ def preprocess(i):
  run_state = i['run_script_input']['run_state']
  state['mlperf-inference-implementation']['script_id'] = run_state['script_id']+":"+",".join(run_state['script_variation_tags'])
 
+ if env.get('CM_VLLM_SERVER_MODEL_NAME', '') != '' and env.get('CM_ML_MODEL_FULL_NAME', '') == '':
+ env['CM_ML_MODEL_FULL_NAME'] = env['CM_VLLM_SERVER_MODEL_NAME'].replace("/", "_")
+
  return {'return':0}
 
 def postprocess(i):
@@ -288,20 +291,20 @@ def postprocess(i):
  cmd = ""
  xcmd = ""
 
- readme_init = "This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/ck).\n\n"
+ readme_init = "This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/cm4mlops).\n\n"
 
- readme_init+= "*Check [CM MLPerf docs](https://mlcommons.github.io/inference) for more details.*\n\n"
+ readme_init+= "*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.*\n\n"
 
  readme_body = "## Host platform\n\n* OS version: {}\n* CPU version: {}\n* Python version: {}\n* MLCommons CM version: {}\n\n".format(platform.platform(), 
  platform.processor(), sys.version, cm.__version__)
 
  x = repo_name
  if repo_hash!='': x+=' --checkout='+str(repo_hash)
 
- readme_body += "## CM Run Command\n\nSee [CM installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md).\n\n"+ \
+ readme_body += "## CM Run Command\n\nSee [CM installation guide](https://docs.mlcommons.org/inference/install/).\n\n"+ \
  "```bash\npip install -U cmind\n\ncm rm cache -f\n\ncm pull repo {}\n\n{}\n```".format(x, xcmd)
 
- readme_body += "\n*Note that if you want to use the [latest automation recipes](https://access.cknowledge.org/playground/?action=scripts) for MLPerf (CM scripts),\n"+ \
+ readme_body += "\n*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts),\n"+ \
  " you should simply reload {} without checkout and clean CM cache as follows:*\n\n".format(repo_name) + \
  "```bash\ncm rm repo {}\ncm pull repo {}\ncm rm cache -f\n\n```".format(repo_name, repo_name)
 
@@ -415,9 +418,6 @@ def postprocess(i):
  is_valid = checker.check_compliance_perf_dir(COMPLIANCE_DIR)
  state['cm-mlperf-inference-results'][state['CM_SUT_CONFIG_NAME']][model][scenario][test] = "passed" if is_valid else "failed"
 
- else:
- print(test)
-
 
  if state.get('mlperf-inference-implementation') and state['mlperf-inference-implementation'].get('version_info'):
  with open(os.path.join(output_dir, "cm-version-info.json"), "w") as f:

script/get-mlperf-inference-utils/mlperf_utils.py

@@ -70,16 +70,22 @@ def get_accuracy_metric(config, model, path):
  acc_upper_limit = config.get_accuracy_upper_limit(model)
  patterns = []
  acc_targets = []
- acc_limits = []
- up_patterns = []
+ acc_limits = [None] * (len(target)//2)
+ up_patterns = [None] * (len(target)//2)
  acc_types = []
 
  if acc_upper_limit is not None:
  acc_limit_check = True
- for i in range(0, len(acc_upper_limit), 2):
- acc_type, acc_target = acc_upper_limit[i:i+2]
- acc_limits.append(acc_target)
- up_patterns.append(checker.ACC_PATTERN[acc_type])
+
+ for ii in range(0, len(target), 2):
+ acc_type1,tmp = target[ii:ii+2]
+ for i in range(0, len(acc_upper_limit), 2):
+ acc_type, acc_target = acc_upper_limit[i:i+2]
+ if acc_type != acc_type1:
+ continue
+ acc_limits[ii//2] = acc_target
+ up_patterns[ii//2] = checker.ACC_PATTERN[acc_type]
+
 
  for i in range(0, len(target), 2):
  acc_type, acc_target = target[i:i+2]
@@ -109,6 +115,8 @@ def get_accuracy_metric(config, model, path):
  acc = None
  if acc_upper_limit is not None:
  for i, (pattern, acc_limit) in enumerate(zip(up_patterns, acc_limits)):
+ if not pattern:
+ continue
  m = re.match(pattern, line)
  if m:
  acc = m.group(1)
@@ -197,7 +205,7 @@ def get_result_string(version, model, scenario, result_path, has_power, sub_res,
  for i, acc in enumerate(acc_results):
  accuracy_results.append(str(round(float(acc_results[acc]), 5)))
  accuracy_result_string += f"`{acc}`: `{round(float(acc_results[acc]), 5)}`"
- if not acc_limits:
+ if not acc_limits or not acc_limits[i]:
  accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}`"
  else:
  accuracy_result_string += f", Required accuracy for closed division `>= {round(acc_targets[i], 5)}` and `<= {round(acc_limits[i], 5)}`"

script/run-mlperf-inference-app/customize.py

@@ -101,10 +101,13 @@ def preprocess(i):
  test_list = ["TEST01", "TEST05"]
  if env['CM_MODEL'] in ["resnet50"]:
  test_list.append("TEST04")
- if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']:
+ if "gpt" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']:
  test_list.remove("TEST01")
  test_list.remove("TEST05")
 
+ if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']:
+ test_list.append("TEST06")
+
  variation_implementation= "_" + env.get("CM_MLPERF_IMPLEMENTATION", "reference")
  variation_model= ",_" + env["CM_MLPERF_MODEL"]
  variation_backend= ",_" + env["CM_MLPERF_BACKEND"] if env.get("CM_MLPERF_BACKEND","") != "" else ""