SCC24 changes

.github/workflows/cla.yml

@@ -8,6 +8,7 @@ on:
 
 jobs:
  cla-check:
+ if: github.repository_owner == 'mlcommons'
  runs-on: ubuntu-latest
  steps:
  - name: "MLCommons CLA bot check"

.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+name: Publish visualization site
+
+
+on:
+ release:
+ types: [published]
+ push:
+ branches:
+ - mlperf-inference-results-scc24
+ - docs
+
+jobs:
+
+ publish:
+ name: Publish the site
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository normally
+ uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+
+ - name: Init docs site
+ run: bash run.sh
+
+ - name: Run Mkdocs deploy
+ run: mkdocs gh-deploy --force

.github/workflows/run-mlperf-inference-submission-checker.yml

@@ -0,0 +1,31 @@
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference submission checker
+
+on:
+ pull_request:
+ branches: [ "main", "mlperf-inference-results-scc24" ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+ env:
+ CM_INDEX: "on"
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: [ "3.10" ]
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python3 -m pip install cm4mlops
+ - name: Run MLPerf Inference Submission Checker
+ run: |
+ cm run script --tags=run,mlperf,inference,submission,checker,_short-run --adr.submission-checker-src.tags=_repo.https://github.com/gateoverflow/inference --src_version=v4.1 --quiet --extra_args=" --skip-extra-files-in-root-check" --submission_dir=./

.github/workflows/update-results.yml

@@ -0,0 +1,51 @@
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference results updater
+
+
+on:
+ push:
+ branches: [ "main", "mlperf-inference-results-scc24" ]
+
+jobs:
+ build:
+
+ runs-on: ubuntu-latest
+ env:
+ CM_INDEX: "on"
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: [ "3.10" ]
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v3
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Install dependencies
+ run: |
+ python3 -m pip install cm4mlops
+
+ - name: Run MLPerf Inference Submission Checker and generate results summary
+ run: |
+ cm run script --tags=run,mlperf,inference,submission,checker,_short-run --src_version=v4.1 --adr.submission-checker-src.tags=_repo.https://github.com/gateoverflow/inference,_branch.improve_result_generation --quiet --extra_args=" --skip-extra-files-in-root-check" --submission_dir=./ > >(tee -a out.txt) 2> >(tee -a checker_log.txt >&2)
+ cm run script --tags=convert,from-csv,to-md --csv_file=summary.csv --md_file=README.md
+
+ USER="arjunsuresh"
+ [email protected]
+
+ git config --global user.name "$USER"
+ git config --global user.email "$EMAIL"
+ #git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN_TOKEN }}@github.com/${{ github.repository }}
+ git add summary*
+ echo -e 'Please download [summary.xlsx](summary.xlsx) to view the most recent results. \n ```' > temp
+ tail -n 16 checker_log.txt >> temp
+ echo -e '\n```\n' >> temp
+ cat temp | cat - README.md > temp1
+ head -n 100 temp1 > README.md
+ git add README.md
+
+ git diff-index --quiet HEAD || (git commit -am "Updated results summary" && git push origin)

README.md

@@ -0,0 +1,32 @@
+Please download [summary.xlsx](summary.xlsx) to view the most recent results. 
+ ```
+[2024-10-07 21:40:43,179 submission_checker1.py:2936 INFO] Results=9, NoResults=0, Power Results=0
+[2024-10-07 21:40:43,179 submission_checker1.py:2943 INFO] ---
+[2024-10-07 21:40:43,179 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0
+
+[2024-10-07 21:40:43,180 submission_checker1.py:2949 INFO] Open Results=9, Open Power Results=0
+
+[2024-10-07 21:40:43,180 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0
+
+[2024-10-07 21:40:43,180 submission_checker1.py:2959 INFO] ---
+[2024-10-07 21:40:43,180 submission_checker1.py:2961 INFO] Systems=8, Power Systems=0
+[2024-10-07 21:40:43,180 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0
+[2024-10-07 21:40:43,180 submission_checker1.py:2967 INFO] Open Systems=8, Open Power Systems=0
+[2024-10-07 21:40:43,180 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0
+[2024-10-07 21:40:43,180 submission_checker1.py:2977 INFO] ---
+[2024-10-07 21:40:43,180 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK
+INFO:root: ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-submission-checker/customize.py
+
+```
+
+| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types |
+|---:|:---------------|:---------------|:-----------|:-------------|:-------------|:-------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:--------------------------------------------------------------|------------------:|:----------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:----------------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------|
+| 0 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-main | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13292 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 |
+| 1 | MLCommons | available | open | datacenter | e8dbfdd7ca14 | e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13976 | CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 |
+| 2 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13598 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 |
+| 3 | MLCommons | available | open | datacenter | 13fce262fb79 | 13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.375843 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 |
+| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 52.9478 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Tokens/s | fp32 |
+| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.345721 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 |
+| 6 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.373636 | CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 |
+| 7 | MLCommons | available | open | datacenter | f9ac88850adc | f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.376944 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 |
+| 8 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374549 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 |

dbversion

@@ -0,0 +1 @@
+100

docinit.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+if [ ! -e docs ]; then
+ git clone https://github.com/GATEOverflow/inference_results_visualization_template.git docs
+ test $? -eq 0 || exit $?
+fi
+
+python3 -m pip install -r docs/requirements.txt
+
+if [ ! -e overrides ]; then
+ cp -r docs/overrides overrides
+ test $? -eq 0 || exit $?
+fi
+
+repo_owner=${INFERENCE_RESULTS_REPO_OWNER:-mlcommons}
+repo_branch=${INFERENCE_RESULTS_REPO_BRANCH:-main}
+repo_name=${INFERENCE_RESULTS_REPO_NAME:-inference_results_${INFERENCE_RESULTS_VERSION}}
+ver_num=$(cat dbversion)
+let ver_num++
+echo "ver_num=$ver_num" > dbversion
+if [ ! -e docs/javascripts/config.js ]; then
+ if [ -n "${INFERENCE_RESULTS_VERSION}" ]; then
+ echo "const results_version=\"${INFERENCE_RESULTS_VERSION}\";" > docs/javascripts/config.js;
+ echo "var repo_owner=\"${repo_owner}\";" >> docs/javascripts/config.js;
+ echo "var repo_branch=\"${repo_branch}\";" >> docs/javascripts/config.js;
+ echo "var repo_name=\"${repo_name}\";" >> docs/javascripts/config.js;
+ #ver_num=`echo ${INFERENCE_RESULTS_VERSION} | tr -cd '0-9'`
+ echo "const dbVersion =\"${ver_num}\";" >> docs/javascripts/config.js;
+ else
+ echo "Please export INFERENCE_RESULTS_VERSION=v4.1 or the corresponding version";
+ exit 1
+ fi
+fi
+
+if [ ! -e docs/thirdparty/tablesorter ]; then
+ cd docs/thirdparty && git clone https://github.com/Mottie/tablesorter.git && cd -
+ test $? -eq 0 || exit $?
+fi
+
+python3 process.py
+test $? -eq 0 || exit $?
+python3 process_results_table.py
+test $? -eq 0 || exit $?

mkdocs.yml

@@ -0,0 +1,43 @@
+site_name: MLPerf Inference Results Comparison
+repo_url: https://github.com/mlcommons/cm4mlperf-inference
+theme:
+ name: material
+ logo: img/logo_v2.svg
+ favicon: img/logo_v2.svg
+ palette:
+ primary: deep purple
+ accent: yellow
+ custom_dir: overrides
+ features:
+ - content.tabs.link
+ - content.code.copy
+ - navigation.expand
+ - navigation.sections
+ - navigation.indexes
+ - navigation.instant
+ - navigation.tabs
+ - navigation.tabs.sticky
+ - navigation.top
+ - toc.follow
+nav:
+ - Results: index.md
+ - Compare: compare/index.md
+ - Top Results: top_results/index.md
+
+markdown_extensions:
+ - pymdownx.tasklist:
+ custom_checkbox: true
+ - pymdownx.details
+ - admonition
+ - attr_list
+ - def_list
+ - footnotes
+ - pymdownx.superfences:
+ custom_fences:
+ - name: mermaid
+ class: mermaid
+ format: !!python/name:pymdownx.superfences.fence_code_format
+ - pymdownx.tabbed:
+ alternate_style: true
+plugins:
+ - macros

open/MLCommons/code/gptj-99/README.md

@@ -0,0 +1 @@
+TBD

open/MLCommons/code/stable-diffusion-xl/README.md

@@ -0,0 +1 @@
+TBD

open/MLCommons/measurements/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/README.md

@@ -0,0 +1,3 @@
+| Model | Scenario | Accuracy | Throughput | Latency (in ms) |
+|---------------------|------------|-----------------------|--------------|-------------------|
+| stable-diffusion-xl | offline | (15.18544, 235.69504) | 0.376 | - |

open/MLCommons/measurements/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base.json

@@ -0,0 +1,7 @@
+{
+ "starting_weights_filename": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+ "retraining": "no",
+ "input_data_types": "fp32",
+ "weight_data_types": "fp32",
+ "weight_transformations": "no"
+}