sync

.github/workflows/code-review.yml

@@ -0,0 +1,34 @@
+name: OpenAI Code Review
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+    paths:
+      - 'automation/**'
+      - 'script/**'
+      - '!**.md'
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  code_review:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'gateoverflow' && github.event.pull_request.changed_files > 0
+    steps:
+      # Run code review via OpenAI 
+      # Step to run the OpenAI Code Review using the GATEOverflow action
+    - name: Run OpenAI Code Review
+      uses: GATEOverflow/genai-code-review@v1
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}  # GitHub token for authentication
+        openai_api_key: ${{ secrets.OPENAI_API_KEY }}  # OpenAI API key for accessing the GPT model
+        github_pr_id: ${{ github.event.pull_request.number }}  # ID of the pull request to review
+        openai_model: "gpt-4o"  # Model to use for the code review
+        openai_temperature: 0.5  # Temperature setting for the model's output
+        openai_max_tokens: 2048  # Maximum number of tokens for the model's response
+        mode: "files"  # Mode of review, can be "files" or "diff"
+        language: "en"  # Language for the review output
+        custom_prompt: "" # Optional custom prompt for the model
+      continue-on-error: true  # Allow the workflow to continue even if this step fails

.github/workflows/test-mlperf-inference-gptj.yml

@@ -4,22 +4,19 @@
 name: MLPerf inference GPT-J
 
 on:
-  push:
-    branches: [ "main", "dev", "mlperf-inference" ]
-    paths:
-      - '.github/workflows/test-mlperf-inference-gptj.yml'
-      - '**'
-      - '!**.md'
+  schedule:
+    - cron: "1 1 * * */3"
 
 jobs:
   build:
+    if: github.repository_owner == 'gateoverflow'
     runs-on: [ self-hosted, linux, x64 ]
     strategy:
       fail-fast: false
       matrix:
         python-version: [ "3.12" ]
         backend: [ "pytorch" ]
-        precision: [ "bfloat16" ]
+        precision: [ "float16" ]
 
     steps:
     - name: Install dependencies

.github/workflows/test-mlperf-inference-sdxl.yaml

@@ -0,0 +1,47 @@
+name: MLPerf inference SDXL
+
+on:
+  schedule:
+    - cron: "1 2 * * *"
+
+jobs:
+  build_reference:
+    if: github.repository_owner == 'gateoverflow'
+    runs-on: [ self-hosted, linux, x64 ]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        precision: [ "float16" ]
+    steps:
+    - name: Install dependencies
+      run: |
+        source gh_action/bin/deactivate || python3 -m venv gh_action
+        source gh_action/bin/activate
+        export CM_REPOS=$HOME/GH_CM
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Test MLPerf Inference SDXL
+      run: |
+        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean
+
+  build_nvidia:
+      if: github.repository_owner == 'gateoverflow'
+      runs-on: [ self-hosted, linux, x64 ]
+      strategy:
+        fail-fast: false
+        matrix:
+          python-version: [ "3.12" ]
+          backend: [ "tensorrt" ]
+          precision: [ "float16" ]
+          implementation: [ "nvidia" ]
+      steps:
+      - name: Install dependencies
+        run: |
+          source gh_action/bin/deactivate || python3 -m venv gh_action
+          source gh_action/bin/activate
+          export CM_REPOS=$HOME/GH_CM
+          cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+      - name: Test MLPerf Inference SDXL
+        run: |
+          cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --docker --model=sdxl --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cuda --scenario=Offline --test_query_count=1 --precision=${{ matrix.precision }} --target_qps=1 --quiet --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --adr.compiler.tags=gcc --hw_name=gh_action --docker_dt=yes  --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --clean

.github/workflows/test-scc24-sdxl.yaml

@@ -0,0 +1,56 @@
+name: MLPerf inference SDXL
+
+on:
+  schedule:
+    - cron: "43 1 * * *"
+
+jobs:
+  build_reference:
+    if: github.repository_owner == 'gateoverflow'
+    runs-on: [ self-hosted, linux, x64 ]
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        precision: [ "float16" ]
+        device: [ "cuda" ]
+    steps:
+    - name: Install dependencies
+      run: |
+        source gh_action/bin/deactivate || python3 -m venv gh_action
+        source gh_action/bin/activate
+        export CM_REPOS=$HOME/GH_CM
+        cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+    - name: Test MLPerf Inference reference SDXL SCC 
+      env:
+        GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+      run: |
+        cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=reference --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --clean |
+        cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons |
+        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet
+
+  build_nvidia:
+      if: github.repository_owner == 'gateoverflow'
+      runs-on: [ self-hosted, linux, x64 ]
+      strategy:
+        fail-fast: false
+        matrix:
+          python-version: [ "3.12" ]
+          backend: [ "tensorrt" ]
+          precision: [ "float16" ]
+          implementation: [ "nvidia" ]
+      steps:
+      - name: Install dependencies
+        run: |
+          source gh_action/bin/deactivate || python3 -m venv gh_action
+          source gh_action/bin/activate
+          export CM_REPOS=$HOME/GH_CM
+          cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }}
+      - name: Test MLPerf Inference NVIDIA SDXL SCC
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+        run: |
+          cm run script --tags=run-mlperf,inference,_find-performance,_r4.1-dev,_short,_scc24-base --model=sdxl --implementation=nvidia --backend=${{ matrix.backend }} --category=datacenter --scenario=Offline --execution_mode=test --device=${{ matrix.device }} --precision=${{ matrix.precision }} --docker --docker_it=no --docker_cm_repo=gateoverflow@cm4mlops --docker_dt=yes --quiet --results_dir=$HOME/gh_action_results --submission_dir=$HOME/gh_action_submissions --precision=float16 --clean |
+          cm run script --tags=generate,inference,submission --clean --preprocess_submission=yes --run-checker --tar=yes --env.CM_TAR_OUTFILE=submission.tar.gz --division=open --category=datacenter --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --run_style=test --adr.submission-checker.tags=_short-run --quiet --submitter=MLCommons |
+          cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/gateoverflow/cm4mlperf-inference --repo_branch=mlperf-inference-results-scc24 --commit_message="Results from self hosted Github actions - NVIDIARTX4090" --quiet

script/app-mlperf-inference-mlcommons-python/_cm.yaml

@@ -185,8 +185,9 @@ deps:
   ## Pytorch (CPU)
   - tags: get,generic-python-lib,_torch
     names:
-    - ml-engine-pytorch
-    - pytorch
+      - torch
+      - ml-engine-pytorch
+      - pytorch
     skip_if_env:
       CM_MODEL:
         - dlrm-v2-99
@@ -216,6 +217,7 @@ deps:
   - tags: get,generic-python-lib,_torchvision
     names:
     - ml-engine-torchvision
+    - torchvision
     skip_if_env:
       CM_MODEL:
         - dlrm-v2-99
@@ -231,6 +233,7 @@ deps:
   - tags: get,generic-python-lib,_torchvision_cuda
     names:
     - ml-engine-torchvision
+    - torchvision
     enable_if_env:
       CM_MLPERF_BACKEND:
       - pytorch
@@ -695,6 +698,8 @@ variations:
     add_deps_recursive:
       pytorch:
         tags: _rocm
+      torchvision:
+        tags: _rocm
 
   rocm,sdxl:
     add_deps:
@@ -834,6 +839,9 @@ variations:
       MLPERF_TVM_TORCH_QUANTIZED_ENGINE: qnnpack
     deps:
     - tags: get,generic-python-lib,_torch
+      names:
+        - torch
+        - pytorch
     - tags: get,tvm
       names:
       - tvm
@@ -861,7 +869,6 @@ variations:
 
   gptj_:
     deps:
-    - tags: get,generic-python-lib,_torch
     - tags: get,generic-python-lib,_package.datasets
     - tags: get,generic-python-lib,_package.attrs
     - tags: get,generic-python-lib,_package.accelerate
@@ -1095,6 +1102,10 @@ variations:
       - dlrm-src
     # to force the version
     - tags: get,generic-python-lib,_torch
+      names:
+        - torch
+        - pytorch
+        - ml-engine-pytorch
       version: "1.13.1"
     - tags: get,generic-python-lib,_mlperf_logging
     - tags: get,generic-python-lib,_opencv-python
@@ -1247,9 +1258,9 @@ variations:
   bfloat16:
     group: precision
     add_deps_recursive:
-      ml-model-bfloat16:
+      ml-model-float16:
         tags:
-          _fp32
+          _fp16
     env:
       CM_MLPERF_QUANTIZATION: off
       CM_MLPERF_MODEL_PRECISION: bfloat16

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -296,6 +296,8 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio
                  scenario_extra_options + mode_extra_options + \
                 " --output " + env['CM_MLPERF_OUTPUT_DIR'] + \
                 " --model-path " + env['CM_ML_MODEL_PATH']
+        if env.get('CM_COCO2014_SAMPLE_ID_PATH','') != '':
+            cmd += " --ids-path " + env['CM_COCO2014_SAMPLE_ID_PATH']
 
     elif "llama2-70b" in env['CM_MODEL']:
         env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "llama2-70b")

script/app-mlperf-inference-nvidia/_cm.yaml

@@ -423,7 +423,7 @@ variations:
     group: model
     env:
       CM_MODEL: stable-diffusion-xl
-      CM_NOT_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://github.com/mlcommons/cm4mlops/blob/main/script/get-ml-model-stable-diffusion/_cm.json#L174"
+      CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://github.com/mlcommons/cm4mlops/blob/main/script/get-ml-model-stable-diffusion/_cm.json#L174"
       CM_ML_MODEL_WEIGHT_TRANSFORMATIONS: "quantization, affine fusion"
       CM_ML_MODEL_INPUTS_DATA_TYPE: int32
       CM_ML_MODEL_WEIGHTS_DATA_TYPE: int8
@@ -878,6 +878,8 @@ variations:
         tags: build,nvidia,inference,server
 
       - tags: reproduce,mlperf,inference,nvidia,harness,_preprocess_data
+        names:
+          - nvidia-preprocess-data
         inherit_variation_tags: true
         force_cache: true
         skip_inherit_variation_groups:
@@ -988,6 +990,8 @@ variations:
 
       - tags: reproduce,mlperf,inference,nvidia,harness,_preprocess_data
         inherit_variation_tags: true
+        names:
+          - nvidia-preprocess-data
         skip_inherit_variation_groups:
           - run-mode
           - loadgen-scenario

script/app-mlperf-inference-nvidia/customize.py

@@ -73,10 +73,11 @@ def preprocess(i):
     elif "stable-diffusion" in env["CM_MODEL"]:
         target_data_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'data', 'coco', 'SDXL')
         if not os.path.exists(target_data_path):
-            cmds.append("make download_data BENCHMARKS='stable-diffusion-xl'")
+            os.makedirs(target_data_path)
+            #cmds.append("make download_data BENCHMARKS='stable-diffusion-xl'")
             env['CM_REQUIRE_COCO2014_DOWNLOAD'] = 'yes'
-            cmds.append(f"cp -r \${CM_DATASET_PATH_ROOT}/captions/captions.tsv {target_data_path}/captions_5k_final.tsv" )
-            cmds.append(f"cp -r \${CM_DATASET_PATH_ROOT}/latents/latents.pt {target_data_path}/latents.pt" )
+            cmds.append(f"cp -r \$CM_DATASET_PATH_ROOT/captions/captions.tsv {target_data_path}/captions_5k_final.tsv" )
+            cmds.append(f"cp -r \$CM_DATASET_PATH_ROOT/latents/latents.pt {target_data_path}/latents.pt" )
         fp16_model_path = os.path.join(env['MLPERF_SCRATCH_PATH'], 'models', 'SDXL', 'official_pytorch', 'fp16', 'stable_diffusion_fp16')
 
         if not os.path.exists(os.path.dirname(fp16_model_path)):

script/app-mlperf-inference/_cm.yaml

@@ -385,6 +385,7 @@ variations:
       CM_IMAGENET_ACCURACY_DTYPE: int32
       CM_CNNDM_ACCURACY_DTYPE: int32
       CM_LIBRISPEECH_ACCURACY_DTYPE: int8
+      CM_DOCKER_USE_VIRTUAL_PYTHON: no
     prehook_deps:
       - names:
          - nvidia-original-mlperf-inference
@@ -1162,7 +1163,7 @@ variations:
       mlperf-inference-implementation:
         tags: _cuda
     deps:
-      - tags: get,cuda-devices
+      - tags: get,cuda-devices,_with-pycuda
         skip_if_env:
           CM_CUDA_DEVICE_PROP_GLOBAL_MEMORY:
             - "yes"

script/build-dockerfile/customize.py

@@ -180,8 +180,11 @@ def preprocess(i):
 
     f.write(EOL+'# Install python packages' + EOL)
     python = get_value(env, config, 'PYTHON', 'CM_DOCKERFILE_PYTHON')
-    f.write('RUN {} -m venv /home/cmuser/venv/cm'.format(python) + " " + EOL)
-    f.write('ENV PATH="/home/cmuser/venv/cm/bin:$PATH"' + EOL)
+
+    docker_use_virtual_python = env.get('CM_DOCKER_USE_VIRTUAL_PYTHON', "yes")
+    if str(docker_use_virtual_python).lower() not in [ "no", "0", "false"]:
+        f.write('RUN {} -m venv /home/cmuser/venv/cm'.format(python) + " " + EOL)
+        f.write('ENV PATH="/home/cmuser/venv/cm/bin:$PATH"' + EOL)
     #f.write('RUN . /opt/venv/cm/bin/activate' + EOL)
     f.write('RUN {} -m pip install '.format(python) + " ".join(get_value(env, config, 'python-packages')) + ' ' + pip_extra_flags + ' ' + EOL)
 

script/clean-nvidia-mlperf-inference-scratch-space/_cm.yaml

@@ -10,6 +10,8 @@ tags:
 - mlperf
 - inference
 uid: bb41f6e3608e4e8a
+input_mapping:
+  extra_cache_rm_tags: CM_CLEAN_EXTRA_CACHE_RM_TAGS
 deps:
   # Get Nvidia scratch space where data and models get downloaded
   - tags: get,mlperf,inference,nvidia,scratch,space

script/clean-nvidia-mlperf-inference-scratch-space/customize.py

@@ -16,22 +16,29 @@ def preprocess(i):
 
     clean_cmd = ''
     cache_rm_tags = ''
+    extra_cache_rm_tags = env.get('CM_CLEAN_EXTRA_CACHE_RM_TAGS', '')
 
     if env.get('CM_MODEL', '') == 'sdxl':
         if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'downloaded_data':
             clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "data", "coco", "SDXL")} """
-            cache_rm_tags  = "nvidia-harness,_preprocessed_data,_sdxl"
+            cache_rm_tags  = "nvidia-harness,_preprocess_data,_sdxl"
         if env.get('CM_CLEAN_ARTIFACT_NAME', '') == 'preprocessed_data':
             clean_cmd = f"""rm -rf {os.path.join(env['CM_NVIDIA_MLPERF_SCRATCH_PATH'], "preprocessed_data", "coco2014-tokenized-sdxl")} """
-            cache_rm_tags  = "nvidia-harness,_preprocessed_data,_sdxl"
+            cache_rm_tags  = "nvidia-harness,_preprocess_data,_sdxl"
 
-    if clean_cmd != '':
-        env['CM_RUN_CMD'] = clean_cmd
+    cache_rm_tags = cache_rm_tags + extra_cache_rm_tags
 
     if cache_rm_tags:
-        r = cm.access({'action': 'rm', 'automation': 'cache', 'tags': cache_rm_tags})
+        r = cm.access({'action': 'rm', 'automation': 'cache', 'tags': cache_rm_tags, 'f': True})
+        print(r)
         if r['return'] != 0 and r['return'] != 16: ## ignore missing ones
             return r
+        if r['return'] == 0: # cache entry found
+            if clean_cmd != '':
+                env['CM_RUN_CMD'] = clean_cmd
+    else:
+        if clean_cmd != '':
+            env['CM_RUN_CMD'] = clean_cmd
 
     return {'return':0}
 

script/download-file/customize.py

@@ -108,6 +108,8 @@ def preprocess(i):
                 elif "no such file" in checksum_result.stderr.lower():
                     #print(f"No file {env['CM_DOWNLOAD_FILENAME']}. Downloading through cmutil.")
                     cmutil_require_download = 1
+                elif checksum_result.returncode == 1:
+                    return {"return":1, "error":f"Error while checking checksum: {checksum_result.stderr}"}
                 else:
                     print(f"File {env['CM_DOWNLOAD_FILENAME']} already present, original checksum and computed checksum matches! Skipping Download..")
             else: