Add llama3.1-8b reference implementation

script/app-mlperf-inference-mlcommons-python/customize.py

@@ -493,7 +493,7 @@ def get_run_cmd_reference(
         if env.get('MLC_ACTIVATE_RGAT_IN_MEMORY', '') == "yes":
             cmd += " --in-memory "
 
-    elif "llama3" in env['MLC_MODEL']:
+    elif "llama3_1-405b" in env['MLC_MODEL']:
         env['RUN_DIR'] = os.path.join(
             env['MLC_MLPERF_INFERENCE_SOURCE'],
             "language",
@@ -517,6 +517,30 @@ def get_run_cmd_reference(
         cmd = cmd.replace("--count", "--total-sample-count")
         cmd = cmd.replace("--max-batchsize", "--batch-size")
 
+    elif "llama3_1-8b" in env['MLC_MODEL']:
+        env['RUN_DIR'] = os.path.join(
+            env['MLC_MLPERF_INFERENCE_SOURCE'],
+            "language",
+            "llama3.1-8b")
+
+        if int(env.get('MLC_MLPERF_INFERENCE_TP_SIZE', '1')) > 1:
+            env['VLLM_WORKER_MULTIPROC_METHOD'] = "spawn"
+
+        cmd = f"""{x}{env['MLC_PYTHON_BIN_WITH_PATH']}{x} main.py \
+            --scenario {env['MLC_MLPERF_LOADGEN_SCENARIO']} \
+            --dataset-path {x}{env['MLC_DATASET_CNNDM_EVAL_PATH']}{x} \
+            --output-log-dir {x}{env['MLC_MLPERF_OUTPUT_DIR']}{x} \
+            --dtype {env['MLC_MLPERF_MODEL_PRECISION']} \
+            --model-path {x}{env['MLC_ML_MODEL_LLAMA3_CHECKPOINT_PATH']}{x} \
+            --tensor-parallel-size {env['MLC_MLPERF_INFERENCE_TP_SIZE']} \
+            --vllm"""
+
+        if env.get('MLC_MLPERF_INFERENCE_NUM_WORKERS', '') != '':
+            cmd += f" --num-workers {env['MLC_MLPERF_INFERENCE_NUM_WORKERS']}"
+
+        cmd = cmd.replace("--count", "--total-sample-count")
+        cmd = cmd.replace("--max-batchsize", "--batch-size")
+
     elif "pointpainting" in env['MLC_MODEL']:
         env['RUN_DIR'] = os.path.join(
             env['MLC_MLPERF_INFERENCE_SOURCE'],

script/app-mlperf-inference-mlcommons-python/meta.yaml

@@ -503,6 +503,19 @@ deps:
         - "yes"
       MLC_RUN_STATE_DOCKER:
         - "yes"
+
+  ## LLAMA3_1-8B
+  - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct
+    names:
+      - llama3-8b-model
+    enable_if_env:
+      MLC_MODEL:
+        - llama3_1-8b
+    skip_if_env:
+      MLC_USE_MODEL_FROM_HOST:
+        - "yes"
+      MLC_RUN_STATE_DOCKER:
+        - "yes"
 
   ## pointpainting
   - tags: get,ml-model,pointpainting
@@ -1408,6 +1421,54 @@ variations:
     env:
       MLC_GENERIC_PYTHON_PIP_EXTRA_FIND_LINKS_URL: "https://data.pyg.org/whl/torch-<<<MLC_TORCH_VERSION>>>+cpu.html"
 
+  llama3_1-8b:
+    group: models
+    env:
+      MLC_MODEL: llama3_1-8b
+    deps:
+      - tags: get,generic-python-lib,_package.transformers
+      - tags: get,generic-python-lib,_package.sentencepiece
+      - tags: get,generic-python-lib,_package.accelerate
+      - tags: get,generic-python-lib,_package.vllm
+      - tags: get,generic-python-lib,_package.pybind11
+      - tags: get,generic-python-lib,_package.nltk
+      - tags: get,generic-python-lib,_package.evaluate
+      - tags: get,generic-python-lib,_package.absl-py
+      - tags: get,generic-python-lib,_package.rouge-score
+      - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct
+        skip_if_env:
+          MLC_RUN_STATE_DOCKER:
+          - "yes"
+          MLC_USE_ML_MODEL_FROM_HOST:
+          - 'yes'
+        names:
+          - llama3_1-8b
+          - llama3-8b
+      ## CNNDM for Llama3 8B model - edge
+      - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone
+        names:
+          - cnndm-llama3-edge
+        enable_if_env:
+          MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: 
+            - edge
+        skip_if_env:
+          MLC_RUN_STATE_DOCKER:
+          - "yes"
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+      ## CNNDM for Llama3 8B model - datacenter
+      - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone
+        names:
+          - cnndm-llama3-datacenter
+        enable_if_env:
+          MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: 
+            - datacenter
+        skip_if_env:
+          MLC_RUN_STATE_DOCKER:
+          - "yes"
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+
   # Target devices
   cpu:
     group: device

script/app-mlperf-inference/meta.yaml

@@ -302,6 +302,10 @@ variations:
     default_variations:
       backend: pytorch
 
+  reference,llama3_1-8b:
+    default_variations:
+      backend: vllm
+
   all-models,nvidia-original:
     docker:
       deps:
@@ -919,6 +923,56 @@ variations:
       - waymo-accuracy-script
       tags: run,accuracy,mlperf,_waymo
 
+  llama3_1-8b:
+    group:
+      model
+    add_deps_recursive:
+      mlperf-inference-implementation:
+        tags: _llama3_1-8b
+    env:
+      MLC_MODEL:
+        llama3_1-8b
+    posthook_deps:
+    - enable_if_env:
+        MLC_MLPERF_LOADGEN_MODE:
+        - accuracy
+        - all
+        MLC_MLPERF_ACCURACY_RESULTS_DIR:
+        - 'on'
+      skip_if_env:
+        MLC_MLPERF_IMPLEMENTATION:
+        - nvidia
+      names:
+      - mlperf-accuracy-script
+      - llama3_1-8b-accuracy-script
+      tags: run,accuracy,mlperf,_dataset_llama3
+    docker:
+      deps:
+      - tags: get,ml-model,llama3,_hf,_meta-llama/Llama-3.1-8B-Instruct
+        enable_if_env:
+          MLC_USE_ML_MODEL_FROM_HOST:
+          - 'yes'
+        names:
+          - llama3_1-8b
+          - llama3-8b
+      ## CNNDM for Llama3 8B model - edge
+      - tags: get,dataset,cnndm,_validation,_edge,_llama3,_mlc,_rclone
+        names:
+          - cnndm-llama3-edge
+        enable_if_env:
+          MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: 
+            - edge
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
+      ## CNNDM for Llama3 8B model - datacenter
+      - tags: get,dataset,cnndm,_validation,_datacenter,_llama3,_mlc,_rclone
+        names:
+          - cnndm-llama3-datacenter
+        enable_if_env:
+          MLC_MLPERF_SUBMISSION_SYSTEM_TYPE: 
+            - datacenter
+          MLC_USE_DATASET_FROM_HOST:
+          - 'yes'
 
   llama3_1-405b:
     group:
@@ -947,7 +1001,7 @@ variations:
       deps:
       - tags: get,ml-model,llama3
         enable_if_env:
-          MLC_USE_DATASET_FROM_HOST:
+          MLC_USE_ML_MODEL_FROM_HOST:
           - 'yes'
         names:
           - llama3_1-405b
@@ -1948,6 +2002,27 @@ variations:
       MLC_REGENERATE_MEASURE_FILES: 'yes'
       MLC_MLPERF_INFERENCE_VERSION: '5.0'
 
+  r5.1-dev_default:
+    group:
+      reproducibility
+    add_deps_recursive:
+      nvidia-inference-common-code:
+        tags: _mlcommons,_v5.0
+      nvidia-inference-server:
+        version: r5.0
+        tags: _mlcommons
+      nvidia-harness:   
+        tags: _v5.0
+      intel-harness:
+        tags: _v4.1
+      inference-src:
+         version: r5.0
+      nvidia-scratch-space:
+        tags: _version.5.1-dev
+    default_env:
+      MLC_SKIP_SYS_UTILS: 'yes'
+      MLC_REGENERATE_MEASURE_FILES: 'yes'
+      MLC_MLPERF_INFERENCE_VERSION: '5.1-dev'
 
 invalid_variation_combinations:
   -

script/get-dataset-cnndm/customize.py

@@ -8,30 +8,37 @@ def preprocess(i):
 
     env = i['env']
 
-    if is_true(env.get('MLC_CNNDM_INTEL_VARIATION', '')):
-        i['run_script_input']['script_name'] = "run-intel"
+    if env.get('MLC_TMP_ML_MODEL', '') != "llama3_1-8b":
+        if is_true(env.get('MLC_CNNDM_INTEL_VARIATION', '')):
+            i['run_script_input']['script_name'] = "run-intel"
+        else:
+            print("Using MLCommons Inference source from '" +
+                  env['MLC_MLPERF_INFERENCE_SOURCE'] + "'")
     else:
-        print("Using MLCommons Inference source from '" +
-              env['MLC_MLPERF_INFERENCE_SOURCE'] + "'")
+        env['MLC_TMP_REQUIRE_DOWNLOAD'] = "yes"
+
+    if env.get('MLC_OUTDIRNAME', '') != '':
+        env['MLC_DOWNLOAD_PATH'] = env['MLC_OUTDIRNAME']
 
     return {'return': 0}
 
 
 def postprocess(i):
     env = i['env']
 
-    if is_false(env.get('MLC_DATASET_CALIBRATION', '')):
-        env['MLC_DATASET_PATH'] = os.path.join(os.getcwd(), 'install')
-        env['MLC_DATASET_EVAL_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'cnn_eval.json')
-        env['MLC_DATASET_CNNDM_EVAL_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'cnn_eval.json')
-        env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PATH']
-    else:
-        env['MLC_CALIBRATION_DATASET_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
-        env['MLC_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join(
-            os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
-        env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_CALIBRATION_DATASET_PATH']
+    if env.get('MLC_TMP_ML_MODEL', '') != "llama3_1-8b":
+        if is_false(env.get('MLC_DATASET_CALIBRATION', '')):
+            env['MLC_DATASET_PATH'] = os.path.join(os.getcwd(), 'install')
+            env['MLC_DATASET_EVAL_PATH'] = os.path.join(
+                os.getcwd(), 'install', 'cnn_eval.json')
+            env['MLC_DATASET_CNNDM_EVAL_PATH'] = os.path.join(
+                os.getcwd(), 'install', 'cnn_eval.json')
+            env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_DATASET_PATH']
+        else:
+            env['MLC_CALIBRATION_DATASET_PATH'] = os.path.join(
+                os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
+            env['MLC_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join(
+                os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
+            env['MLC_GET_DEPENDENT_CACHED_PATH'] = env['MLC_CALIBRATION_DATASET_PATH']
 
     return {'return': 0}

script/get-dataset-cnndm/meta.yaml

@@ -33,6 +33,53 @@ tags:
 - original
 uid: aed298c156e24257
 variations:
+  llama3:
+    env:
+      MLC_TMP_ML_MODEL: llama3_1-8b
+  edge:
+    group: category
+  datacenter:
+    group: category
+  rclone:
+    group: download-tool
+    add_deps_recursive:
+      dae:
+        tags: _rclone
+  dry-run:
+    group: run-mode
+    env:
+      MLC_DOWNLOAD_MODE: dry
+  dry-run,rclone:
+    env:
+      MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run
+  mlc:
+    group: download-src
+    prehook_deps:
+      - tags: get,rclone
+        enable_if_env:
+          MLC_TMP_REQUIRE_DOWNLOAD:
+          - yes
+      - tags: get,rclone-config,_mlc-inference
+        force_cache: true
+        enable_if_env:
+          MLC_TMP_REQUIRE_DOWNLOAD:
+          - yes
+      - enable_if_env:
+          MLC_TMP_REQUIRE_DOWNLOAD:
+          - 'yes'
+        env:
+          MLC_RCLONE_COPY_USING: copy
+        force_cache: true
+        names:
+          - dae
+        tags: download-and-extract
+        force_env_keys:
+          - MLC_OUTDIRNAME  
+        update_tags_from_env_with_prefix:
+          _url.:
+            - MLC_DOWNLOAD_URL
+    env:
+      MLC_DOWNLOAD_SRC: mlcommons
   calibration:
     env:
       MLC_DATASET_CALIBRATION: 'yes'
@@ -53,3 +100,31 @@ variations:
     - MLC_DATASET_PATH
     - MLC_DATASET_EVAL_PATH
     - MLC_DATASET_CNNDM_EVAL_PATH
+  validation,edge,llama3,mlc,rclone:
+    adr:
+      dae:
+        extra_cache_tags: cnndm,dataset,llama3,val,edge
+    env:
+      MLC_DATASET_CNNDM_FILENAME: sample_cnn_eval_5000.json
+      MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<<MLC_DATASET_CNNDM_FILENAME>>>
+      MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH
+      MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH
+  validation,datacenter,llama3,mlc,rclone:
+    adr:
+      dae:
+        extra_cache_tags: cnndm,dataset,llama3,val,datacenter
+    env:
+      MLC_DATASET_CNNDM_FILENAME: cnn_eval.json
+      MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<<MLC_DATASET_CNNDM_FILENAME>>>
+      MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH
+      MLC_EXTRACT_FINAL_ENV_NAME: MLC_DATASET_CNNDM_EVAL_PATH
+  calibation,llama3,mlc,rclone:
+    adr:
+      dae:
+        extra_cache_tags: cnndm,dataset,llama3,calib
+    env:
+      MLC_DATASET_CNNDM_FILENAME: cnn_dailymail_calibration.json
+      MLC_DOWNLOAD_URL: mlc-inference:mlcommons-inference-wg-public/llama3.1_8b/<<<MLC_DATASET_CNNDM_FILENAME>>>
+      MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH
+      MLC_EXTRACT_FINAL_ENV_NAME: MLC_CALIBRATION_DATASET_CNNDM_PATH
+