Update benchmarks.sh

Author: jainapurva

torchao/_models/llama/benchmarks.sh

@@ -21,6 +21,7 @@ export MODEL_REPO=meta-llama/Meta-Llama-3.1-8B
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization int8wo --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization int4wo-64 --write_result benchmark_results.txt
+# Runs on H100, float8 is not supported on CUDA arch < 8.9
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8wo --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8dq-tensor --write_result benchmark_results.txt
 python generate.py --checkpoint_path $CHECKPOINT_PATH/$MODEL_REPO/model.pth --compile --quantization float8dq-wo --write_result benchmark_results.txt

torchao/_models/llama/generate.py

@@ -255,10 +255,7 @@ def main(
             elif granularity=="row":
                 granularity = PerRow()
             else:
-                if granularity=="float8dq":
-                    granularity = PerTensor()
-                else:
-                    raise ValueError(f"Unknown granularity {granularity}")
+                granularity = PerTensor()
             quantize_(model, float8_dynamic_activation_float8_weight(granularity=granularity))
         if "autoquant" in quantization:
             if "autoquant-int4" == quantization:

torchao/quantization/README.md

@@ -139,9 +139,6 @@ change_linear_weights_to_int8_dqtensors(model)
 from torchao.quantization import quantize_, float8_dynamic_activation_float8_weight
 from torchao.quantization.observer import PerTensor
 quantize_(model, float8_dynamic_activation_float8_weight(granularity=PerTensor()))
-from torchao.quantization.observer import PerTensor
-quantize_(model, float8_dynamic_activation_float8_weight(granularity=PerTensor()))
-
 ```
 
 #### A16W6 Floating Point WeightOnly Quantization