fix

Author: jlamypoirier

scripts/run_all_textgen_benchmark_breakdown.sh

@@ -9,12 +9,12 @@
 ./scripts/run_textgen_benchmark_breakdown.sh santacoder bigcode/gpt_bigcode-santacoder 256 2040 11 1
 
 # Large model
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 1 8190 11 0
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 8 8190 11 0
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 32 8190 11 0
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 256 8190 11 0 # OOM?
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 1 8190 11 0
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 8 8190 11 0
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 32 8190 11 0
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 256 8190 11 0 # OOM?
 
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 1 8190 29 1 1
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 8 8190 29 1 1
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 32 8190 29 1 1
-./scripts/run_textgen_benchmark_breakdown.sh large_model ./data/bigcode_large-model 256 8190 29 1 1 # OOM?
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 1 8190 29 1 1
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 8 8190 29 1 1
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 32 8190 29 1 1
+./scripts/run_textgen_benchmark_breakdown.sh starcoder ./data/bigcode_large-model 256 8190 29 1 1 # OOM?

scripts/run_textgen_benchmark_breakdown.sh

@@ -12,12 +12,11 @@ TOKEN_STEP=${5:-5}
 STEP_ID=${6:-""}
 CYCLES=${7:-10}
 
-SAVE_DIR=data/benchmarks/v4
-#BATCH_SIZES="1 2 4 8 16 24 32 48 64 96 128 160 224 256"
+SAVE_DIR=data/benchmarks/v5
 RUN="python3 -m src.main --pipeline_class=TG_Pipeline --max_log_outputs=0 --dtype=float16 --device=cuda  --custom_generate  --breakdown_latency --ignore_oom --no_fast_init "
 
 
-IMPL=("flash" "causal" "vector" "bigcode" "bigcode2")
+IMPL=("flash" "causal" "vector" "bigcode" "bigcode2" "bigcode3")
 
 
 STEP=("" "--no_cache")
@@ -38,7 +37,7 @@ run () { # run(step, runtime, attn)
   fi
 }
 
-for impl in {0..4}
+for impl in {0..5}
 do
   if [ "${STEP_ID}" -eq "0" ]
   then

src/pipeline.py

@@ -491,7 +491,7 @@ def __init__(self, **kwargs):
         if self.device != torch.device("cuda:0"):
             raise ValueError(f"Textgen does not support device {self.device}")
 
-        self.config = self.model.model.transformer.config
+        self.config = getattr(self.model, "config", None) or self.model.model.transformer.config
 
     def _get_config(
         self,