Update yaml configs. Add validation dataset

recipes/configs/gemma/2B_full.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma.gemma_2b

recipes/configs/gemma/2B_lora.yaml

@@ -22,13 +22,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma.lora_gemma_2b

recipes/configs/gemma/7B_full.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-7b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma.gemma_7b

recipes/configs/gemma/7B_lora.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-7b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma.lora_gemma_7b

recipes/configs/gemma2/27B_full.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-27b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.gemma2_27b

recipes/configs/gemma2/27B_lora.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-27b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.lora_gemma2_27b

recipes/configs/gemma2/2B_full.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-2b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.gemma2_2b

recipes/configs/gemma2/2B_lora.yaml

@@ -22,13 +22,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-2b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.lora_gemma2_2b

recipes/configs/gemma2/9B_full.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-9b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.gemma2_9b

recipes/configs/gemma2/9B_lora.yaml

@@ -23,13 +23,21 @@ tokenizer:
   _component_: torchtune.models.gemma.gemma_tokenizer
   path: /tmp/gemma-2-9b/tokenizer.model
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.gemma2.lora_gemma2_9b

recipes/configs/llama2/13B_full.yaml

@@ -43,13 +43,21 @@ tokenizer:
   path: /tmp/Llama-2-13b-hf/tokenizer.model
   max_seq_len: null
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Fine-tuning arguments
 batch_size: 2
 epochs: 1

recipes/configs/llama2/13B_lora.yaml

@@ -56,9 +56,16 @@ tokenizer:
 dataset:
   _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
-batch_size: 2
+
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
 
 # Optimizer and Scheduler
 optimizer:
@@ -75,6 +82,7 @@ loss:
 
 # Training
 epochs: 1
+batch_size: 2
 max_steps_per_epoch: null
 gradient_accumulation_steps: 8  # Use to increase effective batch size
 clip_grad_norm: null

recipes/configs/llama2/70B_lora.yaml

@@ -40,11 +40,18 @@ save_adapter_weights_only: False
 
 # Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
-batch_size: 2
+
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
 
 # Optimizer and Scheduler
 optimizer:
@@ -61,6 +68,7 @@ loss:
 
 # Training
 epochs: 1
+batch_size: 2
 max_steps_per_epoch: null
 clip_grad_norm: null
 compile: False  # torch.compile the model + loss, True increases speed + decreases memory

recipes/configs/llama2/70B_qlora.yaml

@@ -45,12 +45,18 @@ save_adapter_weights_only: False
 
 # Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
-  train_on_input: True
+  split: train[:95%]
 seed: null
 shuffle: True
-batch_size: 2
+
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
 
 # Optimizer and Scheduler
 optimizer:
@@ -70,6 +76,7 @@ fsdp:
 
 # Training
 epochs: 1
+batch_size: 2
 max_steps_per_epoch: null
 gradient_accumulation_steps: 1  # Use to increase effective batch size
 clip_grad_norm: null

recipes/configs/llama2/7B_full.yaml

@@ -26,13 +26,21 @@ tokenizer:
   path: /tmp/Llama-2-7b-hf/tokenizer.model
   max_seq_len: null
 
-# Dataset
+# Dataset and Sampler
 dataset:
-  _component_: torchtune.datasets.alpaca_dataset
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
 
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
+
 # Model Arguments
 model:
   _component_: torchtune.models.llama2.llama2_7b

recipes/configs/llama2/7B_lora.yaml

@@ -53,9 +53,16 @@ save_adapter_weights_only: False
 dataset:
   _component_: torchtune.datasets.alpaca_cleaned_dataset
   packed: False  # True increases speed
+  split: train[:95%]
 seed: null
 shuffle: True
-batch_size: 2
+
+# Validation
+run_val_every_n_steps: null  # Change to an integer to enable validation every N steps
+dataset_val:
+  _component_: torchtune.datasets.alpaca_cleaned_dataset
+  split: train[95%:]
+batch_size_val: ${batch_size}
 
 # Optimizer and Scheduler
 optimizer:
@@ -72,6 +79,7 @@ loss:
 
 # Training
 epochs: 1
+batch_size: 2
 max_steps_per_epoch: null
 clip_grad_norm: null
 compile: False  # torch.compile the model + loss, True increases speed + decreases memory