[RL] Fix flashmask reward training (#10443)

* [RL] Fix flashmask reward training

* [RL] Fix flashmask reward training

* [RL] Fix flashmask reward training

* [RL] Fix flashmask reward training

Author: KB-Ding

docs/zh/llm/alignment/rm/README.md

@@ -0,0 +1 @@
+../../../../../llm/alignment/rm/README.md

llm/alignment/rm/flashmask/README.md renamed to llm/alignment/rm/README.md

@@ -39,4 +39,6 @@ tar -zxvf ultrafeedback_binarized.tar.gz
 
 ```bash
 # RM 启动命令参考
-python -u  -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" ./alignment/rm/flashmask/run_reward.py ./config/llama/rm_flashmask_argument.json
+cd llm/alignment/rm
+export PYTHONPATH=../../../:$PYTHONPATH
+python -u  -m paddle.distributed.launch --gpus "0,1,2,3,4,5,6,7" run_reward.py  ../../config/llama/rm_flashmask_argument.json

llm/alignment/rm/flashmask/data.py renamed to llm/alignment/rm/data.py

@@ -130,7 +130,7 @@ def preference_collate_fn(batch, max_seq_len=None, pad_token_id=0):
         difference = max_seq_len - len(sequence["input_ids"])
 
         input_dict["input_ids"].append(sequence["input_ids"] + [pad_token_id] * difference)
-        input_dict["position_ids"].append(sequence["position_ids"] + [pad_token_id] * difference)
+        input_dict["position_ids"].append(sequence["position_ids"] + [0] * difference)
         if use_attn_mask_startend_row_indices:
             input_dict["attn_mask_startend_row_indices"].append(
                 [
@@ -281,7 +281,7 @@ def zero_padding_process_collate_fn(batch, max_seq_len=None, pad_token_id=0):
         difference = max_seq_len - len(sequence["input_ids"])
 
         input_dict["input_ids"].append(sequence["input_ids"] + [pad_token_id] * difference)
-        input_dict["position_ids"].append(sequence["position_ids"] + [pad_token_id] * difference)
+        input_dict["position_ids"].append(sequence["position_ids"] + [0] * difference)
         input_dict["labels"].append(sequence["labels"] + [-100] * difference)
         if use_attn_mask_startend_row_indices:
             input_dict["attn_mask_startend_row_indices"].append(
@@ -334,7 +334,7 @@ def process_collate_fn(batch, pad_token_id=0):
 
         # input_ids: Tensor(seqL, ); position_ids: list, len(seqL); labels: Tensor(seqL, )
         input_dict["input_ids"].append(sequence["input_ids"].tolist() + [pad_token_id] * difference)
-        input_dict["position_ids"].append(sequence["position_ids"] + [pad_token_id] * difference)
+        input_dict["position_ids"].append(sequence["position_ids"] + [0] * difference)
         input_dict["labels"].append(sequence["labels"].tolist() + [-100] * difference)
         if use_attn_mask_startend_row_indices:
             input_dict["attn_mask_startend_row_indices"].append(