Commit ddcab381 authored by jameskrw's avatar jameskrw
Browse files

debugging token level reward

parent 2b8a9347

vagen/env/sokoban/env.py

+2 −0
Original line number Diff line number Diff line
@@ -375,6 +375,8 @@ class SokobanInterface(BaseInterface): 
        # deal with format

        if think and answer: # format is correct

            reward += self.FORMAT_REWARD

        else:

            reward -= self.FORMAT_REWARD*0.1



        info = {}

        for action, valid in zip(action_list, valid_list):

vagen/examples/sokoban/debug_qwen0_5_1_gpu_grpo.sh

+3 −4
Original line number Diff line number Diff line
@@ -22,7 +22,7 @@ python3 -m vagen.trainer.main_ppo \ 
    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban-text-1-step/train.parquet \

    data.val_files=data/sokoban-text-1-step/test.parquet \

    data.train_batch_size=128 \

    data.train_batch_size=16 \

    data.max_prompt_length=1024 \

    data.max_response_length=128 \

    data.max_trajectory_length=1024 \
@@ -30,7 +30,7 @@ python3 -m vagen.trainer.main_ppo \ 
    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_mini_batch_size=128 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \
@@ -41,7 +41,7 @@ python3 -m vagen.trainer.main_ppo \ 
    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \
@@ -62,6 +62,5 @@ python3 -m vagen.trainer.main_ppo \ 
    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=5 \

    # grpo sampling param

    rollout_manger.n_trajectory=8 \

    2>&1 | tee debug_qwen0_5_1_gpu_grpo.log

vagen/examples/sokoban/debug_qwen0_5_4_gpu_grpo.sh

+2 −3
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ python3 -m vagen.trainer.main_ppo \ 
    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban-text-1-step/train.parquet \

    data.val_files=data/sokoban-text-1-step/test.parquet \

    data.train_batch_size=256 \

    data.train_batch_size=16 \

    data.max_prompt_length=768 \

    data.max_response_length=128 \

    data.max_trajectory_length=1024 \
@@ -29,7 +29,7 @@ python3 -m vagen.trainer.main_ppo \ 
    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=64 \

    actor_rollout_ref.actor.ppo_mini_batch_size=128 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \
@@ -61,7 +61,6 @@ python3 -m vagen.trainer.main_ppo \ 
    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=5 \

    # grpo sampling param

    rollout_manger.n_trajectory=8 \

    2>&1 | tee debug_qwen0_5_4_gpu_grpo.log

vagen/examples/sokoban/debug_qwen2_5_vl_4gpu_grpo.sh

+0 −1
Original line number Diff line number Diff line
@@ -59,6 +59,5 @@ python3 -m vagen.trainer.main_ppo \ 
    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=5 \

    # grpo sampling param

    rollout_manger.n_trajectory=8 \

    2>&1 | tee debug_qwen2_5_vl_4gpu_grpo.log
 
 No newline at end of file

vagen/trainer/ppo/ray_trainer.py

+8 −2
Original line number Diff line number Diff line
@@ -143,7 +143,8 @@ def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_re 
        response_mask = attention_mask[:, -response_length:]

        token_level_rewards = data.batch['token_level_rewards']

        if "loss_mask" in data.batch.keys():

            loss_mask = data.batch['loss_mask']

            # get non zero position of token level rewards

            loss_mask = data.batch['loss_mask'][:, -response_length:]

            advantages, returns =core_algos.compute_gae_advantage_return_with_loss_mask(token_level_rewards=token_level_rewards,

                                                                    values=values,

                                                                    eos_mask=loss_mask,
@@ -165,7 +166,12 @@ def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_re 
        attention_mask = data.batch['attention_mask']

        response_mask = attention_mask[:, -response_length:]

        if "loss_mask" in data.batch.keys():

            loss_mask = data.batch['loss_mask']

            loss_mask = data.batch['loss_mask'][:, -response_length:]

            

            valid_token_level_rewards_positions = token_level_rewards[0].nonzero(as_tuple=True)[0]

            valid_loss_positions = loss_mask[0].nonzero(as_tuple=True)[0]

            print(f"[DEBUG]valid_token_level_rewards_positions={valid_token_level_rewards_positions}")

            print(f"[DEBUG]valid_loss_positions={valid_loss_positions}")

            # seems here only need to replace eos_mask with loss_mask

            advantages, returns = core_algos.compute_grpo_outcome_advantage(token_level_rewards=token_level_rewards,

                                                                        eos_mask=loss_mask,

CRA Git | Maintained and supported by SUSTech CRA and CCSE