Commit 96d4fa84 authored by jameskrw's avatar jameskrw
Browse files

updated mask_gae and mask_loss

parent 12da24c6

vagen/examples/release_experiments/mask_gae.sh

0 → 100644
+83 −0
Original line number Diff line number Diff line 
set -x





export VLLM_ATTENTION_BACKEND=XFORMERS

export PYTHONHASHSEED=0



python -m vagen.env.sokoban.create_dataset \

    --data_dir data/sokoban-vision-6-step \

    --max_action_length 6 \

    --dim_room 6 6 \

    --num_boxes 1 \

    --max_steps 100 \

    --search_depth 30 \

    --start_seed 0 \

    --train_ratio 0.8 \

    --visual_env \

    --max_action_per_step 3 \

    --max_action_penalty -0.1 \

    --format_reward 0.5 \

    --n_candidate 20000



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=masked_gae \

    algorithm.high_level_gamma=0.95 \

    data.train_files=data/sokoban-vision-6-step/train.parquet \

    data.val_files=data/sokoban-vision-6-step/test.parquet \

    data.train_batch_size=128 \

    data.max_prompt_length=1024 \

    data.max_response_length=128 \

    data.max_trajectory_length=1800 \

    data.image_key=images \

    data.truncation=left \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=False \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=mse \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.2 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    actor_rollout_ref.rollout.top_p=0.95 \

    actor_rollout_ref.rollout.temperature=0.7 \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=True \

    critic.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    critic.model.fsdp_config.optimizer_offload=False \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='mask_gae' \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=100 \

    trainer.test_freq=20 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=3 \

    rollout_manager.window_size=5 \

    rollout_manager.use_multi_turn_reward=False \

    rollout_manager.use_loss_mask=False \

    rollout_manager.use_gae_mask=True \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=8 \

    rollout_manager.n_trajectory=1 \

    2>&1 | tee mask_gae.log

vagen/examples/release_experiments/mask_loss.sh

0 → 100644
+83 −0
Original line number Diff line number Diff line 
set -x





export VLLM_ATTENTION_BACKEND=XFORMERS

export PYTHONHASHSEED=0



python -m vagen.env.sokoban.create_dataset \

    --data_dir data/sokoban-vision-6-step \

    --max_action_length 6 \

    --dim_room 6 6 \

    --num_boxes 1 \

    --max_steps 100 \

    --search_depth 30 \

    --start_seed 0 \

    --train_ratio 0.8 \

    --visual_env \

    --max_action_per_step 3 \

    --max_action_penalty -0.1 \

    --format_reward 0.5 \

    --n_candidate 20000



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=gae \

    algorithm.high_level_gamma=0.95 \

    data.train_files=data/sokoban-vision-6-step/train.parquet \

    data.val_files=data/sokoban-vision-6-step/test.parquet \

    data.train_batch_size=128 \

    data.max_prompt_length=1024 \

    data.max_response_length=128 \

    data.max_trajectory_length=1800 \

    data.image_key=images \

    data.truncation=left \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=False \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=mse \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.2 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    actor_rollout_ref.rollout.top_p=0.95 \

    actor_rollout_ref.rollout.temperature=0.7 \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=True \

    critic.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    critic.model.fsdp_config.optimizer_offload=False \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='mask_loss' \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=100 \

    trainer.test_freq=20 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=3 \

    rollout_manager.window_size=5 \

    rollout_manager.use_multi_turn_reward=False \

    rollout_manager.use_loss_mask=True \

    rollout_manager.use_gae_mask=False \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=8 \

    rollout_manager.n_trajectory=1 \

    2>&1 | tee mask_loss.log

vagen/mllm_agent/rollout.py

+4 −0
Original line number Diff line number Diff line
@@ -511,6 +511,10 @@ class QwenVLRolloutManger(): 
            row_dict["multi_turn_token_level_rewards"] = multi_turn_token_level_rewards # (seq_len,) 

        if self.config.use_loss_mask:

            row_dict['loss_mask'] = loss_mask

        if self.config.use_gae_mask:

            row_dict['gae_mask'] = loss_mask

        if self.config.use_reward_mask:

            row_dict['reward_mask'] = end_of_response_position_mask

        row_dict["end_of_response_position_mask"] = end_of_response_position_mask # 

        position_ids = torch.cat([position_ids_prompt, position_ids_response], dim=-1)

        row_dict['prompts'] = input_ids_prompt

vagen/trainer/config/ppo_trainer.yaml

+2 −0
Original line number Diff line number Diff line
@@ -198,5 +198,7 @@ rollout_manager: 
  n_trajectory: 1 # >1 for grpo, please set actor_rollout_ref.rollout.n = 1 since we don't use vllm n sample parameter, set it >1 here for grpo agentic setting

  use_multi_turn_reward: True # for ablation study

  use_loss_mask: True # for ablation study

  use_reward_mask: True

  use_gae_mask: True

  special_token_for_loss_mask: ['<|box_start|>', '<|box_end|>']

  truncation: ${data.truncation}
 
 No newline at end of file

vagen/trainer/ppo/ray_trainer.py

+4 −6
Original line number Diff line number Diff line
@@ -160,11 +160,10 @@ def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_re 
        attention_mask = data.batch['attention_mask']

        response_mask = attention_mask[:, -response_length:]

        token_level_rewards = data.batch['token_level_rewards']

        assert "loss_mask" in data.batch.keys()

        loss_mask = data.batch['loss_mask'][:, -response_length:]

        gae_mask = data.batch['gae_mask'][:, -response_length:]

        advantages, returns =core_algos.compute_gae_advantage_return_with_loss_mask(token_level_rewards=token_level_rewards,

                                                                values=values,

                                                                loss_mask=loss_mask,

                                                                loss_mask=gae_mask,

                                                                gamma=gamma,

                                                                lam=lam)

        data.batch['advantages'] = advantages
@@ -208,10 +207,9 @@ def compute_advantage(data: DataProto, adv_estimator, gamma=1.0, lam=1.0, num_re 
        advantages, returns = core_algos.compute_turn_wise_gae_advantage_return(token_level_rewards=data.batch['token_level_rewards'],

                                                                        values=values,

                                                                        loss_mask=loss_mask,

                                                                        reward_masks=data.batch['end_of_response_position_mask'][:, -response_length:],

                                                                        lam=lam,

                                                                        high_level_gamma=high_level_gamma,

                                                                        reward_masks=data.batch['end_of_response_position_mask'][:, -response_length:])

                                                                        reward_mask=data.batch['end_of_response_position_mask'][:, -response_length:])

        

        data.batch['advantages'] = advantages

        data.batch['returns'] = returns
@@ -566,7 +564,7 @@ class RayPPOTrainer(object): 
            if config.critic.ppo_micro_batch_size is not None:

                assert config.critic.ppo_mini_batch_size % config.critic.ppo_micro_batch_size == 0

                assert config.critic.ppo_micro_batch_size * sp_size >= n_gpus

            if config.core_algos.adv_estimator == AdvantageEstimator.TURN_WISE_GAE:

            if config.algorithm.adv_estimator == AdvantageEstimator.TURN_WISE_GAE:

                assert config.critic.get('use_reward_mask', False), \

                    "TURN_WISE_GAE needs reward mask"

CRA Git | Maintained and supported by SUSTech CRA and CCSE