Commit 3ec8a5cf authored by williamzhangNU's avatar williamzhangNU
Browse files

minor update

parent 006a3d09

vagen/env/sokoban/env.py

+3 −3
Original line number Diff line number Diff line
@@ -206,9 +206,9 @@ class PreprocessResult: 
class SokobanInterface(BaseInterface):



    INVALID_ACTION = 0

    FORMAT_REWARD = 0.1

    FORMAT_PENALTY = -0.1

    VALID_ACTION_REWARD = 0.2

    FORMAT_REWARD = 0.5

    FORMAT_PENALTY = -0.5

    VALID_ACTION_REWARD = 0.5

    ACTION_LOOKUP = {

        0: "None",

        1: "Up",

vagen/examples/sokoban/debug_qwen0_5_1_gpu_grpo.sh

+20 −19
Original line number Diff line number Diff line
@@ -5,8 +5,8 @@ export VLLM_ATTENTION_BACKEND=XFORMERS 
export PYTHONHASHSEED=0



python -m vagen.env.sokoban.create_dataset \

    --data_dir data/sokoban-text-1-step \

    --max_action_length 1 \

    --data_dir data/sokoban-text-2-step \

    --max_action_length 2 \

    --dim_room 6 6 \

    --num_boxes 1 \

    --max_steps 100 \
@@ -20,47 +20,48 @@ python -m vagen.env.sokoban.create_dataset \ 


python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban-text-1-step/train.parquet \

    data.val_files=data/sokoban-text-1-step/test.parquet \

    data.train_batch_size=16 \

    data.max_prompt_length=1024 \

    data.train_files=data/sokoban-text-2-step/train.parquet \

    data.val_files=data/sokoban-text-2-step/test.parquet \

    data.train_batch_size=128 \

    data.max_prompt_length=768 \

    data.max_response_length=128 \

    data.max_trajectory_length=1024 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-1.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=128 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.ppo_mini_batch_size=64 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=low_var_kl \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    +actor_rollout_ref.ref.use_ref=True \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='debug_qwen0_5_1_gpu_grpo' \

    trainer.experiment_name='debug_qwen0_5_1_gpu_grpo_2_steps' \

    trainer.n_gpus_per_node=1 \

    trainer.nnodes=1 \

    trainer.save_freq=50 \

    trainer.test_freq=2 \

    trainer.test_freq=5 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=1 \

    rollout_manager.window_size=5 \

    rollout_manger.max_turns=2 \

    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=2 \

    rollout_manager.n_trajectory=8 \

    2>&1 | tee debug_qwen0_5_1_gpu_grpo.log
 
    trainer.val_generations_to_log_to_wandb=4 \

    rollout_manger.n_trajectory=2 \

    2>&1 | tee debug_qwen0_5_1_gpu_grpo_2_steps.log

vagen/examples/sokoban/debug_qwen0_5_1_gpu_ppo_multi_turn_ppo.sh

+9 −8
Original line number Diff line number Diff line
@@ -4,8 +4,8 @@ export VLLM_ATTENTION_BACKEND=XFORMERS 
export PYTHONHASHSEED=0



python -m vagen.env.sokoban.create_dataset \

    --data_dir data/sokoban-text-1-step \

    --max_action_length 1 \

    --data_dir data/sokoban-text-3-step \

    --max_action_length 3 \

    --dim_room 6 6 \

    --num_boxes 1 \

    --max_steps 100 \
@@ -18,12 +18,13 @@ python -m vagen.env.sokoban.create_dataset \ 


python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=multi_turn_gae \

    data.train_files=data/sokoban-text-1-step/train.parquet \

    data.val_files=data/sokoban-text-1-step/test.parquet \

    algorithm.high_level_gamma=0.95 \

    data.train_files=data/sokoban-text-3-step/train.parquet \

    data.val_files=data/sokoban-text-3-step/test.parquet \

    data.train_batch_size=64 \

    data.max_prompt_length=768 \

    data.max_prompt_length=1408 \

    data.max_response_length=128 \

    data.max_trajectory_length=1024 \

    data.max_trajectory_length=1536 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \
@@ -63,9 +64,9 @@ python3 -m vagen.trainer.main_ppo \ 
    trainer.save_freq=100 \

    trainer.test_freq=5 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=2 \

    rollout_manager.max_turns=3 \

    rollout_manager.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=2 \

    trainer.val_generations_to_log_to_wandb=4 \

    rollout_manager.n_trajectory=2 \

    2>&1 | tee debug_qwen0_5_1_gpu_ppo_multi_turn_ppo.log

vagen/examples/sokoban/debug_qwen2_5_vl_4gpu_grpo.sh

+6 −6
Original line number Diff line number Diff line
@@ -5,8 +5,8 @@ export PYTHONHASHSEED=0 


python -m vagen.env.sokoban.create_dataset \

    --visual_env \

    --data_dir data/sokoban-vision-1-step \

    --max_action_length 1 \

    --data_dir data/sokoban-vision-2-step \

    --max_action_length 2 \

    --dim_room 6 6 \

    --num_boxes 1 \

    --max_steps 100 \
@@ -17,12 +17,12 @@ python -m vagen.env.sokoban.create_dataset \ 


python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban-vision-1-step/train.parquet \

    data.val_files=data/sokoban-vision-1-step/test.parquet \

    data.train_files=data/sokoban-vision-2-step/train.parquet \

    data.val_files=data/sokoban-vision-2-step/test.parquet \

    data.train_batch_size=16 \

    data.max_prompt_length=768 \

    data.max_prompt_length=1920 \

    data.max_response_length=128 \

    data.max_trajectory_length=1024 \

    data.max_trajectory_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

vagen/mllm_agent/rollout.py

+1 −0
Original line number Diff line number Diff line
@@ -500,6 +500,7 @@ class QwenVLRolloutManger(): 
            for idx,reward in enumerate(rewards):

                multi_turn_token_level_rewards[reward_positions[idx]] = reward

            row_dict["multi_turn_token_level_rewards"] = multi_turn_token_level_rewards # (seq_len,) 

            row_dict["reward_masks"] = token_level_rewards_mask

        if self.config.use_loss_mask:

            row_dict['loss_mask'] = loss_mask

        position_ids = torch.cat([position_ids_prompt, position_ids_response], dim=-1)

CRA Git | Maintained and supported by SUSTech CRA and CCSE