Commit bb7e2d30 authored by jameskrw's avatar jameskrw
Browse files

sokoban text is now runnable (grpo, 1/4 A100s)

parent 9cd80974

README.md

+13 −0
Original line number Diff line number Diff line
@@ -11,3 +11,16 @@ cd vagen 
bash scripts/install.sh

```



## Run

```

bash vagen/examples/sokoban/debug_qwen0_5_1_gpu_grpo.sh

bash vagen/examples/sokoban/debug_qwen0_5_4_gpu_ppo.sh



# Verified on 1 and 4 A100 GPUs

```



## TODO

1. Implement real grpo: rollout.n>1

2. Make PPO runnable (modify ppo scripts and code)

3. Add more metrics and image visualization

vagen/examples/sokoban/debug_qwen2_5_vl.sh→vagen/examples/sokoban/debug_qwen0_5_1_gpu_grpo.sh

+20 −17
Original line number Diff line number Diff line
@@ -2,24 +2,24 @@ set -x 


export VLLM_ATTENTION_BACKEND=XFORMERS



python -m vagen.env.sokoban.create_dataset --visual_env --data_dir data/sokoban

python -m vagen.env.sokoban.create_dataset --data_dir data/sokoban-text



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban/train.parquet \

    data.val_files=data/sokoban/test.parquet \

    data.train_batch_size=2 \

    data.max_prompt_length=512 \

    data.max_response_length=1536 \

    data.train_files=data/sokoban-text/train.parquet \

    data.val_files=data/sokoban-text/test.parquet \

    data.train_batch_size=32 \

    data.max_prompt_length=1024 \

    data.max_response_length=128 \

    data.max_trajectory_length=2048 \

    +data.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_trajectory_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=2 \

    #Set use_remove_padding to false, if true, causing batch size must be postive error in vllm

    actor_rollout_ref.model.use_remove_padding=False \ 

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \
@@ -28,7 +28,7 @@ python3 -m vagen.trainer.main_ppo \ 
    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \
@@ -41,11 +41,14 @@ python3 -m vagen.trainer.main_ppo \ 
    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='qwen2_5_vl_3b_function_rm' \

    trainer.n_gpus_per_node=2 \

    trainer.experiment_name='qwen2_5_05b_function_rm' \

    trainer.n_gpus_per_node=1 \

    trainer.nnodes=1 \

    trainer.save_freq=50 \

    trainer.test_freq=2 \

    trainer.total_epochs=15 \

    +max_turns=2 \

    2>&1 | tee debug.log
 
    rollout_manger.max_turns=2 \

    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=5 \

    2>&1 | tee debug_qwen0_5_1_gpu_grpo.log

vagen/examples/sokoban/debug_qwen0_5_1_gpu_ppo.sh

0 → 100644
+52 −0
Original line number Diff line number Diff line 
# set -x



# export VLLM_ATTENTION_BACKEND=XFORMERS



# python -m vagen.env.sokoban.create_dataset --data_dir data/sokoban-text



# # max_trajectory_length = max_prompt_length + max_response_length



# python3 -m vagen.trainer.main_ppo \

#     algorithm.adv_estimator=ppo \

#     data.train_files=data/sokoban-text/train.parquet \

#     data.val_files=data/sokoban-text/test.parquet \

#     data.train_batch_size=32 \

#     data.max_prompt_length=2048 \

#     data.max_response_length=128 \

#     data.max_trajectory_length=3072 \

#     data.image_key=images \

#     actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

#     actor_rollout_ref.actor.optim.lr=1e-6 \

#     actor_rollout_ref.model.use_remove_padding=True \

#     actor_rollout_ref.actor.ppo_mini_batch_size=32 \

#     actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \

#     actor_rollout_ref.actor.use_kl_loss=True \

#     actor_rollout_ref.actor.kl_loss_coef=0.001 \

#     actor_rollout_ref.actor.kl_loss_type=low_var_kl \

#     actor_rollout_ref.model.enable_gradient_checkpointing=True \

#     actor_rollout_ref.actor.fsdp_config.param_offload=False \

#     actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

#     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \

#     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

#     actor_rollout_ref.rollout.name=vllm \

#     actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \

#     actor_rollout_ref.rollout.enable_chunked_prefill=False \

#     actor_rollout_ref.rollout.enforce_eager=False \

#     actor_rollout_ref.rollout.free_cache_engine=False \

#     actor_rollout_ref.rollout.n=1 \

#     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \

#     actor_rollout_ref.ref.fsdp_config.param_offload=True \

#     algorithm.kl_ctrl.kl_coef=0.001 \

#     trainer.critic_warmup=0 \

#     trainer.logger=['console','wandb'] \

#     trainer.project_name='vagen' \

#     trainer.experiment_name='qwen2_5_05b_function_rm' \

#     trainer.n_gpus_per_node=1 \

#     trainer.nnodes=1 \

#     trainer.save_freq=50 \

#     trainer.test_freq=2 \

#     trainer.total_epochs=15 \

#     rollout_manger.max_turns=2 \

#     rollout_manger.window_size=5 \

#     trainer.val_before_train=True \

#     2>&1 | tee debug_qwen0_5_1_gpu_ppo.log

vagen/examples/sokoban/debug_qwen0_5.sh→vagen/examples/sokoban/debug_qwen0_5_4_gpu_grpo.sh

+17 −17
Original line number Diff line number Diff line
@@ -11,43 +11,43 @@ python3 -m vagen.trainer.main_ppo \ 
    data.train_files=data/sokoban-text/train.parquet \

    data.val_files=data/sokoban-text/test.parquet \

    data.train_batch_size=32 \

    data.max_prompt_length=512 \

    data.max_response_length=1536 \

    +data.max_trajectory_length=2048 \

    +data.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_trajectory_length=2048 \

    data.max_prompt_length=1024 \

    data.max_response_length=128 \

    data.max_trajectory_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=8 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=2 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=low_var_kl \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=2 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=2 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='qwen2_5_05b_function_rm' \

    trainer.n_gpus_per_node=1 \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=-1 \

    trainer.test_freq=-1 \

    trainer.save_freq=50 \

    trainer.test_freq=2 \

    trainer.total_epochs=15 \

    +max_turns=5 \

    2>&1 | tee debug.log
 
    rollout_manger.max_turns=2 \

    rollout_manger.window_size=5 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=5 \

    2>&1 | tee debug_qwen0_5_4_gpu_grpo.log

vagen/examples/sokoban/debug_qwen0_5_4_gpu_ppo.sh

0 → 100644
+53 −0
Original line number Diff line number Diff line 
# set -x



# export VLLM_ATTENTION_BACKEND=XFORMERS



# python -m vagen.env.sokoban.create_dataset --data_dir data/sokoban-text



# # max_trajectory_length = max_prompt_length + max_response_length



# python3 -m vagen.trainer.main_ppo \

#     algorithm.adv_estimator=grpo \

#     data.train_files=data/sokoban-text/train.parquet \

#     data.val_files=data/sokoban-text/test.parquet \

#     data.train_batch_size=32 \

#     data.max_prompt_length=512 \

#     data.max_response_length=1536 \

#     +data.max_trajectory_length=2048 \

#     +data.max_response_per_turn=256 \

#     +actor_rollout_ref.rollout.max_response_per_turn=256 \

#     +actor_rollout_ref.rollout.max_trajectory_length=2048 \

#     data.image_key=images \

#     actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

#     actor_rollout_ref.actor.optim.lr=1e-6 \

#     actor_rollout_ref.model.use_remove_padding=True \

#     actor_rollout_ref.actor.ppo_mini_batch_size=8 \

#     actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \

#     actor_rollout_ref.actor.use_kl_loss=True \

#     actor_rollout_ref.actor.kl_loss_coef=0.001 \

#     actor_rollout_ref.actor.kl_loss_type=low_var_kl \

#     actor_rollout_ref.model.enable_gradient_checkpointing=True \

#     actor_rollout_ref.actor.fsdp_config.param_offload=False \

#     actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

#     actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \

#     actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

#     actor_rollout_ref.rollout.name=vllm \

#     actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \

#     actor_rollout_ref.rollout.enable_chunked_prefill=False \

#     actor_rollout_ref.rollout.enforce_eager=False \

#     actor_rollout_ref.rollout.free_cache_engine=False \

#     actor_rollout_ref.rollout.n=1 \

#     actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \

#     actor_rollout_ref.ref.fsdp_config.param_offload=True \

#     algorithm.kl_ctrl.kl_coef=0.001 \

#     trainer.critic_warmup=0 \

#     trainer.logger=['console','wandb'] \

#     trainer.project_name='vagen' \

#     trainer.experiment_name='qwen2_5_05b_function_rm' \

#     trainer.n_gpus_per_node=1 \

#     trainer.nnodes=1 \

#     trainer.save_freq=-1 \

#     trainer.test_freq=-1 \

#     trainer.total_epochs=15 \

#     +max_turns=5 \

#     2>&1 | tee debug_qwen0_5_4_gpu_ppo.log

CRA Git | Maintained and supported by SUSTech CRA and CCSE