Commit be81cb27 authored by williamzhangNU's avatar williamzhangNU
Browse files

minor update

parent cf4d0815

vagen/examples/sokoban/debug_qwen0_5.sh

+14 −9
Original line number Diff line number Diff line
@@ -4,42 +4,47 @@ export VLLM_ATTENTION_BACKEND=XFORMERS 


python -m vagen.env.sokoban.create_dataset --data_dir data/sokoban-text



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban-text/train.parquet \

    data.val_files=data/sokoban-text/test.parquet \

    data.train_batch_size=16 \

    data.train_batch_size=32 \

    data.max_prompt_length=512 \

    data.max_response_length=512 \

    data.max_trajectory_length=3072 \

    data.max_response_length=1536 \

    +data.max_trajectory_length=2048 \

    +data.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_trajectory_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=128 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \

    actor_rollout_ref.actor.ppo_mini_batch_size=8 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=low_var_kl \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='qwen2_5_05b_function_rm' \

    trainer.n_gpus_per_node=4 \

    trainer.n_gpus_per_node=1 \

    trainer.nnodes=1 \

    trainer.save_freq=-1 \

    trainer.test_freq=-1 \

vagen/examples/sokoban/debug_qwen2_5_vl.sh

+8 −5
Original line number Diff line number Diff line
@@ -9,9 +9,12 @@ python3 -m vagen.trainer.main_ppo \ 
    data.train_files=data/sokoban/train.parquet \

    data.val_files=data/sokoban/test.parquet \

    data.train_batch_size=16 \

    data.max_prompt_length=2048 \

    data.max_response_length=256 \

    data.max_trajectory_length=3072 \

    data.max_prompt_length=512 \

    data.max_response_length=1536 \

    data.max_trajectory_length=2048 \

    +data.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_response_per_turn=256 \

    +actor_rollout_ref.rollout.max_trajectory_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \
@@ -25,7 +28,7 @@ python3 -m vagen.trainer.main_ppo \ 
    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \
@@ -39,7 +42,7 @@ python3 -m vagen.trainer.main_ppo \ 
    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='qwen2_5_vl_3b_function_rm' \

    trainer.n_gpus_per_node=4 \

    trainer.n_gpus_per_node=2 \

    trainer.nnodes=1 \

    trainer.save_freq=50 \

    trainer.test_freq=2 \

vagen/mllm_agent/rollout.py

+3 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ from vagen.env.base import EnvConfig,IMAGE_PLACEHOLDER 
class QwenVLRolloutConifg:

    window_size: int = 5

    max_trajectory_length: int = 3072

    max_response_per_turn: int = 256

    max_turns: int = 5

    n_gpu_per_node: int = 1 # used for multigpu batch balancing

    sptk_for_loss_mask: List[str] = field(default_factory=lambda: ['<|box_start|>', '<|box_end|>'])
@@ -536,6 +537,8 @@ class QwenVLRolloutManger(): 
                    raw_prompt_ids_array[i] = raw_prompt_ids[i]

                else:

                    raw_prompt_ids_array[i] = raw_prompt_ids[i].tolist()

                print(f"[DEBUG] raw_prompt_ids_array({i}) length: {len(raw_prompt_ids_array[i])}")

                print(f"[DEBUG] raw_prompt_ids_array({i}) content: {self.tokenizer.decode(raw_prompt_ids_array[i])}")

            gen_batch.non_tensor_batch['raw_prompt_ids'] = raw_prompt_ids_array

            

            output_batch = self.actor_rollout_wg.generate_sequences(gen_batch)

vagen/trainer/ppo/ray_trainer.py

+2 −7
Original line number Diff line number Diff line
@@ -968,8 +968,9 @@ class RayPPOTrainer(object): 




        rollout_config = QwenVLRolloutConifg(

            max_trajectory_length=self.config.data.max_trajectory_length,

            max_turns=self.config.max_turns,

            max_trajectory_length=self.config.data.max_trajectory_length,

            max_response_per_turn=self.config.data.max_response_per_turn,

            n_gpu_per_node=self.config.trainer.n_gpus_per_node,

        )

        rollout_manager = QwenVLRolloutManger(
@@ -1049,12 +1050,6 @@ class RayPPOTrainer(object): 
                        final_gen_batch_output = rollout_manager.get_final_trajectory()



                    with torch.no_grad():

                        print(f"[DEBUG] shape of input_ids: {final_gen_batch_output.batch['input_ids'].shape}")

                        print(f"[DEBUG] shape of attention_mask: {final_gen_batch_output.batch['attention_mask'].shape}")

                        print(f"[DEBUG] shape of position_ids: {final_gen_batch_output.batch['position_ids'].shape}")

                        print(f"[DEBUG] shape of loss_mask: {final_gen_batch_output.batch['loss_mask'].shape}")

                        print(f"[DEBUG] shape of prompts: {final_gen_batch_output.batch['prompts'].shape}")

                        print(f"[DEBUG] shape of responses: {final_gen_batch_output.batch['responses'].shape}")

                        output = self.actor_rollout_wg.compute_log_prob(final_gen_batch_output)

                        final_gen_batch_output = final_gen_batch_output.union(output)

                    batch.non_tensor_batch['uid'] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))],

CRA Git | Maintained and supported by SUSTech CRA and CCSE