minor update (cf4d0815) · Commits · 张泽凯 / VAGEN

vagen/examples/sokoban/debug_qwen0_5.sh

+7 −7

Original line number	Diff line number	Diff line
		@@ -16,33 +16,33 @@ python3 -m vagen.trainer.main_ppo \
		actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \
		actor_rollout_ref.actor.optim.lr=1e-6 \
		actor_rollout_ref.model.use_remove_padding=True \
		actor_rollout_ref.actor.ppo_mini_batch_size=4 \
		actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.actor.ppo_mini_batch_size=128 \
		actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=8 \
		actor_rollout_ref.actor.use_kl_loss=True \
		actor_rollout_ref.actor.kl_loss_coef=0.001 \
		actor_rollout_ref.actor.kl_loss_type=low_var_kl \
		actor_rollout_ref.model.enable_gradient_checkpointing=True \
		actor_rollout_ref.actor.fsdp_config.param_offload=False \
		actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
		actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
		actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \
		actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
		actor_rollout_ref.rollout.name=vllm \
		actor_rollout_ref.rollout.gpu_memory_utilization=0.7 \
		actor_rollout_ref.rollout.enable_chunked_prefill=False \
		actor_rollout_ref.rollout.enforce_eager=False \
		actor_rollout_ref.rollout.free_cache_engine=False \
		actor_rollout_ref.rollout.n=1 \
		actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=8 \
		actor_rollout_ref.ref.fsdp_config.param_offload=True \
		algorithm.kl_ctrl.kl_coef=0.001 \
		trainer.critic_warmup=0 \
		trainer.logger=['console','wandb'] \
		trainer.project_name='vagen' \
		trainer.experiment_name='qwen2_5_05b_function_rm' \
		trainer.n_gpus_per_node=1 \
		trainer.n_gpus_per_node=4 \
		trainer.nnodes=1 \
		trainer.save_freq=-1 \
		trainer.test_freq=-1 \
		trainer.total_epochs=15 \
		+max_turns=2 \
		+max_turns=5 \
		2>&1 \| tee debug.log

vagen/mllm_agent/rollout.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -452,7 +452,6 @@ class QwenVLRolloutManger():
		# if self.image_key in row_dict:
		if has_images:
		from verl.models.transformers.qwen2_vl import get_rope_index
		# NOTE please check the way position ids are computed in vllm_rollout_spmd.py
		position_ids_response = get_rope_index(
		self.processor,
		image_grid_thw=image_grid_thw,

vagen/trainer/ppo/ray_trainer.py

+6 −0

Original line number	Diff line number	Diff line
		@@ -1049,6 +1049,12 @@ class RayPPOTrainer(object):
		final_gen_batch_output = rollout_manager.get_final_trajectory()

		with torch.no_grad():
		print(f"[DEBUG] shape of input_ids: {final_gen_batch_output.batch['input_ids'].shape}")
		print(f"[DEBUG] shape of attention_mask: {final_gen_batch_output.batch['attention_mask'].shape}")
		print(f"[DEBUG] shape of position_ids: {final_gen_batch_output.batch['position_ids'].shape}")
		print(f"[DEBUG] shape of loss_mask: {final_gen_batch_output.batch['loss_mask'].shape}")
		print(f"[DEBUG] shape of prompts: {final_gen_batch_output.batch['prompts'].shape}")
		print(f"[DEBUG] shape of responses: {final_gen_batch_output.batch['responses'].shape}")
		output = self.actor_rollout_wg.compute_log_prob(final_gen_batch_output)
		final_gen_batch_output = final_gen_batch_output.union(output)
		batch.non_tensor_batch['uid'] = np.array([str(uuid.uuid4()) for _ in range(len(batch.batch))],

Admin message