udated service (44d3df9a) · Commits · 张泽凯 / VAGEN

vagen/examples/debug_frozen_lake_vision_grpo/env_config.yaml→vagen/examples/debug_frozen_lake_text_grpo_service/env_config.yaml

+1 −1

Original line number	Diff line number	Diff line
		env1:
		env_name: frozenlake
		env_config:
		render_mode: vision
		render_mode: text
		train_size: 10000
		test_size: 512
		No newline at end of file

vagen/examples/debug_frozen_lake_vision_grpo/run.sh→vagen/examples/debug_frozen_lake_text_grpo_service/run.sh

+8 −7

Original line number	Diff line number	Diff line
		@@ -8,23 +8,23 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

		python -m vagen.env.create_dataset \
		--yaml_path "$SCRIPT_DIR/env_config.yaml" \
		--train_path "data/frozenlake-vision-debug/train.parquet" \
		--test_path "data/frozenlake-vision-debug/test.parquet" \
		--train_path "data/frozenlake-text-debug/train.parquet" \
		--test_path "data/frozenlake-text-debug/test.parquet" \

		# max_trajectory_length = max_prompt_length + max_response_length

		python3 -m vagen.trainer.main_ppo \
		algorithm.adv_estimator=grpo \
		algorithm.high_level_gamma=0.95 \
		data.train_files=data/frozenlake-vision-debug/train.parquet \
		data.val_files=data/frozenlake-vision-debug/test.parquet \
		data.train_files=data/frozenlake-text-debug/train.parquet \
		data.val_files=data/frozenlake-text-debug/test.parquet \
		data.train_batch_size=16 \
		data.max_prompt_length=1024 \
		data.max_response_length=128 \
		data.max_trajectory_length=1800 \
		data.image_key=images \
		data.truncation=error \
		actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \
		actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \
		actor_rollout_ref.actor.optim.lr=1e-6 \
		actor_rollout_ref.model.use_remove_padding=True \
		actor_rollout_ref.actor.ppo_mini_batch_size=32 \
		@@ -36,7 +36,7 @@ python3 -m vagen.trainer.main_ppo \
		actor_rollout_ref.actor.fsdp_config.param_offload=False \
		actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
		actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
		actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
		actor_rollout_ref.rollout.name=vllm \
		actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \
		actor_rollout_ref.rollout.enable_chunked_prefill=False \
		@@ -59,7 +59,7 @@ python3 -m vagen.trainer.main_ppo \
		trainer.logger=['console','wandb'] \
		trainer.project_name='vagen_debug' \
		trainer.experiment_name='grpo_mask_loss_frozenlake_vision_debug' \
		trainer.n_gpus_per_node=2 \
		trainer.n_gpus_per_node=1 \
		trainer.nnodes=1 \
		trainer.save_freq=100 \
		trainer.test_freq=20 \
		@@ -71,4 +71,5 @@ python3 -m vagen.trainer.main_ppo \
		trainer.val_before_train=True \
		trainer.val_generations_to_log_to_wandb=8 \
		rollout_manager.n_trajectory=8 \
		rollout_manager.use_service=True \
		2>&1 \| tee grpo_mask_loss.log

vagen/mllm_agent/rollout_service.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -245,8 +245,8 @@ class QwenVLRolloutMangerService():
		id_str = self.split+str(id)
		ids2configs_create[id_str] = cfg
		ids2seeds_reset[id_str] = cfg["seed"]
		self.envs[id_str] = REGISTERED_ENV[cfg["env_name"]]["env_cls"](**cfg["env_config"])
		self.env_client.create_batch(ids2configs_create)
		self.envs[id_str] = REGISTERED_ENV[cfg["env_name"]]["config_cls"](**cfg["env_config"])
		self.env_client.create_environments_batch(ids2configs_create)
		# Step 5: Reset environments
		reset_results=self.env_client.reset_batch(ids2seeds_reset)

vagen/trainer/config/ppo_trainer.yaml

+1 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ actor_rollout_ref:
		use_dynamic_bsz: False
		ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length}
		grad_clip: 1.0
		grad_norm_threshold: 10000
		clip_ratio: 0.2
		entropy_coeff: 0.001
		use_kl_loss: False # True for GRPO

Admin message