temp (e9b261c2) · Commits · 张泽凯 / VAGEN

vagen/env/sokoban/env.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -101,6 +101,7 @@ class SokobanEnv(BaseEnv):
		info["metrics"] = metrics
		metrics['turn_metrics']['action_is_effective'] = not np.array_equal(prev_player_position, self.env.player_position)
		self.total_reward += self.reward

		return self._render(init_obs=False), self.reward, done, info

		def system_prompt(self):

vagen/env/utils/env_utils.py

+1 −0

Original line number	Diff line number	Diff line
		# env_utils.py
		import os
		import random
		import logging
		import numpy as np

vagen/examples/sokoban/debug_single_action_1_turn_grpo_strict_format.sh

+6 −2

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@ python3 -m vagen.trainer.main_ppo \
		algorithm.high_level_gamma=0.95 \
		data.train_files=data/sokoban-text-1-step/train.parquet \
		data.val_files=data/sokoban-text-1-step/test.parquet \
		data.train_batch_size=512 \
		data.train_batch_size=10 \
		data.max_prompt_length=768 \
		data.max_response_length=256 \
		data.max_trajectory_length=1024 \
		@@ -26,6 +26,8 @@ python3 -m vagen.trainer.main_ppo \
		actor_rollout_ref.model.use_remove_padding=False \
		actor_rollout_ref.actor.ppo_mini_batch_size=16 \
		actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.actor.ppo_mini_batch_size=1 \
		actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.actor.use_kl_loss=True \
		actor_rollout_ref.actor.kl_loss_coef=0.001 \
		actor_rollout_ref.actor.kl_loss_type=mse \
		@@ -33,6 +35,7 @@ python3 -m vagen.trainer.main_ppo \
		actor_rollout_ref.actor.fsdp_config.param_offload=False \
		actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \
		actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
		actor_rollout_ref.rollout.name=vllm \
		actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \
		@@ -41,6 +44,7 @@ python3 -m vagen.trainer.main_ppo \
		actor_rollout_ref.rollout.free_cache_engine=False \
		actor_rollout_ref.rollout.n=1 \
		actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \
		actor_rollout_ref.ref.fsdp_config.param_offload=True \
		+actor_rollout_ref.ref.use_ref=True \
		algorithm.kl_ctrl.kl_coef=0.001 \
		@@ -48,7 +52,7 @@ python3 -m vagen.trainer.main_ppo \
		trainer.logger=['console','wandb'] \
		trainer.project_name='vagen' \
		trainer.experiment_name='debug_single_action_single_turn_grpo_0_5B_kl_strict_format' \
		trainer.n_gpus_per_node=1 \
		trainer.n_gpus_per_node=2 \
		trainer.nnodes=1 \
		trainer.save_freq=100 \
		trainer.test_freq=5 \

Admin message