minor udpate

from vagen.env.register import REGISTERED_ENVS, register

from vagen.env.sokoban.env import SokobanInterface

 No newline at end of file

        # parse format and action list

        # if action_list:

        #     reward += self.interface_config['format_reward']

        if not action_list:

            reward -= self.interface_config['format_reward']

        if action_list:

            reward += self.interface_config['format_reward']

        if len(action_list) > self.interface_config['max_action_per_step']:

            reward += self.interface_config['max_action_penalty']

            action_list = action_list[:self.interface_config['max_action_per_step']]

2. Avoid walls (#).

Actions you can take: Up, Down, Left, Right. You can take up to {max_action_per_step} action(s) at a time.

Up: move up to the cell above (to the above row).

Down: move down to the cell below (to the below row).

Left: move left to the cell to the left (to the left column).

Right: move right to the cell to the right (to the right column).

- Up: move up to the cell above

- Down: move down to the cell below

- Left: move left to the cell to the left

- Right: move right to the cell to the right

If there is a box on the cell you want to move to, you will push the box one cell in the same direction.

Rewards:

Box on target: +1.0

Format correct: +0.5

Please think step by step and provide the actions you want to take.

Your response should STRICTLY follow the format: <think>[Your thoughts]</think><answer>[Your actions]</answer>

You should wrap your thought between `<think>` and `</think>` tags, and wrap your answer between `<answer>` and `</answer>` tags.

Your response should STRICTLY follow the format:

<think>...</think><answer>...</answer>

"""

# E.g. <think> There's a box on the upper right of me, the target is on the upper side of the box, I need to push the box it upward. </think><answer> Right,Up,Up </answer>

# Let's try to use a format reward and answer reward

[Initial Observation]:

{observation}

Decide your next action(s).

Your response should STRICTLY follow the format: <think>[Your thoughts]</think><answer>[Your actions]</answer>

Your response should STRICTLY follow the format:

<think>...</think><answer>...</answer>

"""

action_template = """After your answer, the extracted valid action is {valid_action}.\

action_template = """Valid action extracted from your response is {valid_action}.\

After that, the observation is:

{observation}

reward: {reward}

done: {done}

Decide your next action(s).

Your response should STRICTLY follow the format: <think>[Your thoughts]</think><answer>[Your actions]</answer>

Your response should STRICTLY follow the format:

<think>...</think><answer>...</answer>

"""

 No newline at end of file

    --start_seed 0 \

    --train_ratio 0.8 \

    --max_action_per_step 1 \

    --max_action_penalty -0.1 \

    --format_reward 1 \

    --n_candidate 20000 \

    --max_action_penalty 0.0 \

    --format_reward 0.5 \

    --n_candidate 50000 \

    --force-gen

if [ $? -ne 0 ]; then

    data.max_response_length=256 \

    data.max_trajectory_length=1664 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-1.5B-Instruct \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=False \

    critic.model.path=Qwen/Qwen2.5-1.5B-Instruct \

    critic.model.path=Qwen/Qwen2.5-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='debug_single_action_3_turns_ppo_1_5B_kl_penalty_masked_gae' \

    trainer.n_gpus_per_node=2 \

    trainer.experiment_name='debug_single_action_3_turns_ppo_3B_masked_gae_temp_0.7_top_p_0.95' \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=100 \

    trainer.save_freq=400 \

    trainer.test_freq=5 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=3 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=8 \

    rollout_manager.n_trajectory=1 \

    2>&1 | tee debug_single_action_3_turns_ppo_1_5B_kl_penalty_masked_gae.log

    2>&1 | tee debug_single_action_3_turns_ppo_3B_masked_gae_temp_0.7_top_p_0.95.log

    --start_seed 0 \

    --train_ratio 0.8 \

    --max_action_per_step 1 \

    --max_action_penalty -0.1 \

    --format_reward 1 \

    --n_candidate 20000 \

    --max_action_penalty 0.0 \

    --format_reward 0.5 \

    --n_candidate 50000 \

    --force-gen

if [ $? -ne 0 ]; then

    data.max_response_length=256 \

    data.max_trajectory_length=1664 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-1.5B-Instruct \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=False \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=False \

    critic.model.path=Qwen/Qwen2.5-1.5B-Instruct \

    critic.model.path=Qwen/Qwen2.5-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='debug_single_action_3_turns_ppo_1_5B_kl_penalty_multi_turn_gae' \

    trainer.n_gpus_per_node=2 \

    trainer.experiment_name='debug_single_action_3_turns_ppo_3B_multi_turn_gae_temp_0.7_top_p_0.95' \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=100 \

    trainer.save_freq=400 \

    trainer.test_freq=5 \

    trainer.total_epochs=15 \

    rollout_manager.max_turns=3 \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=8 \

    rollout_manager.n_trajectory=1 \

    2>&1 | tee debug_single_action_3_turns_ppo_1_5B_kl_penalty_multi_turn_gae.log

    2>&1 | tee debug_single_action_3_turns_ppo_3B_multi_turn_gae_temp_0.7_top_p_0.95.log