Commit 7aa338ab authored by jameskrw's avatar jameskrw Committed by YaningGao
Browse files

updated navigation and maniskill format exps

parent 5b4e7abe

scripts/exps/format/maniskill/freethink/env_config.yaml

0 → 100644
+36 −0
Original line number Diff line number Diff line 
env1:

    env_name: primitive_skill 

    env_config:

        render_mode: vision

        prompt_format: free_think

        use_accuracy_reward: false

        env_id: "AlignTwoCube" # AlignTwoCube,PlaceTwoCube,PutAppleInDrawer,StackThreeCube

    train_size: 10000  

    test_size: 8

env2:

    env_name: primitive_skill 

    env_config:

        render_mode: vision

        prompt_format: free_think

        use_accuracy_reward: false

        env_id: "PlaceTwoCube" # AlignTwoCube,PlaceTwoCube,PutAppleInDrawer,StackThreeCube

    train_size: 10000  

    test_size: 8

env3:

    env_name: primitive_skill 

    env_config:

        render_mode: vision

        prompt_format: free_think

        use_accuracy_reward: false

        env_id: "PutAppleInDrawer" # AlignTwoCube,PlaceTwoCube,PutAppleInDrawer,StackThreeCube

    train_size: 10000  

    test_size: 8

env4:

    env_name: primitive_skill 

    env_config:

        render_mode: vision

        prompt_format: free_think

        use_accuracy_reward: false

        env_id: "StackThreeCube" # AlignTwoCube,PlaceTwoCube,PutAppleInDrawer,StackThreeCube

    train_size: 10000  

    test_size: 8

scripts/exps/format/maniskill/freethink/run.sh

0 → 100644
+91 −0
Original line number Diff line number Diff line 
set -x





export VLLM_ATTENTION_BACKEND=XFORMERS

export PYTHONHASHSEED=0



SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"



# run 

# python -m vagen.server.server

# in a tmux session first



# Extract experiment name from the path

# This will take the last 3 parts of the path: format/sokoban/free_think

EXPERIMENT_NAME=$(echo $SCRIPT_DIR | rev | cut -d'/' -f1-3 | rev | tr '/' '-')



echo "Experiment name: $EXPERIMENT_NAME"

# run 

# python -m vagen.server.server server.port=5001 

# in a tmux session first

python -m vagen.env.create_dataset \

    --yaml_path "$SCRIPT_DIR/env_config.yaml" \

    --train_path "data/$EXPERIMENT_NAME/train.parquet" \

    --test_path "data/$EXPERIMENT_NAME/test.parquet" \



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=masked_gae \

    algorithm.high_level_gamma=0.95 \

    data.train_files=data/$EXPERIMENT_NAME/train.parquet \

    data.val_files=data/$EXPERIMENT_NAME/test.parquet \

    data.train_batch_size=64 \

    data.val_batch_size=32 \

    data.max_prompt_length=1024 \

    data.max_response_length=256 \

    data.max_trajectory_length=3000 \

    data.image_key=images \

    data.truncation=error \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=False \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=mse \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    actor_rollout_ref.rollout.top_p=0.95 \

    actor_rollout_ref.rollout.temperature=0.7 \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=True \

    critic.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    critic.model.fsdp_config.optimizer_offload=False \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen_new' \

    trainer.experiment_name=$EXPERIMENT_NAME \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=90 \

    trainer.test_freq=20 \

    trainer.total_training_steps=200 \

    rollout_manager.max_turns=3 \

    rollout_manager.window_size=3 \

    rollout_manager.use_multi_turn_reward=False \

    rollout_manager.use_loss_mask=True \

    rollout_manager.use_gae_mask=True \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=8 \

    rollout_manager.n_trajectory=2 \

    rollout_manager.use_service=True \

    rollout_manager.timeout=240 \

    rollout_manager.base_url="http://localhost:5000" \

    2>&1 | tee $EXPERIMENT_NAME.log

scripts/exps/format/navigation/freethink/env_config.yaml

0 → 100644
+8 −0
Original line number Diff line number Diff line 
env1:

    env_name: navigation  

    env_config:

        render_mode: vision

        prompt_format: free_think

        use_accuracy_reward: false

    train_size: 10000  

    test_size: 128
 
 No newline at end of file

scripts/exps/format/navigation/freethink/run.sh

0 → 100644
+87 −0
Original line number Diff line number Diff line 
set -x





export VLLM_ATTENTION_BACKEND=XFORMERS

export PYTHONHASHSEED=0



SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"



# Extract experiment name from the path

# This will take the last 3 parts of the path: format/sokoban/free_think

EXPERIMENT_NAME=$(echo $SCRIPT_DIR | rev | cut -d'/' -f1-3 | rev | tr '/' '-')



echo "Experiment name: $EXPERIMENT_NAME"

# run 

# python -m vagen.server.server server.port=5001 

# in a tmux session first

python -m vagen.env.create_dataset \

    --yaml_path "$SCRIPT_DIR/env_config.yaml" \

    --train_path "data/$EXPERIMENT_NAME/train.parquet" \

    --test_path "data/$EXPERIMENT_NAME/test.parquet" \



# max_trajectory_length = max_prompt_length + max_response_length



python3 -m vagen.trainer.main_ppo \

    algorithm.adv_estimator=masked_gae \

    algorithm.high_level_gamma=0.95 \

    data.train_files=data/$EXPERIMENT_NAME/train.parquet \

    data.val_files=data/$EXPERIMENT_NAME/test.parquet \

    data.train_batch_size=64 \

    data.val_batch_size=16 \

    data.max_prompt_length=1024 \

    data.max_response_length=256 \

    data.max_trajectory_length=3800 \

    data.image_key=images \

    data.truncation=error \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=32 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=False \

    actor_rollout_ref.actor.kl_loss_coef=0.001 \

    actor_rollout_ref.actor.kl_loss_type=mse \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=4 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.3 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    actor_rollout_ref.rollout.top_p=0.95 \

    actor_rollout_ref.rollout.temperature=0.7 \

    critic.optim.lr=1e-5 \

    critic.model.use_remove_padding=True \

    critic.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    critic.model.enable_gradient_checkpointing=True \

    critic.ppo_micro_batch_size_per_gpu=1 \

    critic.model.fsdp_config.param_offload=False \

    critic.model.fsdp_config.optimizer_offload=False \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen_new' \

    trainer.experiment_name=$EXPERIMENT_NAME \

    trainer.n_gpus_per_node=4 \

    trainer.nnodes=1 \

    trainer.save_freq=90 \

    trainer.test_freq=20 \

    trainer.total_training_steps=200 \

    rollout_manager.max_turns=5 \

    rollout_manager.window_size=5 \

    rollout_manager.use_multi_turn_reward=False \

    rollout_manager.use_loss_mask=True \

    rollout_manager.use_gae_mask=True \

    trainer.val_before_train=True \

    trainer.val_generations_to_log_to_wandb=4 \

    rollout_manager.n_trajectory=1 \

    rollout_manager.use_service=True \

    rollout_manager.timeout=240 \

    rollout_manager.base_url="http://localhost:5001" \

    2>&1 | tee $EXPERIMENT_NAME.log

vagen/env/__init__.py

+8 −8
Original line number Diff line number Diff line 
from .sokoban import SokobanEnv,SokobanEnvConfig

from .frozenlake import FrozenLakeEnv,FrozenLakeEnvConfig, FrozenLakeService

# from .navigation import NavigationEnv, NavigationEnvConfig, NavigationServiceConfig, NavigationService

from .svg import SVGEnv, SvgEnvConfig, SVGService, SVGServiceConfig

from .navigation import NavigationEnv, NavigationEnvConfig, NavigationServiceConfig, NavigationService

# from .svg import SVGEnv, SvgEnvConfig, SVGService, SVGServiceConfig

# from .primitive_skill import PrimitiveSkillEnv, PrimitiveSkillEnvConfig, PrimitiveSkillService, PrimitiveSkillServiceConfig

# from .alfworld import ALFWorldEnv, ALFWorldEnvConfig, ALFWorldService, ALFWorldServiceConfig

REGISTERED_ENV = {
@@ -20,12 +20,12 @@ REGISTERED_ENV = { 
    #     "service_cls": NavigationService,

    #     "service_config_cls": NavigationServiceConfig

    # },

    "svg": {

        "env_cls": SVGEnv,

        "config_cls": SvgEnvConfig,

        "service_cls": SVGService,

        "service_config_cls": SVGServiceConfig

    },

    # "svg": {

    #     "env_cls": SVGEnv,

    #     "config_cls": SvgEnvConfig,

    #     "service_cls": SVGService,

    #     "service_config_cls": SVGServiceConfig

    # },

    # "primitive_skill": {

    #     "env_cls": PrimitiveSkillEnv,

    #     "config_cls": PrimitiveSkillEnvConfig,

CRA Git | Maintained and supported by SUSTech CRA and CCSE