update

**/*.pt

**/checkpoints

**/wget-log

**/_build/

**/*.ckpt

**/outputs

**/*.tar.gz

**/playground

**/wandb

# Byte-compiled / optimized / DLL files

__pycache__/

*.py[cod]

*$py.class

dataset/*

tensorflow/my_graph/*

.idea/

# C extensions

*.so

data

sft/output/*

sft/data/*

sft_configs/

models/

# Distribution / packaging

.Python

build/

develop-eggs/

dist/

downloads/

eggs/

.eggs/

lib/

lib64/

parts/

sdist/

var/

*.egg-info/

.installed.cfg

*.egg

# PyInstaller

#  Usually these files are written by a python script from a template

#  before PyInstaller builds the exe, so as to inject date/other infos into it.

*.manifest

*.spec

# Installer logs

pip-log.txt

pip-delete-this-directory.txt

# Unit test / coverage reports

htmlcov/

.tox/

.coverage

.coverage.*

.cache

nosetests.xml

coverage.xml

*,cover

.hypothesis/

# Translations

*.mo

*.pot

# Django stuff:

*.log

local_settings.py

image_outputs

checkpoints

# Flask stuff:

instance/

.webassets-cache

# Scrapy stuff:

.scrapy

# Sphinx documentation

docs/_build/

# PyBuilder

target/

# IPython Notebook

.ipynb_checkpoints

# pyenv

.python-version

# celery beat schedule file

celerybeat-schedule

# virtualenv

venv/

ENV/

# Spyder project settings

.spyderproject

# Rope project settings

.ropeproject

# vscode

.vscode

# Mac

.DS_Store

# output logs

tests/e2e/toy_examples/deepspeed/synchronous/output.txt

# vim

*.swp

log*

**logs

 No newline at end of file

data:

  tokenizer: null

  train_files: ~/data/rlhf/gsm8k/train.parquet

  val_files: ~/data/rlhf/gsm8k/test.parquet

  prompt_key: prompt

  max_prompt_length: 512

  max_response_length: 512

  train_batch_size: 1024

  val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves

  return_raw_input_ids: False  # This should be set to true when the tokenizer between policy and rm differs

  return_raw_chat: False

  shuffle: True

  image_key: images

actor_rollout_ref:

  hybrid_engine: True

  model:

    path: ~/models/deepseek-llm-7b-chat

    external_lib: null

    override_config: { }

    enable_gradient_checkpointing: True

    use_remove_padding: False

  actor:

    strategy: fsdp  # This is for backward-compatibility

    ppo_mini_batch_size: 256

    ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu

    ppo_micro_batch_size_per_gpu: null

    use_dynamic_bsz: False

    ppo_max_token_len_per_gpu: 16384 # n * ${data.max_prompt_length} + ${data.max_response_length}

    grad_clip: 1.0

    clip_ratio: 0.2

    entropy_coeff: 0.001

    use_kl_loss: False # True for GRPO

    kl_loss_coef: 0.001 # for grpo

    kl_loss_type: low_var_kl # for grpo

    ppo_epochs: 1

    shuffle: False

    ulysses_sequence_parallel_size: 1 # sp size

    optim:

      lr: 1e-6

      lr_warmup_steps_ratio: 0.  # the total steps will be injected during runtime

      min_lr_ratio: null   # only useful for warmup with cosine

      warmup_style: constant  # select from constant/cosine

      total_training_steps: -1  # must be override by program

    fsdp_config:

      wrap_policy:

        # transformer_layer_cls_to_wrap: None

        min_num_params: 0

      param_offload: False

      optimizer_offload: False

      fsdp_size: -1

  ref:

    fsdp_config:

      param_offload: False

      wrap_policy:

        # transformer_layer_cls_to_wrap: None

        min_num_params: 0

    log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu

    log_prob_micro_batch_size_per_gpu: null

    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}

    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}

    ulysses_sequence_parallel_size: ${actor_rollout_ref.actor.ulysses_sequence_parallel_size} # sp size

  rollout:

    name: vllm

    temperature: 1.0

    top_k: -1 # 0 for hf rollout, -1 for vllm rollout

    top_p: 1

    use_fire_sampling: False # https://arxiv.org/abs/2410.21236

    prompt_length: ${data.max_prompt_length}  # not use for opensource

    response_length: ${data.max_response_length}

    # for vllm rollout

    dtype: bfloat16 # should align with FSDP

    gpu_memory_utilization: 0.5

    ignore_eos: False

    enforce_eager: True

    free_cache_engine: True

    load_format: dummy_dtensor

    tensor_model_parallel_size: 2

    max_num_batched_tokens: 8192

    max_model_len: null

    max_num_seqs: 1024

    log_prob_micro_batch_size: null # will be deprecated, use log_prob_micro_batch_size_per_gpu

    log_prob_micro_batch_size_per_gpu: null

    log_prob_use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}

    log_prob_max_token_len_per_gpu: ${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}

    disable_log_stats: True

    enable_chunked_prefill: True # may get higher throughput when set to True. When activated, Please increase max_num_batched_tokens or decrease max_model_len.

    # for hf rollout

    do_sample: True

    # number of responses (i.e. num sample times)

    n: 1 # > 1 for grpo

critic:

  strategy: fsdp

  optim:

    lr: 1e-5

    lr_warmup_steps_ratio: 0.  # the total steps will be injected during runtime

    min_lr_ratio: null   # only useful for warmup with cosine

    warmup_style: constant  # select from constant/cosine

    total_training_steps: -1  # must be override by program

  model:

    path: ~/models/deepseek-llm-7b-chat

    tokenizer_path: ${actor_rollout_ref.model.path}

    override_config: { }

    external_lib: ${actor_rollout_ref.model.external_lib}

    enable_gradient_checkpointing: True

    use_remove_padding: False

    fsdp_config:

      param_offload: False

      optimizer_offload: False

      wrap_policy:

        # transformer_layer_cls_to_wrap: None

        min_num_params: 0

      fsdp_size: -1

  ppo_mini_batch_size: ${actor_rollout_ref.actor.ppo_mini_batch_size}

  ppo_micro_batch_size: null # will be deprecated, use ppo_micro_batch_size_per_gpu

  ppo_micro_batch_size_per_gpu: null

  forward_micro_batch_size: ${critic.ppo_micro_batch_size}

  forward_micro_batch_size_per_gpu: ${critic.ppo_micro_batch_size_per_gpu}

  use_dynamic_bsz: ${actor_rollout_ref.actor.use_dynamic_bsz}

  ppo_max_token_len_per_gpu: 32768 # (${actor_rollout_ref.actor.ppo_max_token_len_per_gpu}) * 2

  forward_max_token_len_per_gpu: ${critic.ppo_max_token_len_per_gpu}

  ulysses_sequence_parallel_size: 1 # sp size

  ppo_epochs: ${actor_rollout_ref.actor.ppo_epochs}

  shuffle: ${actor_rollout_ref.actor.shuffle}

  grad_clip: 1.0

  cliprange_value: 0.5

reward_model:

  enable: False

  strategy: fsdp

  model:

    input_tokenizer: ${actor_rollout_ref.model.path}  # set this to null if the chat template is identical

    path: ~/models/FsfairX-LLaMA3-RM-v0.1

    external_lib: ${actor_rollout_ref.model.external_lib}

    use_remove_padding: False

    fsdp_config:

      min_num_params: 0

      param_offload: False

      fsdp_size: -1

  micro_batch_size: null # will be deprecated, use micro_batch_size_per_gpu

  micro_batch_size_per_gpu: null # set a number

  max_length: null

  ulysses_sequence_parallel_size: 1 # sp size

  use_dynamic_bsz: ${critic.use_dynamic_bsz}

  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}

  reward_manager: naive

algorithm:

  gamma: 1.0

  lam: 1.0

  adv_estimator: gae

  kl_penalty: kl  # how to estimate kl divergence

  kl_ctrl:

    type: fixed

    kl_coef: 0.001

trainer:

  total_epochs: 30

  total_training_steps: null

  project_name: verl_examples

  experiment_name: gsm8k

  logger: [ 'console', 'wandb' ]

  val_generations_to_log_to_wandb: 0

  nnodes: 1

  n_gpus_per_node: 8

  save_freq: -1

  # auto: find the last ckpt to resume. If can't find, start from scratch

  resume_mode: auto # or auto or resume_path if 

  resume_from_path: False

  test_freq: -1

  critic_warmup: 0

  default_hdfs_dir: null

  remove_previous_ckpt_in_save: False

  del_local_ckpt_after_load: False

  default_local_dir: checkpoints/${trainer.project_name}/${trainer.experiment_name}

set -x

export VLLM_ATTENTION_BACKEND=XFORMERS

python3 -m main_ppo \

    algorithm.adv_estimator=grpo \

    data.train_files=data/sokoban/train.parquet \

    data.val_files=data/sokoban/test.parquet \

    data.train_batch_size=8 \

    data.max_prompt_length=1024 \

    data.max_response_length=2048 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    actor_rollout_ref.model.use_remove_padding=True \

    actor_rollout_ref.actor.ppo_mini_batch_size=2 \

    actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.actor.use_kl_loss=True \

    actor_rollout_ref.actor.kl_loss_coef=0.01 \

    actor_rollout_ref.actor.kl_loss_type=low_var_kl \

    actor_rollout_ref.model.enable_gradient_checkpointing=True \

    actor_rollout_ref.actor.fsdp_config.param_offload=False \

    actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \

    actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \

    actor_rollout_ref.rollout.name=vllm \

    actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \

    actor_rollout_ref.rollout.enable_chunked_prefill=False \

    actor_rollout_ref.rollout.enforce_eager=False \

    actor_rollout_ref.rollout.free_cache_engine=False \

    actor_rollout_ref.rollout.n=1 \

    actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1 \

    actor_rollout_ref.ref.fsdp_config.param_offload=True \

    algorithm.kl_ctrl.kl_coef=0.001 \

    trainer.critic_warmup=0 \

    trainer.logger=['console','wandb'] \

    trainer.project_name='vagen' \

    trainer.experiment_name='qwen2_5_vl_3b_function_rm' \

    trainer.n_gpus_per_node=2 \

    trainer.nnodes=1 \

    trainer.save_freq=-1 \

    trainer.test_freq=5 \

    trainer.total_epochs=15 \

    +max_turns=1 2>&1 | tee debug.log

"""

Note that we don't combine the main with ray_trainer as ray_trainer is used by other main.

"""

from verl.trainer.ppo.ray_trainer import RayPPOTrainer

from vagen.trainer.ray_trainer import RayPPOTrainer

from vagen.utils.compute_score import compute_score

import ray

import hydra

@hydra.main(config_path='config', config_name='ppo_trainer', version_base=None)

def main(config):

    run_ppo(config)

    run_ppo(config, compute_score=compute_score)

def run_ppo(config, compute_score=None):

    else:

        raise NotImplementedError

    from verl.trainer.ppo.ray_trainer import ResourcePoolManager, Role

    from vagen.trainer.ray_trainer import ResourcePoolManager, Role

    role_worker_mapping = {

        Role.ActorRollout: ray.remote(ActorRolloutRefWorker),

        return self.system_prompt + '\n' +self.instruction_prompt

# @dataclass

# class EnvFeedback:

#     """Dataclass for managing environment feedback."""

#     env_observation: EnvObservation = field(default_factory=EnvObservation)

#     action_str: str = ""

#     step_reward: float = 0.0

#     done: bool = False

#     env_finished_before: bool = False

#     info: Dict = field(default_factory=dict)

@dataclass

class EnvObservation:

        if "{index}" not in self.placeholder_format:

            raise ValueError("placeholder_format must contain '{index}' for proper formatting")

    def create_observation(self, text: str, contents: List[Any], replace_keys: List[str] = ['observation']) -> None:

    def create_observation(

            self,

            template: str,

            contents: List[Any],

            replace_keys: List[str] = ['observation']

        ) -> None:

        """

        Create an observation template with text and images.

        Args:

            text: The text part of the observation

            template: The text part of the observation

            contents: A list of content objects to include in the observation

            replace_key: The key to replace in the observation template

            replace_keys: The key to replace in the observation template

        This function will:

        1. Create placeholders in the text for each image

        3. Set the observation_template with image placeholders

        """

        # Start with the input text

        result_template = text

        result_template = template

        assert len(replace_keys) == len(contents), "replace_keys and contents must have the same length"

        # Add each image and insert placeholder

    def __init__(self, **env_config):

        self.env_config = env_config

        self.traj_reward = 0

    @classmethod

    def get_task_instruction(cls) -> str:

    def config_repr(cls, config: Dict) -> str:

        """Get the config of the environment."""

    def get_traj_reward(self) -> float:

        """Get the reward of the environment."""

        return self.traj_reward

 No newline at end of file