set max response length smaller and set max trajectory length

    data.train_files=data/sokoban-text/train.parquet \

    data.val_files=data/sokoban-text/test.parquet \

    data.train_batch_size=16 \

    data.max_prompt_length=1024 \

    data.max_response_length=1024 \

    data.max_prompt_length=512 \

    data.max_response_length=512 \

    data.max_trajectory_length=3072 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-0.5B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

    data.val_files=data/sokoban/test.parquet \

    data.train_batch_size=16 \

    data.max_prompt_length=2048 \

    data.max_response_length=2048 \

    data.max_response_length=512 \

    data.max_trajectory_length=3072 \

    data.image_key=images \

    actor_rollout_ref.model.path=Qwen/Qwen2.5-VL-3B-Instruct \

    actor_rollout_ref.actor.optim.lr=1e-6 \

@dataclass

class QwenVLRolloutConifg:

    window_size: int = 5

    max_prompt_length: int = 512 # 1024 

    max_response_length: int = 2048 # 1024 

    max_trajectories_length: int = 3072

    max_turns: int = 5

    n_gpu_per_node: int = 1 # used for multigpu batch balancing

    sptk_for_loss_mask: List[str] = field(default_factory=lambda: ['<|box_start|>', '<|box_end|>'])

        input_ids_response, attention_mask_response = verl_F.tokenize_and_postprocess_data(prompt=response_with_chat_template,

                                                                         tokenizer=self.tokenizer,

                                                                         max_length=self.config.max_response_length+self.config.max_prompt_length,

                                                                         max_length=self.config.max_trajectories_length,

                                                                         pad_token_id=self.tokenizer.pad_token_id,

                                                                         left_pad=False,

                                                                         truncation=self.truncation)

        input_ids_prompt, attention_mask_prompt = verl_F.tokenize_and_postprocess_data(prompt=prompt_with_chat_template,

                                                                         tokenizer=self.tokenizer,

                                                                        #  max_length=self.config.max_response_length+self.config.max_prompt_length,

                                                                         max_length=1,

                                                                         pad_token_id=self.tokenizer.pad_token_id,

                                                                         left_pad=True,

  prompt_key: prompt

  max_prompt_length: 512

  max_response_length: 512

  max_trajectory_length: 3072 # vagen only

  train_batch_size: 1024

  val_batch_size: null # DEPRECATED: Validation datasets are sent to inference engines as a whole batch, which will schedule the memory themselves

  return_raw_input_ids: False  # This should be set to true when the tokenizer between policy and rm differs

  return_raw_chat: False

  shuffle: True

  filter_overlong_prompts: False # for large-scale dataset, filtering overlong prompts could be timeconsuming. You should disable this and set `truncation='left'

  truncation: error

  image_key: images

actor_rollout_ref:

    external_lib: ${actor_rollout_ref.model.external_lib}

    use_remove_padding: False

    fsdp_config:

      wrap_policy:

        min_num_params: 0

      param_offload: False

      fsdp_size: -1

  forward_max_token_len_per_gpu: ${critic.forward_max_token_len_per_gpu}

  reward_manager: naive

custom_reward_function:

  path: null

  name: compute_score

algorithm:

  gamma: 1.0

  lam: 1.0

    kl_coef: 0.001

trainer:

  balance_batch: True

  total_epochs: 30

  total_training_steps: null

  project_name: verl_examples