inference temp

env1:

  env_name: frozenlake

  env_config:

    render_mode: text

    prompt_format: free_think

    use_accuracy_reward: false

  train_size: 10000

  test_size: 2

 No newline at end of file

# Server configuration

server:

  base_url: "http://localhost:5000"  # Environment server URL

  timeout: 600  # Request timeout (seconds)

  max_workers: 10  # Maximum number of concurrent worker threads

# Inference settings

inference:

  max_steps: 10  # Maximum steps per environment

  show_progress: true  # Display progress bar

  debug: false  # Debug mode

  save_images: true  # Save image results

  save_intermediate: false  # Save intermediate results

# Generation parameters (overrides defaults in model config)

generation:

  temperature: 0.7  # Temperature parameter

  top_p: 0.95  # Nucleus sampling parameter

  max_tokens: 512  # Maximum number of tokens to generate

  stop: ["\n\nUser:", "\n\nHuman:"]  # Stop generation at these markers

# Batch processing settings

batch:

  batch_size: 4  # Batch size

  batch_size_multiple: 4  # Batch size multiple (for hardware optimization)

# Evaluation configuration

evaluation:

  metrics:

    - score  # Overall score

    - done  # Completion rate

    - steps  # Step count statistics

# Weights & Biases logging configuration

wandb:

  project: "qwen-vl-eval"  # Project name

  experiment_name: "qwen2.5-vl-3b-eval"  # Experiment name

  entity: null  # Team/entity name (optional)

  val_generations_to_log_to_wandb: 5  # Number of generated samples to log

  log_model: false  # Whether to save the model to wandb

# Output configuration

output:

  format: "json"  # Output format

  include_metrics: true  # Include evaluation metrics

  include_trajectories: true  # Include full trajectories

  pretty_print: true  # Pretty-print the output

type: "vllm"

name: "Qwen2.5-0.5B-Instruct"

# Model path

path: "Qwen/Qwen2.5-0.5B-Instruct"

tokenizer_path: null  # Use the same path as the model

processor_path: null  # Use the same path as the model

# vLLM specific configuration

tensor_parallel_size: 1  # Adjust according to the number of available GPUs

dtype: "bfloat16"  # Options: auto, float16, bfloat16, float32

gpu_memory_utilization: 0.9

trust_remote_code: true

max_model_len: 4096  # Maximum sequence length

# Generation parameters

temperature: 0.7

top_p: 0.95

top_k: 50

max_tokens: 512

repetition_penalty: 1.0

presence_penalty: 0.0

frequency_penalty: 0.0

# Multimodal specific configuration

is_multimodal: true

limit_mm_per_prompt:

  image: 4  # Maximum number of images per prompt

export VLLM_ATTENTION_BACKEND=XFORMERS

export PYTHONHASHSEED=0

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Extract experiment name from the path

# This will take the last 3 parts of the path: format/sokoban/free_think

EXPERIMENT_NAME=$(echo $SCRIPT_DIR | rev | cut -d'/' -f1-2 | rev | tr '/' '-')

echo "Experiment name: $EXPERIMENT_NAME"

# run python -m vagen.server.server in a tmux session first

python -m vagen.env.create_dataset \

    --yaml_path "$SCRIPT_DIR/env_config.yaml" \

    --train_path "data/$EXPERIMENT_NAME/train.parquet" \

    --test_path "data/$EXPERIMENT_NAME/test.parquet" \

    --force_gen

python -m vagen.inference.run_inference \

  --model_config "$SCRIPT_DIR/model_config.yaml" \

  --inference_config "$SCRIPT_DIR/inference_config.yaml" \

  --dataset "data/$EXPERIMENT_NAME/test.parquet" \

  --output_dir results

 No newline at end of file

#!/usr/bin/env python

"""

Main entry point for running inference using the InferenceRolloutService.

This script handles the high-level flow of the inference process.

"""

import sys

import argparse

from pathlib import Path

# Add project root to path

project_root = Path(__file__).parent.parent.parent

sys.path.append(str(project_root))

from vagen.mllm_agent.inference_rollout.model_interface.factory_model import create_model_interface

from vagen.mllm_agent.inference_rollout.inference_rollout_service import InferenceRolloutService

from .utils.config import load_config, save_configs

from .utils.environment import setup_gpu, load_environment_configs

from .utils.logging import setup_output_dir, setup_wandb

from .utils.metrics import run_validation

def parse_args():

    """Parse command line arguments."""

    parser = argparse.ArgumentParser(description="Run inference with model on environments")

    # Required arguments

    parser.add_argument("--model_config", type=str, required=True,

                       help="Path to model configuration YAML")

    parser.add_argument("--inference_config", type=str, required=True,

                       help="Path to inference configuration YAML")

    # Optional arguments

    parser.add_argument("--dataset", type=str, default=None,

                       help="Path to dataset with environment configs")

    parser.add_argument("--output_dir", type=str, default="inference_results",

                       help="Directory to save results")

    parser.add_argument("--split", type=str, default="test",

                       help="Dataset split to use")

    parser.add_argument("--max_envs", type=int, default=100,

                       help="Maximum number of environments to evaluate")

    parser.add_argument("--max_steps", type=int, default=10,

                       help="Maximum number of steps per environment")

    parser.add_argument("--server_url", type=str, default="http://localhost:5000",

                       help="Environment server URL")

    parser.add_argument("--timeout", type=int, default=600,

                       help="Server request timeout in seconds")

    parser.add_argument("--max_workers", type=int, default=10,

                       help="Maximum number of parallel workers")

    parser.add_argument("--seed", type=int, default=42,

                       help="Random seed for reproducibility")

    parser.add_argument("--gpu_id", type=int, default=0,

                       help="GPU ID to use (-1 for CPU)")

    parser.add_argument("--global_steps", type=int, default=0,

                       help="Current global step (for wandb logging)")

    parser.add_argument("--no_wandb", action="store_true",

                       help="Disable wandb logging")

    parser.add_argument("--debug", action="store_true",

                       help="Enable debug logging")

    return parser.parse_args()

def main():

    """Main function with simplified high-level flow."""

    # Parse arguments

    args = parse_args()

    # Set up GPU if available

    setup_gpu(args.gpu_id)

    # Load configurations

    model_config = load_config(args.model_config)

    inference_config = load_config(args.inference_config)

    # Set up output directory

    output_dir = setup_output_dir(args.output_dir)

    print(f"Saving results to {output_dir}")

    # Save configurations

    save_configs(output_dir, model_config, inference_config, args)

    # Set up wandb

    if not args.no_wandb:

        setup_wandb(args, model_config, inference_config)

    try:

        # Create model interface

        model_interface = create_model_interface(model_config)

        # Load environment configs

        env_configs = load_environment_configs(args, inference_config)

        # Print model info

        model_info = model_interface.get_model_info()

        print(f"Model: {model_info['name']} (type: {model_info['type']})")

        print(f"Context length: {model_info['context_length']}")

        # Create inference rollout service

        inference_service = InferenceRolloutService(

            config=inference_config,

            model_interface=model_interface,

            base_url=args.server_url,

            timeout=args.timeout,

            max_workers=args.max_workers,

            split=args.split,

            debug=args.debug

        )

        # Run validation and get metrics

        metrics, results = run_validation(

            inference_service=inference_service,

            env_configs=env_configs,

            max_steps=args.max_steps,

            global_steps=args.global_steps,

            output_dir=output_dir,

            use_wandb=not args.no_wandb

        )

        # Print summary metrics

        print("\n===== Inference Results =====")

        print(f"Total environments: {len(env_configs)}")

        print(f"Completed environments: {sum(1 for r in results if r['metrics']['done'])}")

        print(f"Mean score: {metrics.get('mean_score', 0):.4f}")

        print(f"Completion rate: {metrics.get('percent_done', 0):.1f}%")

    finally:

        # Clean up

        if 'inference_service' in locals():

            inference_service.close()

        # Finish wandb run if active

        if not args.no_wandb:

            try:

                import wandb

                if wandb.run:

                    wandb.finish()

            except:

                pass

if __name__ == "__main__":

    main()

 No newline at end of file