init

# Service Benchmark Configuration

# Server configuration

server:

  base_url: http://localhost:5000

  timeout: 600

  max_workers: 48

# Benchmark parameters

benchmark:

  # Service functions to benchmark

  functions: 

    - create_environments_batch

    - reset_batch

    - step_batch

    - compute_reward_batch

    - get_system_prompts_batch

    - close_batch

  # Number of iterations for statistical significance

  iterations: 3

  # Number of steps to perform for step_batch

  step_count: 5

  # Batch sizes to test

  batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128]

  # Directory to save benchmark results

  output_dir: benchmark_results

# Dataset configurations - use paths to your actual dataset files

datasets:

  # ALFWorld with vision mode

  - name: alfworld-vision

    train_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/train.parquet

    test_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/test.parquet

    use_split: both  # Use both train and test datasets

  # Uncomment and adjust for additional environments

  # - name: frozenlake

  #   train_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/train.parquet

  #   test_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/test.parquet

  #   use_split: both

 No newline at end of file

# Dataset Generation Configuration for Benchmark

# ALFWorld with text mode

alfworld-text:

  env_name: alfworld

  env_config:

    render_mode: text

    alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"

    max_actions_per_step: 1

  train_size: 128  # Generate 128 training examples

  test_size: 32    # Generate 32 test examples

# ALFWorld with vision mode

# alfworld-vision:

#   env_name: alfworld

#   env_config:

#     render_mode: vision

#     alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"

#     max_actions_per_step: 1

#   train_size: 128

#   test_size: 32

# frozenlake:

#   env_name: frozenlake

#   env_config:

#     is_slippery: false

#     size: 4

#     render_mode: text

#   train_size: 128

#   test_size: 32

 No newline at end of file

#!/bin/bash

# Setup script for environment service benchmark

# Create required directories

mkdir -p benchmark_results

# Generate datasets

echo "Generating datasets for benchmark..."

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Generate ALFWorld vision mode dataset

python -m vagen.env.create_dataset \

    --yaml_path "$SCRIPT_DIR/dataset_config.yaml" \

    --train_path data/alfworld-vision-benchmark/train.parquet \

    --test_path data/alfworld-vision-benchmark/test.parquet \

# Run environment service benchmark

# Set environment variables

export PYTHONPATH=$(pwd):$PYTHONPATH

# Create output directory

mkdir -p benchmark_results

# Run benchmark

echo "Running service benchmark"

python -m vagen.benchmark.service_benchmark --config "$SCRIPT_DIR/benchmark_config.yaml"

echo "Benchmark complete. Results saved to benchmark_results directory."

 No newline at end of file

#!/usr/bin/env python3

"""

Service Benchmark

A benchmark tool to measure the performance of BaseService functions 

(create_environments_batch, reset_batch, step_batch, etc.) with different batch sizes.

Usage:

    python service_benchmark.py --config benchmark_config.yaml

"""

import argparse

import time

import json

import random

import statistics

import os

import yaml

import numpy as np

import matplotlib.pyplot as plt

from typing import Dict, List, Any

from datasets import load_dataset

from tqdm import tqdm

from vagen.server.client import BatchEnvClient

from vagen.env import REGISTERED_ENV

def generate_random_action(available_commands):

    """Generate a random action from available commands."""

    if not available_commands:

        return "<think>Let me explore.</think><answer>look</answer>"

    if isinstance(available_commands, list) and len(available_commands) > 0:

        if isinstance(available_commands[0], list):

            available_commands = available_commands[0]

    action = random.choice(available_commands) if available_commands else "look"

    return f"<think>I'll try this action.</think><answer>{action}</answer>"

def benchmark_service(config_path):

    """

    Main benchmark function.

    Args:

        config_path: Path to YAML configuration file

    """

    # Load configuration

    with open(config_path, 'r') as f:

        config = yaml.safe_load(f)

    # Initialize client

    server_config = config.get('server', {})

    client = BatchEnvClient(

        base_url=server_config.get('base_url', 'http://localhost:5000'),

        timeout=server_config.get('timeout', 600),

        max_workers=server_config.get('max_workers', 48)

    )

    # Get benchmark parameters

    benchmark_config = config.get('benchmark', {})

    batch_sizes = benchmark_config.get('batch_sizes', [1, 2, 4, 8, 16, 32, 64, 128])

    iterations = benchmark_config.get('iterations', 3)

    step_count = benchmark_config.get('step_count', 5)

    output_dir = benchmark_config.get('output_dir', 'benchmark_results')

    functions = benchmark_config.get('functions', [

        'create_environments_batch',

        'reset_batch', 

        'step_batch',

        'compute_reward_batch',

        'get_system_prompts_batch',

        'close_batch'

    ])

    # Create output directory

    os.makedirs(output_dir, exist_ok=True)

    # Check if server is alive

    if not client.wait_for_server():

        print("Server not available. Exiting.")

        return

    # Load environment configs from datasets

    datasets_config = config.get('datasets', [])

    all_env_configs = {}

    for dataset_config in datasets_config:

        name = dataset_config.get('name')

        train_path = dataset_config.get('train_path')

        test_path = dataset_config.get('test_path')

        use_split = dataset_config.get('use_split', 'both')

        # Load train dataset if needed

        if use_split in ['train', 'both'] and os.path.exists(train_path):

            try:

                train_dataset = load_dataset('parquet', data_files=train_path, split="train")

                print(f"Loaded train dataset from {train_path} with {len(train_dataset)} examples")

                # Extract environment configs

                for i in range(len(train_dataset)):

                    example = train_dataset[i]

                    env_config = {

                        'env_name': example['extra_info']['env_name'],

                        'env_config': example['extra_info']['env_config'],

                        'seed': example['extra_info']['seed']

                    }

                    all_env_configs.setdefault(name, []).append(env_config)

            except Exception as e:

                print(f"Failed to load train dataset from {train_path}: {e}")

        # Load test dataset if needed

        if use_split in ['test', 'both'] and os.path.exists(test_path):

            try:

                test_dataset = load_dataset('parquet', data_files=test_path, split="train")

                print(f"Loaded test dataset from {test_path} with {len(test_dataset)} examples")

                # Extract environment configs

                for i in range(len(test_dataset)):

                    example = test_dataset[i]

                    env_config = {

                        'env_name': example['extra_info']['env_name'],

                        'env_config': example['extra_info']['env_config'],

                        'seed': example['extra_info']['seed']

                    }

                    all_env_configs.setdefault(name, []).append(env_config)

            except Exception as e:

                print(f"Failed to load test dataset from {test_path}: {e}")

    # Dictionary to store all benchmark results

    results = {}

    # Run benchmark for each environment type

    for env_name, env_configs in all_env_configs.items():

        print(f"\n===== Benchmarking {env_name} =====")

        # Store results for this environment

        env_results = {

            'batch_sizes': [],

            'timings': {func: [] for func in functions},

            'per_env_timings': {func: [] for func in functions}

        }

        # Test different batch sizes

        for batch_size in batch_sizes:

            if batch_size > len(env_configs):

                print(f"Skipping batch size {batch_size}: not enough configs available")

                continue

            print(f"\nBatch size: {batch_size}")

            # Run multiple iterations for statistical significance

            batch_timings = {func: [] for func in functions}

            for iteration in range(iterations):

                print(f"  Iteration {iteration+1}/{iterations}")

                # Sample environment configs for this batch

                batch_configs = random.sample(env_configs, batch_size)

                env_ids = [f"{env_name}_{i}" for i in range(batch_size)]

                # Dictionary of environment configurations

                ids2configs = {env_id: config for env_id, config in zip(env_ids, batch_configs)}

                # Dictionary of seeds for resetting environments

                ids2seeds = {env_id: config['seed'] for env_id, config in zip(env_ids, batch_configs)}

                # ----- Benchmark create_environments_batch -----

                if 'create_environments_batch' in functions:

                    print("    Creating environments...", end='', flush=True)

                    start_time = time.time()

                    client.create_environments_batch(ids2configs)

                    end_time = time.time()

                    create_time = end_time - start_time

                    batch_timings['create_environments_batch'].append(create_time)

                    print(f" {create_time:.4f}s ({create_time/batch_size:.6f}s per env)")

                # ----- Benchmark reset_batch -----

                if 'reset_batch' in functions:

                    print("    Resetting environments...", end='', flush=True)

                    start_time = time.time()

                    reset_results = client.reset_batch(ids2seeds)

                    end_time = time.time()

                    reset_time = end_time - start_time

                    batch_timings['reset_batch'].append(reset_time)

                    print(f" {reset_time:.4f}s ({reset_time/batch_size:.6f}s per env)")

                # ----- Benchmark step_batch -----

                if 'step_batch' in functions:

                    step_times = []

                    # Need to reset to get initial observations

                    if 'reset_batch' not in functions:

                        reset_results = client.reset_batch(ids2seeds)

                    # Run multiple steps

                    for step in range(step_count):

                        print(f"    Step {step+1}/{step_count}...", end='', flush=True)

                        # Generate actions for each environment

                        ids2actions = {}

                        for env_id in env_ids:

                            if env_id in reset_results:

                                observation, info = reset_results[env_id]

                                available_commands = info.get('admissible_commands', [])

                                ids2actions[env_id] = generate_random_action(available_commands)

                        # Measure step time

                        start_time = time.time()

                        step_results = client.step_batch(ids2actions)

                        end_time = time.time()

                        step_time = end_time - start_time

                        step_times.append(step_time)

                        print(f" {step_time:.4f}s ({step_time/batch_size:.6f}s per env)")

                        # Use step results as observations for next step

                        reset_results = {env_id: (obs, info) for env_id, (obs, _, _, info) in step_results.items()}

                    # Store average step time

                    avg_step_time = statistics.mean(step_times)

                    batch_timings['step_batch'].append(avg_step_time)

                    print(f"    Average step time: {avg_step_time:.4f}s ({avg_step_time/batch_size:.6f}s per env)")

                # ----- Benchmark compute_reward_batch -----

                if 'compute_reward_batch' in functions:

                    print("    Computing rewards...", end='', flush=True)

                    start_time = time.time()

                    client.compute_reward_batch(env_ids)

                    end_time = time.time()

                    reward_time = end_time - start_time

                    batch_timings['compute_reward_batch'].append(reward_time)

                    print(f" {reward_time:.4f}s ({reward_time/batch_size:.6f}s per env)")

                # ----- Benchmark get_system_prompts_batch -----

                if 'get_system_prompts_batch' in functions:

                    print("    Getting system prompts...", end='', flush=True)

                    start_time = time.time()

                    client.get_system_prompts_batch(env_ids)

                    end_time = time.time()

                    prompt_time = end_time - start_time

                    batch_timings['get_system_prompts_batch'].append(prompt_time)

                    print(f" {prompt_time:.4f}s ({prompt_time/batch_size:.6f}s per env)")

                # ----- Benchmark close_batch -----

                if 'close_batch' in functions:

                    print("    Closing environments...", end='', flush=True)

                    start_time = time.time()

                    client.close_batch(env_ids)

                    end_time = time.time()

                    close_time = end_time - start_time

                    batch_timings['close_batch'].append(close_time)

                    print(f" {close_time:.4f}s ({close_time/batch_size:.6f}s per env)")

                else:

                    # Make sure environments are closed even if not benchmarking close

                    client.close_batch(env_ids)

            # Compute average timings for this batch size

            env_results['batch_sizes'].append(batch_size)

            for func in functions:

                avg_time = statistics.mean(batch_timings[func])

                env_results['timings'][func].append(avg_time)

                env_results['per_env_timings'][func].append(avg_time / batch_size)

        # Store results for this environment

        results[env_name] = env_results

    # Save results as JSON

    timestamp = time.strftime("%Y%m%d-%H%M%S")

    results_file = os.path.join(output_dir, f"benchmark_results_{timestamp}.json")

    with open(results_file, 'w') as f:

        json.dump(results, f, indent=4)

    print(f"\nResults saved to {results_file}")

    # Generate plots

    generate_plots(results, output_dir, timestamp)

    # Print summary

    print_summary(results)

def generate_plots(results, output_dir, timestamp):

    """

    Generate plots from benchmark results.

    Args:

        results: Dictionary of benchmark results

        output_dir: Directory to save plots

        timestamp: Timestamp string for filenames

    """

    # Create directory for plots

    plots_dir = os.path.join(output_dir, 'plots')

    os.makedirs(plots_dir, exist_ok=True)

    # Plot for each function

    functions = next(iter(results.values()))['timings'].keys()

    for func in functions:

        plt.figure(figsize=(12, 8))

        # Plot total time

        plt.subplot(2, 1, 1)

        for env_name, env_results in results.items():

            batch_sizes = env_results['batch_sizes']

            timings = env_results['timings'][func]

            plt.plot(batch_sizes, timings, 'o-', label=env_name)

        plt.title(f'{func} - Total Time')

        plt.xlabel('Batch Size')

        plt.ylabel('Time (seconds)')

        plt.xscale('log')

        plt.yscale('log')

        plt.grid(True)

        plt.legend()

        # Plot time per environment

        plt.subplot(2, 1, 2)

        for env_name, env_results in results.items():

            batch_sizes = env_results['batch_sizes']

            per_env_timings = env_results['per_env_timings'][func]

            plt.plot(batch_sizes, per_env_timings, 'o-', label=env_name)

        plt.title(f'{func} - Time per Environment')

        plt.xlabel('Batch Size')

        plt.ylabel('Time per Environment (seconds)')

        plt.xscale('log')

        plt.yscale('log')

        plt.grid(True)

        plt.legend()

        plt.tight_layout()

        plt.savefig(os.path.join(plots_dir, f"{func}_{timestamp}.png"))

        plt.close()

    # Generate comparative plot for throughput

    plt.figure(figsize=(12, 10))

    for i, func in enumerate(functions):

        plt.subplot(len(functions), 1, i+1)

        for env_name, env_results in results.items():

            batch_sizes = env_results['batch_sizes']

            timings = env_results['timings'][func]

            throughput = [size / time for size, time in zip(batch_sizes, timings)]

            plt.plot(batch_sizes, throughput, 'o-', label=env_name)

        plt.title(f'{func} - Throughput')

        plt.xlabel('Batch Size')

        plt.ylabel('Environments per Second')

        plt.xscale('log')

        plt.grid(True)

        plt.legend()

    plt.tight_layout()

    plt.savefig(os.path.join(plots_dir, f"throughput_{timestamp}.png"))

    plt.close()

def print_summary(results):

    """

    Print a summary of the benchmark results.

    Args:

        results: Dictionary of benchmark results

    """

    print("\n===== Summary =====")

    # Get list of all functions

    functions = next(iter(results.values()))['timings'].keys()

    for env_name, env_results in results.items():

        print(f"\n{env_name}:")

        batch_sizes = env_results['batch_sizes']

        for func in functions:

            timings = env_results['timings'][func]

            per_env_timings = env_results['per_env_timings'][func]

            print(f"  {func}:")

            for i, batch_size in enumerate(batch_sizes):

                print(f"    Batch size {batch_size}: {timings[i]:.4f}s total, {per_env_timings[i]:.6f}s per env")

            # Print scaling behavior

            if len(batch_sizes) > 1:

                scaling = timings[-1] / timings[0] * batch_sizes[0] / batch_sizes[-1]

                print(f"    Scaling efficiency (batch {batch_sizes[0]} -> {batch_sizes[-1]}): {scaling:.2f}x")

                if scaling < 0.5:

                    print(f"    ⚠️  Poor scaling for {func} in {env_name}")

                elif scaling > 0.8:

                    print(f"    ✅ Good scaling for {func} in {env_name}")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Benchmark service functions")

    parser.add_argument("--config", type=str, default="benchmark_config.yaml", help="Path to configuration YAML")

    args = parser.parse_args()

    benchmark_service(args.config)

 No newline at end of file

"""

start_x11.py

A self‑contained script to launch a virtual X11 server using NVIDIA and Xorg,

A self-contained script to launch a virtual X11 server using NVIDIA and Xorg,

enabling AI2-THOR to run headlessly on your remote server.

Usage:

    # find NVIDIA GPUs

    buses = []

    for r in pci_records():

<<<<<<< HEAD

<<<<<<< HEAD

=======

>>>>>>> a8c743f (alfworld update)

        if r.get('Vendor') == 'NVIDIA Corporation' and (

            r.get('Class','').startswith('VGA') or 

            r.get('Class','').startswith('3D')

        ):

<<<<<<< HEAD

=======

        if r.get('Vendor') == 'NVIDIA Corporation' and r.get('Class','').startswith('VGA'):

>>>>>>> 6b8f828 (alfworld update)

=======

>>>>>>> a8c743f (alfworld update)

            slot = r['Slot']  # e.g. '01:00.0'

            parts = re.split(r'[:\.]', slot)

            buses.append('PCI:' + ':'.join(str(int(x,16)) for x in parts))

    with os.fdopen(fd, 'w') as f:

        f.write(conf)

<<<<<<< HEAD