revise prompt

svg_analysis_results/

temp/

benchmark_results/

env_benchmark_results/

 No newline at end of file

  step_count: 5

  # Batch sizes to test

  batch_sizes: [8, 16, 32]

  batch_sizes: [8, 128]

  # Maximum number of parallel workers

  max_workers: 8

# Benchmark parameters

benchmark:

  # Functions to benchmark

  functions: 

    - system_prompt

    - reset

    - step

    - compute_reward

    - close

  # Number of iterations for statistical significance

  iterations: 3

  # Number of steps to perform for each environment

  step_count: 5

  # Batch sizes to test

  batch_sizes: [8, 128]

  # Maximum number of parallel workers

  max_workers: 8

  # Directory to save benchmark results

  output_dir: "env_benchmark_results"

  # Valid commands for testing (optional, environment-specific)

  valid_commands: ["Left", "Right", "Up", "Down"]

# Datasets to use for benchmarking

datasets:

  - name: svg-vision

    train_path: data/svg-vision-benchmark/train.parquet

    test_path: data/svg-vision-benchmark/test.parquet

    use_split: both

 No newline at end of file

svg-vision:

  env_name: svg

  env_config:

    split: train

    dino_weight: 1.0

    structural_weight: 1.0

    dreamsim_weight: 1.0

  train_size: 1000

  test_size: 64

#!/bin/bash

# Setup script for environment benchmark

# Create required directories

mkdir -p env_benchmark_results

mkdir -p data

echo "Generating datasets for benchmark..."

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# Generate dataset

python -m vagen.env.create_dataset \

    --yaml_path "$SCRIPT_DIR/dataset_config.yaml" \

    --train_path data/svg-vision-benchmark/train.parquet \

    --test_path data/svg-vision-benchmark/test.parquet \

    --force_gen

# Set environment variables

export PYTHONPATH=$(pwd):$PYTHONPATH

# Run benchmark

echo "Running environment benchmark"

python -m vagen.utils_benchmark.env_benchmark --config "$SCRIPT_DIR/benchmark_config.yaml"

echo "Benchmark complete. Results saved to env_benchmark_results directory."

 No newline at end of file