Commit 050cafe3 authored by root's avatar root
Browse files

update

parent 080666b4

vagen/benchmark/benchmark_config.yaml→vagen/benchmark/alfworld/benchmark_config.yaml

+9 −13
Original line number Diff line number Diff line 
# Service Benchmark Configuration



# Server configuration

server:

  base_url: http://localhost:5000
@@ -24,21 +22,19 @@ benchmark: 
  step_count: 5

  

  # Batch sizes to test

  batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128]

  batch_sizes: [8]

  

  # Directory to save benchmark results

  output_dir: benchmark_results



# Dataset configurations - use paths to your actual dataset files

datasets:

  # ALFWorld with vision mode

  - name: alfworld-vision

    train_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/train.parquet

    test_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/test.parquet

    use_split: both  # Use both train and test datasets



  # Uncomment and adjust for additional environments

  # - name: frozenlake

  #   train_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/train.parquet

  #   test_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/test.parquet

  # - name: alfworld-vision

  #   train_path: data/alfworld-vision-benchmark/train.parquet

  #   test_path: data/alfworld-vision-benchmark/test.parquet

  #   use_split: both



  - name: navigation

    train_path: data/navigation-benchmark/train.parquet

    test_path: data/navigation-benchmark/test.parquet

    use_split: both
 
 No newline at end of file

vagen/benchmark/alfworld/dataset_config.yaml

0 → 100644
+16 −0
Original line number Diff line number Diff line 
# ALFWorld with vision mode

# alfworld-vision:

#   env_name: alfworld

#   env_config:

#     render_mode: vision

#     alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"

#     max_actions_per_step: 1

#   train_size: 8

#   test_size: 4



navigation:

  env_name: navigation

  env_config:

    render_mode: vision

  train_size: 8

  test_size: 4
 
 No newline at end of file

vagen/benchmark/run.sh→vagen/benchmark/alfworld/run.sh

+2 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 


# Create required directories

mkdir -p benchmark_results

mkdir -p data



# Generate datasets

echo "Generating datasets for benchmark..."
@@ -13,6 +14,7 @@ python -m vagen.env.create_dataset \ 
    --yaml_path "$SCRIPT_DIR/dataset_config.yaml" \

    --train_path data/alfworld-vision-benchmark/train.parquet \

    --test_path data/alfworld-vision-benchmark/test.parquet \

    --force_gen



# Run environment service benchmark

vagen/benchmark/dataset_config.yaml

deleted100644 → 0
+0 −30
Original line number Diff line number Diff line 
# Dataset Generation Configuration for Benchmark



# ALFWorld with text mode

alfworld-text:

  env_name: alfworld

  env_config:

    render_mode: text

    alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"

    max_actions_per_step: 1

  train_size: 128  # Generate 128 training examples

  test_size: 32    # Generate 32 test examples



# ALFWorld with vision mode

# alfworld-vision:

#   env_name: alfworld

#   env_config:

#     render_mode: vision

#     alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"

#     max_actions_per_step: 1

#   train_size: 128

#   test_size: 32



# frozenlake:

#   env_name: frozenlake

#   env_config:

#     is_slippery: false

#     size: 4

#     render_mode: text

#   train_size: 128

#   test_size: 32
 
 No newline at end of file

vagen/benchmark/navigation/benchmark_config.yaml

0 → 100644
+40 −0
Original line number Diff line number Diff line 
# Server configuration

server:

  base_url: http://localhost:5000

  timeout: 600

  max_workers: 48



# Benchmark parameters

benchmark:

  # Service functions to benchmark

  functions: 

    - create_environments_batch

    - reset_batch

    - step_batch

    - compute_reward_batch

    - get_system_prompts_batch

    - close_batch

  

  # Number of iterations for statistical significance

  iterations: 3

  

  # Number of steps to perform for step_batch

  step_count: 5

  

  # Batch sizes to test

  batch_sizes: [8]

  

  # Directory to save benchmark results

  output_dir: benchmark_results



datasets:

  # ALFWorld with vision mode

  # - name: alfworld-vision

  #   train_path: data/alfworld-vision-benchmark/train.parquet

  #   test_path: data/alfworld-vision-benchmark/test.parquet

  #   use_split: both



  - name: navigation

    train_path: data/navigation-benchmark/train.parquet

    test_path: data/navigation-benchmark/test.parquet

    use_split: both
 
 No newline at end of file

CRA Git | Maintained and supported by SUSTech CRA and CCSE