benchmark update (9df11f99) · Commits · 张泽凯 / VAGEN

vagen/benchmark/alfworld/benchmark_config.yaml

+1 −6

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ benchmark:
		step_count: 5

		# Batch sizes to test
		batch_sizes: [8]
		batch_sizes: [128]

		# Directory to save benchmark results
		output_dir: benchmark_results
		@@ -32,8 +32,3 @@ datasets:
		train_path: data/alfworld-vision-benchmark/train.parquet
		test_path: data/alfworld-vision-benchmark/test.parquet
		use_split: both
		No newline at end of file

		# - name: navigation
		# train_path: data/navigation-benchmark/train.parquet
		# test_path: data/navigation-benchmark/test.parquet
		# use_split: both
		No newline at end of file

+2 −9

Original line number	Diff line number	Diff line
		@@ -4,12 +4,5 @@ alfworld-vision:
		render_mode: vision
		alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"
		max_actions_per_step: 1
		train_size: 8
		test_size: 4

		# navigation:
		# env_name: navigation
		# env_config:
		# render_mode: vision
		# train_size: 8
		# test_size: 4
		No newline at end of file
		train_size: 1000
		test_size: 64

0 → 100644

+34 −0

Original line number	Diff line number	Diff line
		# Server configuration
		server:
		base_url: http://localhost:5000
		timeout: 600
		max_workers: 48

		# Benchmark parameters
		benchmark:
		# Service functions to benchmark
		functions:
		- create_environments_batch
		- reset_batch
		- step_batch
		- compute_reward_batch
		- get_system_prompts_batch
		- close_batch

		# Number of iterations for statistical significance
		iterations: 3

		# Number of steps to perform for step_batch
		step_count: 5

		# Batch sizes to test
		batch_sizes: [128]

		# Directory to save benchmark results
		output_dir: benchmark_results

		datasets:
		- name: frozenlake-vision
		train_path: data/frozenlake-vision-benchmark/train.parquet
		test_path: data/frozenlake-vision-benchmark/test.parquet
		use_split: both
		No newline at end of file

0 → 100644

+7 −0

0 → 100755

+32 −0

Original line number	Diff line number	Diff line
		#!/bin/bash
		# Setup script for environment service benchmark

		# Create required directories
		mkdir -p benchmark_results
		mkdir -p data

		echo "Generating datasets for benchmark..."

		SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
		# Generatedataset
		python -m vagen.env.create_dataset \
		--yaml_path "$SCRIPT_DIR/dataset_config.yaml" \
		--train_path data/frozenlake-vision-benchmark/train.parquet \
		--test_path data/frozenlake-vision-benchmark/test.parquet \
		--force_gen

		# Run environment service benchmark

		# Set environment variables
		export PYTHONPATH=$(pwd):$PYTHONPATH

		# Create output directory
		mkdir -p benchmark_results



		# Run benchmark
		echo "Running service benchmark"
		python -m vagen.benchmark.service_benchmark --config "$SCRIPT_DIR/benchmark_config.yaml"

		echo "Benchmark complete. Results saved to benchmark_results directory."
		No newline at end of file