Merge branch 'prompt_dev' of github.com:RAGEN-AI/vagen into prompt_dev (9351f379) · Commits · 张泽凯 / VAGEN

.gitignore

+1 −0

+4 −14

Original line number	Diff line number	Diff line
		# Service Benchmark Configuration

		# Server configuration
		server:
		base_url: http://localhost:5000
		@@ -24,21 +22,13 @@ benchmark:
		step_count: 5

		# Batch sizes to test
		batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128]
		batch_sizes: [8]

		# Directory to save benchmark results
		output_dir: benchmark_results

		# Dataset configurations - use paths to your actual dataset files
		datasets:
		# ALFWorld with vision mode
		- name: alfworld-vision
		train_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/train.parquet
		test_path: /home/yaning/workspace/vagen/data/alfworld-vision-benchmark/test.parquet
		use_split: both # Use both train and test datasets

		# Uncomment and adjust for additional environments
		# - name: frozenlake
		# train_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/train.parquet
		# test_path: /home/yaning/workspace/vagen/data/frozenlake-benchmark/test.parquet
		# use_split: both
		No newline at end of file
		train_path: data/alfworld-vision-benchmark/train.parquet
		test_path: data/alfworld-vision-benchmark/test.parquet
		use_split: both
		No newline at end of file

0 → 100644

+8 −0

Original line number	Diff line number	Diff line
		alfworld-vision:
		env_name: alfworld
		env_config:
		render_mode: vision
		alf_config_path: "${env:ALFWORLD_DATA}/alf-config.yaml"
		max_actions_per_step: 1
		train_size: 1000
		test_size: 64

+3 −1

Original line number	Diff line number	Diff line
		@@ -3,6 +3,7 @@

		# Create required directories
		mkdir -p benchmark_results
		mkdir -p data

		# Generate datasets
		echo "Generating datasets for benchmark..."
		@@ -13,6 +14,7 @@ python -m vagen.env.create_dataset \
		--yaml_path "$SCRIPT_DIR/dataset_config.yaml" \
		--train_path data/alfworld-vision-benchmark/train.parquet \
		--test_path data/alfworld-vision-benchmark/test.parquet \
		--force_gen

		# Run environment service benchmark

		@@ -26,6 +28,6 @@ mkdir -p benchmark_results

		# Run benchmark
		echo "Running service benchmark"
		python -m vagen.benchmark.service_benchmark --config "$SCRIPT_DIR/benchmark_config.yaml"
		python -m vagen.utils.service_benchmark --config "$SCRIPT_DIR/benchmark_config.yaml"

		echo "Benchmark complete. Results saved to benchmark_results directory."
		No newline at end of file

0 → 100644

+34 −0

Original line number	Diff line number	Diff line
		# Server configuration
		server:
		base_url: http://localhost:5000
		timeout: 600
		max_workers: 48

		# Benchmark parameters
		benchmark:
		# Service functions to benchmark
		functions:
		- create_environments_batch
		- reset_batch
		- step_batch
		- compute_reward_batch
		- get_system_prompts_batch
		- close_batch

		# Number of iterations for statistical significance
		iterations: 3

		# Number of steps to perform for step_batch
		step_count: 5

		# Batch sizes to test
		batch_sizes: [128]

		# Directory to save benchmark results
		output_dir: benchmark_results

		datasets:
		- name: frozenlake-vision
		train_path: data/frozenlake-vision-benchmark/train.parquet
		test_path: data/frozenlake-vision-benchmark/test.parquet
		use_split: both
		No newline at end of file