update reward model server for svg (81ac638c) · Commits · 张泽凯 / VAGEN

test.py

0 → 100644

+120 −0

Original line number	Diff line number	Diff line
		# test_reward_server.py
		# 测试评分服务器的脚本

		import requests
		import json
		import argparse
		import time

		# 示例SVG数据
		SAMPLE_SVG_PAIRS = [
		{
		"gt_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<circle cx="50" cy="50" r="40" fill="yellow"/>
		<circle cx="35" cy="40" r="5" fill="black"/>
		<circle cx="65" cy="40" r="5" fill="black"/>
		<path d="M30 60 Q50 75 70 60" stroke="black" stroke-width="3" fill="none"/>
		</svg>""",
		"gen_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<circle cx="50" cy="50" r="40" fill="yellow"/>
		<circle cx="35" cy="40" r="5" fill="black"/>
		<circle cx="65" cy="40" r="5" fill="black"/>
		<path d="M30 60 Q50 75 70 60" stroke="black" stroke-width="3" fill="none"/>
		</svg>"""
		},
		{
		"gt_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<circle cx="50" cy="50" r="40" fill="yellow"/>
		<circle cx="35" cy="40" r="5" fill="black"/>
		<circle cx="65" cy="40" r="5" fill="black"/>
		<path d="M30 60 Q50 75 70 60" stroke="black" stroke-width="3" fill="none"/>
		</svg>""",
		"gen_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<circle cx="50" cy="50" r="40" fill="red"/>
		<circle cx="35" cy="40" r="5" fill="black"/>
		<circle cx="65" cy="40" r="5" fill="black"/>
		<path d="M30 70 Q50 55 70 70" stroke="black" stroke-width="3" fill="none"/>
		</svg>"""
		},
		{
		"gt_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<rect x="20" y="20" width="60" height="60" fill="blue"/>
		<circle cx="50" cy="50" r="20" fill="white"/>
		</svg>""",
		"gen_svg_code": """<svg width="100" height="100" viewBox="0 0 100 100">
		<rect x="20" y="20" width="60" height="60" fill="blue"/>
		<circle cx="50" cy="50" r="15" fill="white"/>
		</svg>"""
		}
		]

		def test_health(url):
		"""测试健康检查端点"""
		try:
		response = requests.get(f"{url}/health")
		print(f"健康检查状态码: {response.status_code}")
		if response.status_code == 200:
		print(f"健康检查响应: {response.json()}")
		return True
		return False
		except Exception as e:
		print(f"健康检查出错: {e}")
		return False

		def test_compute_score(url, sample_pairs):
		"""测试评分计算端点"""
		results = []

		for i, pair in enumerate(sample_pairs):
		try:
		print(f"\n测试示例 {i+1}:")
		start_time = time.time()
		response = requests.post(
		f"{url}/compute_score",
		json=pair,
		headers={"Content-Type": "application/json"}
		)
		request_time = time.time() - start_time

		print(f"请求耗时: {request_time:.4f}秒")
		print(f"状态码: {response.status_code}")

		if response.status_code == 200:
		result = response.json()
		print(f"评分结果: {json.dumps(result, indent=2)}")
		results.append(result)
		else:
		print(f"错误: {response.text}")
		results.append({"error": response.text})
		except Exception as e:
		print(f"请求出错: {e}")
		results.append({"error": str(e)})

		return results

		def main():
		parser = argparse.ArgumentParser(description='测试评分服务器')
		parser.add_argument('--url', type=str, default='http://127.0.0.1:5000', help='评分服务器URL')
		args = parser.parse_args()

		print(f"测试评分服务器: {args.url}")

		# 测试健康检查
		if not test_health(args.url):
		print("健康检查失败，退出测试")
		return

		# 测试评分计算
		print("\n测试评分计算...")
		results = test_compute_score(args.url, SAMPLE_SVG_PAIRS)

		# 显示摘要
		print("\n测试摘要:")
		for i, result in enumerate(results):
		if "error" in result:
		print(f"示例 {i+1}: 失败 - {result['error']}")
		else:
		print(f"示例 {i+1}: 成功 - 总分: {result.get('total_score', 'N/A')}")

		if __name__ == "__main__":
		main()
		No newline at end of file

vagen/env/README.md

+5 −1

Original line number	Diff line number	Diff line
		@@ -28,8 +28,12 @@ pip install "gymnasium[toy-text]"
		pip install bs4
		pip install svgpathtools
		pip install cairosvg
		pip install flask

		# To run experiment of SVG simply copy the code below
		# create server for reward model
		python vagen/env/svg/reward_model_server.py

		# Then run experiment of SVG simply copy the code below
		bash vagen\examples\debug_svg_vision_grpo\run_structure_only.sh
		bash vagen\examples\debug_svg_vision_grpo\run_dino_only.sh
		```
		No newline at end of file

vagen/env/svg/config.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ class SVGConfig(BaseConfig):
		format_penalty: float = 0.0
		# Analysis mode for logging
		analysis_mode: bool = False
		reward_url: str = "http://127.0.0.1:5000"

		def config_id(self) -> str:
		"""Generate a unique identifier for this configuration"""

vagen/env/svg/env.py

+78 −11

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@ from vagen.env.svg.score import calculate_total_score
		from vagen.env.utils.context_utils import parse_llm_raw_response, convert_numpy_to_PIL
		from .config import SVGConfig
		from .prompt import system_prompt, init_observation_template, action_template
		from .reward_model_server import compute_svg_score

		import os
		import re
		@@ -45,6 +46,9 @@ class SVGEnv(BaseEnv):
		if self.config.analysis_mode:
		self._setup_analysis_logging()

		# initialize reward model server api url
		self.reward_url = self.config.get("reward_url", "http://127.0.0.1:5000")


		def reset(self, seed=None) -> Tuple[Dict, Dict]:
		"""Reset the environment with an optional seed"""
		@@ -162,13 +166,11 @@ class SVGEnv(BaseEnv):
		score_config["code_weight"] = self.config.code_weight

		try:
		scores = calculate_total_score(
		gt_im=self.gt_image,
		gen_im=gen_image,
		gt_code=self.gt_svg_code,
		gen_code=self.gen_svg_code,
		score_config=score_config
		)
		scores = self._get_scores_from_service(self.gt_svg_code, self.gen_svg_code)

		if not scores or "error" in scores:
		logging.warning(f"ERROR IN SCORE SERVER!!!: {scores.get('error') if scores else 'Unknown error'}")
		scores = {"total_score": 0, "dino_score": 0, "structural_score": 0, "color_score": 0, "code_score": 0}
		except Exception as e:
		print(f"Score calculation failed: {e}")

		@@ -303,17 +305,64 @@ class SVGEnv(BaseEnv):
		success_handler.setFormatter(logging.Formatter('%(message)s'))
		self.success_logger.addHandler(success_handler)

		def _get_scores_from_service(self, gt_svg_code, gen_svg_code):
		try:
		import requests

		data = {
		"gt_svg_code": gt_svg_code,
		"gen_svg_code": gen_svg_code
		}

		response = requests.post(
		f"{self.reward_url}/compute_score",
		json=data,
		headers={"Content-Type": "application/json"},
		timeout=10
		)

		if response.status_code == 200:
		return response.json()
		else:
		logging.error(f"Error in scoring: {response.status_code} - {response.text}")
		return {"error": f"Error in scoring: {response.status_code}"}

		except Exception as e:
		logging.error(f"error in scoring server: {str(e)}")
		return {"error": str(e)}

		if __name__ == "__main__":
		import requests
		import time

		# Create configuration
		config = SVGConfig(
		dataset_name="starvector/svg-emoji-simple",
		data_dir="vagen/env/svg/data",
		split="test",
		model_size="small"
		model_size="small",
		dino_only=True,
		reward_url="http://127.0.0.1:5000" # Default reward server URL
		)

		# Check if reward server is running
		server_available = False
		try:
		response = requests.get("http://127.0.0.1:5000/health", timeout=2)
		if response.status_code == 200:
		server_available = True
		print("Reward server is available")
		else:
		print(f"Reward server returned status code: {response.status_code}")
		except Exception as e:
		print(f"Reward server is not available: {e}")
		print("Please make sure to run the reward server first:")
		print("python simple_reward_server.py --port 5000")

		try:
		# Initialize environment
		env = SVGEnv(config)
		print(f"Successfully loaded dataset")
		print("Successfully loaded dataset")

		# Test with seed
		seed = 42
		@@ -337,11 +386,29 @@ if __name__ == "__main__":
		</svg>
		</answer>"""

		# Execute step
		print("Executing step with SVG action...")
		start_time = time.time()
		obs, reward, done, info = env.step(action)
		elapsed_time = time.time() - start_time

		print(f"Step completed in {elapsed_time:.4f} seconds")
		print(f"Reward: {reward}")
		print(f"Done: {done}")
		print(f"obs:{obs}")
		print(f"Score components: {info.get('scores', {})}")

		# Print score components
		if 'scores' in info:
		print("Score components:")
		for key, value in info['scores'].items():
		print(f" {key}: {value}")
		else:
		print("No scores returned")

		# If server was unavailable, explain the fallback behavior
		if not server_available:
		print("\nNote: Test completed without an active reward server.")
		print("The environment used fallback scoring or empty scores.")
		print("For proper scoring, please start the reward server before running this test.")

		# Test with another seed to verify determinism
		seed = 123

vagen/env/svg/reward_model_server.py

0 → 100644

+129 −0

Original line number	Diff line number	Diff line
		# simple_reward_server.py
		# A simple local server that handles SVG scoring requests

		import json
		import time
		import logging
		import argparse
		from flask import Flask, request, jsonify

		# Import SVG scoring related modules
		# Note: Assuming these modules are installed in your environment
		from vagen.env.svg.score import calculate_total_score
		from vagen.env.svg.svg_utils import process_and_rasterize_svg

		# Configure logging
		logging.basicConfig(
		level=logging.INFO,
		format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
		)
		logger = logging.getLogger(__name__)

		# Create Flask app
		app = Flask(__name__)

		# Global variable
		score_config = None

		def compute_svg_score(gt_svg_code, gen_svg_code):
		"""Compute SVG score"""
		try:
		# Process the SVG code and generate images
		_, gt_image = process_and_rasterize_svg(gt_svg_code)
		_, gen_image = process_and_rasterize_svg(gen_svg_code)

		# Calculate score
		scores = calculate_total_score(
		gt_im=gt_image,
		gen_im=gen_image,
		gt_code=gt_svg_code,
		gen_code=gen_svg_code,
		score_config=score_config
		)

		return scores

		except Exception as e:
		logger.error(f"Error occurred while computing SVG score: {str(e)}")
		return {"error": str(e)}

		@app.route('/health', methods=['GET'])
		def health_check():
		"""Health check endpoint"""
		return jsonify({"status": "ok", "message": "SVG scoring service is running normally"}), 200

		@app.route('/compute_score', methods=['POST'])
		def score_endpoint():
		"""SVG scoring endpoint"""
		start_time = time.time()

		# Retrieve request data
		data = request.json
		if not data:
		return jsonify({"error": "Invalid request data"}), 400

		# Extract SVG code
		gt_svg_code = data.get('gt_svg_code')
		gen_svg_code = data.get('gen_svg_code')

		if not gt_svg_code or not gen_svg_code:
		return jsonify({"error": "Missing required parameters: 'gt_svg_code' and 'gen_svg_code'"}), 400

		# Compute score
		result = compute_svg_score(gt_svg_code, gen_svg_code)

		# Log processing time
		process_time = time.time() - start_time
		logger.info(f"Request processed in: {process_time:.4f} seconds")

		# Add processing time to the response
		if isinstance(result, dict) and "error" not in result:
		result["process_time"] = process_time

		return jsonify(result)

		def main():
		"""Main function"""
		global score_config

		parser = argparse.ArgumentParser(description='Local SVG scoring server')
		parser.add_argument('--port', type=int, default=5000, help='Local server port (default: 5000)')
		parser.add_argument('--host', type=str, default='127.0.0.1', help='Local server host (default: 127.0.0.1)')
		parser.add_argument('--model-size', type=str, default='small', help='Model size (default: small)')
		parser.add_argument('--dino-only', action='store_true', help='Use DINO scoring only')
		parser.add_argument('--dino-weight', type=float, help='DINO scoring weight')
		parser.add_argument('--structural-weight', type=float, help='Structural scoring weight')
		parser.add_argument('--color-weight', type=float, help='Color scoring weight')
		parser.add_argument('--code-weight', type=float, help='Code scoring weight')

		args = parser.parse_args()

		# Set score configuration
		score_config = {
		"model_size": args.model_size,
		"dino_only": args.dino_only,
		}

		# Add optional weights
		if args.dino_weight is not None:
		score_config["dino_weight"] = args.dino_weight
		if args.structural_weight is not None:
		score_config["structural_weight"] = args.structural_weight
		if args.color_weight is not None:
		score_config["color_weight"] = args.color_weight
		if args.code_weight is not None:
		score_config["code_weight"] = args.code_weight

		logger.info(f"Score configuration: {score_config}")

		# Start Flask application
		logger.info(f"Starting Flask server on {args.host}:{args.port}")
		app.run(host=args.host, port=args.port)

		if __name__ == "__main__":
		try:
		main()
		except KeyboardInterrupt:
		logger.info("Received interrupt signal, shutting down server...")
		except Exception as e:
		logger.error(f"An error occurred: {str(e)}")

Admin message