minor update (d294b891) · Commits · 张泽凯 / VAGEN

vagen/env/base.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -139,7 +139,7 @@ class BaseInterface(ABC):
		for image in obs["multi_modal_data"][IMAGE_PLACEHOLDER]:
		assert isinstance(image, Image.Image), f"image must be PIL.Image.Image, got {type(image)}"
		len_of_images = len(obs["multi_modal_data"][IMAGE_PLACEHOLDER])
		len_of_image_in_text_template = len(re.findall(r'IMAGE_PLACEHOLDER', obs["text_template"]))
		len_of_image_in_text_template = len(re.findall(IMAGE_PLACEHOLDER, obs["text_template"]))
		assert len_of_images == len_of_image_in_text_template, f"len_of_images must be equal to len_of_image_in_text_template, got {len_of_images} and {len_of_image_in_text_template}"
		return obs, reward, done, info

		@@ -159,7 +159,7 @@ class BaseInterface(ABC):
		for image in obs["multi_modal_data"][IMAGE_PLACEHOLDER]:
		assert isinstance(image, Image.Image), f"image must be PIL.Image.Image, got {type(image)}"
		len_of_images = len(obs["multi_modal_data"][IMAGE_PLACEHOLDER])
		len_of_image_in_text_template = len(re.findall(r'IMAGE_PLACEHOLDER', obs["text_template"]))
		len_of_image_in_text_template = len(re.findall(IMAGE_PLACEHOLDER, obs["text_template"]))
		assert len_of_images == len_of_image_in_text_template, f"len_of_images must be equal to len_of_image_in_text_template, got {len_of_images} and {len_of_image_in_text_template}"
		return obs, info

+4 −2

Original line number	Diff line number	Diff line
		@@ -74,6 +74,7 @@ class QwenVLRolloutManger():
		- For do_embedding=True, replace <image> with <\|vision_start\|>{image_token}<\|vision_end\|> -> prompt_template
		- where {image_token} is the length of image embedding
		"""
		assert len(image_data) == prompt_template.count('<image>'), 'Number of images does not match number of <image> in the prompt template'
		raw_prompt = prompt_template.replace('<image>', '<\|vision_start\|><\|image_pad\|><\|vision_end\|>')
		row_dict['multi_modal_data'] = {'image': image_data}
		image_grid_thw = None
		@@ -426,7 +427,8 @@ class QwenVLRolloutManger():
		truncation=self.truncation)
		input_ids_prompt, attention_mask_prompt = verl_F.tokenize_and_postprocess_data(prompt=prompt_with_chat_template,
		tokenizer=self.tokenizer,
		max_length=self.config.max_response_length+self.config.max_prompt_length,
		# max_length=self.config.max_response_length+self.config.max_prompt_length,
		max_length=1,
		pad_token_id=self.tokenizer.pad_token_id,
		left_pad=True,
		truncation=self.truncation)
		@@ -452,7 +454,7 @@ class QwenVLRolloutManger():
		# if self.image_key in row_dict:
		if has_images:
		from verl.models.transformers.qwen2_vl import get_rope_index

		# NOTE please check the way position ids are computed in vllm_rollout_spmd.py
		position_ids_response = get_rope_index(
		self.processor,
		image_grid_thw=image_grid_thw,