init alfworld (83ac8dd3) · Commits · 张泽凯 / VAGEN

vagen/env/init.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -2,7 +2,7 @@ from .sokoban import SokobanEnv,SokobanEnvConfig
		from .frozenlake import FrozenLakeEnv,FrozenLakeEnvConfig, FrozenLakeService
		# from .navigation import NavigationEnv, NavigationEnvConfig, NavigationServiceConfig, NavigationService
		# from .svg import SVGEnv, SvgEnvConfig, SVGService, SVGServiceConfig
		# from .primitive_skill import PrimitiveSkillEnv, PrimitiveSkillEnvConfig, PrimitiveSkillService, PrimitiveSkillConfig
		# from .primitive_skill import PrimitiveSkillEnv, PrimitiveSkillEnvConfig, PrimitiveSkillService, PrimitiveSkillServiceConfig
		from .alfworld import ALFWorldEnv, ALFWorldEnvConfig, ALFWorldService, ALFWorldServiceConfig
		REGISTERED_ENV = {
		"sokoban": {
		@@ -30,7 +30,7 @@ REGISTERED_ENV = {
		# "env_cls": PrimitiveSkillEnv,
		# "config_cls": PrimitiveSkillEnvConfig,
		# "service_cls": PrimitiveSkillService,
		# "service_config_cls": PrimitiveSkillConfig
		# "service_config_cls": PrimitiveSkillServiceConfig
		# },
		"alfworld": {
		"env_cls": ALFWorldEnv,

vagen/env/alfworld/alf_utils.py

0 → 100644

+146 −0

Original line number	Diff line number	Diff line
		import os
		import yaml
		import torchvision.transforms as T
		from alfworld.agents.environment.alfred_thor_env import AlfredThorEnv
		import gymnasium as gym
		from gymnasium import spaces
		import alfworld.agents.environment as environment
		from typing import Optional
		import numpy as np
		import torch
		import random


		ALF_ACTION_LIST=["pass", "goto", "pick", "put", "open", "close", "toggle", "heat", "clean", "cool", "slice", "inventory", "examine", "look"]
		# ALF_ITEM_LIST =

		def load_config_file(path):
		print(f"[DEBUG]: alfconfig path: {path}")
		assert os.path.exists(path), "Invalid config file"
		with open(path) as reader:
		config = yaml.safe_load(reader)
		return config

		def get_obs_image(env):
		transform = T.Compose([T.ToTensor()])
		current_frames = env.get_frames()
		image_tensors = [transform(i).cuda() for i in current_frames]
		for i in range(len(image_tensors)):
		image_tensors[i] = image_tensors[i].permute(1, 2, 0)
		image_tensors[i]*= 255
		image_tensors[i] = image_tensors[i].int()
		image_tensors[i] = image_tensors[i][:,:,[2,1,0]]
		image_tensors = torch.stack(image_tensors, dim=0)
		return image_tensors

		class AlfEnv(gym.Env):
		def __init__(self, config_file):
		config = load_config_file(config_file)
		env_type = config['env']['type']
		env = getattr(environment, env_type)(config, train_eval='train')
		self.env = env.init_env(batch_size=1)
		self.action_space = spaces.Discrete(len(ALF_ACTION_LIST))
		self.observation_space = spaces.Box(low=0, high=255, shape=(300, 300, 3), dtype=np.uint8)
		# Add the previous admissible commands for step
		self.prev_admissible_commands = None
		self.num_envs = 1
		def step(self, action):
		## SZ.3.4: sanity checking legal action as rewards
		action, legal_action = process_action(self.env, action, self.prev_admissible_commands)
		obs, scores, dones, infos = self.env.step(action)
		infos['observation_text'] = obs
		reward = compute_reward(infos, legal_action)
		self.prev_admissible_commands = list(infos['admissible_commands'])[0]
		return self._get_obs(), reward, dones, infos

		def reset(
		self,
		seed=42,
		):
		self.env.seed(seed)
		obs, infos = self.env.reset()
		infos['observation_text'] = obs
		self.prev_admissible_commands = list(infos['admissible_commands'])[0]
		return self._get_obs(), infos

		def _get_obs(self):
		image = get_obs_image(self.env)
		return image

		def process_action(env, action=None, action_list=None):
		"""
		An function to process the action
		env: the environment should be of type AlfredThorEnv
		action: the list of action to be processeed, it is a list of strings.
		"""
		if type(env) != AlfEnv and type(env) != AlfredThorEnv:
		pass
		else:
		legal_action = False
		for i in range(len(action)):
		action[i] = action[i].lower()
		# TODO: need to figure this out
		if len(action[i]) == 0:
		print("Action is empty!!!!")
		# randomly choose an action from the action list if illegal
		action[i] = action_list[random.randint(0, len(action_list)-1)]
		else:
		try:
		action_index = action[i].find('"action":')
		# string has the following format '"action": "look"\n}'
		if action_index == -1:
		# if we cannot find "action":, then we pick the last 30 characters
		string = action[i][-30:]
		else:
		string = action[i][action_index:]
		# post processing by removing the first and last part of the string
		for act in action_list:
		if act in string:
		action[i] = act
		# if found legal action, set legal_action = True
		legal_action = True
		break
		except:
		# randomly choose an action from the action list if illegal
		action[i] = action_list[random.randint(0, len(action_list)-1)]

		return action, legal_action


		def compute_reward(infos, legal_action):
		# A function to compute the shaped reward for the alfworld environment
		# infos: the info returned by the environment
		# legal_action: a boolean value to indicate if the action is legal
		## Tentative rewards: r = success_reward * 10 + goal_conditioned_r - 1*illegal_action
		reward = 50*float(infos['won'][0]) + float(infos['goal_condition_success_rate'][0])
		if not legal_action:
		# adding a reward penalty to illegal actions
		reward -= 1
		reward = [reward]
		return torch.tensor(reward)

		def get_encoded_text(observation_text, tokenizer, model):

		encoded_input = tokenizer(observation_text, return_tensors='pt')
		outputs = model(**encoded_input)
		cls_embeddings = outputs.last_hidden_state[:,0,:]

		return cls_embeddings

		def get_concat(obs, infos, tokenizer, model, device):
		assert 'observation_text' in infos.keys(), 'observation_text not in infos!'
		obs_text = infos['observation_text']
		obs_text_encode = get_encoded_text(obs_text, tokenizer, model)
		obs_text_encode = obs_text_encode.to(device)
		obs_cat = torch.cat((obs.flatten(start_dim=1), obs_text_encode), dim=1)
		return obs_cat

		def get_cards_concat(obs, infos, tokenizer, model, device):
		## Need to move these codes to a CNN utils or something
		assert 'Formula' in infos[0].keys(), 'Formula not in infos!'
		infos = infos[0]
		formula_list = infos['Formula']
		formula = "".join([str("".join([str(x) for x in formula_list]))])
		obs_text_encode = get_encoded_text(formula, tokenizer, model).to(device)
		obs_cat = torch.cat((obs.flatten(start_dim=1), obs_text_encode), dim=1)
		return obs_cat
		No newline at end of file

vagen/env/alfworld/data/alf-config.yaml

+9 −0

Original line number	Diff line number	Diff line
		@@ -2,15 +2,24 @@ dataset:
		data_path: '$ALFWORLD_DATA/json_2.1.1/train'
		eval_id_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_seen' # null/None to disable
		eval_ood_data_path: '$ALFWORLD_DATA/json_2.1.1/valid_unseen' # null/None to disable
		<<<<<<< HEAD
		num_train_games: 1 # max training games (<=0 indicates full dataset)
		num_eval_games: 1 # max evaluation games (<=0 indicates full dataset)
		=======
		num_train_games: 10 # max training games (<=0 indicates full dataset)
		num_eval_games: 2 # max evaluation games (<=0 indicates full dataset)
		>>>>>>> 581e528 (init alfworld)

		logic:
		domain: '$ALFWORLD_DATA/logic/alfred.pddl' # PDDL domain file that defines the world dynamics
		grammar: '$ALFWORLD_DATA/logic/alfred.twl2' # Grammar file that defines the text feedbacks

		env:
		<<<<<<< HEAD
		type: 'AlfredThorEnv' # 'AlfredTWEnv' or 'AlfredThorEnv' or 'AlfredHybrid'
		=======
		type: 'AlfredThorEnv' # 'AlfredTWEnv' or 'AlfredThorEnv' or 'AlfredHybrid'
		>>>>>>> 581e528 (init alfworld)
		regen_game_files: False # check if game is solvable by expert and save to game.tw-pddl file
		domain_randomization: False # shuffle Textworld print order and object id nums
		task_types: [1, 2, 3, 4, 5, 6] # task-type ids: 1 - Pick & Place, 2 - Examine in Light, 3 - Clean & Place, 4 - Heat & Place, 5 - Cool & Place, 6 - Pick Two & Place

vagen/env/alfworld/service.py

+21 −0

Original line number	Diff line number	Diff line
		import os
		import time
		import logging
		import threading
		import multiprocessing as mp
		from queue import Empty
		from concurrent.futures import ThreadPoolExecutor
		@@ -60,6 +61,19 @@ class ALFWorldService(BaseService):
		# Thread pool for potential intra-process parallelism
		thread_pool = ThreadPoolExecutor(max_workers=max_thread_workers)
		running = True

		#@ TODO revise this
		def send_heartbeat():
		while running:
		try:
		result_queue.put((-999, "heartbeat", process_id))
		time.sleep(120)
		except:
		pass

		heartbeat_thread = threading.Thread(target=send_heartbeat, daemon=True)
		heartbeat_thread.start()

		while running:
		try:
		command, task_id, args = task_queue.get()
		@@ -179,6 +193,13 @@ class ALFWorldService(BaseService):
		while True:
		try:
		rid, status, result = self.result_queues[pid].get(timeout=self.timeout)

		#@ TODO revise this
		if rid == -999 and status == "heartbeat":
		last_heartbeat = time.time()
		self.logger.debug(f"Received heartbeat from worker {pid}")
		continue

		if rid != task_id:
		continue
		if status == 'success':

vagen/examples/debug/debug_alfworld_text_grpo_service/run.sh

+4 −0

Original line number	Diff line number	Diff line
		@@ -59,7 +59,11 @@ python3 -m vagen.trainer.main_ppo \
		trainer.critic_warmup=0 \
		trainer.logger=['console','wandb'] \
		trainer.project_name='vagen_debug' \
		<<<<<<< HEAD
		trainer.experiment_name='grpo_mask_loss_alfworld_text_debug' \
		=======
		trainer.experiment_name='grpo_mask_loss_frozenlake_vision_debug' \
		>>>>>>> 581e528 (init alfworld)
		trainer.n_gpus_per_node=1 \
		trainer.nnodes=1 \
		trainer.save_freq=100 \

vagen/env/alfworld/env.py

+1 −1

File changed.

Contains only whitespace changes.

vagen/env/alfworld/service_config.py

+1 −1

File changed.

Contains only whitespace changes.

Admin message