diff --git a/reverie_src/agent.py b/reverie_src/agent.py index 505a830..63cece0 100644 --- a/reverie_src/agent.py +++ b/reverie_src/agent.py @@ -245,9 +245,11 @@ class Seq2SeqAgent(BaseAgent): """ def take_action(i, idx, name): if type(name) is int: # Go to the next view - self.env.env.sims[idx].makeAction(name, 0, 0) + self.env.env.sims[idx].makeAction([name], [0], [0]) else: # Adjust - self.env.env.sims[idx].makeAction(*self.env_actions[name]) + action_params = self.env_actions[name] + + self.env.env.sims[idx].makeAction([action_params[0]], [action_params[1]], [action_params[2]]) if perm_idx is None: perm_idx = range(len(perm_obs)) @@ -266,13 +268,13 @@ class Seq2SeqAgent(BaseAgent): while src_level > trg_level: # Tune down take_action(i, idx, 'down') src_level -= 1 - while self.env.env.sims[idx].getState().viewIndex != trg_point: # Turn right until the target + while self.env.env.sims[idx].getState()[0].viewIndex != trg_point: # Turn right until the target take_action(i, idx, 'right') assert select_candidate['viewpointId'] == \ - self.env.env.sims[idx].getState().navigableLocations[select_candidate['idx']].viewpointId + self.env.env.sims[idx].getState()[0].navigableLocations[select_candidate['idx']].viewpointId take_action(i, idx, select_candidate['idx']) - state = self.env.env.sims[idx].getState() + state = self.env.env.sims[idx].getState()[0] if traj is not None: traj[i]['path'].append((state.location.viewpointId, state.heading, state.elevation)) diff --git a/reverie_src/env.py b/reverie_src/env.py index a836daa..4bc25a4 100644 --- a/reverie_src/env.py +++ b/reverie_src/env.py @@ -50,15 +50,15 @@ class EnvBatch(): sim.setDiscretizedViewingAngles(True) # Set increment/decrement to 30 degree. (otherwise by radians) sim.setCameraResolution(self.image_w, self.image_h) sim.setCameraVFOV(math.radians(self.vfov)) - sim.init() + sim.initialize() self.sims.append(sim) def _make_id(self, scanId, viewpointId): return scanId + '_' + viewpointId def newEpisodes(self, scanIds, viewpointIds, headings): - for i, (scanId, viewpointId, heading) in enumerate(zip(scanIds, viewpointIds, headings)): - self.sims[i].newEpisode(scanId, viewpointId, heading, 0) + for i, (scanId, viewpointId, heading) in enumerate(zip(scanIds[0], viewpointIds[0], headings[0])): + self.sims[i].newEpisode([scanId], [viewpointId], [heading], [0]) def getStates(self): """ @@ -69,7 +69,7 @@ class EnvBatch(): """ feature_states = [] for i, sim in enumerate(self.sims): - state = sim.getState() + state = sim.getState()[0] long_id = self._make_id(state.scanId, state.location.viewpointId) if self.features: @@ -83,7 +83,7 @@ class EnvBatch(): ''' Take an action using the full state dependent action interface (with batched input). Every action element should be an (index, heading, elevation) tuple. ''' for i, (index, heading, elevation) in enumerate(actions): - self.sims[i].makeAction(index, heading, elevation) + self.sims[i].makeAction([index], [heading], [elevation]) class R2RBatch(): @@ -231,13 +231,13 @@ class R2RBatch(): if long_id not in self.buffered_state_dict: for ix in range(36): if ix == 0: - self.sim.newEpisode(scanId, viewpointId, 0, math.radians(-30)) + self.sim.newEpisode([scanId], [viewpointId], [0], [math.radians(-30)]) elif ix % 12 == 0: - self.sim.makeAction(0, 1.0, 1.0) + self.sim.makeAction([0], [1.0], [1.0]) else: - self.sim.makeAction(0, 1.0, 0) + self.sim.makeAction([0], [1.0], [0]) - state = self.sim.getState() + state = self.sim.getState()[0] assert state.viewIndex == ix # Heading and elevation for the viewpoint center @@ -360,7 +360,7 @@ class R2RBatch(): scanIds = [item['scan'] for item in self.batch] viewpointIds = [item['path'][0] for item in self.batch] headings = [item['heading'] for item in self.batch] - self.env.newEpisodes(scanIds, viewpointIds, headings) + self.env.newEpisodes([scanIds], [viewpointIds], [headings]) return self._get_obs() def step(self, actions): diff --git a/reverie_src/train.py b/reverie_src/train.py index 08d32be..92254ce 100644 --- a/reverie_src/train.py +++ b/reverie_src/train.py @@ -55,7 +55,7 @@ def train(train_env, tok, n_iters, log_every=2000, val_envs={}, aug_env=None): start = time.time() print('\nListener training starts, start iteration: %s' % str(start_iter)) - best_val = {'val_unseen': {"spl": 0., "sr": 0., "state":"", 'update':False}} + best_val = {'val_unseen': {"spl": 0., "sr": 0., "sspl": 0. ,"state":"", 'update':False}} for idx in range(start_iter, start_iter+n_iters, log_every): listner.logs = defaultdict(list) @@ -214,7 +214,7 @@ def train_val(test_only=False): val_env_names = ['val_train_seen'] else: featurized_scans = set([key.split("_")[0] for key in list(feat_dict.keys())]) - val_env_names = ['val_train_seen', 'val_seen', 'val_unseen'] + val_env_names = ['val_seen', 'val_unseen'] train_env = R2RBatch(feat_dict, obj_feats, batch_size=args.batchSize, splits=['train'], tokenizer=tok) from collections import OrderedDict @@ -252,6 +252,7 @@ def train_val_augment(test_only=False): # Load the env img features feat_dict = read_img_features(features, test_only=test_only) + # load object feature with open('data/REVERIE/BBoxS/REVERIE_obj_feats.pkl', 'rb') as f_obj: obj_feats = pkl.load(f_obj) @@ -261,7 +262,8 @@ def train_val_augment(test_only=False): val_env_names = ['val_train_seen'] else: featurized_scans = set([key.split("_")[0] for key in list(feat_dict.keys())]) - val_env_names = ['val_train_seen', 'val_seen', 'val_unseen'] + val_env_names = ['val_seen', 'val_unseen'] + # val_env_names = ['val_train_seen', 'val_seen', 'val_unseen'] # Load the augmentation data aug_path = args.aug diff --git a/reverie_src/utils.py b/reverie_src/utils.py index fb577f1..d9bee1a 100644 --- a/reverie_src/utils.py +++ b/reverie_src/utils.py @@ -346,7 +346,7 @@ def new_simulator(): sim.setCameraResolution(WIDTH, HEIGHT) sim.setCameraVFOV(math.radians(VFOV)) sim.setDiscretizedViewingAngles(True) - sim.init() + sim.initialize() return sim @@ -357,13 +357,13 @@ def get_point_angle_feature(baseViewId=0): base_heading = (baseViewId % 12) * math.radians(30) for ix in range(36): if ix == 0: - sim.newEpisode('ZMojNkEp431', '2f4d90acd4024c269fb0efe49a8ac540', 0, math.radians(-30)) + sim.newEpisode(['ZMojNkEp431'], ['2f4d90acd4024c269fb0efe49a8ac540'], [0], [math.radians(-30)]) elif ix % 12 == 0: - sim.makeAction(0, 1.0, 1.0) + sim.makeAction([0], [1.0], [1.0]) else: - sim.makeAction(0, 1.0, 0) + sim.makeAction([0], [1.0], [0]) - state = sim.getState() + state = sim.getState()[0] assert state.viewIndex == ix heading = state.heading - base_heading @@ -567,7 +567,7 @@ def print_progress(iteration, total, prefix='', suffix='', decimals=1, bar_lengt str_format = "{0:." + str(decimals) + "f}" percents = str_format.format(100 * (iteration / float(total))) filled_length = int(round(bar_length * iteration / float(total))) - bar = '█' * filled_length + '-' * (bar_length - filled_length) + bar = 'L' * filled_length + '-' * (bar_length - filled_length) sys.stdout.write('\r%s |%s| %s%s %s' % (prefix, bar, percents, '%', suffix)), diff --git a/reverie_src/vlnbert/vlnbert_init.py b/reverie_src/vlnbert/vlnbert_init.py index 6a4a164..a7dfb76 100644 --- a/reverie_src/vlnbert/vlnbert_init.py +++ b/reverie_src/vlnbert/vlnbert_init.py @@ -1,7 +1,7 @@ import os def get_tokenizer(args): - from transformers.pytorch_transformers import BertTokenizer + from pytorch_transformers import BertTokenizer tokenizer_class = BertTokenizer diff --git a/scripts/train_reverie_agent.sh b/scripts/train_reverie_agent.sh index 1000189..47bf98e 100644 --- a/scripts/train_reverie_agent.sh +++ b/scripts/train_reverie_agent.sh @@ -1,7 +1,7 @@ export AIRBERT_ROOT=$(pwd) export PYTHONPATH=${PYTHONPATH}:${AIRBERT_ROOT}/build -name=REVERIE-RC-VLN-BERT/train-init.airbert +name=REVERIE-RC-VLN-BERT-original/train-init.airbert flag="--vlnbert vilbert @@ -26,4 +26,4 @@ flag="--vlnbert vilbert --dropout 0.5" mkdir -p snap/$name -CUDA_VISIBLE_DEVICES=$1 python reverie_src/train.py $flag --name $name +CUDA_VISIBLE_DEVICES=$1 python3 reverie_src/train.py $flag --name $name diff --git a/scripts/valid_reverie_agent.sh b/scripts/valid_reverie_agent.sh index 8a8f627..89b714c 100644 --- a/scripts/valid_reverie_agent.sh +++ b/scripts/valid_reverie_agent.sh @@ -1,6 +1,8 @@ export AIRBERT_ROOT=$(pwd) export PYTHONPATH=${PYTHONPATH}:${AIRBERT_ROOT}/build +echo $PYTHONPATH + name=REVERIE-RC-VLN-BERT/init.airbert flag="--vlnbert vilbert @@ -25,5 +27,5 @@ flag="--vlnbert vilbert --featdropout 0.4 --dropout 0.5" -CUDA_VISIBLE_DEVICES=$1 python reverie_src/train.py $flag --name $name +CUDA_VISIBLE_DEVICES=$1 python3 reverie_src/train.py $flag --name $name