map_nav_src/reverie/agent_base.py

@@ -27,13 +27,7 @@ class BaseAgent(object):
     def get_results(self, detailed_output=False):
         output = []
         for k, v in self.results.items():
-            output.append({
+            output.append({'instr_id': k, 'trajectory': v['path'], 'pred_objid': v['pred_objid'], 'found': v['found'], 'gt_found': v['gt_found']})
-                'instr_id': k,
-                'trajectory': v['path'],
-                'pred_objid': v['pred_objid'],
-                'found': v['found'],
-                'gt_found': v['gt_found']
-            })
             if detailed_output:
                 output[-1]['details'] = v['details']
         return output

map_nav_src/reverie/agent_obj.py

@@ -416,14 +416,13 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     else:
                         og = None
 
-
                     # 如果有找到，og 會是 object id
                     # 如果是 not found，og 會是 -1
                     # 如果這個 viewpoint 看不到物件，og 會是 None
                     gmap.node_stop_scores[i_vp] = {
                         'stop': nav_probs[i, 0].data.item(),
                         'og': og,
-                        'og_details': {'objids': i_objids, 'logits': torch.cat([i_obj_logits[:len(i_objids)], i_obj_logits[[-1]] ], dim=0)},
+                        'og_details': {'objids': i_objids, 'logits': i_obj_logits[:len(i_objids)]},
                     }
                                         
             if train_ml is not None:
@@ -443,9 +442,9 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     )
                     ml_loss += self.criterion(nav_outs['local_logits'], local_nav_targets)  # local
                 # objec grounding 
-                # obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
+                obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
                 # print(t, obj_targets[6], obj_logits[6], obs[6]['obj_ids'], pano_inputs['view_lens'][i], obs[6]['gt_obj_id'])
-                # og_loss += self.criterion(obj_logits, obj_targets)
+                og_loss += self.criterion(obj_logits, obj_targets)
                 # print(F.cross_entropy(obj_logits, obj_targets, reduction='none'))
                 # print(t, 'og_loss', og_loss.item(), self.criterion(obj_logits, obj_targets).item())
                                                    
@@ -505,7 +504,7 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     if stop_node is not None and obs[i]['viewpoint'] != stop_node:
                         traj[i]['path'].append(gmaps[i].graph.path(obs[i]['viewpoint'], stop_node))
                     traj[i]['pred_objid'] = stop_score['og']
-                    if stop_score['og'] == -1 or stop_score['og'] == None:
+                    if stop_score['og'] == -1:
                         traj[i]['found'] = False
                     else:
                         traj[i]['found'] = True
@@ -532,11 +531,11 @@ class GMapObjectNavAgent(Seq2SeqAgent):
 
         if train_ml is not None:
             ml_loss = ml_loss * train_ml / batch_size
-            # og_loss = og_loss * train_ml / batch_size
+            og_loss = og_loss * train_ml / batch_size
             self.loss += ml_loss
-            # self.loss += og_loss
+            self.loss += og_loss
             self.logs['IL_loss'].append(ml_loss.item())
-            # self.logs['OG_loss'].append(og_loss.item())
+            self.logs['OG_loss'].append(og_loss.item())
 
         '''
         print("TRAJ:")

map_nav_src/reverie/data_utils.py

@@ -87,7 +87,6 @@ def construct_instrs(anno_dir, dataset, splits, tokenizer, max_instr_len=512):
                 new_item['objId'] = None
             new_item['instruction'] = instr
             new_item['instr_encoding'] = item['instr_encodings'][j][:max_instr_len]
-            new_item['path'] = item['path'][j]
             new_item['found'] = item['found'][j]
             del new_item['instructions']
             del new_item['instr_encodings']

map_nav_src/reverie/env.py

@@ -8,22 +8,12 @@ import random
 import networkx as nx
 from collections import defaultdict
 import copy
-from glob import glob
-
 
 import MatterSim
 
 from utils.data import load_nav_graphs, new_simulator
 from utils.data import angle_feature, get_all_point_angle_feature
 
-with open('./node_region.json') as fp:
-    node_region = json.load(fp)
-
-with open('region2objs.json') as fp:
-    region2objs = json.load(fp)
-
-with open('vp2objs.json') as fp:
-    vp2objs = json.load(fp)
 
 class EnvBatch(object):
     ''' A simple wrapper for a batch of MatterSim environments,
@@ -370,10 +360,6 @@ class ReverieObjectNavBatch(object):
         path = sum(pred_path, [])
         assert gt_path[0] == path[0], 'Result trajectories should include the start position'
 
-        pred_stop_region = node_region[scan][path[-1]]
-        gt_stop_region = node_region[scan][gt_path[-1]]
-
-
         scores['action_steps'] = len(pred_path) - 1
         scores['trajectory_steps'] = len(path) - 1
         scores['trajectory_lengths'] = np.sum([shortest_distances[a][b] for a, b in zip(path[:-1], path[1:])])
@@ -383,98 +369,10 @@ class ReverieObjectNavBatch(object):
         goal_viewpoints = set(self.obj2vps['%s_%s'%(scan, str(gt_objid))])
         assert len(goal_viewpoints) > 0, '%s_%s'%(scan, str(gt_objid))
 
-        scores['found_success'] = float(pred_found == gt_found) 
-
         scores['success'] = float(path[-1] in goal_viewpoints)
-        scores['room_success'] = float(pred_stop_region == gt_stop_region)
+        scores['found_success'] = float(pred_found == gt_found)
         scores['oracle_success'] = float(any(x in goal_viewpoints for x in path))
-
-        gt_room_start_vp = None
-        gt_back_path = []
-        gt_front_path = []
-        for vp in gt_path[::-1]:
-            if node_region[scan][vp] == gt_stop_region and gt_front_path == []:
-                gt_back_path.append(vp)
-                gt_room_start_vp = vp
-            else:
-                gt_front_path.append(vp)
-
-        gt_front_path = gt_front_path[::-1]
-        gt_back_path = gt_back_path[::-1]
-
-        assert (gt_front_path + gt_back_path) == gt_path, "Front path & Back path error"
-
-        gt_front_path += [gt_room_start_vp]
-
-        '''
-        if scores['success'] == 1.0:
-            scores['found_success'] = float(pred_found == gt_found) 
-        else:
-            scores['found_success'] = 0.0
-        '''
-        gt_reach_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_front_path[:-1], gt_front_path[1:])])
-        gt_explore_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_back_path[:-1], gt_back_path[1:])])
-
-        if scores['room_success'] != 0.0:
-            # corse-grained
-
-            # get the reach_path & explore_path
-            room_start_vp = None
-            back_path = []
-            front_path = []
-            for vp in path[::-1]:
-                if node_region[scan][vp] == gt_stop_region and front_path == []:
-                    back_path.append(vp)
-                    room_start_vp = vp
-                else:
-                    front_path.append(vp)
-
-            front_path = front_path[::-1]
-            back_path = back_path[::-1]
-            assert (front_path + back_path) == path, "Front path & Back path error"
-
-            # front_path = ... room_start_vp
-            # back_path = room_start_vp ...
-            front_path += [room_start_vp]
-
-            reach_length = np.sum([shortest_distances[a][b] for a, b in zip(front_path[:-1], front_path[1:])]) if len(front_path) != 1 else 0.01
-            explore_length = np.sum([shortest_distances[a][b] for a, b in zip(back_path[:-1], back_path[1:])]) if len(back_path) != 1 else 0.01
-
-            
-            scores['room_spl'] = scores['room_success'] * gt_reach_length / max(reach_length, gt_reach_length, 0.01)
-            if scores['found_success'] != 0.0:
-                # fine-grained score
-                # p is converage rate
-                if gt_found:
-                    p = 1.0
-                else:
-                    explore_objs = set()
-                    for vp in back_path:
-                        explore_objs.update(vp2objs[vp])
-                    p = len(explore_objs) / len(region2objs[scan][gt_stop_region])
-                scores['coverage_rate'] = p
-                scores['explore_spl'] = scores['room_success'] * scores['found_success'] * gt_explore_length / max(gt_explore_length, explore_length, 0.01) * p
-            else:
-                scores['coverage_rate'] = 0
-                scores['explore_spl'] = 0
-        else:
-            scores['room_spl'] = 0.0
-            scores['coverage_rate'] = 0
-            scores['explore_spl'] = 0
-
-
-
-        
         scores['spl'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01)
-        '''
-        scores['sspl_1'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
-        scores['sspl_2'] = scores['room_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
-        scores['sspl_3'] = scores['oracle_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
-
-        scores['ss_1'] = scores['success'] * scores['found_success']
-        scores['ss_2'] = scores['room_success'] * scores['found_success']
-        scores['ss_3'] = scores['oracle_success'] * scores['found_success']
-        '''
         scores['sspl'] = scores['spl'] * scores['found_success']
 
         scores['rgs'] = str(pred_objid) == str(gt_objid)
@@ -487,7 +385,6 @@ class ReverieObjectNavBatch(object):
         print('eval %d predictions' % (len(preds)))
         print(preds[0])
 
-
         metrics = defaultdict(list)
         for item in preds:
             instr_id = item['instr_id']
@@ -496,14 +393,7 @@ class ReverieObjectNavBatch(object):
             scan, gt_traj, gt_objid = self.gt_trajs[instr_id]
             pred_found = item['found']
             gt_found = item['gt_found']
-
-
             traj_scores = self._eval_item(scan, traj, pred_objid, gt_traj, gt_objid, pred_found, gt_found)
-
-            # record "success" in the result file
-            # let the visualization tool can get the success status 
-            item['success'] = traj_scores['success']
-
             for k, v in traj_scores.items():
                 metrics[k].append(v)
             metrics['instr_id'].append(instr_id)
@@ -515,15 +405,10 @@ class ReverieObjectNavBatch(object):
             'sr': np.mean(metrics['success']) * 100,
             'oracle_sr': np.mean(metrics['oracle_success']) * 100,
             'spl': np.mean(metrics['spl']) * 100,
-            'sspl': np.mean(metrics['sspl']) * 100,
             'rgs': np.mean(metrics['rgs']) * 100,
             'rgspl': np.mean(metrics['rgspl']) * 100,
+            'sspl': np.mean(metrics['sspl']) * 100,
             'found_sr': np.mean(metrics['found_success']) * 100,
-            'room_sr': np.mean(metrics['room_success']) * 100,
-            'room_spl': np.mean(metrics['room_spl']) * 100,
-            'coverage_rate': np.mean(metrics['coverage_rate']) * 100,
-            'explore_spl': np.mean(metrics['explore_spl']) * 100,
         }
         return avg_metrics, metrics
 
-

map_nav_src/reverie/main_nav_obj.py

@@ -69,9 +69,7 @@ def build_dataset(args, rank=0):
     val_env_names = [ 'val_seen', 'val_unseen']
 
     if args.submit:
-        include_test = input('Include test dataset? (y/n)')
+        val_env_names.append('test')
-        if include_test == 'y' or include_test == 'Y':
-            val_env_names.append('test')
         
     val_envs = {}
     for split in val_env_names:
@@ -138,7 +136,7 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
             '\nListener training starts, start iteration: %s' % str(start_iter), record_file
         )
 
-    best_val = {'val_unseen': {"spl": 0., "sr": 0., "room_sr": 0., "state":"", "sspl": 0., 'found_sr': 0., 'explore_spl': 0.}}
+    best_val = {'val_unseen': {"spl": 0., "sr": 0., "state":"", "sspl": 0., 'found_sr': 0.}}
 
     for idx in range(start_iter, start_iter+args.iters, args.log_every):
         listner.logs = defaultdict(list)
@@ -203,15 +201,11 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
 
                 # select model by spl
                 if env_name in best_val:
-                    if score_summary['explore_spl'] >= best_val[env_name]['explore_spl']:
+                    if score_summary['sspl'] >= best_val[env_name]['sspl']:
                         best_val[env_name]['spl'] = score_summary['spl']
                         best_val[env_name]['sspl'] = score_summary['sspl']
-                        best_val[env_name]['explore_spl'] = score_summary['explore_spl']
-                        best_val[env_name]['coverage_rate'] = score_summary['coverage_rate']
-                        best_val[env_name]['room_spl'] = score_summary['room_spl']
                         best_val[env_name]['sr'] = score_summary['sr']
                         best_val[env_name]['found_sr'] = score_summary['found_sr']
-                        best_val[env_name]['room_sr'] = score_summary['room_sr']
                         best_val[env_name]['state'] = 'Iter %d %s' % (iter, loss_str)
                         listner.save(idx, os.path.join(args.ckpt_dir, "best_%s" % (env_name)))
                 
@@ -245,14 +239,11 @@ def valid(args, train_env, val_envs, rank=-1):
         write_to_record_file(str(args) + '\n\n', record_file)
 
     for env_name, env in val_envs.items():
-        print(env_name)
         prefix = 'submit' if args.detailed_output is False else 'detail'
         output_file = os.path.join(args.pred_dir, "%s_%s_%s.json" % (
             prefix, env_name, args.fusion))
         if os.path.exists(output_file):
-            replace = input(f"{output_file} exists. Replace? (y/n): ")
+            continue
-            if replace != 'y' and replace != 'Y':
-                continue
         agent.logs = defaultdict(list)
         agent.env = env
 

map_nav_src/reverie/parser.py

@@ -71,7 +71,7 @@ def parse_args():
     parser.add_argument('--test', action='store_true', default=False)
     parser.add_argument("--submit", action='store_true', default=False)
     parser.add_argument('--no_backtrack', action='store_true', default=False)
-    parser.add_argument('--detailed_output', action='store_true', default=True)
+    parser.add_argument('--detailed_output', action='store_true', default=False)
 
     # Training Configurations
     parser.add_argument(

map_nav_src/scripts/run_reverie.sh

@@ -10,7 +10,7 @@ obj_ft_dim=768
 ngpus=1
 seed=0
 
-name=${train_alg}-${features}-new-reverie-all
+name=${train_alg}-${features}
 name=${name}-seed.${seed} #-${ngpus}gpus
 
 outdir=${DATA_ROOT}/REVERIE/exprs_map/finetune/${name}
@@ -59,14 +59,13 @@ flag="--root_dir ${DATA_ROOT}
       --gamma 0."
 
 # train
-CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag  \
+CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag  \
       --tokenizer bert \
-      --bert_ckpt_file '../datasets/REVERIE/exprs_map/pretrain/cmt-vitbase-mlm.mrc.sap.og-init.lxmert-aug.speaker/ckpts/model_step_100000.pt' \
+      --bert_ckpt_file 'put the pretrained model (see pretrain_src) here' \
       --eval_first
 
 # test
-echo /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen
+CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag  \
-CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag  \
       --tokenizer bert \
-      --resume_file /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen \
+      --resume_file ../datasets/REVERIE/trained_models/best_val_unseen \
-      --test --submit
+      --test --submit