fix: evaluation metrics for new reverie (path)

it's a "new" new reverie, I only choose the front path & the back path >
1's instructions.

map_nav_src/reverie/agent_base.py

@@ -27,7 +27,13 @@ class BaseAgent(object):
     def get_results(self, detailed_output=False):
         output = []
         for k, v in self.results.items():
-            output.append({'instr_id': k, 'trajectory': v['path'], 'pred_objid': v['pred_objid'], 'found': v['found'], 'gt_found': v['gt_found']})
+            output.append({
+                'instr_id': k,
+                'trajectory': v['path'],
+                'pred_objid': v['pred_objid'],
+                'found': v['found'],
+                'gt_found': v['gt_found']
+            })
             if detailed_output:
                 output[-1]['details'] = v['details']
         return output

map_nav_src/reverie/agent_obj.py

@@ -416,13 +416,14 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     else:
                         og = None
 
+
                     # 如果有找到，og 會是 object id
                     # 如果是 not found，og 會是 -1
                     # 如果這個 viewpoint 看不到物件，og 會是 None
                     gmap.node_stop_scores[i_vp] = {
                         'stop': nav_probs[i, 0].data.item(),
                         'og': og,
-                        'og_details': {'objids': i_objids, 'logits': i_obj_logits[:len(i_objids)]},
+                        'og_details': {'objids': i_objids, 'logits': torch.cat([i_obj_logits[:len(i_objids)], i_obj_logits[[-1]] ], dim=0)},
                     }
                                         
             if train_ml is not None:
@@ -442,9 +443,9 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     )
                     ml_loss += self.criterion(nav_outs['local_logits'], local_nav_targets)  # local
                 # objec grounding 
-                obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
+                # obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
                 # print(t, obj_targets[6], obj_logits[6], obs[6]['obj_ids'], pano_inputs['view_lens'][i], obs[6]['gt_obj_id'])
-                og_loss += self.criterion(obj_logits, obj_targets)
+                # og_loss += self.criterion(obj_logits, obj_targets)
                 # print(F.cross_entropy(obj_logits, obj_targets, reduction='none'))
                 # print(t, 'og_loss', og_loss.item(), self.criterion(obj_logits, obj_targets).item())
                                                    
@@ -504,7 +505,7 @@ class GMapObjectNavAgent(Seq2SeqAgent):
                     if stop_node is not None and obs[i]['viewpoint'] != stop_node:
                         traj[i]['path'].append(gmaps[i].graph.path(obs[i]['viewpoint'], stop_node))
                     traj[i]['pred_objid'] = stop_score['og']
-                    if stop_score['og'] == -1:
+                    if stop_score['og'] == -1 or stop_score['og'] == None:
                         traj[i]['found'] = False
                     else:
                         traj[i]['found'] = True
@@ -531,11 +532,11 @@ class GMapObjectNavAgent(Seq2SeqAgent):
 
         if train_ml is not None:
             ml_loss = ml_loss * train_ml / batch_size
-            og_loss = og_loss * train_ml / batch_size
+            # og_loss = og_loss * train_ml / batch_size
             self.loss += ml_loss
-            self.loss += og_loss
+            # self.loss += og_loss
             self.logs['IL_loss'].append(ml_loss.item())
-            self.logs['OG_loss'].append(og_loss.item())
+            # self.logs['OG_loss'].append(og_loss.item())
 
         '''
         print("TRAJ:")

map_nav_src/reverie/data_utils.py

@@ -87,6 +87,7 @@ def construct_instrs(anno_dir, dataset, splits, tokenizer, max_instr_len=512):
                 new_item['objId'] = None
             new_item['instruction'] = instr
             new_item['instr_encoding'] = item['instr_encodings'][j][:max_instr_len]
+            new_item['path'] = item['path'][j]
             new_item['found'] = item['found'][j]
             del new_item['instructions']
             del new_item['instr_encodings']

map_nav_src/reverie/env.py

@@ -8,12 +8,22 @@ import random
 import networkx as nx
 from collections import defaultdict
 import copy
+from glob import glob
+
 
 import MatterSim
 
 from utils.data import load_nav_graphs, new_simulator
 from utils.data import angle_feature, get_all_point_angle_feature
 
+with open('./node_region.json') as fp:
+    node_region = json.load(fp)
+
+with open('region2objs.json') as fp:
+    region2objs = json.load(fp)
+
+with open('vp2objs.json') as fp:
+    vp2objs = json.load(fp)
 
 class EnvBatch(object):
     ''' A simple wrapper for a batch of MatterSim environments,
@@ -360,6 +370,10 @@ class ReverieObjectNavBatch(object):
         path = sum(pred_path, [])
         assert gt_path[0] == path[0], 'Result trajectories should include the start position'
 
+        pred_stop_region = node_region[scan][path[-1]]
+        gt_stop_region = node_region[scan][gt_path[-1]]
+
+
         scores['action_steps'] = len(pred_path) - 1
         scores['trajectory_steps'] = len(path) - 1
         scores['trajectory_lengths'] = np.sum([shortest_distances[a][b] for a, b in zip(path[:-1], path[1:])])
@@ -369,10 +383,98 @@ class ReverieObjectNavBatch(object):
         goal_viewpoints = set(self.obj2vps['%s_%s'%(scan, str(gt_objid))])
         assert len(goal_viewpoints) > 0, '%s_%s'%(scan, str(gt_objid))
 
-        scores['success'] = float(path[-1] in goal_viewpoints)
         scores['found_success'] = float(pred_found == gt_found) 
+
+        scores['success'] = float(path[-1] in goal_viewpoints)
+        scores['room_success'] = float(pred_stop_region == gt_stop_region)
         scores['oracle_success'] = float(any(x in goal_viewpoints for x in path))
+
+        gt_room_start_vp = None
+        gt_back_path = []
+        gt_front_path = []
+        for vp in gt_path[::-1]:
+            if node_region[scan][vp] == gt_stop_region and gt_front_path == []:
+                gt_back_path.append(vp)
+                gt_room_start_vp = vp
+            else:
+                gt_front_path.append(vp)
+
+        gt_front_path = gt_front_path[::-1]
+        gt_back_path = gt_back_path[::-1]
+
+        assert (gt_front_path + gt_back_path) == gt_path, "Front path & Back path error"
+
+        gt_front_path += [gt_room_start_vp]
+
+        '''
+        if scores['success'] == 1.0:
+            scores['found_success'] = float(pred_found == gt_found) 
+        else:
+            scores['found_success'] = 0.0
+        '''
+        gt_reach_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_front_path[:-1], gt_front_path[1:])])
+        gt_explore_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_back_path[:-1], gt_back_path[1:])])
+
+        if scores['room_success'] != 0.0:
+            # corse-grained
+
+            # get the reach_path & explore_path
+            room_start_vp = None
+            back_path = []
+            front_path = []
+            for vp in path[::-1]:
+                if node_region[scan][vp] == gt_stop_region and front_path == []:
+                    back_path.append(vp)
+                    room_start_vp = vp
+                else:
+                    front_path.append(vp)
+
+            front_path = front_path[::-1]
+            back_path = back_path[::-1]
+            assert (front_path + back_path) == path, "Front path & Back path error"
+
+            # front_path = ... room_start_vp
+            # back_path = room_start_vp ...
+            front_path += [room_start_vp]
+
+            reach_length = np.sum([shortest_distances[a][b] for a, b in zip(front_path[:-1], front_path[1:])]) if len(front_path) != 1 else 0.01
+            explore_length = np.sum([shortest_distances[a][b] for a, b in zip(back_path[:-1], back_path[1:])]) if len(back_path) != 1 else 0.01
+
+            
+            scores['room_spl'] = scores['room_success'] * gt_reach_length / max(reach_length, gt_reach_length, 0.01)
+            if scores['found_success'] != 0.0:
+                # fine-grained score
+                # p is converage rate
+                if gt_found:
+                    p = 1.0
+                else:
+                    explore_objs = set()
+                    for vp in back_path:
+                        explore_objs.update(vp2objs[vp])
+                    p = len(explore_objs) / len(region2objs[scan][gt_stop_region])
+                scores['coverage_rate'] = p
+                scores['explore_spl'] = scores['room_success'] * scores['found_success'] * gt_explore_length / max(gt_explore_length, explore_length, 0.01) * p
+            else:
+                scores['coverage_rate'] = 0
+                scores['explore_spl'] = 0
+        else:
+            scores['room_spl'] = 0.0
+            scores['coverage_rate'] = 0
+            scores['explore_spl'] = 0
+
+
+
+        
         scores['spl'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01)
+        '''
+        scores['sspl_1'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
+        scores['sspl_2'] = scores['room_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
+        scores['sspl_3'] = scores['oracle_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
+
+        scores['ss_1'] = scores['success'] * scores['found_success']
+        scores['ss_2'] = scores['room_success'] * scores['found_success']
+        scores['ss_3'] = scores['oracle_success'] * scores['found_success']
+        '''
         scores['sspl'] = scores['spl'] * scores['found_success']
 
         scores['rgs'] = str(pred_objid) == str(gt_objid)
@@ -385,6 +487,7 @@ class ReverieObjectNavBatch(object):
         print('eval %d predictions' % (len(preds)))
         print(preds[0])
 
+
         metrics = defaultdict(list)
         for item in preds:
             instr_id = item['instr_id']
@@ -393,7 +496,14 @@ class ReverieObjectNavBatch(object):
             scan, gt_traj, gt_objid = self.gt_trajs[instr_id]
             pred_found = item['found']
             gt_found = item['gt_found']
+
+
             traj_scores = self._eval_item(scan, traj, pred_objid, gt_traj, gt_objid, pred_found, gt_found)
+
+            # record "success" in the result file
+            # let the visualization tool can get the success status 
+            item['success'] = traj_scores['success']
+
             for k, v in traj_scores.items():
                 metrics[k].append(v)
             metrics['instr_id'].append(instr_id)
@@ -405,10 +515,15 @@ class ReverieObjectNavBatch(object):
             'sr': np.mean(metrics['success']) * 100,
             'oracle_sr': np.mean(metrics['oracle_success']) * 100,
             'spl': np.mean(metrics['spl']) * 100,
+            'sspl': np.mean(metrics['sspl']) * 100,
             'rgs': np.mean(metrics['rgs']) * 100,
             'rgspl': np.mean(metrics['rgspl']) * 100,
-            'sspl': np.mean(metrics['sspl']) * 100,
             'found_sr': np.mean(metrics['found_success']) * 100,
+            'room_sr': np.mean(metrics['room_success']) * 100,
+            'room_spl': np.mean(metrics['room_spl']) * 100,
+            'coverage_rate': np.mean(metrics['coverage_rate']) * 100,
+            'explore_spl': np.mean(metrics['explore_spl']) * 100,
         }
         return avg_metrics, metrics
 
+

map_nav_src/reverie/main_nav_obj.py

@@ -69,7 +69,9 @@ def build_dataset(args, rank=0):
     val_env_names = [ 'val_seen', 'val_unseen']
 
     if args.submit:
-        val_env_names.append('test')
+        include_test = input('Include test dataset? (y/n)')
+        if include_test == 'y' or include_test == 'Y':
+            val_env_names.append('test')
         
     val_envs = {}
     for split in val_env_names:
@@ -136,7 +138,7 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
             '\nListener training starts, start iteration: %s' % str(start_iter), record_file
         )
 
-    best_val = {'val_unseen': {"spl": 0., "sr": 0., "state":"", "sspl": 0., 'found_sr': 0.}}
+    best_val = {'val_unseen': {"spl": 0., "sr": 0., "room_sr": 0., "state":"", "sspl": 0., 'found_sr': 0., 'explore_spl': 0.}}
 
     for idx in range(start_iter, start_iter+args.iters, args.log_every):
         listner.logs = defaultdict(list)
@@ -201,11 +203,15 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
 
                 # select model by spl
                 if env_name in best_val:
-                    if score_summary['sspl'] >= best_val[env_name]['sspl']:
+                    if score_summary['explore_spl'] >= best_val[env_name]['explore_spl']:
                         best_val[env_name]['spl'] = score_summary['spl']
                         best_val[env_name]['sspl'] = score_summary['sspl']
+                        best_val[env_name]['explore_spl'] = score_summary['explore_spl']
+                        best_val[env_name]['coverage_rate'] = score_summary['coverage_rate']
+                        best_val[env_name]['room_spl'] = score_summary['room_spl']
                         best_val[env_name]['sr'] = score_summary['sr']
                         best_val[env_name]['found_sr'] = score_summary['found_sr']
+                        best_val[env_name]['room_sr'] = score_summary['room_sr']
                         best_val[env_name]['state'] = 'Iter %d %s' % (iter, loss_str)
                         listner.save(idx, os.path.join(args.ckpt_dir, "best_%s" % (env_name)))
                 
@@ -239,11 +245,14 @@ def valid(args, train_env, val_envs, rank=-1):
         write_to_record_file(str(args) + '\n\n', record_file)
 
     for env_name, env in val_envs.items():
+        print(env_name)
         prefix = 'submit' if args.detailed_output is False else 'detail'
         output_file = os.path.join(args.pred_dir, "%s_%s_%s.json" % (
             prefix, env_name, args.fusion))
         if os.path.exists(output_file):
-            continue
+            replace = input(f"{output_file} exists. Replace? (y/n): ")
+            if replace != 'y' and replace != 'Y':
+                continue
         agent.logs = defaultdict(list)
         agent.env = env
 

map_nav_src/reverie/parser.py

@@ -71,7 +71,7 @@ def parse_args():
     parser.add_argument('--test', action='store_true', default=False)
     parser.add_argument("--submit", action='store_true', default=False)
     parser.add_argument('--no_backtrack', action='store_true', default=False)
-    parser.add_argument('--detailed_output', action='store_true', default=False)
+    parser.add_argument('--detailed_output', action='store_true', default=True)
 
     # Training Configurations
     parser.add_argument(

map_nav_src/scripts/run_reverie.sh

@@ -10,7 +10,7 @@ obj_ft_dim=768
 ngpus=1
 seed=0
 
-name=${train_alg}-${features}
+name=${train_alg}-${features}-new-reverie-all
 name=${name}-seed.${seed} #-${ngpus}gpus
 
 outdir=${DATA_ROOT}/REVERIE/exprs_map/finetune/${name}
@@ -59,13 +59,14 @@ flag="--root_dir ${DATA_ROOT}
       --gamma 0."
 
 # train
-CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag  \
+CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag  \
       --tokenizer bert \
-      --bert_ckpt_file 'put the pretrained model (see pretrain_src) here' \
+      --bert_ckpt_file '../datasets/REVERIE/exprs_map/pretrain/cmt-vitbase-mlm.mrc.sap.og-init.lxmert-aug.speaker/ckpts/model_step_100000.pt' \
       --eval_first
 
 # test
-CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag  \
+echo /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen
+CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag  \
       --tokenizer bert \
-      --resume_file ../datasets/REVERIE/trained_models/best_val_unseen \
+      --resume_file /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen \
       --test --submit