diff --git a/map_nav_src/reverie/env.py b/map_nav_src/reverie/env.py index 7234ebc..c89c175 100644 --- a/map_nav_src/reverie/env.py +++ b/map_nav_src/reverie/env.py @@ -8,12 +8,17 @@ import random import networkx as nx from collections import defaultdict import copy +from glob import glob + import MatterSim from utils.data import load_nav_graphs, new_simulator from utils.data import angle_feature, get_all_point_angle_feature +with open('./node_region.json') as fp: + node_region = json.load(fp) + class EnvBatch(object): ''' A simple wrapper for a batch of MatterSim environments, @@ -360,6 +365,9 @@ class ReverieObjectNavBatch(object): path = sum(pred_path, []) assert gt_path[0] == path[0], 'Result trajectories should include the start position' + pred_stop_region = node_region[scan][path[-1]] + gt_stop_region = node_region[scan][gt_path[-1]] + scores['action_steps'] = len(pred_path) - 1 scores['trajectory_steps'] = len(path) - 1 scores['trajectory_lengths'] = np.sum([shortest_distances[a][b] for a, b in zip(path[:-1], path[1:])]) @@ -369,15 +377,29 @@ class ReverieObjectNavBatch(object): goal_viewpoints = set(self.obj2vps['%s_%s'%(scan, str(gt_objid))]) assert len(goal_viewpoints) > 0, '%s_%s'%(scan, str(gt_objid)) + scores['found_success'] = float(pred_found == gt_found) + scores['success'] = float(path[-1] in goal_viewpoints) + scores['room_success'] = float(pred_stop_region == gt_stop_region) + scores['oracle_success'] = float(any(x in goal_viewpoints for x in path)) + ''' if scores['success'] == 1.0: scores['found_success'] = float(pred_found == gt_found) else: scores['found_success'] = 0.0 + ''' - scores['oracle_success'] = float(any(x in goal_viewpoints for x in path)) + + scores['spl'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) - scores['sspl'] = scores['spl'] * scores['found_success'] + scores['sspl_1'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success'] + scores['sspl_2'] = scores['room_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success'] + scores['sspl_3'] = scores['oracle_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success'] + + scores['ss_1'] = scores['success'] * scores['found_success'] + scores['ss_2'] = scores['room_success'] * scores['found_success'] + scores['ss_3'] = scores['oracle_success'] * scores['found_success'] + # scores['sspl'] = scores['spl'] * scores['found_success'] scores['rgs'] = str(pred_objid) == str(gt_objid) scores['rgspl'] = scores['rgs'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) @@ -419,8 +441,14 @@ class ReverieObjectNavBatch(object): 'spl': np.mean(metrics['spl']) * 100, 'rgs': np.mean(metrics['rgs']) * 100, 'rgspl': np.mean(metrics['rgspl']) * 100, - 'sspl': np.mean(metrics['sspl']) * 100, + 'sspl_1': np.mean(metrics['sspl_1']) * 100, + 'sspl_2': np.mean(metrics['sspl_2']) * 100, + 'sspl_3': np.mean(metrics['sspl_3']) * 100, + 'ss_1': np.mean(metrics['ss_1']) * 100, + 'ss_2': np.mean(metrics['ss_2']) * 100, + 'ss_3': np.mean(metrics['ss_3']) * 100, 'found_sr': np.mean(metrics['found_success']) * 100, + 'room_sr': np.mean(metrics['room_success']) * 100, } return avg_metrics, metrics