Compare commits
No commits in common. "main" and "adversarial" have entirely different histories.
main
...
adversaria
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -27,13 +27,7 @@ class BaseAgent(object):
|
||||
def get_results(self, detailed_output=False):
|
||||
output = []
|
||||
for k, v in self.results.items():
|
||||
output.append({
|
||||
'instr_id': k,
|
||||
'trajectory': v['path'],
|
||||
'pred_objid': v['pred_objid'],
|
||||
'found': v['found'],
|
||||
'gt_found': v['gt_found']
|
||||
})
|
||||
output.append({'instr_id': k, 'trajectory': v['path'], 'pred_objid': v['pred_objid'], 'found': v['found'], 'gt_found': v['gt_found']})
|
||||
if detailed_output:
|
||||
output[-1]['details'] = v['details']
|
||||
return output
|
||||
|
||||
@ -416,14 +416,13 @@ class GMapObjectNavAgent(Seq2SeqAgent):
|
||||
else:
|
||||
og = None
|
||||
|
||||
|
||||
# 如果有找到，og 會是 object id
|
||||
# 如果是 not found，og 會是 -1
|
||||
# 如果這個 viewpoint 看不到物件，og 會是 None
|
||||
gmap.node_stop_scores[i_vp] = {
|
||||
'stop': nav_probs[i, 0].data.item(),
|
||||
'og': og,
|
||||
'og_details': {'objids': i_objids, 'logits': torch.cat([i_obj_logits[:len(i_objids)], i_obj_logits[[-1]] ], dim=0)},
|
||||
'og_details': {'objids': i_objids, 'logits': i_obj_logits[:len(i_objids)]},
|
||||
}
|
||||
|
||||
if train_ml is not None:
|
||||
@ -443,9 +442,9 @@ class GMapObjectNavAgent(Seq2SeqAgent):
|
||||
)
|
||||
ml_loss += self.criterion(nav_outs['local_logits'], local_nav_targets) # local
|
||||
# objec grounding
|
||||
# obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
|
||||
obj_targets = self._teacher_object(obs, ended, pano_inputs['view_lens'], obj_logits)
|
||||
# print(t, obj_targets[6], obj_logits[6], obs[6]['obj_ids'], pano_inputs['view_lens'][i], obs[6]['gt_obj_id'])
|
||||
# og_loss += self.criterion(obj_logits, obj_targets)
|
||||
og_loss += self.criterion(obj_logits, obj_targets)
|
||||
# print(F.cross_entropy(obj_logits, obj_targets, reduction='none'))
|
||||
# print(t, 'og_loss', og_loss.item(), self.criterion(obj_logits, obj_targets).item())
|
||||
|
||||
@ -505,7 +504,7 @@ class GMapObjectNavAgent(Seq2SeqAgent):
|
||||
if stop_node is not None and obs[i]['viewpoint'] != stop_node:
|
||||
traj[i]['path'].append(gmaps[i].graph.path(obs[i]['viewpoint'], stop_node))
|
||||
traj[i]['pred_objid'] = stop_score['og']
|
||||
if stop_score['og'] == -1 or stop_score['og'] == None:
|
||||
if stop_score['og'] == -1:
|
||||
traj[i]['found'] = False
|
||||
else:
|
||||
traj[i]['found'] = True
|
||||
@ -532,11 +531,11 @@ class GMapObjectNavAgent(Seq2SeqAgent):
|
||||
|
||||
if train_ml is not None:
|
||||
ml_loss = ml_loss * train_ml / batch_size
|
||||
# og_loss = og_loss * train_ml / batch_size
|
||||
og_loss = og_loss * train_ml / batch_size
|
||||
self.loss += ml_loss
|
||||
# self.loss += og_loss
|
||||
self.loss += og_loss
|
||||
self.logs['IL_loss'].append(ml_loss.item())
|
||||
# self.logs['OG_loss'].append(og_loss.item())
|
||||
self.logs['OG_loss'].append(og_loss.item())
|
||||
|
||||
'''
|
||||
print("TRAJ:")
|
||||
|
||||
@ -87,7 +87,6 @@ def construct_instrs(anno_dir, dataset, splits, tokenizer, max_instr_len=512):
|
||||
new_item['objId'] = None
|
||||
new_item['instruction'] = instr
|
||||
new_item['instr_encoding'] = item['instr_encodings'][j][:max_instr_len]
|
||||
new_item['path'] = item['path'][j]
|
||||
new_item['found'] = item['found'][j]
|
||||
del new_item['instructions']
|
||||
del new_item['instr_encodings']
|
||||
|
||||
@ -8,22 +8,12 @@ import random
|
||||
import networkx as nx
|
||||
from collections import defaultdict
|
||||
import copy
|
||||
from glob import glob
|
||||
|
||||
|
||||
import MatterSim
|
||||
|
||||
from utils.data import load_nav_graphs, new_simulator
|
||||
from utils.data import angle_feature, get_all_point_angle_feature
|
||||
|
||||
with open('./node_region.json') as fp:
|
||||
node_region = json.load(fp)
|
||||
|
||||
with open('region2objs.json') as fp:
|
||||
region2objs = json.load(fp)
|
||||
|
||||
with open('vp2objs.json') as fp:
|
||||
vp2objs = json.load(fp)
|
||||
|
||||
class EnvBatch(object):
|
||||
''' A simple wrapper for a batch of MatterSim environments,
|
||||
@ -370,10 +360,6 @@ class ReverieObjectNavBatch(object):
|
||||
path = sum(pred_path, [])
|
||||
assert gt_path[0] == path[0], 'Result trajectories should include the start position'
|
||||
|
||||
pred_stop_region = node_region[scan][path[-1]]
|
||||
gt_stop_region = node_region[scan][gt_path[-1]]
|
||||
|
||||
|
||||
scores['action_steps'] = len(pred_path) - 1
|
||||
scores['trajectory_steps'] = len(path) - 1
|
||||
scores['trajectory_lengths'] = np.sum([shortest_distances[a][b] for a, b in zip(path[:-1], path[1:])])
|
||||
@ -383,98 +369,10 @@ class ReverieObjectNavBatch(object):
|
||||
goal_viewpoints = set(self.obj2vps['%s_%s'%(scan, str(gt_objid))])
|
||||
assert len(goal_viewpoints) > 0, '%s_%s'%(scan, str(gt_objid))
|
||||
|
||||
scores['found_success'] = float(pred_found == gt_found)
|
||||
|
||||
scores['success'] = float(path[-1] in goal_viewpoints)
|
||||
scores['room_success'] = float(pred_stop_region == gt_stop_region)
|
||||
scores['found_success'] = float(pred_found == gt_found)
|
||||
scores['oracle_success'] = float(any(x in goal_viewpoints for x in path))
|
||||
|
||||
gt_room_start_vp = None
|
||||
gt_back_path = []
|
||||
gt_front_path = []
|
||||
for vp in gt_path[::-1]:
|
||||
if node_region[scan][vp] == gt_stop_region and gt_front_path == []:
|
||||
gt_back_path.append(vp)
|
||||
gt_room_start_vp = vp
|
||||
else:
|
||||
gt_front_path.append(vp)
|
||||
|
||||
gt_front_path = gt_front_path[::-1]
|
||||
gt_back_path = gt_back_path[::-1]
|
||||
|
||||
assert (gt_front_path + gt_back_path) == gt_path, "Front path & Back path error"
|
||||
|
||||
gt_front_path += [gt_room_start_vp]
|
||||
|
||||
'''
|
||||
if scores['success'] == 1.0:
|
||||
scores['found_success'] = float(pred_found == gt_found)
|
||||
else:
|
||||
scores['found_success'] = 0.0
|
||||
'''
|
||||
gt_reach_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_front_path[:-1], gt_front_path[1:])])
|
||||
gt_explore_length = np.sum([shortest_distances[a][b] for a, b in zip(gt_back_path[:-1], gt_back_path[1:])])
|
||||
|
||||
if scores['room_success'] != 0.0:
|
||||
# corse-grained
|
||||
|
||||
# get the reach_path & explore_path
|
||||
room_start_vp = None
|
||||
back_path = []
|
||||
front_path = []
|
||||
for vp in path[::-1]:
|
||||
if node_region[scan][vp] == gt_stop_region and front_path == []:
|
||||
back_path.append(vp)
|
||||
room_start_vp = vp
|
||||
else:
|
||||
front_path.append(vp)
|
||||
|
||||
front_path = front_path[::-1]
|
||||
back_path = back_path[::-1]
|
||||
assert (front_path + back_path) == path, "Front path & Back path error"
|
||||
|
||||
# front_path = ... room_start_vp
|
||||
# back_path = room_start_vp ...
|
||||
front_path += [room_start_vp]
|
||||
|
||||
reach_length = np.sum([shortest_distances[a][b] for a, b in zip(front_path[:-1], front_path[1:])]) if len(front_path) != 1 else 0.01
|
||||
explore_length = np.sum([shortest_distances[a][b] for a, b in zip(back_path[:-1], back_path[1:])]) if len(back_path) != 1 else 0.01
|
||||
|
||||
|
||||
scores['room_spl'] = scores['room_success'] * gt_reach_length / max(reach_length, gt_reach_length, 0.01)
|
||||
if scores['found_success'] != 0.0:
|
||||
# fine-grained score
|
||||
# p is converage rate
|
||||
if gt_found:
|
||||
p = 1.0
|
||||
else:
|
||||
explore_objs = set()
|
||||
for vp in back_path:
|
||||
explore_objs.update(vp2objs[vp])
|
||||
p = len(explore_objs) / len(region2objs[scan][gt_stop_region])
|
||||
scores['coverage_rate'] = p
|
||||
scores['explore_spl'] = scores['room_success'] * scores['found_success'] * gt_explore_length / max(gt_explore_length, explore_length, 0.01) * p
|
||||
else:
|
||||
scores['coverage_rate'] = 0
|
||||
scores['explore_spl'] = 0
|
||||
else:
|
||||
scores['room_spl'] = 0.0
|
||||
scores['coverage_rate'] = 0
|
||||
scores['explore_spl'] = 0
|
||||
|
||||
|
||||
|
||||
|
||||
scores['spl'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01)
|
||||
'''
|
||||
scores['sspl_1'] = scores['success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
|
||||
scores['sspl_2'] = scores['room_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
|
||||
scores['sspl_3'] = scores['oracle_success'] * gt_lengths / max(scores['trajectory_lengths'], gt_lengths, 0.01) * scores['found_success']
|
||||
|
||||
scores['ss_1'] = scores['success'] * scores['found_success']
|
||||
scores['ss_2'] = scores['room_success'] * scores['found_success']
|
||||
scores['ss_3'] = scores['oracle_success'] * scores['found_success']
|
||||
'''
|
||||
scores['sspl'] = scores['spl'] * scores['found_success']
|
||||
|
||||
scores['rgs'] = str(pred_objid) == str(gt_objid)
|
||||
@ -487,7 +385,6 @@ class ReverieObjectNavBatch(object):
|
||||
print('eval %d predictions' % (len(preds)))
|
||||
print(preds[0])
|
||||
|
||||
|
||||
metrics = defaultdict(list)
|
||||
for item in preds:
|
||||
instr_id = item['instr_id']
|
||||
@ -496,14 +393,7 @@ class ReverieObjectNavBatch(object):
|
||||
scan, gt_traj, gt_objid = self.gt_trajs[instr_id]
|
||||
pred_found = item['found']
|
||||
gt_found = item['gt_found']
|
||||
|
||||
|
||||
traj_scores = self._eval_item(scan, traj, pred_objid, gt_traj, gt_objid, pred_found, gt_found)
|
||||
|
||||
# record "success" in the result file
|
||||
# let the visualization tool can get the success status
|
||||
item['success'] = traj_scores['success']
|
||||
|
||||
for k, v in traj_scores.items():
|
||||
metrics[k].append(v)
|
||||
metrics['instr_id'].append(instr_id)
|
||||
@ -515,15 +405,10 @@ class ReverieObjectNavBatch(object):
|
||||
'sr': np.mean(metrics['success']) * 100,
|
||||
'oracle_sr': np.mean(metrics['oracle_success']) * 100,
|
||||
'spl': np.mean(metrics['spl']) * 100,
|
||||
'sspl': np.mean(metrics['sspl']) * 100,
|
||||
'rgs': np.mean(metrics['rgs']) * 100,
|
||||
'rgspl': np.mean(metrics['rgspl']) * 100,
|
||||
'sspl': np.mean(metrics['sspl']) * 100,
|
||||
'found_sr': np.mean(metrics['found_success']) * 100,
|
||||
'room_sr': np.mean(metrics['room_success']) * 100,
|
||||
'room_spl': np.mean(metrics['room_spl']) * 100,
|
||||
'coverage_rate': np.mean(metrics['coverage_rate']) * 100,
|
||||
'explore_spl': np.mean(metrics['explore_spl']) * 100,
|
||||
}
|
||||
return avg_metrics, metrics
|
||||
|
||||
|
||||
|
||||
@ -69,9 +69,7 @@ def build_dataset(args, rank=0):
|
||||
val_env_names = [ 'val_seen', 'val_unseen']
|
||||
|
||||
if args.submit:
|
||||
include_test = input('Include test dataset? (y/n)')
|
||||
if include_test == 'y' or include_test == 'Y':
|
||||
val_env_names.append('test')
|
||||
val_env_names.append('test')
|
||||
|
||||
val_envs = {}
|
||||
for split in val_env_names:
|
||||
@ -138,7 +136,7 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
|
||||
'\nListener training starts, start iteration: %s' % str(start_iter), record_file
|
||||
)
|
||||
|
||||
best_val = {'val_unseen': {"spl": 0., "sr": 0., "room_sr": 0., "state":"", "sspl": 0., 'found_sr': 0., 'explore_spl': 0.}}
|
||||
best_val = {'val_unseen': {"spl": 0., "sr": 0., "state":"", "sspl": 0., 'found_sr': 0.}}
|
||||
|
||||
for idx in range(start_iter, start_iter+args.iters, args.log_every):
|
||||
listner.logs = defaultdict(list)
|
||||
@ -203,15 +201,11 @@ def train(args, train_env, val_envs, aug_env=None, rank=-1):
|
||||
|
||||
# select model by spl
|
||||
if env_name in best_val:
|
||||
if score_summary['explore_spl'] >= best_val[env_name]['explore_spl']:
|
||||
if score_summary['sspl'] >= best_val[env_name]['sspl']:
|
||||
best_val[env_name]['spl'] = score_summary['spl']
|
||||
best_val[env_name]['sspl'] = score_summary['sspl']
|
||||
best_val[env_name]['explore_spl'] = score_summary['explore_spl']
|
||||
best_val[env_name]['coverage_rate'] = score_summary['coverage_rate']
|
||||
best_val[env_name]['room_spl'] = score_summary['room_spl']
|
||||
best_val[env_name]['sr'] = score_summary['sr']
|
||||
best_val[env_name]['found_sr'] = score_summary['found_sr']
|
||||
best_val[env_name]['room_sr'] = score_summary['room_sr']
|
||||
best_val[env_name]['state'] = 'Iter %d %s' % (iter, loss_str)
|
||||
listner.save(idx, os.path.join(args.ckpt_dir, "best_%s" % (env_name)))
|
||||
|
||||
@ -245,14 +239,11 @@ def valid(args, train_env, val_envs, rank=-1):
|
||||
write_to_record_file(str(args) + '\n\n', record_file)
|
||||
|
||||
for env_name, env in val_envs.items():
|
||||
print(env_name)
|
||||
prefix = 'submit' if args.detailed_output is False else 'detail'
|
||||
output_file = os.path.join(args.pred_dir, "%s_%s_%s.json" % (
|
||||
prefix, env_name, args.fusion))
|
||||
if os.path.exists(output_file):
|
||||
replace = input(f"{output_file} exists. Replace? (y/n): ")
|
||||
if replace != 'y' and replace != 'Y':
|
||||
continue
|
||||
continue
|
||||
agent.logs = defaultdict(list)
|
||||
agent.env = env
|
||||
|
||||
|
||||
@ -71,7 +71,7 @@ def parse_args():
|
||||
parser.add_argument('--test', action='store_true', default=False)
|
||||
parser.add_argument("--submit", action='store_true', default=False)
|
||||
parser.add_argument('--no_backtrack', action='store_true', default=False)
|
||||
parser.add_argument('--detailed_output', action='store_true', default=True)
|
||||
parser.add_argument('--detailed_output', action='store_true', default=False)
|
||||
|
||||
# Training Configurations
|
||||
parser.add_argument(
|
||||
|
||||
@ -10,7 +10,7 @@ obj_ft_dim=768
|
||||
ngpus=1
|
||||
seed=0
|
||||
|
||||
name=${train_alg}-${features}-new-reverie-all
|
||||
name=${train_alg}-${features}
|
||||
name=${name}-seed.${seed} #-${ngpus}gpus
|
||||
|
||||
outdir=${DATA_ROOT}/REVERIE/exprs_map/finetune/${name}
|
||||
@ -59,14 +59,13 @@ flag="--root_dir ${DATA_ROOT}
|
||||
--gamma 0."
|
||||
|
||||
# train
|
||||
CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag \
|
||||
CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag \
|
||||
--tokenizer bert \
|
||||
--bert_ckpt_file '../datasets/REVERIE/exprs_map/pretrain/cmt-vitbase-mlm.mrc.sap.og-init.lxmert-aug.speaker/ckpts/model_step_100000.pt' \
|
||||
--bert_ckpt_file 'put the pretrained model (see pretrain_src) here' \
|
||||
--eval_first
|
||||
|
||||
# test
|
||||
echo /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen
|
||||
CUDA_VISIBLE_DEVICES='0' python3 reverie/main_nav_obj.py $flag \
|
||||
CUDA_VISIBLE_DEVICES='0' python reverie/main_nav_obj.py $flag \
|
||||
--tokenizer bert \
|
||||
--resume_file /root/mount/Matterport3DSimulator/VLN-DUET/datasets/REVERIE/exprs_map/finetune/${name}/ckpts/best_val_unseen \
|
||||
--test --submit
|
||||
--resume_file ../datasets/REVERIE/trained_models/best_val_unseen \
|
||||
--test --submit
|
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue
Block a user