151 lines
3.8 KiB
Plaintext
151 lines
3.8 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#1. load skybox image of the path[-3:]\n",
|
|
"#2. display instruction, adversarial instruction, (navgpt captions and objects)\n",
|
|
"\n",
|
|
"from PIL import Image\n",
|
|
"import json, os\n",
|
|
"\n",
|
|
"def load_json(fn):\n",
|
|
" with open(fn) as f:\n",
|
|
" ret = json.load(f)\n",
|
|
" return ret\n",
|
|
"\n",
|
|
"def dump_json(data, fn, force=False):\n",
|
|
" if not force:\n",
|
|
" assert not os.path.exists(fn)\n",
|
|
" with open(fn, 'w') as f:\n",
|
|
" json.dump(data, f)\n",
|
|
" \n",
|
|
"def concat_images(images):\n",
|
|
" #images = [Image.open(x) for x in ['Test1.jpg', 'Test2.jpg', 'Test3.jpg']]\n",
|
|
" widths, heights = zip(*(i.size for i in images))\n",
|
|
"\n",
|
|
" total_width = sum(widths)+(len(images)-1)*10\n",
|
|
" max_height = max(heights)\n",
|
|
"\n",
|
|
" new_im = Image.new('RGB', (total_width, max_height))\n",
|
|
"\n",
|
|
" x_offset = 0\n",
|
|
" for im in images:\n",
|
|
" new_im.paste(im, (x_offset,0))\n",
|
|
" x_offset += im.size[0]+10\n",
|
|
" return new_im"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')\n",
|
|
"reverie_val_unseen_fnf = load_json(\"..//fnf/reverie_val_unseen_fnf.json\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fnf_pairs = []\n",
|
|
"for idx, r in enumerate(reverie_val_unseen_fnf):\n",
|
|
" if not r['found']:\n",
|
|
" fnf_pairs.append((reverie_val_unseen_fnf[idx-1], r))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 25,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"idx = 1\n",
|
|
"fnf_pair = fnf_pairs[idx]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"scan_id, path = fnf_pair[0]['scan'], fnf_pair[0]['path']\n",
|
|
"fn_template = '/work/ganymede9487/mp3d/unzipped/{}/matterport_skybox_images/{}_skybox{}_sami.jpg'\n",
|
|
"skybox_image_files = [fn_template.format(scan_id, path[-1], skybox_idx) for skybox_idx in range(6)]\n",
|
|
"skybox_images = []\n",
|
|
"for skybox_image_file in skybox_image_files:\n",
|
|
" skybox_images.append(Image.open(skybox_image_file))\n",
|
|
"\n",
|
|
"concat_images(skybox_images)\n",
|
|
"\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 27,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n",
|
|
"Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print(fnf_pair[0]['instruction'])\n",
|
|
"print(fnf_pair[1]['instruction'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#0 ok\n",
|
|
"#1 error 2 (not reasonable)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "minigpt4",
|
|
"language": "python",
|
|
"name": "minigpt4"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.9.18"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|