snsd0805/Found-Not-Found
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

instrgen

Browse Source
This commit is contained in:
htsu 2024-01-15 09:42:55 +08:00
commit 9b465309b7
3 changed files with 3555 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3255
instruction_generation/instruction_editing.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

150
visualization/.ipynb_checkpoints/visualizer-checkpoint.ipynb Normal file
View File

@ -0,0 +1,150 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#1. load skybox image of the path[-3:]\n",
"#2. display instruction, adversarial instruction, (navgpt captions and objects)\n",
"\n",
"from PIL import Image\n",
"import json, os\n",
"\n",
"def load_json(fn):\n",
" with open(fn) as f:\n",
" ret = json.load(f)\n",
" return ret\n",
"\n",
"def dump_json(data, fn, force=False):\n",
" if not force:\n",
" assert not os.path.exists(fn)\n",
" with open(fn, 'w') as f:\n",
" json.dump(data, f)\n",
" \n",
"def concat_images(images):\n",
" #images = [Image.open(x) for x in ['Test1.jpg', 'Test2.jpg', 'Test3.jpg']]\n",
" widths, heights = zip(*(i.size for i in images))\n",
"\n",
" total_width = sum(widths)+(len(images)-1)*10\n",
" max_height = max(heights)\n",
"\n",
" new_im = Image.new('RGB', (total_width, max_height))\n",
"\n",
" x_offset = 0\n",
" for im in images:\n",
" new_im.paste(im, (x_offset,0))\n",
" x_offset += im.size[0]+10\n",
" return new_im"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')\n",
"reverie_val_unseen_fnf = load_json(\"..//fnf/reverie_val_unseen_fnf.json\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"fnf_pairs = []\n",
"for idx, r in enumerate(reverie_val_unseen_fnf):\n",
" if not r['found']:\n",
" fnf_pairs.append((reverie_val_unseen_fnf[idx-1], r))"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"idx = 1\n",
"fnf_pair = fnf_pairs[idx]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scan_id, path = fnf_pair[0]['scan'], fnf_pair[0]['path']\n",
"fn_template = '/work/ganymede9487/mp3d/unzipped/{}/matterport_skybox_images/{}_skybox{}_sami.jpg'\n",
"skybox_image_files = [fn_template.format(scan_id, path[-1], skybox_idx) for skybox_idx in range(6)]\n",
"skybox_images = []\n",
"for skybox_image_file in skybox_image_files:\n",
" skybox_images.append(Image.open(skybox_image_file))\n",
"\n",
"concat_images(skybox_images)\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n",
"Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls\n"
]
}
],
"source": [
"print(fnf_pair[0]['instruction'])\n",
"print(fnf_pair[1]['instruction'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#0 ok\n",
"#1 error 2 (not reasonable)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "minigpt4",
"language": "python",
"name": "minigpt4"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

150
visualization/visualizer.ipynb Normal file
View File

@ -0,0 +1,150 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#1. load skybox image of the path[-3:]\n",
"#2. display instruction, adversarial instruction, (navgpt captions and objects)\n",
"\n",
"from PIL import Image\n",
"import json, os\n",
"\n",
"def load_json(fn):\n",
" with open(fn) as f:\n",
" ret = json.load(f)\n",
" return ret\n",
"\n",
"def dump_json(data, fn, force=False):\n",
" if not force:\n",
" assert not os.path.exists(fn)\n",
" with open(fn, 'w') as f:\n",
" json.dump(data, f)\n",
" \n",
"def concat_images(images):\n",
" #images = [Image.open(x) for x in ['Test1.jpg', 'Test2.jpg', 'Test3.jpg']]\n",
" widths, heights = zip(*(i.size for i in images))\n",
"\n",
" total_width = sum(widths)+(len(images)-1)*10\n",
" max_height = max(heights)\n",
"\n",
" new_im = Image.new('RGB', (total_width, max_height))\n",
"\n",
" x_offset = 0\n",
" for im in images:\n",
" new_im.paste(im, (x_offset,0))\n",
" x_offset += im.size[0]+10\n",
" return new_im"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')\n",
"reverie_val_unseen_fnf = load_json(\"..//fnf/reverie_val_unseen_fnf.json\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"fnf_pairs = []\n",
"for idx, r in enumerate(reverie_val_unseen_fnf):\n",
" if not r['found']:\n",
" fnf_pairs.append((reverie_val_unseen_fnf[idx-1], r))"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"idx = 1\n",
"fnf_pair = fnf_pairs[idx]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scan_id, path = fnf_pair[0]['scan'], fnf_pair[0]['path']\n",
"fn_template = '/work/ganymede9487/mp3d/unzipped/{}/matterport_skybox_images/{}_skybox{}_sami.jpg'\n",
"skybox_image_files = [fn_template.format(scan_id, path[-1], skybox_idx) for skybox_idx in range(6)]\n",
"skybox_images = []\n",
"for skybox_image_file in skybox_image_files:\n",
" skybox_images.append(Image.open(skybox_image_file))\n",
"\n",
"concat_images(skybox_images)\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n",
"Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls\n"
]
}
],
"source": [
"print(fnf_pair[0]['instruction'])\n",
"print(fnf_pair[1]['instruction'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#0 ok\n",
"#1 error 2 (not reasonable)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "minigpt4",
"language": "python",
"name": "minigpt4"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
}
},
"nbformat": 4,
"nbformat_minor": 4
}