From 9b465309b711d83da8517f36099fbd8c629991d8 Mon Sep 17 00:00:00 2001 From: htsu Date: Mon, 15 Jan 2024 09:42:55 +0800 Subject: [PATCH] instrgen --- .../instruction_editing.ipynb | 3255 +++++++++++++++++ .../visualizer-checkpoint.ipynb | 150 + visualization/visualizer.ipynb | 150 + 3 files changed, 3555 insertions(+) create mode 100644 instruction_generation/instruction_editing.ipynb create mode 100644 visualization/.ipynb_checkpoints/visualizer-checkpoint.ipynb create mode 100644 visualization/visualizer.ipynb diff --git a/instruction_generation/instruction_editing.ipynb b/instruction_generation/instruction_editing.ipynb new file mode 100644 index 0000000..e0d6280 --- /dev/null +++ b/instruction_generation/instruction_editing.ipynb @@ -0,0 +1,3255 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import json, os\n", + "\n", + "def load_json(fn):\n", + " with open(fn) as f:\n", + " ret = json.load(f)\n", + " return ret\n", + "\n", + "def dump_json(data, fn, force=False):\n", + " if not force:\n", + " assert not os.path.exists(fn)\n", + " with open(fn, 'w') as f:\n", + " json.dump(data, f)\n", + " \n", + "reverie_train = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_train_first_ins.json')\n", + "reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#one-time code: process reverie_train\n", + "'''\n", + "reverie_train = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_train.json')\n", + "for r in reverie_train:\n", + " r['instruction'] = r['instructions'][0]\n", + "dump_json(reverie_train, '../REVERIE/tasks/REVERIE/data/REVERIE_train_first_ins.json')\n", + "'''\n", + "1" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "'''\n", + "reverie_idx_scan_map = {}\n", + "for r in reverie_train+reverie_val_unseen:\n", + " if r['instruction'] in reverie_instr_scan_map:\n", + " assert r['scan'] == reverie_instr_scan_map[r['instruction']]\n", + " else:\n", + " reverie_instr_scan_map[r['instruction']] = r['scan']\n", + "'''\n", + "#archived\n", + "1" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'distance': 13.42,\n", + " 'ix': 1205,\n", + " 'scan': '759xd9YjKW5',\n", + " 'id': '5483_138',\n", + " 'instructions_l': ['Turn hard left and walk towards the kitchen area. Go towards the door to the right. Walk across the room past he carpet. Walk through the doorway right of the mirror and wait by the chairs. ',\n", + " 'Walk through the doorway into the kitchen. Turn right and then turn immediately left through the white door. Walk across the large run and past the mirror on the wall. Stop between the white armchair and the large wooden case. ',\n", + " 'Exit workout room to kitchen, make a right, then a left to doorway, walk toward living area, and wait at entrance. '],\n", + " 'path_id': 5483,\n", + " 'objId': 138,\n", + " 'path': ['f5f240dba39b4c19a1354c89c56d7a59',\n", + " 'd3bbb56e888d4b659563d6e135406870',\n", + " 'd16ee55e1e434c8abc07c79b08204bf1',\n", + " '87449aea807942e0998e230c344f0e42',\n", + " '5c8f1d2de7c44d4fb95fa752c5fdce24',\n", + " '6e1e523c608942daae201e0edb7cdde0',\n", + " '217b34f2c019478ea2b734c13e16b62b'],\n", + " 'heading': 5.471,\n", + " 'instructions': ['Go to the living room and sit down in the armchair',\n", + " 'Go to the living room on the first floor and bring the armchair near the wardrobe into the library',\n", + " 'Go to the living room and sit in the arm chair closest to the entrance'],\n", + " 'instruction': 'Go to the living room and sit down in the armchair'}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append(\"/home/ganymede9487/llama/llama\")\n", + "from gpt_utils import GPT\n", + "\n", + "api_key = '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'\n", + "gpt = GPT(\"./instruction_parsing.json\", api_key)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#one-time code: load instruction_parsing_lagacy.json to instruction_parsing.json\n", + "#change query_name from instr_id (duplicate) to reverie index e.g. train_13_0\n", + "'''\n", + "import pysondb\n", + "old_responses = pysondb.db.getDb('./instruction_parsing_lagacy.json').getAll()\n", + "\n", + "for idx, (reverie_r, old_res_r) in enumerate(zip(reverie_val_unseen, old_responses)):\n", + " assert reverie_r['instruction'] in old_res_r['prompt']\n", + " assert reverie_r['instr_id'] == old_res_r['query_name'].split(\"__\")[-1]\n", + " query_name = f\"reverie__val_unseen__{idx}__0\"\n", + " gpt.db.add({'query_name':query_name, 'prompt':old_res_r['prompt'], 'response':old_res_r['response']})\n", + "'''\n", + "1" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#reverie[-1]\n", + "#inputs = {\n", + " #'instruction': 'In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam'\n", + "#}\n", + "inputs = 'In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam'\n", + "outputs = {\n", + " 'room': 'kitchen',\n", + " 'goal': 'ceiling beam',\n", + " 'goal_relations':[\n", + " {'relation': 'above', 'reference': 'brown shelves'},\n", + " {'relation': 'above', 'reference': 'basket'},\n", + " {'relation': 'on the top', 'reference': 'shelf'},\n", + " ]\n", + "}\n", + "\n", + "template = f\"\"\"Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "{inputs}\n", + "outputs:\n", + "{json.dumps(outputs, indent=4)}\n", + "Now it is your turn:\n", + "inputs: \n", + "___inputs___\n", + "outputs:\n", + "\"\"\"\n", + "#print(prompt)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-03 18:17:34.821782 0 4150\n", + "reverie__train__0__0: Response already existed\n", + "reverie__train__1__0: Response already existed\n", + "reverie__train__2__0: Response already existed\n", + "reverie__train__3__0: Response already existed\n", + "reverie__train__4__0: Response already existed\n", + "reverie__train__5__0: Response already existed\n", + "reverie__train__6__0: Response already existed\n", + "reverie__train__7__0: Response already existed\n", + "reverie__train__8__0: Response already existed\n", + "reverie__train__9__0: Response already existed\n", + "reverie__train__10__0: Response already existed\n", + "reverie__train__11__0: Response already existed\n", + "reverie__train__12__0: Response already existed\n", + "reverie__train__13__0: Response already existed\n", + "reverie__train__14__0: Response already existed\n", + "reverie__train__15__0: Response already existed\n", + "reverie__train__16__0: Response already existed\n", + "reverie__train__17__0: Response already existed\n", + "reverie__train__18__0: Response already existed\n", + "reverie__train__19__0: Response already existed\n", + "reverie__train__20__0: Response already existed\n", + "reverie__train__21__0: Response already existed\n", + "reverie__train__22__0: Response already existed\n", + "reverie__train__23__0: Response already existed\n", + "reverie__train__24__0: Response already existed\n", + "2023-11-03 18:17:35.189910 25 4150\n", + "reverie__train__25__0: Response already existed\n", + "reverie__train__26__0: Response already existed\n", + "reverie__train__27__0: Response already existed\n", + "reverie__train__28__0: Response already existed\n", + "reverie__train__29__0: Response already existed\n", + "reverie__train__30__0: Response already existed\n", + "reverie__train__31__0: Response already existed\n", + "reverie__train__32__0: Response already existed\n", + "reverie__train__33__0: Response already existed\n", + "reverie__train__34__0: Response already existed\n", + "reverie__train__35__0: Response already existed\n", + "reverie__train__36__0: Response already existed\n", + "reverie__train__37__0: Response already existed\n", + "reverie__train__38__0: Response already existed\n", + "reverie__train__39__0: Response already existed\n", + "reverie__train__40__0: Response already existed\n", + "reverie__train__41__0: Response already existed\n", + "reverie__train__42__0: Response already existed\n", + "reverie__train__43__0: Response already existed\n", + "reverie__train__44__0: Response already existed\n", + "reverie__train__45__0: Response already existed\n", + "reverie__train__46__0: Response already existed\n", + "reverie__train__47__0: Response already existed\n", + "reverie__train__48__0: Response already existed\n", + "reverie__train__49__0: Response already existed\n", + "2023-11-03 18:17:35.531960 50 4150\n", + "reverie__train__50__0: Response already existed\n", + "reverie__train__51__0: Response already existed\n", + "reverie__train__52__0: Response already existed\n", + "reverie__train__53__0: Response already existed\n", + "reverie__train__54__0: Response already existed\n", + "reverie__train__55__0: Response already existed\n", + "reverie__train__56__0: Response already existed\n", + "reverie__train__57__0: Response already existed\n", + "reverie__train__58__0: Response already existed\n", + "reverie__train__59__0: Response already existed\n", + "reverie__train__60__0: Response already existed\n", + "reverie__train__61__0: Response already existed\n", + "reverie__train__62__0: Response already existed\n", + "reverie__train__63__0: Response already existed\n", + "reverie__train__64__0: Response already existed\n", + "reverie__train__65__0: Response already existed\n", + "reverie__train__66__0: Response already existed\n", + "reverie__train__67__0: Response already existed\n", + "reverie__train__68__0: Response already existed\n", + "reverie__train__69__0: Response already existed\n", + "reverie__train__70__0: Response already existed\n", + "reverie__train__71__0: Response already existed\n", + "reverie__train__72__0: Response already existed\n", + "reverie__train__73__0: Response already existed\n", + "reverie__train__74__0: Response already existed\n", + "2023-11-03 18:17:35.870754 75 4150\n", + "reverie__train__75__0: Response already existed\n", + "reverie__train__76__0: Response already existed\n", + "reverie__train__77__0: Response already existed\n", + "reverie__train__78__0: Response already existed\n", + "reverie__train__79__0: Response already existed\n", + "reverie__train__80__0: Response already existed\n", + "reverie__train__81__0: Response already existed\n", + "reverie__train__82__0: Response already existed\n", + "reverie__train__83__0: Response already existed\n", + "reverie__train__84__0: Response already existed\n", + "reverie__train__85__0: Response already existed\n", + "reverie__train__86__0: Response already existed\n", + "reverie__train__87__0: Response already existed\n", + "reverie__train__88__0: Response already existed\n", + "reverie__train__89__0: Response already existed\n", + "reverie__train__90__0: Response already existed\n", + "reverie__train__91__0: Response already existed\n", + "reverie__train__92__0: Response already existed\n", + "reverie__train__93__0: Response already existed\n", + "reverie__train__94__0: Response already existed\n", + "reverie__train__95__0: Response already existed\n", + "reverie__train__96__0: Response already existed\n", + "reverie__train__97__0: Response already existed\n", + "reverie__train__98__0: Response already existed\n", + "reverie__train__99__0: Response already existed\n", + "2023-11-03 18:17:36.201295 100 4150\n", + "reverie__train__100__0: Response already existed\n", + "reverie__train__101__0: Response already existed\n", + "reverie__train__102__0: Response already existed\n", + "reverie__train__103__0: Response already existed\n", + "reverie__train__104__0: Response already existed\n", + "reverie__train__105__0: Response already existed\n", + "reverie__train__106__0: Response already existed\n", + "reverie__train__107__0: Response already existed\n", + "reverie__train__108__0: Response already existed\n", + "reverie__train__109__0: Response already existed\n", + "reverie__train__110__0: Response already existed\n", + "reverie__train__111__0: Response already existed\n", + "reverie__train__112__0: Response already existed\n", + "reverie__train__113__0: Response already existed\n", + "reverie__train__114__0: Response already existed\n", + "reverie__train__115__0: Response already existed\n", + "reverie__train__116__0: Response already existed\n", + "reverie__train__117__0: Response already existed\n", + "reverie__train__118__0: Response already existed\n", + "reverie__train__119__0: Response already existed\n", + "reverie__train__120__0: Response already existed\n", + "reverie__train__121__0: Response already existed\n", + "reverie__train__122__0: Response already existed\n", + "reverie__train__123__0: Response already existed\n", + "reverie__train__124__0: Response already existed\n", + "2023-11-03 18:17:36.536267 125 4150\n", + "reverie__train__125__0: Response already existed\n", + "reverie__train__126__0: Response already existed\n", + "reverie__train__127__0: Response already existed\n", + "reverie__train__128__0: Response already existed\n", + "reverie__train__129__0: Response already existed\n", + "reverie__train__130__0: Response already existed\n", + "reverie__train__131__0: Response already existed\n", + "reverie__train__132__0: Response already existed\n", + "reverie__train__133__0: Response already existed\n", + "reverie__train__134__0: Response already existed\n", + "reverie__train__135__0: Response already existed\n", + "reverie__train__136__0: Response already existed\n", + "reverie__train__137__0: Response already existed\n", + "reverie__train__138__0: Response already existed\n", + "reverie__train__139__0: Response already existed\n", + "reverie__train__140__0: Response already existed\n", + "reverie__train__141__0: Response already existed\n", + "reverie__train__142__0: Response already existed\n", + "reverie__train__143__0: Response already existed\n", + "reverie__train__144__0: Response already existed\n", + "reverie__train__145__0: Response already existed\n", + "reverie__train__146__0: Response already existed\n", + "reverie__train__147__0: Response already existed\n", + "reverie__train__148__0: Response already existed\n", + "reverie__train__149__0: Response already existed\n", + "2023-11-03 18:17:36.878317 150 4150\n", + "reverie__train__150__0: Response already existed\n", + "reverie__train__151__0: Response already existed\n", + "reverie__train__152__0: Response already existed\n", + "reverie__train__153__0: Response already existed\n", + "reverie__train__154__0: Response already existed\n", + "reverie__train__155__0: Response already existed\n", + "reverie__train__156__0: Response already existed\n", + "reverie__train__157__0: Response already existed\n", + "reverie__train__158__0: Response already existed\n", + "reverie__train__159__0: Response already existed\n", + "reverie__train__160__0: Response already existed\n", + "reverie__train__161__0: Response already existed\n", + "reverie__train__162__0: Response already existed\n", + "reverie__train__163__0: Response already existed\n", + "reverie__train__164__0: Response already existed\n", + "reverie__train__165__0: Response already existed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "reverie__train__166__0: Response already existed\n", + "reverie__train__167__0: Response already existed\n", + "reverie__train__168__0: Response already existed\n", + "reverie__train__169__0: Response already existed\n", + "reverie__train__170__0: Response already existed\n", + "reverie__train__171__0: Response already existed\n", + "reverie__train__172__0: Response already existed\n", + "reverie__train__173__0: Response already existed\n", + "reverie__train__174__0: Response already existed\n", + "2023-11-03 18:17:37.226115 175 4150\n", + "reverie__train__175__0: Response already existed\n", + "reverie__train__176__0: Response already existed\n", + "reverie__train__177__0: Response already existed\n", + "reverie__train__178__0: Response already existed\n", + "reverie__train__179__0: Response already existed\n", + "reverie__train__180__0: Response already existed\n", + "reverie__train__181__0: Response already existed\n", + "reverie__train__182__0: Response already existed\n", + "reverie__train__183__0: Response already existed\n", + "reverie__train__184__0: Response already existed\n", + "reverie__train__185__0: Response already existed\n", + "reverie__train__186__0: Response already existed\n", + "reverie__train__187__0: Response already existed\n", + "reverie__train__188__0: Response already existed\n", + "reverie__train__189__0: Response already existed\n", + "reverie__train__190__0: Response already existed\n", + "reverie__train__191__0: Response already existed\n", + "reverie__train__192__0: Response already existed\n", + "reverie__train__193__0: Response already existed\n", + "reverie__train__194__0: Response already existed\n", + "reverie__train__195__0: Response already existed\n", + "reverie__train__196__0: Response already existed\n", + "reverie__train__197__0: Response already existed\n", + "reverie__train__198__0: Response already existed\n", + "reverie__train__199__0: Response already existed\n", + "2023-11-03 18:17:37.555939 200 4150\n", + "reverie__train__200__0: Response already existed\n", + "reverie__train__201__0: Response already existed\n", + "reverie__train__202__0: Response already existed\n", + "reverie__train__203__0: Response already existed\n", + "reverie__train__204__0: Response already existed\n", + "reverie__train__205__0: Response already existed\n", + "reverie__train__206__0: Response already existed\n", + "reverie__train__207__0: Response already existed\n", + "reverie__train__208__0: Response already existed\n", + "reverie__train__209__0: Response already existed\n", + "reverie__train__210__0: Response already existed\n", + "reverie__train__211__0: Response already existed\n", + "reverie__train__212__0: Response already existed\n", + "reverie__train__213__0: Response already existed\n", + "reverie__train__214__0: Response already existed\n", + "reverie__train__215__0: Response already existed\n", + "reverie__train__216__0: Response already existed\n", + "reverie__train__217__0: Response already existed\n", + "reverie__train__218__0: Response already existed\n", + "reverie__train__219__0: Response already existed\n", + "reverie__train__220__0: Response already existed\n", + "reverie__train__221__0: Response already existed\n", + "reverie__train__222__0: Response already existed\n", + "reverie__train__223__0: Response already existed\n", + "reverie__train__224__0: Response already existed\n", + "2023-11-03 18:17:37.886875 225 4150\n", + "reverie__train__225__0: Response already existed\n", + "reverie__train__226__0: Response already existed\n", + "reverie__train__227__0: Response already existed\n", + "reverie__train__228__0: Response already existed\n", + "reverie__train__229__0: Response already existed\n", + "reverie__train__230__0: Response already existed\n", + "reverie__train__231__0: Response already existed\n", + "reverie__train__232__0: Response already existed\n", + "reverie__train__233__0: Response already existed\n", + "reverie__train__234__0: Response already existed\n", + "reverie__train__235__0: Response already existed\n", + "reverie__train__236__0: Response already existed\n", + "reverie__train__237__0: Response already existed\n", + "reverie__train__238__0: Response already existed\n", + "reverie__train__239__0: Response already existed\n", + "reverie__train__240__0: Response already existed\n", + "reverie__train__241__0: Response already existed\n", + "reverie__train__242__0: Response already existed\n", + "reverie__train__243__0: Response already existed\n", + "reverie__train__244__0: Response already existed\n", + "reverie__train__245__0: Response already existed\n", + "reverie__train__246__0: Response already existed\n", + "reverie__train__247__0: Response already existed\n", + "reverie__train__248__0: Response already existed\n", + "reverie__train__249__0: Response already existed\n", + "2023-11-03 18:17:38.219850 250 4150\n", + "reverie__train__250__0: Response already existed\n", + "reverie__train__251__0: Response already existed\n", + "reverie__train__252__0: Response already existed\n", + "reverie__train__253__0: Response already existed\n", + "reverie__train__254__0: Response already existed\n", + "reverie__train__255__0: Response already existed\n", + "reverie__train__256__0: Response already existed\n", + "reverie__train__257__0: Response already existed\n", + "reverie__train__258__0: Response already existed\n", + "reverie__train__259__0: Response already existed\n", + "reverie__train__260__0: Response already existed\n", + "reverie__train__261__0: Response already existed\n", + "reverie__train__262__0: Response already existed\n", + "reverie__train__263__0: Response already existed\n", + "reverie__train__264__0: Response already existed\n", + "reverie__train__265__0: Response already existed\n", + "reverie__train__266__0: Response already existed\n", + "reverie__train__267__0: Response already existed\n", + "reverie__train__268__0: Response already existed\n", + "reverie__train__269__0: Response already existed\n", + "reverie__train__270__0: Response already existed\n", + "reverie__train__271__0: Response already existed\n", + "reverie__train__272__0: Response already existed\n", + "reverie__train__273__0: Response already existed\n", + "reverie__train__274__0: Response already existed\n", + "2023-11-03 18:17:38.558654 275 4150\n", + "reverie__train__275__0: Response already existed\n", + "reverie__train__276__0: Response already existed\n", + "reverie__train__277__0: Response already existed\n", + "reverie__train__278__0: Response already existed\n", + "reverie__train__279__0: Response already existed\n", + "reverie__train__280__0: Response already existed\n", + "reverie__train__281__0: Response already existed\n", + "reverie__train__282__0: Response already existed\n", + "reverie__train__283__0: Response already existed\n", + "reverie__train__284__0: Response already existed\n", + "reverie__train__285__0: Response already existed\n", + "reverie__train__286__0: Response already existed\n", + "reverie__train__287__0: Response already existed\n", + "reverie__train__288__0: Response already existed\n", + "reverie__train__289__0: Response already existed\n", + "reverie__train__290__0: Response already existed\n", + "reverie__train__291__0: Response already existed\n", + "reverie__train__292__0: Response already existed\n", + "reverie__train__293__0: Response already existed\n", + "reverie__train__294__0: Response already existed\n", + "reverie__train__295__0: Response already existed\n", + "reverie__train__296__0: Response already existed\n", + "reverie__train__297__0: Response already existed\n", + "reverie__train__298__0: Response already existed\n", + "reverie__train__299__0: Response already existed\n", + "2023-11-03 18:17:38.890873 300 4150\n", + "reverie__train__300__0: Response already existed\n", + "reverie__train__301__0: Response already existed\n", + "reverie__train__302__0: Response already existed\n", + "reverie__train__303__0: Response already existed\n", + "reverie__train__304__0: Response already existed\n", + "reverie__train__305__0: Response already existed\n", + "reverie__train__306__0: Response already existed\n", + "reverie__train__307__0: Response already existed\n", + "reverie__train__308__0: Response already existed\n", + "reverie__train__309__0: Response already existed\n", + "reverie__train__310__0: Response already existed\n", + "reverie__train__311__0: Response already existed\n", + "reverie__train__312__0: Response already existed\n", + "reverie__train__313__0: Response already existed\n", + "reverie__train__314__0: Response already existed\n", + "reverie__train__315__0: Response already existed\n", + "reverie__train__316__0: Response already existed\n", + "reverie__train__317__0: Response already existed\n", + "reverie__train__318__0: Response already existed\n", + "reverie__train__319__0: Response already existed\n", + "reverie__train__320__0: Response already existed\n", + "reverie__train__321__0: Response already existed\n", + "reverie__train__322__0: Response already existed\n", + "reverie__train__323__0: Response already existed\n", + "reverie__train__324__0: Response already existed\n", + "2023-11-03 18:17:39.221674 325 4150\n", + "reverie__train__325__0: Response already existed\n", + "reverie__train__326__0: Response already existed\n", + "reverie__train__327__0: Response already existed\n", + "reverie__train__328__0: Response already existed\n", + "reverie__train__329__0: Response already existed\n", + "reverie__train__330__0: Response already existed\n", + "reverie__train__331__0: Response already existed\n", + "reverie__train__332__0: Response already existed\n", + "reverie__train__333__0: Response already existed\n", + "reverie__train__334__0: Response already existed\n", + "reverie__train__335__0: Response already existed\n", + "reverie__train__336__0: Response already existed\n", + "reverie__train__337__0: Response already existed\n", + "reverie__train__338__0: Response already existed\n", + "reverie__train__339__0: Response already existed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "reverie__train__340__0: Response already existed\n", + "reverie__train__341__0: Response already existed\n", + "reverie__train__342__0: Response already existed\n", + "reverie__train__343__0: Response already existed\n", + "reverie__train__344__0: Response already existed\n", + "reverie__train__345__0: Response already existed\n", + "reverie__train__346__0: Response already existed\n", + "reverie__train__347__0: Response already existed\n", + "reverie__train__348__0: Response already existed\n", + "reverie__train__349__0: Response already existed\n", + "2023-11-03 18:17:39.557288 350 4150\n", + "reverie__train__350__0: Response already existed\n", + "reverie__train__351__0: Response already existed\n", + "reverie__train__352__0: Response already existed\n", + "reverie__train__353__0: Response already existed\n", + "reverie__train__354__0: Response already existed\n", + "reverie__train__355__0: Response already existed\n", + "reverie__train__356__0: Response already existed\n", + "reverie__train__357__0: Response already existed\n", + "reverie__train__358__0: Response already existed\n", + "reverie__train__359__0: Response already existed\n", + "reverie__train__360__0: Response already existed\n", + "reverie__train__361__0: Response already existed\n", + "reverie__train__362__0: Response already existed\n", + "reverie__train__363__0: Response already existed\n", + "reverie__train__364__0: Response already existed\n", + "reverie__train__365__0: Response already existed\n", + "reverie__train__366__0: Response already existed\n", + "reverie__train__367__0: Response already existed\n", + "reverie__train__368__0: Response already existed\n", + "reverie__train__369__0: Response already existed\n", + "reverie__train__370__0: Response already existed\n", + "reverie__train__371__0: Response already existed\n", + "reverie__train__372__0: Response already existed\n", + "reverie__train__373__0: Response already existed\n", + "reverie__train__374__0: Response already existed\n", + "2023-11-03 18:17:39.906177 375 4150\n", + "reverie__train__375__0: Response already existed\n", + "reverie__train__376__0: Response already existed\n", + "reverie__train__377__0: Response already existed\n", + "reverie__train__378__0: Response already existed\n", + "reverie__train__379__0: Response already existed\n", + "reverie__train__380__0: Response already existed\n", + "reverie__train__381__0: Response already existed\n", + "reverie__train__382__0: Response already existed\n", + "reverie__train__383__0: Response already existed\n", + "reverie__train__384__0: Response already existed\n", + "reverie__train__385__0: Response already existed\n", + "reverie__train__386__0: Response already existed\n", + "reverie__train__387__0: Response already existed\n", + "reverie__train__388__0: Response already existed\n", + "reverie__train__389__0: Response already existed\n", + "reverie__train__390__0: Response already existed\n", + "reverie__train__391__0: Response already existed\n", + "reverie__train__392__0: Response already existed\n", + "reverie__train__393__0: Response already existed\n", + "reverie__train__394__0: Response already existed\n", + "reverie__train__395__0: Response already existed\n", + "reverie__train__396__0: Response already existed\n", + "reverie__train__397__0: Response already existed\n", + "reverie__train__398__0: Response already existed\n", + "reverie__train__399__0: Response already existed\n", + "2023-11-03 18:17:40.253340 400 4150\n", + "reverie__train__400__0: Response already existed\n", + "reverie__train__401__0: Response already existed\n", + "reverie__train__402__0: Response already existed\n", + "reverie__train__403__0: Response already existed\n", + "reverie__train__404__0: Response already existed\n", + "reverie__train__405__0: Response already existed\n", + "reverie__train__406__0: Response already existed\n", + "reverie__train__407__0: Response already existed\n", + "reverie__train__408__0: Response already existed\n", + "reverie__train__409__0: Response already existed\n", + "reverie__train__410__0: Response already existed\n", + "reverie__train__411__0: Response already existed\n", + "reverie__train__412__0: Response already existed\n", + "reverie__train__413__0: Response already existed\n", + "reverie__train__414__0: Response already existed\n", + "reverie__train__415__0: Response already existed\n", + "reverie__train__416__0: Response already existed\n", + "reverie__train__417__0: Response already existed\n", + "reverie__train__418__0: Response already existed\n", + "reverie__train__419__0: Response already existed\n", + "reverie__train__420__0: Response already existed\n", + "reverie__train__421__0: Response already existed\n", + "reverie__train__422__0: Response already existed\n", + "reverie__train__423__0: Response already existed\n", + "reverie__train__424__0: Response already existed\n", + "2023-11-03 18:17:40.594747 425 4150\n", + "reverie__train__425__0: Response already existed\n", + "reverie__train__426__0: Response already existed\n", + "reverie__train__427__0: Response already existed\n", + "reverie__train__428__0: Response already existed\n", + "reverie__train__429__0: Response already existed\n", + "reverie__train__430__0: Response already existed\n", + "reverie__train__431__0: Response already existed\n", + "reverie__train__432__0: Response already existed\n", + "reverie__train__433__0: Response already existed\n", + "reverie__train__434__0: Response already existed\n", + "reverie__train__435__0: Response already existed\n", + "reverie__train__436__0: Response already existed\n", + "reverie__train__437__0: Response already existed\n", + "reverie__train__438__0: Response already existed\n", + "reverie__train__439__0: Response already existed\n", + "reverie__train__440__0: Response already existed\n", + "reverie__train__441__0: Response already existed\n", + "reverie__train__442__0: Response already existed\n", + "reverie__train__443__0: Response already existed\n", + "reverie__train__444__0: Response already existed\n", + "reverie__train__445__0: Response already existed\n", + "reverie__train__446__0: Response already existed\n", + "reverie__train__447__0: Response already existed\n", + "reverie__train__448__0: Response already existed\n", + "reverie__train__449__0: Response already existed\n", + "2023-11-03 18:17:40.926482 450 4150\n", + "reverie__train__450__0: Response already existed\n", + "reverie__train__451__0: Response already existed\n", + "reverie__train__452__0: Response already existed\n", + "reverie__train__453__0: Response already existed\n", + "reverie__train__454__0: Response already existed\n", + "reverie__train__455__0: Response already existed\n", + "reverie__train__456__0: Response already existed\n", + "reverie__train__457__0: Response already existed\n", + "reverie__train__458__0: Response already existed\n", + "reverie__train__459__0: Response already existed\n", + "reverie__train__460__0: Response already existed\n", + "reverie__train__461__0: Response already existed\n", + "reverie__train__462__0: Response already existed\n", + "reverie__train__463__0: Response already existed\n", + "reverie__train__464__0: Response already existed\n", + "reverie__train__465__0: Response already existed\n", + "reverie__train__466__0: Response already existed\n", + "reverie__train__467__0: Response already existed\n", + "reverie__train__468__0: Response already existed\n", + "reverie__train__469__0: Response already existed\n", + "reverie__train__470__0: Response already existed\n", + "reverie__train__471__0: Response already existed\n", + "reverie__train__472__0: Response already existed\n", + "reverie__train__473__0: Response already existed\n", + "reverie__train__474__0: Response already existed\n", + "2023-11-03 18:17:41.258333 475 4150\n", + "reverie__train__475__0: Response already existed\n", + "reverie__train__476__0: Response already existed\n", + "reverie__train__477__0: Response already existed\n", + "reverie__train__478__0: Response already existed\n", + "reverie__train__479__0: Response already existed\n", + "reverie__train__480__0: Response already existed\n", + "reverie__train__481__0: Response already existed\n", + "reverie__train__482__0: Response already existed\n", + "reverie__train__483__0: Response already existed\n", + "reverie__train__484__0: Response already existed\n", + "reverie__train__485__0: Response already existed\n", + "reverie__train__486__0: Response already existed\n", + "reverie__train__487__0: Response already existed\n", + "reverie__train__488__0: Response already existed\n", + "reverie__train__489__0: Response already existed\n", + "reverie__train__490__0: Response already existed\n", + "reverie__train__491__0: Response already existed\n", + "reverie__train__492__0: Response already existed\n", + "reverie__train__493__0: Response already existed\n", + "reverie__train__494__0: Response already existed\n", + "reverie__train__495__0: Response already existed\n", + "reverie__train__496__0: Response already existed\n", + "reverie__train__497__0: Response already existed\n", + "reverie__train__498__0: Response already existed\n", + "reverie__train__499__0: Response already existed\n", + "2023-11-03 18:17:41.595003 500 4150\n", + "reverie__train__500__0: Response already existed\n", + "reverie__train__501__0: Response already existed\n", + "reverie__train__502__0: Response already existed\n", + "reverie__train__503__0: Response already existed\n", + "reverie__train__504__0: Response already existed\n", + "reverie__train__505__0: Response already existed\n", + "reverie__train__506__0: Response already existed\n", + "reverie__train__507__0: Response already existed\n", + "reverie__train__508__0: Response already existed\n", + "reverie__train__509__0: Response already existed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "reverie__train__510__0: Response already existed\n", + "reverie__train__511__0: Response already existed\n", + "reverie__train__512__0: Response already existed\n", + "reverie__train__513__0: Response already existed\n", + "reverie__train__514__0: Response already existed\n", + "reverie__train__515__0: Response already existed\n", + "reverie__train__516__0: Response already existed\n", + "reverie__train__517__0: Response already existed\n", + "reverie__train__518__0: Response already existed\n", + "reverie__train__519__0: Response already existed\n", + "reverie__train__520__0: Response already existed\n", + "reverie__train__521__0: Response already existed\n", + "reverie__train__522__0: Response already existed\n", + "reverie__train__523__0: Response already existed\n", + "reverie__train__524__0: Response already existed\n", + "2023-11-03 18:17:41.925264 525 4150\n", + "reverie__train__525__0: Response already existed\n", + "reverie__train__526__0: Response already existed\n", + "reverie__train__527__0: Response already existed\n", + "reverie__train__528__0: Response already existed\n", + "reverie__train__529__0: Response already existed\n", + "reverie__train__530__0: Response already existed\n", + "reverie__train__531__0: Response already existed\n", + "reverie__train__532__0: Response already existed\n", + "reverie__train__533__0: Response already existed\n", + "reverie__train__534__0: Response already existed\n", + "reverie__train__535__0: Response already existed\n", + "reverie__train__536__0: Response already existed\n", + "reverie__train__537__0: Response already existed\n", + "reverie__train__538__0: Response already existed\n", + "reverie__train__539__0: Response already existed\n", + "reverie__train__540__0: Response already existed\n", + "reverie__train__541__0: Response already existed\n", + "reverie__train__542__0: Response already existed\n", + "reverie__train__543__0: Response already existed\n", + "reverie__train__544__0: Response already existed\n", + "reverie__train__545__0: Response already existed\n", + "reverie__train__546__0: Response already existed\n", + "reverie__train__547__0: Response already existed\n", + "reverie__train__548__0: Response already existed\n", + "reverie__train__549__0: Response already existed\n", + "2023-11-03 18:17:42.256847 550 4150\n", + "reverie__train__550__0: Response already existed\n", + "reverie__train__551__0: Response already existed\n", + "reverie__train__552__0: Response already existed\n", + "reverie__train__553__0: Response already existed\n", + "reverie__train__554__0: Response already existed\n", + "reverie__train__555__0: Response already existed\n", + "reverie__train__556__0: Response already existed\n", + "reverie__train__557__0: Response already existed\n", + "reverie__train__558__0: Response already existed\n", + "reverie__train__559__0: Response already existed\n", + "reverie__train__560__0: Response already existed\n", + "reverie__train__561__0: Response already existed\n", + "reverie__train__562__0: Response already existed\n", + "reverie__train__563__0: Response already existed\n", + "reverie__train__564__0: Response already existed\n", + "reverie__train__565__0: Response already existed\n", + "reverie__train__566__0: Response already existed\n", + "reverie__train__567__0: Response already existed\n", + "reverie__train__568__0: Response already existed\n", + "reverie__train__569__0: Response already existed\n", + "reverie__train__570__0: Response already existed\n", + "reverie__train__571__0: Response already existed\n", + "reverie__train__572__0: Response already existed\n", + "reverie__train__573__0: Response already existed\n", + "reverie__train__574__0: Response already existed\n", + "2023-11-03 18:17:42.589035 575 4150\n", + "reverie__train__575__0: Response already existed\n", + "reverie__train__576__0: Response already existed\n", + "reverie__train__577__0: Response already existed\n", + "reverie__train__578__0: Response already existed\n", + "reverie__train__579__0: Response already existed\n", + "reverie__train__580__0: Response already existed\n", + "reverie__train__581__0: Response already existed\n", + "reverie__train__582__0: Response already existed\n", + "reverie__train__583__0: Response already existed\n", + "reverie__train__584__0: Response already existed\n", + "reverie__train__585__0: Response already existed\n", + "reverie__train__586__0: Response already existed\n", + "reverie__train__587__0: Response already existed\n", + "reverie__train__588__0: Response already existed\n", + "reverie__train__589__0: Response already existed\n", + "reverie__train__590__0: Response already existed\n", + "reverie__train__591__0: Response already existed\n", + "reverie__train__592__0: Response already existed\n", + "reverie__train__593__0: Response already existed\n", + "reverie__train__594__0: Response already existed\n", + "reverie__train__595__0: Response already existed\n", + "reverie__train__596__0: Response already existed\n", + "reverie__train__597__0: Response already existed\n", + "reverie__train__598__0: Response already existed\n", + "reverie__train__599__0: Response already existed\n", + "2023-11-03 18:17:42.919508 600 4150\n", + "reverie__train__600__0: Response already existed\n", + "reverie__train__601__0: Response already existed\n", + "reverie__train__602__0: Response already existed\n", + "reverie__train__603__0: Response already existed\n", + "reverie__train__604__0: Response already existed\n", + "reverie__train__605__0: Response already existed\n", + "reverie__train__606__0: Response already existed\n", + "reverie__train__607__0: Response already existed\n", + "reverie__train__608__0: Response already existed\n", + "reverie__train__609__0: Response already existed\n", + "reverie__train__610__0: Response already existed\n", + "reverie__train__611__0: Response already existed\n", + "reverie__train__612__0: Response already existed\n", + "reverie__train__613__0: Response already existed\n", + "reverie__train__614__0: Response already existed\n", + "reverie__train__615__0: Response already existed\n", + "reverie__train__616__0: Response already existed\n", + "reverie__train__617__0: Response already existed\n", + "reverie__train__618__0: Response already existed\n", + "reverie__train__619__0: Response already existed\n", + "reverie__train__620__0: Response already existed\n", + "reverie__train__621__0: Response already existed\n", + "reverie__train__622__0: Response already existed\n", + "reverie__train__623__0: Response already existed\n", + "reverie__train__624__0: Response already existed\n", + "2023-11-03 18:17:43.251016 625 4150\n", + "reverie__train__625__0: Response already existed\n", + "reverie__train__626__0: Response already existed\n", + "reverie__train__627__0: Response already existed\n", + "reverie__train__628__0: Response already existed\n", + "reverie__train__629__0: Response already existed\n", + "reverie__train__630__0: Response already existed\n", + "reverie__train__631__0: Response already existed\n", + "reverie__train__632__0: Response already existed\n", + "reverie__train__633__0: Response already existed\n", + "reverie__train__634__0: Response already existed\n", + "reverie__train__635__0: Response already existed\n", + "reverie__train__636__0: Response already existed\n", + "reverie__train__637__0: Response already existed\n", + "reverie__train__638__0: Response already existed\n", + "reverie__train__639__0: Response already existed\n", + "reverie__train__640__0: Response already existed\n", + "reverie__train__641__0: Response already existed\n", + "reverie__train__642__0: Response already existed\n", + "reverie__train__643__0: Response already existed\n", + "reverie__train__644__0: Response already existed\n", + "reverie__train__645__0: Response already existed\n", + "reverie__train__646__0: Response already existed\n", + "reverie__train__647__0: Response already existed\n", + "reverie__train__648__0: Response already existed\n", + "reverie__train__649__0: Response already existed\n", + "2023-11-03 18:17:43.587570 650 4150\n", + "reverie__train__650__0: Response already existed\n", + "reverie__train__651__0: Response already existed\n", + "reverie__train__652__0: Response already existed\n", + "reverie__train__653__0: Response already existed\n", + "reverie__train__654__0: Response already existed\n", + "reverie__train__655__0: Response already existed\n", + "reverie__train__656__0: Response already existed\n", + "reverie__train__657__0: Response already existed\n", + "reverie__train__658__0: Response already existed\n", + "reverie__train__659__0: Response already existed\n", + "reverie__train__660__0: Response already existed\n", + "reverie__train__661__0: Response already existed\n", + "reverie__train__662__0: Response already existed\n", + "reverie__train__663__0: Response already existed\n", + "reverie__train__664__0: Response already existed\n", + "reverie__train__665__0: Response already existed\n", + "reverie__train__666__0: Response already existed\n", + "reverie__train__667__0: Response already existed\n", + "reverie__train__668__0: Response already existed\n", + "reverie__train__669__0: Response already existed\n", + "reverie__train__670__0: Response already existed\n", + "reverie__train__671__0: Response already existed\n", + "reverie__train__672__0: Response already existed\n", + "reverie__train__673__0: Response already existed\n", + "reverie__train__674__0: Response already existed\n", + "2023-11-03 18:17:43.930937 675 4150\n", + "reverie__train__675__0: Response already existed\n", + "reverie__train__676__0: Response already existed\n", + "reverie__train__677__0: Response already existed\n", + "reverie__train__678__0: Response already existed\n", + "reverie__train__679__0: Response already existed\n", + "reverie__train__680__0: Response already existed\n", + "reverie__train__681__0: Response already existed\n", + "reverie__train__682__0: Response already existed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "reverie__train__683__0: Response already existed\n", + "reverie__train__684__0: Response already existed\n", + "reverie__train__685__0: Response already existed\n", + "reverie__train__686__0: Response already existed\n", + "reverie__train__687__0: Response already existed\n", + "reverie__train__688__0: Response already existed\n", + "reverie__train__689__0: Response already existed\n", + "reverie__train__690__0: Response already existed\n", + "reverie__train__691__0: Response already existed\n", + "reverie__train__692__0: Response already existed\n", + "reverie__train__693__0: Response already existed\n", + "reverie__train__694__0: Response already existed\n", + "reverie__train__695__0: Response already existed\n", + "reverie__train__696__0: Response already existed\n", + "reverie__train__697__0: Response already existed\n", + "reverie__train__698__0: Response already existed\n", + "reverie__train__699__0: Response already existed\n", + "2023-11-03 18:17:44.280731 700 4150\n", + "reverie__train__700__0: Response already existed\n", + "reverie__train__701__0: Response already existed\n", + "reverie__train__702__0: Response already existed\n", + "reverie__train__703__0: Response already existed\n", + "reverie__train__704__0: Response already existed\n", + "reverie__train__705__0: Response already existed\n", + "reverie__train__706__0: Response already existed\n", + "reverie__train__707__0: Response already existed\n", + "reverie__train__708__0: Response already existed\n", + "reverie__train__709__0: Response already existed\n", + "reverie__train__710__0: Response already existed\n", + "reverie__train__711__0: Response already existed\n", + "reverie__train__712__0: Response already existed\n", + "reverie__train__713__0: Response already existed\n", + "reverie__train__714__0: Response already existed\n", + "reverie__train__715__0: Response already existed\n", + "reverie__train__716__0: Response already existed\n", + "reverie__train__717__0: Response already existed\n", + "reverie__train__718__0: Response already existed\n", + "reverie__train__719__0: Response already existed\n", + "reverie__train__720__0: Response already existed\n", + "reverie__train__721__0: Response already existed\n", + "reverie__train__722__0: Response already existed\n", + "reverie__train__723__0: Response already existed\n", + "reverie__train__724__0: Response already existed\n", + "2023-11-03 18:17:44.613817 725 4150\n", + "reverie__train__725__0: Response already existed\n", + "reverie__train__726__0: Response already existed\n", + "reverie__train__727__0: Response already existed\n", + "reverie__train__728__0: Response already existed\n", + "reverie__train__729__0: Response already existed\n", + "reverie__train__730__0: Response already existed\n", + "reverie__train__731__0: Response already existed\n", + "reverie__train__732__0: Response already existed\n", + "reverie__train__733__0: Response already existed\n", + "reverie__train__734__0: Response already existed\n", + "reverie__train__735__0: Response already existed\n", + "reverie__train__736__0: Response already existed\n", + "reverie__train__737__0: Response already existed\n", + "reverie__train__738__0: Response already existed\n", + "reverie__train__739__0: Response already existed\n", + "reverie__train__740__0: Response already existed\n", + "reverie__train__741__0: Response already existed\n", + "reverie__train__742__0: Response already existed\n", + "reverie__train__743__0: Response already existed\n", + "reverie__train__744__0: Response already existed\n", + "reverie__train__745__0: Response already existed\n", + "reverie__train__746__0: Response already existed\n", + "reverie__train__747__0: Response already existed\n", + "reverie__train__748__0: Response already existed\n", + "reverie__train__749__0: Response already existed\n", + "2023-11-03 18:17:44.943575 750 4150\n", + "reverie__train__750__0: Response already existed\n", + "reverie__train__751__0: Response already existed\n", + "reverie__train__752__0: Response already existed\n", + "reverie__train__753__0: Response already existed\n", + "reverie__train__754__0: Response already existed\n", + "reverie__train__755__0: Response already existed\n", + "reverie__train__756__0: Response already existed\n", + "reverie__train__757__0: Response already existed\n", + "reverie__train__758__0: Response already existed\n", + "reverie__train__759__0: Response already existed\n", + "reverie__train__760__0: Response already existed\n", + "reverie__train__761__0: Response already existed\n", + "reverie__train__762__0: Response already existed\n", + "reverie__train__763__0: Response already existed\n", + "reverie__train__764__0: Response already existed\n", + "reverie__train__765__0: Response already existed\n", + "reverie__train__766__0: Response already existed\n", + "reverie__train__767__0: Response already existed\n", + "reverie__train__768__0: Response already existed\n", + "reverie__train__769__0: Response already existed\n", + "reverie__train__770__0: Response already existed\n", + "reverie__train__771__0: Response already existed\n", + "reverie__train__772__0: Response already existed\n", + "reverie__train__773__0: Response already existed\n", + "reverie__train__774__0: Response already existed\n", + "2023-11-03 18:17:45.281375 775 4150\n", + "reverie__train__775__0: Response already existed\n", + "reverie__train__776__0: Response already existed\n", + "reverie__train__777__0: Response already existed\n", + "reverie__train__778__0: Response already existed\n", + "reverie__train__779__0: Response already existed\n", + "reverie__train__780__0: Response already existed\n", + "reverie__train__781__0: Response already existed\n", + "reverie__train__782__0: Response already existed\n", + "reverie__train__783__0: Response already existed\n", + "reverie__train__784__0: Response already existed\n", + "reverie__train__785__0: Response already existed\n", + "reverie__train__786__0: Response already existed\n", + "reverie__train__787__0: Response already existed\n", + "reverie__train__788__0: Response already existed\n", + "reverie__train__789__0: Response already existed\n", + "reverie__train__790__0: Response already existed\n", + "reverie__train__791__0: Response already existed\n", + "reverie__train__792__0: Response already existed\n", + "reverie__train__793__0: Response already existed\n", + "reverie__train__794__0: Response already existed\n", + "reverie__train__795__0: Response already existed\n", + "reverie__train__796__0: Response already existed\n", + "reverie__train__797__0: Response already existed\n", + "reverie__train__798__0: Response already existed\n", + "reverie__train__799__0: Response already existed\n", + "2023-11-03 18:17:45.612762 800 4150\n", + "reverie__train__800__0: Response already existed\n", + "reverie__train__801__0: Response already existed\n", + "reverie__train__802__0: Response already existed\n", + "reverie__train__803__0: Response already existed\n", + "reverie__train__804__0: Response already existed\n", + "reverie__train__805__0: Response already existed\n", + "reverie__train__806__0: Response already existed\n", + "reverie__train__807__0: Response already existed\n", + "reverie__train__808__0: Response already existed\n", + "reverie__train__809__0: Response already existed\n", + "reverie__train__810__0: Response already existed\n", + "reverie__train__811__0: Response already existed\n", + "reverie__train__812__0: Response already existed\n", + "reverie__train__813__0: Response already existed\n", + "reverie__train__814__0: Response already existed\n", + "reverie__train__815__0: Response already existed\n", + "reverie__train__816__0: Response already existed\n", + "reverie__train__817__0: Response already existed\n", + "reverie__train__818__0: Response already existed\n", + "reverie__train__819__0: Response already existed\n", + "reverie__train__820__0: Response already existed\n", + "reverie__train__821__0: Response already existed\n", + "reverie__train__822__0: Response already existed\n", + "reverie__train__823__0: Response already existed\n", + "reverie__train__824__0: Response already existed\n", + "2023-11-03 18:17:45.946423 825 4150\n", + "reverie__train__825__0: Response already existed\n", + "reverie__train__826__0: Response already existed\n", + "reverie__train__827__0: Response already existed\n", + "reverie__train__828__0: Response already existed\n", + "reverie__train__829__0: Response already existed\n", + "reverie__train__830__0: Response already existed\n", + "reverie__train__831__0: Response already existed\n", + "reverie__train__832__0: Response already existed\n", + "reverie__train__833__0: Response already existed\n", + "reverie__train__834__0: Response already existed\n", + "reverie__train__835__0: Response already existed\n", + "reverie__train__836__0: Response already existed\n", + "reverie__train__837__0: Response already existed\n", + "reverie__train__838__0: Response already existed\n", + "reverie__train__839__0: Response already existed\n", + "reverie__train__840__0: Response already existed\n", + "reverie__train__841__0: Response already existed\n", + "reverie__train__842__0: Response already existed\n", + "reverie__train__843__0: Response already existed\n", + "reverie__train__844__0: Response already existed\n", + "reverie__train__845__0: Response already existed\n", + "reverie__train__846__0: Response already existed\n", + "reverie__train__847__0: Response already existed\n", + "reverie__train__848__0: Response already existed\n", + "reverie__train__849__0: Response already existed\n", + "2023-11-03 18:17:46.281965 850 4150\n", + "reverie__train__850__0: Response already existed\n", + "reverie__train__851__0: Response already existed\n", + "reverie__train__852__0: Response already existed\n", + "reverie__train__853__0: Response already existed\n", + "reverie__train__854__0: Response already existed\n", + "reverie__train__855__0: Response already existed\n", + "reverie__train__856__0: Response already existed\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "reverie__train__857__0: Response already existed\n", + "reverie__train__858__0: Response already existed\n", + "reverie__train__859__0: Response already existed\n", + "reverie__train__860__0: Response already existed\n", + "reverie__train__861__0: Response already existed\n", + "reverie__train__862__0: Response already existed\n", + "reverie__train__863__0: Response already existed\n", + "reverie__train__864__0: Response already existed\n", + "reverie__train__865__0: Response already existed\n", + "reverie__train__866__0: Response already existed\n", + "reverie__train__867__0: Response already existed\n", + "reverie__train__868__0: Response already existed\n", + "reverie__train__869__0: Response already existed\n", + "reverie__train__870__0: Response already existed\n", + "reverie__train__871__0: Response already existed\n", + "reverie__train__872__0: Response already existed\n", + "reverie__train__873__0: Response already existed\n", + "reverie__train__874__0: Response already existed\n", + "2023-11-03 18:17:46.613060 875 4150\n", + "reverie__train__875__0: Response already existed\n", + "reverie__train__876__0: Response already existed\n", + "reverie__train__877__0: Response already existed\n", + "reverie__train__878__0: Response already existed\n", + "reverie__train__879__0: Response already existed\n", + "reverie__train__880__0: Response already existed\n", + "reverie__train__881__0: Response already existed\n", + "reverie__train__882__0: Response already existed\n", + "reverie__train__883__0: Response already existed\n", + "reverie__train__884__0: Response already existed\n", + "reverie__train__885__0: Response already existed\n", + "reverie__train__886__0: Response already existed\n", + "reverie__train__887__0: Response already existed\n", + "reverie__train__888__0: Response already existed\n", + "reverie__train__889__0: Response already existed\n", + "reverie__train__890__0: Response already existed\n", + "reverie__train__891__0: Response already existed\n", + "reverie__train__892__0: Response already existed\n", + "reverie__train__893__0: Response already existed\n", + "reverie__train__894__0: Response already existed\n", + "reverie__train__895__0: Response already existed\n", + "reverie__train__896__0: Response already existed\n", + "reverie__train__897__0: Response already existed\n", + "reverie__train__898__0: Response already existed\n", + "reverie__train__899__0: Response already existed\n", + "2023-11-03 18:17:46.941832 900 4150\n", + "reverie__train__900__0: Response already existed\n", + "reverie__train__901__0: Response already existed\n", + "reverie__train__902__0: Response already existed\n", + "reverie__train__903__0: Response already existed\n", + "reverie__train__904__0: Response already existed\n", + "reverie__train__905__0: Response already existed\n", + "reverie__train__906__0: Response already existed\n", + "reverie__train__907__0: Response already existed\n", + "reverie__train__908__0: Response already existed\n", + "reverie__train__909__0: Response already existed\n", + "reverie__train__910__0: Response already existed\n", + "reverie__train__911__0: Response already existed\n", + "reverie__train__912__0: Response already existed\n", + "reverie__train__913__0: Response already existed\n", + "reverie__train__914__0: Response already existed\n", + "reverie__train__915__0: Response already existed\n", + "reverie__train__916__0: Response already existed\n", + "reverie__train__917__0: Response already existed\n", + "reverie__train__918__0: Response already existed\n", + "reverie__train__919__0: Response already existed\n", + "reverie__train__920__0: Response already existed\n", + "reverie__train__921__0: Response already existed\n", + "reverie__train__922__0: Response already existed\n", + "reverie__train__923__0: Response already existed\n", + "reverie__train__924__0: Response already existed\n", + "2023-11-03 18:17:47.278084 925 4150\n", + "reverie__train__925__0: Response already existed\n", + "reverie__train__926__0: Response already existed\n", + "reverie__train__927__0: Response already existed\n", + "reverie__train__928__0: Response already existed\n", + "reverie__train__929__0: Response already existed\n", + "reverie__train__930__0: Response already existed\n", + "reverie__train__931__0: Response already existed\n", + "2023-11-03 18:18:28.823948 950 4150\n", + "2023-11-03 18:29:26.855792 975 4150\n", + "2023-11-03 18:30:16.935044 1000 4150\n", + "2023-11-03 18:41:10.896918 1025 4150\n", + "2023-11-03 18:42:07.502219 1050 4150\n", + "2023-11-03 18:53:03.065171 1075 4150\n", + "2023-11-03 18:53:54.877647 1100 4150\n", + "2023-11-03 18:54:48.791501 1125 4150\n", + "2023-11-03 18:55:43.289232 1150 4150\n", + "2023-11-03 18:56:41.644274 1175 4150\n", + "2023-11-03 18:57:38.180216 1200 4150\n", + "2023-11-03 18:58:38.035448 1225 4150\n", + "2023-11-03 19:09:30.757411 1250 4150\n", + "2023-11-03 19:10:25.609829 1275 4150\n", + "2023-11-03 19:11:26.171085 1300 4150\n", + "2023-11-03 19:12:21.486296 1325 4150\n", + "2023-11-03 19:13:20.494973 1350 4150\n", + "2023-11-03 19:14:14.761849 1375 4150\n", + "2023-11-03 19:15:09.973554 1400 4150\n", + "2023-11-03 19:26:04.641323 1425 4150\n", + "2023-11-03 19:27:01.282772 1450 4150\n", + "2023-11-03 19:27:55.772304 1475 4150\n", + "2023-11-03 19:28:48.837705 1500 4150\n", + "2023-11-03 19:39:42.795517 1525 4150\n", + "2023-11-03 19:40:44.042447 1550 4150\n", + "2023-11-03 19:41:39.169004 1575 4150\n", + "2023-11-03 19:42:35.648877 1600 4150\n", + "2023-11-03 19:43:35.719115 1625 4150\n", + "2023-11-03 19:44:33.633646 1650 4150\n", + "2023-11-03 19:45:27.907864 1675 4150\n", + "2023-11-03 19:46:23.763293 1700 4150\n", + "2023-11-03 19:47:18.141973 1725 4150\n", + "2023-11-03 19:48:18.713898 1750 4150\n", + "2023-11-03 19:49:15.430759 1775 4150\n", + "2023-11-03 19:50:09.554255 1800 4150\n", + "2023-11-03 19:51:05.540993 1825 4150\n", + "2023-11-03 19:51:59.147400 1850 4150\n", + "2023-11-03 19:52:50.535837 1875 4150\n", + "2023-11-03 19:53:47.538290 1900 4150\n", + "2023-11-03 19:54:45.598844 1925 4150\n", + "2023-11-03 19:55:41.487436 1950 4150\n", + "2023-11-03 19:56:35.598625 1975 4150\n", + "2023-11-03 19:57:29.747924 2000 4150\n", + "2023-11-03 19:58:22.559164 2025 4150\n", + "2023-11-03 20:09:19.580777 2050 4150\n", + "2023-11-03 20:10:22.105658 2075 4150\n", + "2023-11-03 20:11:21.982692 2100 4150\n", + "2023-11-03 20:12:16.658451 2125 4150\n", + "2023-11-03 20:13:17.025381 2150 4150\n", + "2023-11-03 20:24:18.237495 2175 4150\n", + "2023-11-03 20:25:17.356171 2200 4150\n", + "2023-11-03 20:36:12.947244 2225 4150\n", + "2023-11-03 20:37:10.440993 2250 4150\n", + "2023-11-03 20:48:07.283968 2275 4150\n", + "2023-11-03 20:49:03.668390 2300 4150\n", + "2023-11-03 20:50:01.510116 2325 4150\n", + "2023-11-03 20:50:52.135443 2350 4150\n", + "2023-11-03 20:51:41.762370 2375 4150\n", + "2023-11-03 21:12:36.635155 2400 4150\n", + "2023-11-03 21:23:30.997602 2425 4150\n", + "2023-11-03 21:24:30.433886 2450 4150\n", + "2023-11-03 21:25:19.994474 2475 4150\n", + "2023-11-03 21:26:11.707307 2500 4150\n", + "2023-11-03 21:27:11.617450 2525 4150\n", + "2023-11-03 21:28:05.615026 2550 4150\n", + "2023-11-03 21:39:03.425169 2575 4150\n", + "2023-11-03 21:50:01.152045 2600 4150\n", + "2023-11-03 22:01:11.406074 2625 4150\n", + "2023-11-03 22:12:15.630115 2650 4150\n", + "2023-11-03 22:23:18.925613 2675 4150\n", + "2023-11-03 22:24:22.198089 2700 4150\n", + "2023-11-03 22:25:21.189951 2725 4150\n", + "2023-11-03 22:26:25.069070 2750 4150\n", + "2023-11-03 22:27:22.743949 2775 4150\n", + "2023-11-03 22:28:27.142443 2800 4150\n", + "2023-11-03 22:29:22.864917 2825 4150\n", + "2023-11-03 22:30:21.736120 2850 4150\n", + "2023-11-03 22:31:25.172575 2875 4150\n", + "2023-11-03 22:32:34.667822 2900 4150\n", + "2023-11-03 22:33:30.707156 2925 4150\n", + "2023-11-03 22:44:30.920467 2950 4150\n", + "2023-11-03 22:45:36.004002 2975 4150\n", + "2023-11-03 22:46:37.877049 3000 4150\n", + "2023-11-03 22:47:37.350368 3025 4150\n", + "2023-11-03 22:48:42.039747 3050 4150\n", + "2023-11-03 22:49:41.122465 3075 4150\n", + "2023-11-03 22:50:37.015301 3100 4150\n", + "2023-11-03 22:51:32.623823 3125 4150\n", + "2023-11-03 22:52:27.413485 3150 4150\n", + "2023-11-03 23:03:24.999827 3175 4150\n", + "2023-11-03 23:04:18.735021 3200 4150\n", + "2023-11-03 23:15:17.407308 3225 4150\n", + "2023-11-03 23:16:23.923856 3250 4150\n", + "2023-11-03 23:27:31.539260 3275 4150\n", + "2023-11-03 23:28:37.650127 3300 4150\n", + "2023-11-03 23:49:42.276140 3325 4150\n", + "2023-11-03 23:50:43.416254 3350 4150\n", + "2023-11-03 23:51:45.940531 3375 4150\n", + "2023-11-03 23:52:53.539010 3400 4150\n", + "2023-11-03 23:53:54.684306 3425 4150\n", + "2023-11-03 23:55:01.449183 3450 4150\n", + "2023-11-03 23:56:02.099644 3475 4150\n", + "2023-11-04 00:07:08.362792 3500 4150\n", + "2023-11-04 00:18:11.960242 3525 4150\n", + "2023-11-04 00:19:11.881544 3550 4150\n", + "2023-11-04 00:20:12.852495 3575 4150\n", + "2023-11-04 00:21:12.440021 3600 4150\n", + "2023-11-04 00:32:16.289157 3625 4150\n", + "2023-11-04 00:33:20.898961 3650 4150\n", + "2023-11-04 00:34:21.252484 3675 4150\n", + "2023-11-04 00:35:17.969487 3700 4150\n", + "2023-11-04 00:36:18.207138 3725 4150\n", + "2023-11-04 00:37:16.416946 3750 4150\n", + "2023-11-04 00:38:15.261926 3775 4150\n", + "2023-11-04 00:39:13.761617 3800 4150\n", + "2023-11-04 00:40:10.796242 3825 4150\n", + "2023-11-04 00:41:12.868656 3850 4150\n", + "2023-11-04 00:42:22.174490 3875 4150\n", + "2023-11-04 00:43:25.801307 3900 4150\n", + "2023-11-04 00:44:25.731902 3925 4150\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-04 00:45:25.986293 3950 4150\n", + "2023-11-04 00:46:25.993308 3975 4150\n", + "2023-11-04 00:47:20.908407 4000 4150\n", + "2023-11-04 00:48:17.180307 4025 4150\n", + "2023-11-04 00:49:13.321578 4050 4150\n", + "2023-11-04 00:50:08.811854 4075 4150\n", + "2023-11-04 00:51:06.818990 4100 4150\n", + "2023-11-04 00:52:00.923809 4125 4150\n" + ] + } + ], + "source": [ + "from datetime import datetime\n", + "for idx, r in enumerate(reverie_train):\n", + " if idx%25==0:\n", + " print(datetime.now(), idx, len(reverie_train))\n", + " prompt = template.replace(\"___inputs___\", r['instruction'])\n", + " #query_name = f'reverie__{r[\"instr_id\"]}'\n", + " query_name = f'reverie__train__{idx}__0'\n", + " response = gpt.chatgpt_query(prompt, query_name, model='gpt-3.5-turbo', max_tokens=512, retry=True)\n", + " #if idx==10:\n", + " #raise" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'room': 'laundryroom',\n", + " 'goal': 'leopard trinket',\n", + " 'goal_relations': [{'relation': 'on', 'reference': 'shelf'},\n", + " {'relation': 'on the first level', 'reference': 'laundryroom'}]}" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json.loads(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Go to the laundryroom on the first level and remove the leopard trinket from the shelf'" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r['instruction']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "#cluster scan_id/room/object mapping\n", + "#swap with same room/w/o that object\n", + "\n", + "#1. pool construction\n", + "def make_distractor_pools(reverie, gpt, split):\n", + " from collections import defaultdict\n", + " room_object_mapping = defaultdict(lambda: defaultdict(list))\n", + "\n", + " for idx, reverie_r in enumerate(reverie):\n", + " query_name = f'reverie__{split}__{idx}__0'\n", + " gpt_r = gpt.db.getByQuery({'query_name': query_name})\n", + " assert len(gpt_r)==1\n", + " gpt_r = gpt_r[0]\n", + " res = json.loads(gpt_r['response'])\n", + " \n", + " #instr_id = r['query_name'].split(\"__\")[-1]\n", + " \n", + " room_object_mapping[reverie_r['scan']][res['room']].append(res['goal'])\n", + "\n", + " for scan_id, rom in room_object_mapping.items():\n", + " for k in rom.keys():\n", + " rom[k] = list(set(rom[k]))\n", + " \n", + " return room_object_mapping\n", + "\n", + "def generate_distractor_examples(reverie, gpt, split, room_object_mapping):\n", + " import copy\n", + " #reverie search and swap\n", + " #make sense to apply the original instructions?\n", + " new_reverie = []\n", + " #for idx, (reverie_r, gpt_r) in enumerate(zip(reverie, gpt.db.getAll())):\n", + " for idx, reverie_r in enumerate(reverie):\n", + " query_name = f'reverie__{split}__{idx}__0'\n", + " gpt_r = gpt.db.getByQuery({'query_name': query_name})\n", + " assert len(gpt_r)==1\n", + " gpt_r = gpt_r[0]\n", + " \n", + " #assert reverie_r['instr_id'] == gpt_r['query_name'].split(\"__\")[-1]\n", + " instr = reverie_r['instruction']\n", + " scan_id = reverie_r['scan']\n", + " response = json.loads(gpt_r['response'])\n", + " #goal = response['goal']\n", + "\n", + " objects_existed_in_room = [obj for obj in room_object_mapping[scan_id][response['room']]]\n", + " distractor_pool = []\n", + " for k,v in room_object_mapping.items():\n", + " if scan_id==k:\n", + " continue\n", + " distractor_pool += v[response['room']]\n", + " distractor_pool = list(set(distractor_pool))\n", + "\n", + " if (len(distractor_pool))==0 or response['goal'] is None:\n", + " print(idx)\n", + " continue\n", + " new_instr = instr.replace(response['goal'], distractor_pool[idx%len(distractor_pool)])\n", + "\n", + " reverie_r['found'] = True\n", + " new_reverie.append(reverie_r)\n", + " new_r = copy.deepcopy(reverie_r)\n", + " new_r['instruction'] = new_instr\n", + " new_r['found'] = False\n", + " new_reverie.append(new_r)\n", + " return new_reverie" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "room_object_mapping = make_distractor_pools(reverie_train, gpt, 'train')\n", + "#reverie_train_with_distractor_examples = generate_distractor_examples(reverie_train, gpt, 'train')" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "9\n", + "15\n", + "25\n", + "40\n", + "62\n", + "64\n", + "65\n", + "120\n", + "128\n", + "140\n", + "156\n", + "170\n", + "191\n", + "193\n", + "201\n", + "237\n", + "238\n", + "241\n", + "275\n", + "276\n", + "293\n", + "295\n", + "299\n", + "328\n", + "339\n", + "340\n", + "341\n", + "353\n", + "364\n", + "368\n", + "374\n", + "380\n", + "387\n", + "399\n", + "405\n", + "414\n", + "417\n", + "426\n", + "438\n", + "456\n", + "457\n", + "460\n", + "465\n", + "497\n", + "519\n", + "547\n", + "551\n", + "554\n", + "558\n", + "583\n", + "607\n", + "640\n", + "661\n", + "663\n", + "679\n", + "736\n", + "760\n", + "786\n", + "789\n", + "803\n", + "812\n", + "828\n", + "868\n", + "879\n", + "905\n", + "915\n", + "924\n", + "945\n", + "948\n", + "953\n", + "974\n", + "999\n", + "1001\n", + "1008\n", + "1020\n", + "1040\n", + "1054\n", + "1061\n", + "1066\n", + "1070\n", + "1079\n", + "1080\n", + "1082\n", + "1114\n", + "1120\n", + "1124\n", + "1125\n", + "1126\n", + "1127\n", + "1135\n", + "1137\n", + "1142\n", + "1145\n", + "1153\n", + "1154\n", + "1181\n", + "1184\n", + "1190\n", + "1201\n", + "1211\n", + "1219\n", + "1226\n", + "1229\n", + "1232\n", + "1234\n", + "1250\n", + "1255\n", + "1257\n", + "1263\n", + "1270\n", + "1279\n", + "1298\n", + "1310\n", + "1314\n", + "1326\n", + "1333\n", + "1343\n", + "1355\n", + "1382\n", + "1413\n", + "1430\n", + "1431\n", + "1438\n", + "1449\n", + "1452\n", + "1460\n", + "1469\n", + "1477\n", + "1482\n", + "1484\n", + "1498\n", + "1512\n", + "1517\n", + "1519\n", + "1550\n", + "1554\n", + "1571\n", + "1580\n", + "1581\n", + "1583\n", + "1587\n", + "1594\n", + "1595\n", + "1596\n", + "1605\n", + "1611\n", + "1613\n", + "1622\n", + "1625\n", + "1627\n", + "1629\n", + "1630\n", + "1635\n", + "1639\n", + "1643\n", + "1650\n", + "1651\n", + "1670\n", + "1686\n", + "1701\n", + "1707\n", + "1710\n", + "1715\n", + "1732\n", + "1757\n", + "1771\n", + "1772\n", + "1773\n", + "1776\n", + "1778\n", + "1792\n", + "1798\n", + "1825\n", + "1827\n", + "1843\n", + "1844\n", + "1849\n", + "1850\n", + "1866\n", + "1871\n", + "1879\n", + "1883\n", + "1894\n", + "1895\n", + "1901\n", + "1969\n", + "1975\n", + "1979\n", + "1984\n", + "1988\n", + "1990\n", + "1994\n", + "2039\n", + "2040\n", + "2057\n", + "2061\n", + "2073\n", + "2078\n", + "2103\n", + "2111\n", + "2147\n", + "2161\n", + "2169\n", + "2199\n", + "2207\n", + "2211\n", + "2218\n", + "2229\n", + "2252\n", + "2258\n", + "2293\n", + "2355\n", + "2357\n", + "2379\n", + "2404\n", + "2408\n", + "2419\n", + "2431\n", + "2437\n", + "2458\n", + "2466\n", + "2481\n", + "2540\n", + "2554\n", + "2578\n", + "2601\n", + "2613\n", + "2621\n", + "2631\n", + "2642\n", + "2660\n", + "2663\n", + "2664\n", + "2670\n", + "2752\n", + "2767\n", + "2770\n", + "2776\n", + "2782\n", + "2783\n", + "2795\n", + "2798\n", + "2807\n", + "2832\n", + "2839\n", + "2842\n", + "2869\n", + "2883\n", + "2885\n", + "2903\n", + "2906\n", + "2908\n", + "2913\n", + "2917\n", + "2918\n", + "2924\n", + "2925\n", + "2926\n", + "2936\n", + "2937\n", + "2938\n", + "2945\n", + "2969\n", + "2990\n", + "2996\n", + "3066\n", + "3080\n", + "3083\n", + "3085\n", + "3104\n", + "3114\n", + "3117\n", + "3119\n", + "3158\n", + "3165\n", + "3169\n", + "3181\n", + "3185\n", + "3192\n", + "3197\n", + "3200\n", + "3220\n", + "3221\n", + "3223\n", + "3233\n", + "3239\n", + "3243\n", + "3249\n", + "3252\n", + "3260\n", + "3261\n", + "3277\n", + "3278\n", + "3281\n", + "3283\n", + "3285\n", + "3294\n", + "3301\n", + "3302\n", + "3309\n", + "3318\n", + "3326\n", + "3328\n", + "3344\n", + "3350\n", + "3358\n", + "3369\n", + "3370\n", + "3379\n", + "3381\n", + "3388\n", + "3389\n", + "3395\n", + "3397\n", + "3410\n", + "3411\n", + "3415\n", + "3421\n", + "3426\n", + "3429\n", + "3431\n", + "3442\n", + "3443\n", + "3458\n", + "3464\n", + "3465\n", + "3479\n", + "3487\n", + "3491\n", + "3503\n", + "3521\n", + "3525\n", + "3528\n", + "3530\n", + "3538\n", + "3541\n", + "3544\n", + "3545\n", + "3547\n", + "3553\n", + "3563\n", + "3570\n", + "3581\n", + "3601\n", + "3602\n", + "3608\n", + "3613\n", + "3630\n", + "3636\n", + "3640\n", + "3660\n", + "3670\n", + "3674\n", + "3679\n", + "3698\n", + "3701\n", + "3702\n", + "3708\n", + "3716\n", + "3721\n", + "3745\n", + "3750\n", + "3768\n", + "3784\n", + "3794\n", + "3798\n", + "3802\n", + "3804\n", + "3806\n", + "3807\n", + "3810\n", + "3822\n", + "3829\n", + "3874\n", + "3905\n", + "3914\n", + "3915\n", + "3919\n", + "3920\n", + "3924\n", + "3925\n", + "3931\n", + "3947\n", + "3950\n", + "3958\n", + "3961\n", + "3963\n", + "3989\n", + "4015\n", + "4022\n", + "4023\n", + "4026\n", + "4032\n", + "4070\n", + "4081\n", + "4084\n", + "4094\n", + "4110\n", + "4112\n", + "4122\n", + "4123\n", + "4128\n", + "4134\n", + "4138\n" + ] + } + ], + "source": [ + "reverie_train_with_distractor_examples = generate_distractor_examples(reverie_train, gpt, 'train')" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(60, 7492)" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(room_object_mapping), len(reverie_train_with_distractor_examples)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "D7N2EKCX4Sj None\n" + ] + } + ], + "source": [ + "for scan_id, rooms in room_object_mapping.items():\n", + " for room_name, objs in rooms.items():\n", + " if None in objs:\n", + " print(scan_id, room_name)\n", + "#D7N2EKCX4Sj None\n", + "del room_object_mapping['D7N2EKCX4Sj'][None]" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [], + "source": [ + "dump_json(reverie_train_with_distractor_examples, './reverie_train_fnf.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "room_object_mapping_val_unseen = make_distractor_pools(reverie_val_unseen, gpt, 'val_unseen')" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "for scan_id, rooms in room_object_mapping_val_unseen.items():\n", + " for room_name, objs in rooms.items():\n", + " if None in objs:\n", + " print(scan_id, room_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12\n", + "37\n", + "48\n", + "55\n", + "59\n", + "60\n", + "62\n", + "65\n", + "67\n", + "68\n", + "81\n", + "83\n", + "88\n", + "92\n", + "93\n", + "109\n", + "110\n", + "112\n", + "119\n", + "121\n", + "127\n", + "133\n", + "137\n", + "138\n", + "141\n", + "142\n", + "148\n", + "154\n", + "169\n", + "171\n", + "187\n", + "193\n", + "202\n", + "203\n", + "205\n", + "212\n", + "218\n", + "225\n", + "232\n", + "234\n", + "240\n", + "241\n", + "245\n", + "254\n", + "258\n", + "263\n", + "271\n", + "277\n", + "288\n", + "301\n", + "305\n", + "325\n", + "327\n", + "332\n", + "333\n", + "344\n", + "348\n", + "354\n", + "356\n", + "357\n", + "360\n", + "362\n", + "367\n", + "370\n", + "379\n", + "387\n", + "391\n", + "393\n", + "398\n", + "408\n", + "427\n", + "453\n", + "455\n", + "456\n", + "457\n", + "469\n", + "477\n", + "481\n", + "507\n", + "509\n", + "520\n", + "528\n", + "534\n", + "535\n", + "537\n", + "539\n", + "566\n", + "570\n", + "582\n", + "587\n", + "598\n", + "599\n", + "603\n", + "605\n", + "613\n", + "617\n", + "630\n", + "633\n", + "634\n", + "640\n", + "648\n", + "653\n", + "662\n", + "664\n", + "669\n", + "677\n", + "681\n", + "694\n", + "702\n", + "703\n", + "704\n", + "706\n", + "720\n", + "725\n", + "731\n", + "753\n", + "757\n", + "780\n", + "803\n", + "806\n", + "807\n", + "809\n", + "814\n", + "822\n", + "823\n", + "845\n", + "846\n", + "849\n", + "850\n", + "852\n", + "854\n", + "856\n", + "865\n", + "871\n", + "873\n", + "874\n", + "877\n", + "878\n", + "882\n", + "917\n", + "925\n", + "929\n", + "940\n", + "941\n", + "942\n", + "962\n", + "964\n", + "968\n", + "972\n", + "990\n", + "994\n", + "997\n", + "1002\n", + "1004\n", + "1006\n", + "1009\n", + "1015\n", + "1018\n", + "1022\n", + "1025\n", + "1028\n", + "1029\n", + "1030\n", + "1032\n", + "1033\n", + "1034\n", + "1036\n", + "1037\n", + "1038\n", + "1044\n", + "1048\n", + "1049\n", + "1050\n", + "1051\n", + "1055\n", + "1056\n", + "1058\n", + "1059\n", + "1060\n", + "1061\n", + "1063\n", + "1064\n", + "1065\n", + "1071\n", + "1073\n", + "1075\n", + "1084\n", + "1085\n", + "1087\n", + "1090\n", + "1092\n", + "1093\n", + "1097\n", + "1103\n", + "1105\n", + "1112\n", + "1121\n", + "1122\n", + "1132\n", + "1141\n", + "1142\n", + "1164\n", + "1165\n", + "1167\n", + "1188\n", + "1193\n", + "1201\n", + "1209\n", + "1225\n", + "1230\n", + "1233\n", + "1236\n", + "1237\n", + "1243\n", + "1247\n", + "1252\n", + "1256\n", + "1263\n", + "1265\n", + "1267\n", + "1289\n", + "1291\n", + "1292\n", + "1293\n", + "1298\n", + "1299\n", + "1303\n", + "1309\n", + "1311\n", + "1312\n", + "1314\n", + "1316\n" + ] + } + ], + "source": [ + "reverie_val_unseen_with_distractor_examples = generate_distractor_examples(reverie_val_unseen, gpt, 'val_unseen', room_object_mapping_val_unseen)\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [], + "source": [ + "dump_json(reverie_val_unseen_with_distractor_examples, './reverie_val_unseen_fnf.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'distance': 10.14,\n", + " 'ix': 137,\n", + " 'scan': 'Z6MFQCViBuw',\n", + " 'heading': 1.605,\n", + " 'instructions_l': ['Continue down the red carpet path past the white statue of two people in the corner two the right. Stop on the path with the two vases on the ground to your right. ',\n", + " 'Walk down the hallway. Wait in front of the next set of doors. ',\n", + " 'Walk forward along red carpet. Stop at the fire extinguisher. '],\n", + " 'path_id': 2853,\n", + " 'objId': 310,\n", + " 'path': ['ae2aeba150fd4fe4b6714b63e1a8ba51',\n", + " '099d792dbfb2402ca355fd7ed860128f',\n", + " '80cd596c480c4f72a7457433b862d179',\n", + " '30cacfc75a8c4f09bde7fc10899f0a53',\n", + " 'e0f34035822e46f7b431406cd82ac871',\n", + " '8acc5cd5a6dd4da1ae3fc3088ff549c2'],\n", + " 'id': '2853_310',\n", + " 'instructions': ['Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls',\n", + " 'Go to the lounge on level 1 with the fireplace and wallpaper panels with plants in blue pots and clean off the rope in front of the black wooden desk'],\n", + " 'instruction': 'Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls',\n", + " 'instr_id': '2853_0',\n", + " 'instr_encodings': None,\n", + " 'found': True}" + ] + }, + "execution_count": 77, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reverie_val_unseen_with_distractor_examples[2]" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'distance': 10.14,\n", + " 'ix': 137,\n", + " 'scan': 'Z6MFQCViBuw',\n", + " 'heading': 1.605,\n", + " 'instructions_l': ['Continue down the red carpet path past the white statue of two people in the corner two the right. Stop on the path with the two vases on the ground to your right. ',\n", + " 'Walk down the hallway. Wait in front of the next set of doors. ',\n", + " 'Walk forward along red carpet. Stop at the fire extinguisher. '],\n", + " 'path_id': 2853,\n", + " 'objId': 310,\n", + " 'path': ['ae2aeba150fd4fe4b6714b63e1a8ba51',\n", + " '099d792dbfb2402ca355fd7ed860128f',\n", + " '80cd596c480c4f72a7457433b862d179',\n", + " '30cacfc75a8c4f09bde7fc10899f0a53',\n", + " 'e0f34035822e46f7b431406cd82ac871',\n", + " '8acc5cd5a6dd4da1ae3fc3088ff549c2'],\n", + " 'id': '2853_310',\n", + " 'instructions': ['Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls',\n", + " 'Go to the lounge on level 1 with the fireplace and wallpaper panels with plants in blue pots and clean off the rope in front of the black wooden desk'],\n", + " 'instruction': 'Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls',\n", + " 'instr_id': '2853_0',\n", + " 'instr_encodings': None,\n", + " 'found': False}" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reverie_val_unseen_with_distractor_examples[3]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "#2. search and swap" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['query_name', 'prompt', 'response', 'id'])" + ] + }, + "execution_count": 67, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json.loads(response)\n", + "r.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12\n", + "37\n", + "48\n", + "55\n", + "59\n", + "60\n", + "62\n", + "65\n", + "67\n", + "68\n", + "81\n", + "83\n", + "88\n", + "92\n", + "93\n", + "109\n", + "110\n", + "112\n", + "119\n", + "121\n", + "127\n", + "133\n", + "137\n", + "138\n", + "141\n", + "142\n", + "148\n", + "154\n", + "169\n", + "171\n", + "187\n", + "193\n", + "202\n", + "203\n", + "205\n", + "212\n", + "218\n", + "225\n", + "232\n", + "234\n", + "240\n", + "241\n", + "245\n", + "254\n", + "258\n", + "263\n", + "271\n", + "277\n", + "288\n", + "301\n", + "305\n", + "325\n", + "327\n", + "332\n", + "333\n", + "344\n", + "348\n", + "354\n", + "356\n", + "357\n", + "360\n", + "362\n", + "367\n", + "370\n", + "379\n", + "387\n", + "391\n", + "393\n", + "398\n", + "408\n", + "427\n", + "453\n", + "455\n", + "456\n", + "457\n", + "469\n", + "477\n", + "481\n", + "507\n", + "509\n", + "520\n", + "528\n", + "534\n", + "535\n", + "537\n", + "539\n", + "566\n", + "570\n", + "582\n", + "587\n", + "598\n", + "599\n", + "603\n", + "605\n", + "613\n", + "617\n", + "630\n", + "633\n", + "634\n", + "640\n", + "648\n", + "653\n", + "662\n", + "664\n", + "669\n", + "677\n", + "681\n", + "694\n", + "702\n", + "703\n", + "704\n", + "706\n", + "720\n", + "725\n", + "731\n", + "753\n", + "757\n", + "780\n", + "803\n", + "806\n", + "807\n", + "809\n", + "814\n", + "822\n", + "823\n", + "845\n", + "846\n", + "849\n", + "850\n", + "852\n", + "854\n", + "856\n", + "865\n", + "871\n", + "873\n", + "874\n", + "877\n", + "878\n", + "882\n", + "917\n", + "925\n", + "929\n", + "940\n", + "941\n", + "942\n", + "962\n", + "964\n", + "968\n", + "972\n", + "990\n", + "994\n", + "997\n", + "1002\n", + "1004\n", + "1006\n", + "1009\n", + "1015\n", + "1018\n", + "1022\n", + "1025\n", + "1028\n", + "1029\n", + "1030\n", + "1032\n", + "1033\n", + "1034\n", + "1036\n", + "1037\n", + "1038\n", + "1044\n", + "1048\n", + "1049\n", + "1050\n", + "1051\n", + "1055\n", + "1056\n", + "1058\n", + "1059\n", + "1060\n", + "1061\n", + "1063\n", + "1064\n", + "1065\n", + "1071\n", + "1073\n", + "1075\n", + "1084\n", + "1085\n", + "1087\n", + "1090\n", + "1092\n", + "1093\n", + "1097\n", + "1103\n", + "1105\n", + "1112\n", + "1121\n", + "1122\n", + "1132\n", + "1141\n", + "1142\n", + "1164\n", + "1165\n", + "1167\n", + "1188\n", + "1193\n", + "1201\n", + "1209\n", + "1225\n", + "1230\n", + "1233\n", + "1236\n", + "1237\n", + "1243\n", + "1247\n", + "1252\n", + "1256\n", + "1263\n", + "1265\n", + "1267\n", + "1289\n", + "1291\n", + "1292\n", + "1293\n", + "1298\n", + "1299\n", + "1303\n", + "1309\n", + "1311\n", + "1312\n", + "1314\n", + "1316\n" + ] + } + ], + "source": [ + "import copy\n", + "#reverie search and swap\n", + "#make sense to apply the original instructions?\n", + "new_reverie = []\n", + "for idx, (reverie_r, gpt_r) in enumerate(zip(reverie, gpt.db.getAll())):\n", + " assert reverie_r['instr_id'] == gpt_r['query_name'].split(\"__\")[-1]\n", + " instr = reverie_r['instruction']\n", + " scan_id = reverie_r['scan']\n", + " response = json.loads(gpt_r['response'])\n", + " #goal = response['goal']\n", + " \n", + " objects_existed_in_room = [obj for obj in room_object_mapping[scan_id][response['room']]]\n", + " distractor_pool = []\n", + " for k,v in room_object_mapping.items():\n", + " if scan_id==k:\n", + " continue\n", + " distractor_pool += v[response['room']]\n", + " distractor_pool = list(set(distractor_pool))\n", + " \n", + " if (len(distractor_pool))==0:\n", + " print(idx)\n", + " continue\n", + " new_instr = instr.replace(response['goal'], distractor_pool[idx%len(distractor_pool)])\n", + " \n", + " reverie_r['found'] = True\n", + " new_reverie.append(reverie_r)\n", + " new_r = copy.deepcopy(reverie_r)\n", + " new_r['instruction'] = new_instr\n", + " new_r['found'] = False\n", + " new_reverie.append(new_r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#check swap results\n", + "#try to implement scan-and-swap (scan room and objects), or just use chatgpt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 117, + "metadata": {}, + "outputs": [], + "source": [ + "#relation swap\n", + "#object swap\n", + "#room swap\n" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1328, 315, 12)" + ] + }, + "execution_count": 114, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(reverie), len(reverie_dict), idx" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [], + "source": [ + "reverie_r\n" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "====== reverie__4634_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Enter the dining room and pick up the objects on the table\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'dining room', 'goal': 'objects', 'goal_relations': [{'relation': 'on', 'reference': 'table'}]}\n", + "====== reverie__2853_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'lounge', 'goal': 'rope', 'goal_relations': [{'relation': 'around', 'reference': 'table'}, {'relation': 'closer to', 'reference': 'walls'}]}\n", + "====== reverie__2804_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the lounge with green walls and polish the candle on the right\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'lounge', 'goal': 'candle', 'goal_relations': [{'relation': 'on the right', 'reference': 'N/A'}, {'relation': 'with green walls', 'reference': 'N/A'}]}\n", + "====== reverie__3503_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the hallway on level 1 next to the kitchen with the grill and turn on the grill\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'hallway', 'goal': 'grill', 'goal_relations': [{'relation': 'next to', 'reference': 'kitchen'}, {'relation': 'on level 1', 'reference': 'hallway'}]}\n", + "====== reverie__3822_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the living room and wipe down the end table\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'living room', 'goal': 'end table', 'goal_relations': [{'relation': 'wipe down', 'reference': 'none'}]}\n", + "====== reverie__6863_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go into the dining room with the bay window and the round table and extinguish the chandelier\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'dining room', 'goal': 'chandelier', 'goal_relations': [{'relation': 'with', 'reference': 'bay window'}, {'relation': 'with', 'reference': 'round table'}]}\n", + "====== reverie__2978_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the kitchen on level 1 and pull out the stool that's second from the left\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'kitchen', 'goal': 'stool', 'goal_relations': [{'relation': 'on level 1', 'reference': ''}, {'relation': 'second from the left', 'reference': ''}]}\n", + "====== reverie__3531_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go into the gym and roll the big ball on the floor\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'gym', 'goal': 'big ball', 'goal_relations': [{'relation': 'on', 'reference': 'floor'}]}\n", + "====== reverie__5138_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the level 2 dining room and pull the chair closest to the living room away from the dining table\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'dining room', 'goal': 'chair', 'goal_relations': [{'relation': 'closest to', 'reference': 'living room'}, {'relation': 'away from', 'reference': 'dining table'}]}\n", + "====== reverie__2204_0\n", + "Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\n", + "Example:\n", + "inputs:\n", + "In the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\n", + "outputs:\n", + "{\n", + " \"room\": \"kitchen\",\n", + " \"goal\": \"ceiling beam\",\n", + " \"goal_relations\": [\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"brown shelves\"\n", + " },\n", + " {\n", + " \"relation\": \"above\",\n", + " \"reference\": \"basket\"\n", + " },\n", + " {\n", + " \"relation\": \"on the top\",\n", + " \"reference\": \"shelf\"\n", + " }\n", + " ]\n", + "}\n", + "Now it is your turn:\n", + "inputs: \n", + "Go to the bathroom that has a praying angel wall decoration and bring me the hand towel on the left side\n", + "outputs:\n", + "\n", + "---\n", + "{'room': 'bathroom', 'goal': 'hand towel', 'goal_relations': [{'relation': 'on the left side', 'reference': 'praying angel wall decoration'}]}\n" + ] + } + ], + "source": [ + "for r in gpt.db.getAll()[:10]:\n", + " print(f\"====== {r['query_name']}\")\n", + " print(r['prompt'])\n", + " print(\"---\")\n", + " print(json.loads(r['response']))" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'distance': 13.81,\n", + " 'ix': 362,\n", + " 'scan': 'zsNo4HB9uLZ',\n", + " 'heading': 5.531,\n", + " 'instructions_l': ['Walk into and through the kitchen. Stop next to the long wooden dining table. ',\n", + " 'Turn around and go straight into the kitchen. Go straight and stop by the table near the window. ',\n", + " 'Go to the kitchen, then to the dining room table inside. '],\n", + " 'path_id': 4634,\n", + " 'objId': 215,\n", + " 'path': ['11bdfcf0a0984901a1d6f60fdd8ee4a6',\n", + " 'a80acc2328154455965a8be4133dee71',\n", + " '9b1e2472265c46989eb1c3911aa5971b',\n", + " 'de7a74588c2842e9ba391abb153a0257',\n", + " '6416ac70316f44d1ae7ec5a3029b1703',\n", + " '487a4cc75db94e56aa4d1d35866736fc',\n", + " '0753202108e24c0094f09c60b8f36127'],\n", + " 'id': '4634_215',\n", + " 'instructions': ['Enter the dining room and pick up the objects on the table',\n", + " 'go to the dining room and pickup all the objects on the table inside and bring them to me',\n", + " 'Rearrange the objects on the dining room table'],\n", + " 'instruction': 'Enter the dining room and pick up the objects on the table',\n", + " 'instr_id': '4634_0',\n", + " 'instr_encodings': None}" + ] + }, + "execution_count": 90, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reverie[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'query_name': 'reverie__4634_0',\n", + " 'prompt': 'Please extract the target room, the goal object, and the relations between the goal object and other reference objects.\\nExample:\\ninputs:\\nIn the kitchen above the brown shelves and above the basket on the top shelf there is a ceiling beam Please firm this beam\\noutputs:\\n{\\n \"room\": \"kitchen\",\\n \"goal\": \"ceiling beam\",\\n \"goal_relations\": [\\n {\\n \"relation\": \"above\",\\n \"reference\": \"brown shelves\"\\n },\\n {\\n \"relation\": \"above\",\\n \"reference\": \"basket\"\\n },\\n {\\n \"relation\": \"on the top\",\\n \"reference\": \"shelf\"\\n }\\n ]\\n}\\nNow it is your turn:\\ninputs: \\nEnter the dining room and pick up the objects on the table\\noutputs:\\n',\n", + " 'response': '{\\n \"room\": \"dining room\",\\n \"goal\": \"objects\",\\n \"goal_relations\": [\\n {\\n \"relation\": \"on\",\\n \"reference\": \"table\"\\n }\\n ]\\n}',\n", + " 'id': 126150531885790335}" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#visualization tools\n", + "gpt.db.getAll()[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'room': 'bathroom',\n", + " 'goal': 'window',\n", + " 'goal_relations': [{'relation': 'on the second level',\n", + " 'reference': 'bathroom'},\n", + " {'relation': 'with a sink countertop', 'reference': 'bathroom'},\n", + " {'relation': 'in the far-left corner', 'reference': 'sink countertop'},\n", + " {'relation': 'where the vase touches the mirror',\n", + " 'reference': 'far-left corner'}]}" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json.loads(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Go to the bathroom on the second level that has a sink countertop with a lime-green vase in the far-left corner where the vase touches the mirror and open the window'" + ] + }, + "execution_count": 36, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r['instruction']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "minigpt4", + "language": "python", + "name": "minigpt4" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/visualization/.ipynb_checkpoints/visualizer-checkpoint.ipynb b/visualization/.ipynb_checkpoints/visualizer-checkpoint.ipynb new file mode 100644 index 0000000..00995d0 --- /dev/null +++ b/visualization/.ipynb_checkpoints/visualizer-checkpoint.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#1. load skybox image of the path[-3:]\n", + "#2. display instruction, adversarial instruction, (navgpt captions and objects)\n", + "\n", + "from PIL import Image\n", + "import json, os\n", + "\n", + "def load_json(fn):\n", + " with open(fn) as f:\n", + " ret = json.load(f)\n", + " return ret\n", + "\n", + "def dump_json(data, fn, force=False):\n", + " if not force:\n", + " assert not os.path.exists(fn)\n", + " with open(fn, 'w') as f:\n", + " json.dump(data, f)\n", + " \n", + "def concat_images(images):\n", + " #images = [Image.open(x) for x in ['Test1.jpg', 'Test2.jpg', 'Test3.jpg']]\n", + " widths, heights = zip(*(i.size for i in images))\n", + "\n", + " total_width = sum(widths)+(len(images)-1)*10\n", + " max_height = max(heights)\n", + "\n", + " new_im = Image.new('RGB', (total_width, max_height))\n", + "\n", + " x_offset = 0\n", + " for im in images:\n", + " new_im.paste(im, (x_offset,0))\n", + " x_offset += im.size[0]+10\n", + " return new_im" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')\n", + "reverie_val_unseen_fnf = load_json(\"..//fnf/reverie_val_unseen_fnf.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "fnf_pairs = []\n", + "for idx, r in enumerate(reverie_val_unseen_fnf):\n", + " if not r['found']:\n", + " fnf_pairs.append((reverie_val_unseen_fnf[idx-1], r))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "idx = 1\n", + "fnf_pair = fnf_pairs[idx]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scan_id, path = fnf_pair[0]['scan'], fnf_pair[0]['path']\n", + "fn_template = '/work/ganymede9487/mp3d/unzipped/{}/matterport_skybox_images/{}_skybox{}_sami.jpg'\n", + "skybox_image_files = [fn_template.format(scan_id, path[-1], skybox_idx) for skybox_idx in range(6)]\n", + "skybox_images = []\n", + "for skybox_image_file in skybox_image_files:\n", + " skybox_images.append(Image.open(skybox_image_file))\n", + "\n", + "concat_images(skybox_images)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n", + "Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls\n" + ] + } + ], + "source": [ + "print(fnf_pair[0]['instruction'])\n", + "print(fnf_pair[1]['instruction'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#0 ok\n", + "#1 error 2 (not reasonable)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "minigpt4", + "language": "python", + "name": "minigpt4" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/visualization/visualizer.ipynb b/visualization/visualizer.ipynb new file mode 100644 index 0000000..00995d0 --- /dev/null +++ b/visualization/visualizer.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#1. load skybox image of the path[-3:]\n", + "#2. display instruction, adversarial instruction, (navgpt captions and objects)\n", + "\n", + "from PIL import Image\n", + "import json, os\n", + "\n", + "def load_json(fn):\n", + " with open(fn) as f:\n", + " ret = json.load(f)\n", + " return ret\n", + "\n", + "def dump_json(data, fn, force=False):\n", + " if not force:\n", + " assert not os.path.exists(fn)\n", + " with open(fn, 'w') as f:\n", + " json.dump(data, f)\n", + " \n", + "def concat_images(images):\n", + " #images = [Image.open(x) for x in ['Test1.jpg', 'Test2.jpg', 'Test3.jpg']]\n", + " widths, heights = zip(*(i.size for i in images))\n", + "\n", + " total_width = sum(widths)+(len(images)-1)*10\n", + " max_height = max(heights)\n", + "\n", + " new_im = Image.new('RGB', (total_width, max_height))\n", + "\n", + " x_offset = 0\n", + " for im in images:\n", + " new_im.paste(im, (x_offset,0))\n", + " x_offset += im.size[0]+10\n", + " return new_im" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "reverie_val_unseen = load_json('../REVERIE/tasks/REVERIE/data/REVERIE_val_unseen_first_ins.json')\n", + "reverie_val_unseen_fnf = load_json(\"..//fnf/reverie_val_unseen_fnf.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "fnf_pairs = []\n", + "for idx, r in enumerate(reverie_val_unseen_fnf):\n", + " if not r['found']:\n", + " fnf_pairs.append((reverie_val_unseen_fnf[idx-1], r))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "idx = 1\n", + "fnf_pair = fnf_pairs[idx]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scan_id, path = fnf_pair[0]['scan'], fnf_pair[0]['path']\n", + "fn_template = '/work/ganymede9487/mp3d/unzipped/{}/matterport_skybox_images/{}_skybox{}_sami.jpg'\n", + "skybox_image_files = [fn_template.format(scan_id, path[-1], skybox_idx) for skybox_idx in range(6)]\n", + "skybox_images = []\n", + "for skybox_image_file in skybox_image_files:\n", + " skybox_images.append(Image.open(skybox_image_file))\n", + "\n", + "concat_images(skybox_images)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Go to the lounge on level 1 with the fire extinguisher and push the rope around the table closer to the walls\n", + "Go to the lounge on level 1 with the fire extinguisher and push the towel around the table closer to the walls\n" + ] + } + ], + "source": [ + "print(fnf_pair[0]['instruction'])\n", + "print(fnf_pair[1]['instruction'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#0 ok\n", + "#1 error 2 (not reasonable)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "minigpt4", + "language": "python", + "name": "minigpt4" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}