update trainint tasks

README.md

@@ -3,9 +3,9 @@
 
 ### Lirui Wang, Yiyang Ling, Zhecheng Yuan, Mohit Shridhar, Chen Bao, Yuzhe Qin, Bailin Wang, Huazhe Xu, Xiaolong Wang
 
-[Project Page](https://liruiw.github.io/gensim) | [Arxiv](https://arxiv.org/abs/2310.01361) | [Gradio Demo](https://huggingface.co/spaces/Gen-Sim/Gen-Sim) | [Huggingface Dataset](https://huggingface.co/datasets/Gen-Sim/Gen-Sim) | [Finetuned Code-LLama Model](https://huggingface.co/Gen-Sim/Gen-Sim)
+[Project Page](https://liruiw.github.io/gensim) | [Arxiv](https://arxiv.org/abs/2310.01361) | [Gradio Demo](https://huggingface.co/spaces/Gen-Sim/Gen-Sim) | [Huggingface Dataset](https://huggingface.co/datasets/Gen-Sim/Gen-Sim) | [Finetuned Code-LLama Model](https://huggingface.co/Gen-Sim/Gen-Sim) | [GPTs](https://chat.openai.com/g/g-rqxeNpjxd-gensim)
 
-This repo explores the use of an LLM code generation pipeline to write simulation environments and expert goals to augment diverse simulation tasks. Strongly recommend also checking out the [Gradio Demo](https://huggingface.co/spaces/Gen-Sim/Gen-Sim).
+This repo explores the use of an LLM code generation pipeline to write simulation environments and expert goals to augment diverse simulation tasks. Strongly recommend also checking out the [Gradio Demo](https://huggingface.co/spaces/Gen-Sim/Gen-Sim) and [GPTs](https://chat.openai.com/g/g-rqxeNpjxd-gensim).
 
 
 ![](media/gensim_teaser_v1.gif)
@@ -65,12 +65,13 @@ python gensim/run_simulation.py  disp=True  prompt_folder=topdown_chain_of_thoug
 
 8. offline eval: `python -m gensim.evaluate_finetune_model_offline model_output_dir=after_finetune_CodeLlama-13b-Instruct-hf_fewshot_False_epoch_10_0`
 
-## 🤖 Policy Training Benchmark
-0. Note that the 100+ generated tasks by GenSim can be used for benchmarking algorithms in multitask policy training. See `prompts/policy_training_list.json` for a list of training tasks.
+## 🤖 Policy Benchmark
+0. Note that the 100+ generated tasks by GenSim can be used for benchmarking algorithms in multitask policy training. See `scripts/task_list/GPT_*.json` for a list of benchmark settings. Pretrained multitask models can be found [here](https://drive.google.com/drive/folders/1RRSa4hXQKuN1ABuUVEdfV6urqZ99KZ57?usp=drive_link).
 1. Generate multitask demonstrations. For example, run  `bash scripts/generate_datasets.sh data 'align-box-corner assembling-kits block-insertion' `
-2. Single-task training  `sh scripts/train_test_multi_task.sh data "[align-rope,align-box-corner] `
+2. Single-task training  `sh scripts/train_test_multi_task.sh data "[align-rope,align-box-corner]`
 3. Multi-task training   `sh scripts/train_test_single_task.sh data align-box-corner`
 
+
 ## ✅ Note
 0. Temperature `0.5-0.8 `is good range for diversity, `0.0-0.2` is for stable results.
 1. The generation pipeline will print out statistics regarding compilation, runtime, task design, and diversity scores. Note that these metric depend on the task compexity that LLM tries to generate.

cliport/environments/environment.py

@@ -187,6 +187,10 @@ def add_object(self, urdf, pose, category='rigid', color=None, **kwargs):
           print(f"missing urdf error: {os.path.join(self.assets_root, urdf)}. use dummy block.")
           urdf = 'stacking/block.urdf'
 
+        if len(pose) == 3 and (not hasattr(pose[0], '__len__')):
+            # add default orientation if missing
+            pose = (pose, (0,0,0,1))
+
         obj_id = pybullet_utils.load_urdf(
             p,
             os.path.join(self.assets_root, urdf),

cliport/tasks/task.py

@@ -27,6 +27,7 @@ def __init__(self):
         self.sixdof = False
         self.primitive = primitives.PickPlace()
         self.oracle_cams = cameras.Oracle.CONFIG
+        self.rng = None
 
         # Evaluation epsilons (for pose evaluation metric).
         self.pos_eps = 0.01

cliport/utils/utils.py

@@ -20,6 +20,7 @@
 import os
 import torch
 import torchvision
+from collections import defaultdict
 
 
 # -----------------------------------------------------------------------------
@@ -1047,6 +1048,7 @@ def KDTree(node, min_object_dim, margin, bboxes):
     'gold': [255.0 / 255.0, 215.0 / 255.0, 0.0 / 255.0],    
 
 }
+COLORS = defaultdict(lambda: [255.0 / 255.0, 0.0, 0.], COLORS)
 
 COLORS_NAMES = list(COLORS.keys())
 TRAIN_COLORS = ['blue', 'red', 'green', 'yellow', 'brown', 'gray', 'cyan']

gensim/agent.py

@@ -130,9 +130,6 @@ def template_reference_prompt(self):
                     task_code_reference_replace_prompt += f'```\n{self.memory.online_code_buffer[key]}\n```\n\n'
                 else:
                     print("missing task reference code:", key)
-        else:
-            task_code_reference_replace_prompt = sample_list_reference(base_task_codes, sample_num=cfg['task_code_candidate_num'])
-        # print("Template Reference Code PROMPT: ", task_code_reference_replace_prompt)
 
         return task_code_reference_replace_prompt
 

gensim/run_simulation.py

@@ -42,7 +42,7 @@ def main(cfg):
         simulation_runner.task_creation()
         simulation_runner.simulate_task()
         simulation_runner.print_current_stats()
-        # clear_messages()
+        clear_messages()
 
     simulation_runner.save_stats()
 

scripts/task_list/CLIPort_5.json

@@ -0,0 +1 @@
+["place-red-in-green","stack-block-pyramid","align-box-corner","packing-boxes","block-insertion"]

scripts/task_list/GPT_20.json

@@ -0,0 +1 @@
+["insert-blocks-lineup", "color-coordinated-insertion", "insert-ell-along-square-path", "place-ball-in-elevated-bowl", "ball-in-bowl-obstacle-course-new", "vertical-insertion-blocks", "ball-sorting-with-blocks-barrier", "color-ordered-insertion-new", "colored-balls-sorting-in-corner", "corner-sort-cylinders", "color-coordinated-cylinder-tower", "sphere-align-stand", "align-cylinders-in-zones", "color-coordinated-box-ball-matching", "color-cued-ball-corner-sorting", "color-specific-container-fill", "align-balls-in-colored-boxes", "stack-three-layer-red-wall", "create-pyramid-blocks-and-container", "block-on-cylinder-on-pallet"]

scripts/task_list/GPT_50.json

@@ -0,0 +1 @@
+["color-coordinated-box-ball-matching", "ball-on-box-on-container", "insert-sphere-into-container", "sphere-container-color-match", "colored-balls-sorting-in-corner", "color-cued-ball-corner-sorting", "place-ball-in-elevated-bowl", "stack-blocks-in-container", "sorting-blocks-into-pallets", "ball-sorting-with-blocks-barrier", "put-kit-in-bowl", "ball-in-bowl-obstacle-course-new", "color-ordered-blocks-on-pallet", "color-coordinated-block-shifting", "color-coordinated-zone-arrangement", "color-coordinated-insertion", "sort-insert-color-coordinated-blocks", "color-coordinated-sphere-insertion", "insert-blocks-lineup", "color-specific-container-fill", "color-coordinated-zone-stacking", "color-coordinated-sphere-on-pallet-pyramid", "color-coordinated-block-tower", "create-pyramid-blocks-and-container", "stack-color-coordinated-blocks", "colorful-block-tower-on-cylinder-base", "color-coordinated-cylinder-pyramid", "block-pyramid-with-limited-space", "multicolor-block-bridge", "construct-colorful-arch", "mixed-color-block-barrier-insertion", "insert-ell-along-square-path", "block-on-cylinder-on-pallet", "vertical-insertion-blocks", "guided-block-path", "color-ordered-insertion-new", "color-blocks-in-cylinder-maze", "colored-cylinder-in-square", "construct-corner-blocks", "build-car", "corner-sort-cylinders", "place-blue-on-line-ends", "align-pair-colored-blocks-along-line", "align-rope-along-line", "align-balls-in-colored-boxes", "sphere-align-stand", "align-rope-cross-zone", "kit-in-bowl-in-zone", "sort-and-stack-clr-blocks", "color-ordered-container-arrangement"]