################### The following prompts are used in Open Jar ################### (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "planner" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: grasp the jar with right hand and grasp the lid of the jar with left hand to unscrew it in an anti_clockwise direction until it is removed from the jar. composer("move the right hand 10 cm behind of the jar") composer("grasp the jar with right hand") composer("grasp the lid of the jar with left hand") composer("turn left hand counter-clockwise by 90 degrees") composer("move left hand upward for 6cm") (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move the right hand 10 cm behind of the jar. movable = parse_query_obj('right hand') affordance_map = get_affordance_map('a point 10 cm behind of and 5 cm above the jar') gripper_map = get_gripper_map('open everywhere') rotation_map = get_rotation_map('face the jar and turn counter-clockwise by 270 degrees and down by 90 degrees') execute(movable, rotation_map=rotation_map, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 10 cm behind of and 5 cm above the jar. # 10 cm behind of and 5 cm above the jar affordance_map = get_empty_affordance_map() jar = parse_query_obj('jar') (min_x, min_y, min_z), (max_x, max_y, max_z) = jar.aabb center_x, center_y, center_z = jar.position x = center_x # 10 cm behind so we subtract from min_y y = min_y - cm2index(10, 'y') # 5 above so we add to center_z z = center_z + cm2index(5, 'z') affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: face the jar and turn counter-clockwise by 270 degrees and down by 90 degrees. rotation_map = get_empty_rotation_map() jar = parse_query_obj('jar') curr_rotation = pointat2quat(-jar.normal) rotation_delta = euler2quat(0, -np.radians(90), -np.radians(270)) rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta) ret_val = rotation_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: jar. jar = detect('jar') ret_val = jar (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: jar. jar = detect('jar') ret_val = jar (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the jar with right hand. movable = parse_query_obj('right hand') affordance_map = get_affordance_map('a point 5cm below the center of the jar') gripper_map = get_gripper_map('open everywhere except 1cm around the jar') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 5cm below the center of the jar. affordance_map = get_empty_affordance_map() jar = parse_query_obj('jar') x, y, z = jar.position z = z - cm2index(5, 'z') affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the jar. # open everywhere except 1cm around the jar gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 1cm around the jar jar = parse_query_obj('jar') set_voxel_by_radius(gripper_map, jar.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: jar. jar = detect('jar') ret_val = jar (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: jar. jar = detect('jar') ret_val = jar (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the lid of the jar with left hand. movable = parse_query_obj('left hand') affordance_map = get_affordance_map('a point 2cm below the center of the lid') gripper_map = get_gripper_map('open everywhere except 1cm around the lid') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 2cm below the center of the lid. lid = parse_query_obj('lid') x, y, z = lid.position z = z - cm2index(2, 'z') affordance_map = get_empty_affordance_map() affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the lid. lid = parse_query_obj('lid') gripper_map = get_empty_gripper_map() gripper_map[:, :, :] = 1 set_voxel_by_radius(gripper_map, lid.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: lid. lid = detect('lid') ret_val = lid (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: lid. lid = detect('lid') ret_val = lid (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: turn left hand counter-clockwise by 90 degrees. movable = parse_query_obj('left hand') rotation_map = get_rotation_map('turn counter-clockwise by 90 degrees') execute(movable, rotation_map=rotation_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: turn counter-clockwise by 90 degrees. rotation_map = get_empty_rotation_map() curr_rotation = rotation_map[0, 0, 0] rotation_delta = euler2quat(0, 0, -np.radians(90)) rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta) ret_val = rotation_map [interfaces.py | 15:51:9] finished executing path via controller (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move left hand upward for 6cm. movable = parse_query_obj('left hand') affordance_map = get_affordance_map(f'a point 6cm above {movable.position}') execute(movable, affordance_map=affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['jar', 'lid']" ######################################## objects = ['jar', 'lid'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 6cm above [55 55 9]. affordance_map = get_empty_affordance_map() position = np.array([55, 55, 9]) x, y, z = position z = z + cm2index(6, 'z') affordance_map[x, y, z] = 1 ret_val = affordance_map ################### The following prompts are used in Open Drawer ################### (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "planner" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: hold the drawer with right hand and open the bottom drawer with left hand. composer("hold the top drawer with right hand") composer("grasp the bottom drawer handle with left hand while moving at 0.1x speed") composer("move left hand away from the bottom drawer handle by 15cm, at 0.1x speed") # done (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: hold the top drawer with right hand. movable = parse_query_obj('right hand') affordance_map = get_affordance_map('a point in the center offset by 10 cm in y-axis and 1cm below the top drawer') gripper_map = get_gripper_map('open everywhere except 1cm above the top drawer') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point in the center offset by 10 cm in y-axis and 1cm below the top drawer. affordance_map = get_empty_affordance_map() top_drawer = parse_query_obj('top drawer') (min_x, min_y, min_z), (max_x, max_y, max_z) = top_drawer.aabb center_x, center_y, center_z = top_drawer.position x = center_x # in the center offset by 10 cm so we subtract from center_y y = center_y - cm2index(10, 'y') # 1cm below so we subtract from max_z z = max_z - cm2index(1, 'z') affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm above the top drawer. # open everywhere gripper_map = get_empty_gripper_map() gripper_map[:, :, :] = 1 # close when 1cm above the top drawer top_drawer = parse_query_obj('top drawer') (min_x, min_y, min_z), (max_x, max_y, max_z) = top_drawer.aabb center_x, center_y, center_z = top_drawer.position x = center_x y = center_y z = max_z + cm2index(1, 'z') set_voxel_by_radius(gripper_map, (x, y, z), radius_cm=0, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: top drawer. top_drawer = detect('top drawer') ret_val = top_drawer (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: top drawer. top_drawer = detect('top drawer') ret_val = top_drawer (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the bottom drawer handle with left hand while moving at 0.1x speed. movable = parse_query_obj('left hand') affordance_map = get_affordance_map('a point at the center of the bottom drawer handle') velocity_map = get_velocity_map('0.1x speed') rotation_map = get_rotation_map('face the bottom drawer handle') gripper_map = get_gripper_map('open everywhere except 1cm around the bottom drawer handle') execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, rotation_map=rotation_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point at the center of the bottom drawer handle. affordance_map = get_empty_affordance_map() bottom_drawer_handle = parse_query_obj('bottom drawer handle') x, y, z = bottom_drawer_handle.position affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_velocity_map" generated code ######################################## # Query: 0.1x speed. velocity_map = get_empty_velocity_map() velocity_map[:] = 0.1 ret_val = velocity_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: face the bottom drawer handle. rotation_map = get_empty_rotation_map() bottom_drawer_handle = parse_query_obj('bottom drawer handle') target_rotation = pointat2quat(-bottom_drawer_handle.normal) rotation_map[:, :, :] = target_rotation ret_val = rotation_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the bottom drawer handle. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 1cm around the bottom drawer handle bottom_drawer_handle = parse_query_obj('bottom drawer handle') set_voxel_by_radius(gripper_map, bottom_drawer_handle.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: bottom drawer handle. bottom_drawer_handle = detect('bottom drawer handle') ret_val = bottom_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: bottom drawer handle. bottom_drawer_handle = detect('bottom drawer handle') ret_val = bottom_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: bottom drawer handle. bottom_drawer_handle = detect('bottom drawer handle') ret_val = bottom_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move left hand away from the bottom drawer handle by 15cm, at 0.1x speed. movable = parse_query_obj('left hand') velocity_map = get_velocity_map('0.1x speed') affordance_map = get_affordance_map('a point 15cm away from the bottom drawer handle') execute(movable, velocity_map=velocity_map, affordance_map=affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_velocity_map" generated code ######################################## # Query: 0.1x speed. velocity_map = get_empty_velocity_map() velocity_map[:] = 0.1 ret_val = velocity_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 15cm away from the bottom drawer handle. affordance_map = get_empty_affordance_map() bottom_drawer_handle = parse_query_obj('bottom drawer handle') # positive normal because we are moving away from the handle. moving_dir = bottom_drawer_handle.normal # add to the moving_dir affordance_xyz = bottom_drawer_handle.position + cm2index(15, moving_dir) affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer', 'bottom drawer handle']" ######################################## objects = ['top drawer', 'bottom drawer handle'] # Query: bottom drawer handle. bottom_drawer_handle = detect('bottom drawer handle') ret_val = bottom_drawer_handle ################### The following prompts are used in Put Item in Drawer ################### (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "planner" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: open the top drawer with right hand and put the item in the top drawer with left hand. composer("grasp the top drawer handle with right hand while moving at 0.1x speed") composer("move right hand away from the top drawer handle by 15cm, at 0.1x speed") composer("grasp the item with left hand") composer("move left hand to a point 5 cm into and 8 cm above the top drawer handle and open gripper when it's 8 cm behind of the top drawer handle") # done (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the top drawer handle with right hand while moving at 0.1x speed. movable = parse_query_obj('right hand') affordance_map = get_affordance_map('a point at the center of top drawer handle') velocity_map = get_velocity_map('0.1x speed') rotation_map = get_rotation_map('face the top drawer handle') gripper_map = get_gripper_map('open everywhere except 0.1cm around the top drawer handle') execute(movable, affordance_map=affordance_map, velocity_map=velocity_map, rotation_map=rotation_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point at the center of top drawer handle. affordance_map = get_empty_affordance_map() top_drawer_handle = parse_query_obj('top drawer handle') center_x, center_y, center_z = top_drawer_handle.position affordance_map[center_x, center_y, center_z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_velocity_map" generated code ######################################## # Query: 0.1x speed. velocity_map = get_empty_velocity_map() velocity_map[:] = 0.1 ret_val = velocity_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: face the top drawer handle. rotation_map = get_empty_rotation_map() top_drawer_handle = parse_query_obj('top drawer handle') target_rotation = pointat2quat(-top_drawer_handle.normal) rotation_map[:, :, :] = target_rotation ret_val = rotation_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 0.1cm around the top drawer handle. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 0.1cm around the top drawer handle top_drawer_handle = parse_query_obj('top drawer handle') set_voxel_by_radius(gripper_map, top_drawer_handle.position, radius_cm=0.1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move right hand away from the top drawer handle by 15cm, at 0.1x speed. movable = parse_query_obj('right hand') velocity_map = get_velocity_map('0.1x speed') affordance_map = get_affordance_map('a point 15cm away from the top drawer handle') execute(movable, velocity_map=velocity_map, affordance_map=affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_velocity_map" generated code ######################################## # Query: 0.1x speed. velocity_map = get_empty_velocity_map() velocity_map[:] = 0.1 ret_val = velocity_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 15cm away from the top drawer handle. affordance_map = get_empty_affordance_map() top_drawer_handle = parse_query_obj('top drawer handle') # positive normal because we are moving away from the handle. moving_dir = top_drawer_handle.normal # add to the moving_dir affordance_xyz = top_drawer_handle.position + cm2index(15, moving_dir) affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the item with left hand. movable = parse_query_obj('left hand') affordance_map = get_affordance_map('a point at the center of the item') gripper_map = get_gripper_map('open everywhere except 1cm around the item') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point at the center of the item. affordance_map = get_empty_affordance_map() item = parse_query_obj('item') x, y, z = item.position affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the item. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 1cm around the item item = parse_query_obj('item') set_voxel_by_radius(gripper_map, item.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: item. item = detect('item') ret_val = item (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: item. item = detect('item') ret_val = item (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move left hand to a point 5 cm into and 8 cm above the top drawer handle and open gripper when it's 8 cm behind of the top drawer handle. movable = parse_query_obj('left hand') affordance_map = get_affordance_map('a point 5 cm into and 8 cm above the top drawer handle') gripper_map = get_gripper_map('close everywhere except 8cm around the top drawer handle') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 5 cm into and 8 cm above the top drawer handle. affordance_map = get_empty_affordance_map() top_drawer_handle = parse_query_obj('top drawer handle') center_x, center_y, center_z = top_drawer_handle.position x = center_x y = center_y # 8 above so we add to center_z z = center_z + cm2index(8, 'z') # negative normal because we are moving into the handle. moving_dir = -top_drawer_handle.normal # add to the moving_dir affordance_xyz = [x, y, z] + cm2index(5, moving_dir) affordance_map[affordance_xyz[0], affordance_xyz[1], affordance_xyz[2]] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: close everywhere except 8cm around the top drawer handle. gripper_map = get_empty_gripper_map() # close everywhere gripper_map[:, :, :] = 0 # open when 8cm around the top drawer handle top_drawer_handle = parse_query_obj('top drawer handle') set_voxel_by_radius(gripper_map, top_drawer_handle.position, radius_cm=8, value=1) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['top drawer handle', 'item']" ######################################## objects = ['top drawer handle', 'item'] # Query: top drawer handle. top_drawer_handle = detect('top drawer handle') ret_val = top_drawer_handle ################### The following prompts are used in Hand Over Item ################### (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "planner" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: grasp the block with left hand and hand it over to the right hand. composer("grasp the cube with left hand") composer("move left hand upward for 20cm") composer("rotate 90 degrees upward and sideway 90 degrees with left hand") composer("rotate right hand 90 degrees upward") composer("move right hand down for 40cm") composer("grasp the cube with right hand") composer("open left hand gripper") composer("move right hand forward for 20cm") # done (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the cube with left hand. movable = parse_query_obj('left hand') affordance_map = get_affordance_map('a point at the center of the cube') gripper_map = get_gripper_map('open everywhere except 1cm around the cube') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point at the center of the cube. affordance_map = get_empty_affordance_map() cube = parse_query_obj('cube') x, y, z = cube.position affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the cube. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 1cm around the cube cube = parse_query_obj('cube') set_voxel_by_radius(gripper_map, cube.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: cube. cube = detect('cube') ret_val = cube (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: cube. cube = detect('cube') ret_val = cube (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move left hand upward for 20cm. movable = parse_query_obj('left hand') affordance_map = get_affordance_map(f'a point 20cm above {movable.position}') execute(movable, affordance_map=affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand *** OpenAI API call took 1.22s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 20cm above [63 44 4]. affordance_map = get_empty_affordance_map() x = 63 y = 44 z = cm2index(4, 'z') + 20 affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: rotate 90 degrees upward and sideway 90 degrees with left hand. movable = parse_query_obj('left hand') rotation_map = get_rotation_map('rotate 90 degrees upward and sideway 90 degrees with left hand') execute(movable, rotation_map=rotation_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: rotate 90 degrees upward and sideway 90 degrees with left hand. rotation_map = get_empty_rotation_map() curr_rotation = rotation_map[0, 0, 0] rotation_delta = euler2quat(-np.radians(90), 0, np.radians(90)) rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta) ret_val = rotation_map [interfaces.py | 23:32:47] finished executing path via controller (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: rotate right hand 90 degrees upward. movable = parse_query_obj('right hand') rotation_map = get_rotation_map('rotate 90 degrees upward') execute(movable, rotation_map=rotation_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_rotation_map" generated code ######################################## # Query: rotate 90 degrees upward. rotation_map = get_empty_rotation_map() curr_rotation = rotation_map[0, 0, 0] rotation_delta = euler2quat(0, -np.radians(90), 0) rotation_map[:, :, :] = qmult(curr_rotation, rotation_delta) ret_val = rotation_map [interfaces.py | 23:32:53] finished executing path via controller (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move right hand down for 40cm. movable = parse_query_obj('right hand') affordance_map = get_affordance_map(f'a point 40cm below {movable.position}') execute(movable, affordance_map=affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 40cm below [52 49 71]. affordance_map = get_empty_affordance_map() x = 52 y = 49 z = cm2index(71, 'z') - 40 affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: grasp the cube with right hand. movable = parse_query_obj('right hand') affordance_map = get_affordance_map('a point at the center of the cube') gripper_map = get_gripper_map('open everywhere except 1cm around the cube') execute(movable, affordance_map=affordance_map, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point at the center of the cube. affordance_map = get_empty_affordance_map() cube = parse_query_obj('cube') x, y, z = cube.position affordance_map[x, y, z] = 1 ret_val = affordance_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere except 1cm around the cube. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 # close when 1cm around the cube cube = parse_query_obj('cube') set_voxel_by_radius(gripper_map, cube.position, radius_cm=1, value=0) ret_val = gripper_map (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: cube. cube = detect('cube') ret_val = cube (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: cube. cube = detect('cube') ret_val = cube (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: open left hand gripper. movable = parse_query_obj('left hand') gripper_map = get_gripper_map('open everywhere') execute(movable, gripper_map=gripper_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: left hand. left_hand = detect('left hand') ret_val = left_hand (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "get_gripper_map" generated code ######################################## # Query: open everywhere. gripper_map = get_empty_gripper_map() # open everywhere gripper_map[:, :, :] = 1 ret_val = gripper_map [interfaces.py | 23:33:35] finished executing path via controller (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "composer" generated code ######################################## # Query: move right hand forward for 20cm. movable = parse_query_obj('right hand') affordance_map = get_affordance_map(f'a point 20cm in front of {movable.position}') execute(movable, affordance_map) (using cache) *** OpenAI API call took 0.00s *** ######################################## ## "parse_query_obj" generated code ## context: "objects = ['cube']" ######################################## objects = ['cube'] # Query: right hand. right_hand = detect('right hand') ret_val = right_hand *** OpenAI API call took 1.17s *** ######################################## ## "get_affordance_map" generated code ######################################## # Query: a point 20cm in front of [59 45 4]. affordance_map = get_empty_affordance_map() # 20cm in front of so we subtract from x-axis x = cm2index(59, 'x') - 20 y = 45 z = 4 affordance_map[x, y, z] = 1 ret_val = affordance_map