Get VLM Predicate Demo to work with actual movement skills

predicators/approaches/bilevel_planning_approach.py

@@ -72,6 +72,11 @@ def _solve(self, task: Task, timeout: int) -> Callable[[State], Action]:
             for act in nsrt_plan:
                 logging.debug(act)
 
+            for act in nsrt_plan:
+                print(act)
+            import ipdb
+            ipdb.set_trace()
+
         # Run full bilevel planning.
         else:
             option_plan, nsrt_plan, metrics = self._run_sesame_plan(

predicators/approaches/documentation/grammar_search_invention_approach.md

@@ -62,7 +62,7 @@ apple_coring__vlm_demos__456__2
     | options.txt
 
 ### Running predicate invention using these image demos
-To use the Gemini VLM, you need to set the `GOOGLE_API_KEY` environment variable in your terminal. You can make/get an API key [here](https://aistudio.google.com/app/apikey).
+To use the Gemini VLM, you need to set the `GOOGLE_API_KEY` environment variable in your terminal. You can make/get an API key [here](https://aistudio.google.com/app/apikey). To use an Open AI VLM, the `OPENAI_API_KEY` variable needs to be set.
 
 Example command: `python predicators/main.py --env apple_coring --seed 456 --approach grammar_search_invention --excluded_predicates all --num_train_tasks 1 --num_test_tasks 0 --offline_data_method saved_vlm_img_demos_folder --vlm_trajs_folder_name apple_coring__vlm_demos__456__1`
 

predicators/approaches/spot_wrapper_approach.py

@@ -18,6 +18,7 @@
 from predicators import utils
 from predicators.approaches import BaseApproach, BaseApproachWrapper
 from predicators.envs.spot_env import get_detection_id_for_object, get_robot
+from predicators.settings import CFG
 from predicators.spot_utils.skills.spot_find_objects import find_objects
 from predicators.spot_utils.skills.spot_stow_arm import stow_arm
 from predicators.spot_utils.utils import get_allowed_map_regions
@@ -66,7 +67,7 @@ def _policy(state: State) -> Action:
                     state.get(obj, "lost") > 0.5:
                     lost_objects.add(obj)
             # Need to find the objects.
-            if lost_objects:
+            if lost_objects and len(CFG.spot_vlm_teleop_demo_folderpath) == 0:
                 logging.info(f"[Spot Wrapper] Lost objects: {lost_objects}")
                 # Reset the base approach policy.
                 base_approach_policy = None