diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75978a63e..c54220ebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: # Setup python version and caching - uses: actions/setup-python@v4 with: - python-version: '3.10.12' # matching python version in container env. + python-version: '3.8.10' # matching python version in container env. cache: 'poetry' # Install standard packages - run: | @@ -78,7 +78,7 @@ jobs: # # Setup python version and caching # - uses: actions/setup-python@v4 # with: -# python-version: '3.10.12' # matching python version in container env. +# python-version: '3.8.10' # matching python version in container env. # cache: 'poetry' # # Install standard packages # - run: | @@ -102,7 +102,7 @@ jobs: # Setup python version and caching - uses: actions/setup-python@v4 with: - python-version: '3.10.12' # matching python version in container env. + python-version: '3.8.10' # matching python version in container env. cache: 'poetry' # Install standard packages - run: | diff --git a/docs/system/testing_with_bbn.rst b/docs/system/testing_with_bbn.rst new file mode 100644 index 000000000..e5cbfb9dd --- /dev/null +++ b/docs/system/testing_with_bbn.rst @@ -0,0 +1,44 @@ +============================ +Integration Testing with BBN +============================ + +Testing BBNUpdate Message to YAML Transmission +============================================== +Relevant Node: ``ros2 run bbn_integration ZmqIntegrationClient`` + +Relevant Source Files: +* ``ros/bbn_integration/src/nodes/zmq_integration_client.cxx`` +* ``ros/bbn_integration/src/ros_to_yaml.cxx`` + +The files contained here are manually crafted YAML serializations of the ROS2 +BBNUpdate message format. +These can be emitted as ROS2 messages via ``ros2 topic pub ...`` to test the +functionality of our ``ZmqIntegrationClient`` converter node defined in this +package. + +How to publish message from YAML file +------------------------------------- +Example Message YAML serializations: +* ``ros/bbn_integration/example_files/example_output.yml`` +* ``ros/bbn_integration/example_files/example_output_seq_1.yml`` +* ``ros/bbn_integration/example_files/example_output_seq_2.yml`` +* ``ros/bbn_integration/example_files/example_output_seq_3.yml`` + +The ``*_seq_*.yml`` files are intended to represent a sequence of BBNUpdate +messages showing the current step changing. +These are useful to send in succession to BBN's own service in order to +validate if our output YAML format is valid and that they are appropriately +seeing the expected content changes. + +Example CLI to manually publish a BBNUpdate message from a YAML file: + +.. code-block:: bash + + ros2 topic pub --once \ + /BBNUpdates \ + bbn_integration_msgs/BBNUpdate \ + "$(cat ros/bbn_integration/example_files/example_output_seq_3.yml)" + +The ``/BBNUpdates`` topic used in the example above is arbitrary and should be +replaced with whatever topic you parameterize ``-p topic_update_msg:=...`` for +the ``ZmqIntegrationClient`` node to be. diff --git a/ros/bbn_integration/example_files/README.md b/ros/bbn_integration/example_files/README.md new file mode 100644 index 000000000..7b06bdba5 --- /dev/null +++ b/ros/bbn_integration/example_files/README.md @@ -0,0 +1 @@ +See documentation in `./docs/system/testing_with_bbn.rst` diff --git a/ros/bbn_integration_py/bbn_integration_py/nodes/task_to_bbn_update.py b/ros/bbn_integration_py/bbn_integration_py/nodes/task_to_bbn_update.py index ffd4c7bdc..a8a45e602 100644 --- a/ros/bbn_integration_py/bbn_integration_py/nodes/task_to_bbn_update.py +++ b/ros/bbn_integration_py/bbn_integration_py/nodes/task_to_bbn_update.py @@ -213,16 +213,20 @@ def _update_task_graph(self, msg: TaskUpdate) -> bool: # rclpy.spin_until_future_complete(self, future) # resp: QueryTaskGraph.Response = future.result() log.info("Querying for task-monitor task graph -- Done") - if resp.task_title != msg.task_name: - self.get_logger().warn( - f"Received QueryTaskGraph response with mismatching title " - f"({resp.task_title}) compared to the current task update " - f"name ({msg.task_name})." - ) - return False + if msg.task_name not in resp.task_titles: + self.get_logger().warn( + f"Received QueryTaskGraph response with mismatching title " + f"({resp.task_title}) compared to the current task update " + f"name ({msg.task_name})." + ) + return False + + # There are multiple task graphs in this message, get the index via the + # tile and extract the steps from that specific graph. + graph_idx = resp.task_titles.index(msg.task_name) # Store the list of step strings for this task - self._task_to_step_list[msg.task_name] = resp.task_graph.task_steps + self._task_to_step_list[msg.task_name] = resp.task_graphs[graph_idx].task_steps return True def _callback_input_task_update(self, msg: TaskUpdate) -> None: @@ -325,8 +329,10 @@ def _callback_input_task_update(self, msg: TaskUpdate) -> None: # Current KW step has no mapping into BBN steps, skipping. continue + # BBN expects 1-based indexing in the `number` field as opposed to + # 0-based indexing that is output by python's `enumerate()`. task_step_state_map[step_i] = BBNStepState( - number=step_i, name=step_name, state=state, confidence=1.0 + number=step_i + 1, name=step_name, state=state, confidence=1.0 ) # Check that the index-to-step mapping is contiguous across step indices. diff --git a/tmux/demos/medical/BBN-R18.yml b/tmux/demos/medical/BBN-R18.yml new file mode 100644 index 000000000..8f1ad2d20 --- /dev/null +++ b/tmux/demos/medical/BBN-R18.yml @@ -0,0 +1,139 @@ +# +# System configuration to run the ANGEL system for the BBN system. +# +# This configuration is for the M2 tourniquet task. +# + +name: R18-Chest-Seal +root: <%= ENV["ANGEL_WORKSPACE_DIR"] %> + +# Optional tmux socket +# socket_name: foo + +# Note that the pre and post options have been deprecated and will be replaced by +# project hooks. + +# Project hooks + +# Runs on project start, always +# on_project_start: command +on_project_start: | + export ROS_NAMESPACE=${ROS_NAMESPACE:-/kitware} + export HL2SS_API_URL=${HL2SS_API_URL:-128.33.193.178:8000} + export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6669} + export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config + export NODE_CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs + export MODEL_DIR=${ANGEL_WORKSPACE_DIR}/model_files +# Run on project start, the first time +# on_project_first_start: command + +# Run on project start, after the first time +# on_project_restart: command + +# Run on project exit ( detaching from tmux session ) +# on_project_exit: command + +# Run on project stop +# on_project_stop: command + +# Runs in each window and pane before window/pane specific commands. Useful for setting up interpreter versions. +# pre_window: rbenv shell 2.0.0-p247 + +# Pass command line options to tmux. Useful for specifying a different tmux.conf. +# tmux_options: -f ~/.tmux.mac.conf +tmux_options: -f <%= ENV["ANGEL_WORKSPACE_DIR"] %>/tmux/tmux.conf + +windows: + # Connection with the HL2 app via HL2SS/REDIS sensor data streaming. + - sensor_input: ros2 run angel_system_nodes redis_ros_bridge --ros-args + -r __ns:=${ROS_NAMESPACE} + -p url:=${HL2SS_API_URL} + -p image_topic:=image_0 + -p hand_pose_topic:=HandJointPoseData + + - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_topic:=image_0 + -p output_topic:=image_0_ts + + - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_topic:=image_0 + -p det_topic:=pose_dets + -p pose_topic:=PatientPose + -p det_net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/pose_det_model.pth + -p pose_net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/pose_model.pth + -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml + -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py + -p cuda_device_id:=0 + + - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_topic:=image_0 + -p det_topic:=ObjectDetections2d + -p net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/r18_det.pt + -p hand_net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/hands_model.pt + + - simple_2d_overlay: ros2 run angel_utils Simple2dDetectionOverlay --ros-args + -r __ns:=${ROS_NAMESPACE} + -p topic_input_images:=image_0 + -p topic_input_det_2d:=ObjectDetections2d + -p topic_output_images:=pv_image_detections_2d + -p filter_top_k:=5 + + - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_ts_topic:=image_0_ts + -p det_topic:=ObjectDetections2d + -p pose_topic:=PatientPose + -p model_weights:=${ANGEL_WORKSPACE_DIR}/model_files/models/r18_tcn.ckpt + -p model_mapping:=${ANGEL_WORKSPACE_DIR}/model_files/data_mapping/r18_mapping.txt + -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json + -p act_topic:=activity_topic + + - task_monitor: + layout: even-vertical + panes: + - gsp: ros2 run angel_system_nodes global_step_predictor --ros-args + -r __ns:=${ROS_NAMESPACE} + -p config_file:=${CONFIG_DIR}/tasks/multi-task-config-medical-r18.yaml + -p activity_config_file:=${CONFIG_DIR}/activity_labels/r18.yaml + -p task_state_topic:=TaskUpdates + -p task_error_topic:=TaskErrors + -p system_command_topic:=SystemCommands + -p det_topic:=activity_topic + -p model_file:=${ANGEL_WORKSPACE_DIR}/model_files/coco/r18_test_activity_preds.mscoco.json + -p thresh_frame_count:=3 + -p deactivate_thresh_frame_count:=10 + -p threshold_multiplier_weak:=0.00 + -p threshold_frame_count_weak:=3 + -p step_mode:=granular + -p query_task_graph_topic:=query_task_graph + - echo: sleep 0.5 && ros2 topic echo --no-arr "${ROS_NAMESPACE}/TaskUpdates" + + # - engineering-ui: + # layout: even-vertical + # panes: + # - websocket: ros2 launch rosbridge_server rosbridge_websocket_launch.xml port:=9090 + # - engineering_ui_server: node ros/angel_utils/multi_task_demo_ui/index.js + # --namespace=${ROS_NAMESPACE} + # --image_topic=pv_image_detections_2d/compressed + # --query_task_graph_topic=query_task_graph + # --task_updates_topic=TaskUpdates + # --activity_detections_topic=activity_topic + # --task_errors_topic=TaskErrors + + - BBN Interface: + layout: even-vertical + panes: + - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args + -r __ns:=${ROS_NAMESPACE} + -p task_update_topic:=TaskUpdates + -p bbn_update_topic:=BBNUpdates + -p task_graph_srv_topic:=query_task_graph + -p config:=${CONFIG_DIR}/bbn_integration/m2_tourniquet.yml + - echo: sleep 0.5 && ros2 topic echo "${ROS_NAMESPACE}/BBNUpdates" + - zmq-publisher: ros2 run bbn_integration ZmqIntegrationClient --ros-args + -r __ns:=${ROS_NAMESPACE} + -p topic_update_msg:=BBNUpdates + -p server_address:=${BBN_OUTPUT_URL} diff --git a/tmux/demos/medical/Kitware-R18.yml b/tmux/demos/medical/Kitware-R18.yml index c283aeec9..7356eed7a 100644 --- a/tmux/demos/medical/Kitware-R18.yml +++ b/tmux/demos/medical/Kitware-R18.yml @@ -20,7 +20,7 @@ root: <%= ENV["ANGEL_WORKSPACE_DIR"] %> on_project_start: | export ROS_NAMESPACE=${ROS_NAMESPACE:-/kitware} export HL2SS_API_URL=${HL2SS_API_URL:-128.33.193.178:8000} - export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6667} + export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6669} export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config export NODE_CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs export MODEL_DIR=${ANGEL_WORKSPACE_DIR}/model_files @@ -44,10 +44,9 @@ on_project_start: | tmux_options: -f <%= ENV["ANGEL_WORKSPACE_DIR"] %>/tmux/tmux.conf windows: -# Connection with the HL2 app via HL2SS sensor data streaming and the -# ROS-TCP-Endpoint - - sensor_input: ros2 run angel_system_nodes generate_images - # - sensor_input: ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_02-19_57_59-R18_naked_mannequin_no_blue_tarp-timestamp_unconfident/rosbag2_2024_04_02-19_57_59_0.db3 + + # Read sensor input from bag file. + - sensor_input: echo ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_10-18_57_36-R18_Chris_live/ - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args -r __ns:=${ROS_NAMESPACE} @@ -65,13 +64,20 @@ windows: -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py -p cuda_device_id:=0 - - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args - -r __ns:=${ROS_NAMESPACE} + - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args + -r __ns:=${ROS_NAMESPACE} -p image_topic:=image_0 -p det_topic:=ObjectDetections2d -p net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/r18_det.pt -p hand_net_checkpoint:=${ANGEL_WORKSPACE_DIR}/model_files/models/hands_model.pt + - simple_2d_overlay: ros2 run angel_utils Simple2dDetectionOverlay --ros-args + -r __ns:=${ROS_NAMESPACE} + -p topic_input_images:=image_0 + -p topic_input_det_2d:=ObjectDetections2d + -p topic_output_images:=pv_image_detections_2d + -p filter_top_k:=5 + - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args -r __ns:=${ROS_NAMESPACE} -p image_ts_topic:=image_0_ts @@ -82,22 +88,37 @@ windows: -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json -p act_topic:=activity_topic - - task_monitor: ros2 run angel_system_nodes global_step_predictor --ros-args - -r __ns:=${ROS_NAMESPACE} - -p config_file:=${CONFIG_DIR}/tasks/multi-task-config-medical-r18.yaml - -p activity_config_file:=${CONFIG_DIR}/activity_labels/r18.yaml - -p task_state_topic:=TaskUpdates - -p task_error_topic:=TaskErrors - -p system_command_topic:=SystemCommands - -p det_topic:=activity_topic - -p model_file:=${ANGEL_WORKSPACE_DIR}/model_files/coco/r18_test_activity_preds.mscoco.json - -p thresh_frame_count:=3 - -p deactivate_thresh_frame_count:=10 - -p threshold_multiplier_weak:=0.00 - -p threshold_frame_count_weak:=3 - -p step_mode:=granular - -p query_task_graph_topic:=query_task_graph - + - task_monitor: + layout: even-vertical + panes: + - gsp: ros2 run angel_system_nodes global_step_predictor --ros-args + -r __ns:=${ROS_NAMESPACE} + -p config_file:=${CONFIG_DIR}/tasks/multi-task-config-medical-r18.yaml + -p activity_config_file:=${CONFIG_DIR}/activity_labels/r18.yaml + -p task_state_topic:=TaskUpdates + -p task_error_topic:=TaskErrors + -p system_command_topic:=SystemCommands + -p det_topic:=activity_topic + -p model_file:=${ANGEL_WORKSPACE_DIR}/model_files/coco/r18_test_activity_preds.mscoco.json + -p thresh_frame_count:=3 + -p deactivate_thresh_frame_count:=10 + -p threshold_multiplier_weak:=0.00 + -p threshold_frame_count_weak:=3 + -p step_mode:=granular + -p query_task_graph_topic:=query_task_graph + - echo: sleep 0.5 && ros2 topic echo --no-arr "${ROS_NAMESPACE}/TaskUpdates" + + # - engineering-ui: + # layout: even-vertical + # panes: + # - websocket: ros2 launch rosbridge_server rosbridge_websocket_launch.xml port:=9090 + # - engineering_ui_server: node ros/angel_utils/multi_task_demo_ui/index.js + # --namespace=${ROS_NAMESPACE} + # --image_topic=pv_image_detections_2d/compressed + # --query_task_graph_topic=query_task_graph + # --task_updates_topic=TaskUpdates + # --activity_detections_topic=activity_topic + # --task_errors_topic=TaskErrors - BBN Interface: layout: even-vertical @@ -108,6 +129,7 @@ windows: -p bbn_update_topic:=BBNUpdates -p task_graph_srv_topic:=query_task_graph -p config:=${CONFIG_DIR}/bbn_integration/m2_tourniquet.yml + - echo: sleep 0.5 && ros2 topic echo "${ROS_NAMESPACE}/BBNUpdates" - zmq-publisher: ros2 run bbn_integration ZmqIntegrationClient --ros-args -r __ns:=${ROS_NAMESPACE} -p topic_update_msg:=BBNUpdates diff --git a/tmux/demos/medical/M3-Kitware.yml b/tmux/demos/medical/M3-Kitware.yml index 42f008f25..997e0a05d 100644 --- a/tmux/demos/medical/M3-Kitware.yml +++ b/tmux/demos/medical/M3-Kitware.yml @@ -20,7 +20,7 @@ root: <%= ENV["ANGEL_WORKSPACE_DIR"] %> on_project_start: | export ROS_NAMESPACE=${ROS_NAMESPACE:-/kitware} export HL2SS_API_URL=${HL2SS_API_URL:-128.33.193.178:8000} - export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:5557} + export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6669} export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config export NODE_CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs export MODEL_DIR=${ANGEL_WORKSPACE_DIR}/model_files @@ -47,7 +47,7 @@ windows: # Connection with the HL2 app via HL2SS sensor data streaming and the # ROS-TCP-Endpoint # - sensor_input: ros2 run angel_system_nodes generate_images - - sensor_input: ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_02-20_01_26-M3_naked_mannequin_no_blue_tarp/rosbag2_2024_04_02-20_01_26_0.db3 + - sensor_input: echo ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_02-20_01_26-M3_naked_mannequin_no_blue_tarp/rosbag2_2024_04_02-20_01_26_0.db3 - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args -r __ns:=${ROS_NAMESPACE}