Merge pull request #418 from derkmed/dialog_refactor

Refactor all dialogue system nodes to inherit from same abstract class

ros/angel_system_nodes/angel_system_nodes/audio/asr.py

@@ -8,10 +8,9 @@
 
 from nltk.tokenize import sent_tokenize
 import rclpy
-from rclpy.node import Node
-import simpleaudio as sa
 
 from angel_msgs.msg import HeadsetAudioData, Utterance
+from angel_system_nodes.audio import dialogue
 from angel_utils import make_default_main
 
 
@@ -27,10 +26,9 @@
 WAV_SAMPLE_WIDTH = 4  # Derived from audio_player.py
 
 
-class ASR(Node):
+class ASR(dialogue.AbstractDialogueNode):
     def __init__(self):
-        super().__init__(self.__class__.__name__)
-        self.log = self.get_logger()
+        super().__init__()
 
         parameter_names = [
             AUDIO_TOPIC,

ros/angel_system_nodes/angel_system_nodes/audio/dialogue.py

@@ -0,0 +1,33 @@
+from rclpy.node import Node
+
+from angel_msgs.msg import DialogueUtterance
+
+
+class AbstractDialogueNode(Node):
+    def __init__(self):
+        super().__init__(self.__class__.__name__)
+        self.log = self.get_logger()
+
+    def _copy_dialogue_utterance(
+        src_msg: DialogueUtterance, node_name: str, copy_time
+    ) -> DialogueUtterance:
+        msg = DialogueUtterance()
+
+        msg.header.frame_id = node_name
+        msg.header.stamp = copy_time
+
+        msg.utterance_text = src_msg.utterance_text
+
+        # Copy all optional fields below.
+
+        # Copy over intent classification information if present.
+        if src_msg.intent:
+            msg.intent = src_msg.intent
+            msg.intent_confidence_score = src_msg.intent_confidence_score
+
+        # Copy over emotion classification information if present.
+        if msg.emotion:
+            msg.emotion = src_msg.emotion
+            msg.emotion_confidence_score = src_msg.emotion_confidence_score
+
+        return msg

ros/angel_system_nodes/angel_system_nodes/audio/emotion/base_emotion_detector.py

@@ -1,10 +1,10 @@
 import queue
-from rclpy.node import Node
 from termcolor import colored
 import threading
 from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
 
 from angel_msgs.msg import InterpretedAudioUserEmotion, InterpretedAudioUserIntent
+from angel_system_nodes.audio import dialogue
 from angel_utils import declare_and_get_parameters
 from angel_utils import make_default_main
 
@@ -24,15 +24,14 @@
 VADER_POSITIVE_COMPOUND_THRESHOLD = 0.05
 
 
-class BaseEmotionDetector(Node):
+class BaseEmotionDetector(dialogue.AbstractDialogueNode):
     """
     As of Q22023, emotion detection is derived via VaderSentiment
     (https://github.com/cjhutto/vaderSentiment).
     """
 
     def __init__(self):
-        super().__init__(self.__class__.__name__)
-        self.log = self.get_logger()
+        super().__init__()
 
         # Handle parameterization.
         param_values = declare_and_get_parameters(

ros/angel_system_nodes/angel_system_nodes/audio/emotion/gpt_emotion_detector.py

@@ -27,7 +27,6 @@
 class GptEmotionDetector(BaseEmotionDetector):
     def __init__(self):
         super().__init__()
-        self.log = self.get_logger()
 
         # This node additionally includes fields for interacting with OpenAI
         # via LangChain.

ros/angel_system_nodes/angel_system_nodes/audio/intent/base_intent_detector.py

@@ -4,6 +4,7 @@
 import threading
 
 from angel_msgs.msg import InterpretedAudioUserIntent, Utterance
+from angel_system_nodes.audio import dialogue
 from angel_utils import declare_and_get_parameters
 from angel_utils import make_default_main
 
@@ -23,10 +24,9 @@
 PARAM_INTERP_USER_INTENT_TOPIC = "interp_user_intent_topic"
 
 
-class BaseIntentDetector(Node):
+class BaseIntentDetector(dialogue.AbstractDialogueNode):
     def __init__(self):
-        super().__init__(self.__class__.__name__)
-        self.log = self.get_logger()
+        super().__init__()
 
         # Handle parameterization.
         param_values = declare_and_get_parameters(

ros/angel_system_nodes/angel_system_nodes/audio/intent/gpt_intent_detector.py

@@ -27,7 +27,6 @@
 class GptIntentDetector(BaseIntentDetector):
     def __init__(self):
         super().__init__()
-        self.log = self.get_logger()
 
         # This node additionally includes fields for interacting with OpenAI
         # via LangChain.

ros/angel_system_nodes/angel_system_nodes/audio/question_answerer.py

@@ -2,12 +2,13 @@
 import openai
 import os
 import queue
-from rclpy.node import Node
+
 import requests
 from termcolor import colored
 import threading
 
 from angel_msgs.msg import InterpretedAudioUserEmotion, SystemTextResponse
+from angel_system_nodes.audio import dialogue
 from angel_utils import declare_and_get_parameters
 from angel_utils import make_default_main
 
@@ -20,10 +21,9 @@
 FEW_SHOT_PROMPT = "few_shot_prompt_file"
 
 
-class QuestionAnswerer(Node):
+class QuestionAnswerer(dialogue.AbstractDialogueNode):
     def __init__(self):
-        super().__init__(self.__class__.__name__)
-        self.log = self.get_logger()
+        super().__init__()
 
         param_values = declare_and_get_parameters(
             self,

ros/angel_system_nodes/setup.py

@@ -20,11 +20,11 @@
     entry_points={
         "console_scripts": [
             "video_listener = angel_system_nodes.video_subscriber:main",
-            "base_intent_detector = angel_system_nodes.base_intent_detector:main",
-            "gpt_intent_detector = angel_system_nodes.gpt_intent_detector:main",
-            "base_emotion_detector = angel_system_nodes.base_emotion_detector:main",
-            "gpt_emotion_detector = angel_system_nodes.gpt_emotion_detector:main",
-            "question_answerer = angel_system_nodes.question_answerer:main",
+            "base_intent_detector = angel_system_nodes.audio.intent.base_intent_detector:main",
+            "gpt_intent_detector = angel_system_nodes.audio.intent.gpt_intent_detector:main",
+            "base_emotion_detector = angel_system_nodes.audio.emotion.base_emotion_detector:main",
+            "gpt_emotion_detector = angel_system_nodes.audio.emotion.gpt_emotion_detector:main",
+            "question_answerer = angel_system_nodes.audio.question_answerer:main",
             "intent_detector = angel_system_nodes.intent_detector:main",
             "spatial_mapper = angel_system_nodes.spatial_mapper:main",
             "feedback_generator = angel_system_nodes.feedback_generator:main",

tmux/demos/cooking/eval_vocalized_question_answering.yml

@@ -25,7 +25,7 @@ root: <%= ENV["ANGEL_WORKSPACE_DIR"] %>
 # on_project_start: command
 on_project_start: |
   export ROS_NAMESPACE=${ROS_NAMESPACE:-/debug}
-  export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs
+  export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config
   export NODE_RESOURCES_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/resource
 # Run on project start, the first time
 # on_project_first_start: command
@@ -60,15 +60,15 @@ tmux_options: -f <%= ENV["ANGEL_WORKSPACE_DIR"] %>/tmux/tmux.conf
 
 windows:
   # - ros_bag_play: ros2 bag play <<PATH_TO_BAG_FILE>>
-  - ros_bag_play: sleep 5; ros2 bag play /angel_workspace/ros_bags/rosbag2_2023_03_01-17_28_00/rosbag2_2023_03_01-17_28_00_0.db3
+  - ros_bag_play: sleep 5; ros2 bag play /angel_workspace/ros_bags/rosbag2_2023_07_12-17_51_14_0.db3
   - vocal:
       layout: even-vertical
       panes:
         - vad: ros2 run angel_system_nodes voice_activity_detector --ros-args
             -r __ns:=${ROS_NAMESPACE}
             -p input_audio_topic:=HeadsetAudioData
             -p output_voice_activity_topic:=DetectedVoiceData
-            -p vad_server_url:=http://communication.cs.columbia.edu:55667/vad
+            -p vad_server_url:=127.0.0.1:55667/vad
             -p vad_cadence:=3
             -p vad_margin:=0.20
             -p max_accumulation_length:=10
@@ -77,7 +77,7 @@ windows:
             -r __ns:=${ROS_NAMESPACE}
             -p audio_topic:=DetectedVoiceData
             -p utterances_topic:=utterances_topic
-            -p asr_server_url:=http://communication.cs.columbia.edu:55667/asr
+            -p asr_server_url:=127.0.0.1:55667/asr
             -p asr_req_segment_duration:=1
             -p is_sentence_tokenize:=False
             -p debug_mode:=True