Merge pull request fabiofelix#49 from fabiofelix/updates

General commit

README.md

@@ -6,8 +6,8 @@ It can process videos and predict task (skill) steps such as the ones related to
 
 > [!NOTE] 
 > These are the used skills:  
-> (June/2024 demo) Apply tourniquet (M2), Pressure Dressing (M3), X-Stat (M5), and Apply Chest seal (R18)
-> (December/2024 demo) Nasopharyngeal Airway (NPA) (A8), Wound Packing (M4), Ventilate (BVM) (R16), Needle Chest Decompression (R19)
+> - (June/2024 demo) Apply tourniquet (M2), Pressure Dressing (M3), X-Stat (M5), and Apply Chest seal (R18)
+> - (December/2024 demo) Nasopharyngeal Airway (NPA) (A8), Wound Packing (M4), Ventilate with Bag-Valve-Mask (BVM) (R16), Needle Chest Decompression (R19)
 
 ## **Install**
 
@@ -130,7 +130,9 @@ python tools/test.py --cfg config/M3.yaml
 1. Main code: `toos/run_step_recog.py` (function *train_kfold*)
 2. Training/evaluation routines: `step_recog/iterators.py` (functions *train*, *evaluate*)
 3. Model classes: `step_recog/models.py`
-4. Dataloader: `step_recog/datasets/milly.py` (class *Milly_multifeature_v4* and methods *_construct_loader* and *__getitem\__*)
+4. Dataloader: `step_recog/datasets/milly.py` (methods *_construct_loader* and *__getitem\__*)
+- class *Milly_multifeature_v4* loads video frames and returns formated features
+- class *Milly_multifeature_v5* loads (preprocessed) features and returns formated features 
 5. Image augmentation: `tools/augmentation.py` (function *get_augmentation*)
 6. Basic configuration: `step_recog/config/defaults.py` (more important), `act_recog/config/defaults.py`, `auditory_slowfast/config/defaults.py`
 6. Visualizer: `step_recog/full/visualize.py` implements a specific code that combines dataloading, model prediction, and a state machine. It uses the user interface with the trained models.

config/A8.yaml

@@ -5,9 +5,9 @@ MODEL:
   YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_A8.pt'
 
 DATASET:
-  TR_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv"
-  VL_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv"
-  TS_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv"
+  TR_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv"
+  VL_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv"
+  TS_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv"
 
 OUTPUT:
   LOCATION: "/home/user/output"

config/ALL_SKILLS.yaml

@@ -72,7 +72,7 @@ SKILLS:
     - Push plunger firmly to deploy sponges into wound.
     - Apply manual pressure if necessary.    
 
-  - NAME: R16 - Ventilate (BVM)
+  - NAME: R16 - Ventilate with Bag-Valve-Mask (BVM)
     STEPS:
     - Place casualty into 'sniffing position' with head tilted back and nostrils exposed.
     - Open the BVM packaging.

config/M4.yaml

@@ -5,9 +5,9 @@ MODEL:
   YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_M4.pt'  
 
 DATASET:
-  TR_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv"
-  VL_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv"
-  TS_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv"
+  TR_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv"
+  VL_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv"
+  TS_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv"
 
 OUTPUT:
   LOCATION: "/home/user/output"

config/R16.yaml

@@ -5,15 +5,15 @@ MODEL:
   YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_R16.pt'  
 
 DATASET:
-  TR_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv"
-  VL_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv"
-  TS_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv"
+  TR_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv"
+  VL_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv"
+  TS_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv"
 
 OUTPUT:
   LOCATION: "/home/user/output"
 
 SKILLS:
-  - NAME: R16 - Ventilate (BVM)
+  - NAME: R16 - Ventilate with Bag-Valve-Mask (BVM)
     STEPS:
       - Place casualty into 'sniffing position' with head tilted back and nostrils exposed.
       - Open the BVM packaging.

config/R19.yaml

@@ -5,9 +5,9 @@ MODEL:
   YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_R19.pt'  
 
 DATASET:
-  TR_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv"
-  VL_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv"
-  TS_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv"
+  TR_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv"
+  VL_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv"
+  TS_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv"
 
 OUTPUT:
   LOCATION: "/home/user/output"

config/STEPGRU_BASE.yaml

@@ -11,6 +11,7 @@ MODEL:
   SLOWFAST_CONFIG: '/home/user/data/auditory_slowfast/configs/BBN/SLOWFAST_R50.yaml'
 DATASET:
   NAME: 'Milly'
+  CLASS: 'Milly_multifeature_v4'
   LOCATION: '/frame' 
   AUDIO_LOCATION: '/sound'
   INCLUDE_IMAGE_AUGMENTATIONS: True

scripts/extract_frames.sh

@@ -6,7 +6,7 @@ SKILL=$3
 TYPE=$4 ## frame sound
 EXTRACT=$5
 SLOW_FAST_AUDIO_DEFAULT_SAMPLE_RATE=24000
-POSSIBLE_SKILLS=("M1" "M2" "M3" "M5" "R18")
+POSSIBLE_SKILLS=("A8" "M1" "M2" "M3" "M4" "M5" "R16" "R18" "R19")
 POSSIBLE_TYPES=("frame" "sound")
 POSSIBLE_BOOLEAN=("true" "false")
 

step_recog/iterators.py

@@ -586,7 +586,16 @@ def save_evaluation(expected, predicted, classes, cfg, label_order = None, norma
     file.write("Balanced accuracy: {:.2f}\n".format(my_balanced_accuracy_score(expected, predicted)))
   finally:
     file.close() 
+
+  #========================================================================================================================#
+  cm = confusion_matrix(expected, predicted, normalize = None, labels = classes)
+
+  df = pd.DataFrame(cm, columns = label_order, index = label_order)
+  df.index.name   = 'Expected'
+  df.columns.name = 'Predicted'
+  df.to_csv(os.path.join(cfg.OUTPUT.LOCATION, file_name.replace('.png', '_raw.csv')))
   #========================================================================================================================#
+
   cm = confusion_matrix(expected, predicted, normalize = normalize, labels = classes)
 
   if pad is not None:

tools/run_step_recog.py

@@ -102,6 +102,8 @@ def my_train_test_split(cfg, videos):
       videos, video_test = train_test_split(videos, test_size=0.10, random_state=2359) #M5  1030: only with BBN 041624.zip
     elif "R18" in cfg.SKILLS[0]["NAME"]:      
       videos, video_test = train_test_split(videos, test_size=0.10, random_state=2343) #R18 1740: only with BBN seal_videos.zip
+    else: #A8, M4, R16, R19      
+      videos, video_test = train_test_split(videos, test_size=0.10, random_state=1030) 
 
   return videos, video_test