diff --git a/README.md b/README.md index b8bc70d..c792138 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ It can process videos and predict task (skill) steps such as the ones related to > [!NOTE] > These are the used skills: -> (June/2024 demo) Apply tourniquet (M2), Pressure Dressing (M3), X-Stat (M5), and Apply Chest seal (R18) -> (December/2024 demo) Nasopharyngeal Airway (NPA) (A8), Wound Packing (M4), Ventilate (BVM) (R16), Needle Chest Decompression (R19) +> - (June/2024 demo) Apply tourniquet (M2), Pressure Dressing (M3), X-Stat (M5), and Apply Chest seal (R18) +> - (December/2024 demo) Nasopharyngeal Airway (NPA) (A8), Wound Packing (M4), Ventilate with Bag-Valve-Mask (BVM) (R16), Needle Chest Decompression (R19) ## **Install** @@ -130,7 +130,9 @@ python tools/test.py --cfg config/M3.yaml 1. Main code: `toos/run_step_recog.py` (function *train_kfold*) 2. Training/evaluation routines: `step_recog/iterators.py` (functions *train*, *evaluate*) 3. Model classes: `step_recog/models.py` -4. Dataloader: `step_recog/datasets/milly.py` (class *Milly_multifeature_v4* and methods *_construct_loader* and *__getitem\__*) +4. Dataloader: `step_recog/datasets/milly.py` (methods *_construct_loader* and *__getitem\__*) +- class *Milly_multifeature_v4* loads video frames and returns formated features +- class *Milly_multifeature_v5* loads (preprocessed) features and returns formated features 5. Image augmentation: `tools/augmentation.py` (function *get_augmentation*) 6. Basic configuration: `step_recog/config/defaults.py` (more important), `act_recog/config/defaults.py`, `auditory_slowfast/config/defaults.py` 6. Visualizer: `step_recog/full/visualize.py` implements a specific code that combines dataloading, model prediction, and a state machine. It uses the user interface with the trained models. diff --git a/config/A8.yaml b/config/A8.yaml index 67ee5bb..b0f5a7c 100644 --- a/config/A8.yaml +++ b/config/A8.yaml @@ -5,9 +5,9 @@ MODEL: YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_A8.pt' DATASET: - TR_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv" - VL_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv" - TS_ANNOTATIONS_FILE: "/home/user/labels/A8_professional_videos.csv" + TR_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv" + VL_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv" + TS_ANNOTATIONS_FILE: "/home/user/labels/A8_Alabama_videos.csv" OUTPUT: LOCATION: "/home/user/output" diff --git a/config/ALL_SKILLS.yaml b/config/ALL_SKILLS.yaml index 8048119..d4064b3 100644 --- a/config/ALL_SKILLS.yaml +++ b/config/ALL_SKILLS.yaml @@ -72,7 +72,7 @@ SKILLS: - Push plunger firmly to deploy sponges into wound. - Apply manual pressure if necessary. - - NAME: R16 - Ventilate (BVM) + - NAME: R16 - Ventilate with Bag-Valve-Mask (BVM) STEPS: - Place casualty into 'sniffing position' with head tilted back and nostrils exposed. - Open the BVM packaging. diff --git a/config/M4.yaml b/config/M4.yaml index 50a286c..b1993a1 100644 --- a/config/M4.yaml +++ b/config/M4.yaml @@ -5,9 +5,9 @@ MODEL: YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_M4.pt' DATASET: - TR_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv" - VL_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv" - TS_ANNOTATIONS_FILE: "/home/user/labels/M4_professional_videos.csv" + TR_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv" + VL_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv" + TS_ANNOTATIONS_FILE: "/home/user/labels/M4_Alabama_videos.csv" OUTPUT: LOCATION: "/home/user/output" diff --git a/config/R16.yaml b/config/R16.yaml index 4f95639..f29ece6 100644 --- a/config/R16.yaml +++ b/config/R16.yaml @@ -5,15 +5,15 @@ MODEL: YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_R16.pt' DATASET: - TR_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv" - VL_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv" - TS_ANNOTATIONS_FILE: "/home/user/labels/R16_professional_videos.csv" + TR_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv" + VL_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv" + TS_ANNOTATIONS_FILE: "/home/user/labels/R16_Alabama_videos.csv" OUTPUT: LOCATION: "/home/user/output" SKILLS: - - NAME: R16 - Ventilate (BVM) + - NAME: R16 - Ventilate with Bag-Valve-Mask (BVM) STEPS: - Place casualty into 'sniffing position' with head tilted back and nostrils exposed. - Open the BVM packaging. diff --git a/config/R19.yaml b/config/R19.yaml index 9a3c4a9..4c49794 100644 --- a/config/R19.yaml +++ b/config/R19.yaml @@ -5,9 +5,9 @@ MODEL: YOLO_CHECKPOINT_URL: '/home/user/models/bbn_yolo_R19.pt' DATASET: - TR_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv" - VL_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv" - TS_ANNOTATIONS_FILE: "/home/user/labels/R19_professional_videos.csv" + TR_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv" + VL_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv" + TS_ANNOTATIONS_FILE: "/home/user/labels/R19_Alabama_videos.csv" OUTPUT: LOCATION: "/home/user/output" diff --git a/config/STEPGRU_BASE.yaml b/config/STEPGRU_BASE.yaml index 273e5e6..cfc9ddb 100644 --- a/config/STEPGRU_BASE.yaml +++ b/config/STEPGRU_BASE.yaml @@ -11,6 +11,7 @@ MODEL: SLOWFAST_CONFIG: '/home/user/data/auditory_slowfast/configs/BBN/SLOWFAST_R50.yaml' DATASET: NAME: 'Milly' + CLASS: 'Milly_multifeature_v4' LOCATION: '/frame' AUDIO_LOCATION: '/sound' INCLUDE_IMAGE_AUGMENTATIONS: True diff --git a/scripts/extract_frames.sh b/scripts/extract_frames.sh index 17aeec2..119903e 100755 --- a/scripts/extract_frames.sh +++ b/scripts/extract_frames.sh @@ -6,7 +6,7 @@ SKILL=$3 TYPE=$4 ## frame sound EXTRACT=$5 SLOW_FAST_AUDIO_DEFAULT_SAMPLE_RATE=24000 -POSSIBLE_SKILLS=("M1" "M2" "M3" "M5" "R18") +POSSIBLE_SKILLS=("A8" "M1" "M2" "M3" "M4" "M5" "R16" "R18" "R19") POSSIBLE_TYPES=("frame" "sound") POSSIBLE_BOOLEAN=("true" "false") diff --git a/step_recog/iterators.py b/step_recog/iterators.py index 7222fb5..eb8a8a0 100644 --- a/step_recog/iterators.py +++ b/step_recog/iterators.py @@ -586,7 +586,16 @@ def save_evaluation(expected, predicted, classes, cfg, label_order = None, norma file.write("Balanced accuracy: {:.2f}\n".format(my_balanced_accuracy_score(expected, predicted))) finally: file.close() + + #========================================================================================================================# + cm = confusion_matrix(expected, predicted, normalize = None, labels = classes) + + df = pd.DataFrame(cm, columns = label_order, index = label_order) + df.index.name = 'Expected' + df.columns.name = 'Predicted' + df.to_csv(os.path.join(cfg.OUTPUT.LOCATION, file_name.replace('.png', '_raw.csv'))) #========================================================================================================================# + cm = confusion_matrix(expected, predicted, normalize = normalize, labels = classes) if pad is not None: diff --git a/tools/run_step_recog.py b/tools/run_step_recog.py index 5587b9d..27de2e5 100644 --- a/tools/run_step_recog.py +++ b/tools/run_step_recog.py @@ -102,6 +102,8 @@ def my_train_test_split(cfg, videos): videos, video_test = train_test_split(videos, test_size=0.10, random_state=2359) #M5 1030: only with BBN 041624.zip elif "R18" in cfg.SKILLS[0]["NAME"]: videos, video_test = train_test_split(videos, test_size=0.10, random_state=2343) #R18 1740: only with BBN seal_videos.zip + else: #A8, M4, R16, R19 + videos, video_test = train_test_split(videos, test_size=0.10, random_state=1030) return videos, video_test