TETFN and CENET models added

thuiar · Dec 20, 2023 · 80995e4 · 80995e4
1 parent 36b03b8
commit 80995e4
Show file tree

Hide file tree

Showing 21 changed files with 1,626 additions and 27 deletions.
diff --git a/README.md b/README.md
@@ -145,11 +145,13 @@ MMSA uses feature files that are organized as follows:
 | Multi-Task  |     [MLF_DNN](src/MMSA/models/multiTask/MLF_DNN.py)     |                         [MMSA](https://github.com/thuiar/MMSA)                         | ACL 2020          |
 | Multi-Task  |        [MTFN](src/MMSA/models/multiTask/MTFN.py)        |                         [MMSA](https://github.com/thuiar/MMSA)                         | ACL 2020          |
 | Multi-Task  |        [MLMF](src/MMSA/models/multiTask/MLMF.py)        |                         [MMSA](https://github.com/thuiar/MMSA)                         | ACL 2020          |
+| Multi-Task  |     [SELF_MM](src/MMSA/models/multiTask/SELF_MM.py)     |                      [Self-MM](https://github.com/thuiar/Self-MM)                      | AAAI 2021    
+| Multi-Task  |     [TETFN](src/MMSA/models/multiTask/TETFN.py)     |                      TETFN                      | PR 2023    
 | Single-Task |   [BERT-MAG](src/MMSA/models/singleTask/BERT_MAG.py)    |        [MAG-BERT](https://github.com/WasifurRahman/BERT_multimodal_transformer)        | ACL 2020          |
-| Single-Task |       [MISA](src/MMSA/models/singleTask/MISA.py)        |                      [MISA](https://github.com/declare-lab/MISA)                       | ACMMM 2020    |
-| Single-Task |     [SELF_MM](src/MMSA/models/multiTask/SELF_MM.py)     |                      [Self-MM](https://github.com/thuiar/Self-MM)                      | AAAI 2021          |
+| Single-Task |       [MISA](src/MMSA/models/singleTask/MISA.py)        |                      [MISA](https://github.com/declare-lab/MISA)                       | ACMMM 2020    |      |
 | Single-Task |       [MMIM](src/MMSA/models/singleTask/MMIM.py)        |            [MMIM](https://github.com/declare-lab/Multimodal-Infomax)                   | EMNLP 2021    |
 | Single-Task |           BBFN (Work in Progress)                       |               [BBFN](https://github.com/declare-lab/BBFN)                              | ICMI 2021          |
+| Single-Task |           [CENET](src/MMSA/models/singleTask/CENET.py)    |               [CENET](https://github.com/Say2L/CENet)                              | TMM 2022          |
 
 
 ## 4. Results

diff --git a/results/result-stat.md b/results/result-stat.md
@@ -13,20 +13,24 @@
 | mult |79.71 |79.63 |80.98 |80.95 |42.68 |36.91 |87.99 |70.22 | Unaligned |
 | misa |81.84 |81.82 |83.54 |83.58 |47.08 |41.37 |77.65 |77.81 | Unaligned |
 | self_mm |83.44 |83.36 |85.46 |85.43 |53.47 |46.67 |70.80 |79.63 | Unaligned |
+| tetfn |83.24 |83.13 |85.37 |85.33 |53.64 |45.77 |70.84 |79.84 | Aligned |
+| cenet |83.53 |83.49 |85.21 |85.22 |50.87 |44.90 |72.54 |79.53 | Unaligned |
 
 - MOSEI
 
 | Model |Has0_acc_2 |Has0_F1_score |Non0_acc_2 |Non0_F1_score |Mult_acc_5 |Mult_acc_7 |MAE |Corr | Data Setting|
 | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
 | ef_lstm |77.84 |78.34 |80.79 |80.67 |51.16 |50.01 |60.05 |68.25 | Aligned |
-| lf_dnn |80.6 |80.85 |82.74 |82.52 |51.97 |50.83 |58.02 |70.87 | Unaligned |
-| tfn |78.50 |78.96 |81.89 |81.74 |53.1 |51.6 |57.26 |71.41 | Unaligned |
+| lf_dnn |80.60 |80.85 |82.74 |82.52 |51.97 |50.83 |58.02 |70.87 | Unaligned |
+| tfn |78.50 |78.96 |81.89 |81.74 |53.10 |51.60 |57.26 |71.41 | Unaligned |
 | lmf |80.54 |80.94 |83.48 |83.36 |52.99 |51.59 |57.57 |71.69 | Unaligned |
 | mfn |78.94 |79.55 |82.86 |82.85 |52.76 |51.34 |57.33 |71.82 | Aligned |
 | graph_mfn |81.28 |81.48 |83.48 |83.23 |52.69 |51.37 |57.45 |71.33 | Aligned |
 | mult |81.15 |81.56 |84.63 |84.52 |54.18 |52.84 |55.93 |73.31 | Unaligned |
 | misa |80.67 |81.12 |84.67 |84.66 |53.63 |52.05 |55.75 |75.15 | Unaligned |
 | self_mm |83.76 |83.82 |85.15 |84.90 |55.53 |53.87 |53.09 |76.49 | Unaligned |
+| tetfn |84.12 |84.35 |86.21 |86.11 |55.78 |53.90 |53.73 |76.96 | Aligned |
+| cenet |83.52 |83.85 |86.38 |86.32 |56.15 |54.26 |52.59 |77.75 | Unaligned |
 
 - SIMS
 
@@ -43,6 +47,8 @@
 | mtfn |81.09 |68.80 |40.31 |81.01 |39.54 |66.58 | Unaligned |
 | mlmf |79.34 |68.36 |41.05 |79.07 |40.91 |63.90 | Unaligned |
 | self_mm |80.04 |65.47 |41.53 |80.44 |42.50 |59.52 | Unaligned |
+| tetfn |81.18 |63.24 |41.79 |80.24 |42.00 |57.65 | Unaligned |
+| cenet |77.90 |62.58 |33.92 |77.53 |47.09 |53.95 | Unaligned |
 
 ## Classification
 > Data setting is the same as `Regression`

diff --git a/setup.cfg b/setup.cfg
@@ -34,6 +34,7 @@ install_requires =
     nvidia-ml-py3 >= 7.352.0
     scikit-learn >= 0.24.2
     easydict >= 1.9
+    pytorch_transformers >= 1.2.0
 
 [options.packages.find]
 where = src
diff --git a/src/MMSA/__main__.py b/src/MMSA/__main__.py
@@ -6,8 +6,8 @@ def parse_args():
     parser = argparse.ArgumentParser()
 
     parser.add_argument('-m', '--model', type=str, default='lf_dnn', help='Name of model',
-                        choices=['lf_dnn', 'ef_lstm', 'tfn', 'lmf', 'mfn', 'graph_mfn', 'mult', 'bert_mag', 
-                                 'misa', 'mfm', 'mlf_dnn', 'mtfn', 'mlmf', 'self_mm', 'mmim'])
+                        choices=['lf_dnn', 'ef_lstm', 'tfn', 'mctn','lmf', 'mfn', 'graph_mfn', 'mult', 'bert_mag', 
+                                 'misa', 'mfm', 'mlf_dnn', 'mtfn', 'mlmf', 'self_mm', 'mmim','tfr_net','tetfn','cenet'])
     parser.add_argument('-d', '--dataset', type=str, default='sims',
                         choices=['sims', 'mosi', 'mosei'], help='Name of dataset')
     parser.add_argument('-c', '--config', type=str, default='',

diff --git a/src/MMSA/config/config_regression.json b/src/MMSA/config/config_regression.json
@@ -61,6 +61,19 @@
         "missing_rate": [0.2, 0.2, 0.2],
         "missing_seed": [1111, 1111, 1111]
       }
+    },
+    "simsv2": {
+      "unaligned": {
+        "featurePath": "SIMSv2/Processed/sims_unaligned.pkl",
+        "seq_lens": [39, 400, 55],
+        "feature_dims": [768, 33, 709],
+        "train_samples": 2722,
+        "num_classes": 3,
+        "language": "cn",
+        "KeyEval": "Loss",
+        "missing_rate": [0.2, 0.2, 0.2],
+        "missing_seed": [1111, 1111, 1111]
+      }
     }
   },
   "tfn": {
@@ -1171,5 +1184,235 @@
         "num_temporal_head": 25
       }
     }
+  },
+  "tetfn": {
+    "commonParams": {
+      "need_data_aligned": true,
+      "need_model_aligned": true,
+      "need_normalized": false,
+      "use_bert": true,
+      "use_finetune": true,
+      "save_labels": false,
+      "early_stop": 8,
+      "update_epochs": 4
+    },
+    "datasetParams": {
+      "mosi": {
+        "batch_size": 64,
+        "transformers": "bert",
+        "pretrained": "bert-base-uncased",
+        "a_lstm_hidden_size":16,
+        "a_lstm_layers":1,
+        "a_lstm_dropout":0.0,
+        "v_lstm_hidden_size":64,
+        "v_lstm_layers":1,
+        "v_lstm_dropout":0.0,
+        "conv1d_kernel_size_l":1,
+        "conv1d_kernel_size_a":1,
+        "conv1d_kernel_size_v":3,
+        "dst_feature_dims":50,
+        "nheads":5,
+        "attn_dropout":0.1,
+        "attn_dropout_a":0.0,
+        "attn_dropout_v":0.1,
+        "relu_dropout":0.0,
+        "embed_dropout":0.0,
+        "res_dropout":0.1,
+        "post_fusion_dropout":0.0,
+        "post_fusion_dim":64,
+        "post_text_dropout":0.0,
+        "post_text_dim":32,
+        "post_audio_dropout":0.0,
+        "post_audio_dim":32,
+        "post_video_dropout":0.0,
+        "post_video_dim":16,
+        "train_samples":1284,
+        "excludeZero":true,
+        "update_epochs":4,
+        "H":3,
+        "decay":false,
+        "weight_decay_bert":0.001,
+        "learning_rate_bert":3e-5,
+        "weight_decay_audio":0.01,
+        "weight_decay_video":0.0,
+        "weight_decay_other":0.01,
+        "learning_rate_audio":0.0005,
+        "learning_rate_video":0.0003,
+        "learning_rate_other":0.0003
+      },
+      "mosei": {
+        "batch_size": 32,
+        "transformers": "bert",
+        "pretrained": "bert-base-uncased",
+        "a_lstm_hidden_size":32,
+        "a_lstm_layers":1,
+        "a_lstm_dropout":0.0,
+        "v_lstm_hidden_size":32,
+        "v_lstm_layers":1,
+        "v_lstm_dropout":0.0,
+        "conv1d_kernel_size_l":1,
+        "conv1d_kernel_size_a":1,
+        "conv1d_kernel_size_v":1,
+        "dst_feature_dims":50,
+        "nheads":5,
+        "attn_dropout":0.1,
+        "attn_dropout_a":0.0,
+        "attn_dropout_v":0.1,
+        "relu_dropout":0.1,
+        "embed_dropout":0.1,
+        "res_dropout":0.0,
+        "post_fusion_dropout":0.0,
+        "post_fusion_dim":64,
+        "post_text_dropout":0.1,
+        "post_text_dim":64,
+        "post_audio_dropout":0.0,
+        "post_audio_dim":32,
+        "post_video_dropout":0.1,
+        "post_video_dim":16,
+        "train_samples":16326,
+        "excludeZero":true,
+        "update_epochs":4,
+        "H":3,
+        "decay":false,
+        "weight_decay_bert":0.001,
+        "learning_rate_bert":3e-5,
+        "weight_decay_audio":0.0,
+        "weight_decay_video":0.001,
+        "weight_decay_other":0.01,
+        "learning_rate_audio":0.001,
+        "learning_rate_video":0.005,
+        "learning_rate_other":0.0001
+      },
+      "sims": {
+        "batch_size": 64,
+        "transformers": "bert",
+        "pretrained": "bert-base-chinese",
+        "a_lstm_hidden_size":16,
+        "a_lstm_layers":1,
+        "a_lstm_dropout":0.0,
+        "v_lstm_hidden_size":64,
+        "v_lstm_layers":1,
+        "v_lstm_dropout":0.0,
+        "conv1d_kernel_size_l":3,
+        "conv1d_kernel_size_a":5,
+        "conv1d_kernel_size_v":1,
+        "dst_feature_dims":50,
+        "nheads":5,
+        "attn_dropout":0.0,
+        "attn_dropout_a":0.1,
+        "attn_dropout_v":0.0,
+        "relu_dropout":0.1,
+        "embed_dropout":0.0,
+        "res_dropout":0.0,
+        "post_fusion_dropout":0.1,
+        "post_fusion_dim":64,
+        "post_text_dropout":0.0,
+        "post_text_dim":64,
+        "post_audio_dropout":0.0,
+        "post_audio_dim":16,
+        "post_video_dropout":0.0,
+        "post_video_dim":32,
+        "train_samples":1368,
+        "excludeZero":true,
+        "update_epochs":4,
+        "H":3,
+        "decay":false,
+        "weight_decay_bert":0.001,
+        "learning_rate_bert":3e-5,
+        "weight_decay_audio":0.0,
+        "weight_decay_video":0.01,
+        "weight_decay_other":0.001,
+        "learning_rate_audio":0.003,
+        "learning_rate_video":0.0005,
+        "learning_rate_other":0.003
+      },
+      "simsv2": {
+        "batch_size": 64,
+        "transformers": "bert",
+        "pretrained": "bert-base-chinese",
+        "a_lstm_hidden_size":16,
+        "a_lstm_layers":1,
+        "a_lstm_dropout":0.0,
+        "v_lstm_hidden_size":32,
+        "v_lstm_layers":1,
+        "v_lstm_dropout":0.0,
+        "conv1d_kernel_size_l":5,
+        "conv1d_kernel_size_a":3,
+        "conv1d_kernel_size_v":1,
+        "dst_feature_dims":50,
+        "nheads":5,
+        "attn_dropout":0.1,
+        "attn_dropout_a":0.0,
+        "attn_dropout_v":0.1,
+        "relu_dropout":0.0,
+        "embed_dropout":0.0,
+        "res_dropout":0.1,
+        "post_fusion_dropout":0.1,
+        "post_fusion_dim":128,
+        "post_text_dropout":0.1,
+        "post_text_dim":64,
+        "post_audio_dropout":0.0,
+        "post_audio_dim":16,
+        "post_video_dropout":0.0,
+        "post_video_dim":32,
+        "train_samples":2722,
+        "excludeZero":true,
+        "update_epochs":4,
+        "H":3,
+        "decay":false,
+        "weight_decay_bert":0.001,
+        "learning_rate_bert":5e-6,
+        "weight_decay_audio":0.001,
+        "weight_decay_video":0.001,
+        "weight_decay_other":0.001,
+        "learning_rate_audio":0.0005,
+        "learning_rate_video":0.005,
+        "learning_rate_other":0.0003
+      }
+    }
+  },
+  "cenet":{
+    "commonParams": {
+      "need_data_aligned": false,
+      "need_model_aligned": false,
+      "need_normalized": false,
+      "use_bert": true,
+      "use_finetune": true,
+      "early_stop": 8
+    },
+    "datasetParams": {
+      "mosi": {
+        "pretrained": "bert-base-uncased",
+        "learning_rate":1e-5,
+        "weight_decay":0.0001,
+        "max_grad_norm":2,
+        "adam_epsilon":3e-8,
+        "batch_size":64
+      },
+      "mosei":{
+        "pretrained": "bert-base-uncased",
+        "learning_rate":1e-5,
+        "weight_decay":0.0001,
+        "max_grad_norm":2,
+        "adam_epsilon":1e-8,
+        "batch_size":64
+      },
+      "sims":{
+        "pretrained": "bert-base-chinese",
+        "learning_rate":2e-6,
+        "weight_decay":0.0,
+        "max_grad_norm":2,
+        "adam_epsilon":2e-8,
+        "batch_size":32
+      },
+      "simsv2":{
+        "pretrained": "bert-base-chinese",
+        "learning_rate":3e-5,
+        "weight_decay":0.0,
+        "max_grad_norm":2,
+        "adam_epsilon":3e-8,
+        "batch_size":64
+      }
+    }
   }
 }