Updates Default Configs For New Census Data (#47)

* Resolves Issue #37

* Resolves UMAP runtime issues and VAE optimizer parameters

* Removes outdata data pre-processing scripts

* Adds data download scripts

* Adds option to download a random subset

* Adds stat taking functions

* Adds multi-processing to the data download

* Updates stat functions to be per-category

* Adds documentation

* Fixes bug with transposing the metadata counts dataframe

* Updates default configs to match the new census data release

configs/data/local.yaml

@@ -6,78 +6,78 @@ init_args:
       name: human
       return_dense: true
       batch_size: 128
-      directory_path: /mnt/projects/debruinz_project/summer_census_data/3m_subset
+      directory_path: /mnt/projects/debruinz_project/july2024_census_data/subset
       train_npz_masks: 
-      - 3m_human_counts_1.npz
-      - 3m_human_counts_2.npz
-      - 3m_human_counts_3.npz
-      - 3m_human_counts_4.npz
-      - 3m_human_counts_5.npz
-      - 3m_human_counts_6.npz
-      - 3m_human_counts_7.npz
-      - 3m_human_counts_8.npz
-      - 3m_human_counts_9.npz
-      - 3m_human_counts_10.npz
-      - 3m_human_counts_11.npz
-      - 3m_human_counts_12.npz
-      - 3m_human_counts_13.npz
+      - human_counts_1.npz
+      - human_counts_2.npz
+      - human_counts_3.npz
+      - human_counts_4.npz
+      - human_counts_5.npz
+      - human_counts_6.npz
+      - human_counts_7.npz
+      - human_counts_8.npz
+      - human_counts_9.npz
+      - human_counts_10.npz
+      - human_counts_11.npz
+      - human_counts_12.npz
+      - human_counts_13.npz
       train_metadata_masks:
-      - 3m_human_metadata_1.pkl
-      - 3m_human_metadata_2.pkl
-      - 3m_human_metadata_3.pkl
-      - 3m_human_metadata_4.pkl
-      - 3m_human_metadata_5.pkl
-      - 3m_human_metadata_6.pkl
-      - 3m_human_metadata_7.pkl
-      - 3m_human_metadata_8.pkl
-      - 3m_human_metadata_9.pkl
-      - 3m_human_metadata_10.pkl
-      - 3m_human_metadata_11.pkl
-      - 3m_human_metadata_12.pkl
-      - 3m_human_metadata_13.pkl
-      val_npz_masks: 3m_human_counts_14.npz
-      val_metadata_masks: 3m_human_metadata_14.pkl
-      test_npz_masks: 3m_human_counts_15.npz
-      test_metadata_masks: 3m_human_metadata_15.pkl
+      - human_metadata_1.pkl
+      - human_metadata_2.pkl
+      - human_metadata_3.pkl
+      - human_metadata_4.pkl
+      - human_metadata_5.pkl
+      - human_metadata_6.pkl
+      - human_metadata_7.pkl
+      - human_metadata_8.pkl
+      - human_metadata_9.pkl
+      - human_metadata_10.pkl
+      - human_metadata_11.pkl
+      - human_metadata_12.pkl
+      - human_metadata_13.pkl
+      val_npz_masks: human_counts_14.npz
+      val_metadata_masks: human_metadata_14.pkl
+      test_npz_masks: human_counts_15.npz
+      test_metadata_masks: human_metadata_15.pkl
       verbose: false
   - class_path: cmmvae.data.local.SpeciesManager
     init_args:
       name: mouse
       return_dense: true
       batch_size: 128
-      directory_path: /mnt/projects/debruinz_project/summer_census_data/3m_subset
+      directory_path: /mnt/projects/debruinz_project/july2024_census_data/subset
       train_npz_masks: 
-      - 3m_mouse_counts_1.npz
-      - 3m_mouse_counts_2.npz
-      - 3m_mouse_counts_3.npz
-      - 3m_mouse_counts_4.npz
-      - 3m_mouse_counts_5.npz
-      - 3m_mouse_counts_6.npz
-      - 3m_mouse_counts_7.npz
-      - 3m_mouse_counts_8.npz
-      - 3m_mouse_counts_9.npz
-      - 3m_mouse_counts_10.npz
-      - 3m_mouse_counts_11.npz
-      - 3m_mouse_counts_12.npz
-      - 3m_mouse_counts_13.npz
+      - mouse_counts_1.npz
+      - mouse_counts_2.npz
+      - mouse_counts_3.npz
+      - mouse_counts_4.npz
+      - mouse_counts_5.npz
+      - mouse_counts_6.npz
+      - mouse_counts_7.npz
+      - mouse_counts_8.npz
+      - mouse_counts_9.npz
+      - mouse_counts_10.npz
+      - mouse_counts_11.npz
+      - mouse_counts_12.npz
+      - mouse_counts_13.npz
       train_metadata_masks:
-      - 3m_mouse_metadata_1.pkl
-      - 3m_mouse_metadata_2.pkl
-      - 3m_mouse_metadata_3.pkl
-      - 3m_mouse_metadata_4.pkl
-      - 3m_mouse_metadata_5.pkl
-      - 3m_mouse_metadata_6.pkl
-      - 3m_mouse_metadata_7.pkl
-      - 3m_mouse_metadata_8.pkl
-      - 3m_mouse_metadata_9.pkl
-      - 3m_mouse_metadata_10.pkl
-      - 3m_mouse_metadata_11.pkl
-      - 3m_mouse_metadata_12.pkl
-      - 3m_mouse_metadata_13.pkl
-      val_npz_masks: 3m_mouse_counts_14.npz
-      val_metadata_masks: 3m_mouse_metadata_14.pkl
-      test_npz_masks: 3m_mouse_counts_15.npz
-      test_metadata_masks: 3m_mouse_metadata_15.pkl
+      - mouse_metadata_1.pkl
+      - mouse_metadata_2.pkl
+      - mouse_metadata_3.pkl
+      - mouse_metadata_4.pkl
+      - mouse_metadata_5.pkl
+      - mouse_metadata_6.pkl
+      - mouse_metadata_7.pkl
+      - mouse_metadata_8.pkl
+      - mouse_metadata_9.pkl
+      - mouse_metadata_10.pkl
+      - mouse_metadata_11.pkl
+      - mouse_metadata_12.pkl
+      - mouse_metadata_13.pkl
+      val_npz_masks: mouse_counts_14.npz
+      val_metadata_masks: mouse_metadata_14.pkl
+      test_npz_masks: mouse_counts_15.npz
+      test_metadata_masks: mouse_metadata_15.pkl
       verbose: false
   num_workers: 2
   n_val_workers: 1

configs/model/config.yaml

@@ -53,15 +53,15 @@ init_args:
               encoder_config:
                 class_path: cmmvae.modules.base.FCBlockConfig
                 init_args:
-                  layers: [ 60664, 1024, 512 ]
+                  layers: [ 60530, 1024, 512 ]
                   dropout_rate: [ 0.1, 0.1 ]
                   use_batch_norm: True
                   use_layer_norm: False
                   activation_fn: torch.nn.ReLU
               decoder_config:
                 class_path: cmmvae.modules.base.FCBlockConfig
                 init_args:
-                  layers: [ 512, 1024, 60664 ]
+                  layers: [ 512, 1024, 60530 ]
                   dropout_rate: 0.0
                   use_batch_norm: False
                   use_layer_norm: False
@@ -72,15 +72,15 @@ init_args:
               encoder_config:
                 class_path: cmmvae.modules.base.FCBlockConfig
                 init_args:
-                  layers: [ 52417, 1024, 512 ]
+                  layers: [ 52437, 1024, 512 ]
                   dropout_rate: [ 0.1, 0.1 ]
                   use_batch_norm: True
                   use_layer_norm: False
                   activation_fn: torch.nn.ReLU
               decoder_config:
                 class_path: cmmvae.modules.base.FCBlockConfig
                 init_args:
-                  layers: [ 512, 1024, 52417 ]
+                  layers: [ 512, 1024, 52437 ]
                   dropout_rate: 0.0
                   use_batch_norm: False
                   use_layer_norm: False