eval_post_ecapa_cos_ft.yaml

root_dir: .
exp_dir: !ref <root_dir>/exp

datasets: !include:datasets_vpc2022_official_eval.yaml

eval_steps:  # all metrics in this list will be computed in the evaluation. Remove entry to skip
  privacy:
    - asv
  utility:
    - asr
    - gvd

anon_data_suffix: anon  # suffix for dataset to signal that it is anonymized
eval_data_dir: !PLACEHOLDER  # path to anonymized evaluation data in kaldi format, e.g. <eval_data_dir>/libri_test_enrolls/wav.scp etc.

privacy:
  asv:
    model_dir: !ref <exp_dir>/asv_anon_<privacy[asv][vec_type]>_ft  # path to existing ASV model or output for trained ASV model
    vec_type: ecapa  # ecapa or xvector

    training:
      anon: true  # true or false, depending on whether the training data for the ASV is anonymized or original
      train_data_dir: !ref <eval_data_dir>/train-clean-360  # path to original or anonymized training data for ASV
      train_config: evaluation/privacy/asv_train/hparams/train_ecapa_tdnn_small_ft.yaml
      finetuning: true # true (ft) or false (scratch)
      pretrained_model: !ref <exp_dir>/asv_pre_<privacy[asv][vec_type]>/CKPT+10  # path to pretrained model, only used for finetuning
      lr: 0.01
      epochs: 10
      batch_size: 256
      num_utt: 50  # ALL or specific number, number of utterances per speaker
      utt_selection: spk-diverse-sess  # select utterances per speaker and session (spk-sess), per speaker and randomly across all sessions (spk-random), per speaker and balanced across sessions (spk-diverse-sess)
      num_spk: ALL  # ALL or specific number, number of speakers
      retrain: false  # retrain in any case (true) or skip training if model exists (false)

    evaluation:
      results_dir: !ref <privacy[asv][model_dir]>  # path to save evaluation results
      distance: cosine  # cosine or plda
      plda:   # ignored if distance is not plda
        model_dir: null  # path to trained PLDA or output of PLDA training
        train_data_dir: null # path to PLDA training data
        anon: !ref <privacy[asv][training][anon]>  # trained on anonymized (true) or original (false) data

utility:
  asr:
    libri_dir: !PLACEHOLDER  # path to parent dir of original LibriSpeech for data preparation, needs the structure <libri_dir>/LibriSpeech/LICENSE.TXT etc.
    model_name: asr_anon_ft  # name for ASR model
    model_dir: !ref <exp_dir>/<utility[asr][model_name]>  # path to existing ASR model or output for trained ASR model

    training:
      anon: true  # true or false, depending on whether the training data for the ASR is anonymized or original
      train_data_dir: !ref <eval_data_dir>/train-clean-360  # path to original or anonymized training data for ASR
      num_gpus: 2  # how many GPUs to use for training (if available)
      finetuning: true  # true (ft) or false (scratch)
      pretrained_model: !ref <exp_dir>/asr_pre/asr_train_asr_transformer_raw_en_bpe5000/valid.acc.ave.pth  # path to pretrained model, only used for finetuning
      num_utt: 50  # ALL or specific number, number of utterances per speaker
      num_spk: ALL  # ALL or specific number, number of speakers
      nj: 4  # number of parallel jobs
      retrain: false  # retrain in any case (true) or skip training if model exists (false)

    evaluation:
      num_gpus: 4  # how many GPUs to use for evaluation (if available)
      lm_dir: !ref <exp_dir>/asr_pre/lm_train_lm_transformer2_en_bpe5000  # path to trained language model
      nj: 4  # number of parallel jobs

  gvd:
    num_utt: 5  # all or specific number, number of utterances per speaker
    asv_params:  # settings for ASV using for GVD evaluation
      model_dir: !ref <privacy[asv][model_dir]>
      vec_type: !ref <privacy[asv][vec_type]>
      evaluation:
        plda: !ref <privacy[asv][evaluation][plda]>
        distance: !ref <privacy[asv][evaluation][distance]>
    results_dir: !ref <privacy[asv][model_dir]>  # path to results dir for GVD computation
    recompute: false  # recompute similarity matrices in any case (true) or skip if matrices exist (false)