experiments/eval/eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preliminaries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running the models\n",
    "\n",
    "We need to run each of the models on each of our test sets and in both decoding modes (greedy and sampling) and store the results in the `out` subdirectory of the model directory. The 4 commands to run for each model are in the `run_model_eval.sh` script, which expects the model directory as an argument, e.g. `./run_model_eval.sh ../v01`. For a speedup, consider parallelizing this on a cluster and tweaking the batch size.\n",
    "\n",
    "**Note:** The script will try to guess whether or not to input drums by detecting whether the model directory name contains `drums`. If your models are named differently or expect different sets of instruments, you will need to modify the script."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing style profiles\n",
    "\n",
    "For evaluation on the synthetic test set, we need to compute the reference style profiles for all styles in the test set. This is done using the script `compute.sh` in the `style_profiles` directory. The different types of style profiles are defined in the `config.yaml` and `config_drums.yaml` files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!(cd style_profiles; ./compute.sh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import concurrent.futures as cf\n",
    "import csv\n",
    "import functools\n",
    "import gzip\n",
    "import itertools\n",
    "import json\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from note_seq import notebook_utils\n",
    "from note_seq import sequences_lib\n",
    "from note_seq import midi_synth\n",
    "from note_seq.protobuf import music_pb2\n",
    "from confugue import Configuration\n",
    "from museflow.io.note_sequence_io import NoteSequenceDB\n",
    "from museflow.note_sequence_utils import filter_sequence\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from groove2groove.eval.style_profiles import extract_all_stats\n",
    "from groove2groove.eval.notes_chroma_similarity import chroma_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = '../../data/synth'\n",
    "BODHIDHARMA_DIR = '../../data/bodhidharma'\n",
    "STYLE_PROFILE_DIR = 'style_profiles'\n",
    "STYLE_PROFILE_CFG_PATH = os.path.join(STYLE_PROFILE_DIR, 'config.yaml')\n",
    "STYLE_PROFILE_DRUMS_CFG_PATH = os.path.join(STYLE_PROFILE_DIR, 'config_drums.yaml')\n",
    "OUT_PREFIX = 'out/'\n",
    "INSTRUMENTS = ['Bass', 'Piano', 'Guitar', 'Strings']\n",
    "DRUMS = ['Drums']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load reference style profiles\n",
    "ref_profiles = {}\n",
    "for instr in INSTRUMENTS + DRUMS:\n",
    "    ref_profiles[instr] = collections.defaultdict(dict)\n",
    "    with open(os.path.join(STYLE_PROFILE_DIR, f'{instr}.json')) as f:\n",
    "        for profile_type, style_dict in json.load(f).items():\n",
    "            for style, profile in style_dict.items():\n",
    "                assert profile is not None\n",
    "                if profile is not None:\n",
    "                    ref_profiles[instr][style][profile_type] = np.asarray(profile)\n",
    "    ref_profiles[instr] = dict(ref_profiles[instr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_sequences(sequences, **kwargs):\n",
    "    return [filter_sequence(seq, copy=True, **kwargs) for seq in sequences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_similarity(hist1, hist2):\n",
    "    return 1. - scipy.spatial.distance.cosine(hist1.reshape(1, -1), hist2.reshape(1, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(STYLE_PROFILE_CFG_PATH, 'rb') as f:\n",
    "    STYLE_PROFILE_FN = Configuration.from_yaml(f).bind(extract_all_stats)\n",
    "with open(STYLE_PROFILE_DRUMS_CFG_PATH, 'rb') as f:\n",
    "    STYLE_PROFILE_DRUMS_FN = Configuration.from_yaml(f).bind(extract_all_stats)\n",
    "\n",
    "def evaluate_style(sequences, ref_stats=None, ref_sequences=None, is_drum=False, separate_drums=False):\n",
    "    \"\"\"Evaluate the style similarity of a set of sequences to a reference.\"\"\"\n",
    "    extract_fn = STYLE_PROFILE_FN if not is_drum else STYLE_PROFILE_DRUMS_FN\n",
    "    stats = extract_fn(data=sequences)\n",
    "    if ref_stats is None:\n",
    "        ref_stats = extract_fn(data=ref_sequences)\n",
    "    metrics = {name + ('_drums' if is_drum and separate_drums else ''):\n",
    "                   cosine_similarity(stats[name], ref_stats[name])\n",
    "               for name in stats if name in ref_stats}\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_content(sequence, reference):\n",
    "    \"\"\"Evaluate the content similarity of a sequence to a reference.\"\"\"\n",
    "    sequence = filter_sequence(sequence, drums=False, copy=True)\n",
    "    reference = filter_sequence(reference, drums=False, copy=True)\n",
    "    return {\n",
    "        'content': chroma_similarity(sequence, reference,\n",
    "                                     sampling_rate=12, window_size=24, stride=12, use_velocity=False)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_content_par(sequences, references):\n",
    "    \"\"\"Evaluate the content similarity of a list of sequences to a list of references.\"\"\"\n",
    "    with cf.ProcessPoolExecutor(max_workers=12) as pool:\n",
    "        results = pool.map(_evaluate_content_par_task, zip(sequences, references))\n",
    "        return list(tqdm(results, total=len(sequences), desc='content', leave=False))\n",
    "\n",
    "def _evaluate_content_par_task(args):\n",
    "    return evaluate_content(*args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_style_one_instrument(data, outputs, out_dict, instr, ref_instr=None, micro=False):\n",
    "    \"\"\"Evaluate one model on all style metrics for one instrument.\"\"\"\n",
    "    if not ref_instr:\n",
    "        ref_instr = instr\n",
    "\n",
    "    # Filter the outputs to include only the desired instrument, then join them with the metadata\n",
    "    outputs_filtered = filter_sequences(outputs, instrument_re=f'^{instr}$')\n",
    "    outputs_and_metadata = pd.concat([\n",
    "        data[['src_style', 'tgt_style']],\n",
    "        pd.Series(outputs_filtered, name='output')\n",
    "    ], axis=1)\n",
    "\n",
    "    if micro:\n",
    "        grouped = outputs_and_metadata.groupby(['src_style', 'tgt_style'])\n",
    "    else:\n",
    "        grouped = outputs_and_metadata.groupby(['tgt_style'])\n",
    "\n",
    "    for key, df in tqdm(grouped, desc='style', leave=False):\n",
    "        tgt_style = key[1] if micro else key\n",
    "        if tgt_style not in ref_profiles[ref_instr]:\n",
    "            continue\n",
    "\n",
    "        out_dict['style'].append(\n",
    "            evaluate_style(df['output'], ref_stats=ref_profiles[ref_instr][tgt_style], is_drum=instr in DRUMS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_style_nano(data, outputs, out_dict, max_over_programs=False):\n",
    "    \"\"\"Evaluate one model on all 'nano' style metrics.\"\"\"\n",
    "    with cf.ProcessPoolExecutor(max_workers=12) as pool:\n",
    "        task_fn = functools.partial(_evaluate_style_nano_par_task, max_over_programs=max_over_programs)\n",
    "        task_results = pool.map(task_fn, ((output, data_row) for output, (_, data_row) in zip(outputs, data.iterrows())))\n",
    "        task_results = tqdm(task_results, total=len(outputs), desc='style', leave=False)\n",
    "        for out_rows in task_results:\n",
    "            out_dict['style'].extend(out_rows)\n",
    "\n",
    "def _evaluate_style_nano_par_task(args, max_over_programs):\n",
    "    output, data_row = args\n",
    "    reference = data_row['style_seq']\n",
    "    reference_programs = sorted(set((n.program, n.is_drum) for n in reference.notes))\n",
    "    out_rows = []\n",
    "    for program, is_drum in reference_programs:\n",
    "        reference_filtered = filter_sequence(reference, programs=[program], drums=is_drum, copy=True)\n",
    "\n",
    "        if max_over_programs:\n",
    "            output_programs = sorted(set((n.program, n.is_drum) for n in output.notes))\n",
    "        else:\n",
    "            output_programs = [(program, is_drum)]\n",
    "\n",
    "        metrics_to_maximize_over = []\n",
    "        for out_program, out_is_drum in output_programs:\n",
    "            if out_is_drum != is_drum:\n",
    "                continue\n",
    "\n",
    "            output_filtered = filter_sequence(output, programs=[out_program], drums=out_is_drum, copy=True)\n",
    "            metrics_to_maximize_over.append(\n",
    "                evaluate_style([output_filtered], ref_sequences=[reference_filtered], is_drum=is_drum))\n",
    "        out_row = dict(pd.DataFrame(metrics_to_maximize_over).max())\n",
    "        out_row['src_key'], out_row['style_key'] = data_row['src_key'], data_row['style_key']\n",
    "        out_rows.append(out_row)\n",
    "    return out_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def melt_results(results, **kwargs):\n",
    "    \"\"\"Convert the results table to the long format.\"\"\"\n",
    "    results_long = []\n",
    "    for col in results:\n",
    "        for group in results[col]:\n",
    "            df_melted = results[col][group].melt(**kwargs)\n",
    "            df_melted['group'] = group\n",
    "            df_melted['model'] = col\n",
    "            results_long.append(df_melted)\n",
    "    return pd.concat(results_long)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We cache the metric values for each model in `metrics_cache`, using a hash of the serialized note sequences as the key. This ensures that when we re-run the evaluation with new outputs, we save time by only recomputing the values we don't have yet. However, if you change the references or the metric definitions, you need to clear the cache, otherwise you will get incorrect results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_cache = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True forces all requested values to be re-computed from scratch and overwrite the values in\n",
    "# the cache. However, this does not clear old, unused keys from the cache.\n",
    "overwrite_cache = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('metrics_cache.pickle', 'rb') as f:\n",
    "#     metrics_cache = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metrics_cache_key(outputs_series, tag=None):\n",
    "    serialized = outputs_series.map(lambda x: x.SerializeToString())\n",
    "    key = (outputs_series.name, pd.util.hash_pandas_object(serialized).sum())\n",
    "    if tag:\n",
    "        key = (*key, tag)\n",
    "    return key\n",
    "\n",
    "def metrics_cache_clear(series_name):\n",
    "    keys = [key for key in metrics_cache if key[0] == series_name]\n",
    "    for key in keys:\n",
    "        del metrics_cache[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_all(data, outputs, eval_style=True, tag=None):\n",
    "    \"\"\"Evaluate all models on all metrics.\"\"\"\n",
    "    metrics = collections.defaultdict(lambda: collections.defaultdict(list))\n",
    "    assert np.array_equal(outputs.index, data.index)\n",
    "\n",
    "    for col in tqdm(outputs.columns):\n",
    "        cache_key = metrics_cache_key(outputs[col], tag)\n",
    "        if not overwrite_cache and cache_key in metrics_cache:\n",
    "            metrics[col] = metrics_cache[cache_key]\n",
    "            continue\n",
    "\n",
    "        # Style metrics\n",
    "        if eval_style:\n",
    "            if 'tgt_style' in data.columns:\n",
    "                # We have style labels\n",
    "                if col == 'source':\n",
    "                    # We don't know which source instrument to choose, so we compute the maximum over all instruments.\n",
    "                    for ref_instr in tqdm(INSTRUMENTS + DRUMS, leave=False):\n",
    "                        metrics_tmp = collections.defaultdict(list)\n",
    "                        for src_instr in tqdm(INSTRUMENTS if ref_instr not in DRUMS else DRUMS, leave=False):\n",
    "                            out_dict = collections.defaultdict(list)\n",
    "                            evaluate_style_one_instrument(data, outputs[col], out_dict, src_instr, ref_instr,\n",
    "                                                          micro=True)  # group by source-target style pairs\n",
    "                            for key, vals in out_dict.items():\n",
    "                                metrics_tmp[key].append(vals)\n",
    "\n",
    "                        for key in metrics_tmp:\n",
    "                            # Check that the lists are of the same length\n",
    "                            assert len({len(m) for m in metrics_tmp[key]}) == 1\n",
    "                            for metrics_to_maximize_over in zip(*metrics_tmp[key]):\n",
    "                                metrics[col][key].append(dict(pd.DataFrame(metrics_to_maximize_over).max()))\n",
    "                else:\n",
    "                    for instr in tqdm(INSTRUMENTS + DRUMS, leave=False):\n",
    "                        evaluate_style_one_instrument(data, outputs[col], metrics[col], instr)\n",
    "            else:\n",
    "                # No style labels; compute nano metrics\n",
    "                evaluate_style_nano(data, outputs[col], metrics[col],\n",
    "                                    max_over_programs=(col == 'source'))\n",
    "\n",
    "        # Content metric\n",
    "        outputs_and_metadata = pd.concat([\n",
    "            data[['src_seq']],\n",
    "            pd.Series(outputs[col], name='output')\n",
    "        ], axis=1)\n",
    "        metrics[col]['content'].extend(evaluate_content_par(*zip(*(\n",
    "            (row['output'], row['src_seq']) for _, row in outputs_and_metadata.iterrows()))))\n",
    "\n",
    "        metrics_cache[cache_key] = metrics[col].copy()\n",
    "\n",
    "    metrics = {\n",
    "        col: {\n",
    "            m: pd.DataFrame(metrics[col][m])\n",
    "            for m in metrics[col]\n",
    "        }\n",
    "        for col in metrics\n",
    "    }\n",
    "            \n",
    "    results = pd.DataFrame()\n",
    "    results_err = pd.DataFrame()\n",
    "    for col in metrics:\n",
    "        results = results.join(pd.concat(\n",
    "            [metrics[col]['style'].mean(),\n",
    "             metrics[col]['content'].mean()]\n",
    "        ).rename(col), how='outer')\n",
    "        results_err = results_err.join(pd.concat(\n",
    "            [metrics[col]['style'].std(),\n",
    "             metrics[col]['content'].std()]\n",
    "        ).rename(col), how='outer')\n",
    "    \n",
    "    return results, results_err, metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Synthetic test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdirs = ['v01', 'v01_vel', 'v01_drums', 'v01_drums_vel', 'v01_drums_vel_perf']\n",
    "section = 'test'\n",
    "\n",
    "data = pd.DataFrame()\n",
    "outputs = pd.DataFrame()\n",
    "\n",
    "with gzip.open(os.path.join(DATA_DIR, section, 'final', 'meta.json.gz'), 'rb') as f:\n",
    "    metadata = json.load(f)\n",
    "\n",
    "# Load triplets of keys: source, style, target\n",
    "with open(f'triplets_{section}.tsv') as f:\n",
    "    key_triplets = list(csv.reader(f, delimiter='\\t'))\n",
    "    data['src_key'], data['style_key'], data['tgt_key'] = zip(*key_triplets)\n",
    "\n",
    "# Add style names\n",
    "data['src_style'] = pd.Series(metadata[key]['style'] for key in data['src_key'])\n",
    "data['tgt_style'] = pd.Series(metadata[key]['style'] for key in data['style_key'])\n",
    "\n",
    "# Load source and target sequences\n",
    "with NoteSequenceDB(os.path.join(DATA_DIR, section, 'final', 'all.db')) as db, db.begin() as txn:\n",
    "    outputs['source'], outputs['style'], outputs['target'] = zip(*((txn.get(src), txn.get(sty), txn.get(tgt)) for src, sty, tgt in key_triplets))\n",
    "    for seq in itertools.chain(outputs['source'], outputs['style'], outputs['target']):\n",
    "        for instrument_info in seq.instrument_infos:\n",
    "            assert instrument_info.name.startswith('BB ')\n",
    "            instrument_info.name = instrument_info.name[len('BB '):]\n",
    "    data['src_seq'] = outputs['source']\n",
    "\n",
    "# Load model outputs\n",
    "for logdir in logdirs:\n",
    "    for decoding in ['greedy', 'sample06']:\n",
    "        col = f'{logdir}_{decoding}'\n",
    "        with NoteSequenceDB(os.path.join('..', logdir, f'{OUT_PREFIX}{section}_{decoding}.db')) as db, db.begin() as txn:\n",
    "            outputs[col] = pd.Series(txn.get(f'{src}_{style}') for src, style, _ in key_triplets)\n",
    "\n",
    "# Compute the metrics\n",
    "results_test, results_test_err, results_test_all = evaluate_all(data, outputs)\n",
    "results_test_long = melt_results(results_test_all)\n",
    "\n",
    "results_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Synthetic test set (nano)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdirs = ['v01', 'v01_vel', 'v01_drums', 'v01_drums_vel', 'v01_drums_vel_perf']\n",
    "section = 'test'\n",
    "\n",
    "data = pd.DataFrame()\n",
    "outputs = pd.DataFrame()\n",
    "\n",
    "with gzip.open(os.path.join(DATA_DIR, section, 'final', 'meta.json.gz'), 'rb') as f:\n",
    "    metadata = json.load(f)\n",
    "\n",
    "# Load triplets of keys: source, style, target\n",
    "with open(f'triplets_{section}.tsv') as f:\n",
    "    key_triplets = list(csv.reader(f, delimiter='\\t'))\n",
    "    data['src_key'], data['style_key'], data['tgt_key'] = zip(*key_triplets)\n",
    "\n",
    "# Load source and target sequences\n",
    "with NoteSequenceDB(os.path.join(DATA_DIR, section, 'final', 'all.db')) as db, db.begin() as txn:\n",
    "    outputs['source'], outputs['style'], outputs['target'] = zip(*((txn.get(src), txn.get(sty), txn.get(tgt)) for src, sty, tgt in key_triplets))\n",
    "    for seq in itertools.chain(outputs['source'], outputs['style'], outputs['target']):\n",
    "        for instrument_info in seq.instrument_infos:\n",
    "            assert instrument_info.name.startswith('BB ')\n",
    "            instrument_info.name = instrument_info.name[len('BB '):]\n",
    "    data['src_seq'] = outputs['source']\n",
    "    data['style_seq'] = outputs['style']\n",
    "\n",
    "# Load model outputs\n",
    "for logdir in logdirs:\n",
    "    for decoding in ['greedy', 'sample06']:\n",
    "        col = f'{logdir}_{decoding}'\n",
    "        with NoteSequenceDB(os.path.join('..', logdir, f'{OUT_PREFIX}{section}_{decoding}.db')) as db, db.begin() as txn:\n",
    "            outputs[col] = pd.Series(txn.get(f'{src}_{style}') for src, style, _ in key_triplets)\n",
    "\n",
    "# Compute the metrics\n",
    "results_test_nano, results_test_nano_err, results_test_nano_all = evaluate_all(data, outputs, tag='nano')\n",
    "results_test_nano_long = melt_results(results_test_nano_all)\n",
    "\n",
    "results_test_nano"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bodhidharma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logdirs = ['v01', 'v01_vel', 'v01_drums', 'v01_drums_vel', 'v01_drums_vel_perf']\n",
    "section = 'bodh'\n",
    "\n",
    "data = pd.DataFrame()\n",
    "outputs = pd.DataFrame()\n",
    "\n",
    "with gzip.open(os.path.join(BODHIDHARMA_DIR, 'final', 'meta.json.gz'), 'rb') as f:\n",
    "    metadata = json.load(f)\n",
    "\n",
    "# Load pairs of keys: source, style\n",
    "with open(f'pairs_{section}.tsv') as f:\n",
    "    key_pairs = list(csv.reader(f, delimiter='\\t'))\n",
    "    data['src_key'], data['style_key'] = zip(*key_pairs)\n",
    "\n",
    "# Load source and style sequences\n",
    "with NoteSequenceDB(os.path.join(BODHIDHARMA_DIR, 'final', 'vel_norm_biab', 'all.db')) as db, db.begin() as txn:\n",
    "    data['src_seq'], data['style_seq'] = zip(*((txn.get(src), txn.get(sty)) for src, sty in key_pairs))\n",
    "outputs['source'] = data['src_seq']\n",
    "outputs['style'] = data['style_seq']\n",
    "\n",
    "# Load model outputs\n",
    "for logdir in logdirs:\n",
    "    for decoding in ['greedy', 'sample06']:\n",
    "        col = f'{logdir}_{decoding}'\n",
    "        with NoteSequenceDB(os.path.join('..', logdir, f'{OUT_PREFIX}{section}_{decoding}.db')) as db, db.begin() as txn:\n",
    "            outputs[col] = pd.Series(txn.get(f'{src}_{style}', music_pb2.NoteSequence())\n",
    "                                     for src, style in key_pairs)\n",
    "\n",
    "# Compute the metrics\n",
    "results_bodh, results_bodh_err, results_bodh_all = evaluate_all(data, outputs)\n",
    "results_bodh_long = melt_results(results_bodh_all)\n",
    "\n",
    "results_bodh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('metrics_cache.pickle', 'wb') as f:\n",
    "    pickle.dump(metrics_cache, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "groove2groove TF1.12",
   "language": "python",
   "name": "groove2groove"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}