From 6b801511692772aa6b8c4b4b3f9ac7d4bf694339 Mon Sep 17 00:00:00 2001
From: Colin Raffel <craffel@gmail.com>
Date: Wed, 24 Feb 2016 18:26:50 -0800
Subject: [PATCH] Add tests and data for key detection

This data is synthetic; there are relatively few cases to test so I
think this is ok for now. It would be better to get real data though.
---
 tests/data/key/est00.txt     |  1 +
 tests/data/key/est01.txt     |  1 +
 tests/data/key/est02.txt     |  1 +
 tests/data/key/est03.txt     |  1 +
 tests/data/key/est04.txt     |  1 +
 tests/data/key/est05.txt     |  1 +
 tests/data/key/est06.txt     |  1 +
 tests/data/key/est07.txt     |  1 +
 tests/data/key/est08.txt     |  1 +
 tests/data/key/est09.txt     |  1 +
 tests/data/key/output00.json |  1 +
 tests/data/key/output01.json |  1 +
 tests/data/key/output02.json |  1 +
 tests/data/key/output03.json |  1 +
 tests/data/key/output04.json |  1 +
 tests/data/key/output05.json |  1 +
 tests/data/key/output06.json |  1 +
 tests/data/key/output07.json |  1 +
 tests/data/key/output08.json |  1 +
 tests/data/key/output09.json |  1 +
 tests/data/key/ref00.txt     |  1 +
 tests/data/key/ref01.txt     |  1 +
 tests/data/key/ref02.txt     |  1 +
 tests/data/key/ref03.txt     |  1 +
 tests/data/key/ref04.txt     |  1 +
 tests/data/key/ref05.txt     |  1 +
 tests/data/key/ref06.txt     |  1 +
 tests/data/key/ref07.txt     |  1 +
 tests/data/key/ref08.txt     |  1 +
 tests/data/key/ref09.txt     |  1 +
 tests/generate_data.py       |  2 ++
 tests/test_key.py            | 66 ++++++++++++++++++++++++++++++++++++
 32 files changed, 98 insertions(+)
 create mode 100644 tests/data/key/est00.txt
 create mode 100644 tests/data/key/est01.txt
 create mode 100644 tests/data/key/est02.txt
 create mode 100644 tests/data/key/est03.txt
 create mode 100644 tests/data/key/est04.txt
 create mode 100644 tests/data/key/est05.txt
 create mode 100644 tests/data/key/est06.txt
 create mode 100644 tests/data/key/est07.txt
 create mode 100644 tests/data/key/est08.txt
 create mode 100644 tests/data/key/est09.txt
 create mode 100644 tests/data/key/output00.json
 create mode 100644 tests/data/key/output01.json
 create mode 100644 tests/data/key/output02.json
 create mode 100644 tests/data/key/output03.json
 create mode 100644 tests/data/key/output04.json
 create mode 100644 tests/data/key/output05.json
 create mode 100644 tests/data/key/output06.json
 create mode 100644 tests/data/key/output07.json
 create mode 100644 tests/data/key/output08.json
 create mode 100644 tests/data/key/output09.json
 create mode 100644 tests/data/key/ref00.txt
 create mode 100644 tests/data/key/ref01.txt
 create mode 100644 tests/data/key/ref02.txt
 create mode 100644 tests/data/key/ref03.txt
 create mode 100644 tests/data/key/ref04.txt
 create mode 100644 tests/data/key/ref05.txt
 create mode 100644 tests/data/key/ref06.txt
 create mode 100644 tests/data/key/ref07.txt
 create mode 100644 tests/data/key/ref08.txt
 create mode 100644 tests/data/key/ref09.txt
 create mode 100644 tests/test_key.py

diff --git a/tests/data/key/est00.txt b/tests/data/key/est00.txt
new file mode 100644
index 00000000..e62df4c7
--- /dev/null
+++ b/tests/data/key/est00.txt
@@ -0,0 +1 @@
+C	major
\ No newline at end of file
diff --git a/tests/data/key/est01.txt b/tests/data/key/est01.txt
new file mode 100644
index 00000000..d6d5964d
--- /dev/null
+++ b/tests/data/key/est01.txt
@@ -0,0 +1 @@
+G	minor
\ No newline at end of file
diff --git a/tests/data/key/est02.txt b/tests/data/key/est02.txt
new file mode 100644
index 00000000..804254e5
--- /dev/null
+++ b/tests/data/key/est02.txt
@@ -0,0 +1 @@
+Ab	minor
diff --git a/tests/data/key/est03.txt b/tests/data/key/est03.txt
new file mode 100644
index 00000000..9180b375
--- /dev/null
+++ b/tests/data/key/est03.txt
@@ -0,0 +1 @@
+D	major
\ No newline at end of file
diff --git a/tests/data/key/est04.txt b/tests/data/key/est04.txt
new file mode 100644
index 00000000..19eb38b5
--- /dev/null
+++ b/tests/data/key/est04.txt
@@ -0,0 +1 @@
+F#	minor
\ No newline at end of file
diff --git a/tests/data/key/est05.txt b/tests/data/key/est05.txt
new file mode 100644
index 00000000..132f59d5
--- /dev/null
+++ b/tests/data/key/est05.txt
@@ -0,0 +1 @@
+A	major
diff --git a/tests/data/key/est06.txt b/tests/data/key/est06.txt
new file mode 100644
index 00000000..e1036665
--- /dev/null
+++ b/tests/data/key/est06.txt
@@ -0,0 +1 @@
+Gb	minor
\ No newline at end of file
diff --git a/tests/data/key/est07.txt b/tests/data/key/est07.txt
new file mode 100644
index 00000000..8c0d8003
--- /dev/null
+++ b/tests/data/key/est07.txt
@@ -0,0 +1 @@
+G	major
\ No newline at end of file
diff --git a/tests/data/key/est08.txt b/tests/data/key/est08.txt
new file mode 100644
index 00000000..e62df4c7
--- /dev/null
+++ b/tests/data/key/est08.txt
@@ -0,0 +1 @@
+C	major
\ No newline at end of file
diff --git a/tests/data/key/est09.txt b/tests/data/key/est09.txt
new file mode 100644
index 00000000..354c63db
--- /dev/null
+++ b/tests/data/key/est09.txt
@@ -0,0 +1 @@
+C	minor
\ No newline at end of file
diff --git a/tests/data/key/output00.json b/tests/data/key/output00.json
new file mode 100644
index 00000000..29693fb6
--- /dev/null
+++ b/tests/data/key/output00.json
@@ -0,0 +1 @@
+{"Weighted Score": 1.0}
\ No newline at end of file
diff --git a/tests/data/key/output01.json b/tests/data/key/output01.json
new file mode 100644
index 00000000..29693fb6
--- /dev/null
+++ b/tests/data/key/output01.json
@@ -0,0 +1 @@
+{"Weighted Score": 1.0}
\ No newline at end of file
diff --git a/tests/data/key/output02.json b/tests/data/key/output02.json
new file mode 100644
index 00000000..e9216527
--- /dev/null
+++ b/tests/data/key/output02.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.5}
\ No newline at end of file
diff --git a/tests/data/key/output03.json b/tests/data/key/output03.json
new file mode 100644
index 00000000..c5815450
--- /dev/null
+++ b/tests/data/key/output03.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.0}
\ No newline at end of file
diff --git a/tests/data/key/output04.json b/tests/data/key/output04.json
new file mode 100644
index 00000000..73c07220
--- /dev/null
+++ b/tests/data/key/output04.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.3}
\ No newline at end of file
diff --git a/tests/data/key/output05.json b/tests/data/key/output05.json
new file mode 100644
index 00000000..73c07220
--- /dev/null
+++ b/tests/data/key/output05.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.3}
\ No newline at end of file
diff --git a/tests/data/key/output06.json b/tests/data/key/output06.json
new file mode 100644
index 00000000..0d4f47fe
--- /dev/null
+++ b/tests/data/key/output06.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.2}
\ No newline at end of file
diff --git a/tests/data/key/output07.json b/tests/data/key/output07.json
new file mode 100644
index 00000000..0d4f47fe
--- /dev/null
+++ b/tests/data/key/output07.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.2}
\ No newline at end of file
diff --git a/tests/data/key/output08.json b/tests/data/key/output08.json
new file mode 100644
index 00000000..c5815450
--- /dev/null
+++ b/tests/data/key/output08.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.0}
\ No newline at end of file
diff --git a/tests/data/key/output09.json b/tests/data/key/output09.json
new file mode 100644
index 00000000..c5815450
--- /dev/null
+++ b/tests/data/key/output09.json
@@ -0,0 +1 @@
+{"Weighted Score": 0.0}
\ No newline at end of file
diff --git a/tests/data/key/ref00.txt b/tests/data/key/ref00.txt
new file mode 100644
index 00000000..e62df4c7
--- /dev/null
+++ b/tests/data/key/ref00.txt
@@ -0,0 +1 @@
+C	major
\ No newline at end of file
diff --git a/tests/data/key/ref01.txt b/tests/data/key/ref01.txt
new file mode 100644
index 00000000..d6d5964d
--- /dev/null
+++ b/tests/data/key/ref01.txt
@@ -0,0 +1 @@
+G	minor
\ No newline at end of file
diff --git a/tests/data/key/ref02.txt b/tests/data/key/ref02.txt
new file mode 100644
index 00000000..fb8eb3f4
--- /dev/null
+++ b/tests/data/key/ref02.txt
@@ -0,0 +1 @@
+Db	minor
\ No newline at end of file
diff --git a/tests/data/key/ref03.txt b/tests/data/key/ref03.txt
new file mode 100644
index 00000000..de2d3e66
--- /dev/null
+++ b/tests/data/key/ref03.txt
@@ -0,0 +1 @@
+A	minor
\ No newline at end of file
diff --git a/tests/data/key/ref04.txt b/tests/data/key/ref04.txt
new file mode 100644
index 00000000..2ffad075
--- /dev/null
+++ b/tests/data/key/ref04.txt
@@ -0,0 +1 @@
+A	major
\ No newline at end of file
diff --git a/tests/data/key/ref05.txt b/tests/data/key/ref05.txt
new file mode 100644
index 00000000..e7a294cc
--- /dev/null
+++ b/tests/data/key/ref05.txt
@@ -0,0 +1 @@
+F#	minor
diff --git a/tests/data/key/ref06.txt b/tests/data/key/ref06.txt
new file mode 100644
index 00000000..ec25c12b
--- /dev/null
+++ b/tests/data/key/ref06.txt
@@ -0,0 +1 @@
+Gb	major
\ No newline at end of file
diff --git a/tests/data/key/ref07.txt b/tests/data/key/ref07.txt
new file mode 100644
index 00000000..d6d5964d
--- /dev/null
+++ b/tests/data/key/ref07.txt
@@ -0,0 +1 @@
+G	minor
\ No newline at end of file
diff --git a/tests/data/key/ref08.txt b/tests/data/key/ref08.txt
new file mode 100644
index 00000000..8c0d8003
--- /dev/null
+++ b/tests/data/key/ref08.txt
@@ -0,0 +1 @@
+G	major
\ No newline at end of file
diff --git a/tests/data/key/ref09.txt b/tests/data/key/ref09.txt
new file mode 100644
index 00000000..de2d3e66
--- /dev/null
+++ b/tests/data/key/ref09.txt
@@ -0,0 +1 @@
+A	minor
\ No newline at end of file
diff --git a/tests/generate_data.py b/tests/generate_data.py
index e2bed131..97a7b606 100755
--- a/tests/generate_data.py
+++ b/tests/generate_data.py
@@ -68,6 +68,8 @@ def load_separation_data(folder):
     tasks['transcription'] = (mir_eval.transcription,
                               mir_eval.io.load_valued_intervals,
                               'tests/data/transcription/{}*.txt')
+    tasks['key'] = (mir_eval.key, mir_eval.io.load_key,
+                    'tests/data/key/{}*.txt')
     # Get task keys from argv
     for task in sys.argv[1:]:
         print('Generating data for {}'.format(task))
diff --git a/tests/test_key.py b/tests/test_key.py
new file mode 100644
index 00000000..eda7bd31
--- /dev/null
+++ b/tests/test_key.py
@@ -0,0 +1,66 @@
+'''
+Tests for mir_eval.key
+'''
+
+import mir_eval
+import nose.tools
+import glob
+import json
+import numpy as np
+
+A_TOL = 1e-12
+
+# Path to the fixture files
+REF_GLOB = 'tests/data/key/ref*.txt'
+EST_GLOB = 'tests/data/key/est*.txt'
+SCORES_GLOB = 'tests/data/key/output*.json'
+
+
+def __unit_test_key_function(metric):
+
+    good_keys = ['C major', 'c major', 'C# major', 'Bb minor', 'db minor']
+    # All of these are invalid key strings
+    bad_keys = ['C maj', 'Cb major', 'C', 'K major', 'F## minor']
+
+    for good_key in good_keys:
+        for bad_key in bad_keys:
+            # Should raise an error whether we pass a bad key as ref or est
+            nose.tools.assert_raises(ValueError, metric, good_key, bad_key)
+            nose.tools.assert_raises(ValueError, metric, bad_key, good_key)
+
+    for good_key in good_keys:
+        # When the same key is passed for est and ref, score should be 1
+        assert mir_eval.key.weighted_score(good_key, good_key) == 1.
+
+
+def __check_score(sco_f, metric, score, expected_score):
+    assert np.allclose(score, expected_score, atol=A_TOL)
+
+
+def test_key_functions():
+    # Load in all files in the same order
+    ref_files = sorted(glob.glob(REF_GLOB))
+    est_files = sorted(glob.glob(EST_GLOB))
+    sco_files = sorted(glob.glob(SCORES_GLOB))
+
+    assert len(ref_files) == len(est_files) == len(sco_files) > 0
+
+    # Unit test all metrics (one for now)
+    for metric in [mir_eval.key.weighted_score]:
+        yield __unit_test_key_function, metric
+
+    # Regression tests
+    for ref_f, est_f, sco_f in zip(ref_files, est_files, sco_files):
+        with open(sco_f, 'r') as f:
+            expected_scores = json.load(f)
+        # Load in an example key annotation
+        reference_key = mir_eval.io.load_key(ref_f)
+        # Load in an example key detector output
+        estimated_key = mir_eval.io.load_key(est_f)
+        # Compute scores
+        scores = mir_eval.key.evaluate(reference_key, estimated_key)
+        # Compare them
+        for metric in scores:
+            # This is a simple hack to make nosetest's messages more useful
+            yield (__check_score, sco_f, metric, scores[metric],
+                   expected_scores[metric])