Add tests and data for key detection

This data is synthetic; there are relatively few cases to test so I
think this is ok for now. It would be better to get real data though.

tests/data/key/est00.txt

@@ -0,0 +1 @@
+C	major

tests/data/key/est01.txt

@@ -0,0 +1 @@
+G	minor

tests/data/key/est02.txt

@@ -0,0 +1 @@
+Ab	minor

tests/data/key/est03.txt

@@ -0,0 +1 @@
+D	major

tests/data/key/est04.txt

@@ -0,0 +1 @@
+F#	minor

tests/data/key/est05.txt

@@ -0,0 +1 @@
+A	major

tests/data/key/est06.txt

@@ -0,0 +1 @@
+Gb	minor

tests/data/key/est07.txt

@@ -0,0 +1 @@
+G	major

tests/data/key/est08.txt

@@ -0,0 +1 @@
+C	major

tests/data/key/est09.txt

@@ -0,0 +1 @@
+C	minor

tests/data/key/output00.json

@@ -0,0 +1 @@
+{"Weighted Score": 1.0}

tests/data/key/output01.json

@@ -0,0 +1 @@
+{"Weighted Score": 1.0}

tests/data/key/output02.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.5}

tests/data/key/output03.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.0}

tests/data/key/output04.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.3}

tests/data/key/output05.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.3}

tests/data/key/output06.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.2}

tests/data/key/output07.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.2}

tests/data/key/output08.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.0}

tests/data/key/output09.json

@@ -0,0 +1 @@
+{"Weighted Score": 0.0}

tests/data/key/ref00.txt

@@ -0,0 +1 @@
+C	major

tests/data/key/ref01.txt

@@ -0,0 +1 @@
+G	minor

tests/data/key/ref02.txt

@@ -0,0 +1 @@
+Db	minor

tests/data/key/ref03.txt

@@ -0,0 +1 @@
+A	minor

tests/data/key/ref04.txt

@@ -0,0 +1 @@
+A	major

tests/data/key/ref05.txt

@@ -0,0 +1 @@
+F#	minor

tests/data/key/ref06.txt

@@ -0,0 +1 @@
+Gb	major

tests/data/key/ref07.txt

@@ -0,0 +1 @@
+G	minor

tests/data/key/ref08.txt

@@ -0,0 +1 @@
+G	major

tests/data/key/ref09.txt

@@ -0,0 +1 @@
+A	minor

tests/generate_data.py

@@ -68,6 +68,8 @@ def load_separation_data(folder):
     tasks['transcription'] = (mir_eval.transcription,
                               mir_eval.io.load_valued_intervals,
                               'tests/data/transcription/{}*.txt')
+    tasks['key'] = (mir_eval.key, mir_eval.io.load_key,
+                    'tests/data/key/{}*.txt')
     # Get task keys from argv
     for task in sys.argv[1:]:
         print('Generating data for {}'.format(task))

tests/test_key.py

@@ -0,0 +1,66 @@
+'''
+Tests for mir_eval.key
+'''
+
+import mir_eval
+import nose.tools
+import glob
+import json
+import numpy as np
+
+A_TOL = 1e-12
+
+# Path to the fixture files
+REF_GLOB = 'tests/data/key/ref*.txt'
+EST_GLOB = 'tests/data/key/est*.txt'
+SCORES_GLOB = 'tests/data/key/output*.json'
+
+
+def __unit_test_key_function(metric):
+
+    good_keys = ['C major', 'c major', 'C# major', 'Bb minor', 'db minor']
+    # All of these are invalid key strings
+    bad_keys = ['C maj', 'Cb major', 'C', 'K major', 'F## minor']
+
+    for good_key in good_keys:
+        for bad_key in bad_keys:
+            # Should raise an error whether we pass a bad key as ref or est
+            nose.tools.assert_raises(ValueError, metric, good_key, bad_key)
+            nose.tools.assert_raises(ValueError, metric, bad_key, good_key)
+
+    for good_key in good_keys:
+        # When the same key is passed for est and ref, score should be 1
+        assert mir_eval.key.weighted_score(good_key, good_key) == 1.
+
+
+def __check_score(sco_f, metric, score, expected_score):
+    assert np.allclose(score, expected_score, atol=A_TOL)
+
+
+def test_key_functions():
+    # Load in all files in the same order
+    ref_files = sorted(glob.glob(REF_GLOB))
+    est_files = sorted(glob.glob(EST_GLOB))
+    sco_files = sorted(glob.glob(SCORES_GLOB))
+
+    assert len(ref_files) == len(est_files) == len(sco_files) > 0
+
+    # Unit test all metrics (one for now)
+    for metric in [mir_eval.key.weighted_score]:
+        yield __unit_test_key_function, metric
+
+    # Regression tests
+    for ref_f, est_f, sco_f in zip(ref_files, est_files, sco_files):
+        with open(sco_f, 'r') as f:
+            expected_scores = json.load(f)
+        # Load in an example key annotation
+        reference_key = mir_eval.io.load_key(ref_f)
+        # Load in an example key detector output
+        estimated_key = mir_eval.io.load_key(est_f)
+        # Compute scores
+        scores = mir_eval.key.evaluate(reference_key, estimated_key)
+        # Compare them
+        for metric in scores:
+            # This is a simple hack to make nosetest's messages more useful
+            yield (__check_score, sco_f, metric, scores[metric],
+                   expected_scores[metric])