experiment-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##################\n",
    "# Classification #\n",
    "##################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score\n",
    "from sklearn import svm, linear_model, naive_bayes, neural_network, neighbors, ensemble\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, HashingVectorizer\n",
    "import random, math\n",
    "import numpy as np\n",
    "import scipy.sparse as sp\n",
    "from datetime import datetime\n",
    "from collections import Counter\n",
    "from itertools import combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('mxl-list.txt', 'r') as f:\n",
    "    dataset = [piece.strip() for piece in f.readlines()]\n",
    "    bach_data = [f for f in dataset if f.replace('-', '_').split('_')[0] == 'bach']\n",
    "    beethoven_data = [f for f in dataset if f.replace('-', '_').split('_')[0] == 'beethoven']\n",
    "    debussy_data = [f for f in dataset if f.replace('-', '_').split('_')[0] == 'debussy']\n",
    "    scarlatti_data = [f for f in dataset if f.replace('-', '_').split('_')[0] == 'scarlatti']\n",
    "    victoria_data = [f for f in dataset if f.replace('-', '_').split('_')[0] == 'victoria']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('bach-chordsequence.txt', 'r') as f:\n",
    "    BACH = [' '.join(piece.strip('[]\\n').split(', ')) for piece in f.readlines()]\n",
    "    BACH = [(BACH[i], bach_data[i]) for i in range(len(BACH))]\n",
    "with open('beethoven-chordsequence.txt', 'r') as f:\n",
    "    BEETHOVEN = [' '.join(piece.strip('[]\\n').split(', ')) for piece in f.readlines()]\n",
    "    BEETHOVEN = [(BEETHOVEN[i], beethoven_data[i]) for i in range(len(BEETHOVEN))]\n",
    "with open('debussy-chordsequence.txt', 'r') as f:\n",
    "    DEBUSSY = [' '.join(piece.strip('[]\\n').split(', ')) for piece in f.readlines()]\n",
    "    DEBUSSY = [(DEBUSSY[i], debussy_data[i]) for i in range(len(DEBUSSY))]\n",
    "with open('scarlatti-chordsequence.txt', 'r') as f:\n",
    "    SCARLATTI = [' '.join(piece.strip('[]\\n').split(', ')) for piece in f.readlines()]\n",
    "    SCARLATTI = [(SCARLATTI[i], scarlatti_data[i]) for i in range(len(SCARLATTI))]\n",
    "with open('victoria-chordsequence.txt', 'r') as f:\n",
    "    VICTORIA = [' '.join(piece.strip('[]\\n').split(', ')) for piece in f.readlines()]\n",
    "    VICTORIA = [(VICTORIA[i], victoria_data[i]) for i in range(len(VICTORIA))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_ngrams(input_list, N=4):\n",
    "    return [' '.join(input_list[i:i+N]) for i in range(len(input_list)-N+1)]\n",
    "\n",
    "def ngrams_by_composer(composer): \n",
    "    for i in range(1,5):\n",
    "        ngrams = []\n",
    "        for piece in composer:\n",
    "            ngrams += find_ngrams(piece[0].split(' '), i)\n",
    "        print(len(ngrams), '{}-grams total;'.format(str(i)), len(set(ngrams)), 'unique')\n",
    "    print('-')\n",
    "\n",
    "def show_ngrams(composer_data, composer_name):\n",
    "    print(composer_name, ':', len(composer_data), 'pieces')\n",
    "    ngrams_by_composer(composer_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "show_ngrams(BACH,'bach')\n",
    "show_ngrams(BEETHOVEN,'beethoven')\n",
    "show_ngrams(DEBUSSY,'debussy')\n",
    "show_ngrams(SCARLATTI,'scarlatti')\n",
    "show_ngrams(VICTORIA, 'victoria')\n",
    "show_ngrams(BACH+BEETHOVEN+DEBUSSY+SCARLATTI+VICTORIA, 'all composers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_Xy(composers, size=1):\n",
    "    if size >= 1: # use every rows\n",
    "        indices = [range(len(composer)) for composer in composers]\n",
    "    else:\n",
    "        indices = [random.sample(range(len(composer)), math.ceil(size*len(composer))) for composer in composers]\n",
    "\n",
    "    y = []\n",
    "    for i in range(len(composers)):\n",
    "        y += [i for n in range(len(indices[i]))]\n",
    "    \n",
    "    X = []\n",
    "    for i in range(len(composers)):\n",
    "        X += [composers[i][j] for j in indices[i]]\n",
    "    \n",
    "    return X, np.array(y, dtype='int16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def crossvalidate(X_tuple, y, classifiers, vectorizer, NGRAMRANGE, K=10):    \n",
    "    for clf in classifiers:\n",
    "        clf.cm_sum = np.zeros([len(set(y)),len(set(y))], dtype='int16')\n",
    "        clf.accuracies, clf.fones, clf.misclassified, clf.runningtime = [], [], [], []\n",
    "        clf.fones_micro, clf.fones_macro = [], []\n",
    "        clf.name = str(clf).split('(')[0]\n",
    "\n",
    "    X = np.array([piece[0] for piece in X_tuple])\n",
    "    filenames = np.array([piece[1] for piece in X_tuple])\n",
    "    kf = KFold(n_splits=K, shuffle=True)\n",
    "    for train_index, test_index in kf.split(y):\n",
    "        X_train, X_test, y_train, y_test = X[train_index], X[test_index], y[train_index], y[test_index]\n",
    "        \n",
    "        vct = vectorizer.set_params(lowercase=False, token_pattern=u\"(?u)\\\\b\\\\w+\\\\b\", ngram_range=NGRAMRANGE)\n",
    "        X_train_tfidf = vct.fit_transform(X_train)\n",
    "#         X_test_tfidf = tfidf.transform(X_test)  # i think this computes tf-idf values using the whole test set, but i want each piece to be treated separately\n",
    "        X_test_tfidf = sp.vstack([vct.transform(np.array([piece])) for piece in X_test])\n",
    "        \n",
    "        for clf in classifiers:\n",
    "            t = datetime.now()\n",
    "            clf.fit(X_train_tfidf, y_train)\n",
    "            y_pred = clf.predict(X_test_tfidf)\n",
    "            clf.runningtime.append((datetime.now()-t).total_seconds())\n",
    "            clf.cm_sum += confusion_matrix(y_test, y_pred)\n",
    "            clf.misclassified.append(test_index[np.where(y_test != y_pred)]) # http://stackoverflow.com/a/25570632\n",
    "            clf.accuracies.append(accuracy_score(y_test, y_pred))\n",
    "            clf.fones.append(f1_score(y_test, y_pred, average='weighted'))\n",
    "            clf.fones_micro.append(f1_score(y_test, y_pred, average='micro'))\n",
    "            clf.fones_macro.append(f1_score(y_test, y_pred, average='macro'))\n",
    "\n",
    "    result = dict()\n",
    "    for clf in classifiers:\n",
    "        clf.misclassified = np.sort(np.hstack(clf.misclassified))\n",
    "        result[clf.name] = [clf.cm_sum, clf.accuracies, clf.fones, clf.misclassified, filenames[clf.misclassified], clf.runningtime, clf.fones_micro, clf.fones_macro]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def benchmark_classifiers(composers, NGRAMRANGES, classifiers, vectorizer, n=1, retrieve_title=True):\n",
    "    misclassified_list = []\n",
    "    for NGRAMRANGE in NGRAMRANGES:\n",
    "        print('n-gram range', NGRAMRANGE)\n",
    "        X, y = build_Xy(composers, size=n)\n",
    "        cv_result = crossvalidate(X, y, classifiers, vectorizer, NGRAMRANGE)\n",
    "        for clf, results in cv_result.items():\n",
    "            print(clf)\n",
    "            cm = results[0]\n",
    "            print(cm)\n",
    "            acc = results[1] # using two different f-measures, don't need this\n",
    "#             print('accuracy', round(np.mean(acc)*100,2), '({})'.format(round(np.std(acc, ddof=1)*100,2)))\n",
    "            fones = results[2] # weighted average, don't need this\n",
    "#             print('f1', round(np.mean(fones)*100,2), '({})'.format(round(np.std(fones, ddof=1)*100,2)), fones)\n",
    "            misclassified = results[3]\n",
    "            misclassified_filenames = results[4]\n",
    "            misclassified_list += list(misclassified_filenames)\n",
    "#             print('misclassified',[(misclassified[i], misclassified_filenames[i]) for i in range(len(misclassified))])\n",
    "            runningtime = results[5]\n",
    "#             print('running time', np.sum(runningtime))\n",
    "            fones_micro = results[6]\n",
    "            fones_macro = results[7]\n",
    "            print('micro-averaged f-score (std) & macro-averaged f-score (std)')\n",
    "            print(round(np.mean(fones_micro),4), '({})'.format(round(np.std(fones_micro, ddof=1),4)), '&', round(np.mean(fones_macro),4), '({})'.format(round(np.std(fones_macro, ddof=1),4)))\n",
    "    print('-----')\n",
    "    return misclassified_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "COMPOSERS = [BACH, BEETHOVEN, DEBUSSY, SCARLATTI, VICTORIA]\n",
    "NGRAMRANGES = [(1,1),(2,2),(3,3),(4,4),(1,2),(3,4),(1,4)]\n",
    "CLASSIFIERS = [svm.LinearSVC(penalty='l2', C=5, loss='hinge'),\n",
    "               linear_model.LogisticRegression(penalty='l2', C=100, tol=1, multi_class='multinomial', solver='sag'),\n",
    "               neighbors.KNeighborsClassifier(weights='distance'),\n",
    "               naive_bayes.MultinomialNB(alpha=0.00001, fit_prior=False),\n",
    "               neural_network.MLPClassifier(solver='lbfgs',hidden_layer_sizes=(10,))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Compare different methods of vectorizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "VECTORIZER = TfidfVectorizer(sublinear_tf=True)\n",
    "benchmark_classifiers(COMPOSERS,NGRAMRANGES,CLASSIFIERS,VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "VECTORIZER = CountVectorizer(binary=True)\n",
    "benchmark_classifiers(COMPOSERS,NGRAMRANGES,CLASSIFIERS,VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "VECTORIZER = CountVectorizer()\n",
    "benchmark_classifiers(COMPOSERS,NGRAMRANGES,CLASSIFIERS,VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test pairwise classification "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "NGRAMRANGES = [(1,2)]\n",
    "VECTORIZER = TfidfVectorizer(sublinear_tf=True)\n",
    "for indices in combinations(range(5),2):\n",
    "    print('composer indices',[i for i in indices]) # COMPOSERS = [BACH, BEETHOVEN, DEBUSSY, SCARLATTI, VICTORIA]\n",
    "    benchmark_classifiers([COMPOSERS[i] for i in indices],NGRAMRANGES,CLASSIFIERS,VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Identify the often-misclassified files\n",
    "# do the experiment 100 times with the best classifier, SVM\n",
    "# then find pieces that are misclassified more than 50% of the time\n",
    "COMPOSERS = [BACH, BEETHOVEN, DEBUSSY, SCARLATTI, VICTORIA]\n",
    "CLASSIFIERS = [svm.LinearSVC(penalty='l2', C=5, loss='hinge')] # \n",
    "appendix = []\n",
    "for i in range(100):\n",
    "    appendix += benchmark_classifiers(COMPOSERS,NGRAMRANGES,CLASSIFIERS,VECTORIZER)\n",
    "Counter(appendix).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use both chord sequences and duration data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "with open('bach-durations.txt', 'r') as f:\n",
    "    BD2 = [line.strip() for line in f.readlines()]\n",
    "with open('beethoven-durations.txt', 'r') as f:\n",
    "    BD = [line.strip() for line in f.readlines()]\n",
    "with open('debussy-durations.txt', 'r') as f:\n",
    "    DD = [line.strip() for line in f.readlines()]\n",
    "with open('scarlatti-durations.txt', 'r') as f:\n",
    "    SD = [line.strip() for line in f.readlines()]\n",
    "with open('victoria-durations.txt', 'r') as f:\n",
    "    VD = [line.strip() for line in f.readlines()]\n",
    "    \n",
    "BD2_TYPELENGTH = [piece.split(';') for piece in BD2]\n",
    "BD_TYPELENGTH = [piece.split(';') for piece in BD]\n",
    "DD_TYPELENGTH = [piece.split(';') for piece in DD]\n",
    "SD_TYPELENGTH = [piece.split(';') for piece in SD]\n",
    "VD_TYPELENGTH = [piece.split(';') for piece in VD]\n",
    "\n",
    "typelengths = list(set(flatten(BD2_TYPELENGTH+BD_TYPELENGTH+DD_TYPELENGTH+SD_TYPELENGTH+VD_TYPELENGTH)))\n",
    "typelength_dict = {typelengths[i]: str(i+300) for i in range(len(typelengths))}\n",
    "BD2_T = [(' '.join([typelength_dict[dur] for dur in piece]),'temp') for piece in BD2_TYPELENGTH]\n",
    "BD_T = [(' '.join([typelength_dict[dur] for dur in piece]),'temp') for piece in BD_TYPELENGTH]\n",
    "DD_T = [(' '.join([typelength_dict[dur] for dur in piece]),'temp') for piece in DD_TYPELENGTH]\n",
    "SD_T = [(' '.join([typelength_dict[dur] for dur in piece]),'temp') for piece in SD_TYPELENGTH]\n",
    "VD_T = [(' '.join([typelength_dict[dur] for dur in piece]),'temp') for piece in VD_TYPELENGTH]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# print most common duration by composer, regardless of element type(chord/note/rest)\n",
    "\n",
    "# BD_LENGTHONLY = [[string.split('|')[1] for string in piece.split(';')] for piece in BD]\n",
    "# SD_LENGTHONLY = [[string.split('|')[1] for string in piece.split(';')] for piece in SD]\n",
    "# BD2_LENGTHONLY = [[string.split('|')[1] for string in piece.split(';')] for piece in BD2]\n",
    "# DD_LENGTHONLY = [[string.split('|')[1] for string in piece.split(';')] for piece in DD]\n",
    "# VD_LENGTHONLY = [[string.split('|')[1] for string in piece.split(';')] for piece in VD]\n",
    "# lengths = list(set(flatten(BD2_LENGTHONLY+BD_LENGTHONLY+DD_LENGTHONLY+SD_LENGTHONLY+VD_LENGTHONLY)))\n",
    "# length_dict = {lengths[i]: str(i+200) for i in range(len(lengths))}\n",
    "# BD_L = [(' '.join([length_dict[dur] for dur in piece]),'temp') for piece in BD_LENGTHONLY]\n",
    "# BD2_L = [(' '.join([length_dict[dur] for dur in piece]),'temp') for piece in BD2_LENGTHONLY]\n",
    "# SD_L = [(' '.join([length_dict[dur] for dur in piece]),'temp') for piece in SD_LENGTHONLY]\n",
    "# DD_L = [(' '.join([length_dict[dur] for dur in piece]),'temp') for piece in DD_LENGTHONLY]\n",
    "# VD_L = [(' '.join([length_dict[dur] for dur in piece]),'temp') for piece in VD_LENGTHONLY]\n",
    "# duration_all       = flatten(BD2_LENGTHONLY+BD_LENGTHONLY+DD_LENGTHONLY+SD_LENGTHONLY+VD_LENGTHONLY)\n",
    "# duration_bach      = flatten(BD2_LENGTHONLY)\n",
    "# duration_beethoven = flatten(BD_LENGTHONLY)\n",
    "# duration_debussy   = flatten(DD_LENGTHONLY)\n",
    "# duration_scarlatti = flatten(SD_LENGTHONLY)\n",
    "# duration_victoria  = flatten(VD_LENGTHONLY)\n",
    "\n",
    "# for l in [duration_all,duration_bach,duration_beethoven,duration_debussy,duration_scarlatti,duration_victoria]:\n",
    "#     for key, value in Counter(l).most_common(10):\n",
    "#         print(key, '&', round(100*value/len(l),2))\n",
    "#     print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def crossvalidate_twofeaturesets(X_tuple1, X_tuple2, y, classifiers, vectorizer, range1, range2, K=10):    \n",
    "    for clf in classifiers:\n",
    "        clf.cm_sum = np.zeros([len(set(y)),len(set(y))], dtype='int16')\n",
    "        clf.accuracies, clf.fones, clf.misclassified, clf.runningtime = [], [], [], []\n",
    "        clf.fones_micro, clf.fones_macro = [], []\n",
    "        clf.name = str(clf).split('(')[0]\n",
    "\n",
    "    X1 = np.array([piece[0] for piece in X_tuple1])\n",
    "    X2 = np.array([piece[0] for piece in X_tuple2])\n",
    "    filenames = np.array([piece[1] for piece in X_tuple2])\n",
    "    kf = KFold(n_splits=K, shuffle=True)\n",
    "    for train_index, test_index in kf.split(y):\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "        X_train_new, X_test_new = X1[train_index], X1[test_index]\n",
    "        vct1 = vectorizer.set_params(ngram_range=range1)\n",
    "        X_train, X_test = X2[train_index], X2[test_index] \n",
    "        vct2 = vectorizer.set_params(ngram_range=range2)\n",
    "   \n",
    "        X_train_new_tfidf = vct1.fit_transform(X_train_new) # use two separate vectorizers for each feature set\n",
    "        X_test_new_tfidf = sp.vstack([vct1.transform(np.array([piece])) for piece in X_test_new])\n",
    "        X_train_tfidf = vct2.fit_transform(X_train)\n",
    "        X_test_tfidf = sp.vstack([vct2.transform(np.array([piece])) for piece in X_test])\n",
    "        \n",
    "        X_train_tfidf = sp.hstack((X_train_tfidf, X_train_new_tfidf)) # Merge the two feature sets\n",
    "        X_test_tfidf = sp.hstack((X_test_tfidf, X_test_new_tfidf))\n",
    "        \n",
    "        for clf in classifiers:\n",
    "            t = datetime.now()\n",
    "            clf.fit(X_train_tfidf, y_train)\n",
    "            y_pred = clf.predict(X_test_tfidf)\n",
    "            clf.runningtime.append((datetime.now()-t).total_seconds())\n",
    "            clf.cm_sum += confusion_matrix(y_test, y_pred)\n",
    "            clf.misclassified.append(test_index[np.where(y_test != y_pred)]) # http://stackoverflow.com/a/25570632\n",
    "            clf.accuracies.append(accuracy_score(y_test, y_pred))\n",
    "            clf.fones.append(f1_score(y_test, y_pred, average='weighted'))\n",
    "            clf.fones_micro.append(f1_score(y_test, y_pred, average='micro'))\n",
    "            clf.fones_macro.append(f1_score(y_test, y_pred, average='macro'))\n",
    "\n",
    "    result = dict()\n",
    "    for clf in classifiers:\n",
    "        clf.misclassified = np.sort(np.hstack(clf.misclassified))\n",
    "        result[clf.name] = [clf.cm_sum, clf.accuracies, clf.fones, clf.misclassified, filenames[clf.misclassified], clf.runningtime, clf.fones_micro, clf.fones_macro]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def benchmark_classifiers_twofeaturesets(composers1, composers2, range1, range2, classifiers, vectorizer, n=1, retrieve_title=True):\n",
    "    misclassified_list = []\n",
    "    print('duration n-gram range:', range1, 'chord n-gram range:', range2)\n",
    "    X1, y = build_Xy(composers1, size=n)\n",
    "    X2, y = build_Xy(composers2, size=n)\n",
    "    cv_result = crossvalidate_twofeaturesets(X1, X2, y, classifiers, vectorizer, range1, range2)\n",
    "    for clf, results in cv_result.items():\n",
    "        print(clf)\n",
    "        cm = results[0]\n",
    "        print(cm)\n",
    "        acc = results[1]\n",
    "        fones = results[2]\n",
    "        misclassified = results[3]\n",
    "        misclassified_filenames = results[4]\n",
    "        misclassified_list += list(misclassified_filenames)\n",
    "#             print('misclassified',[(misclassified[i], misclassified_filenames[i]) for i in range(len(misclassified))])\n",
    "        runningtime = results[5]\n",
    "#         print('running time', np.sum(runningtime))\n",
    "        fones_micro = results[6]\n",
    "        fones_macro = results[7]\n",
    "        print('F-measures')\n",
    "        print(round(np.mean(fones_micro),4), '({})'.format(round(np.std(fones_micro, ddof=1),4)), '&', round(np.mean(fones_macro),4), '({})'.format(round(np.std(fones_macro, ddof=1),4)))\n",
    "    print('-----')\n",
    "    return misclassified_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "CLASSIFIERS = [svm.LinearSVC(penalty='l2', C=5, loss='hinge'),\n",
    "               linear_model.LogisticRegression(penalty='l2', C=100, tol=1, multi_class='multinomial', solver='sag'),\n",
    "               neighbors.KNeighborsClassifier(weights='distance'),\n",
    "               naive_bayes.MultinomialNB(alpha=0.00001, fit_prior=False),\n",
    "               neural_network.MLPClassifier(solver='lbfgs',hidden_layer_sizes=(10,))]\n",
    "VECTORIZER = TfidfVectorizer(sublinear_tf=True, lowercase=False, token_pattern=u\"(?u)\\\\b\\\\w+\\\\b\")\n",
    "\n",
    "COMPOSERS1 = [BD2_T,BD_T,DD_T,SD_T,VD_T]\n",
    "COMPOSERS2 = [BACH, BEETHOVEN, DEBUSSY, SCARLATTI, VICTORIA]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "benchmark_classifiers_twofeaturesets(COMPOSERS1, COMPOSERS2, (1,1), (1,2), CLASSIFIERS, VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for indices in combinations(range(5),2):\n",
    "    print('composer indices',[i for i in indices]) \n",
    "    benchmark_classifiers_twofeaturesets([COMPOSERS1[i] for i in indices],[COMPOSERS2[i] for i in indices],(1,1),(1,2),CLASSIFIERS,VECTORIZER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Identify the often-misclassified files, using both feature sets\n",
    "# do the experiment 100 times with the best classifier, SVM\n",
    "# then find pieces that are misclassified more than 50% of the time\n",
    "COMPOSERS1 = [BD2_T,BD_T,DD_T,SD_T,VD_T]\n",
    "COMPOSERS2 = [BACH, BEETHOVEN, DEBUSSY, SCARLATTI, VICTORIA]\n",
    "CLASSIFIERS = [svm.LinearSVC(penalty='l2', C=5, loss='hinge')] # \n",
    "appendix = []\n",
    "for i in range(100):\n",
    "    appendix += benchmark_classifiers_twofeaturesets(COMPOSERS1, COMPOSERS2, (1,1), (1,2), CLASSIFIERS, VECTORIZER)\n",
    "Counter(appendix).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}