numer_Manual_Folds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "os.chdir(\"/Users/PraveenGupta/Downloads/Kaggle/numerai_datasets/New Outputs/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train=pd.read_csv(\"numerai_training_data.csv\")\n",
    "test=pd.read_csv(\"numerai_tournament_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold\n",
    "skf=StratifiedKFold(train['target'],n_folds=5)\n",
    "i=1\n",
    "for train_index, test_index in skf:\n",
    "    print \"Train:\",train_index,\"Test:\",test_index\n",
    "    train.set_value(test_index,'group',i)\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train.to_csv(\"numerai_training_data_withFolds.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_grouped=pd.read_csv(\"numerai_training_data_withFolds.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Custom Function created which would do the CV Manually\n",
    "# Function should be created in such a way that proper calibration should be done w.r.t the LB.. with proper calibration\n",
    "# submission could be done cautiously\n",
    "\n",
    "target = 'target'\n",
    "IDCol = ['t_id']\n",
    "\n",
    "import random\n",
    "random.seed(111)\n",
    "\n",
    "from sklearn import cross_validation, metrics\n",
    "from datetime import datetime\n",
    "\n",
    "def custModelfit(alg2R,dtrain,dtest,predictors,target,IDCol,filename,postfix=True,verbose=True,stack=False):\n",
    "    print datetime.now().time()\n",
    "    Model_Score=[]\n",
    "    CV_Score=[]\n",
    "    predOutofFoldDF=[]\n",
    "    \n",
    "    for i in range(5):\n",
    "        \n",
    "        alg2R.fit(dtrain.loc[dtrain['group']%5!=i,predictors],dtrain.loc[dtrain['group']%5!=i,target])        \n",
    "        dtrain_predprob_model =alg2R.predict_proba(dtrain.loc[dtrain['group']%5!=i,predictors])[:,1]\n",
    "        idsForTrainInFold=dtrain.loc[dtrain['group']%5!=i,\"t_id\"]\n",
    "        predInFoldDF = pd.DataFrame({\"t_id\":idsForTrainInFold,\"predInFold\":dtrain_predprob_model})\n",
    "        \n",
    "        dtrain_predprob=alg2R.predict_proba(dtrain.loc[dtrain['group']%5==i,predictors])[:,1]\n",
    "        idsForTrainOutofFold =  dtrain.loc[dtrain['group']%5==i,\"t_id\"]\n",
    "        predOutofFoldDF.append(pd.DataFrame({\"t_id\":idsForTrainOutofFold,\"predOutofFold\":dtrain_predprob}))\n",
    "        \n",
    "#         dtrain.loc[dtrain['group']%5!=i,\"preds\"]=alg2R.predict_proba(dtrain.loc[dtrain['group']%5!=i,predictors])[:,1]\n",
    "#         dtrain.loc[dtrain['group']%5==i,\"preds\"]=alg2R.predict_proba(dtrain.loc[dtrain['group']%5==i,predictors])[:,1]\n",
    "                \n",
    "#         if postfix==True:\n",
    "#             dtrain.loc[dtrain['var15']<=22,\"preds\"]=0\n",
    "\n",
    "        if stack==True:\n",
    "            dtrain.loc[dtrain['group']%5!=i,\"pred\" + str(i)]=alg2R.predict_proba(dtrain.loc[dtrain['group']%5!=i,predictors])[:,1]\n",
    "            dtrain.loc[dtrain['group']%5==i,\"pred\" + str(i)]=alg2R.predict_proba(dtrain.loc[dtrain['group']%5==i,predictors])[:,1]\n",
    "        \n",
    "        #metrics.roc_auc_score(abc[\"TARGET\"]).values,abc[\"preds\"].values)\n",
    "        train_predDF=dtrain[['t_id','target','group']]\n",
    "        train_predDF = pd.merge(train_predDF,predInFoldDF,on=['t_id'])\n",
    "        Model_Score.append(metrics.log_loss(train_predDF.loc[train_predDF['group']%5!=i,target].values,train_predDF.loc[train_predDF['group']%5!=i,'predInFold'].values))\n",
    "\n",
    "        train_predDF=dtrain[['t_id','target','group']]\n",
    "        train_predDF = pd.merge(train_predDF,predOutofFoldDF[i],on=['t_id'])\n",
    "        CV_Score.append(metrics.log_loss(train_predDF.loc[train_predDF['group']%5==i,target].values,train_predDF.loc[train_predDF['group']%5==i,'predOutofFold'].values))\n",
    "        \n",
    "        if verbose == True:\n",
    "            print \"Log_Loss : %.4g\" % (Model_Score[i])\n",
    "            print \"CV Log_Loss : %.4g\" % (CV_Score[i])\n",
    "        \n",
    "    print \"Overall Model Score : %.4g\" %(np.mean(Model_Score))\n",
    "    print \"Overall CV Score %.4g\" %(np.mean(CV_Score))\n",
    "    \n",
    "    #Predict on testing data\n",
    "    alg2R.fit(dtrain[predictors],dtrain[target])\n",
    "    dtest[target]=alg2R.predict_proba(dtest[predictors])[:,1]\n",
    "    \n",
    "    #Export CV file\n",
    "    predOutofFoldDF_CV = pd.concat(predOutofFoldDF,ignore_index=True)\n",
    "    #cv = pd.DataFrame(dtrain[['t_id','predOutofFold']])\n",
    "    predOutofFoldDF_CV = predOutofFoldDF_CV.rename(columns={'predOutofFold': \"CV_\"+filename[:-4]})\n",
    "    if not os.path.exists(\"CV\"):\n",
    "        os.makedirs(\"CV\")\n",
    "    predOutofFoldDF_CV.to_csv(\"CV/\"+\"CV_\"+filename,index=False)\n",
    "       \n",
    "    #Export submission file\n",
    "    IDCol.append(target)\n",
    "    submission = pd.DataFrame({x : dtest[x] for x in IDCol})\n",
    "    submission = submission.rename(columns={'target': 'probability'})\n",
    "    if not os.path.exists(\"submission\"):\n",
    "        os.makedirs(\"submission\")\n",
    "    submission.to_csv(\"submission/\"+filename,index=False)\n",
    "    \n",
    "    print datetime.now().time()\n",
    "    \n",
    "    if stack==True:\n",
    "        dtrain['pred1'] = dtrain['pred1'].fillna(0.0)\n",
    "        dtrain['pred2'] = dtrain['pred2'].fillna(0.0)\n",
    "        dtrain['pred3'] = dtrain['pred3'].fillna(0.0)\n",
    "        dtrain['pred4'] = dtrain['pred4'].fillna(0.0)\n",
    "        dtrain['pred0'] = dtrain['pred0'].fillna(0.0)\n",
    "        return((dtrain[\"pred1\"]+dtrain[\"pred2\"]+dtrain[\"pred3\"]+dtrain[\"pred4\"]+dtrain[\"pred0\"])/5.0)\n",
    "        # []\"ID\",\"PRED\" whole train data set\n",
    "        # retrun dataframe ...\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:17:48.187000\n",
      "Log_Loss : 0.6911\n",
      "CV Log_Loss : 0.6915\n",
      "Log_Loss : 0.6912\n",
      "CV Log_Loss : 0.6911\n",
      "Log_Loss : 0.691\n",
      "CV Log_Loss : 0.6922\n",
      "Log_Loss : 0.691\n",
      "CV Log_Loss : 0.6922\n",
      "Log_Loss : 0.6912\n",
      "CV Log_Loss : 0.6911\n",
      "Overall Model Score : 0.6911\n",
      "Overall CV Score 0.6916\n",
      "15:17:52.227000\n"
     ]
    }
   ],
   "source": [
    "# Log_Loss : 0.6911\n",
    "# CV Log_Loss : 0.6915\n",
    "# Log_Loss : 0.6912\n",
    "# CV Log_Loss : 0.6911\n",
    "# Log_Loss : 0.691\n",
    "# CV Log_Loss : 0.6922\n",
    "# Log_Loss : 0.691\n",
    "# CV Log_Loss : 0.6922\n",
    "# Log_Loss : 0.6912\n",
    "# CV Log_Loss : 0.6911\n",
    "# Overall Model Score : 0.6911\n",
    "# Overall CV Score 0.6916\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "alg3=LogisticRegression()\n",
    "predictors=[x for x in train_grouped.columns if x not in [target]+IDCol+[\"predictions\",\"group\"]]\n",
    "custModelfit(alg3,train_grouped,test,predictors,target,IDCol,'alg_Logistic.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:40:14.198000\n",
      "Log_Loss : 0.6911\n",
      "CV Log_Loss : 0.6915\n",
      "Log_Loss : 0.6912\n",
      "CV Log_Loss : 0.6911\n",
      "Log_Loss : 0.691\n",
      "CV Log_Loss : 0.6924\n",
      "Log_Loss : 0.691\n",
      "CV Log_Loss : 0.6922\n",
      "Log_Loss : 0.6912\n",
      "CV Log_Loss : 0.6911\n",
      "Overall Model Score : 0.6911\n",
      "Overall CV Score 0.6917\n",
      "20:41:57.066000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "alg3=LogisticRegression()\n",
    "bag3=BaggingClassifier(base_estimator=alg3,n_estimators=20,n_jobs=-1)\n",
    "predictors=[x for x in train_grouped.columns if x not in [target]+IDCol+[\"predictions\",\"group\"]]\n",
    "custModelfit(bag3,train_grouped,test,predictors,target,IDCol,'alg_Logistic_Bag.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:39:34.253000\n",
      "Log_Loss : 0.6871\n",
      "CV Log_Loss : 0.6914\n",
      "Log_Loss : 0.6874\n",
      "CV Log_Loss : 0.6908\n",
      "Log_Loss : 0.6872\n",
      "CV Log_Loss : 0.6913\n",
      "Log_Loss : 0.6869\n",
      "CV Log_Loss : 0.6919\n",
      "Log_Loss : 0.6873\n",
      "CV Log_Loss : 0.6907\n",
      "Overall Model Score : 0.6872\n",
      "Overall CV Score 0.6912\n",
      "20:40:35.854000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "alg1=RandomForestClassifier(n_estimators=200,max_depth=5,min_samples_split=50, min_samples_leaf=25,max_features='sqrt',n_jobs=-1,random_state=111)\n",
    "predictors=[x for x in train_grouped.columns if x not in [target]+IDCol+[\"predictions\",\"group\"]]\n",
    "custModelfit(alg1,train_grouped,test,predictors,target,IDCol,'alg_RandomForest.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:45:34.268000\n",
      "Log_Loss : 0.689\n",
      "CV Log_Loss : 0.6952\n",
      "Log_Loss : 0.689\n",
      "CV Log_Loss : 0.6949\n",
      "Log_Loss : 0.6895\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.689\n",
      "CV Log_Loss : 0.6933\n",
      "Log_Loss : 0.6897\n",
      "CV Log_Loss : 0.6947\n",
      "Overall Model Score : 0.6893\n",
      "Overall CV Score 0.6942\n",
      "20:45:35.772000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "alg2R=DecisionTreeClassifier(max_depth=5,min_samples_split=50, min_samples_leaf=25,max_features='sqrt',random_state=111)\n",
    "custModelfit(alg2R,train_grouped,test,predictors,target,IDCol,'alg_DTree.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:47:59.022000\n",
      "Log_Loss : 0.6905\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6906\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6906\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6905\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6907\n",
      "CV Log_Loss : 0.693\n",
      "Overall Model Score : 0.6906\n",
      "Overall CV Score 0.6931\n",
      "20:50:06.255000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "alg3=DecisionTreeClassifier(max_depth=5,min_samples_split=50, min_samples_leaf=25,max_features='sqrt',random_state=111)\n",
    "ada1=AdaBoostClassifier(base_estimator=alg3,n_estimators=200)\n",
    "custModelfit(ada1,train_grouped,test,predictors,target,IDCol,'alg_DTree_AdaBoost.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20:51:28.945000\n",
      "Log_Loss : 0.6931\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6931\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6931\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6931\n",
      "CV Log_Loss : 0.6931\n",
      "Log_Loss : 0.6931\n",
      "CV Log_Loss : 0.6931\n",
      "Overall Model Score : 0.6931\n",
      "Overall CV Score 0.6931\n",
      "21:02:08.399000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "alg3=LogisticRegression(solver='sag')\n",
    "ada1=AdaBoostClassifier(base_estimator=alg3)\n",
    "custModelfit(ada1,train_grouped,test,predictors,target,IDCol,'alg_Logistic_AdaB.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:03:51.025000\n",
      "Log_Loss : 0.6873\n",
      "CV Log_Loss : 0.6914\n",
      "Log_Loss : 0.6875\n",
      "CV Log_Loss : 0.6907\n",
      "Log_Loss : 0.6873\n",
      "CV Log_Loss : 0.6913\n",
      "Log_Loss : 0.6869\n",
      "CV Log_Loss : 0.692\n",
      "Log_Loss : 0.6874\n",
      "CV Log_Loss : 0.6907\n",
      "Overall Model Score : 0.6873\n",
      "Overall CV Score 0.6912\n",
      "21:04:02.060000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "alg1=RandomForestClassifier(n_estimators=50,max_depth=5,min_samples_split=50, min_samples_leaf=25,max_features='sqrt',n_jobs=-1,random_state=111)\n",
    "ada1=AdaBoostClassifier(base_estimator=alg1,n_estimators=100)\n",
    "predictors=[x for x in train_grouped.columns if x not in [target]+IDCol+[\"predictions\",\"group\"]]\n",
    "custModelfit(alg1,train_grouped,test,predictors,target,IDCol,'alg_RandomForest_AdaB.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:26:54.610000\n",
      "Log_Loss : 0.6912\n",
      "CV Log_Loss : 0.6929\n",
      "Log_Loss : 0.6913\n",
      "CV Log_Loss : 0.6923\n",
      "Log_Loss : 0.6913\n",
      "CV Log_Loss : 0.6926\n",
      "Log_Loss : 0.6913\n",
      "CV Log_Loss : 0.6933\n",
      "Log_Loss : 0.6913\n",
      "CV Log_Loss : 0.6919\n",
      "Overall Model Score : 0.6913\n",
      "Overall CV Score 0.6926\n",
      "21:26:55.676000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import ExtraTreeClassifier\n",
    "algET=ExtraTreeClassifier(max_depth=5,min_samples_split=50, min_samples_leaf=25,max_features='sqrt',random_state=111)\n",
    "custModelfit(algET,train_grouped,test,predictors,target,IDCol,'alg_ET.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:27:29.588000\n",
      "Log_Loss : 0.6629\n",
      "CV Log_Loss : 0.7146\n",
      "Log_Loss : 0.6627\n",
      "CV Log_Loss : 0.7136\n",
      "Log_Loss : 0.6634\n",
      "CV Log_Loss : 0.7189\n",
      "Log_Loss : 0.6623\n",
      "CV Log_Loss : 0.717\n",
      "Log_Loss : 0.663\n",
      "CV Log_Loss : 0.7124\n",
      "Overall Model Score : 0.6629\n",
      "Overall CV Score 0.7153\n",
      "21:34:22.407000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "algKNN=KNeighborsClassifier(leaf_size=100,n_jobs=4,n_neighbors=20)\n",
    "custModelfit(algKNN,train_grouped,test,predictors,target,IDCol,'alg_KNN.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:37:01.976000\n",
      "Log_Loss : 0.7007\n",
      "CV Log_Loss : 0.702\n",
      "Log_Loss : 0.7002\n",
      "CV Log_Loss : 0.6993\n",
      "Log_Loss : 0.701\n",
      "CV Log_Loss : 0.702\n",
      "Log_Loss : 0.7023\n",
      "CV Log_Loss : 0.7055\n",
      "Log_Loss : 0.7009\n",
      "CV Log_Loss : 0.6998\n",
      "Overall Model Score : 0.701\n",
      "Overall CV Score 0.7017\n",
      "21:37:03.257000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "algGNB=GaussianNB()\n",
    "custModelfit(algGNB,train_grouped,test,predictors,target,IDCol,'alg_GNB.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:35:33.292000\n",
      "Log_Loss : 0.6873\n",
      "CV Log_Loss : 0.6914\n",
      "Log_Loss : 0.6875\n",
      "CV Log_Loss : 0.6907\n",
      "Log_Loss : 0.6873\n",
      "CV Log_Loss : 0.6913\n",
      "Log_Loss : 0.6869\n",
      "CV Log_Loss : 0.692\n",
      "Log_Loss : 0.6874\n",
      "CV Log_Loss : 0.6907\n",
      "Overall Model Score : 0.6873\n",
      "Overall CV Score 0.6912\n",
      "21:35:44.621000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "alg8=SVC(random_state=111)\n",
    "custModelfit(alg1,train_grouped,test,predictors,target,IDCol,'alg_svm.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:37:25.508000\n",
      "Log_Loss : 0.7284\n",
      "CV Log_Loss : 0.7306\n",
      "Log_Loss : 0.6984\n",
      "CV Log_Loss : 0.6991\n",
      "Log_Loss : 0.7168\n",
      "CV Log_Loss : 0.7175\n",
      "Log_Loss : 0.7458\n",
      "CV Log_Loss : 0.7484\n",
      "Log_Loss : 0.7087\n",
      "CV Log_Loss : 0.7073\n",
      "Overall Model Score : 0.7196\n",
      "Overall CV Score 0.7206\n",
      "21:37:27.097000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDClassifier\n",
    "alg8=SGDClassifier(loss='log')\n",
    "custModelfit(alg8,train_grouped,test,predictors,target,IDCol,'alg_SGD.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:38:18.060000\n",
      "Log_Loss : 0.6811\n",
      "CV Log_Loss : 0.6919\n",
      "Log_Loss : 0.6812\n",
      "CV Log_Loss : 0.6916\n",
      "Log_Loss : 0.6813\n",
      "CV Log_Loss : 0.6921\n",
      "Log_Loss : 0.6809\n",
      "CV Log_Loss : 0.6928\n",
      "Log_Loss : 0.6813\n",
      "CV Log_Loss : 0.6914\n",
      "Overall Model Score : 0.6811\n",
      "Overall CV Score 0.692\n",
      "21:40:04.350000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "alg1=GradientBoostingClassifier(random_state=111)\n",
    "custModelfit(alg1,train_grouped,test,predictors,target,IDCol,'alg_GBM.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Function for xgboost\n",
    "import xgboost as xgb\n",
    "from datetime import datetime\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "predictors=[x for x in train.columns if x not in [target]+IDCol+[\"predictions\"]]\n",
    "def xGBmodelfit(alg,dtrain,dtest,predictors,filename,useTrainCV=True, cv_folds=5,early_stopping_rounds=200):\n",
    "    print datetime.now().time()\n",
    "    if useTrainCV:\n",
    "        xgb_param=alg.get_xgb_params()\n",
    "        xgtrain = xgb.DMatrix(dtrain[predictors].values,label = dtrain[target].values)\n",
    "        cvresult = xgb.cv(xgb_param,xgtrain,num_boost_round=alg.get_params()['n_estimators'],\n",
    "            nfold=cv_folds,metrics='logloss',early_stopping_rounds=early_stopping_rounds)#,show_progress=False)\n",
    "        alg.set_params(n_estimators=cvresult.shape[0])#,booster='gbtree')\n",
    "    \n",
    "    bestNoOfRounds = cvresult['test-logloss-mean'].argmax()\n",
    "    bestAnsForThisSetting =  cvresult.ix[bestNoOfRounds]\n",
    "    bestModelEval = bestAnsForThisSetting[0]\n",
    "    print \"Best Num of Rounds %.7g\" %bestNoOfRounds\n",
    "#     print \"Best Ans %.7g\" %bestAnsForThisSetting\n",
    "    print \"Best Model Eval (Test-LogLoss) %.7g\" %bestModelEval\n",
    "    \n",
    "    alg.fit(dtrain[predictors],dtrain[target],eval_metric='logloss')#,eta=0.0001)\n",
    "    \n",
    "#   Predicting training data\n",
    "    dtrain[\"predictions\"] = alg.predict_proba(dtrain[predictors])[:,1]\n",
    "    dtrain_predprob=alg.predict_proba(dtrain[predictors])[:,1]       \n",
    "    \n",
    "    print \"\\nModel Report\"\n",
    "    print \"Accuracy : %.4g\" %metrics.accuracy_score(dtrain[target],alg.predict(dtrain[predictors]))\n",
    "    print \"LogLoss Score (train) : %f\" %metrics.log_loss(dtrain[target],dtrain_predprob)\n",
    "    \n",
    "#   Feature important plot\n",
    "    feat_imp = pd.Series(alg.booster().get_fscore()).sort_values(ascending=False)\n",
    "    feat_imp.plot(kind='bar',title = 'Feature Importance')\n",
    "    plt.ylabel('Feature Importance score')\n",
    "    \n",
    "    #Predict on testing data\n",
    "    dtest[target]=alg.predict_proba(dtest[predictors])[:,1]\n",
    "    \n",
    "    #Export submission file\n",
    "    IDCol.append(target)\n",
    "    submission = pd.DataFrame({x : dtest[x] for x in IDCol})\n",
    "    submission = submission.rename(columns={'target': 'probability'})\n",
    "    \n",
    "    if not os.path.exists(\"submission\"):\n",
    "        os.makedirs(\"submission\")\n",
    "    submission.to_csv(\"submission/\"+filename,index=False)\n",
    "    \n",
    "    cv = pd.DataFrame(dtrain[['t_id','predictions']])\n",
    "    cv = cv.rename(columns={'predictions': \"CV_\"+filename[:-4]})\n",
    "    if not os.path.exists(\"CV\"):\n",
    "        os.makedirs(\"CV\")\n",
    "    cv.to_csv(\"CV/\"+\"CV_\"+filename,index=False)\n",
    "    \n",
    "    print datetime.now().time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15:56:15.026000\n",
      "Best Num of Rounds 0\n",
      "Best Model Eval (Test-LogLoss) 0.6928388\n",
      "\n",
      "Model Report\n",
      "Accuracy : 0.5636\n",
      "LogLoss Score (train) : 0.683068\n",
      "15:56:29.952000\n"
     ]
    }
   ],
   "source": [
    "# CV = 0.6929616/0.666240, LB = 0.69468\n",
    "\n",
    "xgb1 = XGBClassifier(\n",
    "objective = 'binary:logistic',\n",
    "learning_rate=0.1,\n",
    "n_estimators = 100,\n",
    "max_depth=3,\n",
    "min_child_weight=1,\n",
    "subsample=0.8,\n",
    "colsample_bytree=0.8,\n",
    "nthread=4,\n",
    "seed=27)\n",
    "\n",
    "xGBmodelfit(xgb1,train,test,predictors,'xgBoost.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: -0.69186, std: 0.00061, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.69194, std: 0.00057, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.69196, std: 0.00061, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.69376, std: 0.00116, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.69350, std: 0.00116, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.69365, std: 0.00100, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.69736, std: 0.00132, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.69619, std: 0.00145, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.69614, std: 0.00105, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.70421, std: 0.00271, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.70197, std: 0.00178, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.70113, std: 0.00178, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 3, 'min_child_weight': 1},\n",
       " -0.69185776020201128)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "param_test1 = {\n",
    " 'max_depth':range(3,10,2),\n",
    " 'min_child_weight':range(1,6,2)\n",
    "}\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=100,\n",
    " gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
    " param_grid = param_test1, scoring='log_loss',n_jobs=4,iid=False, cv=5)\n",
    "gsearch1.fit(train[predictors],train[target])\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}