step4.2_submit_blr_UMAP_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "83c494d3-6ebd-4cde-aff0-8fc9344374dd",
   "metadata": {
    "id": "83c494d3-6ebd-4cde-aff0-8fc9344374dd"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy.stats import gaussian_kde\n",
    "import pcntoolkit as ptk\n",
    "from pcntoolkit.normative import estimate, predict, evaluate\n",
    "from pcntoolkit.util.utils import compute_MSLL, create_design_matrix\n",
    "from pcntoolkit.util.utils import calibration_descriptives\n",
    "from pcntoolkit.model.hbr import bspline_fit, bspline_transform\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7da01c88-7033-498b-a811-79ad58e8c17a",
   "metadata": {
    "id": "7da01c88-7033-498b-a811-79ad58e8c17a"
   },
   "outputs": [],
   "source": [
    "projdir = '/project_cephfs/3022017.02/projects/stijdboe/make_results/more_flexible_models_lifespan_big/'\n",
    "textfiles = os.path.join(projdir,'textfiles')\n",
    "if not os.path.exists(textfiles):\n",
    "    os.mkdir(textfiles)\n",
    "data_dir = '/project_cephfs/3022017.02/projects/stijdboe/Data/UMAP_data'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "896842d7-8913-4137-9d86-4757c42bcf1b",
   "metadata": {
    "id": "896842d7-8913-4137-9d86-4757c42bcf1b"
   },
   "source": [
    "### Fit the models\n",
    "\n",
    "Now we fit the models. This involves looping over the IDPs we have selected. We will use a module from PCNtoolkit to set up the design matrices, containing the covariates, fixed effects for site and nonlinear basis expansion. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b12d20d7-30ae-4440-9e69-ec8ce4120b32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing data in /project_cephfs/3022017.02/projects/stijdboe/make_results/more_flexible_models_lifespan_big/textfiles/Y_UMAP_tr.txt\n",
      "Estimating model  1 of 1\n",
      "configuring BLR ( order 1 )\n",
      "Using default hyperparameters\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 15322.752023\n",
      "         Iterations: 5\n",
      "         Function evaluations: 249\n",
      "Saving model meta-data...\n",
      "Evaluating the model ...\n",
      "Writing outputs ...\n"
     ]
    }
   ],
   "source": [
    "warp =  'WarpSinArcsinh'\n",
    "# For each fold\n",
    "\n",
    "this_identifier = f\"blr_UMAP\"\n",
    "processing_dir = os.path.join(projdir, this_identifier+'/')\n",
    "if not os.path.exists(processing_dir):\n",
    "    os.mkdir(processing_dir)\n",
    "\n",
    "log_dir = os.path.join(processing_dir, 'log')           #\n",
    "if not os.path.isdir(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "# The paths to the data\n",
    "X_tr_path = os.path.join(data_dir, 'cov_tr.txt')\n",
    "Y_tr_path = os.path.join(data_dir, 'trainselect.txt')\n",
    "\n",
    "X_te_path = os.path.join(data_dir, 'cov_te.txt')\n",
    "Y_te_path = os.path.join(data_dir, 'testselect.txt')\n",
    "\n",
    "X_tr = np.loadtxt(X_tr_path)\n",
    "X_te = np.loadtxt(X_te_path)\n",
    "\n",
    "\n",
    "Phi_tr = create_design_matrix(X_tr, basis='bspline')\n",
    "Phi_te = create_design_matrix(X_te, basis='bspline')\n",
    "\n",
    "# Save as text files \n",
    "Phi_tr_path = os.path.join(textfiles, f'Phi_tr.txt' )\n",
    "Phi_te_path = os.path.join(textfiles, f'Phi_te.txt' )\n",
    "np.savetxt(Phi_tr_path, Phi_tr)\n",
    "np.savetxt(Phi_te_path, Phi_te)\n",
    "\n",
    "Y_tr = np.loadtxt(Y_tr_path)\n",
    "Y_te = np.loadtxt(Y_te_path)\n",
    "\n",
    "\n",
    "os.chdir(processing_dir)\n",
    "\n",
    "F_tr = Y_tr\n",
    "F_tr_mean  = F_tr.mean(axis = 0)\n",
    "F_tr_std = F_tr.std(axis= 0)\n",
    "F_tr = (F_tr - F_tr_mean)/F_tr_std\n",
    "F_te = Y_te\n",
    "F_te = (F_te - F_tr_mean)/F_tr_std\n",
    "\n",
    "# Save as text files \n",
    "Y_feature_tr_path = os.path.join(textfiles, f'Y_UMAP_tr.txt' )\n",
    "Y_feature_te_path = os.path.join(textfiles, f'Y_UMAP_te.txt' )\n",
    "np.savetxt(Y_feature_tr_path, F_tr)\n",
    "np.savetxt(Y_feature_te_path, F_te)\n",
    "\n",
    "# Fit the model\n",
    "estimate(\n",
    "         Phi_tr_path,  \n",
    "         Y_feature_tr_path,\n",
    "         testcov=Phi_te_path, \n",
    "         testresp=Y_feature_te_path, \n",
    "         alg='blr', optimizer = 'powell', \n",
    "         savemodel=True, warp=warp, warp_reparam=True)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "1_fit_normative_models.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "stijn1",
   "language": "python",
   "name": "stijn1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}