GetStatsmodelsDataInfo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "950ca397",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/statsmodels/compat/pandas.py:65: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  from pandas import Int64Index as NumericIndex\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "\n",
    "from lifelines import CoxPHFitter, KaplanMeierFitter\n",
    "from lifelines.utils import concordance_index\n",
    "from lifelines import KaplanMeierFitter\n",
    "from lifelines import NelsonAalenFitter\n",
    "\n",
    "from scipy import stats\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ebb21069",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First Entries: ['survival', 'AER', 'AER', 'AER', 'asaur', 'asaur', 'asaur', 'asaur', 'asaur', 'betareg', 'boot', 'boot', 'boot', 'boot', 'boot', 'boot', 'boot', 'boot', 'boot', 'carData', 'carData', 'carData', 'carData', 'carData', 'carData', 'carData', 'carData', 'causaldata', 'causaldata', 'causaldata', 'causaldata', 'causaldata', 'survival', 'survival', 'survival', 'openintro', 'AER', 'openintro', 'openintro', 'stevedata', 'stevedata', 'survival', 'AER', 'Ecdat', 'sampleSelection', 'Ecdat', 'openintro', 'dslabs', 'medicaldata', 'geepack', 'openintro', 'Stat2Data', 'stevedata', 'survival', 'mstate', 'mosaicData', 'tidyr', 'tidyr', 'DAAG', 'openintro', 'DAAG', 'stevedata', 'Ecdat', 'mosaicData', 'Ecdat', 'Ecdat', 'heplots', 'carData', 'tidyr', 'survival', 'mosaicData', 'survival', 'MASS', 'mlmRev', 'mstate', 'openintro', 'mstate', 'mstate', 'geepack', 'modeldata', 'DAAG', 'mstate', 'survival', 'mlmRev', 'survival', 'openintro', 'causaldata', 'causaldata', 'openintro', 'COUNT', 'mosaicData', 'openintro', 'Stat2Data', 'survival', 'Ecdat', 'plm', 'survival', 'Stat2Data', 'DAAG', 'HistData', 'mosaicData', 'openintro', 'MASS', 'openintro', 'openintro', 'openintro', 'openintro', 'openintro', 'AER', 'carData', 'HistData', 'multgee', 'medicaldata', 'survival', 'sampleSelection', 'HistData', 'MASS', 'openintro', 'survival', 'survival', 'texmex', 'HLMdiag', 'medicaldata', 'quantreg', 'boot', 'dslabs', 'HSAUR', 'Ecdat', 'openintro', 'openintro', 'dslabs', 'Stat2Data', 'Stat2Data', 'tidyr']\n",
      "\n",
      "Second Entries: ['flchain', 'DoctorVisits', 'HealthInsurance', 'Medicaid1986', 'hepatoCellular', 'pancreatic', 'pancreatic2', 'pharmacoSmoking', 'prostateSurvival', 'StressAnxiety', 'aids', 'cd4', 'downs.bc', 'melanoma', 'neuro', 'nodal', 'remission', 'survival', 'urine', 'Blackmore', 'Davis', 'KosteckiDillon', 'Leinhardt', 'LoBD', 'Robey', 'WeightLoss', 'Wong', 'abortion', 'nhefs', 'nhefs_complete', 'organ_donations', 'thornton_hiv', 'nafld2 (nafld)', 'nafld3 (nafld)', 'survexp.us (survexp)', 'simpsons_paradox_covid', 'Fertility', 'avandia', 'mammogram', 'gss_abortion', 'mm_nhis', 'survexp.usr (survexp)', 'Fertility2', 'HI', 'RandHIE', 'DoctorContacts', 'health_coverage', 'us_contagious_diseases', 'covid_testing', 'muscatine', 'yrbss', 'YouthRisk', 'wvs_usa_abortion', 'survexp.mn (survexp)', 'ebmt2', 'Births', 'who', 'who2', 'monica', 'smallpox', 'crickete', 'anes_prochoice', 'MedExp', 'BirthsSSA', 'DoctorAUS', 'OFP', 'AddHealth', 'KosteckiDillon', 'population', 'myeloma (cancer)', 'BirthsCDC', 'rotterdam (cancer)', 'Aids2', 'guPrenat', 'ebmt4', 'biontech_adolescents', 'ebmt3', 'prothr', 'ohio', 'cells', 'dengue', 'ebmt1', 'pbcseq (pbc)', 'Contraception', 'colon (cancer)', 'smoking', 'nhefs', 'nhefs_complete', 'healthcare_law_survey', 'medpar', 'HELPfull', 'hfi', 'NCbirths', 'mgus2 (cancer)', 'Cigar', 'Cigar', 'udca2 (udca)', 'Whickham2', 'mifem', 'Snow.streets', 'Gestation', 'babies', 'OME', 'sleep_deprivation', 'ebola_survey', 'scotus_healthcare', 'births14', 'ncbirths', 'Medicaid1986', 'Blackmore', 'GaltonFamilies', 'arthritis', 'opt', 'transplant', 'Smoke', 'CholeraDeaths1849', 'biopsy', 'diabetes2', 'gbsg (cancer)', 'myeloid (cancer)', 'liver', 'autism', 'indo_rct', 'uis', 'aids', 'brca', 'respiratory', 'Cigarette', 'fastfood', 'bdims', 'outlier_example', 'Blood1', 'YouthRisk2009', 'cms_patient_experience']\n",
      "134\n",
      "134\n"
     ]
    }
   ],
   "source": [
    "# Provided data\n",
    "data = \"\"\"\n",
    "survival\tflchain\n",
    "AER\tDoctorVisits\n",
    "AER\tHealthInsurance\n",
    "AER\tMedicaid1986\n",
    "asaur\thepatoCellular\n",
    "asaur\tpancreatic\n",
    "asaur\tpancreatic2\n",
    "asaur\tpharmacoSmoking\n",
    "asaur\tprostateSurvival\n",
    "betareg\tStressAnxiety\n",
    "boot\taids\n",
    "boot\tcd4\n",
    "boot\tdowns.bc\n",
    "boot\tmelanoma\n",
    "boot\tneuro\n",
    "boot\tnodal\n",
    "boot\tremission\n",
    "boot\tsurvival\n",
    "boot\turine\n",
    "carData\tBlackmore\n",
    "carData\tDavis\n",
    "carData\tKosteckiDillon\n",
    "carData\tLeinhardt\n",
    "carData\tLoBD\n",
    "carData\tRobey\n",
    "carData\tWeightLoss\n",
    "carData\tWong\n",
    "causaldata\tabortion\n",
    "causaldata\tnhefs\n",
    "causaldata\tnhefs_complete\n",
    "causaldata\torgan_donations\n",
    "causaldata\tthornton_hiv\n",
    "survival\tnafld2 (nafld)\n",
    "survival\tnafld3 (nafld)\n",
    "survival\tsurvexp.us (survexp)\n",
    "openintro\tsimpsons_paradox_covid\n",
    "AER\tFertility\n",
    "openintro\tavandia\n",
    "openintro\tmammogram\n",
    "stevedata\tgss_abortion\n",
    "stevedata\tmm_nhis\n",
    "survival\tsurvexp.usr (survexp)\n",
    "AER\tFertility2\n",
    "Ecdat\tHI\n",
    "sampleSelection\tRandHIE\n",
    "Ecdat\tDoctorContacts\n",
    "openintro\thealth_coverage\n",
    "dslabs\tus_contagious_diseases\n",
    "medicaldata\tcovid_testing\n",
    "geepack\tmuscatine\n",
    "openintro\tyrbss\n",
    "Stat2Data\tYouthRisk\n",
    "stevedata\twvs_usa_abortion\n",
    "survival\tsurvexp.mn (survexp)\n",
    "mstate\tebmt2\n",
    "mosaicData\tBirths\n",
    "tidyr\twho\n",
    "tidyr\twho2\n",
    "DAAG\tmonica\n",
    "openintro\tsmallpox\n",
    "DAAG\tcrickete\n",
    "stevedata\tanes_prochoice\n",
    "Ecdat\tMedExp\n",
    "mosaicData\tBirthsSSA\n",
    "Ecdat\tDoctorAUS\n",
    "Ecdat\tOFP\n",
    "heplots\tAddHealth\n",
    "carData\tKosteckiDillon\n",
    "tidyr\tpopulation\n",
    "survival\tmyeloma (cancer)\n",
    "mosaicData\tBirthsCDC\n",
    "survival\trotterdam (cancer)\n",
    "MASS\tAids2\n",
    "mlmRev\tguPrenat\n",
    "mstate\tebmt4\n",
    "openintro\tbiontech_adolescents\n",
    "mstate\tebmt3\n",
    "mstate\tprothr\n",
    "geepack\tohio\n",
    "modeldata\tcells\n",
    "DAAG\tdengue\n",
    "mstate\tebmt1\n",
    "survival\tpbcseq (pbc)\n",
    "mlmRev\tContraception\n",
    "survival\tcolon (cancer)\n",
    "openintro\tsmoking\n",
    "causaldata\tnhefs\n",
    "causaldata\tnhefs_complete\n",
    "openintro\thealthcare_law_survey\n",
    "COUNT\tmedpar\n",
    "mosaicData\tHELPfull\n",
    "openintro\thfi\n",
    "Stat2Data\tNCbirths\n",
    "survival\tmgus2 (cancer)\n",
    "Ecdat\tCigar\n",
    "plm\tCigar\n",
    "survival\tudca2 (udca)\n",
    "Stat2Data\tWhickham2\n",
    "DAAG\tmifem\n",
    "HistData\tSnow.streets\n",
    "mosaicData\tGestation\n",
    "openintro\tbabies\n",
    "MASS\tOME\n",
    "openintro\tsleep_deprivation\n",
    "openintro\tebola_survey\n",
    "openintro\tscotus_healthcare\n",
    "openintro\tbirths14\n",
    "openintro\tncbirths\n",
    "AER\tMedicaid1986\n",
    "carData\tBlackmore\n",
    "HistData\tGaltonFamilies\n",
    "multgee\tarthritis\n",
    "medicaldata\topt\n",
    "survival\ttransplant\n",
    "sampleSelection\tSmoke\n",
    "HistData\tCholeraDeaths1849\n",
    "MASS\tbiopsy\n",
    "openintro\tdiabetes2\n",
    "survival\tgbsg (cancer)\n",
    "survival\tmyeloid (cancer)\n",
    "texmex\tliver\n",
    "HLMdiag\tautism\n",
    "medicaldata\tindo_rct\n",
    "quantreg\tuis\n",
    "boot\taids\n",
    "dslabs\tbrca\n",
    "HSAUR\trespiratory\n",
    "Ecdat\tCigarette\n",
    "openintro\tfastfood\n",
    "openintro\tbdims\n",
    "dslabs\toutlier_example\n",
    "Stat2Data\tBlood1\n",
    "Stat2Data\tYouthRisk2009\n",
    "tidyr\tcms_patient_experience\n",
    "\"\"\"\n",
    "\n",
    "# Splitting the data into lines and creating two lists\n",
    "first_entries = []\n",
    "second_entries = []\n",
    "\n",
    "for line in data.strip().split('\\n'):\n",
    "    entries = line.split()\n",
    "    \n",
    "    if len(entries) == 2:\n",
    "        first_entries.append(entries[0])\n",
    "        second_entries.append(entries[1])\n",
    "    elif len(entries) == 3:\n",
    "        first_entries.append(entries[0])\n",
    "        second_entries.append(entries[1] + ' ' + entries[2])\n",
    "    \n",
    "# Displaying the lists\n",
    "print(\"First Entries:\", first_entries)\n",
    "print(\"\\nSecond Entries:\", second_entries)\n",
    "\n",
    "print(len(first_entries))\n",
    "print(len(second_entries))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b238ce93",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "survival flchain     Assay of serum free light chain for 7874 subjects.\n",
      "AER DoctorVisits     Australian Health Service Utilization Data\n",
      "AER HealthInsurance     Medical Expenditure Panel Survey Data\n",
      "AER Medicaid1986     Medicaid Utilization Data\n",
      "asaur hepatoCellular     hepatoCellular\n",
      "asaur pancreatic     pancreatic\n",
      "asaur pancreatic2     pancreatic2\n",
      "asaur pharmacoSmoking     pharmacoSmoking\n",
      "asaur prostateSurvival     prostateSurvival\n",
      "betareg StressAnxiety     Dependency of Anxiety on Stress\n",
      "boot aids     Delay in AIDS Reporting in England and Wales\n",
      "boot cd4     CD4 Counts for HIV-Positive Patients\n",
      "boot downs.bc     Incidence of Down's Syndrome in British Columbia\n",
      "boot melanoma     Survival from Malignant Melanoma\n",
      "boot neuro     Neurophysiological Point Process Data\n",
      "boot nodal     Nodal Involvement in Prostate Cancer\n",
      "boot remission     Cancer Remission and Cell Activity\n",
      "boot survival     Survival of Rats after Radiation Doses\n",
      "boot urine     Urine Analysis Data\n",
      "carData Blackmore     Exercise Histories of Eating-Disordered and Control\n",
      "carData Davis     Self-Reports of Height and Weight\n",
      "carData KosteckiDillon     Treatment of Migraine Headaches\n",
      "carData Leinhardt     Data on Infant-Mortality\n",
      "carData LoBD     Cancer drug data use to provide an example of the use of\n",
      "carData Robey     Fertility and Contraception\n",
      "carData WeightLoss     Weight Loss Data\n",
      "carData Wong     Post-Coma Recovery of IQ\n",
      "causaldata abortion     Data on abortion legalization and sexually transmitted\n",
      "causaldata nhefs     National Health and Nutrition Examination Survey Data I\n",
      "causaldata nhefs_complete     Complete-Data National Health and Nutrition Examination\n",
      "causaldata organ_donations     Organ Donation Data\n",
      "causaldata thornton_hiv     Data from HIV information experiment in Thornton (2008)\n",
      "openintro simpsons_paradox_covid     Simpson's Paradox: Covid\n",
      "AER Fertility     Fertility and Women's Labor Supply\n",
      "openintro avandia     Cardiovascular problems for two types of Diabetes\n",
      "openintro mammogram     Experiment with Mammogram Randomized\n",
      "stevedata gss_abortion     Abortion Opinions in the General Social Survey\n",
      "stevedata mm_nhis     Data from the 2009 National Health Interview Survey\n",
      "AER Fertility2     Fertility and Women's Labor Supply\n",
      "Ecdat HI     Health Insurance and Hours Worked By Wives\n",
      "sampleSelection RandHIE     RAND Health Insurance Experiment\n",
      "Ecdat DoctorContacts     Contacts With Medical Doctor\n",
      "openintro health_coverage     Health Coverage and Health Status\n",
      "dslabs us_contagious_diseases     Contagious disease data for US states\n",
      "medicaldata covid_testing     Deidentified Results of COVID-19 testing at the\n",
      "geepack muscatine     Data on Obesity from the Muscatine Coronary Risk Factor\n",
      "openintro yrbss     Youth Risk Behavior Surveillance System (YRBSS)\n",
      "Stat2Data YouthRisk     Annual survey of health-risk youth behaviors\n",
      "stevedata wvs_usa_abortion     Attitudes on the Justifiability of Abortion in the United\n",
      "mstate ebmt2     Data from the European Society for Blood and Marrow\n",
      "mosaicData Births     US Births\n",
      "tidyr who     World Health Organization TB data\n",
      "tidyr who2     World Health Organization TB data\n",
      "DAAG monica     WHO Monica Data\n",
      "openintro smallpox     Smallpox vaccine results\n",
      "stevedata anes_prochoice     Abortion Attitudes (ANES, 2012)\n",
      "Ecdat MedExp     Structure of Demand for Medical Care\n",
      "mosaicData BirthsSSA     US Births\n",
      "Ecdat DoctorAUS     Doctor Visits in Australia\n",
      "Ecdat OFP     Visits to Physician Office\n",
      "heplots AddHealth     Adolescent Health Data\n",
      "carData KosteckiDillon     Treatment of Migraine Headaches\n",
      "tidyr population     World Health Organization TB data\n",
      "mosaicData BirthsCDC     US Births\n",
      "MASS Aids2     Australian AIDS Survival Data\n",
      "mlmRev guPrenat     Prenatal care in Guatemala\n",
      "mstate ebmt4     Data from the European Society for Blood and Marrow\n",
      "openintro biontech_adolescents     Efficacy of Pfizer-BioNTech COVID-19 vaccine on\n",
      "mstate ebmt3     Data from the European Society for Blood and Marrow\n",
      "mstate prothr     Abnormal prothrombin levels in liver cirrhosis\n",
      "geepack ohio     Ohio Children Wheeze Status\n",
      "modeldata cells     Cell body segmentation\n",
      "DAAG dengue     Dengue prevalence, by administrative region\n",
      "mstate ebmt1     Data from the European Society for Blood and Marrow\n",
      "mlmRev Contraception     Contraceptive use in Bangladesh\n",
      "openintro smoking     UK Smoking Data\n",
      "causaldata nhefs     National Health and Nutrition Examination Survey Data I\n",
      "causaldata nhefs_complete     Complete-Data National Health and Nutrition Examination\n",
      "openintro healthcare_law_survey     Pew Research Center poll on health care, including\n",
      "COUNT medpar     medpar\n",
      "mosaicData HELPfull     Health Evaluation and Linkage to Primary Care\n",
      "openintro hfi     Human Freedom Index\n",
      "Stat2Data NCbirths     North Carolina Birth Records\n",
      "Ecdat Cigar     Cigarette Consumption\n",
      "plm Cigar     Cigarette Consumption\n",
      "Stat2Data Whickham2     Whickham Health Study\n",
      "DAAG mifem     WHO Monica Data\n",
      "HistData Snow.streets     John Snow's Map and Data on the 1854 London Cholera\n",
      "mosaicData Gestation     Data from the Child Health and Development Studies\n",
      "openintro babies     The Child Health and Development Studies\n",
      "MASS OME     Tests of Auditory Perception in Children with OME\n",
      "openintro sleep_deprivation     Survey on sleep deprivation and transportation workers\n",
      "openintro ebola_survey     Survey on Ebola quarantine\n",
      "openintro scotus_healthcare     Public Opinion with SCOTUS ruling on American Healthcare\n",
      "openintro births14     US births\n",
      "openintro ncbirths     North Carolina births, 1000 cases\n",
      "AER Medicaid1986     Medicaid Utilization Data\n",
      "carData Blackmore     Exercise Histories of Eating-Disordered and Control\n",
      "HistData GaltonFamilies     Galton's data on the heights of parents and their\n",
      "multgee arthritis     Rheumatoid Arthritis Clinical Trial\n",
      "medicaldata opt     Obstetrics and Periodontal Therapy Dataset\n",
      "survival transplant     Liver transplant waiting list\n",
      "sampleSelection Smoke     Survey Responses on Smoking Behaviour\n",
      "HistData CholeraDeaths1849     Daily Deaths from Cholera and Diarrhaea in England, 1849\n",
      "MASS biopsy     Biopsy Data on Breast Cancer Patients\n",
      "openintro diabetes2     Type 2 Diabetes Clinical Trial for Patients 10-17 Years\n",
      "texmex liver     Liver related laboratory data\n",
      "HLMdiag autism     Autism data\n",
      "medicaldata indo_rct     RCT of Indomethacin for Prevention of Post-ERCP\n",
      "quantreg uis     UIS Drug Treatment study data\n",
      "boot aids     Delay in AIDS Reporting in England and Wales\n",
      "dslabs brca     Breast Cancer Wisconsin Diagnostic Dataset from UCI\n",
      "HSAUR respiratory     Respiratory Illness Data\n",
      "Ecdat Cigarette     The Cigarette Consumption Panel Data Set\n",
      "openintro fastfood     Nutrition in fast food\n",
      "openintro bdims     Body measurements of 507 physically active individuals.\n",
      "dslabs outlier_example     Adult male heights in feet with outliers\n",
      "Stat2Data Blood1     Blood Pressure, Weight, and Smoking Status\n",
      "Stat2Data YouthRisk2009     Youth Risk Survey\n",
      "tidyr cms_patient_experience     Data from the Centers for Medicare & Medicaid Services\n"
     ]
    }
   ],
   "source": [
    "packages = []\n",
    "items = []\n",
    "labels = []\n",
    "docs = []\n",
    "rows = []\n",
    "cols = []\n",
    "\n",
    "for i, package in enumerate(first_entries):\n",
    "    item = second_entries[i]\n",
    "\n",
    "    try:\n",
    "        df_doc = sm.datasets.get_rdataset(item, package)\n",
    "        \n",
    "    except:\n",
    "        continue\n",
    "\n",
    "    obj = df_doc.__doc__\n",
    "    for line in obj.strip().split('\\n'):\n",
    "        if '.. rubric::' in line:\n",
    "            ln = line.replace('.. rubric::', '')\n",
    "            print(package, item, ln)\n",
    "            \n",
    "            packages.append(package)\n",
    "            items.append(item)\n",
    "            labels.append(ln + ' (' + package + ': ' + item + ')')\n",
    "            docs.append(obj)\n",
    "            \n",
    "            df = df_doc.data\n",
    "            rows.append(df.shape[0])\n",
    "            cols.append(df.shape[1])\n",
    "        \n",
    "            break\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3a807b59",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "114 healthcare datasets from Statsmodels\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package</th>\n",
       "      <th>item</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>No. of rows</th>\n",
       "      <th>No. of columns</th>\n",
       "      <th>docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>survival</td>\n",
       "      <td>flchain</td>\n",
       "      <td>Assay of serum free light chain for 7874 s...</td>\n",
       "      <td>7874</td>\n",
       "      <td>11</td>\n",
       "      <td>.. container::\\n\\n   ======= ===============\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AER</td>\n",
       "      <td>DoctorVisits</td>\n",
       "      <td>Australian Health Service Utilization Data...</td>\n",
       "      <td>5190</td>\n",
       "      <td>12</td>\n",
       "      <td>.. container::\\n\\n   ============ ============...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AER</td>\n",
       "      <td>HealthInsurance</td>\n",
       "      <td>Medical Expenditure Panel Survey Data (AER...</td>\n",
       "      <td>8802</td>\n",
       "      <td>11</td>\n",
       "      <td>.. container::\\n\\n   =============== =========...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AER</td>\n",
       "      <td>Medicaid1986</td>\n",
       "      <td>Medicaid Utilization Data (AER: Medicaid1986)</td>\n",
       "      <td>996</td>\n",
       "      <td>14</td>\n",
       "      <td>.. container::\\n\\n   ============ ============...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>asaur</td>\n",
       "      <td>hepatoCellular</td>\n",
       "      <td>hepatoCellular (asaur: hepatoCellular)</td>\n",
       "      <td>227</td>\n",
       "      <td>48</td>\n",
       "      <td>.. container::\\n\\n   ============== ==========...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    package             item  \\\n",
       "0  survival          flchain   \n",
       "1       AER     DoctorVisits   \n",
       "2       AER  HealthInsurance   \n",
       "3       AER     Medicaid1986   \n",
       "4     asaur   hepatoCellular   \n",
       "\n",
       "                                             Dataset  No. of rows  \\\n",
       "0      Assay of serum free light chain for 7874 s...         7874   \n",
       "1      Australian Health Service Utilization Data...         5190   \n",
       "2      Medical Expenditure Panel Survey Data (AER...         8802   \n",
       "3      Medicaid Utilization Data (AER: Medicaid1986)          996   \n",
       "4             hepatoCellular (asaur: hepatoCellular)          227   \n",
       "\n",
       "   No. of columns                                               docs  \n",
       "0              11  .. container::\\n\\n   ======= ===============\\n...  \n",
       "1              12  .. container::\\n\\n   ============ ============...  \n",
       "2              11  .. container::\\n\\n   =============== =========...  \n",
       "3              14  .. container::\\n\\n   ============ ============...  \n",
       "4              48  .. container::\\n\\n   ============== ==========...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf = pd.DataFrame(columns=['package', 'item', 'Dataset', 'No. of rows', 'No. of columns', 'docs'])\n",
    "tdf['package'] = packages\n",
    "tdf['item'] = items\n",
    "tdf['Dataset'] = labels\n",
    "tdf['No. of rows'] = rows\n",
    "tdf['No. of columns'] = cols\n",
    "tdf['docs'] = docs\n",
    "\n",
    "tdf.drop_duplicates(inplace=True)\n",
    "print(tdf.shape[0],'healthcare datasets from Statsmodels')\n",
    "tdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "40bb5e55",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".. container::\n",
      "\n",
      "   ======= ===============\n",
      "   flchain R Documentation\n",
      "   ======= ===============\n",
      "\n",
      "   .. rubric:: Assay of serum free light chain for 7874 subjects.\n",
      "      :name: flchain\n",
      "\n",
      "   .. rubric:: Description\n",
      "      :name: description\n",
      "\n",
      "   This is a stratified random sample containing 1/2 of the subjects\n",
      "   from a study of the relationship between serum free light chain (FLC)\n",
      "   and mortality. The original sample contains samples on approximately\n",
      "   2/3 of the residents of Olmsted County aged 50 or greater.\n",
      "\n",
      "   .. rubric:: Usage\n",
      "      :name: usage\n",
      "\n",
      "   .. code:: R\n",
      "\n",
      "      flchain\n",
      "      data(flchain, package=\"survival\")\n",
      "\n",
      "   .. rubric:: Format\n",
      "      :name: format\n",
      "\n",
      "   A data frame with 7874 persons containing the following variables.\n",
      "\n",
      "   ``age``\n",
      "      age in years\n",
      "\n",
      "   ``sex``\n",
      "      F=female, M=male\n",
      "\n",
      "   ``sample.yr``\n",
      "      the calendar year in which a blood sample was obtained\n",
      "\n",
      "   ``kappa``\n",
      "      serum free light chain, kappa portion\n",
      "\n",
      "   ``lambda``\n",
      "      serum free light chain, lambda portion\n",
      "\n",
      "   ``flc.grp``\n",
      "      the FLC group for the subject, as used in the original analysis\n",
      "\n",
      "   ``creatinine``\n",
      "      serum creatinine\n",
      "\n",
      "   ``mgus``\n",
      "      1 if the subject had been diagnosed with monoclonal gammapothy\n",
      "      (MGUS)\n",
      "\n",
      "   ``futime``\n",
      "      days from enrollment until death. Note that there are 3 subjects\n",
      "      whose sample was obtained on their death date.\n",
      "\n",
      "   ``death``\n",
      "      0=alive at last contact date, 1=dead\n",
      "\n",
      "   ``chapter``\n",
      "      for those who died, a grouping of their primary cause of death by\n",
      "      chapter headings of the International Code of Diseases ICD-9\n",
      "\n",
      "   .. rubric:: Details\n",
      "      :name: details\n",
      "\n",
      "   In 1995 Dr. Robert Kyle embarked on a study to determine the\n",
      "   prevalence of monoclonal gammopathy of undetermined significance\n",
      "   (MGUS) in Olmsted County, Minnesota, a condition which is normally\n",
      "   only found by chance from a test (serum electrophoresis) which is\n",
      "   ordered for other causes. Later work suggested that one component of\n",
      "   immunoglobulin production, the serum free light chain, might be a\n",
      "   possible marker for immune disregulation. In 2010 Dr. Angela\n",
      "   Dispenzieri and colleagues assayed FLC levels on those samples from\n",
      "   the original study for which they had patient permission and from\n",
      "   which sufficient material remained for further testing. They found\n",
      "   that elevated FLC levels were indeed associated with higher death\n",
      "   rates.\n",
      "\n",
      "   Patients were recruited when they came to the clinic for other\n",
      "   appointments, with a final random sample of those who had not yet had\n",
      "   a visit since the study began. An interesting side question is\n",
      "   whether there are differences between early, mid, and late recruits.\n",
      "\n",
      "   This data set contains an age and sex stratified random sample that\n",
      "   includes 7874 of the original 15759 subjects. The original subject\n",
      "   identifiers and dates have been removed to protect patient identity.\n",
      "   Subsampling was done to further protect this information.\n",
      "\n",
      "   .. rubric:: Source\n",
      "      :name: source\n",
      "\n",
      "   The primary investigator (A Dispenzieri) and statistician (T\n",
      "   Therneau) for the study.\n",
      "\n",
      "   .. rubric:: References\n",
      "      :name: references\n",
      "\n",
      "   A Dispenzieri, J Katzmann, R Kyle, D Larson, T Therneau, C Colby, R\n",
      "   Clark, G Mead, S Kumar, LJ Melton III and SV Rajkumar (2012). Use of\n",
      "   monclonal serum immunoglobulin free light chains to predict overall\n",
      "   survival in the general population, Mayo Clinic Proceedings\n",
      "   87:512-523.\n",
      "\n",
      "   R Kyle, T Therneau, SV Rajkumar, D Larson, M Plevak, J Offord, A\n",
      "   Dispenzieri, J Katzmann, and LJ Melton, III, 2006, Prevalence of\n",
      "   monoclonal gammopathy of undetermined significance, New England J\n",
      "   Medicine 354:1362-1369.\n",
      "\n",
      "   .. rubric:: Examples\n",
      "      :name: examples\n",
      "\n",
      "   .. code:: R\n",
      "\n",
      "      data(flchain)\n",
      "      age.grp <-  cut(flchain$age, c(49,54, 59,64, 69,74,79, 89, 110),\n",
      "                     labels= paste(c(50,55,60,65,70,75,80,90),\n",
      "                                   c(54,59,64,69,74,79,89,109), sep='-'))\n",
      "      table(flchain$sex, age.grp)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "obj = tdf['docs'].iloc[0]\n",
    "print(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "387bd65e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "470 unique labels\n"
     ]
    }
   ],
   "source": [
    "labels = tdf['Dataset'].tolist()\n",
    "\n",
    "str1 = ' '.join(labels)\n",
    "labels = str1.split()\n",
    "\n",
    "labels = sorted(list(set(labels)))\n",
    "print(len(labels), 'unique labels')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "11c476b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "451 unique words\n",
      "413 unique words\n",
      "\n",
      "\"&\",\n",
      "\"10-17\",\n",
      "\"1000\",\n",
      "\"1849\",\n",
      "\"1854\",\n",
      "\"2\",\n",
      "\"2008\",\n",
      "\"2009\",\n",
      "\"2012\",\n",
      "\"507\",\n",
      "\"7874\",\n",
      "\"abnormal\",\n",
      "\"abortion\",\n",
      "\"active\",\n",
      "\"activity\",\n",
      "\"addhealth\",\n",
      "\"administrative\",\n",
      "\"adolescent\",\n",
      "\"adult\",\n",
      "\"aer\",\n",
      "\"after\",\n",
      "\"aids\",\n",
      "\"aids2\",\n",
      "\"american\",\n",
      "\"an\",\n",
      "\"analysis\",\n",
      "\"and\",\n",
      "\"anes\",\n",
      "\"anes_prochoice\",\n",
      "\"annual\",\n",
      "\"anxiety\",\n",
      "\"arthritis\",\n",
      "\"asaur\",\n",
      "\"assay\",\n",
      "\"at\",\n",
      "\"attitudes\",\n",
      "\"auditory\",\n",
      "\"australia\",\n",
      "\"australian\",\n",
      "\"autism\",\n",
      "\"avandia\",\n",
      "\"babies\",\n",
      "\"bangladesh\",\n",
      "\"bdims\",\n",
      "\"behavior\",\n",
      "\"behaviors\",\n",
      "\"behaviour\",\n",
      "\"betareg\",\n",
      "\"biontech_adolescents\",\n",
      "\"biopsy\",\n",
      "\"birth\",\n",
      "\"births\",\n",
      "\"births14\",\n",
      "\"birthscdc\",\n",
      "\"birthsssa\",\n",
      "\"blackmore\",\n",
      "\"blood\",\n",
      "\"blood1\",\n",
      "\"body\",\n",
      "\"boot\",\n",
      "\"brca\",\n",
      "\"breast\",\n",
      "\"british\",\n",
      "\"by\",\n",
      "\"cancer\",\n",
      "\"cardata\",\n",
      "\"cardiovascular\",\n",
      "\"care\",\n",
      "\"carolina\",\n",
      "\"cases\",\n",
      "\"causaldata\",\n",
      "\"cd4\",\n",
      "\"cell\",\n",
      "\"cells\",\n",
      "\"center\",\n",
      "\"centers\",\n",
      "\"chain\",\n",
      "\"child\",\n",
      "\"children\",\n",
      "\"cholera\",\n",
      "\"choleradeaths1849\",\n",
      "\"cigar\",\n",
      "\"cigarette\",\n",
      "\"cirrhosis\",\n",
      "\"clinical\",\n",
      "\"cms_patient_experience\",\n",
      "\"columbia\",\n",
      "\"complete-data\",\n",
      "\"consumption\",\n",
      "\"contacts\",\n",
      "\"contagious\",\n",
      "\"contraception\",\n",
      "\"contraceptive\",\n",
      "\"control\",\n",
      "\"coronary\",\n",
      "\"count\",\n",
      "\"counts\",\n",
      "\"coverage\",\n",
      "\"covid\",\n",
      "\"covid-19\",\n",
      "\"covid_testing\",\n",
      "\"daag\",\n",
      "\"daily\",\n",
      "\"data\",\n",
      "\"dataset\",\n",
      "\"davis\",\n",
      "\"deaths\",\n",
      "\"deidentified\",\n",
      "\"delay\",\n",
      "\"demand\",\n",
      "\"dengue\",\n",
      "\"dependency\",\n",
      "\"deprivation\",\n",
      "\"development\",\n",
      "\"diabetes\",\n",
      "\"diabetes2\",\n",
      "\"diagnostic\",\n",
      "\"diarrhaea\",\n",
      "\"disease\",\n",
      "\"doctor\",\n",
      "\"doctoraus\",\n",
      "\"doctorcontacts\",\n",
      "\"doctorvisits\",\n",
      "\"donation\",\n",
      "\"doses\",\n",
      "\"down's\",\n",
      "\"downs.bc\",\n",
      "\"drug\",\n",
      "\"dslabs\",\n",
      "\"eating-disordered\",\n",
      "\"ebmt1\",\n",
      "\"ebmt2\",\n",
      "\"ebmt3\",\n",
      "\"ebmt4\",\n",
      "\"ebola\",\n",
      "\"ebola_survey\",\n",
      "\"ecdat\",\n",
      "\"efficacy\",\n",
      "\"england\",\n",
      "\"european\",\n",
      "\"evaluation\",\n",
      "\"examination\",\n",
      "\"example\",\n",
      "\"exercise\",\n",
      "\"expenditure\",\n",
      "\"experiment\",\n",
      "\"factor\",\n",
      "\"fast\",\n",
      "\"fastfood\",\n",
      "\"feet\",\n",
      "\"fertility\",\n",
      "\"fertility2\",\n",
      "\"flchain\",\n",
      "\"food\",\n",
      "\"for\",\n",
      "\"free\",\n",
      "\"freedom\",\n",
      "\"from\",\n",
      "\"galton's\",\n",
      "\"galtonfamilies\",\n",
      "\"geepack\",\n",
      "\"general\",\n",
      "\"gestation\",\n",
      "\"gss_abortion\",\n",
      "\"guatemala\",\n",
      "\"guprenat\",\n",
      "\"headaches\",\n",
      "\"health\",\n",
      "\"health-risk\",\n",
      "\"health_coverage\",\n",
      "\"healthcare\",\n",
      "\"healthcare_law_survey\",\n",
      "\"healthinsurance\",\n",
      "\"height\",\n",
      "\"heights\",\n",
      "\"helpfull\",\n",
      "\"hepatocellular\",\n",
      "\"heplots\",\n",
      "\"hfi\",\n",
      "\"hi\",\n",
      "\"histdata\",\n",
      "\"histories\",\n",
      "\"hiv\",\n",
      "\"hiv-positive\",\n",
      "\"hlmdiag\",\n",
      "\"hours\",\n",
      "\"hsaur\",\n",
      "\"human\",\n",
      "\"i\",\n",
      "\"illness\",\n",
      "\"in\",\n",
      "\"incidence\",\n",
      "\"including\",\n",
      "\"index\",\n",
      "\"individuals.\",\n",
      "\"indo_rct\",\n",
      "\"indomethacin\",\n",
      "\"infant-mortality\",\n",
      "\"information\",\n",
      "\"insurance\",\n",
      "\"interview\",\n",
      "\"involvement\",\n",
      "\"iq\",\n",
      "\"john\",\n",
      "\"justifiability\",\n",
      "\"kosteckidillon\",\n",
      "\"labor\",\n",
      "\"laboratory\",\n",
      "\"legalization\",\n",
      "\"leinhardt\",\n",
      "\"levels\",\n",
      "\"light\",\n",
      "\"linkage\",\n",
      "\"list\",\n",
      "\"liver\",\n",
      "\"lobd\",\n",
      "\"london\",\n",
      "\"loss\",\n",
      "\"male\",\n",
      "\"malignant\",\n",
      "\"mammogram\",\n",
      "\"map\",\n",
      "\"marrow\",\n",
      "\"mass\",\n",
      "\"measurements\",\n",
      "\"medexp\",\n",
      "\"medicaid\",\n",
      "\"medicaid1986\",\n",
      "\"medical\",\n",
      "\"medicaldata\",\n",
      "\"medicare\",\n",
      "\"medpar\",\n",
      "\"melanoma\",\n",
      "\"mifem\",\n",
      "\"migraine\",\n",
      "\"mlmrev\",\n",
      "\"mm_nhis\",\n",
      "\"modeldata\",\n",
      "\"monica\",\n",
      "\"mosaicdata\",\n",
      "\"mstate\",\n",
      "\"multgee\",\n",
      "\"muscatine\",\n",
      "\"national\",\n",
      "\"ncbirths\",\n",
      "\"neuro\",\n",
      "\"neurophysiological\",\n",
      "\"nhefs\",\n",
      "\"nhefs_complete\",\n",
      "\"nodal\",\n",
      "\"north\",\n",
      "\"nutrition\",\n",
      "\"obesity\",\n",
      "\"obstetrics\",\n",
      "\"of\",\n",
      "\"office\",\n",
      "\"ofp\",\n",
      "\"ohio\",\n",
      "\"ome\",\n",
      "\"on\",\n",
      "\"openintro\",\n",
      "\"opinion\",\n",
      "\"opinions\",\n",
      "\"opt\",\n",
      "\"organ\",\n",
      "\"organ_donations\",\n",
      "\"organization\",\n",
      "\"outlier_example\",\n",
      "\"outliers\",\n",
      "\"pancreatic\",\n",
      "\"pancreatic2\",\n",
      "\"panel\",\n",
      "\"paradox\",\n",
      "\"parents\",\n",
      "\"patients\",\n",
      "\"perception\",\n",
      "\"periodontal\",\n",
      "\"pew\",\n",
      "\"pfizer-biontech\",\n",
      "\"pharmacosmoking\",\n",
      "\"physically\",\n",
      "\"physician\",\n",
      "\"plm\",\n",
      "\"point\",\n",
      "\"poll\",\n",
      "\"population\",\n",
      "\"post-coma\",\n",
      "\"post-ercp\",\n",
      "\"prenatal\",\n",
      "\"pressure\",\n",
      "\"prevalence\",\n",
      "\"prevention\",\n",
      "\"primary\",\n",
      "\"problems\",\n",
      "\"process\",\n",
      "\"prostate\",\n",
      "\"prostatesurvival\",\n",
      "\"prothr\",\n",
      "\"prothrombin\",\n",
      "\"provide\",\n",
      "\"public\",\n",
      "\"quantreg\",\n",
      "\"quarantine\",\n",
      "\"radiation\",\n",
      "\"rand\",\n",
      "\"randhie\",\n",
      "\"randomized\",\n",
      "\"rats\",\n",
      "\"rct\",\n",
      "\"records\",\n",
      "\"recovery\",\n",
      "\"region\",\n",
      "\"related\",\n",
      "\"remission\",\n",
      "\"reporting\",\n",
      "\"research\",\n",
      "\"respiratory\",\n",
      "\"responses\",\n",
      "\"results\",\n",
      "\"rheumatoid\",\n",
      "\"risk\",\n",
      "\"robey\",\n",
      "\"ruling\",\n",
      "\"sampleselection\",\n",
      "\"scotus\",\n",
      "\"scotus_healthcare\",\n",
      "\"segmentation\",\n",
      "\"self-reports\",\n",
      "\"serum\",\n",
      "\"service\",\n",
      "\"services\",\n",
      "\"set\",\n",
      "\"sexually\",\n",
      "\"simpson's\",\n",
      "\"simpsons_paradox_covid\",\n",
      "\"sleep\",\n",
      "\"sleep_deprivation\",\n",
      "\"smallpox\",\n",
      "\"smoke\",\n",
      "\"smoking\",\n",
      "\"snow's\",\n",
      "\"snow.streets\",\n",
      "\"social\",\n",
      "\"society\",\n",
      "\"stat2data\",\n",
      "\"states\",\n",
      "\"status\",\n",
      "\"stevedata\",\n",
      "\"stress\",\n",
      "\"stressanxiety\",\n",
      "\"structure\",\n",
      "\"studies\",\n",
      "\"study\",\n",
      "\"subjects.\",\n",
      "\"supply\",\n",
      "\"surveillance\",\n",
      "\"survey\",\n",
      "\"survival\",\n",
      "\"syndrome\",\n",
      "\"system\",\n",
      "\"tb\",\n",
      "\"testing\",\n",
      "\"tests\",\n",
      "\"texmex\",\n",
      "\"the\",\n",
      "\"their\",\n",
      "\"therapy\",\n",
      "\"thornton\",\n",
      "\"thornton_hiv\",\n",
      "\"tidyr\",\n",
      "\"to\",\n",
      "\"transmitted\",\n",
      "\"transplant\",\n",
      "\"transportation\",\n",
      "\"treatment\",\n",
      "\"trial\",\n",
      "\"two\",\n",
      "\"type\",\n",
      "\"types\",\n",
      "\"uci\",\n",
      "\"uis\",\n",
      "\"uk\",\n",
      "\"united\",\n",
      "\"urine\",\n",
      "\"us\",\n",
      "\"us_contagious_diseases\",\n",
      "\"use\",\n",
      "\"utilization\",\n",
      "\"vaccine\",\n",
      "\"visits\",\n",
      "\"waiting\",\n",
      "\"wales\",\n",
      "\"weight\",\n",
      "\"weightloss\",\n",
      "\"wheeze\",\n",
      "\"whickham\",\n",
      "\"whickham2\",\n",
      "\"who\",\n",
      "\"who2\",\n",
      "\"wisconsin\",\n",
      "\"with\",\n",
      "\"wives\",\n",
      "\"women's\",\n",
      "\"wong\",\n",
      "\"worked\",\n",
      "\"workers\",\n",
      "\"world\",\n",
      "\"wvs_usa_abortion\",\n",
      "\"years\",\n",
      "\"youth\",\n",
      "\"youthrisk\",\n",
      "\"youthrisk2009\",\n",
      "\"yrbss\",\n"
     ]
    }
   ],
   "source": [
    "# Remove parentheses, commas, and colons\n",
    "cleaned_text = str1.replace(\"(\", \"\").replace(\")\", \"\").replace(\",\", \"\").replace(\":\", \"\")\n",
    "\n",
    "# Split the cleaned text into a list of individual words\n",
    "word_list = cleaned_text.split()\n",
    "word_list = sorted(list(set(word_list)))\n",
    "print(len(word_list), 'unique words')\n",
    "\n",
    "ls2 = []\n",
    "for word in word_list:\n",
    "    ls2.append(word.lower())\n",
    "word_list = sorted(list(set(ls2)))\n",
    "print(len(word_list), 'unique words\\n')\n",
    "\n",
    "# Display the resulting list\n",
    "for i in word_list:\n",
    "    print(\"\\\"\"+i+\"\\\"\"+',')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "450a417f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "tags_ls = [\n",
    "    #\"&\",\n",
    "    #\"10-17\",\n",
    "    #\"1000\",\n",
    "    #\"1849\",\n",
    "    #\"1854\",\n",
    "    #\"2\",\n",
    "    #\"2008\",\n",
    "    #\"2009\",\n",
    "    #\"2012\",\n",
    "    #\"507\",\n",
    "    #\"7874\",\n",
    "    #\"abnormal\",\n",
    "    \"abortion\",\n",
    "    #\"active\",\n",
    "    #\"activity\",\n",
    "    #\"addhealth\",\n",
    "    #\"administrative\",\n",
    "    \"adolescent\",\n",
    "    \"adult\",\n",
    "    #\"aer\",\n",
    "    #\"after\",\n",
    "    \"AIDS \", #\"aids\",\n",
    "    #\"aids2\",\n",
    "    #\"american\",\n",
    "    #\"an\",\n",
    "    #\"analysis\",\n",
    "    #\"and\",\n",
    "    #\"anes\",\n",
    "    #\"anes_prochoice\",\n",
    "    #\"annual\",\n",
    "    \"anxiety\",\n",
    "    \"arthritis\",\n",
    "    #\"asaur\",\n",
    "    #\"assay\",\n",
    "    #\"at\",\n",
    "    #\"attitudes\",\n",
    "    \"auditory\",\n",
    "    #\"australia\",\n",
    "    #\"australian\",\n",
    "    \"autism\",\n",
    "    #\"avandia\",\n",
    "    #\"babies\",\n",
    "    #\"bangladesh\",\n",
    "    #\"bdims\",\n",
    "    \"behavior\",\n",
    "    #\"behaviors\",\n",
    "    #\"behaviour\",\n",
    "    #\"betareg\",\n",
    "    #\"biontech_adolescents\",\n",
    "    \"biopsy\",\n",
    "    \"birth\",\n",
    "    #\"births\",\n",
    "    #\"births14\",\n",
    "    #\"birthscdc\",\n",
    "    #\"birthsssa\",\n",
    "    #\"blackmore\",\n",
    "    \"blood\",\n",
    "    #\"blood1\",\n",
    "    \"body\",\n",
    "    #\"boot\",\n",
    "    #\"brca\",\n",
    "    \"breast\",\n",
    "    #\"british\",\n",
    "    #\"by\",\n",
    "    \"cancer\",\n",
    "    #\"cardata\",\n",
    "    \"cardiovascular\",\n",
    "    #\"care\",\n",
    "    #\"carolina\",\n",
    "    #\"cases\",\n",
    "    #\"causaldata\",\n",
    "    #\"cd4\",\n",
    "    \"cell\",\n",
    "    #\"cells\",\n",
    "    #\"center\",\n",
    "    #\"centers\",\n",
    "    #\"chain\",\n",
    "    \"child\",\n",
    "    #\"children\",\n",
    "    \"cholera\",\n",
    "    #\"choleradeaths1849\",\n",
    "    #\"cigar\",\n",
    "    #\"cigarette\",\n",
    "    \"cirrhosis\",\n",
    "    \"clinical\",\n",
    "    #\"cms_patient_experience\",\n",
    "    #\"columbia\",\n",
    "    #\"complete-data\",\n",
    "    \"consumption\",\n",
    "    #\"contacts\",\n",
    "    \"contagious\",\n",
    "    #\"contraception\",\n",
    "    \"contraceptive\",\n",
    "    #\"control\",\n",
    "    \"coronary\",\n",
    "    #\"count\",\n",
    "    #\"counts\",\n",
    "    #\"coverage\",\n",
    "    \"covid\",\n",
    "    #\"covid-19\",\n",
    "    #\"covid_testing\",\n",
    "    #\"daag\",\n",
    "    #\"daily\",\n",
    "    #\"data\",\n",
    "    #\"dataset\",\n",
    "    #\"davis\",\n",
    "    \"deaths\",\n",
    "    #\"deidentified\",\n",
    "    #\"delay\",\n",
    "    #\"demand\",\n",
    "    \"dengue\",\n",
    "    \"dependency\",\n",
    "    \"deprivation\",\n",
    "    #\"development\",\n",
    "    \"diabetes\",\n",
    "    #\"diabetes2\",\n",
    "    #\"diagnostic\",\n",
    "    \"diarrhaea\",\n",
    "    \"disease\",\n",
    "    \"doctor\",\n",
    "    #\"doctoraus\",\n",
    "    #\"doctorcontacts\",\n",
    "    #\"doctorvisits\",\n",
    "    \"donation\",\n",
    "    #\"doses\",\n",
    "    #\"down's\",\n",
    "    \"Down\", #\"downs.bc\",\n",
    "    \"drug\",\n",
    "    #\"dslabs\",\n",
    "    \"eating\", #\"eating-disordered\",\n",
    "    #\"ebmt1\",\n",
    "    #\"ebmt2\",\n",
    "    #\"ebmt3\",\n",
    "    #\"ebmt4\",\n",
    "    \"ebola\",\n",
    "    #\"ebola_survey\",\n",
    "    #\"ecdat\",\n",
    "    \"efficacy\",\n",
    "    #\"england\",\n",
    "    #\"european\",\n",
    "    #\"evaluation\",\n",
    "    #\"examination\",\n",
    "    #\"example\",\n",
    "    \"exercise\",\n",
    "    #\"expenditure\",\n",
    "    #\"experiment\",\n",
    "    #\"factor\",\n",
    "    #\"fast\",\n",
    "    #\"fastfood\",\n",
    "    \"feet\",\n",
    "    \"fertility\",\n",
    "    #\"fertility2\",\n",
    "    #\"flchain\",\n",
    "    \"food\",\n",
    "    #\"for\",\n",
    "    #\"free\",\n",
    "    \"freedom\",\n",
    "    #\"from\",\n",
    "    #\"galton's\",\n",
    "    #\"galtonfamilies\",\n",
    "    #\"geepack\",\n",
    "    #\"general\",\n",
    "    \"gestation\",\n",
    "    #\"gss_abortion\",\n",
    "    #\"guatemala\",\n",
    "    #\"guprenat\",\n",
    "    \"headaches\",\n",
    "    \"health\",\n",
    "    \"risk\", #\"health-risk\",\n",
    "    #\"health_coverage\",\n",
    "    \"healthcare\",\n",
    "    #\"healthcare_law_survey\",\n",
    "    #\"healthinsurance\",\n",
    "    #\"height\",\n",
    "    #\"heights\",\n",
    "    #\"helpfull\",\n",
    "    \"hepatocellular\",\n",
    "    #\"heplots\",\n",
    "    #\"hfi\",\n",
    "    #\"hi\",\n",
    "    #\"histdata\",\n",
    "    #\"histories\",\n",
    "    \"hiv\",\n",
    "    #\"hiv-positive\",\n",
    "    #\"hlmdiag\",\n",
    "    #\"hours\",\n",
    "    #\"hsaur\",\n",
    "    #\"human\",\n",
    "    #\"i\",\n",
    "    \"illness\",\n",
    "    #\"in\",\n",
    "    #\"incidence\",\n",
    "    #\"including\",\n",
    "    #\"index\",\n",
    "    #\"individuals.\",\n",
    "    #\"indo_rct\",\n",
    "    \"indomethacin\",\n",
    "    \"infant\", #\"infant-mortality\",\n",
    "    #\"information\",\n",
    "    \"insurance\",\n",
    "    #\"interview\",\n",
    "    #\"involvement\",\n",
    "    #\"iq\",\n",
    "    #\"john\",\n",
    "    #\"justifiability\",\n",
    "    #\"kosteckidillon\",\n",
    "    #\"labor\",\n",
    "    #\"laboratory\",\n",
    "    #\"legalization\",\n",
    "    #\"leinhardt\",\n",
    "    #\"levels\",\n",
    "    #\"light\",\n",
    "    #\"linkage\",\n",
    "    #\"list\",\n",
    "    \"liver\",\n",
    "    #\"lobd\",\n",
    "    #\"london\",\n",
    "    #\"loss\",\n",
    "    \"male\",\n",
    "    \"malignant\",\n",
    "    \"mammogram\",\n",
    "    #\"map\",\n",
    "    \"marrow\",\n",
    "    #\"mass\",\n",
    "    #\"measurements\",\n",
    "    #\"medexp\",\n",
    "    \"medicaid\",\n",
    "    #\"medicaid1986\",\n",
    "    \"medical\",\n",
    "    #\"medicaldata\",\n",
    "    \"medicare\",\n",
    "    \"medpar\",\n",
    "    \"melanoma\",\n",
    "    #\"mifem\",\n",
    "    \"migraine\",\n",
    "    #\"mlmrev\",\n",
    "    #\"mm_nhis\",\n",
    "    #\"modeldata\",\n",
    "    #\"monica\",\n",
    "    #\"mosaicdata\",\n",
    "    #\"mstate\",\n",
    "    #\"multgee\",\n",
    "    #\"muscatine\",\n",
    "    #\"national\",\n",
    "    #\"ncbirths\",\n",
    "    \"neuro\",\n",
    "    #\"neurophysiological\",\n",
    "    #\"nhefs\",\n",
    "    #\"nhefs_complete\",\n",
    "    #\"nodal\",\n",
    "    #\"north\",\n",
    "    \"nutrition\",\n",
    "    \"obesity\",\n",
    "    \"obstetrics\",\n",
    "    #\"of\",\n",
    "    #\"office\",\n",
    "    #\"ofp\",\n",
    "    #\"ohio\",\n",
    "    #\"ome\",\n",
    "    #\"on\",\n",
    "    #\"openintro\",\n",
    "    #\"opinion\",\n",
    "    #\"opinions\",\n",
    "    #\"opt\",\n",
    "    \"organ\",\n",
    "    #\"organ_donations\",\n",
    "    #\"organization\",\n",
    "    #\"outlier_example\",\n",
    "    #\"outliers\",\n",
    "    \"pancreatic\",\n",
    "    #\"pancreatic2\",\n",
    "    #\"panel\",\n",
    "    #\"paradox\",\n",
    "    \"parents\",\n",
    "    #\"patients\",\n",
    "    #\"perception\",\n",
    "    \"periodontal\",\n",
    "    #\"pew\",\n",
    "    #\"pfizer\", #pfizer-biontech\",\n",
    "    #\"pharmacosmoking\",\n",
    "    #\"physically\",\n",
    "    \"physician\",\n",
    "    #\"plm\",\n",
    "    #\"point\",\n",
    "    #\"poll\",\n",
    "    #\"population\",\n",
    "    \"coma\",#\"post-coma\",\n",
    "    #\"post-ercp\",\n",
    "    \"prenatal\",\n",
    "    #\"pressure\",\n",
    "    #\"prevalence\",\n",
    "    #\"prevention\",\n",
    "    #\"primary\",\n",
    "    #\"problems\",\n",
    "    #\"process\",\n",
    "    \"prostate\",\n",
    "    #\"prostatesurvival\",\n",
    "    #\"prothr\",\n",
    "    \"prothrombin\",\n",
    "    #\"provide\",\n",
    "    #\"public\",\n",
    "    #\"quantreg\",\n",
    "    \"quarantine\",\n",
    "    \"radiation\",\n",
    "    #\"rand\",\n",
    "    #\"randhie\",\n",
    "    #\"randomized\",\n",
    "    #\"rats\",\n",
    "    #\"rct\",\n",
    "    #\"records\",\n",
    "    \"recovery\",\n",
    "    #\"region\",\n",
    "    #\"related\",\n",
    "    \"remission\",\n",
    "    \"reporting\",  # !!\n",
    "    #\"research\",\n",
    "    \"respiratory\",\n",
    "    #\"responses\",\n",
    "    #\"results\",\n",
    "    #\"rheumatoid\",\n",
    "    #\"risk\",\n",
    "    #\"robey\",\n",
    "    #\"ruling\",\n",
    "    #\"sampleselection\",\n",
    "    #\"scotus\",\n",
    "    #\"scotus_healthcare\",\n",
    "    #\"segmentation\",\n",
    "    #\"self-reports\",\n",
    "    \"serum\",\n",
    "    #\"service\",\n",
    "    #\"services\",\n",
    "    #\"set\",\n",
    "    #\"sexually\",\n",
    "    #\"simpson's\",\n",
    "    #\"simpsons_paradox_covid\",\n",
    "    \"sleep\",\n",
    "    #\"sleep_deprivation\",\n",
    "    \"smallpox\",\n",
    "    #\"smoke\",\n",
    "    \"smoking\",\n",
    "    #\"snow's\",\n",
    "    #\"snow.streets\",\n",
    "    #\"social\",\n",
    "    \"society\",\n",
    "    #\"stat2data\",\n",
    "    #\"states\",\n",
    "    #\"status\",\n",
    "    #\"stevedata\",\n",
    "    \"stress\",\n",
    "    #\"stressanxiety\",\n",
    "    #\"structure\",\n",
    "    #\"studies\",\n",
    "    #\"study\",\n",
    "    #\"subjects.\",\n",
    "    #\"supply\",\n",
    "    #\"surveillance\",\n",
    "    #\"survey\",\n",
    "    #\"survival\",\n",
    "    \"syndrome\",\n",
    "    #\"system\",\n",
    "    #\"tb\",\n",
    "    #\"testing\",\n",
    "    #\"tests\",\n",
    "    #\"texmex\",\n",
    "    #\"the\",\n",
    "    #\"their\",\n",
    "    #\"therapy\",\n",
    "    #\"thornton\",\n",
    "    #\"thornton_hiv\",\n",
    "    #\"tidyr\",\n",
    "    #\"to\",\n",
    "    #\"transmitted\",\n",
    "    \"transplant\",\n",
    "    #\"transportation\",\n",
    "    #\"treatment\",\n",
    "    #\"trial\",\n",
    "    #\"two\",\n",
    "    #\"type\",\n",
    "    #\"types\",\n",
    "    #\"uci\",\n",
    "    #\"uis\",\n",
    "    #\"uk\",\n",
    "    #\"united\",\n",
    "    \"urine\",\n",
    "    #\"us\",\n",
    "    #\"us_contagious_diseases\",\n",
    "    #\"use\",\n",
    "    #\"utilization\",\n",
    "    \"vaccine\",\n",
    "    #\"visits\",\n",
    "    #\"waiting\",\n",
    "    #\"wales\",\n",
    "    \"weight\",\n",
    "    #\"weightloss\",\n",
    "    \"wheeze\",\n",
    "    #\"whickham\",\n",
    "    #\"whickham2\",\n",
    "    \"WHO \", #\"who\",\n",
    "    #\"who2\",\n",
    "    #\"wisconsin\",\n",
    "    #\"with\",\n",
    "    #\"wives\",\n",
    "    \"women\", #\"women's\",\n",
    "    #\"wong\",\n",
    "    #\"worked\",\n",
    "    #\"workers\",\n",
    "    #\"world\",\n",
    "    #\"wvs_usa_abortion\",\n",
    "    #\"years\",\n",
    "    #\"youth\",\n",
    "    #\"youthrisk\",\n",
    "    #\"youthrisk2009\",\n",
    "    #\"yrbss\",\n",
    "]\n",
    "\n",
    "print(len(tags_ls))\n",
    "tags_ls = sorted(list(set(tags_ls)))\n",
    "print(len(tags_ls))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "f176d60f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "114 datasets\n",
      "114 filtered datasets\n",
      "4 max tags per dataset\n",
      "Datasets not included:\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "labels = tdf['Dataset'].tolist()\n",
    "print(len(list(set(labels))), 'datasets')\n",
    "\n",
    "filtered = []\n",
    "cts = []\n",
    "\n",
    "for lab in labels:\n",
    "    ct = 0\n",
    "    for tag in tags_ls:\n",
    "        if tag.lower() in lab.lower():\n",
    "            filtered.append(lab)\n",
    "            ct += 1\n",
    "        \n",
    "    cts.append(ct)\n",
    "            \n",
    "print(len(list(set(filtered))), 'filtered datasets')\n",
    "print(max(cts), 'max tags per dataset')\n",
    "\n",
    "print('Datasets not included:')\n",
    "\n",
    "ls = np.setdiff1d(labels, filtered)\n",
    "print(ls)\n",
    "# yields the elements in `list_2` that are NOT in `list_1`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "c9cbe4fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"AIDS \",\"Down\",\"WHO \",\"abortion\",\"adolescent\",\"adult\",\"anxiety\",\"arthritis\",\"auditory\",\"autism\",\"behavior\",\"biopsy\",\"birth\",\"blood\",\"body\",\"breast\",\"cancer\",\"cardiovascular\",\"cell\",\"child\",\"cholera\",\"cirrhosis\",\"clinical\",\"coma\",\"consumption\",\"contagious\",\"contraceptive\",\"coronary\",\"covid\",\"deaths\",\"dengue\",\"dependency\",\"deprivation\",\"diabetes\",\"diarrhaea\",\"disease\",\"doctor\",\"donation\",\"drug\",\"eating\",\"ebola\",\"efficacy\",\"exercise\",\"feet\",\"fertility\",\"food\",\"freedom\",\"gestation\",\"headaches\",\"health\",\"healthcare\",\"hepatocellular\",\"hiv\",\"illness\",\"indomethacin\",\"infant\",\"insurance\",\"liver\",\"male\",\"malignant\",\"mammogram\",\"marrow\",\"medicaid\",\"medical\",\"medicare\",\"medpar\",\"melanoma\",\"migraine\",\"neuro\",\"nutrition\",\"obesity\",\"obstetrics\",\"organ\",\"pancreatic\",\"parents\",\"periodontal\",\"physician\",\"prenatal\",\"prostate\",\"prothrombin\",\"quarantine\",\"radiation\",\"recovery\",\"remission\",\"reporting\",\"respiratory\",\"risk\",\"serum\",\"sleep\",\"smallpox\",\"smoking\",\"society\",\"stress\",\"syndrome\",\"transplant\",\"urine\",\"vaccine\",\"weight\",\"wheeze\",\"women\",\n"
     ]
    }
   ],
   "source": [
    "tag_str = str()\n",
    "for tag in tags_ls:\n",
    "    tag_str += \"\\\"\"+tag+\"\\\"\"+','\n",
    "    \n",
    "print(tag_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "20013b63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(114, 6)\n",
      "(2, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package</th>\n",
       "      <th>item</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>No. of rows</th>\n",
       "      <th>No. of columns</th>\n",
       "      <th>docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>boot</td>\n",
       "      <td>aids</td>\n",
       "      <td>Delay in AIDS Reporting in England and Wal...</td>\n",
       "      <td>570</td>\n",
       "      <td>6</td>\n",
       "      <td>.. container::\\n\\n   ==== ===============\\n   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>MASS</td>\n",
       "      <td>Aids2</td>\n",
       "      <td>Australian AIDS Survival Data (MASS: Aids2)</td>\n",
       "      <td>2843</td>\n",
       "      <td>7</td>\n",
       "      <td>.. container::\\n\\n   ===== ===============\\n  ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   package   item                                            Dataset  \\\n",
       "10    boot   aids      Delay in AIDS Reporting in England and Wal...   \n",
       "64    MASS  Aids2        Australian AIDS Survival Data (MASS: Aids2)   \n",
       "\n",
       "    No. of rows  No. of columns  \\\n",
       "10          570               6   \n",
       "64         2843               7   \n",
       "\n",
       "                                                 docs  \n",
       "10  .. container::\\n\\n   ==== ===============\\n   ...  \n",
       "64  .. container::\\n\\n   ===== ===============\\n  ...  "
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(tdf.shape)\n",
    "\n",
    "f_df = tdf[tdf['Dataset'].str.contains('AIDS', case=False)]\n",
    "print(f_df.shape)\n",
    "f_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "f858137a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf.to_csv('statsmodels_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bd901d6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}