scrape_jornada_datacatalog.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import requests\n",
    "import pandas as pd\n",
    "#import urllib.request\n",
    "#import time\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape the Jornada Data catalog\n",
    "\n",
    "This ('https://jornada.nmsu.edu/data-catalogs/jornada') seems to be the most comprehensive data catalog on the Jornada websites. It had 172 total packages at last count. Some are duplicates or test packages, so I think this explains why it is different than the LTER data catalog (https://jornada.nmsu.edu/lter/data), which had only 152 at last count."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://jornada.nmsu.edu/data-catalogs/jornada'\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "172"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets = soup.findAll('div', attrs={'class':'views-row'})\n",
    "len(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "ids = []\n",
    "titles = []\n",
    "creators = []\n",
    "files = []\n",
    "packages = []\n",
    "\n",
    "for i, ds in enumerate(datasets):\n",
    "    #dataset IDs\n",
    "    id_div = ds.find('div', attrs={'class':'views-field views-field-field-data-set-id'})\n",
    "    ids.append(id_div.find('div', attrs={'class':'field-content'}).string[12:])\n",
    "    # titles\n",
    "    title_div = ds.find('div', attrs={'class':'views-field views-field-title'})\n",
    "    titles.append(title_div.find('h2').string)\n",
    "    # links\n",
    "    links.append('https://jornada.nmsu.edu' + title_div.find('a')['href'])\n",
    "    # creators\n",
    "    creator_div = ds.find('div', attrs={'class':'views-field views-field-field-person-creator'})\n",
    "    try:\n",
    "        creators.append(creator_div.find('div', attrs={'class':'field-content'}).string[23:])\n",
    "    except: \n",
    "        creators.append('NA')\n",
    "    # data files\n",
    "    file_div = ds.find('div', attrs={'class':'views-field views-field-field-data-source-file'})\n",
    "    try:\n",
    "        files.append(file_div.find('div', attrs={'class':'field-content'}).find('a')['href'])\n",
    "    except:\n",
    "        files.append('NA')\n",
    "    # data package\n",
    "    package_div = ds.find('div', attrs={'class':'views-field views-field-field-related-links'})\n",
    "    try:\n",
    "        packages.append(package_div.find('div', attrs={'class':'field-content'}).find('a')['href'])\n",
    "    except:\n",
    "        packages.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write the table out using pandas\n",
    "\n",
    "jornada_out = pd.DataFrame(\n",
    "    {'link': links,\n",
    "     'ID': ids,\n",
    "     'title': titles,\n",
    "     'creator': creators,\n",
    "     'files': files,\n",
    "     'packages': packages\n",
    "    })\n",
    "\n",
    "jornada_out.to_csv('/Volumes/DataProducts/LTER_IM/Website_data_rescue/Jornada_all_catalog.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check that the website is the same as the archived copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "handle = open('/Users/gmaurer/Desktop/Jornada Data | Jornada.html')\n",
    "soup2 = BeautifulSoup(handle, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "172\n"
     ]
    }
   ],
   "source": [
    "titles2 = soup2.findAll('div', attrs={'class':'views-field views-field-title'})\n",
    "print(len(titles2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape the Jornada \"Long-Term\" data catalog\n",
    "\n",
    "https://jornada.nmsu.edu/data-catalogs/long-term - At last count (9-April-2020) this has 65 datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://jornada.nmsu.edu/data-catalogs/long-term'\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(response.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = soup.findAll('tr')\n",
    "len(rows)\n",
    "# 7 of these are header rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "titles = []\n",
    "resp_pi = []\n",
    "packages = []\n",
    "\n",
    "for i, r in enumerate(rows):\n",
    "    # titles & link\n",
    "    title_div = r.find('td', attrs={'class':'views-field views-field-title views-align-left views-table-width-30'})\n",
    "    try:\n",
    "        titles.append(title_div.find('a').string)\n",
    "        links.append('https://jornada.nmsu.edu' + title_div.find('a')['href'])\n",
    "    except:\n",
    "        titles.append('NA')\n",
    "        links.append('NA')\n",
    "    # creators\n",
    "    resp_div = r.find('td', attrs={'class': 'views-field views-field-field-name views-table-width-15'})\n",
    "    try:\n",
    "        resp_pi.append(resp_div.find('a').string)\n",
    "    except: \n",
    "        resp_pi.append('NA')\n",
    "    # data package\n",
    "    package_div = r.find('td', attrs={'class':'views-field views-field-field-related-links views-table-width-10'})\n",
    "    try:\n",
    "        packages.append(package_div.find('a')['href'])\n",
    "    except:\n",
    "        packages.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write the table out using pandas\n",
    "\n",
    "jornada_out = pd.DataFrame(\n",
    "    {'link': links,\n",
    "     'title': titles,\n",
    "     'responsible_pi': resp_pi,\n",
    "     'packages': packages\n",
    "    })\n",
    "\n",
    "jornada_out.to_csv('/Volumes/DataProducts/LTER_IM/Website_data_rescue/Jornada_long-term_catalog.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape the Jornada \"Climate\" data catalog\n",
    "\n",
    "https://jornada.nmsu.edu/data-catalogs/climate - At last count (9-April-2020) this has 18 datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://jornada.nmsu.edu/data-catalogs/climate'\n",
    "response = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#soup = BeautifulSoup(response.text, 'html.parser')\n",
    "soup=BeautifulSoup(response.text, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_body=soup.find('tbody')\n",
    "rows = table_body.find_all('tr') # This step excludes the header rows\n",
    "#rows = soup.findAll('tr')\n",
    "len(rows)\n",
    "# Missing 3 rows - no idea why!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "titles = []\n",
    "resp_pi = []\n",
    "packages = []\n",
    "\n",
    "for i, r in enumerate(rows):\n",
    "    # titles & link\n",
    "    title_div = r.find('td', attrs={'class':'views-field views-field-title'})\n",
    "    try:\n",
    "        titles.append(title_div.find('a').string)\n",
    "        links.append('https://jornada.nmsu.edu' + title_div.find('a')['href'])\n",
    "    except:\n",
    "        titles.append('NA')\n",
    "        links.append('NA')\n",
    "    # creators\n",
    "    resp_div = r.find('td', attrs={'class': 'views-field views-field-field-name'})\n",
    "    try:\n",
    "        resp_pi.append(resp_div.find('a').string)\n",
    "    except: \n",
    "        resp_pi.append('NA')\n",
    "    # data package\n",
    "    package_div = r.find('td', attrs={'class':'views-field views-field-field-related-links'})\n",
    "    try:\n",
    "        packages.append(package_div.find('a')['href'])\n",
    "    except:\n",
    "        packages.append('NA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_121_biodiversity_precipitation_tipping_bucket_raingauge_event_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_121_biodiversity_precipitation_tipping_bucket_raingauge_daily_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_169_npp_atmospheric_deposition_adc_dust_collection_dryfall_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/Jornada_425001_npp_estimated_daily_precipitation_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/Jornada_127_evaporation_pan_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_126_lter_weather_station_climate_daily_data.zip',\n",
       " 'https://jornada.nmsu.edu/files/dataset_packages/Jornada_126002_lter_weather_station_hourly_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_002_npp_precipitation_graduated_raingauge_monthly_data.zip',\n",
       " 'NA',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/Jornada_338002_tromble_weir_precipitation_data.zip',\n",
       " 'https://jornada.nmsu.edu/files/dataset_packages/Jornada_407_jer_recording_raingauge_network_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_379_noaa_weather_station_climate_daily_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_379_noaa_weather_station_climate_monthly_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_379_noaa_weather_station_evaporation_pan_monthly_data.zip',\n",
       " 'http://jornada.nmsu.edu/files/dataset_packages/JornadaStudy_169_npp_atmospheric_deposition_adc_dust_collection_wetfall_data.zip']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write the table out using pandas\n",
    "\n",
    "jornada_out = pd.DataFrame(\n",
    "    {'link': links,\n",
    "     'title': titles,\n",
    "     'responsible_pi': resp_pi,\n",
    "     'packages': packages\n",
    "    })\n",
    "\n",
    "jornada_out.to_csv('/Volumes/DataProducts/LTER_IM/Website_data_rescue/Jornada_climate_catalog.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}