springer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "69dc82aa",
   "metadata": {},
   "source": [
    "# Springer paper collection\n",
    "\n",
    "### To run this notebook, you need to install the following packages:\n",
    "\n",
    "- ```pyResearchInsights```\n",
    "\n",
    "### The notebook is sectioned into the following parts:\n",
    "\n",
    "- [Metadata collection](#Metadata-collection)\n",
    "- [Fulltext collection](#Fulltext-collection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "399d6856",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metadata collection\n",
    "\n",
    "from pyResearchInsights.common_functions import pre_processing\n",
    "from pyResearchInsights.Scraper import scraper_main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e38b080c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fulltext collection\n",
    "\n",
    "import urllib.request as ur"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba238ab2",
   "metadata": {},
   "source": [
    "## Metadata collection\n",
    "\n",
    "- [Back to top](#Springer-paper-collection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51f59069",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Downloading metadata including abstracts\n",
    "# This block run with modified Scraper.py (in pyResearchInsights) to log fulltext download URLs as well\n",
    "# Modified code is found at the path: \"./Scraper-modified.py\"\n",
    "\n",
    "keywords_to_search = '(\"artificial intelligence\" OR \"machine learning\" OR \"deep learning\") AND (\"climate change\" OR \"climate action\") AND NOT (retracted AND retraction AND retract)'\n",
    "abstracts_log_name, status_logger_name = pre_processing(keywords_to_search)\n",
    "scraper_main(keywords_to_search, abstracts_log_name, status_logger_name)\n",
    "\n",
    "# Log saved"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2639a15",
   "metadata": {},
   "source": [
    "## Fulltext collection\n",
    "\n",
    "- [Back to top](#Springer-paper-collection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db335605",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading download URLs log and separating direct and redirect links\n",
    "\n",
    "download_urls_file = open(\"./log/springer-download-urls.txt\", \"r\")\n",
    "download_urls = download_urls_file.read()\n",
    "urls = download_urls.split(\"\\n\")\n",
    "download_urls_file.close()\n",
    "\n",
    "redirects = []\n",
    "directs = []\n",
    "unavailable_urls_file = open(\"./log/springer-unavailable-urls.txt\", \"w\")\n",
    "for u in urls:\n",
    "    if \"https://wayf.springernature.com\" in u:\n",
    "        unavailable_urls_file.write(u)\n",
    "        unavailable_urls_file.write(\"\\n\")\n",
    "        redirects.append(u)\n",
    "    else:\n",
    "        directs.append(u)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ad896d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Downloading available fulltexts automatically\n",
    "\n",
    "parent_dir = \"/home/gereltuya/Downloads/spbu/ai-for-climate-action/data/springer/papers/\"\n",
    "unavailable = []\n",
    "j = 1\n",
    "\n",
    "for d in directs:\n",
    "    f = parent_dir + \"a\" + str(j) + \".pdf\"\n",
    "    try:\n",
    "        ur.urlretrieve(d, f)\n",
    "        print(str(j), \"Successful -->\", f, \"\\n\")\n",
    "    except:\n",
    "        print(str(j), \"Error -->\", d, \"\\n\")\n",
    "        unavailable.append(d)\n",
    "    j += 1\n",
    "    \n",
    "# Log saved"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}