remove_duplicates.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "64c206ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Duplicate papers removal tool used to screen duplicate papers in systematic reviews.\n",
    "#The tool remove duplicates by similar DOI number and by similar  titles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2deb663e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import relevant libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e74c579",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6979596a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5977, 21)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load exported file from Rayyan (change 'articles.csv' with the name of your file)\n",
    "# Observe number of papers\n",
    "\n",
    "ref = pd.read_csv('articles.csv')\n",
    "ref.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8d695885",
   "metadata": {},
   "outputs": [],
   "source": [
    "REF =ref['doi'].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ac00121",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5d5a4389",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop \"https://doi.org/\"\" from DOI column\n",
    "\n",
    "for num in range(len(REF)):\n",
    "    if REF[num][0] == 'h':\n",
    "        REF[num] = REF[num][16:]\n",
    "\n",
    "        \n",
    "ref['doi'] = REF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "465ab81a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "df1feacd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lower case all titles\n",
    "\n",
    "ref['title'] = ref['title'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5f81a9e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01a9deea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop duplicates according to DOI column\n",
    "# Drop duplicates according to title column \n",
    "\n",
    "ref.drop_duplicates(subset=['doi'], inplace=True)\n",
    "ref.drop_duplicates(subset=['title'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5cde3b0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6ede6686",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3292, 21)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Observe number of papers after duplicate removal\n",
    "ref.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cdb3f4b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "56e911ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export the new file to CSV. \n",
    "ref.to_csv('articles_without_duplicates.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee020772",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1d89489b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Upload the file back to Rayyan (Manually)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55a7a07a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "25eea553",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Acknowledgments\n",
    "# Thanks Stelios Gavras for his help and support!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}