Reddit_Data_Collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Libraries and setting up the client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pymongo\n",
    "from pymongo import MongoClient\n",
    "import praw\n",
    "from praw.models import MoreComments\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "\n",
    "mongo_client = MongoClient()\n",
    "\n",
    "client = MongoClient('mongodb://localhost:27017/')\n",
    "db = client[\"reddit\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "reddit = praw.Reddit(client_id = \"######\",\n",
    "\t\t\t\t\tclient_secret = \"#######\",\n",
    "\t\t\t\t\tuser_agent = \"#######\",\n",
    "\t\t\t\t\tusername = \"########\",\n",
    "\t\t\t\t\tpassword = \"#########\")\n",
    "\n",
    "\n",
    "subreddit = reddit.subreddit('india')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping Reddit and Saving it to MongoDB Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # #List of all the flairs. These will be the keys in classification.\n",
    "flairs = [ \"AskIndia\", \"Non-Political\", \"[R]eddiquette\",\n",
    "          \"Scheduled\", \"Photography\", \"Science/Technology\",\n",
    "          \"Politics\", \"Business/Finance\", \"Policy/Economy\",\n",
    "          \"Sports\", \"Food\", \"AMA\"]\n",
    "\n",
    "\n",
    "for flair in flairs:\n",
    "\n",
    "    #The posts' data is collected by searching by the flair name in the list. Top 200 posts are recorded and stored for analysis.\n",
    "\n",
    "    relevant_subreddits = subreddit.search(flair,limit=200)\n",
    "\n",
    "    for submission in relevant_subreddits:\n",
    "        posts = {\n",
    "        \"title\":str(submission.title),\n",
    "        \"score\":str(submission.score),\n",
    "        \"id\":str(submission.id),\n",
    "        \"url\":str(submission.url),\n",
    "        \"comms_num\":str(submission.num_comments),\n",
    "        \"created\":str(submission.created),\n",
    "        \"body\":str(submission.selftext),\n",
    "        \"author\":str(submission.author),\n",
    "        \"flair\":str(flair),\n",
    "        }\n",
    "\n",
    "#  Only top ten comments and their authors are considered. \n",
    "        submission.comments.replace_more(limit=None)\n",
    "        comment = ''\n",
    "        authors = ''\n",
    "        count = 0\n",
    "        for top_level_comment in submission.comments:\n",
    "            comment = comment + ' ' + top_level_comment.body\n",
    "            authors = authors + ' ' + str(top_level_comment.author)\n",
    "            count+=1     \n",
    "            if(count > 10):\n",
    "                break\n",
    "\n",
    "        posts[\"comment\"] = str(comment)\n",
    "        posts[\"authors\"] = str(authors)\n",
    "        result = db.posts.insert_one(posts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Cleaning "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import utils\n",
    "import gensim.parsing.preprocessing as gsp\n",
    "\n",
    "filters = [\n",
    "           gsp.strip_tags, \n",
    "           gsp.strip_punctuation,\n",
    "           gsp.strip_multiple_whitespaces,\n",
    "           gsp.strip_numeric,\n",
    "           gsp.remove_stopwords, \n",
    "           gsp.strip_short, \n",
    "           gsp.stem_text\n",
    "          ]\n",
    "\n",
    "def clean_text(s):\n",
    "    s = s.lower()\n",
    "    s = utils.to_unicode(s)\n",
    "    for f in filters:\n",
    "        s = f(s)\n",
    "    return s\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Pre-Processing and Saving it into a CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "collection = db.posts\n",
    "posts = pd.DataFrame(list(collection.find()))\n",
    "\n",
    "del posts['_id']\n",
    "\n",
    "posts['title'] = posts['title'].apply(str)\n",
    "posts['body'] = posts['body'].apply(str)\n",
    "posts['comment'] = posts['comment'].apply(str)\n",
    "\n",
    "posts['title'] = posts['title'].apply(clean_text)\n",
    "posts['body'] = posts['body'].apply(clean_text)\n",
    "posts['comment'] = posts['comment'].apply(clean_text)\n",
    "\n",
    "combined_features = posts[\"title\"] + posts[\"comment\"] + posts[\"url\"] + posts[\"body\"]\n",
    "posts = posts.assign(combined_features = combined_features)\n",
    "\n",
    "#saving the csv file\n",
    "posts.to_csv('data.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit",
   "language": "python",
   "name": "python37764bite07536fd0e6c49f6bf9c481dff06ce24"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}