2-preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YIZsKvOBNv8K",
    "outputId": "7c7f9cdb-3215-4a02-a952-e4979781c7c8"
   },
   "outputs": [],
   "source": [
    "!pip install openpyxl\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# for posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = pd.read_csv(r'C:\\users\\sema\\downloads\\science_09.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "2170\n"
     ]
    }
   ],
   "source": [
    "empty_posts_index = []\n",
    "for i in range(len(posts)):\n",
    "    if posts.iloc[i]['title'] == '[deleted by user]':\n",
    "        empty_posts_index.append(i)\n",
    "\n",
    "print(len(empty_posts_index))\n",
    "\n",
    "new_total_posts = posts.drop(index=empty_posts_index)\n",
    "new_total_posts\n",
    "\n",
    "print(len(new_total_posts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>x2sskr</td>\n",
       "      <td>Utilizing lemon peel extract and its nano-emul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>x2t86y</td>\n",
       "      <td>Cureus | Regular Use of Ivermectin as Prophyla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>x2vimp</td>\n",
       "      <td>What was before the big bang?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>x2wuyj</td>\n",
       "      <td>Your blood type could predict your risk of hav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>x2wv4z</td>\n",
       "      <td>Genomes from a medieval mass burial show Ashke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2165</th>\n",
       "      <td>xsf5jp</td>\n",
       "      <td>A majority of Twitter users do not follow eith...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2166</th>\n",
       "      <td>xsf7w6</td>\n",
       "      <td>Traumatic brain injury ‘remains a major global...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2167</th>\n",
       "      <td>xsf9lh</td>\n",
       "      <td>Our cities are warming and urban greenery coul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2168</th>\n",
       "      <td>xsfcsk</td>\n",
       "      <td>Concussions are associated with 60% increase i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2169</th>\n",
       "      <td>xsflwa</td>\n",
       "      <td>“Our results state, resoundingly, that the mos...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2170 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                              title\n",
       "0     x2sskr  Utilizing lemon peel extract and its nano-emul...\n",
       "1     x2t86y  Cureus | Regular Use of Ivermectin as Prophyla...\n",
       "2     x2vimp                      What was before the big bang?\n",
       "3     x2wuyj  Your blood type could predict your risk of hav...\n",
       "4     x2wv4z  Genomes from a medieval mass burial show Ashke...\n",
       "...      ...                                                ...\n",
       "2165  xsf5jp  A majority of Twitter users do not follow eith...\n",
       "2166  xsf7w6  Traumatic brain injury ‘remains a major global...\n",
       "2167  xsf9lh  Our cities are warming and urban greenery coul...\n",
       "2168  xsfcsk  Concussions are associated with 60% increase i...\n",
       "2169  xsflwa  “Our results state, resoundingly, that the mos...\n",
       "\n",
       "[2170 rows x 2 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posts = new_total_posts[['id', 'title']]\n",
    "posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>x2sskr</td>\n",
       "      <td>Utilizing lemon peel extract and its nano-emul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>x2t86y</td>\n",
       "      <td>Cureus | Regular Use of Ivermectin as Prophyla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>x2vimp</td>\n",
       "      <td>What was before the big bang?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>x2wuyj</td>\n",
       "      <td>Your blood type could predict your risk of hav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>x2wv4z</td>\n",
       "      <td>Genomes from a medieval mass burial show Ashke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2165</th>\n",
       "      <td>xsf5jp</td>\n",
       "      <td>A majority of Twitter users do not follow eith...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2166</th>\n",
       "      <td>xsf7w6</td>\n",
       "      <td>Traumatic brain injury ‘remains a major global...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2167</th>\n",
       "      <td>xsf9lh</td>\n",
       "      <td>Our cities are warming and urban greenery coul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2168</th>\n",
       "      <td>xsfcsk</td>\n",
       "      <td>Concussions are associated with 60% increase i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2169</th>\n",
       "      <td>xsflwa</td>\n",
       "      <td>“Our results state, resoundingly, that the mos...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2170 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     link_id                                              title\n",
       "0     x2sskr  Utilizing lemon peel extract and its nano-emul...\n",
       "1     x2t86y  Cureus | Regular Use of Ivermectin as Prophyla...\n",
       "2     x2vimp                      What was before the big bang?\n",
       "3     x2wuyj  Your blood type could predict your risk of hav...\n",
       "4     x2wv4z  Genomes from a medieval mass burial show Ashke...\n",
       "...      ...                                                ...\n",
       "2165  xsf5jp  A majority of Twitter users do not follow eith...\n",
       "2166  xsf7w6  Traumatic brain injury ‘remains a major global...\n",
       "2167  xsf9lh  Our cities are warming and urban greenery coul...\n",
       "2168  xsfcsk  Concussions are associated with 60% increase i...\n",
       "2169  xsflwa  “Our results state, resoundingly, that the mos...\n",
       "\n",
       "[2170 rows x 2 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posts.columns = ['link_id', 'title']\n",
    "posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts['link_id'] = 't3_' + posts['link_id'].astype(str)\n",
    "posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts.to_csv(\"linkid_title_09.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_01.csv')\n",
    "df2 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_02.csv')\n",
    "df3 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_03.csv')\n",
    "df4 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_04.csv')\n",
    "df5 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_05.csv')\n",
    "df6 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_06.csv')\n",
    "df7 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_07.csv')\n",
    "df8 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_08.csv')\n",
    "df9 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_title_09.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [df1, df2, df3, df4, df5, df6, df7, df8, df9]\n",
    "result = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21400"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv('linkid_title_total.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# for comments "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zj8q8hxNOEbd"
   },
   "outputs": [],
   "source": [
    "comments = pd.read_csv(r'C:\\users\\sema\\desktop\\comments\\comments_09.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8AX4RklsS5HF",
    "outputId": "56dfcee6-7c03-404a-b872-e34920d943e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95059\n",
      "82988\n"
     ]
    }
   ],
   "source": [
    "empty_comment_index = []\n",
    "for i in range(len(comments)):\n",
    "    if comments.iloc[i]['body'] == '[removed]' or comments.iloc[i]['body'] == '[deleted]' or comments.iloc[i]['body'].startswith(\"Welcome to r/science!\") or comments.iloc[i]['body'].startswith(\"Your post has been removed\"):\n",
    "        empty_comment_index.append(i)\n",
    "\n",
    "print(len(empty_comment_index))\n",
    "\n",
    "new_total_comment = comments.drop(index=empty_comment_index)\n",
    "new_total_comment\n",
    "\n",
    "print(len(new_total_comment))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "id": "m3R5FO3Elgdo"
   },
   "outputs": [],
   "source": [
    "comments = new_total_comment[['link_id', 'body']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "id": "EMJQGmFplgbH"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link_id</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t3_x2efzp</td>\n",
       "      <td>Keep in mind that you only get up to $100k/yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>t3_x2llzf</td>\n",
       "      <td>Yes but it’s almost like telling someone who i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>t3_x2llzf</td>\n",
       "      <td>True, and overweight people being told their w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>t3_x2llzf</td>\n",
       "      <td>Let's not forget that actual intubation is muc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>t3_x2llzf</td>\n",
       "      <td>&amp;gt; If you had a condition that wasn’t relate...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      link_id                                               body\n",
       "2   t3_x2efzp  Keep in mind that you only get up to $100k/yea...\n",
       "5   t3_x2llzf  Yes but it’s almost like telling someone who i...\n",
       "7   t3_x2llzf  True, and overweight people being told their w...\n",
       "8   t3_x2llzf  Let's not forget that actual intubation is muc...\n",
       "10  t3_x2llzf  &gt; If you had a condition that wasn’t relate..."
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments['body'] = comments.groupby(['link_id'])['body'].transform(lambda x : ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "comments = comments.drop_duplicates()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link_id</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t3_x2efzp</td>\n",
       "      <td>Keep in mind that you only get up to $100k/yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>t3_x2llzf</td>\n",
       "      <td>Yes but it’s almost like telling someone who i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>t3_x2hbo8</td>\n",
       "      <td>Please tell me more I think they're talking ab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>t3_x1j5d4</td>\n",
       "      <td>Yep, I can agree with all of this. nah i feel ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>t3_x2ry0b</td>\n",
       "      <td>This discovery could also advance our understa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176925</th>\n",
       "      <td>t3_xsbpvz</td>\n",
       "      <td>Intellectual disability for folks who don’t wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177056</th>\n",
       "      <td>t3_xscog2</td>\n",
       "      <td>Abstract\\r  \\nIf nipple erection signals/is a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177703</th>\n",
       "      <td>t3_xsf7w6</td>\n",
       "      <td>This is why it should be mandatory to wear a h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177721</th>\n",
       "      <td>t3_xsf9lh</td>\n",
       "      <td>Would be super cool if we used plants that pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177740</th>\n",
       "      <td>t3_xsflwa</td>\n",
       "      <td>Papers on pre-print services such as papers.ss...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1484 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          link_id                                               body\n",
       "2       t3_x2efzp  Keep in mind that you only get up to $100k/yea...\n",
       "5       t3_x2llzf  Yes but it’s almost like telling someone who i...\n",
       "14      t3_x2hbo8  Please tell me more I think they're talking ab...\n",
       "20      t3_x1j5d4  Yep, I can agree with all of this. nah i feel ...\n",
       "21      t3_x2ry0b  This discovery could also advance our understa...\n",
       "...           ...                                                ...\n",
       "176925  t3_xsbpvz  Intellectual disability for folks who don’t wa...\n",
       "177056  t3_xscog2  Abstract\\r  \\nIf nipple erection signals/is a ...\n",
       "177703  t3_xsf7w6  This is why it should be mandatory to wear a h...\n",
       "177721  t3_xsf9lh  Would be super cool if we used plants that pro...\n",
       "177740  t3_xsflwa  Papers on pre-print services such as papers.ss...\n",
       "\n",
       "[1484 rows x 2 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "id": "KBfv7dsDlgYo"
   },
   "outputs": [],
   "source": [
    "comments.to_csv(\"linkid_body_09.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "df1 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_01.csv')\n",
    "df2 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_02.csv')\n",
    "df3 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_03.csv')\n",
    "df4 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_04.csv')\n",
    "df5 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_05.csv')\n",
    "df6 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_06.csv')\n",
    "df7 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_07.csv')\n",
    "df8 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_08.csv')\n",
    "df9 = pd.read_csv(r'C:\\users\\sema\\downloads\\linkid_body_09.csv')\n",
    "\n",
    "print(df1['body'].isna().sum())\n",
    "print(df2['body'].isna().sum())\n",
    "print(df3['body'].isna().sum())\n",
    "print(df4['body'].isna().sum())\n",
    "print(df5['body'].isna().sum())\n",
    "print(df6['body'].isna().sum())\n",
    "print(df7['body'].isna().sum())\n",
    "print(df8['body'].isna().sum())\n",
    "print(df9['body'].isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = [df1, df2, df3, df4, df5, df6, df7, df8, df9]\n",
    "result = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15826"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "result['body'] = result.groupby(['link_id'])['body'].transform(lambda x : ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.drop_duplicates()  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14240"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv('linkid_body_total.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# for merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = pd.read_csv(r'D:\\linkid_title_total.csv')\n",
    "comments = pd.read_csv(r'D:\\linkid_body_total.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21400\n",
      "14240\n"
     ]
    }
   ],
   "source": [
    "print(len(posts))\n",
    "print(len(comments))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'It\\'s the reason for thinking it might not work. And testing confirmed it. Study design and data would be examined by outside parties. Any bias would be pretty quickly uncovered and rejected. As a Uk citizen, in a country where we have socialised healthcare, I WISH something as easy as ivermectin worked. It would save the NHS a fortune and my kids and my kids,kids wouldn’t have to repay the financial destruction the disease has caused. \\n\\nI also wish it worked for USA (and other non socialised countries) so that people wouldn’t get horrendous unmanageable debt from a single hospital visit. \\n\\nThe above reasons are why the “conspiracy theory” argument never holds water when you step outside the USA. What was the fourth drug? You only mentioned three. There’s a few. The researchers used a programme called “AutoDock” to see the interactions between select drugs and SARS-CoV-2.  \\n\\nhttps://pubmed.ncbi.nlm.nih.gov/32871846/\\n\\nhttps://www.frontiersin.org/articles/10.3389/fmicb.2020.592908/full#B22 Another antiviral called Favipiravir. Just because they no longer hold exclusive patent rights does not preclude future sales. They are experts at mass producing it and have well-established, reputable brands associated with the drug. Those articles are dated after people were pushing hydroxychloroquine. Whatever happened with all the initial studies showing benefit from the active ingredient in Pepcid AC? In 2020 and early 2021 I was seeing multiple studies published showing benefit against serious illness and death, and then it dropped off the radar. The revised paper was reviewed by a different group of peers. 33 Billion here. 33 Billion there. Eventually we will talking about “real” money at some point right?  Not profitable?  Quite possibly the dumbest thing I’ve read in here all day. Cause for skepticism ≠ \"It makes no sense for Ivermectin to be used to combat COVID-19. Ivermectin is for parasites and COVID is a virus\" Anything might not work. Do you know the difference between a drug that treats viruses and a drug that treats parasites? Our local farm supply store stopped selling ivermectin. The wording is very technical, but all that the retraction stated was that they reviewed the study, and determined that the methods used to test ivermectin\\'s effectiveness in the human body were not thorough enough to determine it\\'s effectiveness. In other words the results were deemed inconclusive based on the steps taken by researchers.\\n\\nThe redaction was based on a review of a study that was already completed. If you asked a researcher if the redaction meant that ivermectin had no beneficial impact inside a human body against covid-19, they would not be able to tell you yes or no.\\n\\nAt this point the answer doesn\\'t seem to matter, and there\\'s no benefit to researching it any more unless it were helpful in a future pandemic. &gt;the long and short of it was that there was evidence of effectiveness in some countries. the countries it worked to help save were those in which parasites were also prevalent.\\n\\nThere are basically no high-quality randomized controlled trials supporting this claim. Conclusion from [a \"gold standard\" Cochrane Review](https://doi.org/10.1002/14651858.CD015017.pub2):\\n\\n&gt;Based on the current very low‐ to low‐certainty evidence, we are uncertain about the efficacy and safety of ivermectin used to treat or prevent COVID‐19. The completed studies are small and few are considered high quality. Several studies are underway that may produce clearer answers in review updates. Overall, the reliable evidence available does not support the use of ivermectin for treatment or prevention of COVID‐19 outside of well‐designed randomized trial They\\'re not, and there\\'s no need for that to be true. That\\'s still entirely specious reasoning.  There is no reason whatsoever to believe a priori that a particular anthelminthic would *or wouldn\\'t* be useful against a particular virus, or diabetes, or cancer, or any other condition unrelated to worms.\\n\\nThe one and only **scientific** reason to think it wouldn\\'t work is *exactly* because clinical testing demonstrates that it doesn\\'t. Research has strongly indicated that Ivermectin does not work for sars-cov-2.  Even if it did, Merck would have little incentive to promote it, compared to promoting molnupiravir. The cost of Ivermectin is less than 3 cents per dose, according to page 20 of the 2018 WHO report below.  This is compared to a reported cost of over $700 per molnupiravir dose.\\n\\nhttps://www.who.int/selection_medicines/committees/expert/22/applications/s6.6_ivermectin.pdf\\n\\nhttps://theintercept.com/2021/10/05/covid-pill-drug-pricing-merck-ridgeback/ Thank you! These are not the ones I thought I read. I’m Not sure where you argument about the vax is coming from. Nor where the ad hominem is coming from. \\n\\nMy point is that with socialised healthcare, our NHS wants and needs the cheapest possible treatment for diseases. It therefore also wants the best possible prevention of said diseases as it’s cheaper than treating the disease. \\n\\nIf ivermectin worked, countries with socialised healthcare would jump on it immediately. It would save a fortune for the NHS, free up beds for other patients, allow staff to be relocated back to their original areas, allow other patients having to wait for treatments for other diseases to actually get seen etc. \\n\\nThe “big pharma” conspiracy holds no water when socialised healthcare is in use. I believe a new indication and dosage can be non-generic. FDA did that with colchicine. The lack of financial incentive argument doesn\\'t hold.  Not true. Steroid creams inhibit the immune system and make Rosacea worse. Topical Ivermectin doesn\\'t help Rosacea that lacks excess mites. Yes, you are correct that the reliable evidence says you shouldn\\'t use it for covid. The studies surrounding ivermectin have been almost unanimously terrible with fraud, methodological errors, phacking, and/or spurious conclusions. \\n\\nThe meta analysis that was compellingish to me w.r.t. parasites and the early somewhat promising results: \\n\\nhttps://astralcodexten.substack.com/p/ivermectin-much-more-than-you-wanted Look, bleach isn\\'t meant to cure cancer but if you drink enough of it, the cancer dies. \\n\\nThe intent of a drug isn\\'t important, the effect is. And yes, both still kill the patient in high enough dosages. The most commonly theorized cause of rosacea is [an overactive immune system](https://www.mayoclinic.org/diseases-conditions/rosacea/symptoms-causes/syc-20353815), so something to lower the immune response would likely be helpful. But did it not show a protease inhibitor effect? WAIT WAIT. So I understand what this means and someone help?\\n\\n1.) the original paper reviewed a number of clinical trials where Ivermectin was used to treat covid patients. Based on the result of those clinical trials , the review paper concluded that Ivermectin was effective against covid. \\n\\nIs this correct?\\n\\n\\n2.) The original paper was retracted by the journal that published it because.... the clinical trials the review was based upon... were of low quality, should not be relied upon. \\n\\nIs this correct?\\n\\n\\n3.) So has there been another paper reviewing all the clinical trials now that conclusively shows Ivermectin is not effective against covid?\\n\\nI\\'m just wondering because this is only a retraction. It doesn\\'t mean Ivermectin is either effective or ineffective. We would need a number of robust  large clinical trials showing that Ivermectin is ineffective against Covid. \\n\\nIs this correct? \\n\\nIf so does anyone have the link to those clinical trial studies or an article? Yes. But we have a reason to prioritize testing of certain things over another. Like we don\\'t test putting crystals in people\\'s pockets vs placebo when it comes to COVID. We can guess that it wouldn\\'t work, but we don\\'t know until we test it. But we don\\'t want to test it because we have better things to look at. That\\'s fair. I\\'m speaking from frustration of people who clung to Ivermectin that doesn\\'t understand the nuance of the scientific process. \"But doctors tried it and saw it with their own eyes\" even though that makes no sense and isn\\'t in line with what any studies said. Well that’s good news. I’ve seen horror stories of hospital costs in USA. \\n\\nSo if the government pays for uninsured covid treatment, then the government and pharma companies priorities are at odds with one another ?? In the conspiracy world, Pharma companies want sick patients but the government doesn’t??!! So the conspiracy falls over again? \\nI don’t understand how this “money trail” apparently works for the conspiracy??!! &gt;We would need a large number of robust large clinical trials showing that Ivermectin is ineffective against Covid.\\n\\nNo, there would need to be robust clinical trials to show that it was *effective*\\n\\nThere aren’t, and it isn’t. Agree completely!  And I want to apologize, I didn\\'t mean that to sound quite as harsh as I now realize it does on re-reading.\\n\\nThere are plenty of reasons we might form a hypothesis that X does Y - But those are still just hypotheses.  In the case of everyone\\'s favorite dewormer, though, computer models actually did suggest it might be worth taking a look.\\n \\nIf everything about Covid hadn\\'t become a political minefield, if we were talking about, say, warts or IBS - Ivermectin, Chloroquine, and a whole host of other \"crazy\" ideas would be systematically considered, maybe run through a few tightly-controlled preliminary trials to see if there was any reason to dig deeper.  The garbage would get rightly tossed, and nobody would be protesting outside the NIH over it; the promising candidates would be studied further and maybe eventually receive approval (or maybe still get tossed, but without anyone cuffed and dragged out of infusion clinics for demanding they be put on a Lysol drip).\\n\\nI think we all have a new appreciation for the old curse, \"May you live in interesting times\". Yes, exactly. Don\\'t worry about it and I absolutely agree. We are discovering and looking into new treatments that we would think was crazy in the past. Like fecal transplants. And we don\\'t know for sure until we try. But a lot of frustration is when we have people who believe in science and understanding when something probably won\\'t work but still has the possibility talking to people who say it works great despite there being no strong evidence. So from the outside you have one person saying, probably doesn\\'t do anything but maybe it does arguing with someone who is saying that it definitely does something because real doctors are using it and say it works. And someone not familiar with the scientific process looking at this would think the person saying that it works is right because they sound confident and there must be a political or money reason why the media is so against it. So my point is, if we don\\'t have the evidence and we don\\'t have any logic that it stops a virus (other than a computer model), then we have to say that it should not be used for treatment until evidence says otherwise. Doesn\\'t mean it doesn\\'t or can\\'t do anything, just that it likely doesn\\'t and if it does then the effect is likely smaller than other viable treatments. This is actually true, but it\\'s the way the system has been set up. Big money is on surgeries which were way down.\\n\\nThat being said, it\\'s abused by administration because they need money.\\n\\nIt\\'s a very flawed system, but not a conspiracy. What\\'s funny is I think if we started giving Ivermectin they would pivot so hard. It\\'s not about the treatment, its about what \\'they\\' are keeping from \\'us\\'. If we started giving it they would either find a new drug, or say its not real ivermectin The results of the original paper were deemed unreliable and hence retracted. As far as I am aware this is no other evidence to suggest that Ivermectin would be an effective method to fight COVID. \\n\\nIf there is no evidence for a scientific theory it should be presumed false. Flu shots are definitely not profitable. Yea, but among all the drugs studied, ivermectin was believed to need a homodimeric form of the particular enzyme for its inhibitor effect to work - meaning it would need to be coupled with other drugs to even do anything. \\n\\nI think the last thing we want to do is experiment with modifying Covid within the body and then hoping another drug would neutralize it. \\n\\nThere were other drugs: boceprevir, ombitasvir, paritaprevir, tipranavir, and micafungin that inhibited the same enzyme too, you don’t see everyone running around trying to get hepatitis C and HIV drugs for covid… but they sure tout ivermectin as a wonder cure that “they” don’t want you to have. And causes explosive diarrhea The only way some people will believe is if you prove the negative. And even then... But what about the downsides? Why does this sound like something made out of chickpeas? There was a pharmacology paper published alongside a similarly premised paper on ivermectin earlier in the year that pointed out that the relatively mild anti-viral action in-vitro of ivermectin coupled with the way the in-vivo systemic load and elimination happens means that in-vivo anti-viral effects are unlikely below a toxic dosage. Wow... so that hasn\\'t been true for a long time. Historically absolutely true.  Yet this one is massively different as the US paid for development costs and there is zero marketing needed.  \\n\\nMassive profits to covid vaccine makers.  That is not debatable. Actually, pharmacies refused to fill the doctors prescriptions. That’s why people turned to the horse supply. Looks like nope.\\n\\nhttps://www.ncbi.nlm.nih.gov/pmc/articles/PMC7903022/ So, it didnt pan out in computer simulations or did they actually run clinical trails? That’s wrong. A vaccine is free, sometimes even testing. What isn’t covered is hospital care bills, funeral expenses, missed work pay, or post infection disability when your body is fucked up cuz you didn’t get the vaccine and now can’t walk across the room without feeling like you’re gonna die. That comment was wrong. Only the vaccine and sometimes testing are covered. All medical care for Covid treatments or death costs, missed work, or care are not covered Like everything else FEMA is offering $10k for funeral expenses \\n\\nhttps://www.fema.gov/disaster/coronavirus/economic/funeral-assistance Not sure what you are referring to If the numbers were inflated to crisis proportions, everyday work will be hampered and the economy tanks. Without a thriving economy, people lose money and government earns a lot less in taxes. People suffering economic hardship and a government earning little in revenue is not a good sign for reelection. That conspiracy only works for idiots Unfortunately, my anti vax parents didn\\'t get tested before my mom passed away, so the death certificate states natural causes. The conspiracy is that the COVID numbers are inflated, but the truth is, unless there\\'s a positive test, the death isn\\'t listed as COVID.\\n\\nMeant that or family got $0 toward her funeral and my dad has to bear the full cost himself (I\\'m pretty pissed that they didn\\'t have AD&amp;D coverage). You may want to tell the government they can stop paying for treatments then.\\n\\nhttps://www.hrsa.gov/CovidUninsuredClaim\\n\\nNo, the program doesn’t help the folks who elect to not get vaccinated with all the fallout all over their lives if they get sick. But it does insulate the healthcare provider. Somewhat. The program pays out at Medicare rates. Those aren’t typically all that great and while better than having to eat the costs (because contrary to political fiction emergent care can’t be withheld due to ability to pay) the hospitals would rather have privately insured ‘elective’ procedures like ACL repairs happening. I’m vegan so I usually substitute a little favipiravir when the recipe calls for eggs. That’s not good reasoning. Many drugs have potential unintended interactions with various receptors. These need to be fully explored if there is a likelihood for effect. \\n\\nOf course with various tools like high throughput screen or in vitro/ silico modeling we can identify drugs with a higher likelihood of working on our targets of interest don\\'t get me wrong, I\\'m not advocating for ivermectin, I just think it\\'s counter productive to totally dismiss it. that actually feeds the conspiracy theories, in my opinion. it has a known mechanism of action that can work and some small studies show positive results. that does not mean it\\'s good enough to prescribe, but trying to dismiss it totally just feeds the people who can look at the studies that point out the protease inhabitation effect and that show positive results. talking about \"it can\\'t work on viruses because it it\\'s an anti-parasitic\" removes the nuance from the discussion and makes you easy to dismiss because you\\'re saying something that does not make sense (that a medication couldn\\'t possibly have two functions). \\n\\nthanks for the explanation, though. Ivermectin was only successful in vitro testing (Cells is a Petri dish) where Ivermectin did indeed kill Covid cells but you needed 100x more then what is safe for humans.\\n\\nThere has been extensive testing done in multiple countries, Japan/India/US have all done studies based solely on Ivermectin and the results have all come back inconclusive. There’s been no difference between those that took Ivermectin and those that didn’t in all these studies. There are tens of thousands of drugs in existence. Odds are good that the vast majority don\\'t help against COVID. We\\'re not going to test anti seizure meds and diet drugs to rule them out unless there\\'s a reason to pursue it. One typically works on viruses and one typically works on parasites. If someone has a weird infection, it\\'s much more sensible to say \"have you tried Ampicillin?\" than \"have you tried guzzling superglue?\". The article was re-published on Dec 21st and only title and conclusions were changed. Apparently no changes were asked for the remaining parts. Imho retracting for just title and conclusions is plain grotesque.\\n\\n\\nSee by yourself\\n\\nRetracted title: The mechanisms of action of Ivermectin against SARS-CoV-2: An evidence-based clinical review article\\n\\nRetracted conclusions: \"Considering the urgency of the ongoing COVID-19 pandemic, simultaneous detection of various new mutant strains and future potential re-emergence of novel coronaviruses, repurposing of approved drugs such as Ivermectin could be worthy of attention.\"\\n\\n\\n\\nNew article is in https://www.nature.com/articles/s41429-021-00491-6\\n\\nNew title: The mechanisms of action of ivermectin against SARS-CoV-2—an extensive review\\n\\nNew conclusions: \"We have summarized published results on the inhibition of multiple viral and host targets that could be involved in SARS-CoV-2 replication and the disease COVID-19. Although multiple antiviral and host target activities have been reported for ivermectin in SARS-CoV-2 and COVID-19, it is still unclear if any of these activities will play a role in the prevention and treatment of the disease. The controlled clinical trials that are underway will reveal if these activities will translate into clinical efficacy.\" You do realize my comment is entirely about making sure he doesn\\'t draw false equivalencies, right? Even if ivermectin worked, it would still cost a fortune here somehow, American health care is totally awesome. So, forgive the snark, but how does it make sense to peer review a paper and have it published, then retract it with another set of peers? What was the purpose of the first peer review? How does this help the average person\\'s faith in peer reviewed studies? So basically, you have no idea. I don\\'t think ivermectin works, but it\\'s not because it\\'s a \"parasite drug\" There are typically only 2 or 3 reviewers for a given paper. While most editors do their best to vet reviewers to be experts in the field it is possible for a biased, inappropriate or inexperienced reviewer to get selected. A new round of review usually means VERY carefully vetted new reviewers. (source - not in a medical field but am a reasonably well published scientist and have reviewed extensively) If tomatos could miraculously cure covid, u think all tomato producers wouldn\\'t start raising prices? The only way to control prices in this scenario is for government action. If the horse medicine works, its price will rise. The vaccine in the US is quite literally free for any and everyone. It is the exact opposite of monetized. Explosions are fun! You’ll probably survive them. Probably. If it would cost a fortune, that is how you know it actually worked. In the US at least Is that why liquid naproxen is so cheap in the US?\\nOr colchicine?\\nYour faith in the market is misplaced, but worse is your confidence that physicians and scientists would downplay something that works because they value money more than science and medicine. First reviewers messed up. Editor smelled a rat. Second ones found problems. Authors wouldn\\'t retract. Editor pulled it. Revised paper with limited conclusions stays up.\\nAs far as to whether one should believe in the process or not, if this is an area reader understands, that\\'s up to reader. If this is not an area reader  understands, the system eventually worked. \\nPaper shouldn\\'t have been published in the first place.\\nMaybe your question was rhetorical, I don\\'t know. How about insulin? Except that vaccines are notoriously less profitable than regular medicines and treatments. So your point doesn’t make any sense. In an “ordinary” journal _published_ means that there are 2-3 reviewers who do not vehemently disagree with the contents of the text. Is it does not mean that the results are verified or replicated. Think of published as “not obviously false” rather than “the truth”. One published paper on its own does not mean anything in a scientific context, the problem here was that non-scientists interpreted science, or rather a published paper, as truth and the editor in chief reacted by retracting the paper.\\n\\nGoogle “replication crisis” if you are interested some of the meta-issues of science. I’m coming in with a pedantic point, but it is important: government spending is not financed by taxes. Rather, the money is spent into existence. It’s how true money is created. \\nNot saying patents aren’t a serious problem. It did. See how much Americans Frontline Doctors raked in, along with their teleconsults &amp; referrals?\\n\\nIn some cases the cost of the initial consult meant they couldn\\'t afford the (jacked up) price of ivermectin.\\n\\nEdit: It did = it did cost them $$$. Not saying it was a valid prescription or proven treatment. You are absolutely correct. It\\'s the reason it was being used as a scam in South America or Africa, where the con started. The rich countries wouldn\\'t share the vaccines nor give the formula so the governments chose to go with a lie.\\n\\nThe fact that our Propoganda machine just mentioned and it took off like it did is a tribute to the power of the conservative media and the type of people that believe in them.\\n\\nIt\\'s asinine in every logical scientific sense. But throwing away your critical thinking will eventually cost you your life. American medical systems charge a lot for plenty of things which don’t help at all. You know what else is in the public domain? Insulin.\\nI don\\'t see it sold cheap anywhere in the USA.\\nEven if it\\'s a generic, there\\'s nothing forcing manufacturers to sell it cheap, except competition. And since they tend to try to maximise gains rather than accessibility, they would end up agreeing on something expensive. Yep I heard that ventolin cost $100 in the US in Australia it’s about $12 Every single doctor that recommended it should be stripped of their license to practice. That new Alzheimers drug, case in point. Global healthcare spending is about $8T. American healthcare spending accounts for half of that. AND most of us can’t go to a doctor anyway. Were Double blind estudies Carried out? &gt;If you do not have health insurance, talk to any health care provider in your area to see if they will agree to bill the federal government for other COVID-19 related care, like testing and treatment.\\n\\nhttps://www.hhs.gov/coronavirus/covid-19-care-uninsured-individuals/index.html The title and conclusions were rewritten and with this rewrite it passed subsequent peer review. \\n\\nThe paper proposes theoretical mechanisms of action that are supported by study.  There are mechanisms of action replicable in the lab, but this does not necessarily translate into clinical utility (mouse studies, anyone?) \\n\\nThe paper calls for awaiting the conclusions of ongoing controlled clinical trials to check for efficacy en vivo. \\n\\nI think what people keep getting wrong with all of these hasty studies of drugs like ivermectin and famatodine is that placebo is remarkably effective, even when it is the administering doctor that is being deceived. \\n\\nMany, many drugs in common use and with proven utility are only a few percentage points more effective than placebo. \\n\\nThere is also evidence to suggest that “negative placebo” effects are also quite real.  \\n\\nNearly any study will show some  indications of action because of this. \\n\\nStudies need to be very robust in order to negate placebo effects, and this takes time.\\n\\nNot only that, but in cases like famatodine, there are significant confounding factors, such as clear detrimental effects of other common drugs that someone taking famatodine is almost guaranteed -not- to be taking.\\n\\nNone of this is nearly as simple and cut and dried as people tend to make it, and a drug can also be extremely effective in a small subset of (unusual) conditions while being useless or even detrimental for use on average, further confounding analysis.\\n\\nBiological systems are ridiculously complex and subject to nuance due to the high level of imbedded recursive complexity.  \\n\\nThere is a level of self modifying abstraction at the molecular level that quickly becomes intractable, and that’s even before considering effects on a living organism with psychological components that can have effects on the fundamental context of action. \\n\\nThis paper presents solid evidence for proposed mechanisms of action. **That alone means almost nothing at the clinical level.**  This disconnect is counterintuitive, and is the source of almost all of the noise surrounding “treatments” like ivermectin, chloroquine, famatodine, and others.\\n\\nIt’s almost like subjects requiring high level expertise should be left up to people with high level understanding. Latest episode of Tucker Carlson synopsis &gt;Lots of drugs have useful secondary targets / effects\\n\\nRandom but do you mean medication like finasteride which was originally marketed mainly for enlarged prostates but was found to have therapeutic effect for hair loss and same for minoxidl which was for a heart condition but it also stimulated hair growth too I mean, it\\'s not like the medicine is just free because a country has socialized healthcare. The makers of a drug still get paid, the money just comes from the healthcare system rather than individual people\\'s pocket.\\n\\nI don\\'t really understand how what you\\'re saying debunks the conspiracy theory, if the theory is pharmaceutical companies profiting off covid.\\n\\nTo be clear, I\\'m not saying ivermectin works. I have no clue, I\\'m not a scientist. But I do think there are a lot of people getting very rich off Covid, so of course I\\'m skeptical. Covid can also be diagnosed via symptoms. It doesn\\'t have to be a test according to the CDC. My point is that the conspiracy is that governments are somehow in cahoots with “big pharma” to hide the fact that ivermectin works so that “big pharma” can still sell their expensive treatments \\n\\nExcept with socialised healthcare, it’s the governments priority to treat patients as quickly, efficiently and as cheaply as possible. \\nIn the USA you could argue that big pharma are in cahoots with the hospitals because they BOTH make money on people being Ill \\n\\nIn the Uk; the government doesn’t benefit from a sick\\nCountry. A sick workforce not paying taxes and people taking up hospital beds costs money. \\n\\nIf there was a cheap and effective treatment for covid, the UK government would jump on it Ahh that makes sense.\\n\\nDevil\\'s advocate. Couldn\\'t one argue that while the UK government would want to resolve covid cases quickly, there may still be individual leaders who are profiting off pharmaceutical companys enrichment? \\n\\nNot sure if that\\'s as big of a problem in the UK, but in the US we constantly see decisions being made that cost the government money and are worse for the American people, but still go through because individual leaders have a stake in it. What the person you\\'re responding to is implying is that studies where ivermectin seemed to be effective may have been confounded by large parts of the sample possibly having parasitic infections, ie, their condition improved because they had COVID and a parasite in them, and ivermectin killed the parasite Look, antiseptic mouthwash has been found to help prevent COVID infection. As good as vaccine? No but some possible benefit. So if you want to gargle mouthwash go for it but you need the vaccine as well for actual protection. It’s like relying on a side airbag instead of wearing a seatbelt. Well. Kinda. But you’d have to evidence 3 things. \\n1) someone is being paid off and being enriched by it \\n2) the good thing about science is that it’s evidence based. You’d have to still be able to debunk the science as well as being able to prove that the science is wrong.\\n3) you’d have to prove that the people being paid off are somehow making or changing the decisions (influencing ) the entire “board” involved in the decisions. I take ventolin here in US. It used to be $15 now it’s $100. No reason for this, we asthmatics need it to not drop dead This is like saying McDonald\\'s has no interest in selling burgers because they don\\'t have monopoly on them and they\\'re cheap.\\n\\nIf you\\'re the primary producer of something you have a major advantage over the competition, especially in something time sensitive like a pandemic. I tend to think that the first red flag about the efficacy of ivermectin for treating COVID is that there was no price surge.\\n\\nI mean, eventually there was a price surge, but only after people depleted the stocks, not just after some evidence showed it could be used. If you actually read the thing. It’s a way for the hospitals to get money to cover care for uninsured individuals, it doesn’t pay the individuals, or cover anything else such as missed work etc. Well yeah, you\\'d have to prove it. Your second point is totally true but I think you can \"prove\" or \"accuse\" anyone of doing anything and the people in charge can make it look however they want. This could bleed over into point 2 as well.\\n\\nAgain, not saying all this stuff is going on, but for the people who do, you can see why they\\'re so upset. If you read through that it’s a way for the PROVIDER to request payment from the government to pay the bill owed by uninsured individuals they know they won’t get money from. It doesn’t go to the patient. That’s my point. Huh. That’s kinda cool I guess. Other people have posted that care gets paid for by the government sometimes, but that’s a reimbursement of cost for the hospital if the patient is uninsured, it doesn’t address the costs and missed work and such for the patient end of things, it just keeps the hospital funded I don’t see it. The conspiracy just doesn’t stand up. At all. \\nTo be able to carry it off you would need to control soooo many people and variables it just wouldn’t work. \\nTo carry it off, you would need to pay off entire governments (in EVERY country pretty much globally) \\nYou’d have to pay off the scientists doing the studies. The universities they work for. The people doing the peer review. The doctors. Nurses. Pathologists…..the list is nearly endless. \\n\\nThey couldn’t even keep a BJ secret in the white house. And that was only 2 people involved :-) In vitro studies nevermind computer models rarely pan out in clinical practice. People who have studied medicine outside of covid already understand this. Logically you\\'re 100% right. But these people are 100% emotional at this point. \\n\\nA lot of people have chosen to hate them for it. And that\\'s fair and all, but I find empathizing with people gets a lot more done than spewing statistics at people who just aren\\'t going to believe them. Unfortunately, I think a lot of people lack empathy and call it a day after they believe they\\'ve proven that they\\'re right.\\n\\nAnd if the goal is to get everyone vaccinated and get done with COVID, I don\\'t see the current strategy working. Exactly. What a model says is feasible and what should work is different to real world, and in clinical settings are different. In vitro study of ivermectin was a near lethal dose to get good results. Would you consider information from this site to be highly authoritative? I personally do And you don\\'t want to OD on Ivermectin. The drug is a neurotoxin. That\\'s how it kills parasites. That is incorrect. All of the clinical data and claims were removed in the revised manuscript, which now solely focuses on the mechanism of action of ivermectin. \\n\\nFrom the article\\'s Additional Information section:\\n\\n&gt; This is a revised version of the previously retracted article below. The article has been revised to remove references to clinical studies, focusing exclusively on the mechanisms of action of ivermectin. The article has undergone peer review independent of the original article’s review process. Unfortunately, I believe you are correct for a vocal (maybe \"for the\") minority.  The evidence doesn\\'t matter if it doesn\\'t support ones\\' prejudices. Social scientists* (yes I know this is a controversial field) are going to have a field day in the future with this mess. Which is why buses of Americans come across the CDN and MEX borders to buy drugs but leave before they get infected with SOCIALISM. You said hospital bills and funeral costs weren\\'t covered. They both are.\\n\\n&gt;What services are covered at no cost?\\n\\n&gt;COVID-19 testing\\xa0(both diagnostic and antibody)\\n\\n&gt;Testing-related in-person or telehealth visits\\n\\n&gt;COVID-19 treatment and therapeutics\\n\\n&gt;Treatment-related visits at an office, via telehealth, in an emergency room, for inpatient or outpatient/observation, at a skilled nursing facility, or for long-term acute care (LTAC), rehabilitation care, and home health\\n\\n&gt;Use of medical equipment (e.g. oxygen, ventilator, etc.)\\n\\n&gt;Ambulance for emergency transportation and non-emergent transfers Aduhelm has a 90% chance of getting retracted, because people are getting killed from it. But more importantly, Biogen execs got slightly richer. Unfortunately,  peer review is just the opinion of 2-3 scientists, who may be cronies with the authors. Lazy journal Editors ask for suggested peer reviewers and more often than not, send to those reviewers suggested by the authors. \\nIMO, this happens in all research to a small degree, but is out of control on MD-based manuscripts. That should have happened on first review. Many MDs prescribe drugs off label because they \"should\" work. Evidence based medicine is still not pervasive in North America. I\\'ve read states in India and Japan openly stated they use ivermectin and their cases are near zero per day.  Not sure if it\\'s true because I don\\'t remember where I saw the articles and information, but could be useful to look at. Pretty sure this has been explained in other comments under the thread. The retracted article, with the studies it cites, remains available on the journal site linked in the mod post. I\\'d link to it and to some other meta-analyses (your third question) but don\\'t want to inadvertently run afoul of automated filters regarding this subject.\\n\\nThe largest single RCT studying ivermectin was the Together trial in Brazil, which found a 9% reduction in the proportion of patients requiring extended ER observation or hospitalization, and a 19% reduction in mortality. The 95% confidence interval for the first endpoint was -19% to 31%.\\n\\nSource: 31:16 in this presentation from the trial researchers: [https://rethinkingclinicaltrials.org/news/august-6-2021-early-treatment-of-covid-19-with-repurposed-therapies-the-together-adaptive-platform-trial-edward-mills-phd-frcp/](https://rethinkingclinicaltrials.org/news/august-6-2021-early-treatment-of-covid-19-with-repurposed-therapies-the-together-adaptive-platform-trial-edward-mills-phd-frcp/).\\n\\nOn its own, this trial rules out high efficacy for ivermectin, but does not rule out some low but still useful level of efficacy. The current [NIH guidance](https://www.covid19treatmentguidelines.nih.gov/therapies/antiviral-therapy/ivermectin/) is:\\n\\n&gt;There is insufficient evidence for the COVID-19 Treatment Guidelines Panel (the Panel) to recommend either for or against the use of ivermectin for the treatment of COVID-19.\\n\\nOn the other hand, the same Together trial did find [32% efficacy](https://www.thelancet.com/journals/langlo/article/PIIS2214-109X(21)00448-4/fulltext) of fluvoxamine, a widely available antidepressant. US authorities have adopted the same stance on fluvoxamine:\\n\\n&gt;There is insufficient evidence for the COVID-19 Treatment Guidelines Panel (the Panel) to recommend either for or against the use of fluvoxamine for the treatment of COVID-19.\\n\\nThe Together estimates put fluvoxamine way below Pfizer\\'s new drug (89% efficacy), and about on par with Merck\\'s drug (30% efficacy). My guess is that by the time the NIH/CDC are ready to make strong recommendations on repurposed treatments, supply of the Pfizer drug will be more abundant and the recommendations may be moot.\\n\\nOf course that will be too late for current patients. Monoclonal antibodies work and are a super effective treatment though, right? Of course. It\\'s an effective popular treatment if you catch COVID and are hospitalized. \"What isn’t covered is hospital care bills\"\\n\\nThat\\'s your statement.\\n\\nThey\\'re covered. Yes, just like insurance the provider has to bill someone to get paid. They get to bill the government. Then why isn’t it being prescribed for outpatient treatment? Legit question because I don’t understand the hold up of monoclonal antibodies. Isn’t it an essentially limitless supply? For what, Ivermectin? Yes there\\'s at least one that I see isn\\'t retracted but it\\'s a small study. I\\'m not too knowledgeable in it, but it looks like hospitals are running out of it, and the federal government paused shipments and it seems to be related to it not being that effective against Omicron? I don\\'t know the details. Yeah, it\\'s not like time and effort are real Just a side note, off label doesnt necessarily mean unscientific. There are drugs that have a fair amount of evidence for them in a certain indication but it\\'s so niche the drug company doesn\\'t want to go through all the work to get it approved for that purpose. \\n\\nBut you are right a lot of docs don\\'t even follow that. New indication means a new patent basically. Theyd still make money and would jack up the price. There was in vitro evidence and a proposed mechanism for ivermectin working though. It\\'s at the clinical trial phase when it was shown it didn\\'t actually work, yet people continued using it. In a perfect world it would. But reviewing is unpaid work, so soliciting reviewers can be very time consuming for editors and mistakes can be made. &gt; How does this help the average person\\'s faith in peer reviewed studies?\\n\\nI don\\'t think you should put too much faith in one study.  Best is to wait for independent studies that confirm the results.  And even so, each study has a different confidence or strength level.  So it is best to read the studies and see if the conclusion is a strong one and how strong it is.\\n\\nTLDR: The more scientist that agree, the better. “Peer-Reviewed” is the biggest hypocrisy known in the scientific community… It is a who you know club especially when you get into more controversial subjects.\\n\\nTake everything you read in a science journal with a hefty grain of skepticism. It’s interesting you think this.  The paper was accepted on the basis that the mechanism was “correct” just not the supported clinical studies cited by the paper.\\n\\nThe authors did not agree a resubmitted a revised paper on the mechanism only.\\n\\nThe way I read it, the mechanism is sound, just no supporting clinical data to support the original premise of the paper. Half? Do you a source for that?\\nI’m pretty sure I heard dr John abramson say the number 60% more than any other country, but not 50% of the entire world. I take an antihistamine (hydroxizine) for anxiety. CBF checking this properly but some quick googles looks like this roughly checks out.\\n\\n\\n&gt; U.S. health care spending grew 9.7 percent in 2020, reaching $4.1 trillion or $12,530 per person. As a share of the nation\\'s Gross Domestic Product, health spending accounted for 19.7 percent.\\n\\n&gt; Global healthcare spending is forecast to decline 0.1% to USD 8.3 trillion in 2020 before growing 5.8% to USD 8.8 trillion in 2021. ... 3 trillion in 2020, while pharmaceutical sales are expected to decline 1.6% to USD173 billion in 2020 and to grow 3.4% in 2021. Because our prices are 10x everyone else’s I used to get out for$5. Then one day it went up to$30 with insurance.  During 2021 it dropped down to $5 again I started getting a generic again during 2021. Ask your pharmacist because my last two puffers were $5 each You were saying?\\n\\nThere are cheap generics available- some better than others. \\n\\nhttps://www.cnbc.com/2021/06/29/walmart-unveils-low-price-analog-insulin-amid-rising-diabetes-drug-costs.html\\n\\nThe issue is that insulin is just a more expensive medication to make than others. It\\'s not a simple drug you can easily make. There are certainly other markets issues at play (including the government looking to get involved), but the market is working to correct the issue.  Markets are not instantaneous, but they are correcting. That\\'s nice.Still not as cheap as insulin provided through public healthcare programs, but that\\'s nice. In a day and age when anti-intellectualism is rampant,  I think it’s pretty damn important we do literally everything possible to get these things right the first time through. It’s not like we are talking about studies looking into behaviors found in household pets here; this is the most controversial subject around the globe right now. Get it right Again, totally ignorant of the reality of limited resources Ivermectin is listed on the WHOs list of 40 essential medicines. It\\'s not just a horse dewormer. It also has no patent and is thus much less profitable for drug companies. It\\'s crazy how this drug became a political/culture war issue. It\\'s also strange that doctors were barred from prescribing it off label when there are almost no side effects. I\\'m not sure if it helps for covid but I\\'m open minded. The chairman of the Tokyo medical association recommended it for covid. I doubt that Japanese guy is a partisan trump supporter type. None of this madness makes any sense. No wonder so many people in the general public have developed a conspiratorial mindset. Always take an opportunity to shout-out retraction watch\\n \\n\\nhttps://retractionwatch.com/ It does work. Would you reverse that opinion if if learned that it’s been working this whole time? We do NOT know that IVM isn’t effective for treating COVID and the majority of the evidence suggests exactly the opposite. Saying something over and over doesn’t make it true. It underwent a Facebook fact check Ivermectin is being used today with near perfect success by doctors around the world. Get out of the echo chamber and look at what Tyson and Fareed have done (6000+ treated early with ZERO fatalities). Uttar Pradesh (240 million people) wiped COVID out with at home IVM kits sent to every household. My own doctor treated my entire family with IVM (5 people) with barely a sniffle over a handle of days. She’s treated hundreds of others with similar results and her biggest issue is fighting with pharmacies. 73 studies from researchers around the world show better than 50% improvement in every meaningful outcome including mortality, recovery time and hospitalization rate. IVM only doesn’t work if viewed through a political lens which has no place when analyzing a deadly disease. Science is meant to be open minded AND skeptical, not political. Doesn’t happen in the real world. Look up the safety profile, it’s 10x safer than aspirin. Not even close to accurate. The vast majority of the 73 studies show positive effects across every meaningful outcome variable. 73 studies have been done on IVM including over 30 “gold standard” RCTs which provide overwhelming evidence of efficacy. Only highly politicized evaluations of the evidence suggest “no efficacy.” Entire countries are using it successfully today while the US has a death per million rate 25x than Uttar Predesh (with similar populations). We\\'re still not sure. Because any positive effect for COVID is so small if there\\'s any. There are way better treatments that have been available for well over a year. It’s not true that IVM failed to show efficacy. IVM has shown tremendous efficacy across 73 studies with tens of thousands of subjects. The issue is that the politicization of the interpretation of the studies has made people believe that these researchers are not trustworthy and their RCTs are ALL so woefully flawed that we should dismiss 70% efficacy rates without further curiosity. Instead we should trust one half assed study supporting the ultra expensive Remdesivr because Fauci gave a press conference on it. \\n\\nHint: go read the studies yourself. There are lots of issues but even the bad studies move the needle in favor of IVM. In total, there’s zero doubt that IVM combo therapy given early at reasonable doses works far better than SOC and is extremely safe. US and UK governments both [spend](https://data.oecd.org/chart/6zCo) about 8% of GDP on health care (the big jump in 2014 was when the ACA reclassified private health insurance from voluntary to compulsory; ignore that since it\\'s not actually government spending). Furthermore, in the US about 11% of health care spending is [out-of-pocket](https://data.oecd.org/chart/6zCn), compared to 16% in the UK.\\n\\nSo both premises here are wrong. The US government could save even more money with a cheap treatment for COVID-19 than the UK government could, especially when you consider that every American over the age of 65, which is where COVID-19 hospitalization risk is concentrated, is insured by the government via Medicare. Also, the vast majority of Americans have health insurance, so this idea that everyone\\'s just a hospital visit away from total financial ruin is nonsense. It works fine for Tyson and Fareed who have treated over 6000 people early with zero deaths. You don’t really think they all had worms do you? This was debunked over a year ago and it keeps coming back. IVM has been shown in dozens of studies to work fine at normal, super safe doses. Show me the people who lost their stomach lining from taking IVM. You know the Rolling Stone article was retracted/made up, right? C19ivermectin has the full list of IVM studies and is updated regularly. Not accurate. There are and it is. Way better than 68% improvement (early treatment) with the best safety profile you could possibly have and a cost of under one U.S. penny per dose? What treatment tops those stats? There isn\\'t 68% improvement. Also Casirivimab, imdevimab, and Paxlovid are better. Most of those studies are invalid. That simple. But by all means take it Remember, if only two or three of the 73 studies are well done and support IVM, that\\'s easily enough to justify use. Remdesivr was approved with a single BAD study (they changed the end points) that showed no positive impact on mortality (just time in the hospital). There are easily dozens of worthwhile studies in the 73 and even the bad ones have some good data. They\\'re often bad because they are written by people who barely know English who wanted to get their findings out fast. It doesn\\'t mean that their data isn\\'t real or that they should be 100% ignored. When the IVM group is getting 0% death and the control/SOC group is at 14%+, it pays to pay attention even if you do find a glaring typo on page three of the manuscript.\\n\\nI did take IVM when I had COVID and it worked great. I was quarantined but I didn\\'t even lose a day\\'s worth of work while I had COVID.\\n\\nOr, maybe it was all placebo and I\\'m just lucky. It also worked amazing well for my dad (within hours) and my kids (who probably didn\\'t need it and only took a dose or two). It\\'s also worked for 100% of the patients that our doctor has treated (150+ was her estimate). She won\\'t publicize it for fear of reprisal, but her numbers speak for themselves and they are similar to other IVM doctors across the U.S and around the world. Her partner won\\'t use IVM and has lost almost 4% of his patients with a very similar patient base. Imagine watching your medical partner let 4% of their patients unnecessarily die simply because they were in lockstep with the NIH/FDA machine and not willing to read the source research or even look to their own partner. The reckoning when this is over is going to deservedly destroy careers. Most primary care physicians have buckled when they needed to be courageous and abdicated their responsibility when they needed to think for themselves.\\n\\nSorry for the soapbox treatment. It\\'s a really important topic which has had a direct effect on my life for the last year. Well then can you create a new post in the sub linking the studies? I’d be interested to see what people think. So given all the scientific evidence it doesn’t, the retracted bad studies that say it does and the the entire thread on the science Sub explaining how and why it doesn’t work…..your “scientific” response is “It does work”. \\nThanks for the insightful and scientifically sound response. Your stats only really add to my point. They don’t make my premise wrong. I didn’t say the UK would save more money than the USA. \\nSo In fact; BOTH governments would save a lot of money with a cheap treatment. Plus all the insurance companies would save a fortune as well \\nI take your point about Medicare for the elderly and the high insurance rate. But there are still 10% of USA without insurance (which is about equal to 50% of the UK population!) that’s still a lot of people where a single hospital visit could mean a huge debt. \\n\\nI think your argument is that the USA healthcare system isn’t as bad as people make out and that sometimes the hospital costs are sensationalised in the media or something? I wouldn’t disagree based on your stats. (Although I’m biased towards my own socialised healthcare as a better option :-))\\nBut that doesn’t really make my premise wrong Ivermectin doesn\\'t fight covid in the real world. The dosages used as an antiparasitic that are safe are in the magnitude of 9mg in a whopping 2 doses. This is a tiny dose that barely makes it into your bloodstream and is nothing more than a verifiable placebo for covid.\\n\\nWhen you increase the dosage to what would theoretically be needed, the drug is anything but safe, which the CDC has confirmed with a list of symptoms that can occur including seizures, coma, and death, as the drug is, in fact, a neurotoxin. Nope, if a study is valid also depends on size. A small effect requires a much larger sample size. It seems that most of those 73 studies were rather small (under a thousand participants) and could, even if well designed, could not have provided conclusive information. What’s truly laughable that some people believed that meta analysis allowed them to throw all studies together no matter how poorly done. Just shows that IVN either trying to lie through their teeth or have absolutely no idea what they are doing. And no those studies were bad because they used no valid design, ignored controls, weren’t blinded, cherry picked data or outright falsified data. Bad studies can’t be rectified by cherry-picking, that is in fact an absolute no-no. Endpoints can be in fact adjusted, it’s frequently done with approval of regulators. If you take issue with the remdesivir study feel free to write the FDA. Once again feel free to put your trust in bad science. Bothers me none Yeah, you got to watch that socialism. It\\'s a tricky one. I\\'ve heard tell that a school of socialism can skeletonize a cow in under five minutes. Everyone else is saying the opposite. Can you provide links to any of these 30 gold standard studies to support this claim? Remember that lack of evidence does NOT suggest evidence against. For example, in the RECOVERY trial that people often use as evidence against IVM, the IVM group actually did 18% better than the SOC group. Given the sample size, this wasn\\'t statistically significant, but it\\'s completely inappropriate to classify positive evidence as \"evidence against.\" I stress this because you said that IVM isn\\'t effective and there is no real evidence to support that statement, there is only evidence that didn\\'t show statistically significant efficacy (which is not the same thing). Most commonly, the lack of statistical significance is because you need very large sample sizes when examining something that has a low base rate (i.e., COVID mortality).\\n\\nAgain, quite simply, lack of evidence does NOT equal evidence against.\\n\\nBefore I provide the obvious link, I have past evidence as to what people will think so I\\'d like to make a hypotheses. \\n\\nHypotheses: The people who think IVM works won\\'t go to the link because they already \"know\" that it works. The people who think IVM doesn\\'t work, will dismiss ALL 73 studies because they read in Newsweek or WashPo (any pro-pharma media site) that they were \"low quality\" or even that one of them had to be retracted. They\\'ll also mention lack of peer review, and typos and other issues and paint ALL 73 studies with the same brush. They surely won\\'t go through the studies one by one and notice the obvious and strong pattern of positive results. They also won\\'t do their own Bayesian analysis of the studies that they personally find compelling, because that would take too long, or they don\\'t know how. Since this is supposed to be a science group, a minority of people will look for names that they think they can trust like Lancet or NEJM or Cochrane and they\\'ll parrot anything they find from those groups while ignoring an entire world\\'s worth of obvious, on your face and wiggling evidence.\\n\\nI\\'d also predict that a handful of people will become super arrogant and we will go round and round until I get rude and then they\\'ll report me for calling them a dipshit. Even though I\\'m predicting that, I\\'ll also promise to be as nice and respectful as possible to anybody who just wants to have a conversation. I\\'d much rather learn and be uncomfortable than continue to be wrong. My focus is not to persuade others, it is to learn for myself. I only push people on their beliefs because I want to understand what I might be missing and I only breakout the dipshit moniker when they are both arrogant AND wrong.\\n\\nWith that long intro, the website that has all 73 studies on IVM combo therapy is **c19ivermectin dot com**. To put IVM use in the real world in perspective, I\\'d also do some research on Uttar Pradesh (eradicated COVID with IVM), and Tyson/Fareed (6000+ early treated patients with zero deaths). Epidemiological research is super messy, but the pattern with IVM is also incredibly clear. It literally has worked everywhere it\\'s been tried and in cases where it has been implemented fully upon first symptoms (like Uttar Pradesh), it has wiped COVID out. Places in the world that use IVM heavily for other reasons, have near zero COVID. Stack each \"coincidence\" and go on to the next. When the stack get\\'s high enough your brain will say, \"I have no other plausible explanation for this pattern of results.\" Then you still have to remind yourself that there might be an explanation that you haven\\'t thought of which is why people push for well done RCTs. So you are asking someone to prove a negative?? Doesn’t seem like your c19 website is very credible and the studies have major flaws\\n\\nhttps://arstechnica.com/science/2021/09/the-anonymous-meta-analysis-thats-convincing-people-to-use-ivermectin/?amp=1\\n\\nhttps://www.vox.com/platform/amp/future-perfect/22663127/ivermectin-covid-treatments-vaccines-evidence I am not. Why do you think that? It\\'s quite credible. An easy way to figure that out is to go to any one of the 73 studies listed for IVM and search for it on your own. You\\'ll find it and nothing will be altered from the study you found to the study linked on c19. You pointing to articles that claim the site is not credible are evidence of nothing beyond one author\\'s opinion.\\n\\nThink of it this way, I could have given you a laundry list of 73 different links but instead, I gave you a neatly arranged, updated regularly list. You can ignore the additional analyses they do on the source research but ignoring 73 studies because a journalist doesn\\'t approve of how a website is designed is not at all rational.\\n\\nI\\'d also keep in mind that most of the site is just math equations with their work and source material shown. If you don\\'t like their math, you can do your own (I do it all the time). Because you are saying lack of evidence doesn’t not mean evidence against. \\nscience works from proving positives, not negatives. \\nIt’s like going back to the “vaccines cause autism” claim. \\nThere’s is no credible evidence to say it does. There are studies trying to find a link, but couldn’t. \\nSo you CAN say vaccines don’t cause autism, until someone actually proves it does. \\n\\nThere’s a lack of evidence of a dragon under my bed. But unless I prove otherwise I think you can state that there isn’t a dragon under my bed.\\nsilly analogy I know , but the point is you are asking someone to prove a negative. At a dose high enough to be effective it is NOT safer than aspirin. It’s a gish gallop. And a gish gallop that scientists have dismissed based on their lifetime of study and their understanding of good and bad science. Does it also list any studies that show no evidence of ivermectin being effective?\\nEver wonder why that is? Is your argument that ALL of the 73 studies should be ignored because some of them are bad? Do you have a study by study breakdown of what makes them so bad that you wouldn\\'t even look it them? It\\'s one thing to say, I\\'m concerned about their sample size, it\\'s another to dismiss a study altogether that shows 70% efficacy but no .05 level significance. That\\'s clearly a case where you say, \"we need more data!\"\\n\\nEssentially, being small, may make a study inconclusive, it doesn\\'t make it bad. If I do ten studies with 200 people each and none of them show statistical significance, but my data is reported accurately, it is perfectly appropriate (and smart) to combine these studies into a meta analysis with 2000 people. It\\'s nonsensical to say that because the studies weren\\'t good when they were small, they also aren\\'t good when they are large.\\n\\nThe meta analyses show, after exclusions for quality/bias, that IVM works great across a range of meaningful outcomes. Show me the breakdown of how you excluded ALL 73 studies based on reasonable criteria and tell me how you are better at making those calls than Tess Lawrie. The only way you can claim that IVM doesn\\'t work is by making grand conspiracy claims that all of the research was fraudulent and all the people doing the meta analyses were complicit. At a minimum, you should establish your criteria for inclusion, go get that data and add up the numbers (like Tess Lawrie).\\n\\nI\\'m curious, what would it take for you to change your mind? Do you need a singular, grand study, or if they get to 100 studies with the same effect size they have now (66% improvement)? Does somebody from the right Govt agency have to bless it? Basically, where is your finish line? Also, how do you explain doctors successfully using IVM to treat thousands of patients with no deaths? Are they lucky or lying or do you not care because they don\\'t publish in the right journals? Also, why does the U.S. have a death rate 25x vs Uttar Pradesh? Are they lucky or lying too? I\\'m not trying to be hostile, I really am curious how you navigate all of this evidence with such confidence. Post some studies then of high enough doses in human plasma to reach antiviral activity shown in the invitro studies. Well, thanks for telling me that you didn\\'t even glance at the website. \\n\\nYes, 7 of the 73 studies show a negative effect of IVM treatment. This is usually a case where one person was hospitalized in the treatment condition and zero in the control (i.e., statistical noise). Nonetheless, they should be reviewed carefully and included in the overall meta analysis. The goal is not to prove the invitro study, the goal is to prove real world efficacy with humans. If you\\'re looking to understand mechanisms, I\\'d start here [https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8203399/pdf/41429\\\\_2021\\\\_Article\\\\_430.pdf](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8203399/pdf/41429_2021_Article_430.pdf)\\n\\nPersonally, I\\'m more biased toward focusing on research methods and statistics than I am towards understanding the biological mechanisms mainly because I\\'m better at stats. Nonetheless, mechanism is important. If you do ten studies and study design and patient stratification is different in each of those studies than you won\\'t be able to throw them together. In order to combine data they have to have similar attributes. Basic science. That is the reason why good meta-analysis is rather difficult.\\n\\nHow do I negotiate all those data with confidence? \\n\\n(a) read the literature (credible sources like top scientific journals and trade news not some fourth tier pay to publish rags, or even worse, self published online sources)\\n\\n(b) look at recommendations by regulatory agencies and public health agencies (all of the ones that matter anyway)\\n\\nOf course I can delude myself and think it all a giant conspiracy theory coordinated between government, academia and biopharma across dozens of countries and listen to a bunch of doctors that suddenly pose self anointed infectious disease experts like those that run the pro-IVN groups. \\n\\nBTW the data out of Uttar Pradesh are likely to be the result of under-reporting or active manipulation of data. The only reliable statistics will be excess death. Russia reports on 300k COVID death (which made Russia look better than the US) yet its excess death rate sits at over 900k. I\\'m not him, but I certainly wouldn\\'t, it\\'s largely irrelevant. Only 10x safer within the commonly used dosages for it\\'s primary and most commonly prescribed purpose (parasites). So, technically when you are testing a hypothesis and find that your hypothesis is NOT supported, it does NOT mean that the reverse is true, it just means that your hypothesis is unsupported. The simplest way to think of it is that it means nothing. Very technically, I would argue that it may increase the probability of other hypotheses, but it in no way validates them. Take an easy to visualize example, if I test the hypothesis that there is a treasure chest at the bottom of a the Pacific with a COVID cure in it and then I go and scuba dive around a 200 meter area of the ocean bottom and don\\'t find anything, have I proven that there\\'s not treasure chest? Obviously not. I also haven\\'t really moved the needle on the \"there is no treasure chest probability.\" The only time this may change is if the search domain is definitive and eventually you can verify with a high degree of confidence that a hypothesis is false. For example, if you were looking for a watch in a pool and eventually could search it all comprehensively.\\n\\nSo, that\\'s why I said, **lack of evidence for** is **NOT** the same as **evidence against.** In general, if you wanted to find evidence against IVM, you\\'d either have to show a consistent negative result (i.e., it does harm), or you\\'d have to show mechanistically that it couldn\\'t work. This is what people try to do when they emphasize that IVM is an anti parasitic medication. What they don\\'t recognize (or say) is that it\\'s also an anti viral. \\n\\nTo make this more specific to IVM, let\\'s look at the most recent study (73rd) to come out. In the treatment group, zero of 39 people died, in the Standard of Care Group 8 of 49 died (that\\'s not good, relative to IVM group). Statistically, this is a highly significant result (p &lt; .001) meaning it\\'s unlikely it happened by chance. That\\'s all topline breakdown and it looks pretty good in favor of IVM. It looks even better when we dig into the study notes and find that the people who got IVM were sicker, on average, than the people who didn\\'t get IVM (i.e. their probability of dying should have been even higher). From here, all of the weaknesses start coming out, it was not a properly controlled RCT, it was an observational study although it used statistical controls to try to balance the treatment and control groups (good, but not as good as an RCT). Even though we have statistical significance, the sample size is small and it\\'s fairly narrow (only hospitalized patients). It\\'s also a study out of Japan so there may be issues with generalizing results to other populations. These critiques could go on and on but ignoring the overall results because of them would be irresponsible. Instead, this study should be added to the other 73, warts and all and it should give us a bit more confidence that IVM is having a positive effect. In reality, the probability that IVM is having a positive effect is so high already that this study doesn\\'t do much of anything. Based on the c19 analysis, the current probability that all of the positive IVM results seen across the 73 studies is simply a statistical fluke is 1 in 619 billion. Even if I have a meta-analysis of 10,000 studies, it is irrelevant if I only cherry pick the worst quality studies that support my pre-conceived biases, notions, and hypotheses. I could make a meta-analysis of 73 studies that show the complete opposite, for example, just by cherry picking from the opposite side only. Same with alcohol with most things in a petri dish, alcohol will kill covid in a petri dish, but you can drink 2 bottles of vodka a day and then die of covid. And it is largely impossible to prove a negative. Ehh, it\\'s more like relying on your windshield to stop your body and save your life with no seatbelt or airbags just because a few people have survived these accidents. Research the OxyContin Opioid crisis and watch the movie “DOPESICK” which is based on true life events. That information will definitely put things into perspective that it is possible and it has happened. \\n\\nThis was partly how the Opioid epidemic started in the US in 2016/2017.\\n\\nOxyContin was an FDA approved drug that contributed immensely to the Opioid crisis. There was a lot of politics involved in keeping a drug that was addictive in hospitals. BTW it’s borderline hysterical how Tess Lawrie gets pushed as world renowned scientist. Looked her up on Research Gate. She has barely half the citation I got and I am anything but world renowned. Just shows that people use the situation to push their CV by stirring the pot You answer to Uttar Pradesh is that it\\'s simply misreporting? You think their hospitals are actually full of COVID patients and nobody is talking about it? It\\'s not North Korea. You think there\\'s a countrywide conspiracy to prop IVM up (to what end?). Wouldn\\'t you at least acknowledge that as a datapoint it moves the needle in favor of IVM?\\n\\n[https://indianexpress.com/article/cities/lucknow/uttar-pradesh-government-says-ivermectin-helped-to-keep-deaths-low-7311786/](https://indianexpress.com/article/cities/lucknow/uttar-pradesh-government-says-ivermectin-helped-to-keep-deaths-low-7311786/)\\n\\nAlso, how do explain Tyson/Fareed treating 6000+ patients with zero deaths? They aren\\'t alone, there are tens of thousand of documented cases. Again, wouldn\\'t you at least acknowledge that this moves the needle towards IVM? Aren\\'t you curious why no regulatory agencies seem to be reporting on these cases or trying to study them? Aren\\'t you curious why the NIH waited over a year to begin studying IVM in a multi-year trial that will have zero practical effect on treatment?\\n\\nI could give you 100 places to go to see obvious strong effects of early IVM treatment or even modest positive effects of late treatment. You have to do serious mental gymnastics to unsee the pattern. Are all of these people just lying or lucky?\\n\\nNote that I think one thing that is different about our approach to finding truth is that you assume the regulatory agencies to be credible. I don\\'t make the same assumption. To me the evidence strongly suggests that they are agenda driven and political, not driven to spread truth. Sometimes the truth may overlap with their agenda, but you can\\'t count on it. That just tells me you haven\\'t looked at the data. The vast majority of the studies favor IVM. I could tell you, go knock out the ten studies that make IVM look the best and the bottom line meta results wouldn\\'t change at all.\\n\\nSpecifically, 66 of the 73 studies favor IVM. Moreover, 100% of the prophylaxis studies favors IVM! There is no way you can cherry pick your way out of positive efficacy without exposing massive bias.\\n\\nI should stress that there is a lot of very poorly done and poorly written research in this massive pool of studies. Many of the researchers are not native english speakers and many of them have acknowledged that they wanted to get their data out to the world as quickly as possible. This does NOT imply that the mortality numbers or hospitalization rates (i.e., the important stuff) are not rock solid. It mainly implies that when you are reading about \"low quality\" research by a low quality journalist, you should dig a little deeper. The improved survival rates and recovery times for people who get IVM early is amazingly robust. Do the math yourself comparing treatment to control outcomes for only studies that you find credible and you\\'ll see the pattern in ten minutes. Actually I do think is misreporting. Show me excess death data to convince me otherwise.\\nTyson Fareed make big claims yet there is actually no peer reviewed publication or any other check on their data. Worse they spread misinformation about the vaccine. Now that smells very much like an agenda\\nThere is a pattern that claims are made but data are not released for peer review. You can’t give me a 100 place that have shown success for IVN. You can give me 100 places that claim success. If you want acknowledgement show me data from a multi center randomized double blind trial with proper patient stratification. Stop invoking what would be at best anecdotal evidence. \\nWhile you are at it show evidence the regulatory agencies around the world are all suppressing IVN for political reasons. In an absolute sense, you\\'re right in the sense that you\\'re probably not going going to die regardless of what you do. Nonetheless, I\\'d much rather have a 1:300 chance of dying vs 1:100. I certainly wouldn\\'t use a doctor that didn\\'t recommend early treatment of COVID. The ones who follow the NIH \"wait and see\" guidelines should not be practicing medicine. Okay, this is getting interesting. Let me say that you seem reasonably smart. I\\'ve been obsessed in recently months with how smart people, who aren\\'t totally evil, have been ignoring so much evidence that has the potentially to save so many lives. I think I\\'m understanding part of it now. You clearly believe that there are lots and lots of really bad actors who are literally making stuff up. I\\'ve been watching Tyson posting his numbers for over a year and that thought never seriously crossed my mind.\\n\\nBefore I get to why it didn\\'t cross my mind, I have to say that what you\\'re saying about Brian Tyson is an incredibly serious accusation. If he\\'s lying, and intentionally steering people toward an ineffective treatment just to somehow promote himself, that is worthy of serious prison time. For the longest time, he put his treatment numbers on a sandwich board and posted it to Twitter every day. I notice in when he was around around 200 patients well over a year ago. So you\\'re talking about multi-year, highly public fraud that includes full office staff and journalists/reporters and even potentially patients. Why would a family member of somebody who died pretend that they hadn\\'t. Also, I\\'m fuzzy on the details of this but Harvey Risch, a Yale Epidemiologist reviewed Tyson\\'s records at some point, possibly with the intent of publishing, so he would have to be in on it too (or he was given false information). Anyhow, I may be naive, but I think the idea that all of this reporting is simply made up or fraudulent is pure fantasy. I\\'d beg you to either rethink this or find evidence supporting it begins, if you\\'re correct, that could change a lot of minds. If you\\'re incorrect, you\\'d have to face the fact that IVM is indeed working in the real world and has for a long time.\\n\\nKeep in mind, on top of this fraud, you are also arguing that Uttar Pradesh data (240 million strong) is completely fraudulent, your arguing that the vast majority of 73 studies are fraudulent, and you still haven\\'t addressed Mexico City, Bangladesh, and most of South America. Basically, wherever IVM is used, COVID either doesn\\'t happen or it quickly disappears. In this scenario, there are literally 1000s tens of thousands of people involved in a conspiracy (not necessarily planned) and they have zilch to gain and lots to lose. I\\'m completely lost as to why any of this makes sense.\\n\\nFinally, there\\'s a huge difference between, my Grandma ate an olive and it cured her constipation (i.e., anecdote) and an entire nation state recommended that it\\'s inhabitants take IVM at first sign of symptoms and the countries case rate and death rate dropped to near zero. That\\'s NOT anecdotal. It is messy and it does have load of confounders but epidemiological and anecdotal are not the same thing. \\n\\nOh, regarding suppression of evidence, I\\'ll point out that Dr. Tyson isn\\'t allowed on Twitter anymore and I\\'m not allowed all over the place. I\\'m actually surprised that r/science hasn\\'t pulled me yet (they will). At a higher level, the NIH didn\\'t put any money into researching existing therapeutics for the first year of the pandemic (!?) and even now the main IVM study that they are supporting runs for years. In other words, they complain that there\\'s no good research while simultaneously not funding the multi-center randomized U.S. based trial that you would like to see. I\\'m confident that most of the suppression of evidence is because people are concerned about vaccine hesitancy. There has never been a bigger reason for committing atrocities than \"the greater good.\" It allows people to justify all forms of deceit while still feeling like they are doing the right thing. Fauci lying about masks not working is the tip of the iceberg.\\n\\nAnyhow, thanks for listening and giving me a unique (missing) perspective. Afaik there has been no doctors being barred from personally prescribing drugs for off label usage. The case I think you are talking about was about patients that originally had been prescribed IVM by an outside  doctor and then not continuing to be prescribed it by doctors within the hospital when they were hospitalized, because off label use comes down to individual doctors discretion.\\n\\nYou cannot force a doctor to prescribe something for off label use which they don\\'t agree with, that is what it came down to. I mean unless you have some legitimate examples of US doctors specifically being barred completely from prescribing it off label to their own patients with covid, I haven\\'t seen any.\\n\\nAlso, regardless of the chairman of the Tokyo medical association (Dr Haruo Ozaki) making this recommendation 2 times, once in Feb and once in August, as of late November, it is still not approved for use in Japan, because he can only recommend things, he is not part of the ministry of health to actually approve anything. So afaik IVM is still not approved for use in Japan for covid, and is still not widely used, regardless of the whackadoo claims floating around. If it is approved it would have to be within the past month or so, I haven\\'t seen anything about it though. I think your right. They haven\\'t been barred or prohibited, that was incorrect . I know that some Canadian doctors are afraid to prescribe it for covid due to fear of repercussions. Not sure about the situation in the US.\\n\\nhttps://www.cbc.ca/news/canada/edmonton/ivermectin-covid-alberta-nagase-1.6205075 There isn’t much of conspiracy.  Per the FDA website, roughly 50% of their funding comes from their “customers.”  Like every business, the biggest customers get the most preferred treatment.  Nevertheless, there are other confounding political variables.  But really, how else does an Alzheimer’s drug (Aducanumab by Biogen, aka big pharma) with dubious clinical results and bad safety profile get approved (against advisory council recommendations), while a treatment (Remestemcel-L by Mesoblast, aka, small fry company with a treatment that could change the landscape of medical treatments) that reduces mortality by over 50% in children suffering from Graft Verses Host Disease and a pristine safety profile does not (again, against advisory council recommendation)?   I don’t think I’m alone in thinking that the FDA needs a capital and organizational restructuring. This is outdated by over a year. Normal doses work just fine (.4 to .6 mg per kg). Zero side effects deaths ever from this dose. You don’t want to OD on any drug. Regardless, as far as drugs go IVM is as safe as it gets. The safety concerns are a Red Herring. What’s your source? What’s the efficacy against symptomatic infection? What’s the rate of adverse effects? “Gish gallop” is not very precise so I won’t comment on that. I will say that “scientists” do not speak with a single voice. I am a scientist and I have not dismissed the 73 studies at all so your statement is not accurate. My personal analysis of the 73 studies is that collectively they provide overwhelming support for the efficacy of IVM combo therapy, especially when put in the context of real world outcomes in the nations and cities that have utilized IVM. Messy data is not the same as useless data. Dude, Bill GAtes is the biggest vaccine producer on earth and he finances our uiversities here in Belgium with Millions of dollars. Scientist are getting money. Big Pharma has made billions with the jab, it is not a conspiracy, i don\\'t underdetand how you can still disprove the fact that money is the priority and not health, especially after the National Medicament Agency of France announces 110 000 injuries due to the vaccine. Basically pretty much everything you said is false. \\nYou are in the science sub. Not the conspiracy fantasist sub. When did it change? Maybe 5-6 years ago? Not how science works. Pretty sure nobody here has tried it, or has gone through all the studies to confirm if it works or doesn\\'t.  The media and supposed \"experts\" say it does not work without looking at studies or pointing to any study that says otherwise.  Weird how they just say \"it does not work\" and you believe it.  People need to learn to do their own homework and research studies instead of flat out believing what \"an expert\" says.  Ever check who the expert is or why it makes them an expert of their name is even mentioned?  I doubt 99% of people don\\'t.  I know I used to believe most things, but after a while I started wondering and did my own homework.  To each their own of course and hopefully everyone stays healthy in the end. Don’t see why that is largely impossible.  Placebo studies would prove a negative. You can with some things, but generally, it isn\\'t realistic to do so. You can show evidence for something existing, but it is far more difficult to show evidence for something not existing. IE- If someone believes that unicorns exist, and demands evidence that they do not exist, what evidence will you show them? The lack of unicorn fossils in the fossil record? The absence of evidence is not evidence of absence. Is this article also bunk? If so, I\\'d be interested to know how to determine that.\\n\\n[https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8088823/](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8088823/) Nope. KU leuven accepted money from him. And he is officially the biggest vaccinator on eart with his African campaign. And those injuries are official reports from National Medicament Agency of France. And of course you also have VAERS showing us how many deaths we already have with this ineffective vaccine. It is not working, and you have to admit it. We failed, especially with Omicron. It makes no wense to vaccinated with boosters. Bill gates is a vaccine producer. And no, neither the NMA or vaers can be used to establish causality from the reports. It literally states it in the website. you either didn’t see this or deliberately ignored it. \\nThe vaccine is also working at reducing symptoms and deaths hence why the unvaccinated are dying at higher rates a fact which YOU have to admit. \\n\\nYour conspiracy still doesn’t stand up. Does Bill Gates somehow pay every scientist in the world? Fund every university? You’d also have to evidence that the science is somehow wrong and not just because “profit” Ivermectin in dirt cheap, and the pfizer vaccine is about 12.50$USD Nonsense. You simply take away the ability to speak in favor of something like Ivermectin on any and all.social media platforms under the threat of banishment and effectively \"cancel\" anyone and everyone who so much as suggests it could be a viable treatment.(which is precisely what occurred). \\nPlay all the mental gymnastics you want to, a concerted effort was and still is being undertaken to eliminate control groups for drugs like Ivermectin and also those who refuse vaccines.'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments.iloc[2]['body']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(posts, comments, on='link_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0_x</th>\n",
       "      <th>link_id</th>\n",
       "      <th>title</th>\n",
       "      <th>Unnamed: 0_y</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>t3_rt7fnz</td>\n",
       "      <td>Survey of entity encounter experiences occasio...</td>\n",
       "      <td>21</td>\n",
       "      <td>David Nutt , U.K.’s top drugs scientist / rese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>t3_rta2tk</td>\n",
       "      <td>tIny_DE Wünscht dir ein gesundes Neues Jahr ! ...</td>\n",
       "      <td>26</td>\n",
       "      <td>Your submission has been automatically removed...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>t3_rtc9oa</td>\n",
       "      <td>Patients hospitalized with Covid-19 who had a ...</td>\n",
       "      <td>29</td>\n",
       "      <td>So, if you’re unhealthy, you’re more likely to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>t3_rtcc76</td>\n",
       "      <td>Thanks to a symbiotic fungus, many species of ...</td>\n",
       "      <td>47</td>\n",
       "      <td>Key quote:\\n\\n&amp;gt; We have known a lot about t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>t3_rtcorj</td>\n",
       "      <td>New findings suggest an unintended benefit fro...</td>\n",
       "      <td>36</td>\n",
       "      <td>Health care shouldn’t be through a job. It’s t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13172</th>\n",
       "      <td>2163</td>\n",
       "      <td>t3_xsbpvz</td>\n",
       "      <td>New study shows higher rate of fractures in pe...</td>\n",
       "      <td>1479</td>\n",
       "      <td>Intellectual disability for folks who don’t wa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13173</th>\n",
       "      <td>2164</td>\n",
       "      <td>t3_xscog2</td>\n",
       "      <td>Research on sexual and social expectations of ...</td>\n",
       "      <td>1480</td>\n",
       "      <td>Abstract\\r  \\nIf nipple erection signals/is a ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13174</th>\n",
       "      <td>2166</td>\n",
       "      <td>t3_xsf7w6</td>\n",
       "      <td>Traumatic brain injury ‘remains a major global...</td>\n",
       "      <td>1481</td>\n",
       "      <td>This is why it should be mandatory to wear a h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13175</th>\n",
       "      <td>2167</td>\n",
       "      <td>t3_xsf9lh</td>\n",
       "      <td>Our cities are warming and urban greenery coul...</td>\n",
       "      <td>1482</td>\n",
       "      <td>Would be super cool if we used plants that pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13176</th>\n",
       "      <td>2169</td>\n",
       "      <td>t3_xsflwa</td>\n",
       "      <td>“Our results state, resoundingly, that the mos...</td>\n",
       "      <td>1483</td>\n",
       "      <td>Papers on pre-print services such as papers.ss...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13177 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0_x    link_id   \n",
       "0                 0  t3_rt7fnz  \\\n",
       "1                 2  t3_rta2tk   \n",
       "2                 5  t3_rtc9oa   \n",
       "3                 6  t3_rtcc76   \n",
       "4                 7  t3_rtcorj   \n",
       "...             ...        ...   \n",
       "13172          2163  t3_xsbpvz   \n",
       "13173          2164  t3_xscog2   \n",
       "13174          2166  t3_xsf7w6   \n",
       "13175          2167  t3_xsf9lh   \n",
       "13176          2169  t3_xsflwa   \n",
       "\n",
       "                                                   title  Unnamed: 0_y   \n",
       "0      Survey of entity encounter experiences occasio...            21  \\\n",
       "1      tIny_DE Wünscht dir ein gesundes Neues Jahr ! ...            26   \n",
       "2      Patients hospitalized with Covid-19 who had a ...            29   \n",
       "3      Thanks to a symbiotic fungus, many species of ...            47   \n",
       "4      New findings suggest an unintended benefit fro...            36   \n",
       "...                                                  ...           ...   \n",
       "13172  New study shows higher rate of fractures in pe...          1479   \n",
       "13173  Research on sexual and social expectations of ...          1480   \n",
       "13174  Traumatic brain injury ‘remains a major global...          1481   \n",
       "13175  Our cities are warming and urban greenery coul...          1482   \n",
       "13176  “Our results state, resoundingly, that the mos...          1483   \n",
       "\n",
       "                                                    body  \n",
       "0      David Nutt , U.K.’s top drugs scientist / rese...  \n",
       "1      Your submission has been automatically removed...  \n",
       "2      So, if you’re unhealthy, you’re more likely to...  \n",
       "3      Key quote:\\n\\n&gt; We have known a lot about t...  \n",
       "4      Health care shouldn’t be through a job. It’s t...  \n",
       "...                                                  ...  \n",
       "13172  Intellectual disability for folks who don’t wa...  \n",
       "13173  Abstract\\r  \\nIf nipple erection signals/is a ...  \n",
       "13174  This is why it should be mandatory to wear a h...  \n",
       "13175  Would be super cool if we used plants that pro...  \n",
       "13176  Papers on pre-print services such as papers.ss...  \n",
       "\n",
       "[13177 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 13177 entries, 0 to 13176\n",
      "Data columns (total 5 columns):\n",
      " #   Column        Non-Null Count  Dtype \n",
      "---  ------        --------------  ----- \n",
      " 0   Unnamed: 0_x  13177 non-null  int64 \n",
      " 1   link_id       13177 non-null  object\n",
      " 2   title         13177 non-null  object\n",
      " 3   Unnamed: 0_y  13177 non-null  int64 \n",
      " 4   body          13177 non-null  object\n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 514.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = df[['title', 'body']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('merged.csv', index=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8223\n"
     ]
    }
   ],
   "source": [
    "unmatch = []\n",
    "for i in range(len(posts)):\n",
    "    flag = False\n",
    "    for j in range(len(comments)):\n",
    "        if posts.iloc[i]['link_id'] == comments.iloc[j]['link_id']:\n",
    "            flag = True\n",
    "            break\n",
    "    if flag == False:\n",
    "        unmatch.append(posts.iloc[i])\n",
    "print(len(unmatch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "unmatch_df = pd.DataFrame(unmatch)\n",
    "unmatch_df.to_csv('unmatch_df.csv', index=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Energy at Origins: Favorable Thermodynamics of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Evolution of brilliant iridescent feather nano...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cortical thickness abnormalities in patients w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Scientists Tested Cats With Scooped Out Eyebal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>We continue to evolve: more and more human bei...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21389</th>\n",
       "      <td>For marriages formed after 1975, husbands’ lac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21391</th>\n",
       "      <td>The link between depression and meatless diets</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21392</th>\n",
       "      <td>Most Twitter users don’t follow political elit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21395</th>\n",
       "      <td>A majority of Twitter users do not follow eith...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21398</th>\n",
       "      <td>Concussions are associated with 60% increase i...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8223 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    data\n",
       "1      Energy at Origins: Favorable Thermodynamics of...\n",
       "3      Evolution of brilliant iridescent feather nano...\n",
       "4      Cortical thickness abnormalities in patients w...\n",
       "9      Scientists Tested Cats With Scooped Out Eyebal...\n",
       "10     We continue to evolve: more and more human bei...\n",
       "...                                                  ...\n",
       "21389  For marriages formed after 1975, husbands’ lac...\n",
       "21391     The link between depression and meatless diets\n",
       "21392  Most Twitter users don’t follow political elit...\n",
       "21395  A majority of Twitter users do not follow eith...\n",
       "21398  Concussions are associated with 60% increase i...\n",
       "\n",
       "[8223 rows x 1 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = unmatch_df['title']\n",
    "df1 = pd.DataFrame(df1)\n",
    "df1.rename(columns = {'title':'data'}, inplace = True)\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13177"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.read_csv('merged.csv')\n",
    "df2 = df2[['title', 'body']]\n",
    "df2[\"data\"] = df2['title'].astype(str) +\" \"+ df2[\"body\"]\n",
    "df2 = df2[['data']]\n",
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Energy at Origins: Favorable Thermodynamics of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Evolution of brilliant iridescent feather nano...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cortical thickness abnormalities in patients w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Scientists Tested Cats With Scooped Out Eyebal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>We continue to evolve: more and more human bei...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13172</th>\n",
       "      <td>New study shows higher rate of fractures in pe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13173</th>\n",
       "      <td>Research on sexual and social expectations of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13174</th>\n",
       "      <td>Traumatic brain injury ‘remains a major global...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13175</th>\n",
       "      <td>Our cities are warming and urban greenery coul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13176</th>\n",
       "      <td>“Our results state, resoundingly, that the mos...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21400 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    data\n",
       "1      Energy at Origins: Favorable Thermodynamics of...\n",
       "3      Evolution of brilliant iridescent feather nano...\n",
       "4      Cortical thickness abnormalities in patients w...\n",
       "9      Scientists Tested Cats With Scooped Out Eyebal...\n",
       "10     We continue to evolve: more and more human bei...\n",
       "...                                                  ...\n",
       "13172  New study shows higher rate of fractures in pe...\n",
       "13173  Research on sexual and social expectations of ...\n",
       "13174  Traumatic brain injury ‘remains a major global...\n",
       "13175  Our cities are warming and urban greenery coul...\n",
       "13176  “Our results state, resoundingly, that the mos...\n",
       "\n",
       "[21400 rows x 1 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames = [df1, df2]\n",
    "result = pd.concat(frames)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['data'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(\"data_total.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = pd.read_csv(r'D:\\linkid_title_total.csv')\n",
    "comments = pd.read_csv(r'D:\\linkid_body_total.csv')\n",
    "df = pd.merge(posts, comments, on='link_id')\n",
    "df = df[['title', 'body']]\n",
    "df[\"data\"] = df['title'].astype(str) +\" \"+ df[\"body\"]\n",
    "df = df[['data']]\n",
    "df.to_csv(\"data_total.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}