changed arvin date parsing to use lib (fixes #75) (#83)

src/sites/arvin.py

@@ -1,7 +1,9 @@
-from datetime import datetime
 import logging
 import re
+from datetime import datetime
+
 from bs4 import BeautifulSoup
+import dateparser
 
 from db.models import Victim
 from net.proxy import Proxy
@@ -13,22 +15,25 @@ class Arvin(SiteCrawler):
 
     def _handle_page(self, body: str):
         soup = BeautifulSoup(body, "html.parser")
-        victim_list = soup.find_all("article", {"id" : re.compile('post.*')})
+        victim_list = soup.find_all("article", {"id": re.compile("post.*")})
 
         for victim in victim_list:
             victim_name = victim.find("h2", class_="type-list-title").text.strip()
             victim_leak_site = victim.find("h2", class_="type-list-title").find("a").attrs["href"]
+
             published = victim.find("div", class_="type-list-date").text.strip()
-            published_dt = datetime.strptime(published, "%d. %B %Y")
 
+            published_dt = dateparser.parse(published)
+            if published_dt is None and len(published) > 0:
+                logging.warning(f"couldn't parse timestamp: {published}")
 
             q = self.session.query(Victim).filter_by(
                 url=victim_leak_site, site=self.site)
 
             if q.count() == 0:
                 # new victim
                 v = Victim(name=victim_name, url=victim_leak_site, published=published_dt,
-                            first_seen=datetime.utcnow(), last_seen=datetime.utcnow(), site=self.site)
+                           first_seen=datetime.utcnow(), last_seen=datetime.utcnow(), site=self.site)
                 self.session.add(v)
                 self.new_victims.append(v)
             else:
@@ -40,7 +45,6 @@ def _handle_page(self, body: str):
             self.current_victims.append(v)
         self.session.commit()
 
-
     def scrape_victims(self):
         with Proxy() as p:
             r = p.get(f"{self.url}", headers=self.headers)
@@ -49,15 +53,14 @@ def scrape_victims(self):
             # find all pages
             page_nav = soup.find_all("a", class_="page-numbers")
 
-
             site_list = []
             site_list.append(self.url)
-            
+
             for page in page_nav:
                 # might exist repetition
                 if page.attrs["href"] not in site_list:
                     site_list.append(page.attrs["href"])
-            
+
             for site in site_list:
                 r = p.get(site, headers=self.headers)
-                self._handle_page(r.content.decode()) 
+                self._handle_page(r.content.decode())