fix content type

hon9kon9ize · Mar 9, 2025 · 3de71c9 · 3de71c9
1 parent 4cd2682
commit 3de71c9
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 5 deletions.
diff --git a/scraper/api_scraper.py b/scraper/api_scraper.py
@@ -103,6 +103,7 @@ def parse_article(self, item: dict) -> ScraperOutput:
             article_id,
             title,
             category=self.category,
+            content_type=self.content_type,
             content=content,
             author=author,
             date=date,

diff --git a/scraper/html_scraper.py b/scraper/html_scraper.py
@@ -120,6 +120,7 @@ def parse_article(self, tag: "ResultSet[Tag]") -> ScraperOutput:
             article_id,
             title,
             category=self.category,
+            content_type=self.content_type,
             content=content,
             author=author,
             date=date,

diff --git a/scraper/rss_scraper.py b/scraper/rss_scraper.py
@@ -85,6 +85,7 @@ def parse_article(self, item: Dict[str, Any]) -> ScraperOutput:
             article_id,
             title,
             category=self.category,
+            content_type=self.content_type,
             content=content,
             author=author,
             date=date,

diff --git a/scraper/scraper.py b/scraper/scraper.py
@@ -135,11 +135,6 @@ async def get_articles(self) -> List[ScraperOutput]:
         # filter out None values
         articles = [article for article in articles if article is not None]
 
-        # add category and content type
-        for article in articles:
-            article.category = self.category
-            article.content_type = self.content_type
-
         return articles
 
     async def fetch_article(self, item: Any) -> Any: