add abstract

citation_crawler/crawlers/ss.py

@@ -108,6 +108,10 @@ def doi(self) -> Optional[str]:
             doi = re.sub(r"^/+", "", u.path)
             return doi
 
+    def abstract(self) -> Optional[int]:
+        if 'abstract' in self.data:
+            return self.data['abstract']
+
     async def _get_authors_from_author_data(self) -> Iterable[Author]:
         if not self.author_data:
             authors = []
@@ -146,7 +150,7 @@ async def get_citations(self) -> Iterable[Paper]:
             yield paper
 
 
-fields_references = f"title,year,authors,externalIds,publicationTypes,journal"
+fields_references = f"title,abstract,year,authors,externalIds,publicationTypes,journal"
 root_references = f"semanticscholar/references--{fields_references.replace(',', '-')}"
 root_citations = f"semanticscholar/citations--{fields_references.replace(',', '-')}"
 
@@ -192,7 +196,7 @@ def paper_is_valid(text):
 
 
 fields_authors_sub = ','.join([("authors." + f) for f in fields_authors.split(',')])
-fields_paper = f"title,year,publicationDate,{fields_authors_sub},externalIds,publicationTypes,journal"
+fields_paper = f"title,abstract,year,publicationDate,{fields_authors_sub},externalIds,publicationTypes,journal"
 root_paper = f"semanticscholar/paper--{fields_paper.replace(',', '-')}"
 
 

citation_crawler/items.py

@@ -58,6 +58,10 @@ def date(self) -> Optional[str]:
     def doi(self) -> Optional[str]:
         return None
 
+    @abc.abstractmethod
+    def abstract(self) -> Optional[str]:
+        return None
+
     @abc.abstractmethod
     async def authors(self) -> Iterable[Author]:
         return
@@ -91,6 +95,8 @@ async def __dict__(self) -> dict:
             d['date'] = self.date()
         if self.doi():
             d['doi'] = self.doi()
+        if self.abstract():
+            d['abstract'] = self.abstract()
         d['authors'] = []
         async for author in self.authors():
             d['authors'].append(author.__dict__())

citation_crawler/summarizers/neo4j.py

@@ -16,6 +16,8 @@ async def add_paper(tx, paper: Paper):
         "SET p.title=$title, p.year=$year"
     if paper.doi():
         n4jset += ", p.doi=$doi"
+    if paper.abstract():
+        n4jset += ", p.abstract=$abstract"
     if paper.dblp_id():
         n4jset += ", p.dblp_key=$dblp_id"
     if paper.paperId():
@@ -39,6 +41,7 @@ async def add_paper(tx, paper: Paper):
                  paperId=paper.paperId(),
                  dblp_id=paper.dblp_id(),
                  doi=paper.doi(),
+                 abstract=paper.abstract(),
                  date=date)
 
 

setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name='citation_crawler',
-    version='2.8.2',
+    version='2.8.3',
     author='yindaheng98',
     author_email='[email protected]',
     url='https://github.com/yindaheng98/citation-crawler',