Skip to content

Commit

Permalink
just use info
Browse files Browse the repository at this point in the history
  • Loading branch information
yindaheng98 committed Apr 22, 2024
1 parent 06268fa commit 763b9dd
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 5 deletions.
3 changes: 2 additions & 1 deletion citation_crawler/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,8 @@ async def filter_papers(self, papers):
parser_n4j.add_argument("--username", type=str, default=None, help=f'Auth username to neo4j database.')
parser_n4j.add_argument("--password", type=str, default=None, help=f'Auth password to neo4j database.')
parser_n4j.add_argument("--uri", type=str, required=True, help=f'URI to neo4j database.')
parser_n4j.add_argument("--no-skip-exists", action="store_true", help=f'Do not skip exists references. Use it when you want to rewrite all papers.')
parser_n4j.add_argument("--no-skip-exists", action="store_true",
help=f'Do not skip exists references. Use it when you want to rewrite all papers.')


async def func_parser_n4j_async(parser):
Expand Down
6 changes: 3 additions & 3 deletions citation_crawler/crawlers/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,12 @@ async def download_item(url: str, path: str, cache_days: int, is_valid: Callable
async with file_sem:
try:
async with async_open(save_path, 'r') as f:
logger.debug("use cache: %s -> %s" % (path, url))
logger.info("use cache: %s -> %s" % (path, url))
text = await f.read()
assert is_valid(text)
return text
except:
logger.debug(" no cache: %s" % save_path)
logger.info(" no cache: %s" % save_path)
if os.path.exists(save_path):
logger.info("err cache: %s" % save_path)
try:
Expand All @@ -89,7 +89,7 @@ async def download_item(url: str, path: str, cache_days: int, is_valid: Callable
async with session.get(url,
proxy=os.getenv("HTTP_PROXY"),
timeout=os.getenv("HTTP_TIMEOUT") or 30) as response:
logger.debug(" download: %s <- %s" % (path, url))
logger.info(" download: %s <- %s" % (path, url))
text = await response.text()
assert is_valid(text)
os.makedirs(os.path.dirname(save_path), exist_ok=True)
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

setup(
name='citation_crawler',
version='2.9.2',
version='2.9.4',
author='yindaheng98',
author_email='[email protected]',
url='https://github.com/yindaheng98/citation-crawler',
Expand Down

0 comments on commit 763b9dd

Please sign in to comment.