Skip to content

Commit

Permalink
fixed some syncing issues
Browse files Browse the repository at this point in the history
  • Loading branch information
tomlue committed Jul 11, 2024
1 parent 8eb806b commit 1ec1561
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 8 deletions.
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

setup(
name='sysrev',
version='1.3.10',
version='1.3.11',
description='get sysrev project data and use the sysrev api',
long_description=long_description,
long_description_content_type='text/markdown', # Specify the content type here
Expand Down
21 changes: 14 additions & 7 deletions sysrev/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,13 +215,18 @@ def sync_article_info(self, client:Client, project_id, article_ids):
auto_labels = pd.DataFrame([
{**{'article-id': a['article'].get('article-id'), 'label-id': label_id}, **details} for a in article_info
for label_id, details in a['article'].get('auto-labels', {}).items() ])
auto_labels['answer'] = auto_labels['answer'].apply(json.dumps)
auto_labels['answer'] = auto_labels.get('answer',pd.Series([])).apply(json.dumps)

csl_citations = pd.DataFrame([
{**{k: json.dumps(v) if isinstance(v, (dict, list)) else v for k, v in item['itemData'].items()},
'article-id': a['article'].get('article-id')}
for a in article_info for item in a['article'].get('csl-citation', {}).get('citationItems', [])])
csl_citations['issued'] = csl_citations['issued'].apply(json.dumps)
{
**{k: json.dumps(v) if isinstance(v, (dict, list)) else v for k, v in item['itemData'].items()},
'article-id': a['article'].get('article-id')
}
for a in article_info
for item in (a['article'].get('csl-citation') or {}).get('citationItems', [])
])
csl_citations = csl_citations.reindex(columns=['issued', 'author'], fill_value='')
csl_citations['issued'] = csl_citations.get('issued').apply(json.dumps)
csl_citations['author'] = csl_citations['author'].apply(json.dumps)

self.write_df(full_texts,'full_texts')
Expand All @@ -231,7 +236,8 @@ def sync_article_info(self, client:Client, project_id, article_ids):
def sync_labels(self, client, project_id):
labels = client.get_labels(project_id)
labels_df = pd.DataFrame(labels)
labels_df['definition'] = labels_df['definition'].apply(json.dumps)
# convert any dict column to json string
labels_df = labels_df.apply(lambda col: col.map(lambda x: json.dumps(x) if isinstance(x, dict) else x))
self.write_df(labels_df,'labels')

# TODO - this could be made more efficient by checking sqlite state and updating the sysrev api
Expand All @@ -257,7 +263,8 @@ def sync(self, client : Client, project_id):
article_data_df['resolve'] = article_data_df['resolve'].apply(json.dumps)
self.write_df(article_data_df,'article_data')

self.sync_article_info(client, project_id, article_data_df['article-id'])
article_ids = list(article_data_df.get('article_id',[]))
self.sync_article_info(client, project_id, article_ids)
self.sync_labels(client, project_id)


Expand Down

0 comments on commit 1ec1561

Please sign in to comment.