Skip to content

Commit

Permalink
workflow: merge updates. Closes inspirehep#2294
Browse files Browse the repository at this point in the history
Signed-off-by: Antonio Cesarano <[email protected]>
Signed-off-by: Riccardo Candido <[email protected]>
  • Loading branch information
ammirate committed Sep 15, 2017
1 parent c9ed0b2 commit 202d6f6
Show file tree
Hide file tree
Showing 12 changed files with 24,678 additions and 150 deletions.
3 changes: 1 addition & 2 deletions inspirehep/modules/workflows/actions/hep_approval.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.

"""Approval action for INSPIRE arXiv harvesting."""
"""Merge action for INSPIRE."""

from __future__ import absolute_import, division, print_function

Expand Down Expand Up @@ -53,7 +53,6 @@ def resolve(obj, *args, **kwargs):
approved = value in ('accept', 'accept_core')

obj.extra_data["approved"] = approved
obj.remove_action()

obj.extra_data["user_action"] = value
obj.extra_data["core"] = value == "accept_core"
Expand Down
15 changes: 13 additions & 2 deletions inspirehep/modules/workflows/actions/merge_approval.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,27 @@
# granted to it by virtue of its status as an Intergovernmental Organization
# or submit itself to any jurisdiction.

"""Merge action for INSPIRE."""
"""Approval action for INSPIRE arXiv harvesting."""

from __future__ import absolute_import, division, print_function


class MergeApproval(object):
"""Class representing the merge action."""
"""Class representing the approval action."""
name = "Merge records"

@staticmethod
def resolve(obj, *args, **kwargs):
"""Resolve the action taken in the approval action."""

obj.extra_data["approved"] = True

obj.extra_data["conflicts_solved_by_curator"] = True

obj.remove_action()

obj.save()

obj.continue_workflow(delayed=True)

return True
118 changes: 73 additions & 45 deletions inspirehep/modules/workflows/workflows/article.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from workflow.patterns.controlflow import (
IF,
IF_ELSE,
IF_NOT,
)

from inspire_dojson.hep import hep2marc
Expand All @@ -43,7 +42,6 @@
add_core,
halt_record,
is_record_relevant,
in_production_mode,
is_record_accepted,
reject_record,
is_experimental_paper,
Expand All @@ -55,6 +53,14 @@
submission_fulltext_download,
)


from inspirehep.modules.workflows.tasks.merging import (
merge_articles,
put_root_in_extra_data,
update_record,
has_conflicts,
store_root,
)
from inspirehep.modules.workflows.tasks.classifier import (
classify_paper,
filter_core_keywords,
Expand Down Expand Up @@ -203,21 +209,6 @@
]


CHECK_IF_SUBMISSION_AND_ASK_FOR_APPROVAL = [
IF_ELSE(
is_record_relevant,
[halt_record(
action="hep_approval",
message="Submission halted for curator approval.",
)],
[
reject_record("Article automatically rejected"),
stop_processing
]
),
]


NOTIFY_NOT_ACCEPTED = [
IF(
is_submission,
Expand All @@ -237,6 +228,7 @@
context_factory=reply_ticket_context
),
close_ticket(ticket_id_key="ticket_id"),
mark('stop', True)
]


Expand Down Expand Up @@ -280,8 +272,7 @@
[
send_robotupload(
marcxml_processor=hep2marc,
mode="correct",
extra_data_key="update_payload"
mode="replace",
),
], [
send_robotupload(
Expand All @@ -293,22 +284,6 @@
),
]

CHECK_IF_MERGE_AND_STOP_IF_SO = [
IF(
article_exists,
[
IF_ELSE(
is_submission,
NOTIFY_ALREADY_EXISTING,
[
# halt_record(action="merge_approval"),
delete_self_and_stop_processing,
]
),
]
)
]


ADD_MARKS = [
# Query locally or via legacy search API to see if article
Expand All @@ -331,6 +306,65 @@
]


MERGE_IF_UPDATE = [
put_root_in_extra_data,
IF(
article_exists,
[
merge_articles,
mark('merged', True)
# TODO: save record with new non-conflicting merged fields
],
),
]


STOP_IF_EXISTING_SUBMISSION = [
IF(
is_submission,
IF(
article_exists,
NOTIFY_ALREADY_EXISTING
)
)
]


HALT_FOR_APPROVAL = [
IF_ELSE(
is_record_relevant,
[
IF_ELSE(
article_exists,
halt_record(
action="merge_approval",
message="Submission halted for curator approval.",
),
halt_record(
action="hep_approval",
message="Submission halted for curator approval.",
),
)
],
# record not relevant
[
reject_record("Article automatically rejected"),
stop_processing
]
)
]


STORE_RECORD_AND_ROOT = [
IF_ELSE(
article_exists,
update_record,
store_record
),
store_root,
]


class Article(object):
"""Article ingestion workflow for Literature collection."""
name = "HEP"
Expand All @@ -344,23 +378,17 @@ class Article(object):
ADD_MARKS +
DELETE_AND_STOP_IF_NEEDED +
ENHANCE_RECORD +
# TODO: Once we have a way to resolve merges, we should
# use that instead of stopping
CHECK_IF_MERGE_AND_STOP_IF_SO +
CHECK_IF_SUBMISSION_AND_ASK_FOR_APPROVAL +
STOP_IF_EXISTING_SUBMISSION +
MERGE_IF_UPDATE +
HALT_FOR_APPROVAL +
[
IF_ELSE(
is_record_accepted,
(
POSTENHANCE_RECORD +
SEND_TO_LEGACY_AND_WAIT +
NOTIFY_USER_OR_CURATOR +
[
# TODO: once legacy is out, this should become
# unconditional, and remove the SEND_TO_LEGACY_AND_WAIT
# steps
IF_NOT(in_production_mode, [store_record]),
]
STORE_RECORD_AND_ROOT
),
NOTIFY_NOT_ACCEPTED,
),
Expand Down
94 changes: 70 additions & 24 deletions tests/integration/workflows/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,40 +26,19 @@

import os
import pytest
import re
import requests_mock
import sys

from invenio_db import db
from invenio_workflows import (
workflow_object_class,
)

from invenio_db import db
from inspirehep.factory import create_app
from inspirehep.modules.workflows.models import (
WorkflowsAudit,
WorkflowsPendingRecord,
)


# Use the helpers folder to store test helpers.
# See: http://stackoverflow.com/a/33515264/374865
sys.path.append(os.path.join(os.path.dirname(__file__), 'helpers'))


@pytest.fixture(autouse=True)
def cleanup_workflows_tables(small_app):
with small_app.app_context():
obj_types = (
WorkflowsAudit.query.all(),
WorkflowsPendingRecord.query.all(),
workflow_object_class.query(),
)
for obj_type in obj_types:
for obj in obj_type:
obj.delete()

db.session.commit()


@pytest.fixture
def workflow_app():
app = create_app(
Expand All @@ -74,8 +53,75 @@ def workflow_app():
'http://localhost:1234'
),
MAGPIE_API_URL="http://example.com/magpie",
WORKFLOWS_MATCH_REMOTE_SERVER_URL="http://legacy_search.endpoint/",
WTF_CSRF_ENABLED=False,
)

with app.app_context():
yield app


def drop_all(app):
db.drop_all()
_es = app.extensions['invenio-search']
list(_es.delete(ignore=[404]))


def create_all(app):
from inspirehep.modules.fixtures.collections import init_collections
from inspirehep.modules.fixtures.files import init_all_storage_paths
from inspirehep.modules.fixtures.users import init_users_and_permissions

db.create_all()
_es = app.extensions['invenio-search']
list(_es.create(ignore=[400]))

init_all_storage_paths()
init_users_and_permissions()
init_collections()


@pytest.fixture(autouse=True)
def cleanup_workflows(workflow_app):
db.session.close_all()
drop_all(app=workflow_app)
create_all(app=workflow_app)


@pytest.fixture
def mocked_external_services(workflow_app):
with requests_mock.Mocker() as requests_mocker:
requests_mocker.register_uri(
requests_mock.ANY,
re.compile('.*(indexer|localhost).*'),
real_http=True,
)
requests_mocker.register_uri(
'POST',
re.compile(
'https?://localhost:1234.*',
),
text=u'[INFO]',
status_code=200,
)
requests_mocker.register_uri(
requests_mock.ANY,
re.compile(
'.*' +
workflow_app.config['WORKFLOWS_MATCH_REMOTE_SERVER_URL'] +
'.*'
),
status_code=200,
json=[],
)
requests_mocker.register_uri(
requests_mock.ANY,
re.compile(
'.*' +
workflow_app.config['BEARD_API_URL'] +
'/text/phonetic_blocks.*'
),
status_code=200,
json={'phonetic_blocks': {}},
)
yield
Loading

0 comments on commit 202d6f6

Please sign in to comment.