Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MODSOURCE-824: Endpoint /batch/parsed-records/fetch does not return deleted records #655

Merged
merged 7 commits into from
Nov 26, 2024
Merged

MODSOURCE-824: Endpoint /batch/parsed-records/fetch does not return deleted records #655

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
86a863a
MODSOURCE-824: Implementation done.
VRohach Nov 21, 2024
a0e2849
MODSOURCE-824: Tests added+news updated.
VRohach Nov 21, 2024
aafc300
MODSOURCE-824: raml-storage updated.
VRohach Nov 26, 2024
b19740d
Merge branch 'master' of https://github.com/folio-org/mod-source-reco…
VRohach Nov 26, 2024
0e60ddc
MODSOURCE-824: NEWS updated[2]
VRohach Nov 26, 2024
4cbc54f
MODSOURCE-824: raml-storage updated.
VRohach Nov 26, 2024
42fae4b
MODSOURCE-824: raml-storage updated[3].
VRohach Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
## 2024-xx-xx 5.10.0
## 2025-xx-xx 5.10.0-SNAPSHOT
* [MODSOURCE-816](https://folio-org.atlassian.net/browse/MODSOURCE-816) [RRT] Optimize execution plan for streaming SQL
* [MODSOURCE-824](https://folio-org.atlassian.net/browse/MODSOURCE-824) Endpoint /batch/parsed-records/fetch does not return deleted records

## 2024-10-28 5.9.0
* [MODSOURCE-767](https://folio-org.atlassian.net/browse/MODSOURCE-767) Single record overlay creates duplicate OCLC#/035
Expand Down
3 changes: 2 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,11 @@ public interface RecordDao {
* @param externalIds list of ids
* @param idType external id type on which source record will be searched
* @param recordType record type
* @param includeDeleted searching by deleted records
* @param tenantId tenant id
* @return {@link Future} of {@link StrippedParsedRecordCollection}
*/
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, String tenantId);
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, Boolean includeDeleted, String tenantId);

/**
* Searches for {@link Record} by {@link MatchField} with offset and limit
Expand Down
552 changes: 280 additions & 272 deletions mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,8 @@ public Future<StrippedParsedRecordCollection> fetchStrippedParsedRecords(FetchPa
}

var recordType = toRecordType(fetchRequest.getRecordType().name());
return recordDao.getStrippedParsedRecords(ids, idType, recordType, tenantId)
var includeDeleted = fetchRequest.getIncludeDeleted();
return recordDao.getStrippedParsedRecords(ids, idType, recordType, includeDeleted, tenantId)
.onComplete(records -> filterFieldsByDataRange(records, fetchRequest))
.onFailure(ex -> {
LOG.warn("fetchParsedRecords:: Failed to fetch parsed records. {}", ex.getMessage());
Expand Down
216 changes: 214 additions & 2 deletions ...ce-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageBatchApiTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,46 @@ public void shouldPostSourceStorageBatchEdifactRecords(TestContext testContext)
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
public void shouldPostFetchParsedRecordsBatchWithDeletedWhenIncludeDeleteTrue(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
Expand All @@ -243,7 +281,8 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
Expand All @@ -257,6 +296,147 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
async.complete();
}


@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteFalse(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(false);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteNotExists(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}


@Test
public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
Expand All @@ -282,6 +462,38 @@ public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()))
.body("totalRecords", is(externalIds.size()));
async.complete();
}

@Test
public void shouldFailWhenPostSourceStorageBatchRecordsWithMultipleSnapshots(TestContext testContext) {
Async async = testContext.async();
Expand Down
Loading
Loading