Skip to content

Commit

Permalink
Merge remote-tracking branch 'refs/remotes/origin/master' into MODSOU…
Browse files Browse the repository at this point in the history 
…RCE-783

# Conflicts:
#	NEWS.md
  • Loading branch information
@dmytrokrutii
dmytrokrutii committed Jul 29, 2024
2 parents 5f41bff + 2aa801b commit df5923e
Show file tree
Hide file tree
Showing 6 changed files with 120 additions and 37 deletions.
2 changes: 2 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* [MODSOURCE-773](https://folio-org.atlassian.net/browse/MODSOURCE-773) MARC Search omits suppressed from discovery records in default search
* [MODINV-1044](https://folio-org.atlassian.net/browse/MODINV-1044) Additional Requirements - Update Data Import logic to normalize OCLC 035 values
* [MODSOURMAN-1200](https://folio-org.atlassian.net/browse/MODSOURMAN-1200) Find record by match id on update generation
* [MODINV-1049](https://folio-org.atlassian.net/browse/MODINV-1049) Existing "035" field is not retained the original position in imported record
* [MODSOURCE-785](https://folio-org.atlassian.net/browse/MODSOURCE-785) Update 005 field when set MARC for deletion
* [MODSOURMAN-783](https://folio-org.atlassian.net/browse/MODSOURCE-783) Extend MARC-MARC search query to account for qualifiers

## 2024-03-20 5.8.0
Expand Down
1 change: 0 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/dao/util/ParsedRecordDaoUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,5 +258,4 @@ public static JsonObject normalize(Object content) {
? new JsonObject((String) content)
: JsonObject.mapFrom(content);
}

}
13 changes: 13 additions & 0 deletions mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import static java.util.Objects.nonNull;
import static java.util.stream.Collectors.collectingAndThen;
import static java.util.stream.Collectors.toList;
import static org.folio.dao.util.MarcUtil.reorderMarcRecordFields;
import static org.folio.dao.util.RecordDaoUtil.RECORD_NOT_FOUND_TEMPLATE;
import static org.folio.dao.util.RecordDaoUtil.ensureRecordForeignKeys;
import static org.folio.dao.util.RecordDaoUtil.ensureRecordHasId;
Expand Down Expand Up @@ -55,6 +56,7 @@
import org.folio.rest.jaxrs.model.RecordMatchingDto;
import org.folio.rest.jaxrs.model.RecordsIdentifiersCollection;
import org.folio.services.exceptions.DuplicateRecordException;
import org.folio.services.util.AdditionalFieldsUtil;
import org.folio.services.util.TypeConnection;
import org.jooq.Condition;
import org.jooq.OrderField;
Expand Down Expand Up @@ -342,6 +344,7 @@ public Future<Void> deleteRecordById(String id, IdType idType, String tenantId)
return recordDao.getRecordByExternalId(id, idType, tenantId)
.map(recordOptional -> recordOptional.orElseThrow(() -> new NotFoundException(format(NOT_FOUND_MESSAGE, Record.class.getSimpleName(), id))))
.map(record -> {
update005field(record);
record.withState(Record.State.DELETED);
record.setAdditionalInfo(record.getAdditionalInfo().withSuppressDiscovery(true));
ParsedRecordDaoUtil.updateLeaderStatus(record.getParsedRecord(), DELETED_LEADER_RECORD_STATUS);
Expand Down Expand Up @@ -501,4 +504,14 @@ private Future<RecordsIdentifiersCollection> processDefaultMatchField(MatchField
.withIdentifiers(identifiers).withTotalRecords(recordCollection.getTotalRecords()))));
}

private static void update005field(Record targetRecord) {
if (targetRecord.getParsedRecord() != null && targetRecord.getParsedRecord().getContent() != null) {
AdditionalFieldsUtil.updateLatestTransactionDate(targetRecord);
var sourceContent = targetRecord.getParsedRecord().getContent().toString();
var targetContent = targetRecord.getParsedRecord().getContent().toString();
var content = reorderMarcRecordFields(sourceContent, targetContent);
targetRecord.getParsedRecord().setContent(content);
}
}

}
71 changes: 43 additions & 28 deletions ...rce-record-storage-server/src/main/java/org/folio/services/util/AdditionalFieldsUtil.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package org.folio.services.util;

import static java.lang.String.format;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.folio.dao.util.MarcUtil.reorderMarcRecordFields;

import com.github.benmanes.caffeine.cache.CacheLoader;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
Expand All @@ -8,9 +14,19 @@
import io.vertx.core.Vertx;
import io.vertx.core.json.Json;
import io.vertx.core.json.JsonObject;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ForkJoinPool;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
Expand All @@ -33,23 +49,6 @@
import org.marc4j.marc.Subfield;
import org.marc4j.marc.VariableField;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ForkJoinPool;

import static java.lang.String.format;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static org.folio.dao.util.MarcUtil.reorderMarcRecordFields;


/**
* Util to work with additional fields
Expand All @@ -58,6 +57,7 @@ public final class AdditionalFieldsUtil {

public static final String TAG_00X_PREFIX = "00";
public static final String TAG_005 = "005";
private static final String TAG_010 = "010";
public static final String TAG_999 = "999";
public static final String TAG_035 = "035";
public static final char TAG_035_SUB = 'a';
Expand Down Expand Up @@ -498,16 +498,14 @@ private static void updateOclcSubfield(Record recordForUpdate,
MarcFactory factory = MarcFactory.newInstance();
org.marc4j.marc.Record marcRecord = computeMarcRecord(recordForUpdate);
if (marcRecord != null) {
removeOclcField(marcRecord, TAG_035);

DataField dataField = factory.newDataField(TAG_035, TAG_035_IND, TAG_035_IND);

normalizedValues.forEach(value -> {
var v = value.split("&");
dataField.addSubfield(factory.newSubfield(v[0].charAt(0), v[1]));
});

addDataFieldInNumericalOrder(dataField, marcRecord);
replaceOclc035FieldWithNormalizedData(marcRecord, dataField);

// use stream writer to recalculate leader
streamWriter.write(marcRecord);
Expand Down Expand Up @@ -548,12 +546,21 @@ private static List<Subfield> get035oclcSubfields(VariableField field, char subf
return Collections.emptyList();
}

private static void removeOclcField(org.marc4j.marc.Record marcRecord, String fieldName) {
List<VariableField> variableFields = marcRecord.getVariableFields(fieldName);
private static void replaceOclc035FieldWithNormalizedData(org.marc4j.marc.Record marcRecord, DataField dataField) {
var variableFields = marcRecord.getVariableFields(TAG_035);
if (!variableFields.isEmpty()) {
variableFields.stream()
.filter(variableField -> variableField.find(OCLC))
.forEach(marcRecord::removeVariableField);

var dataFields = marcRecord.getDataFields();
for (int i = 0; i < dataFields.size(); i++) {
if (dataFields.get(i).getTag().equals(TAG_010)) {
marcRecord.getDataFields().add(i + 1, dataField);
return;
}
}
addDataFieldInNumericalOrder(dataField, marcRecord);
}
}

Expand Down Expand Up @@ -616,17 +623,25 @@ private static VariableField getSingleFieldByIndicators(List<VariableField> list
/**
* Updates field 005 for case when this field is not protected.
*
* @param record record to update
* @param targetRecord record to update
* @param mappingParameters mapping parameters
*/
public static void updateLatestTransactionDate(Record record, MappingParameters mappingParameters) {
if (isField005NeedToUpdate(record, mappingParameters)) {
public static void updateLatestTransactionDate(Record targetRecord, MappingParameters mappingParameters) {
if (isField005NeedToUpdate(targetRecord, mappingParameters)) {
updateLatestTransactionDate(targetRecord);
}
}

/**
* Updates field 005.
* @param targetRecord record to update
*/
public static void updateLatestTransactionDate(Record targetRecord) {
String date = AdditionalFieldsUtil.dateTime005Formatter.format(ZonedDateTime.ofInstant(Instant.now(), ZoneId.systemDefault()));
boolean isLatestTransactionDateUpdated = AdditionalFieldsUtil.addControlledFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_005, date, true);
boolean isLatestTransactionDateUpdated = AdditionalFieldsUtil.addControlledFieldToMarcRecord(targetRecord, AdditionalFieldsUtil.TAG_005, date, true);
if (!isLatestTransactionDateUpdated) {
throw new PostProcessingException(format("Failed to update field '005' to record with id '%s'", record.getId()));
throw new PostProcessingException(format("Failed to update field '005' to record with id '%s'", targetRecord.getId()));
}
}
}

/**
Expand Down
16 changes: 14 additions & 2 deletions mod-source-record-storage-server/src/test/java/org/folio/rest/impl/RecordApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import static org.hamcrest.Matchers.nullValue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.UUID;

Expand Down Expand Up @@ -832,15 +834,18 @@ public void shouldDeleteExistingMarcRecordOnDeleteByRecordId(TestContext testCon
}

@Test
public void shouldDeleteExistingMarcRecordOnDeleteByInstanceId(TestContext testContext) {
public void shouldDeleteExistingMarcRecordOnDeleteByInstanceIdAndUpdate005FieldWithCurrentDate(TestContext testContext) {
postSnapshots(testContext, snapshot_1);

String srsId = UUID.randomUUID().toString();
String instanceId = UUID.randomUUID().toString();

String currentDate = "20240718132044.6";
ParsedRecord parsedRecord = new ParsedRecord().withId(srsId)
.withContent(new JsonObject().put("leader", "01542ccm a2200361 4500")
.put("fields", new JsonArray().add(new JsonObject().put("999", new JsonObject()
.put("fields", new JsonArray()
.add(new JsonObject().put("005", currentDate))
.add(new JsonObject().put("999", new JsonObject()
.put("subfields", new JsonArray().add(new JsonObject().put("s", srsId)).add(new JsonObject().put("i", instanceId)))))));

Record newRecord = new Record()
Expand Down Expand Up @@ -887,6 +892,13 @@ public void shouldDeleteExistingMarcRecordOnDeleteByInstanceId(TestContext testC
Assert.assertEquals(true, deletedRecord.getAdditionalInfo().getSuppressDiscovery());
Assert.assertEquals("d", ParsedRecordDaoUtil.getLeaderStatus(deletedRecord.getParsedRecord()));

//Complex verifying "005" field is NOT empty inside parsed record.
LinkedHashMap<String, ArrayList<LinkedHashMap<String, String>>> content = (LinkedHashMap<String, ArrayList<LinkedHashMap<String, String>>>) deletedRecord.getParsedRecord().getContent();
LinkedHashMap<String, String> map = content.get("fields").get(0);
String resulted005FieldValue = map.get("005");
Assert.assertNotNull(resulted005FieldValue);
Assert.assertNotEquals(currentDate, resulted005FieldValue);

async.complete();
}

Expand Down
54 changes: 48 additions & 6 deletions ...urce-record-storage-server/src/test/java/org/folio/services/AdditionalFieldsUtilTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,26 @@
package org.folio.services;

import static org.folio.services.util.AdditionalFieldsUtil.*;
import static org.folio.services.util.AdditionalFieldsUtil.TAG_035;
import static org.folio.services.util.AdditionalFieldsUtil.TAG_035_SUB;
import static org.folio.services.util.AdditionalFieldsUtil.addControlledFieldToMarcRecord;
import static org.folio.services.util.AdditionalFieldsUtil.addDataFieldToMarcRecord;
import static org.folio.services.util.AdditionalFieldsUtil.addFieldToMarcRecord;
import static org.folio.services.util.AdditionalFieldsUtil.get035SubfieldOclcValues;
import static org.folio.services.util.AdditionalFieldsUtil.getCacheStats;
import static org.folio.services.util.AdditionalFieldsUtil.getValueFromControlledField;
import static org.folio.services.util.AdditionalFieldsUtil.isFieldExist;
import static org.folio.services.util.AdditionalFieldsUtil.removeField;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.junit.Assert.assertFalse;
import static org.mockito.ArgumentMatchers.any;

import com.github.benmanes.caffeine.cache.stats.CacheStats;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import com.github.benmanes.caffeine.cache.stats.CacheStats;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.folio.TestUtil;
Expand All @@ -23,9 +33,6 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.BlockJUnit4ClassRunner;

import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import org.marc4j.marc.Subfield;

@RunWith(BlockJUnit4ClassRunner.class)
Expand Down Expand Up @@ -381,6 +388,41 @@ public void shouldRemovePeriodsAndSpacesAfterNormalization() {
Assert.assertEquals(expectedParsedContent, parsedRecord.getContent());
}

@Test
public void shouldPreserveOrderOf035FieldsAfterNormalization() {
// given
var parsedContent = "{\"leader\":\"00198cama 22003611a 4500\",\"fields\":[" +
"{\"001\":\"10065352\"}," +
"{\"005\":\"20220127143948.0\"}," +
"{\"008\":\"761216s1853mauch0010eng\"}," +
"{\"906\":{\"subfields\":[{\"a\":\"7\"},{\"b\":\"cbc\"},{\"c\":\"oclcrpl\"},{\"d\":\"u\"},{\"e\":\"ncip\"},{\"f\":\"19\"},{\"g\":\"y-gencatlg\"}],\"ind1\":\"\",\"ind2\":\"\"}}," +
"{\"035\":{\"subfields\":[{\"9\":\"(DLC)01012052\"}],\"ind1\":\"\",\"ind2\":\"\"}}," +
"{\"010\":{\"subfields\":[{\"a\":\"01012052\"}],\"ind1\":\"\",\"ind2\":\"\"}}," +
"{\"035\":{\"subfields\":[{\"a\":\"(OCoLC)2628488\"}],\"ind1\":\"\",\"ind2\":\"\"}}," +
"{\"040\":{\"subfields\":[{\"a\":\"DLC\"},{\"b\":\"eng\"},{\"c\":\"O\"},{\"d\":\"O\"},{\"d\":\"DLC\"}],\"ind1\":\"\",\"ind2\":\"\"}}]}";

var expectedParsedContent = "{\"leader\":\"00291cama 22001211a 4500\",\"fields\":[" +
"{\"001\":\"10065352\"}," +
"{\"005\":\"20220127143948.0\"}," +
"{\"008\":\"761216s1853mauch0010eng\"}," +
"{\"906\":{\"subfields\":[{\"a\":\"7\"},{\"b\":\"cbc\"},{\"c\":\"oclcrpl\"},{\"d\":\"u\"},{\"e\":\"ncip\"},{\"f\":\"19\"},{\"g\":\"y-gencatlg\"}],\"ind1\":\" \",\"ind2\":\" \"}}," +
"{\"035\":{\"subfields\":[{\"9\":\"(DLC)01012052\"}],\"ind1\":\" \",\"ind2\":\" \"}}," +
"{\"010\":{\"subfields\":[{\"a\":\"01012052\"}],\"ind1\":\" \",\"ind2\":\" \"}}," +
"{\"035\":{\"subfields\":[{\"a\":\"(OCoLC)2628488\"}],\"ind1\":\" \",\"ind2\":\" \"}}," +
"{\"040\":{\"subfields\":[{\"a\":\"DLC\"},{\"b\":\"eng\"},{\"c\":\"O\"},{\"d\":\"O\"},{\"d\":\"DLC\"}],\"ind1\":\" \",\"ind2\":\" \"}}]}";

ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedContent);

Record record = new Record().withId(UUID.randomUUID().toString())
.withParsedRecord(parsedRecord)
.withGeneration(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder().withInstanceId("001").withInstanceHrid("in001"));
// when
AdditionalFieldsUtil.normalize035(record);
Assert.assertEquals(expectedParsedContent, parsedRecord.getContent());
}

@Test
public void shouldNotReturnSubfieldIfOclcNotExist() {
// given
Expand Down

0 comments on commit df5923e

Please sign in to comment.