Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change CET time to UTC #379

Merged
merged 2 commits into from
Feb 22, 2024
Merged

Change CET time to UTC #379

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8b652a
Change times from CET to UTC (#372)
dr0i Feb 20, 2024
ca51975
Fix "retry" part
dr0i Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 36 additions & 15 deletions app/apps/ConvertUpdates.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
Expand All @@ -28,6 +30,7 @@
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.EndElement;
import javax.xml.stream.events.Namespace;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;
import javax.xml.transform.TransformerException;
Expand All @@ -37,19 +40,19 @@
import org.oclc.oai.harvester2.app.RawWrite;
import org.xml.sax.SAXException;

import helper.Email;

/**
* Configurable to-the-minute. Start time is read from file which was saved last time of successfully updating.
* OAI-PMH allows UTC queries. The format doesn't allow e.g. milliseconds - see {@link #dateTimeFormatter}.
* Default end time is now, in the format like eg. 2023-11-06T15:59Z .
* Default end time is now, in the format like eg. 2023-11-06T15:59Z.
* The times are interpreted as CET and as OAI-PMH expects UTC the CET times are converted to UTC to query OAI_PMH.
* Files uses the CET timestamps, though.
*/
public class ConvertUpdates {

static private final short MAX_TRIES = 2;
static private final short MAX_TRIES = 1;
static private final int WAIT_PER_RETRY = 14400000; // ms => 4h
static private final int DAY_IN_MINUTES = 1440;
static private final String FAIL_MESSAGE = "Tried to get the update several times, but the data remains to be empty."
static private final String FAIL_MESSAGE = "Tried to get the update the defined amount of times, but the data remains to be empty."
+ "This may or may not be a problem on the side of the data provider.";
static private boolean rawDates = false;
/* OAI-PMH expects this format */
Expand All @@ -65,13 +68,15 @@ public static void main(String[] args) throws IOException, InterruptedException
File dataUpdate = new File(config("data.updates.data"));
short tried = 1;
while (dataUpdate.length() == 0 ) {
tried++;
System.err.println("Tried " + tried + " times to get the data, but it's empty.");
if (MAX_TRIES <= tried)
if (MAX_TRIES >= tried) {
System.err.println(FAIL_MESSAGE);
break;
}
System.err.println("Going to retry in " + WAIT_PER_RETRY / 1000 / 60 + " min");
Thread.sleep(WAIT_PER_RETRY);
startAndEnd = getUpdatesAndConvert(startAndEnd.getLeft(), endOfUpdates);
tried++;
}
if (dataUpdate.length() == 0) {
System.err.println("Tried " + tried
Expand Down Expand Up @@ -112,7 +117,7 @@ private static Pair<ZonedDateTime, ZonedDateTime> getUpdates(final ZonedDateTime
e.printStackTrace();
}
String dataUpdateUrl = config("data.updates.url");
System.out.printf("Trying to get data using %s from %s until %s using a %s days interval , i.e. doing %s lookup(s)\n",
System.out.printf("Trying to get data using %s from %s CET until %s CET using a %s days interval , i.e. doing %s lookup(s)\n",
dataUpdateUrl, start.format(dateTimeFormatter), givenEndOrNow.format(dateTimeFormatter), intervalInDaysSize, intervalInDays);
for (int i = 0; i < intervalInDays; i++) {
if (i == intervalInDays - 1)
Expand All @@ -136,18 +141,34 @@ private static Pair<ZonedDateTime, ZonedDateTime> getUpdates(final ZonedDateTime
return Pair.of(startOfUpdates, end);
}

public static void process(final String baseUrl, ZonedDateTime from, ZonedDateTime until, File result)
/**
* As we call from a CET timed server and OAI-PMH expects UTC time the given parameter "from" and parameter "until"
* are changed to UTC time.
*
* @param baseUrl the basis URL of the OAI-PMH server
* @param from a CET time defining the beginning of the timeframe
* @param until a CET time defining the end of the timeframe
* @param result the file to where the data is written
*/
public static void process(final String baseUrl, final ZonedDateTime from, final ZonedDateTime until, File result)
throws NoSuchFieldException, IOException, ParserConfigurationException, SAXException, TransformerException,
XMLStreamException, XPathException {
ByteArrayOutputStream stream = new ByteArrayOutputStream();
String fromFormatted = from.minusMinutes(1).format(dateTimeFormatter);
String untilFormatted = until.format(dateTimeFormatter);

System.out.printf("Calling OAI-PMH at %s from %s until %s\n", baseUrl, fromFormatted, untilFormatted);
RawWrite.run(baseUrl, fromFormatted, untilFormatted, "RDFxml", "authorities", stream);
String fromUtcFormatted = getUtcFromCet(from).minusMinutes(1).format(dateTimeFormatter);
String untilUtcFormatted = getUtcFromCet(until).format(dateTimeFormatter);

System.out.printf("Calling OAI-PMH at %s from %s UTC until %s UTC\n", baseUrl, fromUtcFormatted, untilUtcFormatted);
ByteArrayOutputStream stream = new ByteArrayOutputStream();
RawWrite.run(baseUrl, fromUtcFormatted, untilUtcFormatted, "RDFxml", "authorities", stream);
writeRdfDescriptions(result, stream);
}

private static ZonedDateTime getUtcFromCet(final ZonedDateTime cetTime) {
ZonedDateTime cetTimeZoned = ZonedDateTime.of(cetTime.toLocalDateTime(), ZoneId.of("CET"));
ZonedDateTime utcTimeZoned = cetTimeZoned.withZoneSameInstant(ZoneOffset.UTC);
return utcTimeZoned;
}

static void writeRdfDescriptions(File result, ByteArrayOutputStream stream)
throws FactoryConfigurationError, XMLStreamException, IOException {
try (FileWriter fileWriter = new FileWriter(result, true)) {
Expand All @@ -173,7 +194,7 @@ static void writeRdfDescriptions(File result, ByteArrayOutputStream stream)
QName descriptionName = new QName(startElement.getName().getNamespaceURI(),
startElement.getName().getLocalPart(), startElement.getName().getPrefix());
StartElement descriptionWithNamespaces = eventFactory.createStartElement(descriptionName,
startElement.getAttributes(), namespaces);
startElement.getAttributes(), (Iterator<? extends Namespace>) namespaces);
nextEvent = descriptionWithNamespaces;
}
} else if (nextEvent.isEndElement()
Expand Down
Loading