Skip to content

Commit

Permalink
Merge branch 'master' into 1.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@joker234
joker234 committed Nov 10, 2020
2 parents d61d02e + d890d5d commit 78305e8
Show file tree
Hide file tree
Showing 4 changed files with 226 additions and 149 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
## current master

## 1.2.1

### Bug Fixes

* fixing bug Ignite trying to load non-serializable objects within lambda functions ([#75](https://github.com/GIScience/ohsome-api/pull/75))

## 1.2.0

Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<name>ohsome API</name>
<description>A public Web-RESTful-API for "ohsome" OpenStreetMap history data.</description>
<packaging>jar</packaging>
<version>1.2.0</version>
<version>1.2.1</version>

<properties>
<apachecommons-csv.version>1.6</apachecommons-csv.version>
Expand Down
163 changes: 15 additions & 148 deletions src/main/java/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,10 @@

import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
Expand All @@ -17,11 +14,8 @@
import org.heigit.bigspatialdata.oshdb.api.mapreducer.MapReducer;
import org.heigit.bigspatialdata.oshdb.api.object.OSMContribution;
import org.heigit.bigspatialdata.oshdb.api.object.OSMEntitySnapshot;
import org.heigit.bigspatialdata.oshdb.osm.OSMEntity;
import org.heigit.bigspatialdata.oshdb.util.celliterator.ContributionType;
import org.heigit.bigspatialdata.oshdb.util.tagtranslator.TagTranslator;
import org.heigit.bigspatialdata.oshdb.util.time.ISODateTimeParser;
import org.heigit.bigspatialdata.oshdb.util.time.TimestampFormatter;
import org.heigit.ohsome.filter.FilterExpression;
import org.heigit.ohsome.ohsomeapi.Application;
import org.heigit.ohsome.ohsomeapi.controller.rawdata.ElementsGeometry;
Expand All @@ -34,7 +28,6 @@
import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
import org.heigit.ohsome.ohsomeapi.output.dataaggregationresponse.Metadata;
import org.heigit.ohsome.ohsomeapi.output.rawdataresponse.DataResponse;
import org.locationtech.jts.geom.Geometry;
import org.wololo.geojson.Feature;

/** Holds executor methods for the following endpoints: /elementsFullHistory, /contributions. */
Expand Down Expand Up @@ -106,9 +99,7 @@ public void extract() throws Exception {
final boolean isContributionsLatestEndpoint =
requestResource.equals(RequestResource.CONTRIBUTIONSLATEST);
final boolean isContributionsEndpoint =
(isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS))
? true
: false;
isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS);
final Set<SimpleFeatureType> simpleFeatureTypes = processingData.getSimpleFeatureTypes();
Optional<FilterExpression> filter = processingData.getFilterExpression();
final boolean requiresGeometryTypeCheck =
Expand All @@ -119,103 +110,14 @@ public void extract() throws Exception {
String endTimestamp = ISODateTimeParser.parseISODateTime(requestParameters.getTime()[1])
.format(DateTimeFormatter.ISO_DATE_TIME);
MapReducer<List<OSMContribution>> mapRedContributions = mapRedContribution.groupByEntity();
MapReducer<Feature> contributionPreResult = mapRedContributions.flatMap(contributions -> {
List<Feature> output = new LinkedList<>();
Map<String, Object> properties;
Geometry currentGeom = null;
OSMEntity currentEntity = null;
String validFrom = null;
String validTo;
boolean skipNext = false;
if (!isContributionsLatestEndpoint) {
// first contribution:
OSMContribution firstContribution = contributions.get(0);
if (firstContribution.is(ContributionType.CREATION) && !isContributionsEndpoint) {
skipNext = true;
} else {
// if not "creation": take "before" as starting "row" (geom, tags), valid_from = t_start
currentEntity = firstContribution.getEntityBefore();
currentGeom = exeUtils.getGeometry(firstContribution, clipGeometries, true);
validFrom = startTimestamp;
}
// then for each contribution:
for (int i = 0; i < contributions.size(); i++) {
if (i == contributions.size() - 1 && isContributionsEndpoint) {
// end the loop when last contribution is reached as it gets added later on
break;
}
OSMContribution contribution = contributions.get(i);
if (isContributionsEndpoint) {
currentEntity = contribution.getEntityAfter();
currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false);
validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
}
// set valid_to of previous row
validTo = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
if (!skipNext && currentGeom != null && !currentGeom.isEmpty()) {
boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes,
requiresGeometryTypeCheck, filterExpression, currentGeom, currentEntity);
if (addToOutput) {
properties = new TreeMap<>();
if (!isContributionsEndpoint) {
properties.put("@validFrom", validFrom);
properties.put("@validTo", validTo);
} else {
properties.put("@timestamp", validTo);
}
output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry,
contribution.getContributionTypes()));
}
}
skipNext = false;
if (contribution.is(ContributionType.DELETION)) {
// if deletion: skip output of next row
skipNext = true;
} else if (!isContributionsEndpoint) {
// else: take "after" as next row
currentEntity = contribution.getEntityAfter();
currentGeom = exeUtils.getGeometry(contribution, clipGeometries, false);
validFrom = TimestampFormatter.getInstance().isoDateTime(contribution.getTimestamp());
}
}
}
// after loop:
OSMContribution lastContribution = contributions.get(contributions.size() - 1);
currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, false);
currentEntity = lastContribution.getEntityAfter();
if (!lastContribution.is(ContributionType.DELETION)) {
// if last contribution was not "deletion": set valid_to = t_end
validTo = endTimestamp;
properties = new TreeMap<>();
if (!isContributionsEndpoint) {
properties.put("@validFrom", validFrom);
properties.put("@validTo", validTo);
} else {
properties.put("@timestamp",
TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp()));
}
if (!currentGeom.isEmpty()) {
boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes,
requiresGeometryTypeCheck, filterExpression, currentGeom, currentEntity);
if (addToOutput) {
output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt,
includeTags, includeOSMMetadata, isContributionsEndpoint, elementsGeometry,
lastContribution.getContributionTypes()));
}
}
} else if (isContributionsEndpoint) {
// adds the deletion feature for a /contributions request
currentGeom = exeUtils.getGeometry(lastContribution, clipGeometries, true);
properties = new TreeMap<>();
properties.put("@timestamp",
TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp()));
output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, false,
includeOSMMetadata, isContributionsEndpoint, elementsGeometry,
lastContribution.getContributionTypes()));
}
return output;
}).filter(Objects::nonNull);
final boolean isContainingSimpleFeatureTypes = processingData.isContainingSimpleFeatureTypes();
FlatMapExecutor flatMapExecutor = new FlatMapExecutor(isContributionsLatestEndpoint,
isContributionsEndpoint, exeUtils, clipGeometries, startTimestamp, utils,
simpleFeatureTypes, requiresGeometryTypeCheck, filterExpression, keysInt, includeTags,
includeOSMMetadata, elementsGeometry, endTimestamp, isContainingSimpleFeatureTypes);
MapReducer<Feature> contributionPreResult = mapRedContributions
.flatMap(flatMapExecutor::buildChangedFeatures)
.filter(Objects::nonNull);
Metadata metadata = null;
if (processingData.isShowMetadata()) {
metadata = new Metadata(null, requestResource.getDescription(),
Expand All @@ -226,34 +128,15 @@ public void extract() throws Exception {
MapReducer<Feature> snapshotPreResult = null;
if (!isContributionsEndpoint) {
// handles cases where valid_from = t_start, valid_to = t_end; i.e. non-modified data
snapshotPreResult = mapRedSnapshot.groupByEntity().filter(snapshots -> snapshots.size() == 2)
snapshotPreResult = mapRedSnapshot
.groupByEntity()
.filter(snapshots -> snapshots.size() == 2)
.filter(snapshots -> snapshots.get(0).getGeometry() == snapshots.get(1).getGeometry()
&& snapshots.get(0).getEntity().getVersion() == snapshots.get(1).getEntity()
.getVersion())
.map(snapshots -> snapshots.get(0)).flatMap(snapshot -> {
Map<String, Object> properties = new TreeMap<>();
OSMEntity entity = snapshot.getEntity();
if (includeOSMMetadata) {
properties.put("@lastEdit", entity.getTimestamp().toString());
}
Geometry geom = snapshot.getGeometry();
if (!clipGeometries) {
geom = snapshot.getGeometryUnclipped();
}
properties.put("@snapshotTimestamp",
TimestampFormatter.getInstance().isoDateTime(snapshot.getTimestamp()));
properties.put("@validFrom", startTimestamp);
properties.put("@validTo", endTimestamp);
boolean addToOutput = addEntityToOutput(processingData, utils, simpleFeatureTypes,
requiresGeometryTypeCheck, filterExpression, geom, entity);
if (addToOutput) {
return Collections.singletonList(
exeUtils.createOSMFeature(entity, geom, properties, keysInt, includeTags,
includeOSMMetadata, isContributionsEndpoint, elementsGeometry, null));
} else {
return Collections.emptyList();
}
}).filter(Objects::nonNull);
.map(snapshots -> snapshots.get(0))
.flatMap(flatMapExecutor::buildUnchangedFeatures)
.filter(Objects::nonNull);
}
try (
Stream<Feature> snapshotStream =
Expand All @@ -263,20 +146,4 @@ public void extract() throws Exception {
Stream.concat(contributionStream, snapshotStream));
}
}

/** Checks whether the given entity should be added to the output (true) or not (false). */
private boolean addEntityToOutput(ProcessingData processingData, InputProcessingUtils utils,
final Set<SimpleFeatureType> simpleFeatureTypes, final boolean requiresGeometryTypeCheck,
FilterExpression filterExpression, Geometry currentGeom, OSMEntity currentEntity) {
boolean addToOutput;
if (processingData.isContainingSimpleFeatureTypes()) {
addToOutput = utils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes);
} else if (requiresGeometryTypeCheck) {
addToOutput = filterExpression.applyOSMGeometry(currentEntity, currentGeom);
} else {
addToOutput = true;
}
return addToOutput;
}

}
Loading

0 comments on commit 78305e8

Please sign in to comment.