Support for Bulk Geocoding (aka. instant/synchronous batch)

ols-geocoder-core/src/main/java/ca/bc/gov/ols/geocoder/config/GeocoderConfig.java

@@ -70,6 +70,8 @@ public class GeocoderConfig {
 	protected EnumMap<MatchPrecision, Integer> matchPrecisionPoints =
 			new EnumMap<MatchPrecision, Integer>(MatchPrecision.class);
 	protected int maxWithinResults;
+	protected int bulkMaxRequests = 0;
+	protected int bulkTimeLimit = 0;
 	protected double blockFaceOffset;
 	protected String moreInfoUrl;
 	protected String privacyStatement;
@@ -140,6 +142,10 @@ public GeocoderConfig(GeocoderConfigurationStore configStore, GeometryFactory gf
 							Integer.parseInt(value));
 				} else if("maxWithinResults".equals(name)) {
 					maxWithinResults = Integer.parseInt(value);
+				} else if("bulkMaxRequests".equals(name)) {
+					bulkMaxRequests = Integer.parseInt(value);
+				} else if("bulkTimeLimit".equals(name)) {
+					bulkTimeLimit = Integer.parseInt(value);
 				} else if("blockFaceOffset".equals(name)) {
 					blockFaceOffset = Double.parseDouble(value);
 				} else if("moreInfoUrl".equals(name)) {
@@ -510,4 +516,12 @@ public String getDataSourceClassName() {
 		return dataSourceClassName;
 	}
 
+	public int getBulkMaxRequests() {
+		return bulkMaxRequests;
+	}
+
+	public int getBulkTimeLimit() {
+		return bulkTimeLimit;
+	}
+
 }

ols-geocoder-core/src/main/resources/bgeo_configuration_parameters.csv

@@ -6,6 +6,8 @@ BGEO,occupantCustomKmlStyleUrl,"http://openmaps.gov.bc.ca/kml/occupant_custom_st
 BGEO,copyrightLicenseURI,"http://www.data.gov.bc.ca/local/dbc/docs/license/OGL-vbc2.0.pdf"
 BGEO,copyrightNotice,"Copyright 2015 Province of British Columbia - Open Government License"
 BGEO,maxWithinResults,"200"
+BGEO,bulkMaxRequests,"1"
+BGEO,bulkTimeLimit,"10"
 BGEO,blockFaceOffset,"0.5"
 BGEO,baseSrsCode,"3005"
 BGEO,baseSrsBounds,"200000,300000,1900000,1800000"

ols-geocoder-web/pom.xml

@@ -61,12 +61,12 @@
 		<dependency>
 			<groupId>commons-io</groupId>
 			<artifactId>commons-io</artifactId>
-			<version>2.3</version>
+			<version>2.8.0</version>
 		</dependency>
 		<dependency>
 			<groupId>commons-fileupload</groupId>
 			<artifactId>commons-fileupload</artifactId>
-			<version>1.3.3</version>
+			<version>1.4</version>
 		</dependency>
 		<dependency>
 			<groupId>org.geotools</groupId>

ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/WebConfig.java

@@ -25,13 +25,15 @@
 import org.springframework.format.FormatterRegistry;
 import org.springframework.http.MediaType;
 import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.web.multipart.commons.CommonsMultipartResolver;
 import org.springframework.web.servlet.config.annotation.ContentNegotiationConfigurer;
 import org.springframework.web.servlet.config.annotation.DefaultServletHandlerConfigurer;
 import org.springframework.web.servlet.config.annotation.EnableWebMvc;
 import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 
 import ca.bc.gov.ols.geocoder.IGeocoder;
+import ca.bc.gov.ols.geocoder.rest.bulk.CSVBulkResponseConverter;
 import ca.bc.gov.ols.geocoder.rest.converters.BooleanConverter;
 import ca.bc.gov.ols.geocoder.rest.converters.InterpolationConverter;
 import ca.bc.gov.ols.geocoder.rest.converters.LocationDescriptorConverter;
@@ -82,7 +84,7 @@ public void configureMessageConverters(List<HttpMessageConverter<?>> converters)
 		converters.add(htmlErrorMessageConverter());
 		converters.add(kmlErrorMessageConverter());
 		converters.add(csvStringConverter());
-		//converters.add(csvBatchResponseConverter());
+		converters.add(csvBulkResponseConverter());
 		converters.add(jsonStringListConverter());
 		//super.configureMessageConverters(converters);
 	}
@@ -142,18 +144,17 @@ public CsvStringConverter csvStringConverter() {
 		return new CsvStringConverter();
 	}
 
-//  For Instant Batch
-//	@Bean
-//	public CSVBatchResponseConverter csvBatchResponseConverter() {
-//		return new CSVBatchResponseConverter();
-//	}
-//	
-//	@Bean
-//	public CommonsMultipartResolver multipartResolver() {
-//	    CommonsMultipartResolver resolver=new CommonsMultipartResolver();
-//	    resolver.setDefaultEncoding("utf-8");
-//	    return resolver;
-//	}
+	@Bean
+	public CSVBulkResponseConverter csvBulkResponseConverter() {
+		return new CSVBulkResponseConverter();
+	}
+
+	@Bean
+	public CommonsMultipartResolver multipartResolver() {
+	    CommonsMultipartResolver resolver=new CommonsMultipartResolver();
+	    resolver.setDefaultEncoding("utf-8");
+	    return resolver;
+	}
 
 	@Bean
 	public JsonStringListConverter jsonStringListConverter() {

ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/batch/AbstractBatchResponseWriter.java → ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/bulk/AbstractBulkResponseWriter.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package ca.bc.gov.ols.geocoder.rest.batch;
+package ca.bc.gov.ols.geocoder.rest.bulk;
 
 import java.io.IOException;
 import java.io.OutputStreamWriter;
@@ -32,22 +32,23 @@
 import ca.bc.gov.ols.geocoder.IGeocoder;
 import ca.bc.gov.ols.geocoder.api.data.GeocodeMatch;
 import ca.bc.gov.ols.geocoder.api.data.SearchResults;
+import ca.bc.gov.ols.geocoder.config.GeocoderConfig;
 import ca.bc.gov.ols.geocoder.rest.LocationReprojector;
 
 // For Instant Batch
-public abstract class AbstractBatchResponseWriter extends AbstractHttpMessageConverter<GeocoderBatchProcessor> {
+public abstract class AbstractBulkResponseWriter extends AbstractHttpMessageConverter<BulkGeocodeProcessor> {
 
 	@Autowired
 	protected IGeocoder geocoder;
 
 
-	public AbstractBatchResponseWriter(MediaType mediaType) {
+	public AbstractBulkResponseWriter(MediaType mediaType) {
 		super(mediaType);
 	}
 
 	@Override
 	protected boolean supports(Class<?> clazz) {
-		return GeocoderBatchProcessor.class.isAssignableFrom(clazz);
+		return BulkGeocodeProcessor.class.isAssignableFrom(clazz);
 	}
 
 	@Override
@@ -56,40 +57,55 @@ public boolean canRead(Class<?> clazz, MediaType mediaType) {
 	}
 
 	@Override
-	protected GeocoderBatchProcessor readInternal(Class<? extends GeocoderBatchProcessor> clazz,
+	protected BulkGeocodeProcessor readInternal(Class<? extends BulkGeocodeProcessor> clazz,
 			HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException {
 		return null;
 	}
 
 	@Override
-	protected void writeInternal(GeocoderBatchProcessor proc, HttpOutputMessage outputMessage)
+	protected void writeInternal(BulkGeocodeProcessor proc, HttpOutputMessage outputMessage)
 			throws IOException, HttpMessageNotWritableException {
-		//GeocoderConfig config = geocoder.getDatastore().getConfig();
+		GeocoderConfig config = geocoder.getDatastore().getConfig();
 		Writer out = new OutputStreamWriter(outputMessage.getBody(), "UTF-8");
 		//response.reproject(config.getBaseSrsCode(), response.getOutputSRS());
 		// output header
 		writeHeader(out, proc);
-		proc.start();
+		BulkStatsCalculator statsCalc = new BulkStatsCalculator();
+		proc.start(statsCalc);
+		int maxRequests = clampMax(config.getBulkMaxRequests(), proc.getParams().getMaxRequests()); 
+		int maxTime = clampMax(config.getBulkTimeLimit(), proc.getParams().getMaxTime()) * 1000; 
 		SearchResults results = proc.next();
-		int seqNum = 0;
+		int seqNum = proc.getStartSeqNum();
 		while(results != null) {
-			seqNum++;
-			int resultNum = 0;
+			int resultNum = 1;
 			// reproject the results
 			//reprojectResults(results, results.getSrsCode());
 			for(GeocodeMatch match : results.getMatches()) {
-				resultNum++;
 				// output the result match
 				writeMatch(out, match, seqNum, resultNum, results.getExecutionTime());
+				resultNum++;
+			}
+			seqNum++;
+			// break out if we've hit either limit
+			if((maxRequests != 0 && statsCalc.getProcessedCount() >= maxRequests)
+					|| (maxTime != 0 && statsCalc.getElapsedTime() >= maxTime)) {
+				break;
 			}
 			results = proc.next();
 		}
 		// output footer
 		proc.stop();
 		writeFooter(out, proc);
 		out.flush();
+		//logger.info("Stats: " + proc.getStats().toString());
 	}
 
+	private int clampMax(int configMax, int paramMax) {
+		if(configMax <= 0) return paramMax;
+		if(paramMax <= 0) return configMax;
+		return Math.min(configMax, paramMax);
+	}
+
 	public void reprojectResults(SearchResults results, Integer srsCode) {
 		LocationReprojector lr = new LocationReprojector(geocoder.getConfig().getBaseSrsCode(), srsCode);
 		if(srsCode != geocoder.getDatastore().getConfig().getBaseSrsCode()) {
@@ -103,11 +119,11 @@ public void reprojectResults(SearchResults results, Integer srsCode) {
 		}
 	}
 
-	protected abstract void writeHeader(Writer out, GeocoderBatchProcessor proc) throws IOException;
+	protected abstract void writeHeader(Writer out, BulkGeocodeProcessor proc) throws IOException;
 
 	protected abstract void writeMatch(Writer out, GeocodeMatch match, int seqNum, int resultNum, 
 			BigDecimal executionTime) throws IOException;
 
-	protected abstract void writeFooter(Writer out, GeocoderBatchProcessor proc) throws IOException;
+	protected abstract void writeFooter(Writer out, BulkGeocodeProcessor proc) throws IOException;
 
 }

ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/batch/GeocoderBatchProcessor.java → ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/bulk/BulkGeocodeProcessor.java

@@ -13,34 +13,43 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package ca.bc.gov.ols.geocoder.rest.batch;
+package ca.bc.gov.ols.geocoder.rest.bulk;
 
 import java.io.IOException;
+import java.time.LocalDateTime;
 
 import ca.bc.gov.ols.rowreader.CsvRowReader;
 import ca.bc.gov.ols.geocoder.GeocoderDataStore;
 import ca.bc.gov.ols.geocoder.IGeocoder;
 import ca.bc.gov.ols.geocoder.api.GeocodeQuery;
+import ca.bc.gov.ols.geocoder.api.data.ModifiableLocation;
 import ca.bc.gov.ols.geocoder.api.data.SearchResults;
-import ca.bc.gov.ols.geocoder.rest.controllers.GeocodeParameters;
+import ca.bc.gov.ols.geocoder.rest.LocationReprojector;
+import ca.bc.gov.ols.geocoder.rest.controllers.BulkGeocodeParameters;
 
-// For Instant Batch
-public class GeocoderBatchProcessor {
+public class BulkGeocodeProcessor {
 
 	private IGeocoder geocoder;
-	private GeocodeParameters params;
+	private BulkGeocodeParameters params;
 	private CsvRowReader rr;
-	private BatchStatsCalculator stats;
+	private BulkStatsCalculator stats;
+	private int fromSrsCode;
+	private int toSrsCode;
+	private LocationReprojector lr;
 
-	public GeocoderBatchProcessor(GeocodeParameters params, IGeocoder geocoder) {
+	public BulkGeocodeProcessor(BulkGeocodeParameters params, IGeocoder geocoder) {
 		this.params = params;
 		this.geocoder = geocoder;
+		fromSrsCode =  geocoder.getConfig().getBaseSrsCode();
+		toSrsCode = params.getOutputSRS();
+		lr = new LocationReprojector(fromSrsCode, toSrsCode);
 	}
 
-	public void start() throws IOException {
+	public void start(BulkStatsCalculator statsCalc) throws IOException {
 		rr = new CsvRowReader(params.getFile().getInputStream(), GeocoderDataStore.getGeometryFactory());
-		stats = new BatchStatsCalculator();
+		stats = statsCalc;
 		stats.start();
+
 	}
 
 	public SearchResults next() {
@@ -68,31 +77,48 @@ public SearchResults next() {
 			qry.setStateProvTerr(rr.getString("province"));
 		}
 
+		String yourId = rr.getString("yourId");
+		if(yourId != null) {
+			qry.setYourId(yourId);
+		}
 		// handle additional query parameters
-//		qry.setMaxResults(params.getmaxResults);
-//		qry.setSetBack(setBack);
-//		qry.setMinScore(minScore);
-//		qry.setQuickMatch(quickMatch);
-//		qry.setEcho(echo);
-//		qry.setInterpolation(interpolation);
-//		qry.setLocationDescriptor(locationDescriptor);
-//
-//
-//		qry.setEcho(echo);
+		qry.setMaxResults(params.getMaxResults());
+		qry.setSetBack(params.getSetBack());
+		qry.setMinScore(params.getMinScore());
+		qry.setEcho(params.isEcho());
+		qry.setInterpolation(params.getInterpolation());
+		qry.setLocationDescriptor(params.getLocationDescriptor());
+		qry.setOutputSRS(params.getOutputSRS());
+
 		SearchResults results = geocoder.geocode(qry);
-//		results.setInterpolation(qry.getInterpolation());
-//		results.setSrsCode(outputSRS);
+		if(toSrsCode != fromSrsCode) {
+			lr.reproject(results.getMatches());
+			lr.reprecision(results.getMatches());
+		} else {
+			lr.reprecision(results.getMatches());
+		}
+
+		results.setInterpolation(qry.getInterpolation());
+		results.setSrsCode(params.getOutputSRS());
 
 		stats.record(results.getExecutionTime().doubleValue(), results.getBestScore());
 		return results;
 	}
 
-	public BatchStatsCalculator stop() {
+	public int getStartSeqNum() {
+		return params.getStartSeqNum();
+	}
+
+	public BulkStatsCalculator stop() {
 		stats.stop();
 		return stats;
 	}
 
-	public BatchStatsCalculator getStats() {
+	public BulkStatsCalculator getStats() {
 		return stats;
 	}
+
+	public BulkGeocodeParameters getParams() {
+		return params;
+	}
 }

ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/batch/BatchStatsCalculator.java → ols-geocoder-web/src/main/java/ca/bc/gov/ols/geocoder/rest/bulk/BulkStatsCalculator.java

@@ -13,16 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package ca.bc.gov.ols.geocoder.rest.batch;
+package ca.bc.gov.ols.geocoder.rest.bulk;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 import ca.bc.gov.ols.util.StopWatch;
 
-//For Instant Batch
-public class BatchStatsCalculator {
+public class BulkStatsCalculator {
 
 		List<Double> times = new ArrayList<Double>();
 		List<Integer> scores = new ArrayList<Integer>();
@@ -77,6 +76,10 @@ public void stop() {
 
 		}
 
+		public int getProcessedCount() {
+			return times.size();
+		}
+
 		public long getElapsedTime() {
 			return clock.getElapsedTime();
 		}