Merge pull request #29 from pelias/osm_addresses_v2

osm addresses v2

api/near.js

@@ -0,0 +1,61 @@
+
+var sqlite3 = require('sqlite3'),
+    polyline = require('polyline'),
+    requireDir = require('require-dir'),
+    query = requireDir('../query'),
+    project = require('../lib/project'),
+    proximity = require('../lib/proximity');
+
+// polyline precision
+var PRECISION = 6;
+
+// export setup method
+function setup( streetDbPath ){
+
+  // connect to db
+  sqlite3.verbose();
+  // @todo: this is required as the query uses the 'street.' prefix for tables
+  var db = new sqlite3.Database( ':memory:', sqlite3.OPEN_READONLY );
+
+  // attach street database
+  query.attach( db, streetDbPath, 'street' );
+
+  // query method
+  var q = function( coord, cb ){
+
+    var point = {
+      lat: parseFloat( coord.lat ),
+      lon: parseFloat( coord.lon )
+    };
+
+    // error checking
+    if( isNaN( point.lat ) ){ return cb( 'invalid latitude' ); }
+    if( isNaN( point.lon ) ){ return cb( 'invalid longitude' ); }
+
+    // perform a db lookup for nearby streets
+    query.near( db, point, function( err, res ){
+
+      // an error occurred or no results were found
+      if( err || !res || !res.length ){ return cb( err, null ); }
+
+      // decode polylines
+      res.forEach( function( street, i ){
+        res[i].coordinates = project.dedupe( polyline.toGeoJSON(street.line, PRECISION).coordinates );
+      });
+
+      // order streets by proximity from point (by projecting it on to each line string)
+      var ordered = proximity.nearest.street( res, [ point.lon, point.lat ] );
+
+      // return streets ordered ASC by distance from point
+      cb( null, ordered );
+    });
+  };
+
+  // return methods
+  return {
+    query: q,
+    close: db.close.bind( db ),
+  };
+}
+
+module.exports = setup;

api/osm.js

@@ -18,6 +18,7 @@ function osm(dataStream, addressDbPath, streetDbPath, done){
   dataStream
     .pipe( stream.split() ) // split file on newline
     .pipe( stream.osm.parse() ) // parse openstreetmap json data
+    // .pipe( stream.osm.augment( db ) ) // find streets for records with only the housenumber
     .pipe( stream.osm.convert() ) // convert openstreetmap data to generic model
     .pipe( stream.address.batch() ) // batch records on the same street
     .pipe( stream.address.lookup( db ) ) // look up from db

api/search.js

@@ -42,9 +42,7 @@ function setup( addressDbPath, streetDbPath ){
     // @todo: perofmance: only query for part of the table
     query.search( db, point, normalized.number, normalized.street, function( err, res ){
 
-      // @todo: results can be from multiple different street ids!
-      // possibly not an issue? except maybe where there is a dual
-      // carriageway and then the projection would be on the median strip.
+      // @note: results can be from multiple different street ids.
 
       // an error occurred or no results were found
       if( err || !res || !res.length ){ return cb( err, null ); }
@@ -85,19 +83,42 @@ function setup( addressDbPath, streetDbPath ){
 
       // attempt to interpolate the position
 
-      // find the records before and after the desired number
-      var before, after;
-      for( var i=0; i<res.length; i++ ){
-        var row = res[i];
-        if( row.housenumber < normalized.number ){ before = row; }
-        if( row.housenumber > normalized.number ){ after = row; break; }
+      // find the records before and after the desired number (group by street segment)
+      var map = {};
+      res.forEach( function( row ){
+        if( !map.hasOwnProperty( row.id ) ){ map[row.id] = {}; }
+        if( row.housenumber < normalized.number ){ map[row.id].before = row; }
+        if( row.housenumber > normalized.number ){ map[row.id].after = row; }
+        if( map[row.id].before && map[row.id].after ){
+          map[row.id].diff = {
+            before: map[row.id].before.housenumber - normalized.number,
+            after: map[row.id].after.housenumber - normalized.number
+          };
+        }
+      });
+
+      // remove segments with less than 2 points; convert map to array
+      var segments = [];
+      for( var id in map ){
+        if( map[id].before && map[id].after ){
+          segments.push( map[id] );
+        }
       }
 
       // could not find two rows to use for interpolation
-      if( !before || !after ){
+      if( !segments.length ){
         return cb( null, null );
       }
 
+      // sort by miniumum housenumber difference from target housenumber ASC
+      segments.sort( function( a, b ){
+        return Math.abs( a.diff.before + a.diff.after ) - Math.abs( b.diff.before + b.diff.after );
+      });
+
+      // select before/after values to use for the interpolation
+      var before = segments[0].before;
+      var after = segments[0].after;
+
       // compute interpolated address
       var A = { lat: project.toRad( before.proj_lat ), lon: project.toRad( before.proj_lon ) };
       var B = { lat: project.toRad( after.proj_lat ), lon: project.toRad( after.proj_lon ) };

api/vertices.js

@@ -15,7 +15,7 @@ function vertices(addressDbPath, streetDbPath, done){
   query.tables.address(db); // create tables only if not already created
   query.attach(db, process.argv[3], 'street'); // attach street database
 
-  stream.each( db, 'street.polyline' )
+  stream.each( db, 'street.polyline', 'WHERE id IN ( SELECT DISTINCT id FROM address )' )
           .pipe( stream.vertices.lookup( db ) )
           .pipe( stream.vertices.augment() )
           .pipe( stream.batch( 1000 ) ) // batch up data to import

cmd/server.js

@@ -5,8 +5,11 @@ var express = require('express'),
     search = require('../api/search'),
     extract = require('../api/extract'),
     street = require('../api/street'),
+    near = require('../api/near'),
     pretty = require('../lib/pretty'),
-    analyze = require('../lib/analyze');
+    analyze = require('../lib/analyze'),
+    project = require('../lib/project'),
+    proximity = require('../lib/proximity');
 
 // optionally override port using env var
 var PORT = process.env.PORT || 3000;
@@ -23,7 +26,8 @@ var app = express();
 var conn = {
   search: search( process.argv[2], process.argv[3] ),
   extract: extract( process.argv[2], process.argv[3] ),
-  street: street( process.argv[3] )
+  street: street( process.argv[3] ),
+  near: near( process.argv[3] )
 };
 
 // search with geojson view
@@ -91,6 +95,38 @@ app.get('/extract/table', function( req, res ){
   });
 });
 
+// get streets near point, ordered by proximity to point ASC
+// eg: http://localhost:3000/street/near/geojson
+app.get('/street/near/geojson', function( req, res ){
+
+  var point = { lat: req.query.lat, lon: req.query.lon };
+
+  conn.near.query( point, function( err, ordered ){
+    if( err ){ return res.status(400).json( err ); }
+    if( !ordered || !ordered.length ){ return res.status(404).json({}); }
+
+    var geojson = {
+      'type': 'FeatureCollection',
+      'features': ordered.map( function( o ){
+        return {
+          'type': 'Feature',
+          'properties': {
+            'id': o.street.id,
+            'name': Array.isArray( o.street.name ) ? o.street.name[0] : o.street.name,
+            'polyline': o.street.line
+          },
+          'geometry': {
+            'type': 'LineString',
+            'coordinates': o.street.coordinates
+          }
+        };
+      })
+    };
+
+    res.json( geojson );
+  });
+});
+
 // get street geometry as geojson
 // eg: http://localhost:3000/street/1/geojson
 app.get('/street/:id/geojson', function( req, res ){

demo/examples.txt

@@ -1,2 +1,10 @@
 curved roads, OSM (Lavender Sweep, London)
 http://localhost:3000/demo/#18/51.46254/-0.16404
+
+good coverage, OSM(Fentiman Road, London)
+http://localhost:3000/demo/#18/51.48022/-0.11724
+
+http://localhost:3000/demo/#18/51.49117/-0.19861
+
+cul-de-sac
+http://interpolation.wiz.co.nz/demo/#17/39.75060/-76.66723

demo/index.html

@@ -131,6 +131,20 @@
       });
     }
 
+    function streetsNear( latlng, cb ){
+      $.ajax({
+        dataType: "json",
+        url: "/street/near/geojson?lat="+latlng.lat+"&lon="+latlng.lng,
+        success: function(data) {
+          // console.error( JSON.stringify( data, null, 2 ) );
+          cb( null, data );
+        }
+      }).error(function( err ) {
+        console.error( err );
+        cb( err );
+      });
+    }
+
     function streetById( ids, cb ){
       $.ajax({
         dataType: "json",
@@ -164,19 +178,20 @@
       // use map center when latlng not supplied
       if( !latlng ){ latlng = map.getCenter(); }
 
-      reverse( latlng, function( err, data ){
+      streetsNear( latlng, function( err, data ){
 
         if( data && data.features.length ){
 
           var centroid = latlng;
-          var names = data.features.map( function( street ){
-            return street.properties.street;
-          });
-
-          $('#housenumber').val( data.features[0].properties.housenumber );
-          $('#streetname').val( data.features[0].properties.street );
-          $('#lon').val( data.features[0].geometry.coordinates[0] );
-          $('#lat').val( data.features[0].geometry.coordinates[1] );
+          // var names = data.features.map( function( street ){
+          //   return street.properties.name;
+          // });
+          var names = [ data.features[0].properties.name ]; // only use first name
+
+          // $('#housenumber').val( data.features[0].properties.housenumber );
+          $('#streetname').val( data.features[0].properties.name );
+          $('#lon').val( data.features[0].geometry.coordinates[0][0] );
+          $('#lat').val( data.features[0].geometry.coordinates[0][1] );
 
           extract( centroid, names, function( err2, data2 ){
 

dev/compile_sqlite3.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e;
+export LC_ALL=en_US.UTF-8;
+
+# install the latest version of sqlite3
+
+# you can fetch the latest version from http://www.sqlite.org/download.html
+# look for the version which says "C source code as an amalgamation. Also includes a "configure" script"
+
+# remove version provided by package manager
+sudo apt-get remove sqlite3;
+
+# download and extract source code
+SOURCE="http://www.sqlite.org/2016/sqlite-autoconf-3150100.tar.gz";
+wget -q $SOURCE;
+tar xvfz $(basename "$SOURCE");
+
+# compile source
+cd $(basename "$SOURCE" ".tar.gz");
+./configure;
+make -j4;
+
+# install
+sudo make install;
+
+# echo new version
+sqlite3 -version;

dev/near.sh

@@ -7,6 +7,8 @@ STREET_DB="/data/street.db";
 LON="175.042589";
 LAT="-39.9225551";
 
-sqlite3 $STREET_DB "SELECT DISTINCT polyline.id, polyline.line FROM polyline \
+sqlite3 $STREET_DB "SELECT polyline.id, names.name, polyline.line FROM polyline \
   JOIN rtree ON rtree.id = polyline.id \
-  WHERE ( rtree.minX<$LON AND rtree.maxX>$LON AND rtree.minY<$LAT AND rtree.maxY>$LAT )";
+  JOIN names ON names.id = polyline.id \
+  WHERE ( rtree.minX<$LON AND rtree.maxX>$LON AND rtree.minY<$LAT AND rtree.maxY>$LAT ) \
+  GROUP BY polyline.id";

dev/osm.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e;
+export LC_ALL=en_US.UTF-8;
+
+# import osm data directly from PBF file
+
+# location of this file in filesystem
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd );
+
+ADDRESS_DB="/tmp/addr.db";
+STREET_DB="/data/street.db";
+
+# location of pbf2json binary
+# download from: https://github.com/pelias/pbf2json
+PBF2JSON="/var/www/pelias/pbf2json/build/pbf2json.linux-x64";
+
+# PBF extract
+PBFFILE="/data/extract/greater-london-latest.osm.pbf";
+
+# clean up
+rm -rf $ADDRESS_DB;
+
+# run import
+$PBF2JSON -tags="addr:housenumber+addr:street" $PBFFILE | $DIR/../interpolate osm $ADDRESS_DB $STREET_DB;

dev/search.sh

@@ -1,32 +1,42 @@
 #!/bin/bash
 
-# search address table records for max 3 records required to perform interpolation
+# search address table records for records required to perform interpolation
 
 ADDRESS_DB="/data/address.db";
 STREET_DB="/data/street.db";
 
-P1="174.766843";
-P2="-41.288788";
+LON="174.766843";
+LAT="-41.288788";
 
 NAME="glasgow street";
 NUMBER="18";
 
 sqlite3 $ADDRESS_DB "ATTACH DATABASE '$STREET_DB' as 'street'; \
-  WITH base AS (
-    SELECT address.* FROM street.rtree
-    JOIN street.names ON street.names.id = street.rtree.id
-    JOIN address ON address.id = street.rtree.id
-    WHERE (
-      street.rtree.minX<=$P1 AND street.rtree.maxX>=$P1 AND
-      street.rtree.minY<=$P2 AND street.rtree.maxY>=$P2
-    )
-    AND ( names.name='$NAME' )
-    ORDER BY address.housenumber ASC
-  )
-  SELECT * FROM (
-    (SELECT * FROM base WHERE housenumber < '$NUMBER' ORDER BY housenumber DESC LIMIT 1)
-  ) UNION SELECT * FROM (
-    (SELECT * FROM base WHERE housenumber >= '$NUMBER' ORDER BY housenumber ASC LIMIT 2)
-  )
-  ORDER BY housenumber ASC
-  LIMIT 3;";
+  WITH base AS ( \
+    SELECT address.* FROM street.rtree \
+    JOIN street.names ON street.names.id = street.rtree.id \
+    JOIN address ON address.id = street.rtree.id \
+    WHERE ( \
+      street.rtree.minX<=$LON AND street.rtree.maxX>=$LON AND \
+      street.rtree.minY<=$LAT AND street.rtree.maxY>=$LAT \
+    ) \
+    AND ( street.names.name='$NAME' ) \
+    ORDER BY address.housenumber ASC \
+  ) \
+  SELECT * FROM ( \
+    ( \
+      SELECT * FROM base \
+      WHERE housenumber < $NUMBER \
+      GROUP BY id HAVING( MAX( housenumber ) ) \
+      ORDER BY housenumber DESC \
+    ) \
+  ) UNION SELECT * FROM ( \
+    ( \
+      SELECT * FROM base \
+      WHERE housenumber >= $NUMBER \
+      GROUP BY id HAVING( MIN( housenumber ) ) \
+      ORDER BY housenumber ASC \
+    ) \
+  ) \
+  ORDER BY housenumber ASC \
+  LIMIT 20;";