Merge pull request #34 from pelias/osm_ops

osm ops

demo/examples.txt

@@ -8,3 +8,9 @@ http://localhost:3000/demo/#18/51.49117/-0.19861
 
 cul-de-sac
 http://interpolation.wiz.co.nz/demo/#17/39.75060/-76.66723
+
+goethepark
+http://localhost:3000/demo/#18/52.50906/13.30287
+
+aro street
+http://interpolation.wiz.co.nz/demo/#18/-41.29560/174.76492

readme.md

@@ -62,7 +62,7 @@ You will need to download one or more `openaddresses` or `openstreetmap` files f
 
 See the [building the databases](https://github.com/pelias/interpolation#building-the-databases) section below for detailed information on which commands to run.
 
-There is also a script named `./script/conflate.sh` in this repository which makes running this process much easier.
+There are scripts named `./script/conflate_oa.sh` and `./script/conflate_osm.sh` in this repository which make running this process much easier.
 
 note: We only support `openaddreses` and `openstreetmap` formats, you will need to create a custom importer for other sources.
 

script/build.sh

@@ -12,24 +12,26 @@ export LC_ALL=en_US.UTF-8;
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd );
 
 export TIMESTAMP=$(date +"%m-%d-%Y-%H:%M:%S");
+export PBF2JSON_BIN="/var/www/pelias/pbf2json/build/pbf2json.linux-x64";
+
+# location of data files
 export POLYLINE_FILE="/data/polyline/planet.polylines"; # the file containing all the streets
 export OAPATH="/data/oa"; # base path of openaddresses file system
+export PBF2JSON_FILE="/data/pbf/planet.osm.pbf";
 
 # a directory where all builds will live
 BUILDS="/data/builds";
 
 # ensure builds dir exists
 [ -d $BUILDS ] || mkdir -p $BUILDS;
 
-# update openaddresses data (optional)
-$DIR/update_oa.sh;
-
 # a directory where this specific build will live
 export BUILDDIR="$BUILDS/$TIMESTAMP";
 [ -d $BUILDDIR ] || mkdir -p $BUILDDIR;
 
-# a directory with enough free space to store sqlite tmp files
-export SQLITE_TMPDIR="$BUILDDIR/tmp";
+# location of temp files
+export SQLITE_TMPDIR="$BUILDDIR/tmp"; # a directory with enough free space to store sqlite tmp files
+export PBF2JSON_TMPDIR="$BUILDDIR/tmp/leveldb"; # a directory with enough free space to store leveldb tmp files
 
 # run polyline importer
 $DIR/import.sh;
@@ -41,7 +43,10 @@ if type pigz >/dev/null
 fi
 
 # run openaddresses conflation
-$DIR/conflate.sh;
+$DIR/conflate_oa.sh;
+
+# run openstreetmap conflation
+$DIR/conflate_osm.sh;
 
 # run vertex interpolation
 $DIR/vertices.sh;
@@ -53,14 +58,14 @@ if type pigz >/dev/null
 fi
 
 # clean up
-rm -rf "$SQLITE_TMPDIR"; # remove tmp files
+rm -rf "$SQLITE_TMPDIR" "$PBF2JSON_TMPDIR"; # remove tmp files
 
 # record build meta data
 METAFILE="$BUILDDIR/build.meta";
 
 echo "-- file system --" > "$METAFILE";
 ls -lah "$BUILDDIR" >> "$METAFILE";
-shasum "$BUILDDIR/*.db*" >> "$METAFILE";
+shasum $BUILDDIR/*.db* >> "$METAFILE";
 
 echo "-- street db --" >> "$METAFILE";
 sqlite3 -echo "$BUILDDIR/street.db" "SELECT * FROM sqlite_master;" >> "$METAFILE";

script/conflate.sh → script/conflate_oa.sh

@@ -19,15 +19,15 @@ ADDRESS_DB=${ADDRESS_DB:-"$BUILDDIR/address.db"};
 STREET_DB=${STREET_DB:-"$BUILDDIR/street.db"};
 
 # location of stdio files
-PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate.out"};
-PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate.err"};
-PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/conflate.skip"};
+PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate_oa.out"};
+PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate_oa.err"};
+PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/conflate_oa.skip"};
 
 # a directory with enough free space to store sqlite tmp files
 export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"};
 
 # ensure tmpdir exists
-[ -d $SQLITE_TMPDIR ] || mkdir $SQLITE_TMPDIR;
+[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR;
 
 # delete previous stdio files
 rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR;

script/conflate_osm.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e;
+export LC_ALL=en_US.UTF-8;
+
+# import openstreetmap data and conflate it with $STREET_DB
+
+# location of this file in filesystem
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd );
+
+# ---- pbf2json ----
+
+# location of pbf2json binary
+PBF2JSON_BIN=${PBF2JSON_BIN:-"pbf2json.linux-x64"};
+
+# ensure pbf2json exists and is executable
+if [[ ! -f $PBF2JSON_BIN || ! -x $PBF2JSON_BIN ]]; then
+  echo "pbf2json not found or is not executable";
+  exit 1;
+fi
+
+# tags to target from the PBF extract
+PBF2JSON_TAGS=${PBF2JSON_TAGS:-"addr:housenumber+addr:street"};
+
+# full path to the .osm.pbf file we would like to import
+PBF2JSON_FILE=${PBF2JSON_FILE:-"planet.osm.pbf"};
+
+# a directory with enough free space to store leveldb tmp files
+PBF2JSON_TMPDIR=${PBF2JSON_TMPDIR:-"$BUILDDIR/tmp/leveldb"};
+
+# ensure tmpdir exists
+[ -d $PBF2JSON_TMPDIR ] || mkdir -p $PBF2JSON_TMPDIR;
+
+# ---- -------- ----
+
+# location where this build will be stored
+BUILDDIR=${BUILDDIR:-"/data"};
+if [ ! -d $BUILDDIR ]; then
+  echo "data dir does not exist";
+  exit 1;
+fi
+
+# location of sql databases
+ADDRESS_DB=${ADDRESS_DB:-"$BUILDDIR/address.db"};
+STREET_DB=${STREET_DB:-"$BUILDDIR/street.db"};
+
+# location of stdio files
+PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate_osm.out"};
+PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate_osm.err"};
+
+# a directory with enough free space to store sqlite tmp files
+export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"};
+
+# ensure tmpdir exists
+[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR;
+
+# delete previous stdio files
+rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR;
+
+# run import
+$PBF2JSON_BIN -tags="$PBF2JSON_TAGS" -leveldb="$PBF2JSON_TMPDIR" $PBF2JSON_FILE |\
+ time -p node $DIR/../cmd/osm.js $ADDRESS_DB $STREET_DB 1>$PROC_STDOUT 2>$PROC_STDERR;

script/cronjob.sh

@@ -22,5 +22,9 @@ git pull -q origin master;
 # update npm dependencies
 npm --loglevel=silent install;
 
+# update openaddresses data (optional)
+export OAPATH="/data/oa"; # base path of openaddresses file system
+$DIR/update_oa.sh;
+
 # run build
 $DIR/build.sh &> build.log;

script/vertices.sh

@@ -27,7 +27,7 @@ PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/vertices.skip"};
 export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"};
 
 # ensure tmpdir exists
-[ -d $SQLITE_TMPDIR ] || mkdir $SQLITE_TMPDIR;
+[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR;
 
 # delete previous stdio files
 rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR;