diff --git a/demo/examples.txt b/demo/examples.txt index 580752b5..d21fe279 100644 --- a/demo/examples.txt +++ b/demo/examples.txt @@ -8,3 +8,9 @@ http://localhost:3000/demo/#18/51.49117/-0.19861 cul-de-sac http://interpolation.wiz.co.nz/demo/#17/39.75060/-76.66723 + +goethepark +http://localhost:3000/demo/#18/52.50906/13.30287 + +aro street +http://interpolation.wiz.co.nz/demo/#18/-41.29560/174.76492 diff --git a/readme.md b/readme.md index d17941cb..55496d98 100644 --- a/readme.md +++ b/readme.md @@ -62,7 +62,7 @@ You will need to download one or more `openaddresses` or `openstreetmap` files f See the [building the databases](https://github.com/pelias/interpolation#building-the-databases) section below for detailed information on which commands to run. -There is also a script named `./script/conflate.sh` in this repository which makes running this process much easier. +There are scripts named `./script/conflate_oa.sh` and `./script/conflate_osm.sh` in this repository which make running this process much easier. note: We only support `openaddreses` and `openstreetmap` formats, you will need to create a custom importer for other sources. diff --git a/script/build.sh b/script/build.sh index 174ea12c..74abca0d 100755 --- a/script/build.sh +++ b/script/build.sh @@ -12,8 +12,12 @@ export LC_ALL=en_US.UTF-8; DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ); export TIMESTAMP=$(date +"%m-%d-%Y-%H:%M:%S"); +export PBF2JSON_BIN="/var/www/pelias/pbf2json/build/pbf2json.linux-x64"; + +# location of data files export POLYLINE_FILE="/data/polyline/planet.polylines"; # the file containing all the streets export OAPATH="/data/oa"; # base path of openaddresses file system +export PBF2JSON_FILE="/data/pbf/planet.osm.pbf"; # a directory where all builds will live BUILDS="/data/builds"; @@ -21,15 +25,13 @@ BUILDS="/data/builds"; # ensure builds dir exists [ -d $BUILDS ] || mkdir -p $BUILDS; -# update openaddresses data (optional) -$DIR/update_oa.sh; - # a directory where this specific build will live export BUILDDIR="$BUILDS/$TIMESTAMP"; [ -d $BUILDDIR ] || mkdir -p $BUILDDIR; -# a directory with enough free space to store sqlite tmp files -export SQLITE_TMPDIR="$BUILDDIR/tmp"; +# location of temp files +export SQLITE_TMPDIR="$BUILDDIR/tmp"; # a directory with enough free space to store sqlite tmp files +export PBF2JSON_TMPDIR="$BUILDDIR/tmp/leveldb"; # a directory with enough free space to store leveldb tmp files # run polyline importer $DIR/import.sh; @@ -41,7 +43,10 @@ if type pigz >/dev/null fi # run openaddresses conflation -$DIR/conflate.sh; +$DIR/conflate_oa.sh; + +# run openstreetmap conflation +$DIR/conflate_osm.sh; # run vertex interpolation $DIR/vertices.sh; @@ -53,14 +58,14 @@ if type pigz >/dev/null fi # clean up -rm -rf "$SQLITE_TMPDIR"; # remove tmp files +rm -rf "$SQLITE_TMPDIR" "$PBF2JSON_TMPDIR"; # remove tmp files # record build meta data METAFILE="$BUILDDIR/build.meta"; echo "-- file system --" > "$METAFILE"; ls -lah "$BUILDDIR" >> "$METAFILE"; -shasum "$BUILDDIR/*.db*" >> "$METAFILE"; +shasum $BUILDDIR/*.db* >> "$METAFILE"; echo "-- street db --" >> "$METAFILE"; sqlite3 -echo "$BUILDDIR/street.db" "SELECT * FROM sqlite_master;" >> "$METAFILE"; diff --git a/script/conflate.sh b/script/conflate_oa.sh similarity index 79% rename from script/conflate.sh rename to script/conflate_oa.sh index ace60fbd..fc38a2bf 100755 --- a/script/conflate.sh +++ b/script/conflate_oa.sh @@ -19,15 +19,15 @@ ADDRESS_DB=${ADDRESS_DB:-"$BUILDDIR/address.db"}; STREET_DB=${STREET_DB:-"$BUILDDIR/street.db"}; # location of stdio files -PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate.out"}; -PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate.err"}; -PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/conflate.skip"}; +PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate_oa.out"}; +PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate_oa.err"}; +PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/conflate_oa.skip"}; # a directory with enough free space to store sqlite tmp files export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"}; # ensure tmpdir exists -[ -d $SQLITE_TMPDIR ] || mkdir $SQLITE_TMPDIR; +[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR; # delete previous stdio files rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR; diff --git a/script/conflate_osm.sh b/script/conflate_osm.sh new file mode 100755 index 00000000..3b40e416 --- /dev/null +++ b/script/conflate_osm.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e; +export LC_ALL=en_US.UTF-8; + +# import openstreetmap data and conflate it with $STREET_DB + +# location of this file in filesystem +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ); + +# ---- pbf2json ---- + +# location of pbf2json binary +PBF2JSON_BIN=${PBF2JSON_BIN:-"pbf2json.linux-x64"}; + +# ensure pbf2json exists and is executable +if [[ ! -f $PBF2JSON_BIN || ! -x $PBF2JSON_BIN ]]; then + echo "pbf2json not found or is not executable"; + exit 1; +fi + +# tags to target from the PBF extract +PBF2JSON_TAGS=${PBF2JSON_TAGS:-"addr:housenumber+addr:street"}; + +# full path to the .osm.pbf file we would like to import +PBF2JSON_FILE=${PBF2JSON_FILE:-"planet.osm.pbf"}; + +# a directory with enough free space to store leveldb tmp files +PBF2JSON_TMPDIR=${PBF2JSON_TMPDIR:-"$BUILDDIR/tmp/leveldb"}; + +# ensure tmpdir exists +[ -d $PBF2JSON_TMPDIR ] || mkdir -p $PBF2JSON_TMPDIR; + +# ---- -------- ---- + +# location where this build will be stored +BUILDDIR=${BUILDDIR:-"/data"}; +if [ ! -d $BUILDDIR ]; then + echo "data dir does not exist"; + exit 1; +fi + +# location of sql databases +ADDRESS_DB=${ADDRESS_DB:-"$BUILDDIR/address.db"}; +STREET_DB=${STREET_DB:-"$BUILDDIR/street.db"}; + +# location of stdio files +PROC_STDOUT=${PROC_STDOUT:-"$BUILDDIR/conflate_osm.out"}; +PROC_STDERR=${PROC_STDERR:-"$BUILDDIR/conflate_osm.err"}; + +# a directory with enough free space to store sqlite tmp files +export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"}; + +# ensure tmpdir exists +[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR; + +# delete previous stdio files +rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR; + +# run import +$PBF2JSON_BIN -tags="$PBF2JSON_TAGS" -leveldb="$PBF2JSON_TMPDIR" $PBF2JSON_FILE |\ + time -p node $DIR/../cmd/osm.js $ADDRESS_DB $STREET_DB 1>$PROC_STDOUT 2>$PROC_STDERR; diff --git a/script/cronjob.sh b/script/cronjob.sh index af74d0d1..21669ab9 100755 --- a/script/cronjob.sh +++ b/script/cronjob.sh @@ -22,5 +22,9 @@ git pull -q origin master; # update npm dependencies npm --loglevel=silent install; +# update openaddresses data (optional) +export OAPATH="/data/oa"; # base path of openaddresses file system +$DIR/update_oa.sh; + # run build $DIR/build.sh &> build.log; diff --git a/script/vertices.sh b/script/vertices.sh index 641940d3..dcf03c8f 100755 --- a/script/vertices.sh +++ b/script/vertices.sh @@ -27,7 +27,7 @@ PROC_CONFERR=${PROC_CONFERR:-"$BUILDDIR/vertices.skip"}; export SQLITE_TMPDIR=${SQLITE_TMPDIR:-"$BUILDDIR/tmp"}; # ensure tmpdir exists -[ -d $SQLITE_TMPDIR ] || mkdir $SQLITE_TMPDIR; +[ -d $SQLITE_TMPDIR ] || mkdir -p $SQLITE_TMPDIR; # delete previous stdio files rm -f $PROC_STDOUT $PROC_STDERR $PROC_CONFERR;