diff --git a/package.json b/package.json index c90f2b3..5567586 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "pelias-config": "^4.0.0", "pelias-logger": "^1.2.1", "pelias-microservice-wrapper": "^1.2.1", - "pelias-whosonfirst": "^3.0.0", + "pelias-whosonfirst": "^4.0.0", "polygon-lookup": "^2.1.0", "request": "^2.83.0", "simplify-js": "^1.2.1", diff --git a/schema.js b/schema.js index c11cc22..b22c62e 100644 --- a/schema.js +++ b/schema.js @@ -16,7 +16,6 @@ module.exports = Joi.object().keys({ Joi.number().integer(), Joi.array().items(Joi.number().integer()) ], - sqlite: Joi.boolean().default(false).truthy('yes').falsy('no') }).unknown(true), services: Joi.object().keys({ pip: Joi.object().keys({ diff --git a/src/pip/index.js b/src/pip/index.js index 05a4750..d218e29 100644 --- a/src/pip/index.js +++ b/src/pip/index.js @@ -12,7 +12,6 @@ const async = require('async'); const _ = require('lodash'); const fs = require('fs'); const missingMetafilesAreFatal = require('pelias-config').generate(require('../../schema')).imports.adminLookup.missingMetafilesAreFatal; -const isSqlite = require('pelias-config').generate(require('../../schema')).imports.whosonfirst.sqlite; let requestCount = 0; // worker processes keyed on layer @@ -43,36 +42,9 @@ module.exports.create = function createPIPService(datapath, layers, localizedAdm // ie - _.intersection([1, 2, 3], [3, 1]) === [1, 3] layers = _.intersection(defaultLayers, _.isEmpty(layers) ? defaultLayers : layers); - if (isSqlite === true) { - const folder = path.join(datapath, 'sqlite'); - if (!fs.existsSync(folder)) { - return callback(`unable to locate sqlite folder`); - } - } else { - // keep track of any missing metafiles for later reporting and error conditions - const missingMetafiles = []; - - // further refine the layers by filtering out layers for which there is no metafile - layers = layers.filter(layer => { - const filename = path.join(datapath, 'meta', `whosonfirst-data-${layer}-latest.csv`); - - if (!fs.existsSync(filename)) { - const message = `unable to locate ${filename}`; - if (missingMetafilesAreFatal) { - logger.error(message); - } else { - logger.warn(message); - } - missingMetafiles.push(`whosonfirst-data-${layer}-latest.csv`); - return false; - } - return true; - }); - - // if there are missing metafiles and this is fatal, then return an error - if (!_.isEmpty(missingMetafiles) && missingMetafilesAreFatal) { - return callback(`unable to locate meta files in ${path.join(datapath, 'meta')}: ${missingMetafiles.join(', ')}`); - } + const folder = path.join(datapath, 'sqlite'); + if (!fs.existsSync(folder)) { + return callback(`unable to locate sqlite folder`); } logger.info(`starting with layers ${layers}`); diff --git a/src/pip/readStream.js b/src/pip/readStream.js index 19ca4c2..07ff786 100644 --- a/src/pip/readStream.js +++ b/src/pip/readStream.js @@ -13,14 +13,6 @@ const SQLiteStream = whosonfirst.SQLiteStream; const SQLITE_REGEX = /whosonfirst-data-[a-z0-9-]+\.db$/; -function readBundleRecords(datapath, layer) { - return whosonfirst.metadataStream(datapath).create(layer) - .pipe(whosonfirst.parseMetaFiles()) - .pipe(whosonfirst.isNotNullIslandRelated()) - .pipe(whosonfirst.recordHasName()) - .pipe(whosonfirst.loadJSON(datapath, false)); -} - function getSqliteFilePaths(root) { return fs.readdirSync(root) .filter(d => SQLITE_REGEX.test(d)) @@ -54,11 +46,7 @@ function readSqliteRecords(datapath, layer) { function readData(datapath, layer, localizedAdminNames, callback) { const features = []; - const stream = config.sqlite === true ? - readSqliteRecords(datapath, layer) : - readBundleRecords(datapath, layer); - - stream + readSqliteRecords(datapath,layer) .pipe(whosonfirst.recordHasIdAndProperties()) .pipe(whosonfirst.isActiveRecord()) .pipe(filterOutPointRecords.create()) @@ -75,4 +63,4 @@ function readData(datapath, layer, localizedAdminNames, callback) { } -module.exports = readData; \ No newline at end of file +module.exports = readData; diff --git a/test/pip/index.js b/test/pip/index.js index 8d663fb..b35088b 100644 --- a/test/pip/index.js +++ b/test/pip/index.js @@ -11,834 +11,622 @@ const customConfig = require('../testUtils').customConfig; const pip = require('../../src/pip/index'); tape('PiP tests', test => { - ['bundle', 'sqlite'].forEach(wofDataType => { - const globalConfig = { imports: { adminLookup: {}, whosonfirst: { sqlite: wofDataType === 'sqlite' } } }; - test.test(`${wofDataType}: empty array should be returned when search_layers is empty even if lat/lon is in a polygon`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta files with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // and write it to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record]); - - const service = pip.create(temp_dir, ['neighbourhood'], false, (err, o) => { - // do a lookup that specifies empty layers - o.lookup(0.5, 0.5, [], (err, results) => { - t.deepEquals(results, [], 'no results should have been returned'); - // must be explicitly ended or the test hangs - o.end(); - config.unset(); - t.end(); - }); + test.test(`empty array should be returned when search_layers is empty even if lat/lon is in a polygon`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); + + // write out the WOF meta files with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,2,2', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[2,1],[2,2],[1,2],[1,1] + ] + ], + type: 'Polygon' + } + }; + // and write it to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record]); + + const service = pip.create(temp_dir, ['neighbourhood'], false, (err, o) => { + // do a lookup that specifies empty layers + o.lookup(0.5, 0.5, [], (err, results) => { + t.deepEquals(results, [], 'no results should have been returned'); + // must be explicitly ended or the test hangs + o.end(); + config.unset(); + t.end(); }); }); }); - test.test(`${wofDataType}: requested lat/lon inside a neighbourhood polygon should return only neighbourhood`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta files with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; + }); - // setup a borough WOF record that doesn't contain the lookup point to - // show that hierarchy is used to establish the response - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '3,3,4,4', - 'geom:latitude': 3.5, - 'geom:longitude': 3.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [3,3],[3,4],[4,4],[4,3],[3,3] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { - // lookup of point that only hits neighbourhood will NOT return borough - o.lookup(1.5, 1.5, undefined, (err, results) => { - t.deepEquals(results, [ - { - Id: 123, - Name: 'neighbourhood name', - Placetype: 'neighbourhood', - Centroid: { - lat: 1.5, - lon: 1.5 - }, - BoundingBox: '1,1,2,2', - Hierarchy: [ [ 123, 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - o.end(); - config.unset(); - t.end(); - }); + test.test(`requested lat/lon inside a neighbourhood polygon should return only neighbourhood`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); - }); + // write out the WOF meta files with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), + `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - }); + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,2,2', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[2,1],[2,2],[1,2],[1,1] + ] + ], + type: 'Polygon' + } + }; - }); + // setup a borough WOF record that doesn't contain the lookup point to + // show that hierarchy is used to establish the response + const borough_record = { + id: 456, + type: 'Feature', + properties: { + 'geom:bbox': '3,3,4,4', + 'geom:latitude': 3.5, + 'geom:longitude': 3.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [], + 'wof:id': 456, + 'wof:name': 'borough name', + 'wof:placetype': 'borough' + }, + geometry: { + coordinates: [ + [ + [3,3],[3,4],[4,4],[4,3],[3,3] + ] + ], + type: 'Polygon' + } + }; - test.test(`${wofDataType}: requested lat/lon inside a neighbourhood and parent borough should return full hierarchy`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta files with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,4,4', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[4,1],[4,4],[1,4],[1,1] - ] - ], - type: 'Polygon' - } - }; + // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - // setup a borough WOF record that doesn't contain the lookup point to - // show that hierarchy is used to establish the response - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '3,3,4,4', - 'geom:latitude': 3.5, - 'geom:longitude': 3.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [3,3],[3,4],[4,4],[4,3],[3,3] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { - // lookup of point that only hits neighbourhood will NOT return borough - o.lookup(3.5, 3.5, undefined, (err, results) => { - t.deepEquals(results, [ - { - Id: 123, - Name: 'neighbourhood name', - Placetype: 'neighbourhood', - Centroid: { - lat: 1.5, - lon: 1.5 - }, - BoundingBox: '1,1,4,4', - Hierarchy: [ [ 123, 456 ] ] + const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { + // lookup of point that only hits neighbourhood will NOT return borough + o.lookup(1.5, 1.5, undefined, (err, results) => { + t.deepEquals(results, [ + { + Id: 123, + Name: 'neighbourhood name', + Placetype: 'neighbourhood', + Centroid: { + lat: 1.5, + lon: 1.5 }, - { - Id: 456, - Name: 'borough name', - Placetype: 'borough', - Centroid: { - lat: 3.5, - lon: 3.5 - }, - BoundingBox: '3,3,4,4', - Hierarchy: [ [ 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - o.end(); - config.unset(); - t.end(); - }); - + BoundingBox: '1,1,2,2', + Hierarchy: [ [ 123, 456 ] ] + } + ]); + // must be explicitly ended or the test hangs + o.end(); + config.unset(); + t.end(); }); }); }); - test.test(`${wofDataType}: querying for only neighbourhood layer should work fine`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta files with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,4,4', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[4,1],[4,4],[1,4],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // setup a borough WOF record that doesn't contain the lookup point to - // show that hierarchy is used to establish the response - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '3,3,4,4', - 'geom:latitude': 3.5, - 'geom:longitude': 3.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [3,3],[3,4],[4,4],[4,3],[3,3] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { - // lookup of point that only hits neighbourhood will NOT return borough - o.lookup(3.5, 3.5, ['neighbourhood'], (err, results) => { - t.deepEquals(results, [ - { - Id: 123, - Name: 'neighbourhood name', - Placetype: 'neighbourhood', - Centroid: { - lat: 1.5, - lon: 1.5 - }, - BoundingBox: '1,1,4,4', - Hierarchy: [ [ 123, 456 ] ] - }, - { - Id: 456, - Name: 'borough name', - Placetype: 'borough', - Centroid: { - lat: 3.5, - lon: 3.5 - }, - BoundingBox: '3,3,4,4', - Hierarchy: [ [ 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - o.end(); - config.unset(); - t.end(); - }); + }); - }); + test.test(`requested lat/lon inside a neighbourhood and parent borough should return full hierarchy`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); - }); + // write out the WOF meta files with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), + `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - }); + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,4,4', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[4,1],[4,4],[1,4],[1,1] + ] + ], + type: 'Polygon' + } + }; - test.test(`${wofDataType}: first layer not containing the point should fallback to other layers`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta files with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; + // setup a borough WOF record that doesn't contain the lookup point to + // show that hierarchy is used to establish the response + const borough_record = { + id: 456, + type: 'Feature', + properties: { + 'geom:bbox': '3,3,4,4', + 'geom:latitude': 3.5, + 'geom:longitude': 3.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [], + 'wof:id': 456, + 'wof:name': 'borough name', + 'wof:placetype': 'borough' + }, + geometry: { + coordinates: [ + [ + [3,3],[3,4],[4,4],[4,3],[3,3] + ] + ], + type: 'Polygon' + } + }; - // setup a borough WOF record that doesn't contain the lookup point to - // show that hierarchy is used to establish the response - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '3,3,4,4', - 'geom:latitude': 3.5, - 'geom:longitude': 3.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [3,3],[3,4],[4,4],[4,3],[3,3] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { - o.lookup(3.5, 3.5, undefined, (err, results) => { - t.deepEquals(results, [ - { - Id: 456, - Name: 'borough name', - Placetype: 'borough', - Centroid: { - lat: 3.5, - lon: 3.5 - }, - BoundingBox: '3,3,4,4', - Hierarchy: [ [ 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - o.end(); - config.unset(); - t.end(); - }); + // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); + const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { + // lookup of point that only hits neighbourhood will NOT return borough + o.lookup(3.5, 3.5, undefined, (err, results) => { + t.deepEquals(results, [ + { + Id: 123, + Name: 'neighbourhood name', + Placetype: 'neighbourhood', + Centroid: { + lat: 1.5, + lon: 1.5 + }, + BoundingBox: '1,1,4,4', + Hierarchy: [ [ 123, 456 ] ] + }, + { + Id: 456, + Name: 'borough name', + Placetype: 'borough', + Centroid: { + lat: 3.5, + lon: 3.5 + }, + BoundingBox: '3,3,4,4', + Hierarchy: [ [ 456 ] ] + } + ]); + // must be explicitly ended or the test hangs + o.end(); + config.unset(); + t.end(); }); }); }); - test.test(`${wofDataType}: no layer containing the point should return an empty array`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta file with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // setup a borough WOF record that doesn't contain the lookup point - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '3,3,4,4', - 'geom:latitude': 3.5, - 'geom:longitude': 3.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [3,3],[3,4],[4,4],[4,3],[3,3] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, service) => { - service.lookup(10, 10, undefined, (err, results) => { - t.deepEquals(results, []); - // must be explicitly ended or the test hangs - service.end(); - config.unset(); - t.end(); - }); + }); - }); + test.test(`querying for only neighbourhood layer should work fine`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); - }); + // write out the WOF meta files with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), + `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - }); + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,4,4', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[4,1],[4,4],[1,4],[1,1] + ] + ], + type: 'Polygon' + } + }; - test.test(`${wofDataType}: explicit layers should skip unincluded layers`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta file with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a neighbourhood WOF record - // this has the same geometry as the borough record to show that it will be skipped - const neighbourhood_record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 123, - 'wof:name': 'neighbourhood name', - 'wof:placetype': 'neighbourhood' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; + // setup a borough WOF record that doesn't contain the lookup point to + // show that hierarchy is used to establish the response + const borough_record = { + id: 456, + type: 'Feature', + properties: { + 'geom:bbox': '3,3,4,4', + 'geom:latitude': 3.5, + 'geom:longitude': 3.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [], + 'wof:id': 456, + 'wof:name': 'borough name', + 'wof:placetype': 'borough' + }, + geometry: { + coordinates: [ + [ + [3,3],[3,4],[4,4],[4,3],[3,3] + ] + ], + type: 'Polygon' + } + }; - // setup a borough WOF record that's the exact same geometry as neighbourhood - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - borough_id: 456 - } - ], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - - pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, service) => { - service.lookup(1.5, 1.5, ['borough'], (err, results) => { - t.deepEquals(results, [ - { - Id: 456, - Name: 'borough name', - Placetype: 'borough', - Centroid: { - lat: 1.5, - lon: 1.5 - }, - BoundingBox: '1,1,2,2', - Hierarchy: [ [ 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - service.end(); - config.unset(); - t.end(); - }); + // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); + const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { + // lookup of point that only hits neighbourhood will NOT return borough + o.lookup(3.5, 3.5, ['neighbourhood'], (err, results) => { + t.deepEquals(results, [ + { + Id: 123, + Name: 'neighbourhood name', + Placetype: 'neighbourhood', + Centroid: { + lat: 1.5, + lon: 1.5 + }, + BoundingBox: '1,1,4,4', + Hierarchy: [ [ 123, 456 ] ] + }, + { + Id: 456, + Name: 'borough name', + Placetype: 'borough', + Centroid: { + lat: 3.5, + lon: 3.5 + }, + BoundingBox: '3,3,4,4', + Hierarchy: [ [ 456 ] ] + } + ]); + // must be explicitly ended or the test hangs + o.end(); + config.unset(); + t.end(); }); }); }); - test.test(`${wofDataType}: only layers specified by create should be loaded and should not be checked`, t => { - const logger = require('pelias-mock-logger')(); - - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta file with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), - 'this is not a valid WOF meta file'); - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), - `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - - // setup a borough WOF record that's the exact same geometry as neighbourhood - const borough_record = { - id: 456, - type: 'Feature', - properties: { - 'geom:bbox': '1,1,2,2', - 'geom:latitude': 1.5, - 'geom:longitude': 1.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - locality_id: 123, - region_id: 456 - } - ], - 'wof:id': 456, - 'wof:name': 'borough name', - 'wof:placetype': 'borough' - }, - geometry: { - coordinates: [ - [ - [1,1],[2,1],[2,2],[1,2],[1,1] - ] - ], - type: 'Polygon' - } - }; + }); - // and write the records to file - fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), 'this isn\'t JSON'); - fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [borough_record]); + test.test(`first layer not containing the point should fallback to other layers`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); - const pip = proxyquire('../../src/pip/index', { - 'pelias-logger': logger - }); + // write out the WOF meta files with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), + `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); - // load only the neighbourhood layer - pip.create(temp_dir, ['borough'], false, (err, service) => { - t.notOk(logger.isInfoMessage(/neighbourhood worker loaded .+/)); - t.ok(logger.isInfoMessage(/borough worker loaded 1 features in \d+\.\d+ seconds/)); - - // but request both neighbourhood and borough layers - service.lookup(1.5, 1.5, ['neighbourhood', 'borough'], (err, results) => { - t.deepEquals(results, [ - { - Id: 456, - Name: 'borough name', - Placetype: 'borough', - Centroid: { - lat: 1.5, - lon: 1.5 - }, - BoundingBox: '1,1,2,2', - Hierarchy: [ [ 123, 456 ] ] - } - ]); - // must be explicitly ended or the test hangs - service.end(); - config.unset(); - t.end(); - }); + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,2,2', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[2,1],[2,2],[1,2],[1,1] + ] + ], + type: 'Polygon' + } + }; + + // setup a borough WOF record that doesn't contain the lookup point to + // show that hierarchy is used to establish the response + const borough_record = { + id: 456, + type: 'Feature', + properties: { + 'geom:bbox': '3,3,4,4', + 'geom:latitude': 3.5, + 'geom:longitude': 3.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [], + 'wof:id': 456, + 'wof:name': 'borough name', + 'wof:placetype': 'borough' + }, + geometry: { + coordinates: [ + [ + [3,3],[3,4],[4,4],[4,3],[3,3] + ] + ], + type: 'Polygon' + } + }; + + // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); + const service = pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, o) => { + o.lookup(3.5, 3.5, undefined, (err, results) => { + t.deepEquals(results, [ + { + Id: 456, + Name: 'borough name', + Placetype: 'borough', + Centroid: { + lat: 3.5, + lon: 3.5 + }, + BoundingBox: '3,3,4,4', + Hierarchy: [ [ 456 ] ] + } + ]); + // must be explicitly ended or the test hangs + o.end(); + config.unset(); + t.end(); }); }); }); - test.test(`${wofDataType}: no layers specified for create should load all layers`, t => { - const logger = require('pelias-mock-logger')(); + }); - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); + test.test(`no layer containing the point should return an empty array`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); - // write out the WOF meta file with the minimum required fields - [ - 'neighbourhood', 'borough', 'locality', 'localadmin', 'county', - 'macrocounty', 'region', 'macroregion', 'dependency', 'country', 'empire', - 'continent', 'marinearea', 'ocean'].forEach(layer => { - fs.writeFileSync( - path.join(temp_dir, 'meta', `whosonfirst-data-${layer}-latest.csv`), `id,name,path${EOL}`); - }); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), []); - - const pip = proxyquire('../../src/pip/index', { - 'pelias-logger': logger - }); + // write out the WOF meta file with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), + `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); + + // setup a neighbourhood WOF record + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,2,2', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[2,1],[2,2],[1,2],[1,1] + ] + ], + type: 'Polygon' + } + }; - // don't specify layers so all are loaded - pip.create(temp_dir, undefined, false, (err, service) => { - t.ok(logger.isInfoMessage(/neighbourhood worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/borough worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/locality worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/localadmin worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/county worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/macrocounty worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/region worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/macroregion worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/dependency worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/country worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/empire worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/continent worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/marinearea worker loaded 0 features in \d+\.\d+ seconds/)); - t.ok(logger.isInfoMessage(/ocean worker loaded 0 features in \d+\.\d+ seconds/)); - - t.equals(logger.getInfoMessages().pop(), 'PIP Service Loading Completed!!!'); + // setup a borough WOF record that doesn't contain the lookup point + const borough_record = { + id: 456, + type: 'Feature', + properties: { + 'geom:bbox': '3,3,4,4', + 'geom:latitude': 3.5, + 'geom:longitude': 3.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [], + 'wof:id': 456, + 'wof:name': 'borough name', + 'wof:placetype': 'borough' + }, + geometry: { + coordinates: [ + [ + [3,3],[3,4],[4,4],[4,3],[3,3] + ] + ], + type: 'Polygon' + } + }; + + // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); + fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); + pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, service) => { + service.lookup(10, 10, undefined, (err, results) => { + t.deepEquals(results, []); + // must be explicitly ended or the test hangs service.end(); config.unset(); t.end(); - }); }); }); - }); - test.test('layers missing metafiles when not fatal should report error and skip lookup at those layers', t => { - const logger = require('pelias-mock-logger')(); + }); + test.test(`explicit layers should skip unincluded layers`, t => { temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); fs.mkdirSync(path.join(temp_dir, 'data')); fs.mkdirSync(path.join(temp_dir, 'meta')); // write out the WOF meta file with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + `id,name,path${EOL}123,place name,neighbourhood_record.geojson${EOL}`); fs.writeFileSync( path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); + // setup a neighbourhood WOF record + // this has the same geometry as the borough record to show that it will be skipped + const neighbourhood_record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '1,1,2,2', + 'geom:latitude': 1.5, + 'geom:longitude': 1.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + locality_id: 123, + region_id: 456 + } + ], + 'wof:id': 123, + 'wof:name': 'neighbourhood name', + 'wof:placetype': 'neighbourhood' + }, + geometry: { + coordinates: [ + [ + [1,1],[2,1],[2,2],[1,2],[1,1] + ] + ], + type: 'Polygon' + } + }; + // setup a borough WOF record that's the exact same geometry as neighbourhood const borough_record = { id: 456, @@ -868,26 +656,12 @@ tape('PiP tests', test => { }; // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), JSON.stringify(neighbourhood_record)); fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [neighbourhood_record, borough_record]); - // Override env because proxyquire can't be used in child_process - const config = customConfig(temp_dir, { imports: { adminLookup: {}, whosonfirst: { sqlite: false } } }); - const pip = proxyquire('../../src/pip/index', { - 'pelias-logger': logger, - 'pelias-config': { - generate: () => { - return config.content; - } - } - }); - - // initialize PiP with neighbourhood/locality (that don't exist) and borough (which does exist) pip.create(temp_dir, ['neighbourhood', 'borough'], false, (err, service) => { - t.deepEquals(logger.getWarnMessages(), [ - 'unable to locate ' + path.join(temp_dir, 'meta', `whosonfirst-data-neighbourhood-latest.csv`) - ]); - - service.lookup(1.5, 1.5, undefined, (err, results) => { + service.lookup(1.5, 1.5, ['borough'], (err, results) => { t.deepEquals(results, [ { Id: 456, @@ -913,14 +687,18 @@ tape('PiP tests', test => { }); - test.test('layers missing metafiles when fatal should return error on callback', t => { + test.test(`only layers specified by create should be loaded and should not be checked`, t => { const logger = require('pelias-mock-logger')(); temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); fs.mkdirSync(path.join(temp_dir, 'data')); fs.mkdirSync(path.join(temp_dir, 'meta')); // write out the WOF meta file with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-neighbourhood-latest.csv'), + 'this is not a valid WOF meta file'); fs.writeFileSync( path.join(temp_dir, 'meta', 'whosonfirst-data-borough-latest.csv'), `id,name,path${EOL}456,borough name,borough_record.geojson${EOL}`); @@ -936,7 +714,8 @@ tape('PiP tests', test => { 'mz:hierarchy_label': 1, 'wof:hierarchy': [ { - borough_id: 456 + locality_id: 123, + region_id: 456 } ], 'wof:id': 456, @@ -954,35 +733,39 @@ tape('PiP tests', test => { }; // and write the records to file + fs.writeFileSync(path.join(temp_dir, 'data', 'neighbourhood_record.geojson'), 'this isn\'t JSON'); fs.writeFileSync(path.join(temp_dir, 'data', 'borough_record.geojson'), JSON.stringify(borough_record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [borough_record]); const pip = proxyquire('../../src/pip/index', { - 'pelias-logger': logger, - 'pelias-config': { - generate: () => { - return { - imports: { - adminLookup: { - missingMetafilesAreFatal: true - }, - whosonfirst: { sqlite: false } - } - }; - } - } + 'pelias-logger': logger }); - // initialize PiP with neighbourhood (that doesn't exist) and borough (which does exist) - pip.create(temp_dir, ['neighbourhood', 'borough', 'locality'], false, (err, service) => { - t.deepEquals(logger.getErrorMessages(), [ - 'unable to locate ' + path.join(temp_dir, 'meta', `whosonfirst-data-neighbourhood-latest.csv`), - 'unable to locate ' + path.join(temp_dir, 'meta', `whosonfirst-data-locality-latest.csv`) - ], 'should have an error message'); + // load only the neighbourhood layer + pip.create(temp_dir, ['borough'], false, (err, service) => { + t.notOk(logger.isInfoMessage(/neighbourhood worker loaded .+/)); + t.ok(logger.isInfoMessage(/borough worker loaded 1 features in \d+\.\d+ seconds/)); - t.deepEquals(err, `unable to locate meta files in ${path.join(temp_dir, 'meta')}` + - ': whosonfirst-data-neighbourhood-latest.csv, whosonfirst-data-locality-latest.csv'); - t.notOk(service); - t.end(); + // but request both neighbourhood and borough layers + service.lookup(1.5, 1.5, ['neighbourhood', 'borough'], (err, results) => { + t.deepEquals(results, [ + { + Id: 456, + Name: 'borough name', + Placetype: 'borough', + Centroid: { + lat: 1.5, + lon: 1.5 + }, + BoundingBox: '1,1,2,2', + Hierarchy: [ [ 123, 456 ] ] + } + ]); + // must be explicitly ended or the test hangs + service.end(); + config.unset(); + t.end(); + }); }); @@ -990,6 +773,54 @@ tape('PiP tests', test => { }); + test.test(`no layers specified for create should load all layers`, t => { + const logger = require('pelias-mock-logger')(); + + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); + + // write out the WOF meta file with the minimum required fields + [ + 'neighbourhood', 'borough', 'locality', 'localadmin', 'county', + 'macrocounty', 'region', 'macroregion', 'dependency', 'country', 'empire', + 'continent', 'marinearea', 'ocean'].forEach(layer => { + fs.writeFileSync( + path.join(temp_dir, 'meta', `whosonfirst-data-${layer}-latest.csv`), `id,name,path${EOL}`); + }); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), []); + + const pip = proxyquire('../../src/pip/index', { + 'pelias-logger': logger + }); + + // don't specify layers so all are loaded + pip.create(temp_dir, undefined, false, (err, service) => { + t.ok(logger.isInfoMessage(/neighbourhood worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/borough worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/locality worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/localadmin worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/county worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/macrocounty worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/region worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/macroregion worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/dependency worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/country worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/empire worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/continent worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/marinearea worker loaded 0 features in \d+\.\d+ seconds/)); + t.ok(logger.isInfoMessage(/ocean worker loaded 0 features in \d+\.\d+ seconds/)); + + t.equals(logger.getInfoMessages().pop(), 'PIP Service Loading Completed!!!'); + + service.end(); + config.unset(); + t.end(); + }); + }); + }); + test.test('Should load SQLite when option is activated', t => { const logger = require('pelias-mock-logger')(); const defaultGeom = { diff --git a/test/pip/worker.js b/test/pip/worker.js index a03d731..966f632 100644 --- a/test/pip/worker.js +++ b/test/pip/worker.js @@ -9,182 +9,179 @@ const generateWOFDB = require('pelias-whosonfirst/test/generateWOFDB'); const customConfig = require('../testUtils').customConfig; tape('worker tests', (test) => { - ['bundle', 'sqlite'].forEach(wofDataType => { - const globalConfig = { imports: { whosonfirst: { sqlite: wofDataType === 'sqlite' } } }; - test.test(`${wofDataType}: requested lat/lon inside a polygon should return that record\'s hierarchy`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta file with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-test_layer-latest.csv'), - `id,name,path${EOL}123,place name,record.geojson${EOL}`); - - // setup a WOF record - const record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '-1,-1,1,1', - 'geom:latitude': 0.5, - 'geom:longitude': 0.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - layer1_id: 11, - layer2_id: 13 - } - ], - 'wof:id': 17, - 'wof:name': 'record name', - 'wof:placetype': 'test_layer' - }, - geometry: { - coordinates: [ - [ - [1,1],[1,-1],[-1,-1],[-1,1],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // and write it to file - fs.writeFileSync(path.join(temp_dir, 'data', 'record.geojson'), JSON.stringify(record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [record]); - - // for a worker process - const worker = childProcess.fork(path.join('src', 'pip', 'worker'), ['test_layer', temp_dir]); - - worker.on('message', msg => { - // when a 'loaded' message is received, send a 'search' request for a lat/lon - if (msg.type === 'loaded') { - t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer'); - t.ok(_.isFinite(msg.seconds), 'time to load should have been returned'); - t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message'); - - // in this case the lat/lon is inside the known polygon - worker.send({ - type: 'search', - id: 'test request id', - coords: { - latitude: 0.5, - longitude: -0.5 - } - }); - - } else if (msg.type === 'results') { - // when the 'results' message is received, make some assertions and kill the worker - temp.cleanupSync(); - - t.deepEquals(msg, { - id: 'test request id', - type: 'results', - layer: 'test_layer', - results: { - Id: 17, - Hierarchy: [ [11, 13] ] - } - }, 'the hierarchy of the WOF record should be returned'); - - t.ok(worker.kill(), 'the worker should be killed'); - config.unset(); - t.end(); - - } - - }); + test.test(`requested lat/lon inside a polygon should return that record\'s hierarchy`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); + + // write out the WOF meta file with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-test_layer-latest.csv'), + `id,name,path${EOL}123,place name,record.geojson${EOL}`); + + // setup a WOF record + const record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '-1,-1,1,1', + 'geom:latitude': 0.5, + 'geom:longitude': 0.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + layer1_id: 11, + layer2_id: 13 + } + ], + 'wof:id': 17, + 'wof:name': 'record name', + 'wof:placetype': 'test_layer' + }, + geometry: { + coordinates: [ + [ + [1,1],[1,-1],[-1,-1],[-1,1],[1,1] + ] + ], + type: 'Polygon' + } + }; + + // and write it to file + fs.writeFileSync(path.join(temp_dir, 'data', 'record.geojson'), JSON.stringify(record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [record]); + + // for a worker process + const worker = childProcess.fork(path.join('src', 'pip', 'worker'), ['test_layer', temp_dir]); + + worker.on('message', msg => { + // when a 'loaded' message is received, send a 'search' request for a lat/lon + if (msg.type === 'loaded') { + t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer'); + t.ok(_.isFinite(msg.seconds), 'time to load should have been returned'); + t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message'); + + // in this case the lat/lon is inside the known polygon + worker.send({ + type: 'search', + id: 'test request id', + coords: { + latitude: 0.5, + longitude: -0.5 + } + }); + + } else if (msg.type === 'results') { + // when the 'results' message is received, make some assertions and kill the worker + temp.cleanupSync(); + + t.deepEquals(msg, { + id: 'test request id', + type: 'results', + layer: 'test_layer', + results: { + Id: 17, + Hierarchy: [ [11, 13] ] + } + }, 'the hierarchy of the WOF record should be returned'); + + t.ok(worker.kill(), 'the worker should be killed'); + config.unset(); + t.end(); + + } }); }); - test.test(`${wofDataType}: requested lat/lon outside any polygons should return empty results`, t => { - temp.mkdir('tmp_wof_data', (err, temp_dir) => { - const config = customConfig(temp_dir, globalConfig); - fs.mkdirSync(path.join(temp_dir, 'data')); - fs.mkdirSync(path.join(temp_dir, 'meta')); - - // write out the WOF meta file with the minimum required fields - fs.writeFileSync( - path.join(temp_dir, 'meta', 'whosonfirst-data-test_layer-latest.csv'), - `id,name,path${EOL}123,place name,record.geojson${EOL}`); - - // setup a WOF record - const record = { - id: 123, - type: 'Feature', - properties: { - 'geom:bbox': '-1,-1,1,1', - 'geom:latitude': 0.5, - 'geom:longitude': 0.5, - 'mz:hierarchy_label': 1, - 'wof:hierarchy': [ - { - layer1_id: 11, - layer2_id: 13 - } - ], - 'wof:id': 17, - 'wof:name': 'record name', - 'wof:placetype': 'test_layer' - }, - geometry: { - coordinates: [ - [ - [1,1],[1,-1],[-1,-1],[-1,1],[1,1] - ] - ], - type: 'Polygon' - } - }; - - // and write it to file - fs.writeFileSync(path.join(temp_dir, 'data', 'record.geojson'), JSON.stringify(record)); - generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [record]); - - const worker = childProcess.fork(path.join('src', 'pip', 'worker'), ['test_layer', temp_dir]); - - worker.on('message', msg => { - // when a 'loaded' message is received, send a 'search' request for a lat/lon - if (msg.type === 'loaded') { - t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer'); - t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message'); - t.ok(_.isFinite(msg.seconds), 'time to load should have been returned'); - - // in this case the lat/lon is outside of any known polygons - worker.send({ - type: 'search', - id: 'test request id', - coords: { - latitude: 1.5, - longitude: -1.5 - } - }); - - } else if (msg.type === 'results') { - // when the 'results' message is received, make some assertions and kill the worker - temp.cleanupSync(); - - t.deepEquals(msg, { - id: 'test request id', - type: 'results', - layer: 'test_layer', - results: {} - }, 'no results should be returned'); - - t.ok(worker.kill(), 'the worker should be killed'); - config.unset(); - t.end(); - - } - - }); + }); + + test.test(`requested lat/lon outside any polygons should return empty results`, t => { + temp.mkdir('tmp_wof_data', (err, temp_dir) => { + const config = customConfig(temp_dir, {}); + fs.mkdirSync(path.join(temp_dir, 'data')); + fs.mkdirSync(path.join(temp_dir, 'meta')); + + // write out the WOF meta file with the minimum required fields + fs.writeFileSync( + path.join(temp_dir, 'meta', 'whosonfirst-data-test_layer-latest.csv'), + `id,name,path${EOL}123,place name,record.geojson${EOL}`); + + // setup a WOF record + const record = { + id: 123, + type: 'Feature', + properties: { + 'geom:bbox': '-1,-1,1,1', + 'geom:latitude': 0.5, + 'geom:longitude': 0.5, + 'mz:hierarchy_label': 1, + 'wof:hierarchy': [ + { + layer1_id: 11, + layer2_id: 13 + } + ], + 'wof:id': 17, + 'wof:name': 'record name', + 'wof:placetype': 'test_layer' + }, + geometry: { + coordinates: [ + [ + [1,1],[1,-1],[-1,-1],[-1,1],[1,1] + ] + ], + type: 'Polygon' + } + }; + + // and write it to file + fs.writeFileSync(path.join(temp_dir, 'data', 'record.geojson'), JSON.stringify(record)); + generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-latest.db'), [record]); + + const worker = childProcess.fork(path.join('src', 'pip', 'worker'), ['test_layer', temp_dir]); + + worker.on('message', msg => { + // when a 'loaded' message is received, send a 'search' request for a lat/lon + if (msg.type === 'loaded') { + t.equals(msg.layer, 'test_layer', 'the worker should respond with its requested layer'); + t.equals(_.size(msg.data), 1, 'a summary of the WOF data should be returned in the message'); + t.ok(_.isFinite(msg.seconds), 'time to load should have been returned'); + + // in this case the lat/lon is outside of any known polygons + worker.send({ + type: 'search', + id: 'test request id', + coords: { + latitude: 1.5, + longitude: -1.5 + } + }); + + } else if (msg.type === 'results') { + // when the 'results' message is received, make some assertions and kill the worker + temp.cleanupSync(); + + t.deepEquals(msg, { + id: 'test request id', + type: 'results', + layer: 'test_layer', + results: {} + }, 'no results should be returned'); + + t.ok(worker.kill(), 'the worker should be killed'); + config.unset(); + t.end(); + + } }); - }); + }); }); + });