feat: Always use sqlite as data source

BREAKING CHANGE: This implements a hard switchover to using SQLite files
for admin lookup. Using bundle files is no longer supported, and as a
result the `imports.whosonfirst.sqlite` setting is now ignored.

`wof-admin-lookup` will now behave as if that flag was always set to
true.

Closes #293

package.json

@@ -32,7 +32,7 @@
     "pelias-config": "^4.0.0",
     "pelias-logger": "^1.2.1",
     "pelias-microservice-wrapper": "^1.2.1",
-    "pelias-whosonfirst": "^3.0.0",
+    "pelias-whosonfirst": "^4.0.0",
     "polygon-lookup": "^2.1.0",
     "request": "^2.83.0",
     "simplify-js": "^1.2.1",

schema.js

@@ -16,7 +16,6 @@ module.exports = Joi.object().keys({
         Joi.number().integer(),
         Joi.array().items(Joi.number().integer())
       ],
-      sqlite: Joi.boolean().default(false).truthy('yes').falsy('no')
     }).unknown(true),
     services: Joi.object().keys({
       pip: Joi.object().keys({

src/pip/index.js

@@ -12,7 +12,6 @@ const async = require('async');
 const _ = require('lodash');
 const fs = require('fs');
 const missingMetafilesAreFatal = require('pelias-config').generate(require('../../schema')).imports.adminLookup.missingMetafilesAreFatal;
-const isSqlite = require('pelias-config').generate(require('../../schema')).imports.whosonfirst.sqlite;
 
 let requestCount = 0;
 // worker processes keyed on layer
@@ -43,36 +42,9 @@ module.exports.create = function createPIPService(datapath, layers, localizedAdm
   // ie - _.intersection([1, 2, 3], [3, 1]) === [1, 3]
   layers = _.intersection(defaultLayers, _.isEmpty(layers) ? defaultLayers : layers);
 
-  if (isSqlite === true) {
-    const folder = path.join(datapath, 'sqlite');
-    if (!fs.existsSync(folder)) {
-      return callback(`unable to locate sqlite folder`);
-    }
-  } else {
-    // keep track of any missing metafiles for later reporting and error conditions
-    const missingMetafiles = [];
-
-    // further refine the layers by filtering out layers for which there is no metafile
-    layers = layers.filter(layer => {
-      const filename = path.join(datapath, 'meta', `whosonfirst-data-${layer}-latest.csv`);
-
-      if (!fs.existsSync(filename)) {
-        const message = `unable to locate ${filename}`;
-        if (missingMetafilesAreFatal) {
-          logger.error(message);
-        } else {
-          logger.warn(message);
-        }
-        missingMetafiles.push(`whosonfirst-data-${layer}-latest.csv`);
-        return false;
-      }
-      return true;
-    });
-
-    // if there are missing metafiles and this is fatal, then return an error
-    if (!_.isEmpty(missingMetafiles) && missingMetafilesAreFatal) {
-      return callback(`unable to locate meta files in ${path.join(datapath, 'meta')}: ${missingMetafiles.join(', ')}`);
-    }
+  const folder = path.join(datapath, 'sqlite');
+  if (!fs.existsSync(folder)) {
+    return callback(`unable to locate sqlite folder`);
   }
 
   logger.info(`starting with layers ${layers}`);

src/pip/readStream.js

@@ -13,14 +13,6 @@ const SQLiteStream = whosonfirst.SQLiteStream;
 
 const SQLITE_REGEX = /whosonfirst-data-[a-z0-9-]+\.db$/;
 
-function readBundleRecords(datapath, layer) {
-  return whosonfirst.metadataStream(datapath).create(layer)
-    .pipe(whosonfirst.parseMetaFiles())
-    .pipe(whosonfirst.isNotNullIslandRelated())
-    .pipe(whosonfirst.recordHasName())
-    .pipe(whosonfirst.loadJSON(datapath, false));
-}
-
 function getSqliteFilePaths(root) {
   return fs.readdirSync(root)
     .filter(d => SQLITE_REGEX.test(d))
@@ -54,11 +46,7 @@ function readSqliteRecords(datapath, layer) {
 function readData(datapath, layer, localizedAdminNames, callback) {
   const features = [];
 
-  const stream = config.sqlite === true ?
-    readSqliteRecords(datapath, layer) :
-    readBundleRecords(datapath, layer);
-
-  stream
+  readSqliteRecords(datapath,layer)
     .pipe(whosonfirst.recordHasIdAndProperties())
     .pipe(whosonfirst.isActiveRecord())
     .pipe(filterOutPointRecords.create())
@@ -75,4 +63,4 @@ function readData(datapath, layer, localizedAdminNames, callback) {
 
 }
 
-module.exports = readData;
+module.exports = readData;