diff --git a/.envTEMPLATE b/.envTEMPLATE index 73ff0ef3..b0ada1c8 100644 --- a/.envTEMPLATE +++ b/.envTEMPLATE @@ -2,7 +2,7 @@ APP_ID=datablue PORT=3000 LOG_LEVEL=debug -REQUEST_LIMIT=100kb +REQUEST_LIMIT=10MB GOOGLE_API_KEY=mykey SESSION_SECRET=mySecret NODE_ENV=development diff --git a/config/locations.ts b/config/locations.ts index 58cebfdd..4382ee49 100644 --- a/config/locations.ts +++ b/config/locations.ts @@ -1,4 +1,4 @@ -import { Translated } from '../server/common/typealias'; +import { BoundingBox, Translated, uncheckedBoundingBoxToChecked } from '../server/common/typealias'; /* * @license @@ -19,14 +19,16 @@ export interface Location { name: string; description: Translated; description_more: Translated; - bounding_box: BoundingBox; + bounding_box: UncheckedBoundingBox; + //TODO @ralf.hauser not used as it seems, remove? operator_fountain_catalog_qid: string; + //TODO @ralf.hauser not used as it seems, remove? issue_api: IssueApi; } // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap // if you change something here, then you need to change it in proximap as well -export interface BoundingBox { +export interface UncheckedBoundingBox { latMin: number; lngMin: number; latMax: number; @@ -521,7 +523,23 @@ export function isCity(s: string): s is City { return cities.includes(s as City); } +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export function getCityBoundingBox(city: City): BoundingBox { + const uncheckedBoundingBox = locationsCollection[city].bounding_box; + try { + return uncheckedBoundingBoxToChecked(uncheckedBoundingBox); + } catch (e: any) { + const newErr = new Error('Could not get city bounding box for ' + city); + newErr.stack += '\nCaused by: ' + e.stack; + throw newErr; + } +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well export type LocationsCollection = Record; + // we don't expose just the internal structure as we also want to be sure that it follows the spec. // However, we allow City union to grow dynamically export const locationsCollection: LocationsCollection = internalLocationsCollection; diff --git a/package-lock.json b/package-lock.json index 1a623c0f..afd82c82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2627,9 +2627,9 @@ } }, "@types/body-parser": { - "version": "1.19.0", - "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", - "integrity": "sha512-W98JrE0j2K78swW4ukqMleo8R7h/pFETjM2DQ90MF6XK2i4LO4W3gQ71Lt4w3bfm2EvVSyWHplECvB5sK22yFQ==", + "version": "1.19.2", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.2.tgz", + "integrity": "sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==", "dev": true, "requires": { "@types/connect": "*", @@ -2637,9 +2637,9 @@ } }, "@types/connect": { - "version": "3.4.34", - "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.34.tgz", - "integrity": "sha512-ePPA/JuI+X0vb+gSWlPKOY0NdNAie/rPUqX2GUPpbZwiKTkSPhjXWuee47E4MtE54QVzGCQMQkAL6JhV2E1+cQ==", + "version": "3.4.35", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", + "integrity": "sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==", "dev": true, "requires": { "@types/node": "*" @@ -2661,9 +2661,9 @@ "dev": true }, "@types/express": { - "version": "4.17.12", - "resolved": "https://nexus.tegonal.com/repository/npm-proxy/@types/express/-/express-4.17.12.tgz", - "integrity": "sha512-pTYas6FrP15B1Oa0bkN5tQMNqOcVXa9j4FTFtO8DWI9kppKib+6NJtfTOOLcwxuuYvcX2+dVG6et1SxW/Kc17Q==", + "version": "4.17.13", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", + "integrity": "sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==", "dev": true, "requires": { "@types/body-parser": "*", @@ -2673,9 +2673,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.21", - "resolved": "https://nexus.tegonal.com/repository/npm-proxy/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz", - "integrity": "sha512-gwCiEZqW6f7EoR8TTEfalyEhb1zA5jQJnRngr97+3pzMaO1RKoI1w2bw07TK72renMUVWcWS5mLI6rk1NqN0nA==", + "version": "4.17.26", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.26.tgz", + "integrity": "sha512-zeu3tpouA043RHxW0gzRxwCHchMgftE8GArRsvYT0ByDMbn19olQHx5jLue0LxWY6iYtXb7rXmuVtSkhy9YZvQ==", "dev": true, "requires": { "@types/node": "*", @@ -2719,21 +2719,21 @@ "dev": true }, "@types/qs": { - "version": "6.9.6", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.6.tgz", - "integrity": "sha512-0/HnwIfW4ki2D8L8c9GVcG5I72s9jP5GSLVF0VIXDW00kmIpA6O33G7a8n59Tmh7Nz0WUC3rSb7PTY/sdW2JzA==", + "version": "6.9.7", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", + "integrity": "sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==", "dev": true }, "@types/range-parser": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", - "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.4.tgz", + "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, "@types/serve-static": { - "version": "1.13.9", - "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.9.tgz", - "integrity": "sha512-ZFqF6qa48XsPdjXV5Gsz0Zqmux2PerNd3a/ktL45mHpa19cuMi/cL8tcxdAx497yRh+QtYPuofjT9oWw9P7nkA==", + "version": "1.13.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", + "integrity": "sha512-nCkHGI4w7ZgAdNkrEu0bv+4xNV/XDqW+DydknebMOQwkpDGx8G+HTlj7R7ABI8i8nKxVw0wtKPi1D+lPOkh4YQ==", "dev": true, "requires": { "@types/mime": "^1", @@ -3170,16 +3170,16 @@ }, "dependencies": { "mime-db": { - "version": "1.40.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", - "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.51.0.tgz", + "integrity": "sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g==" }, "mime-types": { - "version": "2.1.24", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", - "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", + "version": "2.1.34", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.34.tgz", + "integrity": "sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A==", "requires": { - "mime-db": "1.40.0" + "mime-db": "1.51.0" } } } @@ -4882,17 +4882,17 @@ "dev": true }, "content-disposition": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", - "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", "requires": { - "safe-buffer": "5.1.2" + "safe-buffer": "5.2.1" }, "dependencies": { "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, @@ -5998,16 +5998,16 @@ } }, "express": { - "version": "4.17.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", - "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.17.2.tgz", + "integrity": "sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg==", "requires": { "accepts": "~1.3.7", "array-flatten": "1.1.1", - "body-parser": "1.19.0", - "content-disposition": "0.5.3", + "body-parser": "1.19.1", + "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.4.0", + "cookie": "0.4.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "~1.1.2", @@ -6021,23 +6021,93 @@ "on-finished": "~2.3.0", "parseurl": "~1.3.3", "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.5", - "qs": "6.7.0", + "proxy-addr": "~2.0.7", + "qs": "6.9.6", "range-parser": "~1.2.1", - "safe-buffer": "5.1.2", - "send": "0.17.1", - "serve-static": "1.14.1", - "setprototypeof": "1.1.1", + "safe-buffer": "5.2.1", + "send": "0.17.2", + "serve-static": "1.14.2", + "setprototypeof": "1.2.0", "statuses": "~1.5.0", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" }, "dependencies": { + "body-parser": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.1.tgz", + "integrity": "sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA==", + "requires": { + "bytes": "3.1.1", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "on-finished": "~2.3.0", + "qs": "6.9.6", + "raw-body": "2.4.2", + "type-is": "~1.6.18" + } + }, + "bytes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", + "integrity": "sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg==" + }, + "cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" + }, + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "qs": { + "version": "6.9.6", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", + "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==" + }, + "raw-body": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.2.tgz", + "integrity": "sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ==", + "requires": { + "bytes": "3.1.1", + "http-errors": "1.8.1", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" } } }, @@ -6388,9 +6458,9 @@ } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fragment-cache": { "version": "0.2.1", @@ -6981,9 +7051,9 @@ } }, "ipaddr.js": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", - "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "is-accessor-descriptor": { "version": "0.1.6", @@ -7885,12 +7955,11 @@ "integrity": "sha512-Gh39xwJwBKy0OvFmWfBs/vDO4Nl7JhnJtkqNP76OUinQz7BiMoszHYrIDHHAaqVl/QKVxCEy4ZxC/XZninu7nQ==" }, "node-cache": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-4.2.1.tgz", - "integrity": "sha512-BOb67bWg2dTyax5kdef5WfU3X8xu4wPg+zHzkvls0Q/QpYycIFRLEEIdAx9Wma43DxG6Qzn4illdZoYseKWa4A==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", "requires": { - "clone": "2.x", - "lodash": "^4.17.15" + "clone": "2.x" } }, "node-libs-browser": { @@ -8533,12 +8602,12 @@ "dev": true }, "proxy-addr": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", - "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", - "ipaddr.js": "1.9.0" + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" } }, "prr": { @@ -9117,9 +9186,9 @@ } }, "send": { - "version": "0.17.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", - "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.17.2.tgz", + "integrity": "sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww==", "requires": { "debug": "2.6.9", "depd": "~1.1.2", @@ -9128,18 +9197,45 @@ "escape-html": "~1.0.3", "etag": "~1.8.1", "fresh": "0.5.2", - "http-errors": "~1.7.2", + "http-errors": "1.8.1", "mime": "1.6.0", - "ms": "2.1.1", + "ms": "2.1.3", "on-finished": "~2.3.0", "range-parser": "~1.2.1", "statuses": "~1.5.0" }, "dependencies": { + "http-errors": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz", + "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": ">= 1.5.0 < 2", + "toidentifier": "1.0.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" } } }, @@ -9150,14 +9246,14 @@ "dev": true }, "serve-static": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", - "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", + "integrity": "sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ==", "requires": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.17.1" + "send": "0.17.2" } }, "set-value": { diff --git a/package.json b/package.json index 37459e5a..af4b8e67 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "cookie-parser": "^1.4.5", "cors": "^2.8.5", "dotenv": "^10.0.0", - "express": "^4.17.1", + "express": "^4.17.2", "fs": "0.0.1-security", "haversine": "^1.1.1", "helmet": "^4.6.0", @@ -41,7 +41,7 @@ "js-md5": "^0.7.3", "lodash": "^4.17.21", "nocache": "^3.0.1", - "node-cache": "^4.2.1", + "node-cache": "^5.1.2", "query-overpass": "^1.5.5", "source-map-support": "^0.5.20", "swagger-express-middleware": "^4.0.2", @@ -51,7 +51,7 @@ "devDependencies": { "@types/cookie-parser": "^1.4.2", "@types/cors": "^2.8.10", - "@types/express": "^4.17.12", + "@types/express": "^4.17.13", "@types/geojson": "^7946.0.8", "@types/lodash": "^4.14.170", "@types/node": "^14.17.3", diff --git a/server/api/controllers/controller.ts b/server/api/controllers/controller.ts index 68b96576..530a04ee 100644 --- a/server/api/controllers/controller.ts +++ b/server/api/controllers/controller.ts @@ -5,141 +5,86 @@ * and the profit contribution agreement available at https://www.my-d.org/ProfitContributionAgreement */ -import WikidataService from '../services/wikidata.service'; import l from '../../common/logger'; -import generateCityData from '../services/generateLocationData.service'; -import { cities, City, isCity, locationsCollection } from '../../../config/locations'; +import { locationsCollection } from '../../../config/locations'; import { fountain_property_metadata } from '../../../config/fountain.properties'; -import NodeCache from 'node-cache'; -import { essenceOf, fillWikipediaSummary } from '../services/processing.service'; -import { extractProcessingErrors } from './processing-errors.controller'; -import { getImageInfo, getImgsOfCat } from '../services/wikimedia.service'; -import { getCatExtract, getImgClaims } from '../services/claims.wm'; -import { isBlacklisted } from '../services/categories.wm'; -import { - MAX_IMG_SHOWN_IN_GALLERY, - LAZY_ARTIST_NAME_LOADING_i41db, //,CACHE_FOR_HRS_i45db -} from '../../common/constants'; + import sharedConstants from './../../common/shared-constants'; import { Request, Response } from 'express'; import { getSingleBooleanQueryParam, getSingleStringQueryParam } from './utils'; -import { Fountain, FountainCollection, GalleryValue, isDatabase } from '../../common/typealias'; -import { hasWikiCommonsCategories } from '../../common/wikimedia-types'; -import { ImageLike } from '../../../config/text2img'; - -// Configuration of Cache after https://www.npmjs.com/package/node-cache -const cityCache = new NodeCache({ - stdTTL: 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db, // time till cache expires, in seconds - checkperiod: 600, // how often to check for expiration, in seconds - default: 600 - deleteOnExpire: false, // on expire, we want the cache to be recreated not deleted - useClones: false, // do not create a clone of the data when fetching from cache -}); - -/* - * For each location (city), three JSON objects are created. Example for Zurich: - * - "ch-zh": contains the full data for all fountains of the location - * - "ch-zh_essential": contains a summary version of "ch-zh". This is the data loaded for display on the map. It is derived from "ch-zh". - * - "ch-zh_errors": contains a list of errors encountered when processing "ch-zh". - */ - -// when cached data expires, regenerate it (ignore non-essential) -cityCache.on('expired', key => { - // check if cache item key is neither the summary nor the list of errors. These will be updated automatically when the detailed city data are updated. - if (!key.includes('_essential') && !key.includes('_errors')) { - l.info(`controller.js cityCache.on('expired',...): Automatic cache refresh of ${key}`); - generateCityDataAndAddToCache(key, cityCache); - } -}); +import { BoundingBox, isDatabase, parseLngLat } from '../../common/typealias'; +import { + getFountainFromCacheIfNotForceRefreshOrFetch, + getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate, + populateCacheWithCities as populateLocationCacheWithCities, +} from '../services/generateLocationData.service'; +import { illegalState } from '../../common/illegalState'; export class Controller { constructor() { - // In production mode, process all fountains when starting the server so that the data are ready for the first requests + // In production mode, process all fountains of pre-defnied cities when starting the server + // so that the data is ready for the first requests if (process.env.NODE_ENV === 'production') { - cities.forEach(city => { - l.info(`controller.js Generating data for ${city}`); - generateCityData(city).then(fountainCollection => { - // save new data to storage - //TODO @ralfhauser, the old comment states // expire after two hours but CACHE_FOR_HRS_i45db is currently 48, which means after two days - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); - // create a reduced version of the data as well - cityCache.set(city + '_essential', essenceOf(fountainCollection)); - // also create list of processing errors (for proximap#206) - cityCache.set(city + '_errors', extractProcessingErrors(fountainCollection)); - }); + populateLocationCacheWithCities().catch((e: any) => { + l.error('unexpected error occured during populateLocationCacheWithCities\n' + e.stack); }); } } // Function to return detailed fountain information // When requesting detailed information for a single fountain, there are two types of queries - getSingle(req: Request, res: Response): void { - const queryType = getSingleStringQueryParam(req, 'queryType'); - const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true) ?? false; - - if (queryType === 'byId') { - l.info(`controller.js getSingle byId: refresh: ${refresh}`); - byId(req, res, refresh); - } else { - res.status(400).send('only byId supported'); - } + async getSingle(req: Request, res: Response, next: ErrorHandler): Promise { + handlingErrors(next, () => { + const queryType = getSingleStringQueryParam(req, 'queryType'); + const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true) ?? false; + + if (queryType === 'byId') { + l.info(`controller.js getSingle byId: refresh: ${refresh}`); + return this.byId(req, res, refresh); + } else { + illegalState('queryType only byId supported'); + } + }); } - - // Function to return all fountain information for a location. - byLocation(req: Request, res: Response): void { - const start = new Date(); - const city = getSingleStringQueryParam(req, 'city'); - if (!isCity(city)) { - throw Error('unsupported city given: ' + city); + private async byId(req: Request, res: Response, forceRefresh: boolean): Promise { + const loc = parseLngLat(getSingleStringQueryParam(req, 'loc')); + const database = getSingleStringQueryParam(req, 'database'); + if (!isDatabase(database)) { + illegalState('unsupported database given: ' + database); } - const refresh = getSingleBooleanQueryParam(req, 'refresh', /* isOptional = */ true); + const idval = getSingleStringQueryParam(req, 'idval'); - // if a refresh is requested or if no data is in the cache, then reprocess the fountains - if (refresh || cityCache.keys().indexOf(city) === -1) { - l.info(`controller.js byLocation: refresh: ${refresh} , city: ` + city); - generateCityData(city) - .then(fountainCollection => { - // save new data to storage - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); - - // create a reduced version of the data as well - const r_essential = essenceOf(fountainCollection); - cityCache.set(city + '_essential', r_essential); + const fountain = await getFountainFromCacheIfNotForceRefreshOrFetch(forceRefresh, database, idval, loc); + sendJson(res, fountain, 'byId'); + } - // return either the full or reduced version, depending on the "essential" parameter of the query - const essential = getSingleBooleanQueryParam(req, 'essential', /* isOptional = */ true) ?? false; - if (essential) { - sendJson(res, r_essential, 'r_essential'); - } else { - sendJson(res, fountainCollection, 'fountainCollection'); - } + async getByBounds(req: Request, res: Response, next: ErrorHandler): Promise { + handlingErrors(next, () => { + const southWest = parseLngLat(getSingleStringQueryParam(req, 'sw')); + const northEast = parseLngLat(getSingleStringQueryParam(req, 'ne')); + const boundingBox = BoundingBox(southWest, northEast); + const essential = getSingleBooleanQueryParam(req, 'essential'); + const refresh = getSingleBooleanQueryParam(req, 'refresh'); - // also create list of processing errors (for proximap#206) - cityCache.set(city + '_errors', extractProcessingErrors(fountainCollection)); - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; - l.info('controller.js byLocation generateLocationData: finished after ' + elapse.toFixed(1) + ' secs'); - }) - .catch(error => { - if (error.message) { - res.statusMessage = error.message; - } - res.status(500).send(error.stack); - }); - } - // otherwise, get the data from storage - else { - const essential = getSingleBooleanQueryParam(req, 'essential', /* isOptional = */ true) ?? false; - if (essential) { - sendJson(res, cityCache.get(city + '_essential'), 'fromCache essential'); - } else { - sendJson(res, cityCache.get(city), 'fromCache'); - } - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; - l.info('controller.js byLocation: finished after ' + elapse.toFixed(1) + ' secs'); - } + return this.byBoundingBox(res, boundingBox, essential, refresh); + }); } + private async byBoundingBox( + res: Response, + boundingBox: BoundingBox, + essential: boolean, + forceRefresh: boolean + ): Promise { + const collection = await getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + boundingBox, + essential, + 'byBounds', + /* debugAll = */ false + ); + sendJson(res, collection, 'fountainCollection'); + } /** * Function to return metadata regarding all the fountain properties that can be displayed. * (e.g. name translations, definitions, contribution information and tips) @@ -167,27 +112,29 @@ export class Controller { /** * Function to extract processing errors from detailed list of fountains */ - getProcessingErrors(req: Request, res: Response): void { + async getProcessingErrors(_req: Request, res: Response): Promise { + //TODO #148 return processing errors for given boundingBox + sendJson(res, [], 'not yet implemented'); // returns all processing errors for a given location // made for #206 - const city = getSingleStringQueryParam(req, 'city'); - const key = city + '_errors'; + // const city = getSingleStringQueryParam(req, 'city'); + // const key = city + '_errors'; - if (cityCache.keys().indexOf(key) < 0) { - // if data not in cache, create error list - cityCache.set(key, extractProcessingErrors(cityCache.get(city))); - } - cityCache.get(key, (err, value) => { - if (!err) { - sendJson(res, value, 'cityCache.get ' + key); - l.info('controller.js: getProcessingErrors !err sent'); - } else { - const errMsg = 'Error with cache: ' + err; - l.info('controller.js: getProcessingErrors ' + errMsg); - res.statusMessage = errMsg; - res.status(500).send(err.stack); - } - }); + // if (locationCache.keys().indexOf(key) < 0) { + // // if data not in cache, create error list + // locationCache.set(key, extractProcessingErrors(locationCache.get(city))); + // } + // locationCache.get(key, (err, value) => { + // if (!err) { + // sendJson(res, value, 'cityCache.get ' + key); + // l.info('controller.js: getProcessingErrors !err sent'); + // } else { + // const errMsg = 'Error with cache: ' + err; + // l.info('controller.js: getProcessingErrors ' + errMsg); + // res.statusMessage = errMsg; + // res.status(500).send(err.stack); + // } + // }); } } export const controller = new Controller(); @@ -195,22 +142,24 @@ export const controller = new Controller(); function sendJson(resp: Response, obj: Record | undefined, dbg: string): void { //TODO consider using https://github.com/timberio/timber-js/issues/69 or rather https://github.com/davidmarkclements/fast-safe-stringify try { - if (obj == undefined) { + if (obj === undefined) { l.error('controller.js doJson null == obj: ' + dbg); + resp.status(404).send(); + } else { + resp.json(obj); + //TODO @ralfhauser, neihter res.finish nor res.close exist, logging the json would need to be done before hand + // let res = resp.json(obj); + // if(process.env.NODE_ENV !== 'production') { + // // https://nodejs.org/dist/latest-v12.x/docs/api/http.html#http_class_http_serverresponse Event finish + // res.finish = res.close = function (event) { + // //not working :( https://github.com/water-fountains/datablue/issues/40 + // //https://github.com/expressjs/express/issues/4158 https://github.com/expressjs/express/blob/5.0/lib/response.js + // l.info('controller.js doJson length: keys '+Object.keys(obj).length+ + // //'\n responseData.data.length '+resp.responseData.data.length+ + // ' - '+dbg); + // } + // } } - resp.json(obj); - //TODO @ralfhauser, neihter res.finish nor res.close exist, logging the json would need to be done before hand - // let res = resp.json(obj); - // if(process.env.NODE_ENV !== 'production') { - // // https://nodejs.org/dist/latest-v12.x/docs/api/http.html#http_class_http_serverresponse Event finish - // res.finish = res.close = function (event) { - // //not working :( https://github.com/water-fountains/datablue/issues/40 - // //https://github.com/expressjs/express/issues/4158 https://github.com/expressjs/express/blob/5.0/lib/response.js - // l.info('controller.js doJson length: keys '+Object.keys(obj).length+ - // //'\n responseData.data.length '+resp.responseData.data.length+ - // ' - '+dbg); - // } - // } } catch (err: unknown) { const errS = 'controller.js doJson errors: "' + err + '" ' + dbg; l.error(errS); @@ -218,359 +167,9 @@ function sendJson(resp: Response, obj: Record | undefined, dbg: str } } -//TODO @ralfhauser, this function is too long and one does not have a good overview any more. Consider splitting it up into several functions -/** - * Function to respond to request by returning the fountain as defined by the provided identifier - */ -function byId(req: Request, res: Response, forceRefresh: boolean): Promise { - const city = getSingleStringQueryParam(req, 'city'); - if (!isCity(city)) { - return new Promise((_, reject) => reject('unsupported city given: ' + city)); - } - const database = getSingleStringQueryParam(req, 'database'); - if (!isDatabase(database)) { - return new Promise((_, reject) => reject('unsupported database given: ' + database)); - } - const idval = getSingleStringQueryParam(req, 'idval'); - const dbg = idval; - - let name = 'unkNamById'; - // l.info('controller.js byId: '+cityS+' '+dbg); - let fountainCollection = cityCache.get(city); - - // l.info('controller.js byId in promise: '+cityS+' '+dbg); - const cityPromises: Promise[] = []; - if (forceRefresh || fountainCollection === undefined) { - l.info('controller.js byId: ' + city + ' not found in cache ' + dbg + ' - start city lazy load'); - const genLocPrms = generateCityDataAndAddToCache(city, cityCache); - cityPromises.push(genLocPrms); - } - return Promise.all(cityPromises) - .then( - () => { - if (forceRefresh || fountainCollection === undefined) { - fountainCollection = cityCache.get(city); - } - if (fountainCollection !== undefined) { - const fountain = fountainCollection.features.find(f => f.properties['id_' + database]?.value === idval); - const imgMetaPromises: Promise[] = []; - let lazyAdded = 0; - const gl = -1; - if (fountain === undefined) { - l.info('controller.js byId: of ' + city + ' not found in cache ' + dbg); - return undefined; - } else { - const props = fountain.properties; - // l.info('controller.js byId fountain: '+cityS+' '+dbg); - if (null != props) { - name = props.name.value; - if (LAZY_ARTIST_NAME_LOADING_i41db) { - imgMetaPromises.push(WikidataService.fillArtistName(fountain, dbg)); - } - imgMetaPromises.push(WikidataService.fillOperatorInfo(fountain, dbg)); - fillWikipediaSummary(fountain, dbg, 1, imgMetaPromises); - const gallery = props.gallery; - // l.info('controller.js byId props: '+cityS+' '+dbg); - if (null != gallery && null != gallery.value) { - // l.info('controller.js byId gl: '+cityS+' '+dbg); - if (0 < gallery.value.length) { - // l.info('controller.js byId: of '+cityS+' found gal of size '+gl+' "'+name+'" '+dbg); - let i = 0; - let lzAtt = ''; - const showDetails = true; - const singleRefresh = true; - const imgUrlSet = new Set(); - const catPromises: Promise[] = []; - let numberOfCategories = -1; - let numberOfCategoriesLazyAdded = 0; - const imgUrlsLazyByCategory: ImageLike[] = []; - // TODO @ralfhauser, this condition does not make sense, if value.length < 0 means basically if it is empty and if it is empty, then numberOfCategories will always be 0 and the for-loop will do nothing - if (hasWikiCommonsCategories(props) && 0 < props.wiki_commons_name.value.length) { - numberOfCategories = props.wiki_commons_name.value.length; - let j = 0; - for (const cat of props.wiki_commons_name.value) { - j++; - if (null == cat) { - l.info(i + '-' + j + ' controller.js: null == commons category "' + cat + '" "' + dbg); - continue; - } - if (null == cat.c) { - l.info(i + '-' + j + ' controller.js: null == commons cat.c "' + cat + '" "' + dbg); - continue; - } - if (isBlacklisted(cat.c)) { - l.info(i + '-' + j + ' controller.js: commons category blacklisted "' + cat + '" "' + dbg); - continue; - } - const add = 0 > cat.l; - if (add) { - numberOfCategoriesLazyAdded++; - if (0 == imgUrlSet.size) { - for (const img of gallery.value) { - imgUrlSet.add(img.pgTit); - } - } - const catPromise = getImgsOfCat( - cat, - dbg, - city, - imgUrlSet, - imgUrlsLazyByCategory, - 'dbgIdWd', - props, - true - ); - //TODO we might prioritize categories with small number of images to have greater variety of images? - catPromises.push(catPromise); - } - getCatExtract(singleRefresh, cat, catPromises, dbg); - } - } - return Promise.all(catPromises).then( - r => { - for (let k = 0; k < imgUrlsLazyByCategory.length && k < MAX_IMG_SHOWN_IN_GALLERY; k++) { - //between 6 && 50 imgs are on the gallery-preview - const img = imgUrlsLazyByCategory[k]; - //TODO @ralfhauser, val does not exist on GalleryValue but value, changed it - const nImg: GalleryValue = { - s: img.src, - pgTit: img.value, - c: img.cat, - t: img.typ, - }; - gallery.value.push(nImg); - } - if (0 < imgUrlsLazyByCategory.length) { - l.info( - 'controller.js byId lazy img by lazy cat added: attempted ' + - imgUrlsLazyByCategory.length + - ' in ' + - numberOfCategoriesLazyAdded + - '/' + - numberOfCategories + - ' cats, tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - for (const img of gallery.value) { - const imMetaDat = img.metadata; - if (null == imMetaDat && 'wm' == img.t) { - lzAtt += i + ','; - l.info( - 'controller.js byId lazy getImageInfo: ' + - city + - ' ' + - i + - '/' + - gl + - ' "' + - img.pgTit + - '" "' + - name + - '" ' + - dbg - ); - imgMetaPromises.push( - getImageInfo( - img, - i + '/' + gl + ' ' + dbg + ' ' + name + ' ' + city, - showDetails, - props - ).catch(giiErr => { - //TODO @ralfhauser, dbgIdWd does not exist - const dbgIdWd = undefined; - l.info( - 'wikimedia.service.js: fillGallery getImageInfo failed for "' + - img.pgTit + - '" ' + - dbg + - ' ' + - city + - ' ' + - dbgIdWd + - ' "' + - name + - '"' + - '\n' + - giiErr.stack - ); - }) - ); - lazyAdded++; - } else { - // l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg); - } - getImgClaims(singleRefresh, img, imgMetaPromises, i + ': ' + dbg); - i++; - } - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading: attempted ' + - lazyAdded + - '/' + - gl + - ' (' + - lzAtt + - ') of ' + - city + - ' ' + - dbg + - ' "' + - name + - '"' - ); - } - return Promise.all(imgMetaPromises).then( - r => { - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading after promise: attempted ' + - lazyAdded + - ' tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - //TODO @ralfhauser this is a clear smell, we already send the response before we resolve the promise - // it would be better if we return the fountain in a then once the promise completes - sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); - l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); - return fountain; - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - } else { - l.info('controller.js byId: of ' + city + ' gl < 1 ' + dbg); - return Promise.all(imgMetaPromises).then( - r => { - if (0 < lazyAdded) { - l.info( - 'controller.js byId lazy img metadata loading after promise: attempted ' + - lazyAdded + - ' tot ' + - gl + - ' of ' + - city + - ' ' + - dbg + - ' "' + - name + - '" ' + - r.length - ); - } - //TODO @ralfhauser this is a clear smell, we already send the response before we resovle the promise - sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); - l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); - return fountain; - }, - err => { - l.error( - `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city - ); - return undefined; - } - ); - } - } else { - l.info('controller.js byId: of ' + city + ' gallery null || null == gal.value ' + dbg); - return undefined; - } - } else { - l.info('controller.js byId: of ' + city + ' no props ' + dbg); - return undefined; - } - } - } else { - return undefined; - } - // l.info('controller.js byId: end of '+cityS+' '+dbg); - }, - err => { - l.error(`controller.js byId: Failed on genLocPrms: ${err.stack} .` + dbg + ' ' + city); - return undefined; - } - ) - .catch(e => { - //TODO @ralfhauser, this error will never occurr because we already defined an error case two lines above - l.error(`controller.js byId: Error finding fountain in preprocessed data: ${e} , city: ` + city + ' ' + dbg); - l.error(e.stack); - return undefined; - }); +// TODO should no longer be necessary starting with Express 5.x +function handlingErrors(next: ErrorHandler, action: () => Promise) { + action().catch(next); } -export function generateCityDataAndAddToCache(city: City, cityCache: NodeCache): Promise { - // trigger a reprocessing of the location's data, based on the key. - const genLocPrms = generateCityData(city) - .then(fountainCollection => { - // save newly generated fountainCollection to the cache - let numberOfFountains = -1; - if (fountainCollection?.features != null) { - numberOfFountains = fountainCollection.features.length; - } - //TODO @ralfhauser, the old comment states // expire after two hours but CACHE_FOR_HRS_i45db is currently 48, which means after two days - cityCache.set(city, fountainCollection, 60 * 60 * sharedConstants.CACHE_FOR_HRS_i45db); // expire after two hours - - // create a reduced version of the data as well - const essence = essenceOf(fountainCollection); - cityCache.set(city + '_essential', essence); - let ess = -1; - if (null != essence && null != essence.features) { - ess = essence.features.length; - } - - // also create list of processing errors (for proximap#206) - const processingErrors = extractProcessingErrors(fountainCollection); - cityCache.set(city + '_errors', processingErrors); - //TODO @ralfhauser, processingErrors is never null but an array, which also means processingErrors.features never exists and hence this will always be false - // let prcErr = -1; - // if (null != processingErrors && null != processingErrors.features) { - // prcErr = processingErrors.features.length; - // } - const prcErr = processingErrors.length; - l.info( - `generateLocationDataAndCache setting cache of ${city} ` + - ' ftns: ' + - numberOfFountains + - ' ess: ' + - ess + - ' prcErr: ' + - prcErr - ); - return fountainCollection; - }) - .catch(error => { - l.error(`controller.js unable to set Cache. Error: ${error.stack}`); - // TODO @ralfhauser, return void is not so nice IMO but that's what was defined beforehand implicitly - return; - }); - return genLocPrms; -} +type ErrorHandler = (error: unknown) => unknown; diff --git a/server/api/controllers/router.ts b/server/api/controllers/router.ts index 3d6ace0b..7213c5c3 100644 --- a/server/api/controllers/router.ts +++ b/server/api/controllers/router.ts @@ -12,10 +12,10 @@ import { buildInfoController } from './build-info.controller'; // This file maps API routes to functions export const Router = express .Router() - .get('/fountain/', controller.getSingle) - .get('/fountains/', controller.byLocation) - .get('/metadata/fountain_properties/', controller.getPropertyMetadata) - .get('/metadata/locations/', controller.getLocationMetadata) - .get('/metadata/shared-constants/', controller.getSharedConstants) - .get('/processing-errors/', controller.getProcessingErrors) + .get('/fountain', controller.getSingle.bind(controller)) + .get('/fountains', controller.getByBounds.bind(controller)) + .get('/metadata/fountain_properties', controller.getPropertyMetadata.bind(controller)) + .get('/metadata/locations', controller.getLocationMetadata.bind(controller)) + .get('/metadata/shared-constants', controller.getSharedConstants.bind(controller)) + .get('/processing-errors', controller.getProcessingErrors.bind(controller)) .get('/build-info', buildInfoController); diff --git a/server/api/controllers/utils.ts b/server/api/controllers/utils.ts index 7ee1648d..652f065e 100644 --- a/server/api/controllers/utils.ts +++ b/server/api/controllers/utils.ts @@ -116,7 +116,9 @@ function typeCheckAndConvertParam( } else if (typeCheck(param)) { return typeConversion(param); } else { - throw Error(`${paramName} was of a wrong type, expected ${type} was ${JSON.stringify(param)} ${typeof param}`); + throw Error( + `${paramName} was of a wrong type, expected ${type} was ${JSON.stringify(param)} with type ${typeof param}` + ); } } diff --git a/server/api/services/database.service.ts b/server/api/services/database.service.ts deleted file mode 100644 index 7657683a..00000000 --- a/server/api/services/database.service.ts +++ /dev/null @@ -1,122 +0,0 @@ -/* - * @license - * (c) Copyright 2019 - 2020 | MY-D Foundation | Created by Matthew Moy de Vitry - * Use of this code is governed by the GNU Affero General Public License (https://www.gnu.org/licenses/agpl-3.0) - * and the profit contribution agreement available at https://www.my-d.org/ProfitContributionAgreement - */ - -import { essenceOf } from './processing.service'; -//TODO @ralfhauser, CACHE_FOR_HRS_i45db is not defined in constants, please adjust this number -// import {CACHE_FOR_HRS_i45db} from "../../common/constants"; -const CACHE_FOR_HRS_i45db = 1; -import l from '../../common/logger'; -import { generateCityDataAndAddToCache } from '../controllers/controller'; -import _ from 'lodash'; -import haversine from 'haversine'; -import NodeCache from 'node-cache'; -import {} from 'geojson'; -import { Fountain, FountainCollection } from '../../common/typealias'; -import { City } from '../../../config/locations'; - -export function updateCacheWithFountain(cache: NodeCache, fountain: Fountain, city: City): Fountain { - // updates cache and returns fountain with datablue id - // get city data from cache - let fountains = cache.get(city); - const cacheTimeInSecs = 60 * 60 * CACHE_FOR_HRS_i45db; - if (!fountains && !city.includes('_essential') && !city.includes('_errors')) { - l.info( - `updateCacheWithFountain server-side city data disappeared (server restart?) - cache recreation for ${city}` - ); - generateCityDataAndAddToCache(city, cache); - //TODO @ralf.hauser, this is buggy as generateCityDataAndAddToCache returns a promise and most likely did not finish at this point - fountains = cache.get(city); - } - if (fountains) { - // replace fountain - [fountains, fountain] = replaceFountain(fountains, fountain, city); - // send to cache - //TODO consider whether really to fully extend the cache-time for the whole city just because one fountain was refreshed - // a remaining city-cache-time could be calculated with getTtl(cityname) - cache.set(city, fountains, cacheTimeInSecs); - // create a reduced version of the data as well - const r_essential = essenceOf(fountains); - cache.set(city + '_essential', r_essential, cacheTimeInSecs); - return fountain; - } - l.info( - 'database.services.js updateCacheWithFountain: no fountains were in cache of city ' + - city + - ' tried to work on ' + - fountain - ); - return fountain; -} - -function replaceFountain( - fountains: FountainCollection, - fountain: Fountain, - cityName: string -): [FountainCollection, Fountain] { - // update cache with fountain and assign correct datablue id - - const distances: [number, Fountain, number][] = []; - - for (let i = 0; i < fountains.features.length; i++) { - if (isMatch(fountains.features[i], fountain)) { - //replace fountain - fountain.properties.id = fountains.features[i].properties.id; - fountains.features[i] = fountain; - l.info('database.services.js replaceFountain: ismatch ftn ' + i + ', city ' + cityName + ' , ftn ' + fountain); - return [fountains, fountain]; - } else { - // compute distance otherwise - distances.push([ - i, - fountains.features[i], - haversine(fountains.features[i].geometry.coordinates, fountain.geometry.coordinates, { - unit: 'meter', - format: '[lon,lat]', - }), - ]); - } - } - - const triple = _.minBy(distances, p => p[2]); - if (triple !== undefined && triple[2] < 15) { - //TODO @ralf.hauser `f` did not exist here. I assumed that the fountain should be replaced by the fountain which is nearest. Please verify this change is correct - const [index, nearestFountain, distance] = triple; - //replace fountain - // fountain.properties.id = f.properties.id; - l.info( - 'database.services.js replaceFountain: replaced with distance ' + - distance + - ', city ' + - cityName + - ' , ftn ' + - fountain - ); - fountain.properties.id = nearestFountain.properties.id; - fountains.features[index] = fountain; - return [fountains, fountain]; - } else { - // fountain was not found; just add it to the list - fountain.properties.id = _.max(fountains.features.map(f => f.properties.id)) + 1; - fountains.features.push(fountain); - l.info( - 'database.services.js replaceFountain: added with distance ' + - triple?.[2] + - ', city ' + - cityName + - ' , ftn ' + - fountain - ); - return [fountains, fountain]; - } -} - -function isMatch(f1: Fountain, f2: Fountain): boolean { - // returns true if match, otherwise returns distance - return ['id_wikidata', 'id_osm'].some(idName => { - f1.properties && f2.properties && f1.properties[idName].value === f2.properties[idName].value; - }); -} diff --git a/server/api/services/generateLocationData.service.ts b/server/api/services/generateLocationData.service.ts index bb84e4b8..cb732f02 100644 --- a/server/api/services/generateLocationData.service.ts +++ b/server/api/services/generateLocationData.service.ts @@ -6,7 +6,7 @@ */ import l from '../../common/logger'; -import { City, locationsCollection } from '../../../config/locations'; +import { cities, getCityBoundingBox } from '../../../config/locations'; import OsmService from '../services/osm.service'; import WikidataService from '../services/wikidata.service'; import { conflate } from '../services/conflate.data.service'; @@ -14,133 +14,534 @@ import applyImpliedPropertiesOsm from '../services/applyImplied.service'; import { createUniqueIds, defaultCollectionEnhancement, + essenceOf, fillInMissingWikidataFountains, } from '../services/processing.service'; -import { FountainCollection } from '../../common/typealias'; +import { BoundingBox, Database, Fountain, FountainCollection, LngLat } from '../../common/typealias'; import { MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; +import sharedConstants from '../../common/shared-constants'; +import { extractProcessingErrors } from '../controllers/processing-errors.controller'; +import { illegalState } from '../../common/illegalState'; +import '../../common/importAllExtensions'; +import { + cacheEssentialFountainCollection, + cacheFullFountainCollection, + cacheProcessingErrors, + getBoundingBoxOfTiles, + getCachedEssentialFountainCollection, + getCachedFullFountainCollection, + getTileOfLocation, + locationCacheKeyToTile, + splitInTiles, + Tile, + tileToLocationCacheKey, +} from './locationCache'; +import { sleep } from '../../common/sleep'; -/** - * This function creates fountain collections - * @param {string} locationName - the code name of the location for which fountains should be processed - */ -function generateCityData(locationName: City): Promise { - const start = new Date(); - l.info(`generateLocationData.service.js: processing all fountains from "${locationName}" `); - - return new Promise((resolve, reject) => { - const logAndRejectError = function (err: string) { - l.error(err); - reject(new Error(err)); - }; - - // get bounding box of location - const location = locationsCollection[locationName]; - if (location == undefined) { - logAndRejectError(`location not found in config: ${locationName}`); - } else { - //TODO @ralfhauser the following checks are unnecessary IMO as they cannot be null according to the definition - const bbox = location.bounding_box; - if (null == bbox) { - const err = `fatal: null == bbox for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.latMin) { - const err = `fatal: null == bbox.latMin for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.lngMin) { - const err = `fatal: null == bbox.lngMin for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.latMax) { - const err = `fatal: null == bbox.latMax for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (null == bbox.lngMax) { - const err = `fatal: null == bbox.lngMax for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (bbox.lngMin > bbox.lngMax) { - const err = `fatal: bbox.lngMin > bbox.lngMax for ${locationName}`; - l.error(err); - reject(new Error(err)); - } - if (bbox.latMin > bbox.latMax) { - const err = `fatal: bbox.latMin > bbox.latMax for ${locationName}`; - l.error(err); - reject(new Error(err)); +//TODO @ralf.hauser reconsider this functionality. If a user queries a city in the same time then it is more likely +//that we run into a throttling timeout. Maybe only load the default city? +export async function populateCacheWithCities(): Promise { + for (const city of cities) { + if (city !== 'test') continue; + l.info(`Generating data for ${city}`); + await getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + /*forceRefresh= */ false, + getCityBoundingBox(city), + /* essential= */ false, + /* dbg=*/ city, + /* debugAll= */ false, + sharedConstants.CITY_TTL_IN_HOURS + ).catch(async (e: any) => { + // we still want to try to populate the others, thus we are not re-throwing. + //TODO @ralf.hauser it would actually be better if we react to a 429 response from OSM or wikidata + if (e.statusCode === 429) { + console.error('we got throttled, waiting for 2 minutes'); + // wait 2 minutes because OSM or wikidata throttled us + await sleep(2 * 60 * 1000); + } else { + console.error(e.message + '\n' + e.stack); } + }); + // wait 30 seconds between each city to lower the chance that OSM or wikidata throttles us + await sleep(30 * 1000); + } + l.info('finished populating cache with cities'); +} + +export async function getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + forceRefresh: boolean, + boundingBox: BoundingBox, + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined = undefined +): Promise { + const start = new Date(); + const tiles = splitInTiles(boundingBox); + l.info('processing ' + tiles.length + ' tiles'); + const allFountains = await byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + tiles, + essential, + dbg, + debugAll, + ttlInHours + ); + + const end = new Date(); + const elapse = (end.getTime() - start.getTime()) / 1000; + l.info( + 'generateLocationData.service.js: after ' + + elapse.toFixed(1) + + ' secs successfully processed all (size ' + + allFountains.length + + `) fountains from ${dbg} \nstart: ` + + start.toISOString() + + '\nend: ' + + end.toISOString() + ); + return FountainCollection(allFountains, /* lastScan= */ start); +} - // get data from Osm - const osmPromise = OsmService.byBoundingBox(bbox.latMin, bbox.lngMin, bbox.latMax, bbox.lngMax) - .then(r => applyImpliedPropertiesOsm(r)) - .catch(e => { - if ('getaddrinfo' == e.syscall) { - l.info('Are you offline from the internet?'); +async function byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh: boolean, + tiles: BoundingBox[], + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined = undefined +): Promise { + //TODO @ralf.hauser, we could optimise this a bit in case only bounding boxes at the border are not cached yet + const maybeFountains = forceRefresh + ? undefined // don't even search in cache in case of forceRefresh + : tiles.reduce((acc, tile) => { + if (acc === undefined) { + // previous was not in cache, no need to search further + return undefined; + } else { + const cacheEntry = essential + ? getCachedEssentialFountainCollection(tile) + : getCachedFullFountainCollection(tile); + if (cacheEntry !== undefined) { + return acc.concat(cacheEntry.value.features); + } else { + return undefined; } - l.error( - `generateLocationDataService.js: Error collecting OSM data - generateLocationData: ${e.stack} ` + - ' latMi ' + - bbox.latMin + - ', lngMi ' + - bbox.lngMin + - ', latMx ' + - bbox.latMax + - ', lngMx ' + - bbox.lngMax - ); - //TODO @ralfhauser, smelly, using the reject from an outer Promise. IMO better throw the exception - // reject(e); - throw e; - }); - - // get data from Wikidata - const wikidataPromise: Promise = WikidataService.idsByBoundingBox( - bbox.latMin, - bbox.lngMin, - bbox.latMax, - bbox.lngMax, - locationName - ).then(r => WikidataService.byIds(r, locationName)); - - const debugAll = -1 != locationName.indexOf('test'); - - // conflate - Promise.all([osmPromise, wikidataPromise]) - // get any missing wikidata fountains for proximap#212 - .then(r => fillInMissingWikidataFountains(r[0], r[1], locationName)) - .then(r => conflate(r, locationName, debugAll)) - .then(r => defaultCollectionEnhancement(r, locationName, debugAll)) - .then(r => createUniqueIds(r)) - .then(r => { - const end = new Date(); - const elapse = (end.getTime() - start.getTime()) / 1000; - l.info( - 'generateLocationData.service.js: after ' + - elapse.toFixed(1) + - ' secs successfully processed all (size ' + - r.length + - `) fountains from ${locationName} \nstart: ` + - start.toISOString() + - '\nend: ' + - end.toISOString() - ); - resolve({ - type: 'FeatureCollection', - features: r, - }); - }) - .catch(err => { - l.error('generateLocationData.service.js - Promise.all([osmPromise, wikidataPromise]): ' + err.stack); - reject(err); - }); + } + }, /* initial= */ [] as Fountain[] | undefined); + + if (maybeFountains !== undefined) { + l.info('all tiles in cache'); + // all in cache, return immediately + return Promise.resolve(maybeFountains); + } else { + return fetchFountainsFromServerAndUpdateCache(tiles, essential, dbg, debugAll, ttlInHours); + } +} + +async function fetchFountainsFromServerAndUpdateCache( + tiles: Tile[], + essential: boolean, + dbg: string, + debugAll: boolean, + ttlInHours: number | undefined +): Promise { + const boundingBox = getBoundingBoxOfTiles(tiles); + const fountains = await fetchFountainsByBoundingBox(boundingBox, dbg, debugAll); + + const groupedByTile = fountains.groupBy(fountain => + tileToLocationCacheKey( + getTileOfLocation(LngLat(fountain.geometry.coordinates[0], fountain.geometry.coordinates[1])) + ) + ); + + const collections = Array.from(groupedByTile.entries()).map(([cacheKey, fountains]) => { + const tile = locationCacheKeyToTile(cacheKey); + let fountainCollection: FountainCollection | undefined = FountainCollection(fountains); + updateCacheWithFountains(tile, fountainCollection, ttlInHours); + fountainCollection = ( + essential ? getCachedEssentialFountainCollection(tile) : getCachedFullFountainCollection(tile) + )?.value; + if (fountainCollection === undefined) { + illegalState(`fountainCollection ${cacheKey} was undefined after writing it to the cache`); } + return fountainCollection; }); + + return collections.reduce((acc, collection) => acc.concat(collection.features), /* initial= */ new Array()); +} + +function fetchFountainsByBoundingBox(boundingBox: BoundingBox, dbg: string, debugAll: boolean): Promise { + const osmPromise = OsmService.byBoundingBox(boundingBox) + .then(arr => applyImpliedPropertiesOsm(arr)) + .catch(e => { + if ('getaddrinfo' == e.syscall) { + l.info('Are you offline from the internet?'); + } + l.error( + `generateLocationDataService: Error collecting OSM data - generateLocationData: ${e.message}` + + ' latMin ' + + boundingBox.min.lat + + ', lngMim ' + + boundingBox.min.lng + + ', latMax ' + + boundingBox.max.lat + + ', lngMax ' + + boundingBox.max.lng + ); + throw e; + }); + + // get data from Wikidata + const wikidataPromise: Promise = WikidataService.idsByBoundingBox(boundingBox).then(r => + // TODO @ralf.hauser why not fetch the wikidata already in idsByBoundingBox? + WikidataService.byIds(r, dbg) + ); + + // conflate + return ( + Promise.all([osmPromise, wikidataPromise]) + // get any missing wikidata fountains for proximap#212 + .then(arr => fillInMissingWikidataFountains(arr[0], arr[1], dbg)) + .then(arr => conflate(arr, dbg, debugAll)) + .then(arr => defaultCollectionEnhancement(arr, dbg, debugAll)) + //TODO @ralf.hauser really required? + .then(arr => createUniqueIds(arr)) + ); } -export default generateCityData; +function updateCacheWithFountains(tile: Tile, fountainCollection: FountainCollection, ttlInHours: number | undefined) { + // save newly generated fountainCollection to the cache + + const existing = getCachedFullFountainCollection(tile); + const ttl = 60 * 60 * (ttlInHours ?? existing?.ttl ?? sharedConstants.BOUNDING_BOX_TTL_IN_HOURS); + cacheFullFountainCollection(tile, fountainCollection, ttl); + + // create a reduced version of the data as well + const essence = essenceOf(fountainCollection); + cacheEssentialFountainCollection(tile, essence, ttl); + + // also create list of processing errors (for proximap#206) + const processingErrors = extractProcessingErrors(fountainCollection); + cacheProcessingErrors(tile, processingErrors, ttl); + //TODO @ralfhauser, processingErrors is never null but an array, which also means processingErrors.features never exists and hence this will always be false + // let prcErr = -1; + // if (null != processingErrors && null != processingErrors.features) { + // prcErr = processingErrors.features.length; + // } + l.info( + `generateLocationDataAndCache setting cache of ${tileToLocationCacheKey(tile)}` + + ' number of fountains: ' + + (fountainCollection?.features?.length ?? 'unknown') + + ' ess: ' + + (essence?.features?.length ?? 'unknown') + + ' prcErr: ' + + processingErrors.length + ); +} + +export async function getFountainFromCacheIfNotForceRefreshOrFetch( + forceRefresh: boolean, + database: Database, + idval: string, + loc: LngLat +): Promise { + const tile = getTileOfLocation(loc); + const fountains = await byTilesFromCacheIfNotForceRefreshOrPopulate( + forceRefresh, + [tile], + /* essential = */ false, + 'database: ' + database + ' idval: ' + idval, + /* debugAll =*/ false + ); + return fountains.find(f => f.properties['id_' + database]?.value === idval); + + // let name = 'unkNamById'; + + // const cityPromises: Promise[] = []; + // if (forceRefresh || fountains === undefined) { + // //TODO #150 don't load all city data but only the specific fountain? + // const genLocPrms = generateCityDataAndAddToCache(city, locationCache); + // cityPromises.push(genLocPrms); + // } + // return Promise.all(cityPromises) + // .then( + // () => { + // if (forceRefresh || fountains === undefined) { + // fountains = locationCache.get(city); + // } + // if (fountains !== undefined) { + // const fountain = fountains.features.find(f => f.properties['id_' + database]?.value === idval); + // const imgMetaPromises: Promise[] = []; + // let lazyAdded = 0; + // const gl = -1; + // if (fountain === undefined) { + // l.info('controller.js byId: of ' + city + ' not found in cache ' + dbg); + // return undefined; + // } else { + // const props = fountain.properties; + // // l.info('controller.js byId fountain: '+cityS+' '+dbg); + // if (null != props) { + // name = props.name.value; + // if (LAZY_ARTIST_NAME_LOADING_i41db) { + // imgMetaPromises.push(WikidataService.fillArtistName(fountain, dbg)); + // } + // imgMetaPromises.push(WikidataService.fillOperatorInfo(fountain, dbg)); + // fillWikipediaSummary(fountain, dbg, 1, imgMetaPromises); + // const gallery = props.gallery; + // // l.info('controller.js byId props: '+cityS+' '+dbg); + // if (null != gallery && null != gallery.value) { + // // l.info('controller.js byId gl: '+cityS+' '+dbg); + // if (0 < gallery.value.length) { + // // l.info('controller.js byId: of '+cityS+' found gal of size '+gl+' "'+name+'" '+dbg); + // let i = 0; + // let lzAtt = ''; + // const showDetails = true; + // const singleRefresh = true; + // const imgUrlSet = new Set(); + // const catPromises: Promise[] = []; + // let numberOfCategories = -1; + // let numberOfCategoriesLazyAdded = 0; + // const imgUrlsLazyByCategory: ImageLike[] = []; + // // TODO @ralfhauser, this condition does not make sense, if value.length < 0 means basically if it is empty and if it is empty, then numberOfCategories will always be 0 and the for-loop will do nothing + // if (hasWikiCommonsCategories(props) && 0 < props.wiki_commons_name.value.length) { + // numberOfCategories = props.wiki_commons_name.value.length; + // let j = 0; + // for (const cat of props.wiki_commons_name.value) { + // j++; + // if (null == cat) { + // l.info(i + '-' + j + ' controller.js: null == commons category "' + cat + '" "' + dbg); + // continue; + // } + // if (null == cat.c) { + // l.info(i + '-' + j + ' controller.js: null == commons cat.c "' + cat + '" "' + dbg); + // continue; + // } + // if (isBlacklisted(cat.c)) { + // l.info(i + '-' + j + ' controller.js: commons category blacklisted "' + cat + '" "' + dbg); + // continue; + // } + // const add = 0 > cat.l; + // if (add) { + // numberOfCategoriesLazyAdded++; + // if (0 == imgUrlSet.size) { + // for (const img of gallery.value) { + // imgUrlSet.add(img.pgTit); + // } + // } + // const catPromise = getImgsOfCat( + // cat, + // dbg, + // city, + // imgUrlSet, + // imgUrlsLazyByCategory, + // 'dbgIdWd', + // props, + // true + // ); + // //TODO we might prioritize categories with small number of images to have greater variety of images? + // catPromises.push(catPromise); + // } + // getCatExtract(singleRefresh, cat, catPromises, dbg); + // } + // } + // return Promise.all(catPromises).then( + // r => { + // for (let k = 0; k < imgUrlsLazyByCategory.length && k < MAX_IMG_SHOWN_IN_GALLERY; k++) { + // //between 6 && 50 imgs are on the gallery-preview + // const img = imgUrlsLazyByCategory[k]; + // //TODO @ralfhauser, val does not exist on GalleryValue but value, changed it + // const nImg: GalleryValue = { + // s: img.src, + // pgTit: img.value, + // c: img.cat, + // t: img.typ, + // }; + // gallery.value.push(nImg); + // } + // if (0 < imgUrlsLazyByCategory.length) { + // l.info( + // 'controller.js byId lazy img by lazy cat added: attempted ' + + // imgUrlsLazyByCategory.length + + // ' in ' + + // numberOfCategoriesLazyAdded + + // '/' + + // numberOfCategories + + // ' cats, tot ' + + // gl + + // ' of ' + + // city + + // ' ' + + // dbg + + // ' "' + + // name + + // '" ' + + // r.length + // ); + // } + // for (const img of gallery.value) { + // const imMetaDat = img.metadata; + // if (null == imMetaDat && 'wm' == img.t) { + // lzAtt += i + ','; + // l.info( + // 'controller.js byId lazy getImageInfo: ' + + // city + + // ' ' + + // i + + // '/' + + // gl + + // ' "' + + // img.pgTit + + // '" "' + + // name + + // '" ' + + // dbg + // ); + // imgMetaPromises.push( + // getImageInfo( + // img, + // i + '/' + gl + ' ' + dbg + ' ' + name + ' ' + city, + // showDetails, + // props + // ).catch(giiErr => { + // //TODO @ralfhauser, dbgIdWd does not exist + // const dbgIdWd = undefined; + // l.info( + // 'wikimedia.service.js: fillGallery getImageInfo failed for "' + + // img.pgTit + + // '" ' + + // dbg + + // ' ' + + // city + + // ' ' + + // dbgIdWd + + // ' "' + + // name + + // '"' + + // '\n' + + // giiErr.stack + // ); + // }) + // ); + // lazyAdded++; + // } else { + // // l.info('controller.js byId: of '+cityS+' found imMetaDat '+i+' in gal of size '+gl+' "'+name+'" '+dbg); + // } + // getImgClaims(singleRefresh, img, imgMetaPromises, i + ': ' + dbg); + // i++; + // } + // if (0 < lazyAdded) { + // l.info( + // 'controller.js byId lazy img metadata loading: attempted ' + + // lazyAdded + + // '/' + + // gl + + // ' (' + + // lzAtt + + // ') of ' + + // city + + // ' ' + + // dbg + + // ' "' + + // name + + // '"' + // ); + // } + // return Promise.all(imgMetaPromises).then( + // r => { + // if (0 < lazyAdded) { + // l.info( + // 'controller.js byId lazy img metadata loading after promise: attempted ' + + // lazyAdded + + // ' tot ' + + // gl + + // ' of ' + + // city + + // ' ' + + // dbg + + // ' "' + + // name + + // '" ' + + // r.length + // ); + // } + // //TODO @ralfhauser this is a clear smell, we already send the response before we resolve the promise + // // it would be better if we return the fountain in a then once the promise completes + // sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); + // l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); + // return fountain; + // }, + // err => { + // l.error( + // `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city + // ); + // return undefined; + // } + // ); + // }, + // err => { + // l.error( + // `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city + // ); + // return undefined; + // } + // ); + // } else { + // l.info('controller.js byId: of ' + city + ' gl < 1 ' + dbg); + // return Promise.all(imgMetaPromises).then( + // r => { + // if (0 < lazyAdded) { + // l.info( + // 'controller.js byId lazy img metadata loading after promise: attempted ' + + // lazyAdded + + // ' tot ' + + // gl + + // ' of ' + + // city + + // ' ' + + // dbg + + // ' "' + + // name + + // '" ' + + // r.length + // ); + // } + // //TODO @ralfhauser this is a clear smell, we already send the response before we resovle the promise + // sendJson(res, fountain, 'byId ' + dbg); // res.json(fountain); + // l.info('controller.js byId: of ' + city + ' res.json ' + dbg + ' "' + name + '"'); + // return fountain; + // }, + // err => { + // l.error( + // `controller.js: Failed on imgMetaPromises: ${err.stack} .` + dbg + ' "' + name + '" ' + city + // ); + // return undefined; + // } + // ); + // } + // } else { + // l.info('controller.js byId: of ' + city + ' gallery null || null == gal.value ' + dbg); + // return undefined; + // } + // } else { + // l.info('controller.js byId: of ' + city + ' no props ' + dbg); + // return undefined; + // } + // } + // } else { + // return undefined; + // } + // // l.info('controller.js byId: end of '+cityS+' '+dbg); + // }, + // err => { + // l.error(`controller.js byId: Failed on genLocPrms: ${err.stack} .` + dbg + ' ' + city); + // return undefined; + // } + // ) + // .catch(e => { + // //TODO @ralfhauser, this error will never occurr because we already defined an error case two lines above + // l.error(`controller.js byId: Error finding fountain in preprocessed data: ${e} , city: ` + city + ' ' + dbg); + // l.error(e.stack); + // return undefined; + // }); +} diff --git a/server/api/services/locationCache.ts b/server/api/services/locationCache.ts new file mode 100644 index 00000000..aa37ea7a --- /dev/null +++ b/server/api/services/locationCache.ts @@ -0,0 +1,155 @@ +import l from '../../common/logger'; +import NodeCache from 'node-cache'; +import sharedConstants from '../../common/shared-constants'; +import { BoundingBox, FountainCollection, LngLat, parseLngLat } from '../../common/typealias'; +import { ProcessingError } from '../controllers/processing-errors.controller'; +import { getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate } from './generateLocationData.service'; +import { illegalState } from '../../common/illegalState'; + +interface CacheEntry { + value: T; + ttl: number; +} + +// Configuration of Cache after https://www.npmjs.com/package/node-cache +const locationCache = new NodeCache({ + stdTTL: 60 * 60 * sharedConstants.BOUNDING_BOX_TTL_IN_HOURS, // time till cache expires, in seconds + checkperiod: 60 * 15, // how often to check for expiration, in seconds - default: 600 + deleteOnExpire: false, // on expire, we want the cache to be recreated not deleted + useClones: false, // do not create a clone of the data when fetching from cache (because we modify the data I guess) +}); + +export function getCachedFullFountainCollection(tile: Tile): CacheEntry | undefined { + return getCachedFountainCollection(tile, ''); +} +export function getCachedEssentialFountainCollection(tile: Tile): CacheEntry | undefined { + return getCachedFountainCollection(tile, ESSENTIAL_SUFFIX); +} +function getCachedFountainCollection(tile: Tile, suffix: string): CacheEntry | undefined { + return locationCache.get>(tileToLocationCacheKey(tile) + suffix); +} + +export function getCachedProcessingErrors(tile: Tile): CacheEntry | undefined { + return locationCache.get>(tileToLocationCacheKey(tile)); +} + +export function cacheFullFountainCollection(tile: Tile, fountainCollection: FountainCollection, ttl: number): void { + cacheEntry(tile, '', fountainCollection, ttl); +} +export function cacheEssentialFountainCollection( + tile: Tile, + fountainCollection: FountainCollection, + ttl: number +): void { + cacheEntry(tile, ESSENTIAL_SUFFIX, fountainCollection, ttl); +} + +export function cacheProcessingErrors(tile: Tile, errors: ProcessingError[], ttl: number): void { + cacheEntry(tile, PROCESSING_ERRORS_SUFFIX, errors, ttl); +} + +function cacheEntry(tile: Tile, suffix: string, entry: T, ttl: number): void { + locationCache.set>(tileToLocationCacheKey(tile) + suffix, { + value: entry, + ttl: ttl, + }); +} + +//TODO @ralf.hauser, check if it is realy worth it to store essential data +/* + * For each bounding box, 3 JSON objects are created. Example for Zurich: + * - "minLat,minLng:maxLat,maxLng": contains the full data for all fountains within the bounding box + * - "minLat,minLng:maxLat,maxLng_essential": contains the essential data for all fountains within the bounding box. This is the data loaded for display on the map. It is derived from the full data and cached additionally to speed up time + * - "minLat,minLng:maxLat,maxLng_errors": contains a list of errors encountered when processing the fountains within the bounding box + */ +// when cached data expires, regenerate full data (ignore expiration of essential and error data) +locationCache.on('expired', (key: string, cacheEntry: CacheEntry) => { + // check if cache item key is neither the summary nor the list of errors. These will be updated automatically when the detailed city data are updated. + if (isFullDataKey(key)) { + l.info(`controller locationCache.on('expired',...): Automatic cache refresh of ${key}`); + const tile = locationCacheKeyToTile(key); + getByBoundingBoxFromCacheIfNotForceRefreshOrPopulate( + /*forceRefresh= */ true, + tile, + /* essential= */ false, + 'cache expired', + /* debugAll= */ false, + cacheEntry.ttl + ); + } +}); + +const ESSENTIAL_SUFFIX = '_essential'; +const PROCESSING_ERRORS_SUFFIX = '_errors'; +function isFullDataKey(key: string) { + return !key.endsWith(ESSENTIAL_SUFFIX) && !key.endsWith(PROCESSING_ERRORS_SUFFIX); +} + +export function locationCacheKeyToTile(key: string): Tile { + const minMax = key.split(':'); + return BoundingBox(parseLngLat(minMax[0]), parseLngLat(minMax[1])); +} + +export function tileToLocationCacheKey(tile: Tile): string { + return ( + `${tile.min.lat.toFixed(LNG_LAT_STRING_PRECISON)},${tile.min.lng.toFixed(LNG_LAT_STRING_PRECISON)}` + + ':' + + `${tile.max.lat.toFixed(LNG_LAT_STRING_PRECISON)},${tile.max.lng.toFixed(LNG_LAT_STRING_PRECISON)}` + ); +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +// 0.01 lat is ~1km +const TILE_SIZE = 0.01; +const ROUND_FACTOR = 100; +export const LNG_LAT_STRING_PRECISON = 2; +function roundToTilePrecision(n: number): number { + return Math.floor(n * ROUND_FACTOR) / ROUND_FACTOR; +} + +export function getTileOfLocation(lngLat: LngLat): Tile { + // for simplicity reasons we don't take the earth shape into account and + // act as if lng have everywhere the same distance (on all lat) + // + // one tile is 0.1 lat x 0.1 lng + // so the first tile is kind of at 0 - 0.1 lat and 0 - 0.1 lng + const lng = roundToTilePrecision(lngLat.lng); + const lat = roundToTilePrecision(lngLat.lat); + return Tile(lng, lat); +} + +export type Tile = BoundingBox; +export function Tile(lngMin: number, latMin: number): BoundingBox { + const maxLng = lngMin + TILE_SIZE; + const maxLat = latMin + TILE_SIZE; + return BoundingBox(LngLat(lngMin, latMin), LngLat(maxLng, maxLat)); +} + +export function splitInTiles(boundingBox: BoundingBox): Tile[] { + const startTile = getTileOfLocation(boundingBox.min); + const tiles = new Array(); + + for (let lng = startTile.min.lng; lng <= boundingBox.max.lng; lng += TILE_SIZE) { + for (let lat = startTile.min.lat; lat <= boundingBox.max.lat; lat += TILE_SIZE) { + tiles.push(Tile(lng, lat)); + } + } + return tiles; +} + +export function getBoundingBoxOfTiles(tiles: Tile[]): BoundingBox { + if (tiles.length === 0) illegalState('tiles was empty'); + else if (tiles.length === 1) { + return tiles[0]; + } else { + const lngs = tiles.map(x => x.min.lng).sort((a, b) => a - b); + const lats = tiles.map(x => x.min.lat).sort((a, b) => a - b); + const minLng = lngs[0]; + const minLat = lats[0]; + const maxLng = lngs[lngs.length - 1] + TILE_SIZE; + const maxLat = lats[lats.length - 1] + TILE_SIZE; + const boundingBox = BoundingBox(LngLat(minLng, minLat), LngLat(maxLng, maxLat)); + return boundingBox; + } +} diff --git a/server/api/services/osm.service.ts b/server/api/services/osm.service.ts index dcfbf521..6e12d7ed 100644 --- a/server/api/services/osm.service.ts +++ b/server/api/services/osm.service.ts @@ -7,8 +7,8 @@ import l from '../../common/logger'; import osm_fountain_config from '../../../config/fountains.sources.osm'; -import { FountainConfig } from '../../common/typealias'; -//TODO we could use overpas-ts thought I am not sure how well it is maintained and up-to-date +import { BoundingBox, FountainConfig } from '../../common/typealias'; +//TODO @ralf.hauser we could use overpas-ts thought I am not sure how well it is maintained and up-to-date import query_overpass from 'query-overpass'; interface OsmFountainConfigCollection { @@ -44,11 +44,9 @@ class OsmService { }); } - byBoundingBox(latMin: number, lngMin: number, latMax: number, lngMax: number): Promise { - // fetch fountain from OSM by coordinates + byBoundingBox(boundingBox: BoundingBox): Promise { return new Promise((resolve, reject) => { - const query = queryBuilderBox(latMin, lngMin, latMax, lngMax); - // l.info(query); + const query = queryBuilderBoundingBox(boundingBox); query_overpass( query, (error: any, data: OsmFountainConfigCollection) => { @@ -85,13 +83,16 @@ function queryBuilderCenter(lat: number, lng: number, radius = 10): string { `; } -function queryBuilderBox(latMin: number, lngMin: number, latMax: number, lngMax: number): string { +function queryBuilderBoundingBox(boundingBox: BoundingBox): string { // The querybuilder uses the sub_sources defined in osm_fountain_config to know which tags should be queried return ` (${['node', 'way'] .map(e => osm_fountain_config.sub_sources - .map(item => `${e}[${item.tag.name}=${item.tag.value}](${latMin},${lngMin},${latMax},${lngMax});`) + .map( + item => + `${e}[${item.tag.name}=${item.tag.value}](${boundingBox.min.lat},${boundingBox.min.lng},${boundingBox.max.lat},${boundingBox.max.lng});` + ) .join('') ) .join('')} diff --git a/server/api/services/processing.service.ts b/server/api/services/processing.service.ts index 2c4f7399..3123c0db 100644 --- a/server/api/services/processing.service.ts +++ b/server/api/services/processing.service.ts @@ -183,16 +183,7 @@ export function createUniqueIds(fountainArr: Fountain[]): Promise { export function essenceOf(fountainCollection: FountainCollection): FountainCollection { // returns a version of the fountain data with only the essential data - const newCollection: FountainCollection = { - type: 'FeatureCollection', - features: [], - }; - - //TODO @ralfhauser, properties do not exist on the GeoJSON standard for FeatureCollection, in other words, this is a hack. - (newCollection as any).properties = { - // Add last scan time info for https://github.com/water-fountains/proximap/issues/188 - last_scan: new Date(), - }; + const newCollection = FountainCollection([]); // Get list of property names that are marked as essential in the metadata const essentialPropNames: string[] = _.map(fountain_property_metadata, (p, p_name) => { diff --git a/server/api/services/wikidata.service.ts b/server/api/services/wikidata.service.ts index a5c77362..f28540c1 100644 --- a/server/api/services/wikidata.service.ts +++ b/server/api/services/wikidata.service.ts @@ -11,8 +11,10 @@ import { cacheAdapterEnhancer } from 'axios-extensions'; import * as _ from 'lodash'; import wdk from 'wikidata-sdk'; import sharedConstants from '../../common/shared-constants'; -import { Fountain } from '../../common/typealias'; +import { BoundingBox, Fountain } from '../../common/typealias'; import { MediaWikiEntityCollection, MediaWikiEntity, MediaWikiSimplifiedEntity } from '../../common/wikimedia-types'; +import { City } from '../../../config/locations'; +import { LNG_LAT_STRING_PRECISON } from './locationCache'; // Set up caching of http requests const http = axios.create({ @@ -27,7 +29,7 @@ const http = axios.create({ }); class WikidataService { - idsByCenter(lat: number, lng: number, radius = 10, locationName: string): Promise { + idsByCenter(lat: number, lng: number, radius = 10, city: City): Promise { // fetch fountain from OSM by coordinates, within radius in meters const sparql = ` SELECT ?place @@ -49,17 +51,15 @@ class WikidataService { bd:serviceParam wikibase:language "en,de,fr,it,tr" . } }`; - const res = doSparqlRequest(sparql, locationName, 'idsByCenter'); + const res = doSparqlRequest(sparql, 'idsByCenter for city ' + city); return res; } - idsByBoundingBox( - latMin: number, - lngMin: number, - latMax: number, - lngMax: number, - locationName: string - ): Promise { + idsByBoundingBox(bounds: BoundingBox): Promise { + const minLng = bounds.min.lng.toFixed(LNG_LAT_STRING_PRECISON); + const minLat = bounds.min.lat.toFixed(LNG_LAT_STRING_PRECISON); + const maxLng = bounds.max.lng.toFixed(LNG_LAT_STRING_PRECISON); + const maxLat = bounds.max.lat.toFixed(LNG_LAT_STRING_PRECISON); const sparql = ` SELECT ?place WHERE @@ -67,8 +67,8 @@ class WikidataService { SERVICE wikibase:box { # this service allows points within a box to be queried (https://en.wikibooks.org/wiki/SPARQL/SERVICE_-_around_and_box) ?place wdt:P625 ?location . - bd:serviceParam wikibase:cornerWest "Point(${lngMin} ${latMin})"^^geo:wktLiteral. - bd:serviceParam wikibase:cornerEast "Point(${lngMax} ${latMax})"^^geo:wktLiteral. + bd:serviceParam wikibase:cornerSouthWest "Point(${minLng} ${minLat})"^^geo:wktLiteral. + bd:serviceParam wikibase:cornerNorthEast "Point(${maxLng} ${maxLat})"^^geo:wktLiteral. } . # The results of the spatial query are limited to instances or subclasses of water well (Q43483) or fountain (Q483453) @@ -76,14 +76,14 @@ class WikidataService { # the wikibase:label service allows the label to be returned easily. The list of languages provided are fallbacks: if no English label is available, use German etc. SERVICE wikibase:label { - bd:serviceParam wikibase:language "en,de,fr,it,tr" . + bd:serviceParam wikibase:language "${sharedConstants.LANGS.join(',')}" . } }`; - const res = doSparqlRequest(sparql, locationName, 'idsByBoundingBox'); + const res = doSparqlRequest(sparql, 'idsByBoundingBox for ' + JSON.stringify(bounds)); return res; } - byIds(qids: string[], locationName: string): Promise { + byIds(qids: string[], dbg: string): Promise { // fetch fountains by their QIDs const chunkSize = 50; // how many fountains should be fetched at a time (so as to not overload the server) return new Promise((resolve, reject) => { @@ -93,7 +93,7 @@ class WikidataService { chunk(qids, chunkSize).forEach(qidChunk => { chunkCount++; if (chunkSize * chunkCount > qids.length) { - l.info('wikidata.service.js byIds: chunk ' + chunkCount + ' for ' + locationName); + l.info('wikidata.service.js byIds: chunk ' + chunkCount + ' for ' + dbg); } // create sparql url const url = wdk.getEntities({ @@ -108,19 +108,13 @@ class WikidataService { Promise.all(httpPromises) .then(responses => { l.info( - 'wikidata.service.js byIds: ' + - chunkCount + - ' chunks of ' + - chunkSize + - ' prepared for loc "' + - locationName + - '"' + 'wikidata.service.js byIds: ' + chunkCount + ' chunks of ' + chunkSize + ' prepared for loc "' + dbg + '"' ); // holder for data of all fountains let dataAll: MediaWikiSimplifiedEntity[] = []; responses.forEach(r => { // holder for data from each chunk - //TODO should be typed as soon as we update wikidata-sdk to the latest version + //TODO @ralf.hauser should be typed as soon as we update wikidata-sdk to the latest version const data: MediaWikiSimplifiedEntity[] = []; for (const key in r.data.entities) { // simplify object structure of each wikidata entity and add it to 'data' @@ -138,19 +132,19 @@ class WikidataService { if (null != dataAll) { dataAllSize = dataAll.length; } - l.info('wikidata.service.js byIds: dataAll ' + dataAllSize + ' for loc "' + locationName + '"'); + l.info('wikidata.service.js byIds: dataAll ' + dataAllSize + ' for loc "' + dbg + '"'); } // return dataAll to //TODO @ralfhauser that's a smell, we should not use the resolve of an outer promise resolve(dataAll); }) .catch(e => { - l.error('wikidata.service.js byIds: catch e ' + e.stack + ' for loc "' + locationName + '"'); + l.error('wikidata.service.js byIds: catch e ' + e.stack + ' for loc "' + dbg + '"'); //TODO that's a smell, we should not use the reject of an outer promise reject(e); }); } catch (error: any) { - l.error('wikidata.service.js byIds: catch error ' + error.stack + ' for loc "' + locationName + '"'); + l.error('wikidata.service.js byIds: catch error ' + error.stack + ' for loc "' + dbg + '"'); reject(error); } @@ -198,7 +192,6 @@ class WikidataService { const latMin = undefined; const lngMax = undefined; const latMax = undefined; - const locationName = 'undefined'; const newQueryMiro = false; if (newQueryMiro) { @@ -221,7 +214,7 @@ class WikidataService { bd:serviceParam wikibase:language "en,de,fr,it,tr" . } }`; - const res = doSparqlRequest(sparql, locationName, 'fillArtistName'); + const res = doSparqlRequest(sparql, 'fillArtistName'); l.info('wikidata.service.js fillArtistName: new Miro response ' + res + ' "' + idWd + '"'); } @@ -507,7 +500,7 @@ function chunk(arr: T[], len: number): T[][] { return chunks; } -function doSparqlRequest(sparql: string, location: string, dbg: string): Promise { +function doSparqlRequest(sparql: string, dbg: string): Promise { return new Promise((resolve, reject) => { // create url from SPARQL const url = wdk.sparqlQuery(sparql); @@ -521,7 +514,7 @@ function doSparqlRequest(sparql: string, location: string, dbg: string): Promise const error = new Error( `wikidata.service.ts doSparqlRequest Request to Wikidata Failed. Status Code: ${res.status}. Status Message: ${res.statusText}. Url: ${url}` ); - l.error('wikidata.service.js doSparqlRequest: ' + dbg + ', location ' + location + ' ' + error.message); + l.error('wikidata.service.js doSparqlRequest: ' + dbg + ' ' + error.message); // consume response data to free up memory // TODO @ralfhauser, resume does not exist // res.resume(); @@ -533,33 +526,21 @@ function doSparqlRequest(sparql: string, location: string, dbg: string): Promise l.info( 'wikidata.service.js doSparqlRequest: ' + dbg + - ', location ' + - location + ' ' + //+simplifiedResults+' ' simplifiedResults.length + - ' ids found for ' + - location + ' ids found' ); resolve(simplifiedResults); } catch (e: any) { l.error( - 'wikidata.service.js doSparqlRequest: Error occurred simplifying wikidata results.' + - e.stack + - ' ' + - dbg + - ', location ' + - location + 'wikidata.service.js doSparqlRequest: Error occurred simplifying wikidata results.' + e.stack + ' ' + dbg ); reject(e); } }) .catch(error => { l.error( - `'wikidata.service.js doSparqlRequest: Request to Wikidata Failed. Url: ${url}` + - ' ' + - dbg + - ', location ' + - location + `'wikidata.service.js doSparqlRequest: Request to Wikidata Failed. Url: ${url}` + ' ' + dbg + '\n' + error ); reject(error); }); diff --git a/server/common/ArrayExtensions.ts b/server/common/ArrayExtensions.ts new file mode 100644 index 00000000..604cb709 --- /dev/null +++ b/server/common/ArrayExtensions.ts @@ -0,0 +1,131 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ + +import { take } from 'lodash'; +import { compareI18n } from './compare'; +import { illegalState } from './illegalState'; + +export {}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +declare global { + interface Array { + /** + * @author Tegonal GmbH + * @license AGPL + */ + searchUnique(this: T[], predicate: (value: T) => boolean): T; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + isEmpty(this: T[]): boolean; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + nonEmpty(this: T[]): boolean; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + take(number: number): T[]; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + sortI18n(this: T[], propertyAccessFn?: (x: T, y: T) => [string, string]): T[]; + + /** + * @author Tegonal GmbH + * @license AGPL + */ + groupBy(this: T[], keyGetter: (input: T) => K): Map; + } +} + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.searchUnique = function (this: T[], predicate: (value: T) => boolean): T { + const result = this.find(predicate); + if (result !== undefined) { + return result; + } else { + illegalState('searched in array for unique match, nothing found', this, 'used predicate', predicate); + } +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.isEmpty = function (this: T[]): boolean { + return this.length === 0; +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.nonEmpty = function (this: T[]): boolean { + return !this.isEmpty(); +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.take = function (this: T[], number: number): T[] { + return take(this, number); +}; + +/** + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.sortI18n = function (this: T[], propertyAccessFn?: (a: T, b: T) => [string, string]): T[] { + return this.sort((a, b) => { + const [as, bs] = propertyAccessFn ? propertyAccessFn(a, b) : ['' + a, '' + b]; + + return compareI18n(as, bs); + }); +}; + +/** + * @description + * Takes an Array, and a grouping function, + * and returns a Map of the array grouped by the grouping function. + * + * @param this The receiver array of type V. + * @param keyGetter A Function that takes the the Array type V as an input, and returns a value of type K. + * K is generally intended to be a property key of V. + * + * @returns Map of the array grouped by the grouping function. + * + * @author Tegonal GmbH + * @license AGPL + */ +Array.prototype.groupBy = function (this: V[], keyGetter: (input: V) => K): Map { + const map = new Map(); + this.forEach((item: V) => { + const key = keyGetter(item); + const collection = map.get(key); + if (!collection) { + map.set(key, [item]); + } else { + collection.push(item); + } + }); + return map; +}; diff --git a/server/common/build.info.ts b/server/common/build.info.ts index b07f204a..299f0242 100644 --- a/server/common/build.info.ts +++ b/server/common/build.info.ts @@ -1,9 +1,9 @@ // this file is automatically generated by git.version.js script const buildInfo = { version: '', - revision: '7acb2bc', - branch: '#150-forceRefresh-id', - commit_time: '2021-12-20 10:31:35 +0100', - build_time: 'Mon Dec 20 2021 11:11:32 GMT+0100 (Central European Standard Time)', + revision: '11e097b', + branch: '#148-load-by-bound', + commit_time: '2021-11-01 09:01:34 +0000', + build_time: 'Wed Dec 22 2021 16:21:24 GMT+0100 (Central European Standard Time)', }; export default buildInfo; diff --git a/server/common/compare.ts b/server/common/compare.ts new file mode 100644 index 00000000..4e2e7287 --- /dev/null +++ b/server/common/compare.ts @@ -0,0 +1,21 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ +export function compareI18n(a: string, b: string): number { + return new Intl.Collator([], { numeric: true }).compare(a, b); +} + +/** + * @author Tegonal GmbH + * @license AGPL + */ +export function compareBoolean(a: boolean, b: boolean, whenEqual: () => number): number { + if (a && !b) { + return -1; + } else if (!a && b) { + return 1; + } else { + return whenEqual(); + } +} diff --git a/server/common/illegalState.ts b/server/common/illegalState.ts new file mode 100644 index 00000000..d693ca06 --- /dev/null +++ b/server/common/illegalState.ts @@ -0,0 +1,13 @@ +import l from '../common/logger'; + +/** + * Since throw is not an expression in typescript you cannot do thing like ... || throw + * Hence this function + * @author Tegonal GmbH + * @license AGPL + */ +export function illegalState(msg: string, ...optionalParams: any[]): never { + const errorMsg = msg + ' ' + optionalParams.map(x => JSON.stringify(x)).join(' // '); + l.error(errorMsg); + throw new Error('IllegalState detected: ' + errorMsg); +} diff --git a/server/common/importAllExtensions.ts b/server/common/importAllExtensions.ts new file mode 100644 index 00000000..c84b4ed8 --- /dev/null +++ b/server/common/importAllExtensions.ts @@ -0,0 +1,5 @@ +/** + * @author Tegonal GmbH + * @license AGPL + */ +import './ArrayExtensions'; diff --git a/server/common/shared-constants.ts b/server/common/shared-constants.ts index a88c9ec7..38a71573 100644 --- a/server/common/shared-constants.ts +++ b/server/common/shared-constants.ts @@ -14,6 +14,7 @@ export default { //TODO proximap#394 reactivate serbian again, see also TODO in proximap LANGS: ['en', 'de', 'fr', 'it', 'tr' /* 'sr' */], - CACHE_FOR_HRS_i45db: 48, + CITY_TTL_IN_HOURS: 2 * 24, + BOUNDING_BOX_TTL_IN_HOURS: 7 * 24, gak: `${process.env.GOOGLE_API_KEY}`, }; diff --git a/server/common/sleep.ts b/server/common/sleep.ts new file mode 100644 index 00000000..23648f73 --- /dev/null +++ b/server/common/sleep.ts @@ -0,0 +1,8 @@ +/** + * primitive sleep function using setTimeout + * @author Tegonal GmbH + * @license AGPL + */ +export async function sleep(milliseconds: number): Promise { + return new Promise((resolve, _) => setTimeout(resolve, milliseconds)); +} diff --git a/server/common/swagger/Api.yaml b/server/common/swagger/Api.yaml index 20b6f6f9..aa5bcc4d 100644 --- a/server/common/swagger/Api.yaml +++ b/server/common/swagger/Api.yaml @@ -68,12 +68,12 @@ paths: example: wikidata required: true description: database for which the provided identifier is valid - - name: city + - name: loc in: query type: string - example: ch-zh + example: 47.3646083,8.5380421 required: true - description: code of city for which fountains are to be served + description: lat,lng of fountain, necessary in order that we can load - name: idval in: query type: string @@ -91,12 +91,20 @@ paths: get: description: Fetch fountains within bounding box parameters: - - name: city + - name: sw in: query type: string - example: ch-zh + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.3229261255644,8.45960259979614 + required: true + description: lat,lng of the south west location of the bounding box + - name: ne + in: query + type: string + pattern: \d+(.\d+)?,\d+(\.d+)? + example: 47.431119712250506,8.61940272745742 required: true - description: code of location for which fountains are to be served + description: lat,lng of the north east location of the bounding box - name: refresh in: query type: boolean diff --git a/server/common/swagger/index.ts b/server/common/swagger/index.ts index 5d4cc5b7..2f007feb 100644 --- a/server/common/swagger/index.ts +++ b/server/common/swagger/index.ts @@ -32,7 +32,6 @@ export function swaggerify(app: Express, routerProvider: (app: Express) => Route cookie: { secret: process.env.SESSION_SECRET, }, - // Don't allow JSON content over 100kb (default is 1mb) json: { limit: process.env.REQUEST_LIMIT, }, diff --git a/server/common/typealias.ts b/server/common/typealias.ts index 0258b79a..bf1b2c34 100644 --- a/server/common/typealias.ts +++ b/server/common/typealias.ts @@ -1,6 +1,9 @@ import { Feature, FeatureCollection, Geometry, Point } from 'geojson'; +import { UncheckedBoundingBox } from '../../config/locations'; import { ImageLikeCollection, ImageLikeType } from '../../config/text2img'; +import { isNumeric } from '../api/controllers/utils'; import { PropStatus } from './constants'; +import { illegalState } from './illegalState'; import { Category, MediaWikiSimplifiedEntity } from './wikimedia-types'; // TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap @@ -32,7 +35,11 @@ export type Fountain = FeatureCollection< G, FountainPropertyCollection> ->; +> & { last_scan?: Date }; + +export function FountainCollection(fountains: Fountain[], lastScan: Date = new Date()): FountainCollection { + return { type: 'FeatureCollection', features: fountains, last_scan: lastScan }; +} export type FountainConfig = SourceConfig; @@ -136,3 +143,45 @@ export type Database = SourceType; export function isDatabase(d: string): d is Database { return d === 'osm' || d === 'wikidata'; } + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export interface LngLat { + lng: number; + lat: number; +} +export function LngLat(lng: number, lat: number): LngLat { + if (lng < -180 || lng > 180) illegalState('lng out of range [-180, 180]', lng); + if (lat < -90 || lat > 90) illegalState('lat out of range [-180, 180]', lat); + + return { lng: lng, lat: lat }; +} +export function parseLngLat(lngLatAsString: string): LngLat { + const lngLatArr = lngLatAsString.split(','); + if (lngLatArr.length >= 2 && isNumeric(lngLatArr[0]) && isNumeric(lngLatArr[1])) { + const lat = Number(lngLatArr[0]); + const lng = Number(lngLatArr[1]); + return LngLat(lng, lat); + } else { + illegalState('could not parse to LngLat, given string: ' + lngLatAsString); + } +} + +// TODO it would make more sense to move common types to an own library which is consumed by both, datablue and proximap +// if you change something here, then you need to change it in proximap as well +export interface BoundingBox { + min: LngLat; + max: LngLat; +} +export function BoundingBox(min: LngLat, max: LngLat): BoundingBox { + if (min.lng >= max.lng) illegalState('min lng greater or equal max lng.', 'min', min, 'max', max); + if (min.lat >= max.lat) illegalState('min lat greater or equal to max lat', 'min', min, 'max', max); + + return { min: min, max: max }; +} +export function uncheckedBoundingBoxToChecked(uncheckedBoundingBox: UncheckedBoundingBox): BoundingBox { + return BoundingBox( + LngLat(uncheckedBoundingBox.lngMin, uncheckedBoundingBox.latMin), + LngLat(uncheckedBoundingBox.lngMax, uncheckedBoundingBox.latMax) + ); +}