From 60e1ffb84e4e9416207532cc43b2871b591cbb0f Mon Sep 17 00:00:00 2001 From: TheCodemonkey <7ilya@gmx.de> Date: Tue, 19 May 2020 10:29:44 +0200 Subject: [PATCH 1/5] defined weights and calculate completeness index --- .vscode/settings.json | 5 ++ src/common/models/traderProfile.ts | 1 + src/common/services/import/import.service.ts | 48 ++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ad21567 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + // "vscode_custom_css.imports": [ + // "file:///Users/ILeyberman/.vscode/extensions/webrender.synthwave-x-fluoromachine-0.0.11/synthwave-x-fluoromachine.css" + // ] + } \ No newline at end of file diff --git a/src/common/models/traderProfile.ts b/src/common/models/traderProfile.ts index b846567..88c0134 100644 --- a/src/common/models/traderProfile.ts +++ b/src/common/models/traderProfile.ts @@ -18,6 +18,7 @@ export interface TraderProfile { status: TraderProfileStatus; soMeShare: boolean; confirmedLocation?: number[]; + completenessIndex: number; storeType: { gastronomie: boolean; lebensmittel: boolean; diff --git a/src/common/services/import/import.service.ts b/src/common/services/import/import.service.ts index 52af85f..11befa5 100644 --- a/src/common/services/import/import.service.ts +++ b/src/common/services/import/import.service.ts @@ -1,5 +1,31 @@ // import * as traders from '../../repositories/traders.repository'; import admin = require('firebase-admin'); +import { TraderProfile } from '../../models/traderProfile'; + + + +const WEIGHTS_OF_COMP_INDEX: any = { + businessname: 10, + ownerFirstname: 10, + ownerLastname: 10, + postcode: 10, + city: 10, + street: 10, + number: 10, + description: 10, + pickup: 5, + delivery: 5, + email: 25, + telephone: 20, + homepage: 20, + confirmedLocation: 50, + defaultImagePath: 0 +} + +const MAX_COMP_INDEX = Object.values(WEIGHTS_OF_COMP_INDEX) + .map(v => v as number) + .reduce((a,b) => a + b); + function getServiceProvider(source: string){ return require('./' + source +'/provider'); @@ -15,12 +41,34 @@ export async function importData(app: admin.app.App, source: string, options: an for(const trader of traders) { try { // console.log(trader); + buildCompletenessIndex(trader); + // console.log(trader.completenessIndex); // await traders.upsert(app, trader); } catch(e) { console.log('errow while importing', trader); } } + const sortedTraders = (traders as []).sort((a: any, b: any) => a.completenessIndex - b.completenessIndex); + sortedTraders.forEach((t:any) => { + console.log(t.completenessIndex); + }); + console.log('items: ' + traders.length) } +} + +function buildCompletenessIndex(trader:TraderProfile) { + let currentIndex = 0; + + if (trader) { + + for(const key of Object.keys(trader)){ + const prop = (trader as any)[key]; + if (prop && WEIGHTS_OF_COMP_INDEX[key]) + currentIndex += WEIGHTS_OF_COMP_INDEX[key]; + } + + trader.completenessIndex = 100 / MAX_COMP_INDEX * currentIndex; + } } \ No newline at end of file From 308f28d4fe97a40f9a681871248de7f369050e9f Mon Sep 17 00:00:00 2001 From: TheCodemonkey <7ilya@gmx.de> Date: Tue, 19 May 2020 11:18:06 +0200 Subject: [PATCH 2/5] mapp additional osm properties adjust the cmp indx --- src/adminTasks/import.ts | 2 +- src/common/models/traderProfile.ts | 4 +++ src/common/services/import/import.service.ts | 17 ++++++++--- .../services/import/lokalwirkt/mapping.ts | 30 +++++++++++++++++++ 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/adminTasks/import.ts b/src/adminTasks/import.ts index b5f1275..7589e37 100644 --- a/src/adminTasks/import.ts +++ b/src/adminTasks/import.ts @@ -6,7 +6,7 @@ async function importData(app: admin.app.App) { console.log('start import...'); const options = { - region: 'bonn', + region: 'unna', file: '/tmp/lokalkauf/import.csv' } diff --git a/src/common/models/traderProfile.ts b/src/common/models/traderProfile.ts index 88c0134..b7b0591 100644 --- a/src/common/models/traderProfile.ts +++ b/src/common/models/traderProfile.ts @@ -19,6 +19,10 @@ export interface TraderProfile { soMeShare: boolean; confirmedLocation?: number[]; completenessIndex: number; + openingTime: any[]; + osm_id: string; + osm_category: any; + licence: string; storeType: { gastronomie: boolean; lebensmittel: boolean; diff --git a/src/common/services/import/import.service.ts b/src/common/services/import/import.service.ts index 11befa5..3490839 100644 --- a/src/common/services/import/import.service.ts +++ b/src/common/services/import/import.service.ts @@ -15,6 +15,7 @@ const WEIGHTS_OF_COMP_INDEX: any = { description: 10, pickup: 5, delivery: 5, + openingTime: 20, email: 25, telephone: 20, homepage: 20, @@ -22,7 +23,7 @@ const WEIGHTS_OF_COMP_INDEX: any = { defaultImagePath: 0 } -const MAX_COMP_INDEX = Object.values(WEIGHTS_OF_COMP_INDEX) +const MAX_COMP_SCORE = Object.values(WEIGHTS_OF_COMP_INDEX) .map(v => v as number) .reduce((a,b) => a + b); @@ -51,6 +52,7 @@ export async function importData(app: admin.app.App, source: string, options: an const sortedTraders = (traders as []).sort((a: any, b: any) => a.completenessIndex - b.completenessIndex); sortedTraders.forEach((t:any) => { + // console.log(t.completenessIndex); console.log(t.completenessIndex); }); @@ -65,10 +67,17 @@ function buildCompletenessIndex(trader:TraderProfile) { for(const key of Object.keys(trader)){ const prop = (trader as any)[key]; - if (prop && WEIGHTS_OF_COMP_INDEX[key]) - currentIndex += WEIGHTS_OF_COMP_INDEX[key]; + if (prop && WEIGHTS_OF_COMP_INDEX[key]) { + if (Array.isArray(prop)) { + if ((prop as []).length > 0) + currentIndex += WEIGHTS_OF_COMP_INDEX[key]; + } + else if (prop.toString().trim().length > 0) + currentIndex += WEIGHTS_OF_COMP_INDEX[key]; + } } - trader.completenessIndex = 100 / MAX_COMP_INDEX * currentIndex; + // calculates the ratio depends on the max possible score + trader.completenessIndex = 100 / MAX_COMP_SCORE * currentIndex; } } \ No newline at end of file diff --git a/src/common/services/import/lokalwirkt/mapping.ts b/src/common/services/import/lokalwirkt/mapping.ts index 3fe639a..da9c28e 100644 --- a/src/common/services/import/lokalwirkt/mapping.ts +++ b/src/common/services/import/lokalwirkt/mapping.ts @@ -36,6 +36,10 @@ export function mapToTrader(lokalwirktModel:any[]) { street: lw.address, telephone: lw.phone, homepage: lw.website, + openingTime: getOpeningTime(lw["opening-time"]), + osm_id: lw.osm_id, + osm_category: getOSMCategories(lw.category), + licence : lw.licence, confirmedLocation : [Number(lw.lat), Number(lw.lon)], storeType: mapToTraderCategory(lw.category), status: TraderProfileStatus.IMPORTED @@ -69,7 +73,33 @@ export function mapToTraderCategory(category: any) { return out; } +function getOSMCategories (category: any) { + if (category && category.length > 0) { + return (category as []).map((c: any) => { + return { + slug: c.slug, + name: c.name + } + }) + } + + return []; +} + +function getOpeningTime(openingTime:any[]) { + + if (openingTime && openingTime.length > 0) { + return openingTime.map(o => { + return { + weekday: o.weekday, + open: o.open, + close: o.close + } + }) + } + return []; +} // "community_centre", - // "beverages", x From 4067525e227fafe923179fc73a24d65f055e0c36 Mon Sep 17 00:00:00 2001 From: TheCodemonkey <7ilya@gmx.de> Date: Sun, 24 May 2020 14:22:53 +0200 Subject: [PATCH 3/5] #20 import additional osm_ attributes --- package-lock.json | 328 +++++++++++++++++- package.json | 4 +- src/common/models/traderProfile.ts | 1 + src/common/repositories/traders.repository.ts | 15 +- src/common/services/db.backup.ts | 2 +- src/common/services/import/import.service.ts | 9 +- .../services/import/lokalwirkt/mapping.ts | 5 + .../services/import/lokalwirkt/provider.ts | 71 +++- 8 files changed, 415 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4dd4979..eab2e21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -593,6 +593,67 @@ "protobufjs": "^6.8.6" } }, + "@mapbox/geojson-area": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@mapbox/geojson-area/-/geojson-area-0.2.2.tgz", + "integrity": "sha1-GNeBSqNr8j+7zDefjiaiKSfevxA=", + "requires": { + "wgs84": "0.0.0" + } + }, + "@mapbox/geojson-rewind": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@mapbox/geojson-rewind/-/geojson-rewind-0.4.0.tgz", + "integrity": "sha512-b+1uPWBERW4Pet/969BNu61ZPDyH2ilIxBjJDFzxyS9TyszF9UrTQyYIl/G38clux3rtpAGGFSGTCSF/qR6UjA==", + "requires": { + "@mapbox/geojson-area": "0.2.2", + "concat-stream": "~1.6.0", + "minimist": "1.2.0", + "sharkdown": "^0.1.0" + }, + "dependencies": { + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -741,6 +802,12 @@ "@types/node": "*" } }, + "@types/geojson": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-1.0.6.tgz", + "integrity": "sha512-Xqg/lIZMrUd0VRmSRbCAewtwGZiAk3mEUDvV4op1tGl+LvyPcb/MIOSxTl9z+9+J+R4/vpjiCAT4xeKzH9ji1w==", + "optional": true + }, "@types/lodash": { "version": "4.14.149", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", @@ -786,6 +853,22 @@ "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.0.tgz", "integrity": "sha512-flgpHJjntpBAdJD43ShRosQvNC0ME97DCfGvZEDlAThQmnerRXrLbX6YgzRBQCZTthET9eAWFAMaYP0m0Y4HzQ==" }, + "JSONStream": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-0.8.0.tgz", + "integrity": "sha1-78Ri1aW8lOwAf0siVxrNf28q4BM=", + "requires": { + "jsonparse": "0.0.5", + "through": "~2.2.7" + }, + "dependencies": { + "through": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/through/-/through-2.2.7.tgz", + "integrity": "sha1-bo4hIAGR1OtqmfbwEN9Gqhxusr0=" + } + } + }, "abab": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/abab/-/abab-1.0.4.tgz", @@ -850,6 +933,11 @@ "color-convert": "^1.9.0" } }, + "ansicolors": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.2.1.tgz", + "integrity": "sha1-vgiVmQl7dKXJxKhKDNvNtivYeu8=" + }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -956,6 +1044,11 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "blueimp-md5": { + "version": "2.16.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.16.0.tgz", + "integrity": "sha512-j4nzWIqEFpLSbdhUApHRGDwfXbV8ALhqOn+FY5L6XBdKPAXU9BpGgFSbDsgqogfqPPR9R2WooseWCsfhfEC6uQ==" + }, "body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz", @@ -1005,8 +1098,7 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "optional": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "builtin-modules": { "version": "1.1.1", @@ -1019,6 +1111,15 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, + "cardinal": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-0.4.4.tgz", + "integrity": "sha1-ylu2iltRG5D+k7ms6km97lwyv+I=", + "requires": { + "ansicolors": "~0.2.1", + "redeyed": "~0.4.0" + } + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -1078,7 +1179,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "optional": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -1552,6 +1652,27 @@ "resolved": "https://registry.npmjs.org/dom-storage/-/dom-storage-2.1.0.tgz", "integrity": "sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==" }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.2.1.tgz", + "integrity": "sha1-Wd+dzSJ+gIs2Wuc+H2aErD2Ub8I=", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.3.0.tgz", + "integrity": "sha1-mtTVm1r2ymhMYv5tdo7xcOcN8ZI=", + "requires": { + "domelementtype": "1" + } + }, "dot-prop": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", @@ -2131,6 +2252,15 @@ "firebase": ">= 6.0.0" } }, + "geojson-numeric": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/geojson-numeric/-/geojson-numeric-0.2.1.tgz", + "integrity": "sha512-rvItMp3W7pe16o2EQTnRw54v6WHdiE4bYjUsdr3FZskFb6oPC7gjLe4zginP+Wd1B/HLl2acTukfn16Lmwn7lg==", + "requires": { + "concat-stream": "2.0.0", + "optimist": "~0.3.5" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -2315,6 +2445,40 @@ "whatwg-encoding": "^1.0.1" } }, + "htmlparser2": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.5.1.tgz", + "integrity": "sha1-b0L3ZX3RnBP31l3pEYQXOUoL5tA=", + "requires": { + "domelementtype": "1", + "domhandler": "2.2", + "domutils": "1.3", + "readable-stream": "1.1" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "http-errors": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", @@ -2383,6 +2547,11 @@ "resolved": "https://registry.npmjs.org/idb/-/idb-3.0.2.tgz", "integrity": "sha512-+FLa/0sTXqyux0o6C+i2lOR0VoS60LU/jzUo5xjfY6+7sEEgy4Gz1O7yFBXvjd7N0NyIGWIRg8DcQSLEG+VSPw==" }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -2640,6 +2809,11 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsonparse": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-0.0.5.tgz", + "integrity": "sha1-MwVCrT8KZUZlt3jz6y2an6UHrGQ=" + }, "jsonwebtoken": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.1.0.tgz", @@ -2955,6 +3129,14 @@ "mimic-fn": "^2.1.0" } }, + "optimist": { + "version": "0.3.7", + "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.3.7.tgz", + "integrity": "sha1-yQlBrVnkJzMokjB00s8ufLxuwNk=", + "requires": { + "wordwrap": "~0.0.2" + } + }, "optionator": { "version": "0.8.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", @@ -2968,6 +3150,28 @@ "word-wrap": "~1.2.3" } }, + "osm-polygon-features": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/osm-polygon-features/-/osm-polygon-features-0.9.2.tgz", + "integrity": "sha1-IK5BEwxIbkmjsqPCtYoUGcSYZ3g=" + }, + "osmtogeojson": { + "version": "3.0.0-beta.4", + "resolved": "https://registry.npmjs.org/osmtogeojson/-/osmtogeojson-3.0.0-beta.4.tgz", + "integrity": "sha512-GwNy2w5JKOplOBspagcNhCDhBRV6Du2BCvcLkaA7nX12U86Dl2Ciw9zs/VzFFTXfyZlaK+7bGCWN2SNlfn/jOA==", + "requires": { + "@mapbox/geojson-rewind": "0.4.0", + "@types/geojson": "^1.0.2", + "JSONStream": "0.8.0", + "concat-stream": "2.0.0", + "geojson-numeric": "0.2.1", + "htmlparser2": "3.5.1", + "optimist": "~0.3.5", + "osm-polygon-features": "^0.9.1", + "tiny-osmpbf": "^0.1.0", + "xmldom": "~0.1.16" + } + }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -3009,6 +3213,15 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" }, + "pbf": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.2.1.tgz", + "integrity": "sha512-ClrV7pNOn7rtmoQVF4TS1vyU0WhYRnP92fzbfF75jAIwpnzdJXf8iTd4CMEqO4yUenH6NDqLiwjqlh6QgZzgLQ==", + "requires": { + "ieee754": "^1.1.12", + "resolve-protobuf-schema": "^2.1.0" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -3022,8 +3235,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "optional": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "promise-polyfill": { "version": "8.1.3", @@ -3057,6 +3269,11 @@ } } }, + "protocol-buffers-schema": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.4.0.tgz", + "integrity": "sha512-G/2kcamPF2S49W5yaMGdIpkG6+5wZF0fzBteLKgEHjbNzqjZQ85aAs1iJGto31EJaSTkNvHs5IXuHSaTLWBAiA==" + }, "proxy-addr": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", @@ -3136,13 +3353,27 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "optional": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, + "redeyed": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-0.4.4.tgz", + "integrity": "sha1-N+mQpvKyGyoRwuakj9QTVpjLqX8=", + "requires": { + "esprima": "~1.0.4" + }, + "dependencies": { + "esprima": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", + "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=" + } + } + }, "regexp.prototype.flags": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", @@ -3201,6 +3432,14 @@ "path-parse": "^1.0.6" } }, + "resolve-protobuf-schema": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", + "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "requires": { + "protocol-buffers-schema": "^3.3.1" + } + }, "retry-request": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.1.1.tgz", @@ -3307,6 +3546,23 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, + "sharkdown": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/sharkdown/-/sharkdown-0.1.1.tgz", + "integrity": "sha512-exwooSpmo5s45lrexgz6Q0rFQM574wYIX3iDZ7RLLqOb7IAoQZu9nxlZODU972g19sR69OIpKP2cpHTzU+PHIg==", + "requires": { + "cardinal": "~0.4.2", + "minimist": "0.0.5", + "split": "~0.2.10" + }, + "dependencies": { + "minimist": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz", + "integrity": "sha1-16oye87PUY+RBqxrjwA/o7zqhWY=" + } + } + }, "side-channel": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.2.tgz", @@ -3335,6 +3591,14 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "optional": true }, + "split": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/split/-/split-0.2.10.tgz", + "integrity": "sha1-Zwl8YB1pfOE2j0GPBs0gHPBSGlc=", + "requires": { + "through": "2" + } + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -3424,7 +3688,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "optional": true, "requires": { "safe-buffer": "~5.1.0" }, @@ -3432,8 +3695,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "optional": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" } } }, @@ -3477,6 +3739,11 @@ "uuid": "^7.0.0" } }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + }, "through2": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", @@ -3486,6 +3753,20 @@ "readable-stream": "2 || 3" } }, + "tiny-inflate": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", + "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==" + }, + "tiny-osmpbf": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/tiny-osmpbf/-/tiny-osmpbf-0.1.0.tgz", + "integrity": "sha1-ColXFxE+vmquNjxL5e76js2vuSc=", + "requires": { + "pbf": "^3.0.4", + "tiny-inflate": "^1.0.2" + } + }, "tmp": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", @@ -3589,8 +3870,7 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", - "optional": true + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" }, "typedarray-to-buffer": { "version": "3.1.5", @@ -3632,8 +3912,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "optional": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", @@ -3687,6 +3966,11 @@ "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==" }, + "wgs84": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/wgs84/-/wgs84-0.0.0.tgz", + "integrity": "sha1-NP3FVZF7blfPKigu0ENxDASc3HY=" + }, "whatwg-encoding": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz", @@ -3755,11 +4039,24 @@ "is-typed-array": "^1.1.3" } }, + "wikimedia-commons-file-path": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/wikimedia-commons-file-path/-/wikimedia-commons-file-path-1.2.0.tgz", + "integrity": "sha512-nUFMGHmZIUiat6YcX82SIUf7NRzYFCVw8UZSfH0XcUm+BF6+i8POj9C1idMj4OPTgc6ZtGhElDCS0ovphJXGaw==", + "requires": { + "blueimp-md5": "^2.10.0" + } + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, + "wordwrap": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", + "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=" + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -3788,6 +4085,11 @@ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-2.0.1.tgz", "integrity": "sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=" }, + "xmldom": { + "version": "0.1.31", + "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.31.tgz", + "integrity": "sha512-yS2uJflVQs6n+CyjHoaBmVSqIDevTAWrzMmjG1Gc7h1qQ7uVozNhEPJAwZXWyGQ/Gafo3fCwrcaokezLPupVyQ==" + }, "xmlhttprequest": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", diff --git a/package.json b/package.json index 55b9b6c..d2e190d 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,9 @@ "geofirestore": "^3.4.1", "javascript-stringify": "^2.0.1", "moment": "^2.25.2", - "tmp": "^0.2.1" + "osmtogeojson": "^3.0.0-beta.4", + "tmp": "^0.2.1", + "wikimedia-commons-file-path": "^1.2.0" }, "devDependencies": { "tslint": "^5.12.0", diff --git a/src/common/models/traderProfile.ts b/src/common/models/traderProfile.ts index b7b0591..66f8e5b 100644 --- a/src/common/models/traderProfile.ts +++ b/src/common/models/traderProfile.ts @@ -22,6 +22,7 @@ export interface TraderProfile { openingTime: any[]; osm_id: string; osm_category: any; + osm_modified: number; licence: string; storeType: { gastronomie: boolean; diff --git a/src/common/repositories/traders.repository.ts b/src/common/repositories/traders.repository.ts index cf7368f..e1a2b8d 100644 --- a/src/common/repositories/traders.repository.ts +++ b/src/common/repositories/traders.repository.ts @@ -1,6 +1,7 @@ import { TraderEntity } from '../models/traderEntity'; import admin = require('firebase-admin'); import { TraderProfile } from '../models/traderProfile'; +const wikimedia = require("wikimedia-commons-file-path"); export async function loadTraders(app: admin.app.App = admin.app()) : Promise { @@ -18,4 +19,16 @@ export async function upsert(app: admin.app.App = admin.app(), trader: Partial a.completenessIndex - b.completenessIndex); sortedTraders.forEach((t:any) => { // console.log(t.completenessIndex); - console.log(t.completenessIndex); + // console.log(t); }); + await tradersRepo.upsert(app, sortedTraders.reverse()[0]); + console.log('items: ' + traders.length) } } + function buildCompletenessIndex(trader:TraderProfile) { let currentIndex = 0; diff --git a/src/common/services/import/lokalwirkt/mapping.ts b/src/common/services/import/lokalwirkt/mapping.ts index da9c28e..aea914f 100644 --- a/src/common/services/import/lokalwirkt/mapping.ts +++ b/src/common/services/import/lokalwirkt/mapping.ts @@ -1,4 +1,5 @@ import { TraderProfile, TraderProfileStatus } from "../../../models/traderProfile"; +import moment = require("moment"); const CATEGORIES: any = { "blumengarten" : ["florist", "garden_centre", "garden_furniture"], @@ -31,6 +32,9 @@ export function mapToTrader(lokalwirktModel:any[]) { for(const lw of lokalwirktModel) { const trader = { businessname: lw.name, + description: lw.description, + ownerFirstname: lw.firstname, + ownerLastname: lw.lastname, postcode: lw.postalcode, city: lw.locality, street: lw.address, @@ -39,6 +43,7 @@ export function mapToTrader(lokalwirktModel:any[]) { openingTime: getOpeningTime(lw["opening-time"]), osm_id: lw.osm_id, osm_category: getOSMCategories(lw.category), + osm_modified: moment(lw.modified).unix(), licence : lw.licence, confirmedLocation : [Number(lw.lat), Number(lw.lon)], storeType: mapToTraderCategory(lw.category), diff --git a/src/common/services/import/lokalwirkt/provider.ts b/src/common/services/import/lokalwirkt/provider.ts index 2c8f961..5022944 100644 --- a/src/common/services/import/lokalwirkt/provider.ts +++ b/src/common/services/import/lokalwirkt/provider.ts @@ -1,4 +1,5 @@ const axios = require('axios'); +const osmtogeojson = require('osmtogeojson'); import * as fs from 'fs'; import { mapToTrader } from './mapping'; import * as categories from './categories'; @@ -43,12 +44,20 @@ export async function loadData(options: any) { data = cache[d.properties.id]; } else { data = await loadDetails(d.properties.id); - cache[d.properties.id] = data; - fs.writeFileSync(FILE_BUFFER, JSON.stringify(cache), {encoding:'utf8'}); + await updateCache(d.properties.id, cache, data); } if (data) { + // get original osm data... + if ((!data.osm_original || data.osm_original.ERROR) && data.data.osm_id) { + data.osm_original = await loadOriginalOSM_Data(data.data.osm_id, data.data); + await updateCache(d.properties.id, cache, data); + } + + // fill addresses + await fillAddresses(data); + items.push(data.data); } } @@ -73,3 +82,61 @@ export async function loadData(options: any) { return []; } + + + +async function updateCache(itemID:string, cache: any, data: any) { + cache[itemID] = data; + fs.writeFileSync(FILE_BUFFER, JSON.stringify(cache), {encoding:'utf8'}); +} + +async function fillAddresses(data: any) { + + if (data && data.data ) { + + if (!data.data.housenumber && + data.osm_original?.features?.properties && + data.osm_original.features.properties['addr:housenumber']) { + + data.data.housenumber = data.osm_original.features.properties['addr:housenumber']; + + } + + // if (!data.data.address || !data.data.postalcode || !data.data.locality) { + + // } + + // if (data.osm_original) { + // data.data.address = data.data.address = + // } + } +} + +async function loadOriginalOSM_Data(osmID: string, data:any) { + let result = await loadOSMData('node', osmID); + + if (!result) + result = await loadOSMData('relation', osmID); + + if (!result) + result = await loadOSMData('way', osmID); + + return result; +} + +async function loadOSMData(type: string, osmID:string) { + try { + const response = await axios.get('https://www.openstreetmap.org/api/0.6/' + type + '/' + osmID); + + if (response && response.data) { + console.log(response.data); + return osmtogeojson(response.data); + } + + return null; + } catch(e) { + console.log('[' + type + '] error while loading original osm data.' + e, osmID); + return null; + } +} + From f6588bbe9c34519c434770a570b4d88e8ea467cd Mon Sep 17 00:00:00 2001 From: TheCodemonkey <7ilya@gmx.de> Date: Fri, 29 May 2020 14:06:11 +0200 Subject: [PATCH 4/5] extend import db import --- package-lock.json | 20 +++++++-- package.json | 2 + src/common/models/traderProfile.ts | 1 + src/common/repositories/traders.repository.ts | 14 ++++++ src/common/services/import/import.service.ts | 28 ++++++------ .../services/import/lokalwirkt/mapping.ts | 6 ++- .../services/import/lokalwirkt/provider.ts | 43 ++++++++++++------- 7 files changed, 82 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index eab2e21..d2739fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -853,6 +853,11 @@ "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.0.tgz", "integrity": "sha512-flgpHJjntpBAdJD43ShRosQvNC0ME97DCfGvZEDlAThQmnerRXrLbX6YgzRBQCZTthET9eAWFAMaYP0m0Y4HzQ==" }, + "@types/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-xSQfNcvOiE5f9dyd4Kzxbof1aTrLobL278pGLKOZI6esGfZ7ts9Ka16CzIN6Y8hFHE1C7jIBZokULhK1bOgjRw==" + }, "JSONStream": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-0.8.0.tgz", @@ -3737,6 +3742,14 @@ "node-fetch": "^2.2.0", "stream-events": "^1.0.5", "uuid": "^7.0.0" + }, + "dependencies": { + "uuid": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", + "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==", + "optional": true + } } }, "through": { @@ -3920,10 +3933,9 @@ "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, "uuid": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz", - "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==", - "optional": true + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.1.0.tgz", + "integrity": "sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==" }, "vary": { "version": "1.1.2", diff --git a/package.json b/package.json index d2e190d..0e8a5c8 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@sendgrid/mail": "^7.0.0", "@slack/webhook": "^5.0.3", "@types/tmp": "^0.2.0", + "@types/uuid": "^8.0.0", "axios": "^0.19.2", "csvtojson": "^2.0.10", "d3-node": "^2.2.1", @@ -28,6 +29,7 @@ "moment": "^2.25.2", "osmtogeojson": "^3.0.0-beta.4", "tmp": "^0.2.1", + "uuid": "^8.1.0", "wikimedia-commons-file-path": "^1.2.0" }, "devDependencies": { diff --git a/src/common/models/traderProfile.ts b/src/common/models/traderProfile.ts index 66f8e5b..b48760b 100644 --- a/src/common/models/traderProfile.ts +++ b/src/common/models/traderProfile.ts @@ -23,6 +23,7 @@ export interface TraderProfile { osm_id: string; osm_category: any; osm_modified: number; + import_date: number; licence: string; storeType: { gastronomie: boolean; diff --git a/src/common/repositories/traders.repository.ts b/src/common/repositories/traders.repository.ts index e1a2b8d..2610288 100644 --- a/src/common/repositories/traders.repository.ts +++ b/src/common/repositories/traders.repository.ts @@ -1,5 +1,6 @@ import { TraderEntity } from '../models/traderEntity'; import admin = require('firebase-admin'); +import { v4 as uuid } from 'uuid'; import { TraderProfile } from '../models/traderProfile'; const wikimedia = require("wikimedia-commons-file-path"); @@ -26,6 +27,19 @@ export async function upsert(app: admin.app.App = admin.app(), trader: Partial) { + + if (!trader.id) + trader.id = uuid(); + + await app.firestore() + .collection('OSM') + .doc(trader.id) + .set(trader, { merge:true }) + .catch((e) => { + console.log(e); + }); +} async function importWikimediaImage(fileName: string) { diff --git a/src/common/services/import/import.service.ts b/src/common/services/import/import.service.ts index 640b2a4..c0e26f1 100644 --- a/src/common/services/import/import.service.ts +++ b/src/common/services/import/import.service.ts @@ -1,6 +1,7 @@ import * as tradersRepo from '../../repositories/traders.repository'; import admin = require('firebase-admin'); import { TraderProfile } from '../../models/traderProfile'; +import moment = require('moment'); @@ -8,17 +9,17 @@ import { TraderProfile } from '../../models/traderProfile'; const WEIGHTS_OF_COMP_INDEX: any = { businessname: 10, - ownerFirstname: 10, - ownerLastname: 10, + ownerFirstname: 5, + ownerLastname: 5, postcode: 10, city: 10, street: 10, number: 10, - description: 10, - pickup: 5, - delivery: 5, + description: 15, + pickup: 2, + delivery: 2, openingTime: 20, - email: 25, + email: 50, telephone: 20, homepage: 20, confirmedLocation: 50, @@ -41,24 +42,27 @@ export async function importData(app: admin.app.App, source: string, options: an if (traders && traders.length > 0) { - for(const trader of traders) { + for(const trader of (traders as []).filter((t: TraderProfile) => t.storeType) ) { try { // console.log(trader); + (trader as any).import_date = moment(Date.now()).unix(); buildCompletenessIndex(trader); - // console.log(trader.completenessIndex); - // await traders.upsert(app, trader); + + + console.log(trader); + await tradersRepo.import_osm(app, trader); } catch(e) { - console.log('errow while importing', trader); + console.log('errow while importing: ' + (trader as any).businessname, e); } } const sortedTraders = (traders as []).sort((a: any, b: any) => a.completenessIndex - b.completenessIndex); sortedTraders.forEach((t:any) => { - // console.log(t.completenessIndex); + console.log(t.completenessIndex); // console.log(t); }); - await tradersRepo.upsert(app, sortedTraders.reverse()[0]); + // await tradersRepo.upsert(app, sortedTraders.reverse()[0]); console.log('items: ' + traders.length) } diff --git a/src/common/services/import/lokalwirkt/mapping.ts b/src/common/services/import/lokalwirkt/mapping.ts index aea914f..1d2abc9 100644 --- a/src/common/services/import/lokalwirkt/mapping.ts +++ b/src/common/services/import/lokalwirkt/mapping.ts @@ -30,7 +30,7 @@ export function mapToTrader(lokalwirktModel:any[]) { if (lokalwirktModel && lokalwirktModel.length > 0) { for(const lw of lokalwirktModel) { - const trader = { + const trader:Partial = { businessname: lw.name, description: lw.description, ownerFirstname: lw.firstname, @@ -39,6 +39,7 @@ export function mapToTrader(lokalwirktModel:any[]) { city: lw.locality, street: lw.address, telephone: lw.phone, + email: lw.email, homepage: lw.website, openingTime: getOpeningTime(lw["opening-time"]), osm_id: lw.osm_id, @@ -50,6 +51,9 @@ export function mapToTrader(lokalwirktModel:any[]) { status: TraderProfileStatus.IMPORTED }; + if (lw.housenumber) + trader.number = lw.housenumber; + if (trader.storeType) out.push(trader); } diff --git a/src/common/services/import/lokalwirkt/provider.ts b/src/common/services/import/lokalwirkt/provider.ts index 5022944..b39b1fb 100644 --- a/src/common/services/import/lokalwirkt/provider.ts +++ b/src/common/services/import/lokalwirkt/provider.ts @@ -38,8 +38,6 @@ export async function loadData(options: any) { for(const d of response.data.features as any[]) { let data:any; - console.log(d.properties.id); - if (cache[d.properties.id]) { data = cache[d.properties.id]; } else { @@ -94,21 +92,36 @@ async function fillAddresses(data: any) { if (data && data.data ) { - if (!data.data.housenumber && - data.osm_original?.features?.properties && - data.osm_original.features.properties['addr:housenumber']) { - - data.data.housenumber = data.osm_original.features.properties['addr:housenumber']; - + if (!data.data.housenumber && data.osm_original) { + + if (data.osm_original?.features && + data.osm_original?.features.length > 0 && + data.osm_original?.features[0].properties && + data.osm_original.features[0].properties['addr:housenumber']) { + data.data.housenumber = data.osm_original.features[0].properties['addr:housenumber']; + data.data.address = data.osm_original.features[0].properties['addr:street']; + } } - // if (!data.data.address || !data.data.postalcode || !data.data.locality) { - - // } - - // if (data.osm_original) { - // data.data.address = data.data.address = - // } + if (!data.data.postalcode && data.osm_original) { + + if (data.osm_original?.features && + data.osm_original?.features.length > 0 && + data.osm_original?.features[0].properties && + data.osm_original.features[0].properties['addr:postcode']) { + data.data.postalcode = data.osm_original.features[0].properties['addr:postcode']; + } + } + + if (!data.data.locality && data.osm_original) { + + if (data.osm_original?.features && + data.osm_original?.features.length > 0 && + data.osm_original?.features[0].properties && + data.osm_original.features[0].properties['addr:city']) { + data.data.locality = data.osm_original.features[0].properties['addr:city']; + } + } } } From 78b36a5b806ed5287b5be4d3d4bf488dd58eb622 Mon Sep 17 00:00:00 2001 From: TheCodemonkey <7ilya@gmx.de> Date: Fri, 3 Jul 2020 12:20:39 +0200 Subject: [PATCH 5/5] first draft of import as admin script created --- src/adminTasks/import.ts | 4 +- src/common/repositories/traders.repository.ts | 63 +++++++++++++++++-- src/common/services/import/import.service.ts | 42 +++++++++---- .../services/import/lokalwirkt/provider.ts | 56 +++++++++++------ 4 files changed, 127 insertions(+), 38 deletions(-) diff --git a/src/adminTasks/import.ts b/src/adminTasks/import.ts index 7589e37..70ad7c1 100644 --- a/src/adminTasks/import.ts +++ b/src/adminTasks/import.ts @@ -6,8 +6,8 @@ async function importData(app: admin.app.App) { console.log('start import...'); const options = { - region: 'unna', - file: '/tmp/lokalkauf/import.csv' + region: 'unna' + // , file: '/tmp/lokalkauf/import.csv' } await importService.importData(app, 'lokalwirkt', options); diff --git a/src/common/repositories/traders.repository.ts b/src/common/repositories/traders.repository.ts index 2610288..a283055 100644 --- a/src/common/repositories/traders.repository.ts +++ b/src/common/repositories/traders.repository.ts @@ -1,6 +1,6 @@ import { TraderEntity } from '../models/traderEntity'; import admin = require('firebase-admin'); -import { v4 as uuid } from 'uuid'; +// import { v4 as uuid } from 'uuid'; import { TraderProfile } from '../models/traderProfile'; const wikimedia = require("wikimedia-commons-file-path"); @@ -28,19 +28,70 @@ export async function upsert(app: admin.app.App = admin.app(), trader: Partial) { + if (!trader.osm_id) return; - if (!trader.id) - trader.id = uuid(); + const col = app.firestore().collection('Traders'); + + const items = await col.where('osm_id', '==', trader.osm_id) + .get(); + + if(items.docs && items.docs.length > 0) { + // process update + for(const d of items.docs){ + await col.doc(d.id) + .set(trader, { merge:true }) + .catch((e) => { + console.log(e); + }); + } + } + else + { + // process insert + + await col.doc() + .set(trader) + .catch((e) => { + console.log(e); + }); + } +} + +export async function import_osmcache_item(item: any, app: admin.app.App = admin.app()) { + if (!item || !item.lw_id) return; await app.firestore() - .collection('OSM') - .doc(trader.id) - .set(trader, { merge:true }) + .collection('OSM_CACHE') + .doc(item.lw_id + "") + .set(item, { merge:true }) .catch((e) => { console.log(e); }); } +export async function load_osmcache(app: admin.app.App = admin.app()) : Promise { + const traders = await app.firestore().collection('OSM_CACHE').get(); + + return traders.docs.map(d => { + const data = d.data(); + data.id = d.id; + return data; + }); +} + +export async function load_osmcache_item(id: string, app: admin.app.App = admin.app()) : Promise { + + let x = "0"; + x=x; + + const doc = await app.firestore() + .collection('OSM_CACHE') + .doc(id + "") + .get(); + + return (doc)? doc.data() : null; +} + async function importWikimediaImage(fileName: string) { const imgURL = wikimedia(fileName); diff --git a/src/common/services/import/import.service.ts b/src/common/services/import/import.service.ts index c0e26f1..ac815cb 100644 --- a/src/common/services/import/import.service.ts +++ b/src/common/services/import/import.service.ts @@ -1,10 +1,10 @@ import * as tradersRepo from '../../repositories/traders.repository'; import admin = require('firebase-admin'); -import { TraderProfile } from '../../models/traderProfile'; +import { TraderProfile, TraderProfileStatus } from '../../models/traderProfile'; import moment = require('moment'); - +let adminApp: admin.app.App; const WEIGHTS_OF_COMP_INDEX: any = { @@ -37,8 +37,11 @@ function getServiceProvider(source: string){ export async function importData(app: admin.app.App, source: string, options: any) { + if (!adminApp) + adminApp = app; + const provider = getServiceProvider(source); - const traders = await provider.loadData(options); + const traders = await provider.loadData(options, app); if (traders && traders.length > 0) { @@ -49,25 +52,42 @@ export async function importData(app: admin.app.App, source: string, options: an buildCompletenessIndex(trader); - console.log(trader); - await tradersRepo.import_osm(app, trader); + // console.log(trader); + // await tradersRepo.import_osm(app, trader); } catch(e) { console.log('errow while importing: ' + (trader as any).businessname, e); } } - const sortedTraders = (traders as []).sort((a: any, b: any) => a.completenessIndex - b.completenessIndex); - sortedTraders.forEach((t:any) => { - console.log(t.completenessIndex); - // console.log(t); - }); + const trToImport = (traders as []).filter((t: TraderProfile) => t.completenessIndex > 30 && !t.storeType.handwerk); + + console.log('all : ' + traders.length + ' \nimport : ' + trToImport.length); + + for(const ti of trToImport) { + (ti as TraderProfile).status = TraderProfileStatus.PUBLIC; + await tradersRepo.import_osm(app, ti); + } + + // const sortedTraders = (traders as []).sort((a: any, b: any) => a.completenessIndex - b.completenessIndex); + // sortedTraders.forEach((t:any) => { + // console.log(t.completenessIndex); + // // console.log(t); + // }); // await tradersRepo.upsert(app, sortedTraders.reverse()[0]); - console.log('items: ' + traders.length) + console.log('items: ' + trToImport.length) } } +export async function putItemToCache(item: any, app: admin.app.App) { + await tradersRepo.import_osmcache_item(item, adminApp); +} + +export async function getItemFromCache(itemID: any, app: admin.app.App) { + return await tradersRepo.load_osmcache_item(itemID, adminApp); +} + function buildCompletenessIndex(trader:TraderProfile) { let currentIndex = 0; diff --git a/src/common/services/import/lokalwirkt/provider.ts b/src/common/services/import/lokalwirkt/provider.ts index b39b1fb..e8246f3 100644 --- a/src/common/services/import/lokalwirkt/provider.ts +++ b/src/common/services/import/lokalwirkt/provider.ts @@ -1,10 +1,14 @@ const axios = require('axios'); const osmtogeojson = require('osmtogeojson'); -import * as fs from 'fs'; import { mapToTrader } from './mapping'; import * as categories from './categories'; +import * as importService from '../import.service'; +import admin = require('firebase-admin'); + +// import * as fs from 'fs'; +// const FILE_BUFFER = '/tmp/lokalkauf/lokalwirkt.json'; + -const FILE_BUFFER = '/tmp/lokalkauf/lokalwirkt.json'; export async function loadDetails(id: string) { let out:any; @@ -22,27 +26,25 @@ export async function loadDetails(id: string) { } -export async function loadData(options: any) { +export async function loadData(options: any, app: admin.app.App) { const response = await axios.get('https://lokalwirkt.de/api/stores/geo?region-slug=' + options.region); const items: any[] = []; - if (!fs.existsSync(FILE_BUFFER)) { - fs.writeFileSync(FILE_BUFFER, '{}', {encoding: 'utf8'}); - } - - const file_content = fs.readFileSync(FILE_BUFFER, {encoding:'utf8'}); - const cache = JSON.parse(file_content); - + // const cache = await loadCache(); if (response && response.data && response.data.features && response.data.features.length > 0) { for(const d of response.data.features as any[]) { let data:any; - if (cache[d.properties.id]) { - data = cache[d.properties.id]; + const cachedItem = await importService.getItemFromCache(d.properties.id, app); + + if (cachedItem) { + data = cachedItem; } else { - data = await loadDetails(d.properties.id); - await updateCache(d.properties.id, cache, data); + data = await loadDetails(d.properties.id); + data.lw_id = d.properties.id; + await importService.putItemToCache(data, app); + //await updateCache(d.properties.id, cache, data); } if (data) { @@ -50,7 +52,8 @@ export async function loadData(options: any) { // get original osm data... if ((!data.osm_original || data.osm_original.ERROR) && data.data.osm_id) { data.osm_original = await loadOriginalOSM_Data(data.data.osm_id, data.data); - await updateCache(d.properties.id, cache, data); + await importService.putItemToCache(data, app); + // await updateCache(d.properties.id, cache, data); } // fill addresses @@ -83,10 +86,25 @@ export async function loadData(options: any) { -async function updateCache(itemID:string, cache: any, data: any) { - cache[itemID] = data; - fs.writeFileSync(FILE_BUFFER, JSON.stringify(cache), {encoding:'utf8'}); -} +// async function updateCache(itemID:string, cache: any, data: any) { + + +// cache[itemID] = data; +// fs.writeFileSync(FILE_BUFFER, JSON.stringify(cache), {encoding:'utf8'}); +// } + +// async function loadCache() { + +// // return await tradersRepo.load_osmcache(); + +// if (!fs.existsSync(FILE_BUFFER)) { +// fs.mkdirSync('/tmp/lokalkauf'); +// fs.writeFileSync(FILE_BUFFER, '{}', {encoding: 'utf8'}); +// } + +// const file_content = fs.readFileSync(FILE_BUFFER, {encoding:'utf8'}); +// return JSON.parse(file_content); +// } async function fillAddresses(data: any) {