From 6c89c25b7fc6bce9f31465328a0931e543a93143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20J=C3=A4ger?= Date: Mon, 19 Aug 2024 13:13:58 +0000 Subject: [PATCH 1/4] use json files to delete conditions --- .devcontainer/vcpkg.json | 1 + plugins/navteq/converter/Converter.hpp | 3 +- plugins/navteq/converter/StreetConverter.cpp | 52 ++++++++++++++++++++ plugins/navteq/converter/StreetConverter.hpp | 3 ++ plugins/navteq/navteq_plugin.cpp | 10 ++-- 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.devcontainer/vcpkg.json b/.devcontainer/vcpkg.json index de5ebea..1db8240 100644 --- a/.devcontainer/vcpkg.json +++ b/.devcontainer/vcpkg.json @@ -1,5 +1,6 @@ { "dependencies": [ + "boost-json", "boost-log", "boost-program-options", "concurrencpp", diff --git a/plugins/navteq/converter/Converter.hpp b/plugins/navteq/converter/Converter.hpp index 53d3a43..8f718ea 100644 --- a/plugins/navteq/converter/Converter.hpp +++ b/plugins/navteq/converter/Converter.hpp @@ -204,9 +204,10 @@ class Converter { // mutex to protect osmium writer static std::mutex osmiumWriterMutex; + std::filesystem::path executable_path; + private: static std::map lang_code_map; - std::filesystem::path executable_path; }; #endif // CONVERTER_HPP diff --git a/plugins/navteq/converter/StreetConverter.cpp b/plugins/navteq/converter/StreetConverter.cpp index 0cf9ab1..259e7fe 100644 --- a/plugins/navteq/converter/StreetConverter.cpp +++ b/plugins/navteq/converter/StreetConverter.cpp @@ -16,7 +16,9 @@ #include "StreetConverter.hpp" +#include #include +#include #include #include #include @@ -46,6 +48,10 @@ void StreetConverter::convert(const std::filesystem::path &dir, init_cnd_mod(cdms_map, dir); init_cnd_dt_mod(cdms_map, dir); + // patch the conditions with a additional json file + // this is a workaround for the informations that are not fixed by HERE yet + manipulate_cdms_map(cdms_map, dir); + BOOST_LOG_TRIVIAL(info) << " processing country reference"; auto cntry_ref_map = init_g_cntry_ref_map(dir); auto area_to_govt_code_map = init_g_area_to_govt_code_map(cntry_ref_map, dir); @@ -73,6 +79,52 @@ void StreetConverter::convert(const std::filesystem::path &dir, process_way(dir, data, way_end_points_map, z_level_map, writer); } +void StreetConverter::manipulate_cdms_map( + std::multimap &cdms_map, + const std::filesystem::path &dir) { + + std::string filePrefix = dir.parent_path().filename(); + + for (auto const &dir_entry : + std::filesystem::directory_iterator{executable_path}) { + if (dir_entry.is_directory()) + continue; + + if (dir_entry.path().extension() != ".json") + continue; + + if (dir_entry.path().filename().string().find(filePrefix) == + std::string::npos) + continue; + + BOOST_LOG_TRIVIAL(info) << " processing json file: " << dir_entry.path(); + + std::ifstream ifs(dir_entry.path()); + auto jv = boost::json::parse(ifs); + + auto jsonPatch = jv.get_object(); + if (jsonPatch["cdms"].is_null()) + return; + + // check if all Ids are in the cdms_map + auto cdmsJson = jsonPatch["cdms"].get_object(); + + auto condToDelete = + boost::json::value_to>(cdmsJson["delete"]); + + for (auto const &condId : condToDelete) { + auto it = cdms_map.find(condId); + if (it == cdms_map.end()) { + BOOST_LOG_TRIVIAL(error) + << "Patch Failed: Conditional modification with id " << condId + << " not found in database! See " << dir_entry.path(); + exit(1); + } + cdms_map.erase(it); + } + } +} + std::map StreetConverter::process_alt_steets_route_types( const std::filesystem::path &dir) { diff --git a/plugins/navteq/converter/StreetConverter.hpp b/plugins/navteq/converter/StreetConverter.hpp index 5e63994..60676b1 100644 --- a/plugins/navteq/converter/StreetConverter.hpp +++ b/plugins/navteq/converter/StreetConverter.hpp @@ -350,6 +350,9 @@ class StreetConverter : public Converter { const std::string &restriction, int direction, bool is_imperial_units, uint64_t max_value); + void manipulate_cdms_map(std::multimap &cdms_map, + const std::filesystem::path &dir); + // CndMod types (CM) static constexpr std::string_view CM_MOD_TYPE = "MOD_TYPE"; static constexpr std::string_view CM_MOD_VAL = "MOD_VAL"; diff --git a/plugins/navteq/navteq_plugin.cpp b/plugins/navteq/navteq_plugin.cpp index 7a3644a..b3e55df 100644 --- a/plugins/navteq/navteq_plugin.cpp +++ b/plugins/navteq/navteq_plugin.cpp @@ -215,11 +215,11 @@ void navteq_plugin::execute() { BOOST_LOG_TRIVIAL(debug) << "Start converting WaterConverter " << dir; WaterConverter(executable_path).convert(dir, writer); })); - // results.emplace_back(executor->submit([this, &dir, &writer]() { - // BOOST_LOG_TRIVIAL(debug) - // << "Start converting HouseNumberConverter " << dir; - // HouseNumberConverter(executable_path).convert(dir, writer); - // })); + results.emplace_back(executor->submit([this, &dir, &writer]() { + BOOST_LOG_TRIVIAL(debug) + << "Start converting HouseNumberConverter " << dir; + HouseNumberConverter(executable_path).convert(dir, writer); + })); } auto allDone = From ec950e48bd05f71bf12a1ccebd1af0e9a4155c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20J=C3=A4ger?= Date: Mon, 19 Aug 2024 13:14:04 +0000 Subject: [PATCH 2/4] chore: Update CMakeLists.txt to include Boost JSON dependency --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 276fff8..40a3790 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ ENDIF() set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) -find_package(Boost REQUIRED COMPONENTS system log program_options) +find_package(Boost REQUIRED COMPONENTS system log program_options json) find_package(GDAL REQUIRED) find_package(geos REQUIRED) find_path(OSMIUM_INCLUDE_DIRS "osmium/version.hpp") @@ -46,7 +46,7 @@ include_directories(${GDAL_INCLUDE_DIRS} ${GEOS_INCLUDE_DIR} ${OSMIUM_INCLUDE_DI add_executable(${PROJECT_NAME} ${MAIN_CPP_FILES} ${CPP_FILES}) target_link_libraries(${PROJECT_NAME} PRIVATE ${ICU_LIBRARY} - PRIVATE Boost::boost Boost::log Boost::program_options + PRIVATE Boost::boost Boost::log Boost::program_options Boost::json PRIVATE GEOS::geos PRIVATE GDAL::GDAL PRIVATE expat bz2 From 45b791ad21777765163e549398fd1d7161729231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20J=C3=A4ger?= Date: Mon, 19 Aug 2024 13:33:07 +0000 Subject: [PATCH 3/4] refactor: Use std::ranges::find_if for deleting conditions by ID --- plugins/navteq/converter/StreetConverter.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/navteq/converter/StreetConverter.cpp b/plugins/navteq/converter/StreetConverter.cpp index 259e7fe..6e3b60f 100644 --- a/plugins/navteq/converter/StreetConverter.cpp +++ b/plugins/navteq/converter/StreetConverter.cpp @@ -113,7 +113,9 @@ void StreetConverter::manipulate_cdms_map( boost::json::value_to>(cdmsJson["delete"]); for (auto const &condId : condToDelete) { - auto it = cdms_map.find(condId); + auto it = std::ranges::find_if(cdms_map, [condId](auto const &entry) { + return entry.second.cond_id_type == condId; + }); if (it == cdms_map.end()) { BOOST_LOG_TRIVIAL(error) << "Patch Failed: Conditional modification with id " << condId From 0ebaf2e09747458d1b7a120129a09b27e0c8bc5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sven=20J=C3=A4ger?= Date: Tue, 20 Aug 2024 07:38:52 +0000 Subject: [PATCH 4/4] sort osm objects by id and type --- plugins/navteq/navteq_plugin.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/navteq/navteq_plugin.cpp b/plugins/navteq/navteq_plugin.cpp index b3e55df..c5c0df8 100644 --- a/plugins/navteq/navteq_plugin.cpp +++ b/plugins/navteq/navteq_plugin.cpp @@ -16,6 +16,9 @@ #include #include #include +#include +#include +#include #include #include "converter/AdminBoundariesConverter.hpp" @@ -253,9 +256,11 @@ void navteq_plugin::sortPBF() { osmium::io::File tmpfile( output_path.parent_path().append("tmp.pbf").string()); - copyType(writer, tmpfile, osmium::osm_entity_bits::node); - copyType(writer, tmpfile, osmium::osm_entity_bits::way); - copyType(writer, tmpfile, osmium::osm_entity_bits::relation); + for (const auto entity : + {osmium::osm_entity_bits::node, osmium::osm_entity_bits::way, + osmium::osm_entity_bits::relation}) { + copyType(writer, tmpfile, entity); + } writer.close(); @@ -264,11 +269,20 @@ void navteq_plugin::sortPBF() { void navteq_plugin::copyType(osmium::io::Writer &writer, osmium::io::File &file, osmium::osm_entity_bits::type bits) { - osmium::io::Reader reader(file, bits); + std::vector data; + osmium::ObjectPointerCollection objects; + + osmium::io::Reader reader{file, bits}; while (osmium::memory::Buffer buffer = reader.read()) { - writer(std::move(buffer)); + osmium::apply(buffer, objects); + data.push_back(std::move(buffer)); } reader.close(); + + objects.sort(osmium::object_order_type_id_version()); + + auto out = osmium::io::make_output_iterator(writer); + std::copy(objects.begin(), objects.end(), out); } void navteq_plugin::setBoundingBox(double minX, double minY, double maxX,