From 7a556b28f4a452e76e87fc4d225e5cef9fde2d7c Mon Sep 17 00:00:00 2001 From: Anatoly Serdtcev Date: Sat, 28 Dec 2019 12:59:05 +0300 Subject: [PATCH] [generator:regions] Speedup locality index building: parallel regions stripping --- base/thread_pool_computational.hpp | 2 ++ generator/covering_index_generator.cpp | 20 +++++++---- generator/geometry_holder.hpp | 9 +++-- geometry/polygon.hpp | 47 +++++++++++++++++++++++--- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/base/thread_pool_computational.hpp b/base/thread_pool_computational.hpp index 5d0c665..1c99b32 100644 --- a/base/thread_pool_computational.hpp +++ b/base/thread_pool_computational.hpp @@ -125,6 +125,8 @@ class ThreadPool m_joiner.Join(); } + size_t Size() const noexcept { return m_threads.size(); } + private: void Worker() { diff --git a/generator/covering_index_generator.cpp b/generator/covering_index_generator.cpp index d8d5e63..bce7b52 100644 --- a/generator/covering_index_generator.cpp +++ b/generator/covering_index_generator.cpp @@ -23,7 +23,7 @@ #include "base/logging.hpp" #include "base/scope_guard.hpp" #include "base/string_utils.hpp" -#include "base/thread_utils.hpp" +#include "base/thread_pool_computational.hpp" #include "base/timer.hpp" #include "defines.hpp" @@ -43,7 +43,8 @@ namespace generator class CoveredObjectBuilder { public: - CoveredObjectBuilder() + CoveredObjectBuilder(base::thread_pool::computational::ThreadPool & threadPool) + : m_threadPool(threadPool) { m_header.SetGeometryCodingParams(serial::GeometryCodingParams()); m_header.SetScales({scales::GetUpperScale()}); @@ -125,7 +126,7 @@ class CoveredObjectBuilder if (points.size() > 2) { - if (!holder.TryToMakeStrip(points)) + if (!holder.TryToMakeStrip(points, &m_threadPool)) { m2::ConvexHull hull(points, 1e-16); vector hullPoints = hull.Points(); @@ -151,12 +152,14 @@ class CoveredObjectBuilder DataHeader m_header; indexer::CoveredObject m_coveredObject; buffer_vector m_pointsBuffer; + base::thread_pool::computational::ThreadPool & m_threadPool; }; template void CoverFeatures( std::string const & featuresFile, FeatureFilter && featureFilter, IndexBuilder && indexBuilder, unsigned int threadsCount, uint64_t chunkFeaturesCount, + base::thread_pool::computational::ThreadPool & threadPool, covering::ObjectsCovering & objectsCovering) { std::list coveringsParts{}; @@ -164,7 +167,7 @@ void CoverFeatures( coveringsParts.emplace_back(); auto & covering = coveringsParts.back(); - CoveredObjectBuilder localityObjectBuilder; + CoveredObjectBuilder localityObjectBuilder{threadPool}; auto processor = [featureFilter, &indexBuilder, &covering, localityObjectBuilder] (FeatureBuilder & fb, uint64_t /* currPos */) mutable { @@ -229,11 +232,13 @@ bool ParseNodes(string nodesFile, set & nodeIds) bool GenerateRegionsIndex(std::string const & outPath, std::string const & featuresFile, unsigned int threadsCount) { + base::thread_pool::computational::ThreadPool threadPool{threadsCount}; auto const featuresFilter = [](FeatureBuilder & fb) { return fb.IsArea(); }; indexer::RegionsIndexBuilder indexBuilder; + covering::ObjectsCovering objectsCovering; CoverFeatures(featuresFile, featuresFilter, indexBuilder, threadsCount, - 1 /* chunkFeaturesCount */, objectsCovering); + 1 /* chunkFeaturesCount */, threadPool, objectsCovering); LOG(LINFO, ("Build locality index...")); if (!indexBuilder.BuildCoveringIndex(std::move(objectsCovering), outPath)) @@ -248,6 +253,7 @@ bool GenerateGeoObjectsIndex( boost::optional const & nodesFile, boost::optional const & streetsFeaturesFile) { + base::thread_pool::computational::ThreadPool threadPool{threadsCount}; covering::ObjectsCovering objectsCovering; indexer::GeoObjectsIndexBuilder indexBuilder; @@ -268,7 +274,7 @@ bool GenerateGeoObjectsIndex( }; CoverFeatures(geoObjectsFeaturesFile, geoObjectsFilter, indexBuilder, threadsCount, - 10 /* chunkFeaturesCount */, objectsCovering); + 10 /* chunkFeaturesCount */, threadPool, objectsCovering); if (streetsFeaturesFile) { @@ -278,7 +284,7 @@ bool GenerateGeoObjectsIndex( }; CoverFeatures(*streetsFeaturesFile, streetsFilter, indexBuilder, threadsCount, - 1 /* chunkFeaturesCount */, objectsCovering); + 1 /* chunkFeaturesCount */, threadPool, objectsCovering); } LOG(LINFO, ("Build objects index...")); diff --git a/generator/geometry_holder.hpp b/generator/geometry_holder.hpp index 8b4fa60..9af20f5 100644 --- a/generator/geometry_holder.hpp +++ b/generator/geometry_holder.hpp @@ -11,6 +11,8 @@ #include "indexer/data_header.hpp" +#include "base/thread_pool_computational.hpp" + #include #include #include @@ -78,7 +80,8 @@ class GeometryHolder bool NeedProcessTriangles() const { return !m_trgInner || m_buffer.m_innerTrg.empty(); } - bool TryToMakeStrip(Points & points) + bool TryToMakeStrip(Points & points, + base::thread_pool::computational::ThreadPool * threadPool = nullptr) { size_t const count = points.size(); if (!m_trgInner || (count >= 2 && count - 2 > m_maxNumTriangles)) @@ -110,7 +113,8 @@ class GeometryHolder } size_t const index = FindSingleStrip( - count, IsDiagonalVisibleFunctor(points.begin(), points.end())); + count, IsDiagonalVisibleFunctor(points.begin(), points.end()), + threadPool); if (index == count) { @@ -125,7 +129,6 @@ class GeometryHolder return true; } - private: class StripEmitter { diff --git a/geometry/polygon.hpp b/geometry/polygon.hpp index c042cea..5a84a9f 100644 --- a/geometry/polygon.hpp +++ b/geometry/polygon.hpp @@ -7,6 +7,7 @@ #include "base/base.hpp" #include "base/math.hpp" #include "base/stl_helpers.hpp" +#include "base/thread_pool_computational.hpp" #include #include @@ -35,15 +36,51 @@ bool FindSingleStripForIndex(size_t i, size_t n, IsVisibleF isVisible) // If polygon with n vertices is a single strip, return the start index of the strip or n otherwise. template -size_t FindSingleStrip(size_t n, IsVisibleF isVisible) +size_t FindSingleStrip(size_t n, IsVisibleF isVisible, + base::thread_pool::computational::ThreadPool * threadPool = nullptr) { - for (size_t i = 0; i < n; ++i) + // Try to find instantly. + if (FindSingleStripForIndex(0, n, isVisible)) + return 0; + + size_t const & tasksCount = n * n * n / 100'000'000; + if (!threadPool || tasksCount <= 1) { - if (FindSingleStripForIndex(i, n, isVisible)) - return i; + for (size_t i = 1; i < n; ++i) + { + if (FindSingleStripForIndex(i, n, isVisible)) + return i; + } + return n; } - return n; + std::atomic_size_t foundIndex{0}; + std::atomic_size_t unprocessedIndex{1}; + auto find = [n, &isVisible, &foundIndex, &unprocessedIndex]() { + while (foundIndex) + { + auto const i = unprocessedIndex++; + if (i >= n) + return; + + if (FindSingleStripForIndex(i, n, isVisible)) + foundIndex = i; + } + }; + + size_t const tasksPerThread = 10; + size_t const threadsCount = + std::min(std::max(size_t{2}, tasksCount / tasksPerThread), threadPool->Size()); + + std::vector> threadTasks{}; + threadTasks.reserve(threadsCount); + for (size_t i = 0; i < threadsCount; ++i) + threadTasks.push_back(threadPool->Submit(find)); + + for (auto & task : threadTasks) + task.wait(); + + return foundIndex ? foundIndex.load() : n; } #ifdef DEBUG