[generator] Process locality object on memory for interval index buil…

…ding.
mapsme · Dec 19, 2019 · 1fd232e · 1fd232e
1 parent be7a96f
commit 1fd232e
Show file tree

Hide file tree

Showing 15 changed files with 533 additions and 587 deletions.
diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt
@@ -58,8 +58,8 @@ set(
   key_value_concurrent_writer.hpp
   key_value_storage.cpp
   key_value_storage.hpp
-  locality_sorter.cpp
-  locality_sorter.hpp
+  locality_index_generator.cpp
+  locality_index_generator.hpp
   osm2meta.cpp
   osm2meta.hpp
   osm2type.cpp

diff --git a/generator/generator_tests/streets_index_tests.cpp b/generator/generator_tests/streets_index_tests.cpp
@@ -3,10 +3,9 @@
 
 #include "generator/data_version.hpp"
 #include "generator/geo_objects/geo_objects.hpp"
-#include "generator/locality_sorter.hpp"
+#include "generator/locality_index_generator.hpp"
 
 #include "indexer/locality_index.hpp"
-#include "indexer/locality_index_builder.hpp"
 
 #include "base/assert.hpp"
 
@@ -26,17 +25,12 @@ GeoObjectsIndex<IndexReader> GenerateStreetsIndex(std::vector<OsmElementData> co
   ScopedFile const streetsFeatures{"streets"s + DATA_FILE_EXTENSION, ScopedFile::Mode::DoNotCreate};
   WriteFeatures(osmElements, streetsFeatures);
 
-  auto const locDataFile = GetFileName("streets"s + LOC_DATA_FILE_EXTENSION);
-  bool locDataGeneration =
-      feature::GenerateGeoObjectsData(locDataFile, geoObjectsFeatures.GetFullPath(),
-                                      boost::none /* nodesFile */, streetsFeatures.GetFullPath());
-  CHECK(locDataGeneration, ());
-
   ScopedFile const streetsIndex{"streets"s + LOC_IDX_FILE_EXTENSION, ScopedFile::Mode::DoNotCreate};
-  auto streetsIndexBuilding =
-      BuildGeoObjectsIndexFromDataFile(locDataFile, streetsIndex.GetFullPath(), {},
-                                       DataVersion::kFileTag);
-  CHECK(streetsIndexBuilding, ());
+  bool streetsIndexGeneration =
+      GenerateGeoObjectsIndex(streetsIndex.GetFullPath(), geoObjectsFeatures.GetFullPath(),
+                              1 /* threadsCount */, {} /* nodesFile */,
+                              streetsFeatures.GetFullPath());
+  CHECK(streetsIndexGeneration, ());
 
   return ReadIndex<GeoObjectsIndexBox<IndexReader>, MmapReader>(streetsIndex.GetFullPath());
 }

diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp
@@ -1,7 +1,7 @@
 #include "generator/data_version.hpp"
 #include "generator/generate_info.hpp"
 #include "generator/geo_objects/geo_objects_generator.hpp"
-#include "generator/locality_sorter.hpp"
+#include "generator/locality_index_generator.hpp"
 #include "generator/osm_source.hpp"
 #include "generator/processor_factory.hpp"
 #include "generator/raw_generator.hpp"
@@ -294,29 +294,21 @@ int GeneratorToolMain(int argc, char ** argv)
       return EXIT_FAILURE;
     }
 
-    auto const locDataFile =
-        base::FilenameWithoutExt(options.m_geo_objects_index) + LOC_DATA_FILE_EXTENSION;
-
     auto const nodesListPath =
         boost::make_optional(!options.m_nodes_list_path.empty(), options.m_nodes_list_path);
     auto const streetsFeaturesPath =
         boost::make_optional(!options.m_streets_features.empty(), options.m_streets_features);
-    if (!feature::GenerateGeoObjectsData(locDataFile, options.m_geo_objects_features,
-                                         nodesListPath, streetsFeaturesPath))
-    {
-      LOG(LCRITICAL, ("Error generating geo objects data."));
-      return EXIT_FAILURE;
-    }
 
     LOG(LINFO, ("Saving geo objects index to", options.m_geo_objects_index));
-    if (!indexer::BuildGeoObjectsIndexFromDataFile(
-            locDataFile, options.m_geo_objects_index,
-            DataVersion::LoadFromPath(path).GetVersionJson(),
-            DataVersion::kFileTag))
+    if (!GenerateGeoObjectsIndex(options.m_geo_objects_index, options.m_geo_objects_features,
+                                 genInfo.m_threadsCount, nodesListPath, streetsFeaturesPath))
     {
       LOG(LCRITICAL, ("Error generating geo objects index."));
       return EXIT_FAILURE;
     }
+
+    WriteDataVersionSection(options.m_geo_objects_index,
+                            DataVersion::LoadFromPath(genInfo.m_dataPath).GetVersionJson());
   }
 
   if (options.m_generate_regions)
@@ -327,29 +319,23 @@ int GeneratorToolMain(int argc, char ** argv)
       return EXIT_FAILURE;
     }
 
-    auto const locDataFile =
-        base::FilenameWithoutExt(options.m_regions_index) + LOC_DATA_FILE_EXTENSION;
-
-    if (!feature::GenerateRegionsData(locDataFile, options.m_regions_features))
-    {
-      LOG(LCRITICAL, ("Error generating regions data."));
-      return EXIT_FAILURE;
-    }
-
     LOG(LINFO, ("Saving regions index to", options.m_regions_index));
-
-    if (!indexer::BuildRegionsIndexFromDataFile(locDataFile, options.m_regions_index,
-                                                DataVersion::LoadFromPath(path).GetVersionJson(),
-                                                DataVersion::kFileTag))
+    if (!GenerateRegionsIndex(options.m_regions_index, options.m_regions_features,
+                              genInfo.m_threadsCount))
     {
       LOG(LCRITICAL, ("Error generating regions index."));
       return EXIT_FAILURE;
     }
-    if (!feature::GenerateBorders(options.m_regions_index, options.m_regions_features))
+
+    LOG(LINFO, ("Saving regions borders to", options.m_regions_index));
+    if (!GenerateBorders(options.m_regions_index, options.m_regions_features))
     {
       LOG(LCRITICAL, ("Error generating regions borders."));
       return EXIT_FAILURE;
     }
+
+    WriteDataVersionSection(options.m_regions_index,
+                            DataVersion::LoadFromPath(genInfo.m_dataPath).GetVersionJson());
   }
 
   if (options.m_generate_regions_kv)

diff --git a/generator/geo_objects/geo_objects.cpp b/generator/geo_objects/geo_objects.cpp
@@ -3,7 +3,7 @@
 #include "generator/feature_generator.hpp"
 #include "generator/key_value_concurrent_writer.hpp"
 #include "generator/key_value_storage.hpp"
-#include "generator/locality_sorter.hpp"
+#include "generator/locality_index_generator.hpp"
 
 #include "generator/geo_objects/geo_objects.hpp"
 #include "generator/geo_objects/geo_objects_filter.hpp"
@@ -13,7 +13,6 @@
 
 #include "indexer/classificator.hpp"
 #include "indexer/locality_index.hpp"
-#include "indexer/locality_index_builder.hpp"
 
 #include "coding/mmap_reader.hpp"
 
@@ -372,17 +371,9 @@ bool JsonHasBuilding(JsonValue const & json)
 boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
     std::string const & pathToGeoObjectsTmpMwm)
 {
-  auto const dataFile = GetPlatform().TmpPathForFile();
-  SCOPE_GUARD(removeDataFile, std::bind(Platform::RemoveFileIfExists, std::cref(dataFile)));
-  if (!GenerateGeoObjectsData(dataFile, pathToGeoObjectsTmpMwm))
-  {
-    LOG(LCRITICAL, ("Error generating geo objects data."));
-    return {};
-  }
-
   auto const indexFile = GetPlatform().TmpPathForFile();
   SCOPE_GUARD(removeIndexFile, std::bind(Platform::RemoveFileIfExists, std::cref(indexFile)));
-  if (!indexer::BuildGeoObjectsIndexFromDataFile(dataFile, indexFile, std::string(), DataVersion::kFileTag))
+  if (!GenerateGeoObjectsIndex(indexFile, pathToGeoObjectsTmpMwm, 1))
   {
     LOG(LCRITICAL, ("Error generating geo objects index."));
     return {};