Update to version v4.20.0

.github/workflows/install_grpc.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-git clone --branch v1.44.0 https://github.com/grpc/grpc
+git clone --branch v1.54.3 https://github.com/grpc/grpc
 cd grpc
 git submodule update --init
 mkdir -p cmake/build

.github/workflows/test.yml

@@ -47,15 +47,10 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-13]
+        os: [ubuntu-22.04, ubuntu-24.04, macos-13]
         include:
-          - os: ubuntu-24.04
-            cc: gcc-12
-            cxx: g++-12
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
           - os: ubuntu-latest
             sanitizer: TSAN
             cc: gcc-12
@@ -66,6 +61,7 @@ jobs:
       OS: ${{matrix.os}}
       CC: ${{matrix.cc}}
       CXX: ${{matrix.cxx}}
+      SANITIZER: ${{matrix.sanitizer}}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -85,7 +81,11 @@ jobs:
           export CPLUS_INCLUDE_PATH=$GITHUB_WORKSPACE/grpc/third_party/abseil-cpp
           mkdir build && cd build
           if [[ $OS == ubuntu-latest ]]; then
-            cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            if [[ $SANITIZER == 'ASAN' ]]; then
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On -DGTEST_SKIP_SHARDING=On ..
+            else
+              cmake -DENABLE_GRPC=ON -DWITH_${{matrix.sanitizer}}=On ..
+            fi
           elif [[ $OS == macos* ]]; then
             cmake -DGH_CI_OSX=ON ..
           else
@@ -119,8 +119,6 @@ jobs:
         include:
           - os: ubuntu-latest
             sanitizer: ASAN
-            cc: gcc-10
-            cxx: g++-10
             test: 'C++'
           - os: ubuntu-latest
             sanitizer: ASAN

bindings/builtin/builtin.go

@@ -255,7 +255,7 @@ func (binding *Builtin) Init(u []url.URL, eh bindings.EventsHandler, options ...
 		caps: C.int64_t(caps.Value),
 	}
 
-	return err2go(C.reindexer_connect_v4(binding.rx, str2c(u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
+	return err2go(C.reindexer_connect_v4(binding.rx, str2c(u[0].Host+u[0].Path), opts, str2c(bindings.ReindexerVersion), ccaps))
 }
 
 func (binding *Builtin) StartWatchOnCtx(ctx context.Context) (CCtxWrapper, error) {

bindings/consts.go

@@ -2,7 +2,7 @@ package bindings
 
 const CInt32Max = int(^uint32(0) >> 1)
 
-const ReindexerVersion = "v4.19.0"
+const ReindexerVersion = "v4.20.0"
 
 // public go consts from type_consts.h and reindexer_ctypes.h
 const (

changelog.md

@@ -1,7 +1,32 @@
+# Version 4.20.0 (04.02.2025)
+## Core
+- [fea] Optimized indexed strings memory layout (each unique indexed string now requires 20-36 bytes less memery, depending on platform)
+- [fea] Optimized non-built btree-index selection
+- [fea] Optimized selections with large documents count in results
+- [fea] Reduced allocations count in inserts/upserts and select-queries
+- [fea] Changed index compatibility check in `AddIndex`: now `dense` and `regular` indexes are treated compatible
+- [fix] Fixed parsing for [EQUAL_POSITION](readme.md#search-in-array-fields-with-matching-array-indexes) with brackets in SQL
+
+## Replication
+- [fea] Improved full sync performance for namespaces with large amount of empty documents
+
+## Reindexer server
+- [fix] Added more TCP-requests format checks
+- [fix] Fixed [server.yml](cpp_src/server/contrib/server.yml) format errors
+
+## Go connector
+- [fea] Added support for absolute path on Windows platforms in `builtin`-binding
+
+## Build
+- [fix] Fixed build for ARM docker image
+
+## Ported
+- [fea/fix] Ported all the fixes and features from [v3.31.0](https://github.com/Restream/reindexer/releases/tag/v3.31.0)
+
 # Version 4.19.0 (17.12.2024)
 
 ## Core
-- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication` configs in `#config`-namespace)
+- [fea] Added automatic masking for reindexer user's credentials in log files and cluster/sharding JSON's (except for `async_replication`-config in `#config`-namespace)
 - [fix] Fixed assertion on attempt to use 'null'-values with `=`, `IN()`, `<`, `>`, `<=`, `>=` and `RANGE()` operators
 
 ## Replication

cjson/decoder.go

@@ -36,8 +36,7 @@ func fieldByTag(t reflect.Type, tag string) (result reflect.StructField, ok bool
 	}
 	for i := 0; i < t.NumField(); i++ {
 		result = t.Field(i)
-		if ftag := result.Tag.Get("json"); len(ftag) > 0 {
-			ftag, _ = splitStr(ftag, ',')
+		if ftag, _ := splitStr(result.Tag.Get("json"), ','); len(ftag) > 0 {
 			if tag == ftag {
 				return result, true
 			}

clang-tidy/run-clang-tidy-18.py → clang-tidy/run-clang-tidy-19.py

@@ -251,10 +251,10 @@ def main():
                       action='store_true', help='allow alpha checkers from '
                                                 'clang-analyzer.')
   parser.add_argument('-clang-tidy-binary', metavar='PATH',
-                      default='clang-tidy-18',
+                      default='clang-tidy-19',
                       help='path to clang-tidy binary')
   parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
-                      default='clang-apply-replacements-18',
+                      default='clang-apply-replacements-19',
                       help='path to clang-apply-replacements binary')
   parser.add_argument('-checks', default=None,
                       help='checks filter, when not specified, use clang-tidy '

cpp_src/CMakeLists.txt

@@ -50,7 +50,7 @@ else()
   option(LINK_RESOURCES "Link web resources as binary data" ON)
 endif()
 
-set (REINDEXER_VERSION_DEFAULT "4.19.0")
+set (REINDEXER_VERSION_DEFAULT "4.20.0")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
@@ -72,6 +72,13 @@ if(MSVC)
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -Zi")
   set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
   set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG -Zi")
+elseif(WITH_ASAN)
+  # Using O2 instead of O3 to build a bit faster.
+  # Also this allows to avoid SEGFAULT in libasan.so during coroutines interaction on CentOS7 (gcc-12).
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g1")
+  set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
+  set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
 else()
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g1")
   set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g1")

cpp_src/client/itemimplbase.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <deque>
 #include <vector>
 #include "core/keyvalue/variant.h"
 #include "core/payload/payloadiface.h"
@@ -76,7 +75,7 @@ class ItemImplBase {
 
 	std::vector<std::string> precepts_;
 	bool unsafe_ = false;
-	std::deque<std::string> holder_;
+	h_vector<key_string, 16> holder_;
 	std::vector<std::unique_ptr<char[]>> largeJSONStrings_;
 };
 

cpp_src/client/queryresults.h

@@ -37,9 +37,8 @@ class QueryResults {
 
 	class Iterator : public CoroQueryResults::Iterator {
 	public:
-		Iterator(const QueryResults* r, const CoroQueryResults* qr, int idx, int pos, int nextPos,
-				 ResultSerializer::ItemParams itemParams) noexcept
-			: CoroQueryResults::Iterator{qr, idx, pos, nextPos, itemParams, {}}, r_(r) {}
+		Iterator(const QueryResults& r, int idx, int pos, int nextPos, ResultSerializer::ItemParams itemParams) noexcept
+			: CoroQueryResults::Iterator{&r.results_, idx, pos, nextPos, itemParams, {}}, r_(&r) {}
 		Iterator& operator*() { return *this; }
 		Iterator& operator++() noexcept {
 			try {
@@ -57,11 +56,12 @@ class QueryResults {
 			}
 			return *this;
 		}
+
 		const QueryResults* r_;
 	};
 
-	Iterator begin() const noexcept { return Iterator{this, &results_, 0, 0, 0, {}}; }
-	Iterator end() const noexcept { return Iterator{this, &results_, int(results_.Count()), 0, 0, {}}; }
+	Iterator begin() const noexcept { return Iterator{*this, 0, 0, 0, {}}; }
+	Iterator end() const noexcept { return Iterator{*this, int(results_.Count()), 0, 0, {}}; }
 
 	size_t Count() const noexcept { return results_.Count(); }
 	int TotalCount() const noexcept { return results_.TotalCount(); }

cpp_src/cmd/reindexer_tool/CMakeLists.txt

@@ -22,7 +22,7 @@ if (NOT MSVC AND NOT WITH_STDLIB_DEBUG)
     ExternalProject_Add(
       replxx_lib
       GIT_REPOSITORY "https://github.com/Restream/replxx"
-      GIT_TAG "b50b7b7a8c2835b45607cffabc18e4742072e9e6"
+      GIT_TAG "98aa91965d7495e030f31c6f05969177fe5ab81d"
       CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}
     )
     include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)

cpp_src/core/cbinding/reindexer_c.cc

@@ -1,8 +1,7 @@
 #include "reindexer_c.h"
 
-#include <stdlib.h>
 #include <string.h>
-#include <locale>
+#include <clocale>
 #include <mutex>
 #include <shared_mutex>
 
@@ -18,7 +17,7 @@
 
 using namespace reindexer;
 constexpr int kQueryResultsPoolSize = 1024;
-constexpr int kMaxConcurentQueries = 65534;
+constexpr int kMaxConcurrentQueries = 65534;
 constexpr size_t kCtxArrSize = 1024;
 constexpr size_t kWarnLargeResultsLimit = 0x40000000;
 constexpr size_t kMaxPooledResultsCap = 0x10000;
@@ -86,7 +85,7 @@ struct TransactionWrapper {
 };
 
 static std::atomic<int> serializedResultsCount{0};
-static sync_pool<QueryResultsWrapper, kQueryResultsPoolSize, kMaxConcurentQueries> res_pool;
+static sync_pool<QueryResultsWrapper, kQueryResultsPoolSize, kMaxConcurrentQueries> res_pool;
 static CGOCtxPool ctx_pool(kCtxArrSize);
 
 struct put_results_to_pool {
@@ -154,7 +153,7 @@ static void results2c(std::unique_ptr<QueryResultsWrapper> result, struct reinde
 	}
 
 	out->results_ptr = uintptr_t(result.release());
-	if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurentQueries) {
+	if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurrentQueries) {
 		logPrintf(LogWarning, "Too many serialized results: count=%d, alloced=%d", count, res_pool.Alloced());
 	}
 }
@@ -186,7 +185,7 @@ reindexer_error reindexer_ping(uintptr_t rx) {
 	return error2c(db ? Error(errOK) : err_not_init);
 }
 
-static void procces_packed_item(Item& item, int /*mode*/, int state_token, reindexer_buffer data, int format, Error& err) {
+static void proccess_packed_item(Item& item, int /*mode*/, int state_token, reindexer_buffer data, int format, Error& err) {
 	if (item.Status().ok()) {
 		switch (format) {
 			case FormatJson:
@@ -212,7 +211,7 @@ static void procces_packed_item(Item& item, int /*mode*/, int state_token, reind
 
 reindexer_error reindexer_modify_item_packed_tx(uintptr_t rx, uintptr_t tr, reindexer_buffer args, reindexer_buffer data) {
 	auto db = reinterpret_cast<ReindexerWrapper*>(rx);
-	TransactionWrapper* trw = reinterpret_cast<TransactionWrapper*>(tr);
+	auto trw = reinterpret_cast<TransactionWrapper*>(tr);
 	if (!db) {
 		return error2c(err_not_init);
 	}
@@ -226,12 +225,12 @@ reindexer_error reindexer_modify_item_packed_tx(uintptr_t rx, uintptr_t tr, rein
 	int state_token = ser.GetVarUint();
 	Error err = err_not_init;
 	auto item = trw->tr_.NewItem();
-	procces_packed_item(item, mode, state_token, data, format, err);
+	proccess_packed_item(item, mode, state_token, data, format, err);
 	if (err.code() == errTagsMissmatch) {
 		item = db->rx.NewItem(trw->tr_.GetNsName());
 		err = item.Status();
 		if (err.ok()) {
-			procces_packed_item(item, mode, state_token, data, format, err);
+			proccess_packed_item(item, mode, state_token, data, format, err);
 		}
 	}
 	if (err.ok()) {
@@ -263,7 +262,7 @@ reindexer_ret reindexer_modify_item_packed(uintptr_t rx, reindexer_buffer args,
 
 			Item item = rdxKeeper.db().NewItem(ns);
 
-			procces_packed_item(item, mode, state_token, data, format, err);
+			proccess_packed_item(item, mode, state_token, data, format, err);
 
 			query_results_ptr res;
 			if (err.ok()) {
@@ -677,7 +676,7 @@ reindexer_ret reindexer_update_query(uintptr_t rx, reindexer_buffer in, reindexe
 
 reindexer_error reindexer_delete_query_tx(uintptr_t rx, uintptr_t tr, reindexer_buffer in) {
 	auto db = reinterpret_cast<ReindexerWrapper*>(rx);
-	TransactionWrapper* trw = reinterpret_cast<TransactionWrapper*>(tr);
+	auto trw = reinterpret_cast<TransactionWrapper*>(tr);
 	if (!db) {
 		return error2c(err_not_init);
 	}
@@ -698,7 +697,7 @@ reindexer_error reindexer_delete_query_tx(uintptr_t rx, uintptr_t tr, reindexer_
 
 reindexer_error reindexer_update_query_tx(uintptr_t rx, uintptr_t tr, reindexer_buffer in) {
 	auto db = reinterpret_cast<ReindexerWrapper*>(rx);
-	TransactionWrapper* trw = reinterpret_cast<TransactionWrapper*>(tr);
+	auto trw = reinterpret_cast<TransactionWrapper*>(tr);
 	if (!db) {
 		return error2c(err_not_init);
 	}
@@ -761,7 +760,7 @@ reindexer_ret reindexer_enum_meta(uintptr_t rx, reindexer_string ns, reindexer_c
 		out.len = ser.Len();
 		out.data = uintptr_t(ser.Buf());
 		out.results_ptr = uintptr_t(results.release());
-		if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurentQueries) {
+		if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurrentQueries) {
 			logPrintf(LogWarning, "Too many serialized results: count=%d, alloced=%d", count, res_pool.Alloced());
 		}
 	}
@@ -794,7 +793,7 @@ reindexer_ret reindexer_get_meta(uintptr_t rx, reindexer_string ns, reindexer_st
 		out.len = results->ser.Len();
 		out.data = uintptr_t(results->ser.Buf());
 		out.results_ptr = uintptr_t(results.release());
-		if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurentQueries) {
+		if (const auto count{serializedResultsCount.fetch_add(1, std::memory_order_relaxed)}; count > kMaxConcurrentQueries) {
 			logPrintf(LogWarning, "Too many serialized results: count=%d, alloced=%d", count, res_pool.Alloced());
 		}
 	}

cpp_src/core/cjson/cjsondecoder.cc

@@ -40,11 +40,12 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 				const auto& fieldRef{pl.Type().Field(field)};
 				const KeyValueType fieldType{fieldRef.Type()};
 				if (tagType == TAG_ARRAY) {
+					const carraytag atag = rdser.GetCArrayTag();
+					const auto count = atag.Count();
 					if rx_unlikely (!fieldRef.IsArray()) {
 						throwUnexpectedArrayError(fieldRef);
 					}
-					const carraytag atag = rdser.GetCArrayTag();
-					const auto count = atag.Count();
+					validateArrayFieldRestrictions(fieldRef, count, "cjson");
 					const int ofs = pl.ResizeArray(field, count, true);
 					const TagType atagType = atag.Type();
 					if (atagType != TAG_OBJECT) {
@@ -61,6 +62,7 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 					wrser.PutVarUint(count);
 				} else {
 					validateNonArrayFieldRestrictions(objectScalarIndexes_, pl, fieldRef, field, isInArray(), "cjson");
+					validateArrayFieldRestrictions(fieldRef, 1, "cjson");
 					objectScalarIndexes_.set(field);
 					pl.Set(field, cjsonValueToVariant(tagType, rdser, fieldType), true);
 					fieldType.EvaluateOneOf(
@@ -121,8 +123,8 @@ bool CJsonDecoder::decodeCJson(Payload& pl, Serializer& rdser, WrSerializer& wrs
 
 [[nodiscard]] Variant CJsonDecoder::cjsonValueToVariant(TagType tagType, Serializer& rdser, KeyValueType fieldType) {
 	if (fieldType.Is<KeyValueType::String>() && tagType != TagType::TAG_STRING) {
-		storage_.emplace_back(rdser.GetRawVariant(KeyValueType{tagType}).As<std::string>());
-		return Variant(p_string(&storage_.back()), Variant::no_hold_t{});
+		auto& back = storage_.emplace_back(rdser.GetRawVariant(KeyValueType{tagType}).As<key_string>());
+		return Variant(p_string(back), Variant::no_hold_t{});
 	} else {
 		return reindexer::cjsonValueToVariant(tagType, rdser, fieldType);
 	}

cpp_src/core/cjson/cjsondecoder.h

@@ -24,8 +24,9 @@ class Recoder {
 
 class CJsonDecoder {
 public:
-	explicit CJsonDecoder(TagsMatcher& tagsMatcher, std::deque<std::string>& storage) noexcept
-		: tagsMatcher_(tagsMatcher), storage_(storage) {}
+	using StrHolderT = h_vector<key_string, 16>;
+
+	explicit CJsonDecoder(TagsMatcher& tagsMatcher, StrHolderT& storage) noexcept : tagsMatcher_(tagsMatcher), storage_(storage) {}
 	class SkipFilter {
 	public:
 		SkipFilter MakeCleanCopy() const noexcept { return SkipFilter(); }
@@ -165,7 +166,7 @@ class CJsonDecoder {
 	int32_t arrayLevel_{0};
 	ScalarIndexesSetT objectScalarIndexes_;
 	// storage for owning strings obtained from numbers
-	std::deque<std::string>& storage_;
+	StrHolderT& storage_;
 };
 
 extern template bool CJsonDecoder::decodeCJson<CJsonDecoder::DummyFilter, CJsonDecoder::DummyRecoder, CJsonDecoder::NamelessTagOpt>(

cpp_src/core/cjson/cjsontools.cc

@@ -207,6 +207,11 @@ void throwScalarMultipleEncodesError(const Payload& pl, const PayloadFieldType&
 	throw Error(errLogic, "Non-array field '%s' [%d] from '%s' can only be encoded once.", f.Name(), field, pl.Type().Name());
 }
 
+void throwUnexpectedArraySizeError(std::string_view parserName, const PayloadFieldType& f, int arraySize) {
+	throw Error(errParams, "%s array field '%s' for this index type must contain %d elements, but got %d", parserName, f.Name(),
+				f.ArrayDim(), arraySize);
+}
+
 static void dumpCjsonValue(TagType type, Serializer& cjson, std::ostream& dump) {
 	switch (type) {
 		case TAG_VARINT: