From 6653b43360e0819906dc9272797948abaf22b18c Mon Sep 17 00:00:00 2001 From: Jusufadis Bakamovic Date: Tue, 5 May 2020 22:39:10 +0200 Subject: [PATCH] Bug#30050452: CRASH WHEN EXECUTING "SELECT SLEEP(10)" THROUGH CURSOR (WITH THREAD-POOL) Problem: * TempTable segfaults in certain scenarios when ThreadPool (TP) plugin is used. * Cursor-protocol only accentuates the crash condition more. Analysis: * TempTable implements two of its critical parts through the means of thread-local variables, namely its data-backend (storage for tables) and custom memory allocator. * This becomes a design and functional issue as soon as MySQL server is run with the ThreadPool (TP) plugin because presence of that plugin will change the traditional thread-handling MySQL model, which is 1-thread-per-client-connection, to the alternative model where this assumption no longer holds and code relying on such assumption automatically becomes broken. * In particular, TP plugin will not be allocating a new thread for each new connection arriving but instead it will pick a thread from pre-allocated pool of threads and assign one to the statement execution engine. * In practice, this means that it is possible that statements within one client connection are now served with a multitude of different threads (but not simultaneously), which with traditional thread-handling model, such condition was not possible (it was always the same thread). * Furthermore, cursor-protocol makes it possible that even an execution of a single statement within one client connection is served by different threads. * E.g. Cursor-protocol enables fetching the result set of a query (data) in chunks. And TP plugin makes a situation possible where each chunk is fetched from a different thread. * TL;DR thread-local-storage variables cannot really co-exist with TP plugin in their natural form. Solution: * Replace TempTable parts which are implemented in terms of thread-local-storage variables with a solution which: * Is functionally correct in both cases, with and without TP plugin running. * Does not sacrifice the overall performance at any rate. * Does not lock the design but keeps the door open for future advancements (e.g. more control over memory consumption). * Basic idea is to: * Organize data-backend (table-storage) into shards * Ditto for shared-blocks (custom memory allocator external state) * Use unique connection identifier to map between connections and their corresponding shards Addendum: * This patch also fixes some other issues which have been reported but which are directly or indirectly caused by the same limitation in design: * BUG #30050420 CRASHES WHEN CLOSING CONNECTION WITH UNCLOSED CURSOR (WITH THREAD-POOL) * BUG #31116036 TEMPTABLE WASTES 1MB FOR EACH CONNECTION IN THREAD CACHE * BUG #31471863 TEMPTABLE ENGINE USES MUCH MORE MEMORY THAN MEMORY ENGINE FOR TEMPORARY TABLE Reviewed-by: Pawel Olchawa RB: 24497 --- include/my_compiler.h | 24 ++ sql/handler.cc | 2 + .../temptable/include/temptable/allocator.h | 40 +- .../temptable/include/temptable/constants.h | 20 +- storage/temptable/include/temptable/handler.h | 25 +- .../temptable/include/temptable/kv_store.h | 137 +++++++ .../include/temptable/kv_store_logger.h | 98 +++++ .../include/temptable/kv_store_stats.h | 51 +++ .../include/temptable/lock_free_pool.h | 120 ++++++ .../include/temptable/lock_free_type.h | 283 ++++++++++++++ .../include/temptable/sharded_kv_store.h | 103 +++++ .../temptable/sharded_kv_store_logger.h | 104 ++++++ .../include/temptable/shared_block_pool.h | 146 ++++++++ storage/temptable/include/temptable/table.h | 5 +- storage/temptable/src/allocator.cc | 19 +- storage/temptable/src/handler.cc | 135 +++---- storage/temptable/src/plugin.cc | 8 + storage/temptable/src/table.cc | 6 +- unittest/gunit/temptable/allocator-t.cc | 16 +- .../gunit/temptable/temptable-handler-t.cc | 19 +- unittest/gunit/temptable_allocator-t.cc | 353 +++++++++--------- unittest/gunit/temptable_storage-t.cc | 13 +- 22 files changed, 1408 insertions(+), 319 deletions(-) create mode 100644 storage/temptable/include/temptable/kv_store.h create mode 100644 storage/temptable/include/temptable/kv_store_logger.h create mode 100644 storage/temptable/include/temptable/kv_store_stats.h create mode 100644 storage/temptable/include/temptable/lock_free_pool.h create mode 100644 storage/temptable/include/temptable/lock_free_type.h create mode 100644 storage/temptable/include/temptable/sharded_kv_store.h create mode 100644 storage/temptable/include/temptable/sharded_kv_store_logger.h create mode 100644 storage/temptable/include/temptable/shared_block_pool.h diff --git a/include/my_compiler.h b/include/my_compiler.h index 6728280263d8..54f62eca5141 100644 --- a/include/my_compiler.h +++ b/include/my_compiler.h @@ -342,4 +342,28 @@ inline bool unlikely(bool expr) { return expr; } #define MY_COMPILER_CLANG_WORKAROUND_REF_DOCBUG() \ MY_COMPILER_CLANG_DIAGNOSTIC_IGNORE("-Wdocumentation") +/** + * ignore -Wunused-variable compiler warnings for \@see \@ref + * + * @code + * MY_COMPILER_DIAGNOSTIC_PUSH() + * MY_COMPILER_CLANG_WORKAROUND_FALSE_POSITIVE_UNUSED_VARIABLE_WARNING() + * ... + * MY_COMPILER_DIAGNOSTIC_POP() + * @endcode + * + * @see MY_COMPILER_DIAGNOSTIC_PUSH() + * @see MY_COMPILER_DIAGNOSTIC_POP() + * + * allows to work around false positives -Wunused-variable warnings like: + * + * - \@sa \@ref + * - \@see \@ref + * - \@return \@ref + * https://bugs.llvm.org/show_bug.cgi?id=46035 + * + */ +#define MY_COMPILER_CLANG_WORKAROUND_FALSE_POSITIVE_UNUSED_VARIABLE_WARNING() \ + MY_COMPILER_CLANG_DIAGNOSTIC_IGNORE("-Wunused-variable") + #endif /* MY_COMPILER_INCLUDED */ diff --git a/sql/handler.cc b/sql/handler.cc index 799f94038d45..6f9afb5d88c4 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -237,12 +237,14 @@ size_t num_hton2plugins() { return se_plugin_array.size(); } st_plugin_int *insert_hton2plugin(uint slot, st_plugin_int *plugin) { if (se_plugin_array.assign_at(slot, plugin)) return nullptr; + builtin_htons.assign_at(slot, true); return se_plugin_array[slot]; } st_plugin_int *remove_hton2plugin(uint slot) { st_plugin_int *retval = se_plugin_array[slot]; se_plugin_array[slot] = NULL; + builtin_htons.assign_at(slot, false); return retval; } diff --git a/storage/temptable/include/temptable/allocator.h b/storage/temptable/include/temptable/allocator.h index 9b42d14622eb..819ac43d5e32 100644 --- a/storage/temptable/include/temptable/allocator.h +++ b/storage/temptable/include/temptable/allocator.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All Rights Reserved. +/* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the @@ -43,9 +43,6 @@ TempTable custom allocator. */ namespace temptable { -/** Block that is shared between different tables within a given OS thread. */ -extern thread_local Block shared_block; - /* Thin abstraction which enables logging of memory operations. * * Used by the Allocator to implement switching from RAM to MMAP-backed @@ -214,7 +211,7 @@ class Allocator : private MemoryMonitor { }; /** Constructor. */ - Allocator(); + explicit Allocator(Block *shared_block); /** Constructor from allocator of another type. The state is copied into the * new object. */ @@ -293,23 +290,31 @@ class Allocator : private MemoryMonitor { See AllocatorState for details. */ std::shared_ptr m_state; + + /** A block of memory which is a state external to this allocator and can be + * shared among different instances of the allocator (not simultaneously). In + * order to speed up its operations, allocator may decide to consume the + * memory of this shared block. + */ + Block *m_shared_block; }; /* Implementation of inlined methods. */ template -inline Allocator::Allocator() - : m_state(std::make_shared()) {} +inline Allocator::Allocator(Block *shared_block) + : m_state(std::make_shared()), + m_shared_block(shared_block) {} template template inline Allocator::Allocator(const Allocator &other) - : m_state(other.m_state) {} + : m_state(other.m_state), m_shared_block(other.m_shared_block) {} template template inline Allocator::Allocator(Allocator &&other) noexcept - : m_state(std::move(other.m_state)) {} + : m_state(std::move(other.m_state)), m_shared_block(other.m_shared_block) {} template inline Allocator::~Allocator() {} @@ -342,14 +347,15 @@ inline T *Allocator::allocate(size_t n_elements) { Block *block; - if (shared_block.is_empty()) { - shared_block = Block(AllocationScheme::block_size(m_state->number_of_blocks, - n_bytes_requested), - Source::RAM); - block = &shared_block; + if (m_shared_block && m_shared_block->is_empty()) { + *m_shared_block = Block(AllocationScheme::block_size( + m_state->number_of_blocks, n_bytes_requested), + Source::RAM); + block = m_shared_block; ++m_state->number_of_blocks; - } else if (shared_block.can_accommodate(n_bytes_requested)) { - block = &shared_block; + } else if (m_shared_block && + m_shared_block->can_accommodate(n_bytes_requested)) { + block = m_shared_block; } else if (m_state->current_block.is_empty() || !m_state->current_block.can_accommodate(n_bytes_requested)) { const size_t block_size = AllocationScheme::block_size( @@ -396,7 +402,7 @@ inline void Allocator::deallocate(T *chunk_data, const auto remaining_chunks = block.deallocate(Chunk(chunk_data), n_bytes_requested); if (remaining_chunks == 0) { - if (block == shared_block) { + if (m_shared_block && (block == *m_shared_block)) { // Do nothing. Keep the last block alive. } else { DBUG_ASSERT(m_state->number_of_blocks > 0); diff --git a/storage/temptable/include/temptable/constants.h b/storage/temptable/include/temptable/constants.h index 5920ba83a000..102228fbd4cc 100644 --- a/storage/temptable/include/temptable/constants.h +++ b/storage/temptable/include/temptable/constants.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2017, 2017, Oracle and/or its affiliates. All Rights Reserved. +/* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the @@ -26,6 +26,8 @@ TempTable constants. */ #ifndef TEMPTABLE_CONSTANTS_H #define TEMPTABLE_CONSTANTS_H +#include "my_config.h" + namespace temptable { /** Multiply a number by 1024. @@ -68,6 +70,22 @@ constexpr size_t STORAGE_PAGE_SIZE = 64_KiB; /** Number of buckets to have by default in a hash index. */ constexpr size_t INDEX_DEFAULT_HASH_TABLE_BUCKETS = 1024; +/** Store build-type information into the constexpr expression. */ +#ifndef DBUG_OFF +constexpr bool DEBUG_BUILD = true; +#else +constexpr bool DEBUG_BUILD = false; +#endif /* DBUG_OFF */ + +/** Store L1-dcache size information into the constexpr expression. */ +constexpr size_t L1_DCACHE_SIZE = CPU_LEVEL1_DCACHE_LINESIZE; + +/** Number of shards in key-value store. */ +constexpr size_t KV_STORE_SHARDS_COUNT = 16 * 1024; + +/** Size of a pool containing shared-blocks. */ +constexpr size_t SHARED_BLOCK_POOL_SIZE = 16 * 1024; + } /* namespace temptable */ #endif /* TEMPTABLE_CONSTANTS_H */ diff --git a/storage/temptable/include/temptable/handler.h b/storage/temptable/include/temptable/handler.h index e45c2712aaa9..837051e2305c 100644 --- a/storage/temptable/include/temptable/handler.h +++ b/storage/temptable/include/temptable/handler.h @@ -128,10 +128,6 @@ Also called "key", "field", "subkey", "key part", "key segment" elsewhere. #ifndef TEMPTABLE_HANDLER_H #define TEMPTABLE_HANDLER_H -#ifndef DBUG_OFF -#include -#endif /* DBUG_OFF */ - #include "sql/handler.h" #include "sql/table.h" #include "storage/temptable/include/temptable/storage.h" @@ -139,6 +135,9 @@ Also called "key", "field", "subkey", "key part", "key segment" elsewhere. namespace temptable { +/** Forward declarations. */ +class Block; + /** Temptable engine handler. */ class Handler : public ::handler { public: @@ -524,6 +523,10 @@ class Handler : public ::handler { /** Currently opened table, or `nullptr` if none is opened. */ Table *m_opened_table; + /** Pointer to the non-owned shared-block of memory to be re-used by all + * `Allocator` instances or copies made by `Table`. */ + Block *m_shared_block; + /** Iterator used by `rnd_init()`, `rnd_next()` and `rnd_end()` methods. * It points to the row that was retrieved by the last read call (e.g. * `rnd_next()`). */ @@ -554,17 +557,6 @@ class Handler : public ::handler { /** Number of deleted rows by this handler object. */ size_t m_deleted_rows; - -#ifndef DBUG_OFF - /** Check if the current OS thread is the one that created this handler - * object. - * @return true if current thread == creator thread */ - bool current_thread_is_creator() const; - - /** The OS thread that created this handler object. The only thread that is - * supposed to use it and the tables created via it. */ - std::thread::id m_owner; -#endif /* DBUG_OFF */ }; inline void Handler::opened_table_validate() { @@ -588,6 +580,9 @@ inline bool Handler::is_field_type_fixed_size(const Field &mysql_field) const { } } +void kv_store_shards_debug_dump(); +void shared_block_pool_release(THD *thd); + } /* namespace temptable */ #endif /* TEMPTABLE_HANDLER_H */ diff --git a/storage/temptable/include/temptable/kv_store.h b/storage/temptable/include/temptable/kv_store.h new file mode 100644 index 000000000000..bcf947b0c8b3 --- /dev/null +++ b/storage/temptable/include/temptable/kv_store.h @@ -0,0 +1,137 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/kv_store.h +TempTable key-value store implementation. */ + +#ifndef TEMPTABLE_KV_STORE_H +#define TEMPTABLE_KV_STORE_H + +#include +#include +#include + +#include "storage/temptable/include/temptable/constants.h" +#include "storage/temptable/include/temptable/kv_store_logger.h" + +namespace temptable { + +/** Forward-declarations. */ +class Table; + +/** Key-value store, a convenience wrapper class which models a thread-safe + * dictionary type. + * + * Thread-safety is accomplished by using a `Lock` which can be any of the usual + * mutual exclusive algorithms from C++ thread-support library or any other + * which satisfy C++ concurrency named requirements. E.g. mutex, timed_mutex, + * recursive_mutex, recursive_timed_mutex, shared_mutex, shared_timed_mutex, ... + * User code can opt-in for any of those. Also, whether the actual + * implementation will use shared-locks or exclusive-locks (for read-only + * operations) is handled automagically by this class. + * + * Furthermore, user code can similarly opt-in for different key-value + * implementation but whose interface is compatible with the one of + * std::unordered_map. + * */ +template class KeyValueImpl = std::unordered_map> +class Key_value_store + : public Key_value_store_logger, + DEBUG_BUILD> { + /** Do not break encapsulation when using CRTP. */ + friend struct Key_value_store_logger, + DEBUG_BUILD>; + + /** Help ADL to bring debugging/logging symbols into the scope. */ + using Key_value_store_logger, + DEBUG_BUILD>::dbug_print; + using Key_value_store_logger, + DEBUG_BUILD>::log; + + /** Check whether we can use shared locks (which enable multiple concurrent + * readers) or must we rather fallback to exclusive locks. Shared-locks will + * be used only during read-only operations. + * */ + using Exclusive_or_shared_lock = + std::conditional_t::value, + std::shared_lock, std::lock_guard>; + + /** Alias for our key-value store implementation. */ + using Key_value_store_impl = KeyValueImpl; + + /** Container holding (table-name, Table) tuples. */ + Key_value_store_impl m_kv_store; + + /** Lock type. */ + Lock m_lock; + + public: + /** Inserts a new table into the container constructed in-place with the + * given args if there is no table with the key in the container. + * + * [in] args Arguments to forward to the constructor of the table. + * @return Returns a pair consisting of an iterator to the inserted table, + * or the already-existing table if no insertion happened, and a bool + * denoting whether the insertion took place (true if insertion happened, + * false if it did not). + * */ + template + std::pair emplace( + Args &&... args) { + std::lock_guard lock(m_lock); + dbug_print(); + log(Key_value_store_stats::Event::EMPLACE); + return m_kv_store.emplace(args...); + } + + /** Searches for a table with given name (key). + * + * [in] key Name of a table to search for. + * @return Pointer to table if found, nullptr otherwise. + * */ + Table *find(const std::string &key) { + Exclusive_or_shared_lock lock(m_lock); + auto iter = m_kv_store.find(key); + if (iter != m_kv_store.end()) { + return &iter->second; + } else { + return nullptr; + } + } + + /** Removes the table (if one exists) with the given name (key). + * + * [in] key Name of a table to remove.. + * @return Number of elements removed. + * */ + typename Key_value_store_impl::size_type erase(const std::string &key) { + std::lock_guard lock(m_lock); + dbug_print(); + log(Key_value_store_stats::Event::ERASE); + return m_kv_store.erase(key); + } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_KV_STORE_H */ diff --git a/storage/temptable/include/temptable/kv_store_logger.h b/storage/temptable/include/temptable/kv_store_logger.h new file mode 100644 index 000000000000..ede9b7fcf365 --- /dev/null +++ b/storage/temptable/include/temptable/kv_store_logger.h @@ -0,0 +1,98 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/kv_store_logger.h +TempTable key-value store logger implementation. */ + +#ifndef TEMPTABLE_KV_STORE_LOGGER_H +#define TEMPTABLE_KV_STORE_LOGGER_H + +#include +#include + +#include "my_dbug.h" +#include "storage/temptable/include/temptable/kv_store_stats.h" + +namespace temptable { + +/** Default Key_value_store logging facility which turns to no-op in non-debug + * builds. + */ +template +class Key_value_store_logger { + protected: + void log(Key_value_store_stats::Event /*event*/) { + // no-op + } + void dbug_print() { + // no-op + } +}; + +/** Key_value_store logging facility debug builds only. */ +template +class Key_value_store_logger { + /** Container of stats that we will be collecting. */ + std::vector m_stats; + + public: + /** Returns a snapshot of stats collected. To keep up with the thread-safety + * guarantee promise, snapshot must be made under the lock. + * + * @return Stats snapshot. + * */ + std::vector stats() { + auto &kv_store_lock = static_cast(*this).m_lock; + typename T::Exclusive_or_shared_lock lock(kv_store_lock); + return m_stats; + } + + protected: + /** Appends a new entry to stats container with the given event. + * + * [in] event Type of event to be logged. + * */ + void log(Key_value_store_stats::Event event) { + const auto &kv_store = static_cast(*this).m_kv_store; + m_stats.push_back({event, kv_store.size(), kv_store.bucket_count(), + kv_store.load_factor(), kv_store.max_load_factor(), + kv_store.max_bucket_count(), + std::this_thread::get_id()}); + } + + /** DBUG_PRINT's the stats of underlying key-value store implementation. + * */ + void dbug_print() { + const auto &kv_store = static_cast(*this).m_kv_store; + DBUG_PRINT( + "temptable_api_kv_store", + ("this=%p size=%zu; bucket_count=%zu load_factor=%f " + "max_load_factor=%f " + "max_bucket_count=%zu", + this, kv_store.size(), kv_store.bucket_count(), kv_store.load_factor(), + kv_store.max_load_factor(), kv_store.max_bucket_count())); + } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_KV_STORE_LOGGER_H */ diff --git a/storage/temptable/include/temptable/kv_store_stats.h b/storage/temptable/include/temptable/kv_store_stats.h new file mode 100644 index 000000000000..21e43f39c9f9 --- /dev/null +++ b/storage/temptable/include/temptable/kv_store_stats.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/kv_store_stats.h +TempTable key-value store stats description. */ + +#ifndef TEMPTABLE_KV_STORE_STATS_H +#define TEMPTABLE_KV_STORE_STATS_H + +#include + +namespace temptable { + +/** This is a small convenience POD-like type which describes what kind of + * details we are interested in when monitoring the behavior of Key_value_store. + * Details directly correlate to the properties of the underlying data-structure + * that Key_value_store is using. + * */ +struct Key_value_store_stats { + enum class Event { EMPLACE, ERASE }; + Event event; + size_t size; + size_t bucket_count; + double load_factor; + double max_load_factor; + size_t max_bucket_count; + std::thread::id thread_id; +}; + +} // namespace temptable + +#endif /* TEMPTABLE_KV_STORE_STATS_H */ diff --git a/storage/temptable/include/temptable/lock_free_pool.h b/storage/temptable/include/temptable/lock_free_pool.h new file mode 100644 index 000000000000..6aaa130acf97 --- /dev/null +++ b/storage/temptable/include/temptable/lock_free_pool.h @@ -0,0 +1,120 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/lock_free_pool.h +Lock-free pool implementation. */ + +#ifndef TEMPTABLE_LOCK_FREE_POOL_H +#define TEMPTABLE_LOCK_FREE_POOL_H + +#include + +#include "storage/temptable/include/temptable/lock_free_type.h" + +namespace temptable { + +/** Lock-free pool which consists of POOL_SIZE Lock_free_type elements. It has + * all the guarantees and properties of Lock_free_type and its consisting pieces + * so for more details please consult the documentation of those. E.g. user code + * can opt-in for different alignment-requirements and/or lock-free type + * selection algorithms. This type is envisioned to be used as a building block + * for higher-abstraction (lock-free) ADTs. + * */ +template class TypeSelector = + Lock_free_type_selector> +class Lock_free_pool { + std::array, POOL_SIZE> m_lock_free; + + public: + using Type = typename Lock_free_type::Type; + + /** Default constructor. Uses value-initialization to initialize underlying + * T's.*/ + Lock_free_pool() : m_lock_free() {} + + /** Constructor. Uses explicit value to initialize underlying T's. */ + Lock_free_pool(Lock_free_pool::Type default_value) { + for (auto &v : m_lock_free) v.m_value = default_value; + } + + /** Copy-constructor and copy-assignment operator are disabled. */ + Lock_free_pool(const Lock_free_pool &) = delete; + Lock_free_pool &operator=(const Lock_free_pool &) = delete; + + /** Ditto for move-constructor and move-assignment operator. */ + Lock_free_pool(Lock_free_pool &&) = delete; + Lock_free_pool &operator=(Lock_free_pool &&) = delete; + + /** Atomically replaces current value in an array at given index. + * + * [in] idx Index of an element whose value is to be replaced. + * [in] value Value to store into the atomic variable. + * [in] order Memory order constraints to enforce. + * */ + void store(size_t idx, Lock_free_pool::Type value, + std::memory_order order = std::memory_order_seq_cst) { + m_lock_free[idx].m_value.store(value, order); + } + + /** Atomically loads and returns the current value from an array at given + * index. + * + * [in] idx Index of an element whose value is to be returned. + * [in] order Memory order constraints to enforce. + * @return Returns the current value from given index. + * */ + Lock_free_pool::Type load( + size_t idx, std::memory_order order = std::memory_order_seq_cst) const { + return m_lock_free[idx].m_value.load(order); + } + + /** Atomically compares the object representation of an array element at given + * index with the expected, and if those are bitwise-equal, replaces the + * former with desired. + * + * [in] idx Index of an element whose value is to be compared against. + * [in] expected Value expected to be found in the atomic object. Gets stored + * with the actual value of *this if the comparison fails. + * [in] desired Value to store in the atomic object if it is as expected. + * [in] order Memory order constraints to enforce. + * @return True if the underlying atomic value was successfully changed, false + * otherwise. + * */ + bool compare_exchange_strong( + size_t idx, Lock_free_pool::Type &expected, Lock_free_pool::Type desired, + std::memory_order order = std::memory_order_seq_cst) { + return m_lock_free[idx].m_value.compare_exchange_strong(expected, desired, + order); + } + + /** Returns the number of elements this pool contains. + * + * @return Number of elements in the pool. + * */ + constexpr size_t size() const { return POOL_SIZE; } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_LOCK_FREE_POOL_H */ diff --git a/storage/temptable/include/temptable/lock_free_type.h b/storage/temptable/include/temptable/lock_free_type.h new file mode 100644 index 000000000000..691137f5625d --- /dev/null +++ b/storage/temptable/include/temptable/lock_free_type.h @@ -0,0 +1,283 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/lock_free_type.h +Lock-free type (selection) implementation. */ + +#ifndef TEMPTABLE_LOCK_FREE_TYPE_H +#define TEMPTABLE_LOCK_FREE_TYPE_H + +#include +#include + +#include "storage/temptable/include/temptable/constants.h" + +namespace temptable { + +/** Enum class describing alignment-requirements. */ +enum class Alignment { NATURAL, L1_DCACHE_SIZE }; + +/** Lock-free type selector, a helper utility which evaluates during + * the compile-time whether the given type T has a property of being + * always-lock-free for given platform. If true, Lock_free_type_selector::Type + * will hold T, otherwise Lock_free_type_selector::Type will be inexisting in + * which case static-assert will be triggered with a hopefully descriptive + * error-message. In the event of static-assert, one can either try to select + * another type T or, if one does not care about the actual underlying + * type representation, simply utilize the `Largest_lock_free_type_selector` + * utility instead. This utility will work out those details automagically. For + * more information, see documentation on `Largest_lock_free_type_selector`. + * + * In short, reasoning behind this machinery lies in the fact that the standard + * cannot guarantee that underlying implementation of std::atomic is going + * to be able to use lock-free atomic CPU instructions. That obviously depends + * on the given type T but also on the properties of concrete platform. + * Therefore, actual implementation is mostly platform-dependent and is + * free to choose any other locking operation (e.g. mutex) as long as it is + * able to fulfill the atomicity. Lock-freedom is not a pre-requisite. Only + * exception is std::atomic_flag. + * + * For certain types, however, lock-freedom can be claimed upfront during the + * compile-time phase, and this is where this utiliy kicks in. It is essentially + * a C++14 rewrite of std::atomic::is_always_lock_free which is only + * available from C++17 onwards. Once moved to C++17 this utility will become + * obsolete and shall be replaced with standard-compliant implementation. + * + * More details and motivation can be found at: + * http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/p0152r0.html + * */ +template +struct Lock_free_type_selector { + static_assert( + !std::is_same::value, + "No always-lock-free property could be found for given type. " + "Type provided is probably not a built-in (fundamental) type or a " + "pointer which makes it impossible for this particular check to be " + "excercised at compile-time."); +}; + +/** Template-specialization for trivially-copyable classes/structs. + * + * Subset of trivially-copyable classes/structs might have always-lock-free + * property but for this feature to be implemented we would have to go at great + * lengths to implement cross-platform support. Therefore, for simplicity + * reasons let's just detect the overload and fail gracefully. + * */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value and + std::is_trivially_copyable::value>::type> { + static_assert(!std::is_same::value, + "Querying always-lock-free property of trivially-copyable " + "classes or structs is not yet implemented!"); +}; + +/** Template-specialization for pointer types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value>::type> { +#if (ATOMIC_POINTER_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "Pointer type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for long long types. */ +template +struct Lock_free_type_selector< + T, + typename std::enable_if::value or + std::is_same::value>::type> { +#if (ATOMIC_LLONG_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "(unsigned) long long type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for long types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value or + std::is_same::value>::type> { +#if (ATOMIC_LONG_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "(unsigned) long type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for int types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value or + std::is_same::value>::type> { +#if (ATOMIC_INT_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "(unsigned) int type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for short types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value or + std::is_same::value>::type> { +#if (ATOMIC_SHORT_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "(unsigned) short type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for char types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value or + std::is_same::value>::type> { +#if (ATOMIC_CHAR_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "(unsigned) char type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for boolean types. */ +template +struct Lock_free_type_selector< + T, typename std::enable_if::value>::type> { +#if (ATOMIC_BOOL_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "bool type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Largest lock-free type selector, a helper utility very much similar + * to Lock_free_type_selector with the difference being that it tries hard + * not to fail. E.g. it will try to find the largest available T for given + * platform which has a property of being always-lock-free. T which has been + * selected is then found in Largest_lock_free_type_selector::Type. Signedness + * of T is respected. + * */ +template +struct Largest_lock_free_type_selector { + static_assert( + !std::is_same::value, + "No always-lock-free property could be found for given type. " + "Type provided is probably not a built-in (fundamental) type or a " + "pointer which makes it impossible for this particular check to be " + "excercised at compile-time."); +}; + +/** Template-specialization for pointer types. */ +template +struct Largest_lock_free_type_selector< + T, typename std::enable_if::value>::type> { +#if (ATOMIC_POINTER_LOCK_FREE == 2) + using Type = T; +#else + static_assert(false, + "Pointer type on this platform does not have an " + "always-lock-free property. Bailing out ..."); +#endif +}; + +/** Template-specialization for integral types. */ +template +struct Largest_lock_free_type_selector< + T, typename std::enable_if::value>::type> { +#if (ATOMIC_LLONG_LOCK_FREE == 2) + using Type = std::conditional_t::value, + unsigned long long, long long>; +#elif (ATOMIC_LONG_LOCK_FREE == 2) + using Type = + std::conditional_t::value, unsigned long, long>; +#elif (ATOMIC_INT_LOCK_FREE == 2) + using Type = + std::conditional_t::value, unsigned int, int>; +#elif (ATOMIC_SHORT_LOCK_FREE == 2) + using Type = + std::conditional_t::value, unsigned short, short>; +#elif (ATOMIC_CHAR_LOCK_FREE == 2) + using Type = + std::conditional_t::value, unsigned char, char>; +#elif (ATOMIC_BOOL_LOCK_FREE == 2) + using Type = bool; +#else + static_assert( + false, + "No suitable always-lock-free type was found for this platform. " + "Bailing out ..."); +#endif +}; + +/** Representation of an atomic type which is guaranteed to be always-lock-free. + * In case always-lock-free property cannot be satisfied for given T, + * Lock_free_type instantiation will fail with the compile-time error. + * + * Always-lock-free guarantee is implemented through the means of + * Lock_free_type_selector or Largest_lock_free_type_selector. User code can + * opt-in for any of those. By default, Lock_free_type_selector is used. + * + * In addition, this type provides an ability to redefine the + * alignment-requirement of the underlying always-lock-free type, basically + * making it possible to over-align T to the size of the L1-data cache-line + * size. By default, T has a natural alignment. + */ +template class TypeSelector = + Lock_free_type_selector> +struct Lock_free_type { + using Type = typename TypeSelector::Type; + std::atomic m_value; +}; + +/* + * Template-specialization for Lock_free_type with alignment-requirement set to + * L1-data cache size. + * */ +template class TypeSelector> +struct Lock_free_type { + using Type = typename TypeSelector::Type; + alignas(L1_DCACHE_SIZE) std::atomic m_value; +}; + +} // namespace temptable + +#endif /* TEMPTABLE_LOCK_FREE_TYPE_H */ diff --git a/storage/temptable/include/temptable/sharded_kv_store.h b/storage/temptable/include/temptable/sharded_kv_store.h new file mode 100644 index 000000000000..6fcf4f1ba6cb --- /dev/null +++ b/storage/temptable/include/temptable/sharded_kv_store.h @@ -0,0 +1,103 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/sharded_kv_store.h +TempTable sharded key-value store implementation. */ + +#ifndef TEMPTABLE_SHARDED_KV_STORE_H +#define TEMPTABLE_SHARDED_KV_STORE_H + +#include + +#include "storage/temptable/include/temptable/constants.h" +#include "storage/temptable/include/temptable/kv_store.h" +#include "storage/temptable/include/temptable/sharded_kv_store_logger.h" + +namespace temptable { + +/** Sharded key-value store, a convenience wrapper class around Key_value_store + * that creates N_SHARDS instances of Key_value_store and exposes a simple + * interface to further manipulate with them. User code, if it wishes so, can + * opt-in for different key-value implementation or different locking + * mechanisms. Defaults are provided as convenience. + * + * Mapping between a shard and an actual Key_value_store is done at the level of + * implementation detail of this class and it is done via modulo-arithmethic on + * THD (connection) identifier. It is guaranteed that the same shard (same + * Key_value_store instance) will always be returned given the same input: THD + * (connection) identifier. However, there is no guarantee that there will be + * only one such mapping (this function is not a bijection). + * + * In other words, due to the wrap-around property of modulo-arithmetic + * and depending on the actual value of N_SHARDS, it is very much possible to + * get a reference to the same Key_value_store instance for two different THD + * (connection) identifiers. Thread-safety guarantees are excercised at the + * level of Key_value_store instance but are controlled through the `Lock`, this + * class template parameter. + * + * Due to aforementioned modulo-arithmethic, and the questionable performance in + * its more general form, N_SHARDS must be a number which is a power of two. + * This enables us to implement a modulo operation in single bitwise + * instruction. Check whether N_SHARDS is a number which is power of two is run + * during the compile-time. + * */ +template class KeyValueImpl = std::unordered_map> +class Sharded_key_value_store + : public Sharded_key_value_store_logger, + DEBUG_BUILD> { + /** Do not break encapsulation when using CRTP. */ + friend struct Sharded_key_value_store_logger; + + /** Check whether all compile-time pre-conditions are set. */ + static_assert(N_SHARDS && !(N_SHARDS & (N_SHARDS - 1)), + "N_SHARDS template parameter must be a power of two."); + + /** Bitmask which enables us to implement modulo-arithmetic operation in + * single bitwise instruction. */ + static constexpr size_t MODULO_MASK = N_SHARDS - 1; + + /** In the event of inability to express ourselves with something like + * std::array Key_value_store<...>> we have to fallback to this + * method. + * */ + struct L1_dcache_aligned_kv_store { + alignas(L1_DCACHE_SIZE) Key_value_store shard; + }; + + /** N_SHARDS instances of Key_value_store's. */ + std::array m_kv_store_shard; + + public: + /** Returns a reference to one of the underlying Key_value_store instances. + * + * [in] THD (connection) identifier. + * @return A reference to Key_value_store instance. */ + Key_value_store &operator[](size_t thd_id) { + return m_kv_store_shard[thd_id & MODULO_MASK].shard; + } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_SHARDED_KV_STORE_H */ diff --git a/storage/temptable/include/temptable/sharded_kv_store_logger.h b/storage/temptable/include/temptable/sharded_kv_store_logger.h new file mode 100644 index 000000000000..769f6192cb3c --- /dev/null +++ b/storage/temptable/include/temptable/sharded_kv_store_logger.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/sharded_kv_store_logger.h +TempTable sharded key-value store logger implementation. */ + +#ifndef TEMPTABLE_SHARDED_KV_STORE_LOGGER_H +#define TEMPTABLE_SHARDED_KV_STORE_LOGGER_H + +#include +#include +#include + +#include "my_dbug.h" +#include "storage/temptable/include/temptable/kv_store_stats.h" + +namespace temptable { + +/** Default Sharded_key_value_store logging facility which turns to no-op in + * non-debug builds. */ +template +struct Sharded_key_value_store_logger { + void dbug_print() { + // no-op + } +}; + +/** Sharded_key_value_store logging facility debug builds only. */ +template +struct Sharded_key_value_store_logger { + /** DBUG_PRINT's the stats of each shard and additionally some further stats + * which are deduced from it, e.g. ratio between insertions and removals. + * */ + void dbug_print() { + // As representation of std::thread::id is implementation defined, we have + // to convert it to a string first before passing it over to DBUG_PRINT + // machinery. Conversion to string can be done with the overloaded + // std::thread::id::operator<< which is part of the library + // implementation. + auto std_thread_id_to_str = [](const std::thread::id &id) { + std::stringstream ss; + ss << id; + return ss.str(); + }; + + auto &kv_store_shards = static_cast(*this).m_kv_store_shard; + uint32_t shard_id = 0; + for (auto &kv : kv_store_shards) { + auto kv_shard_stats = kv.shard.stats(); + size_t nr_of_emplace_events = std::count_if( + kv_shard_stats.begin(), kv_shard_stats.end(), [](auto &stat) { + return stat.event == Key_value_store_stats::Event::EMPLACE; + }); + + // Silence the -Wunused-variable false-positive warning (bug) from clang. + // MY_COMPILER_CLANG_WORKAROUND_FALSE_POSITIVE_UNUSED_VARIABLE_WARNING + // documentation contains more details (e.g. specific bug-number from LLVM + // Bugzilla) + MY_COMPILER_DIAGNOSTIC_PUSH() + MY_COMPILER_CLANG_WORKAROUND_FALSE_POSITIVE_UNUSED_VARIABLE_WARNING() + size_t nr_of_erase_events = kv_shard_stats.size() - nr_of_emplace_events; + MY_COMPILER_DIAGNOSTIC_POP() + + DBUG_PRINT("temptable_api_sharded_kv_store", + ("shard_id=%u insertions=%zu removals=%zu", shard_id, + nr_of_emplace_events, nr_of_erase_events)); + for (auto &stat : kv_shard_stats) { + DBUG_PRINT( + "temptable_api_sharded_kv_store_debug", + ("shard_id=%u event=%d size=%zu bucket_count=%zu load_factor=%f " + "max_load_factor=%f " + "max_bucket_count=%zu thread_id=%s", + shard_id, static_cast(stat.event), stat.size, + stat.bucket_count, stat.load_factor, stat.max_load_factor, + stat.max_bucket_count, + std_thread_id_to_str(stat.thread_id).c_str())); + } + shard_id++; + } + } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_SHARDED_KV_STORE_LOGGER_H */ diff --git a/storage/temptable/include/temptable/shared_block_pool.h b/storage/temptable/include/temptable/shared_block_pool.h new file mode 100644 index 000000000000..70123f2d2fbb --- /dev/null +++ b/storage/temptable/include/temptable/shared_block_pool.h @@ -0,0 +1,146 @@ +/* Copyright (c) 2020, Oracle and/or its affiliates. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License, version 2.0, as published by the +Free Software Foundation. + +This program is also distributed with certain software (including but not +limited to OpenSSL) that is licensed under separate terms, as designated in a +particular file or component or in included license documentation. The authors +of MySQL hereby grant you an additional permission to link the program and +your derivative works with the separately licensed software that they have +included with MySQL. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0, +for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/** @file storage/temptable/include/temptable/shared_block_pool.h +TempTable shared-block pool implementation. */ + +#ifndef TEMPTABLE_SHARED_BLOCK_POOL_H +#define TEMPTABLE_SHARED_BLOCK_POOL_H + +#include +#include +#include + +#include "storage/temptable/include/temptable/block.h" +#include "storage/temptable/include/temptable/constants.h" +#include "storage/temptable/include/temptable/lock_free_pool.h" +#include "storage/temptable/include/temptable/lock_free_type.h" + +namespace temptable { + +/** Lock-free pool of POOL_SIZE Block elements. + * + * Each Block element in a pool is represented by a slot. Each slot can be + * either free (default) or occupied. Acquiring the Block is possible only from + * an empty slot. Releasing the Block makes the slot free again. + * + * Acquiring and releasing the Block is done via THD identifier on which, + * implicitly, modulo-arithmethic is applied in order to pick the right slot. + * To avoid questionable performance of modulo-arithmetic in its more general + * form, POOL_SIZE is constrained only to numbers which are power of two. This + * enables us to implement a modulo operation in single bitwise instruction. + * Check whether POOL_SIZE is a number which is power of two is run during the + * compile-time. + * + * Given that this pool is envisioned to be used concurrently by multiple + * threads, both the slot and Block are aligned to the size of CPU L1 + * data-cache. This makes sure that we negate the effect of false-sharing + * (threads bouncing each ones data from cache). + */ +template +class Lock_free_shared_block_pool { + /** Check whether all compile-time pre-conditions are set. */ + static_assert(POOL_SIZE && !(POOL_SIZE & (POOL_SIZE - 1)), + "POOL_SIZE template parameter must be a power of two."); + + /** Build a slot type: + * a lock-free pool of L1-DCACHE aligned unsigned long long's + */ + using Shared_block_slot = + Lock_free_pool; + + /** Be pedantic about the element type we used when building the slot type. */ + using Shared_block_slot_element_type = typename Shared_block_slot::Type; + + /** Constexpr variable denoting a non-occupied (free) slot. */ + static constexpr Shared_block_slot_element_type FREE_SLOT = + std::numeric_limits::max(); + + /** Bitmask which enables us to implement modulo-arithmetic operation in + * single bitwise instruction. */ + static constexpr size_t MODULO_MASK = POOL_SIZE - 1; + + /** In the event of inability to express ourselves with something like + * std::array Block> we have to fallback to this method. + * */ + struct L1_dcache_aligned_block { + alignas(L1_DCACHE_SIZE) Block block; + }; + + /** An array of L1-dcache aligned Blocks + * Note: This is _not_ the same as: + * alignas(L1_DCACHE_SIZE) std::array + * */ + std::array m_shared_block; + + /** Lock-free slots. */ + Shared_block_slot m_slot{FREE_SLOT}; + + public: + /** Given the THD identifier, try to acquire an instance of Block. In the + * event of success, non-nullptr instance of Block will be returned and + * underlying slot will be marked as occupied. Otherwise, slot must have been + * already occupied in which case a nullptr will be returned. + * + * Using the same THD identifier to re-acquire the Block (from the same slot) + * is supported but not possible if some other THD identifier is used. + * + * [in] thd_id THD identifier. + * @return Returns a pointer to the Block (success) or nullptr (failure). + * */ + Block *try_acquire(size_t thd_id) { + auto slot_idx = thd_id & MODULO_MASK; + auto slot_thd_id = FREE_SLOT; + if (m_slot.compare_exchange_strong(slot_idx, slot_thd_id, thd_id)) { + return &m_shared_block[slot_idx].block; + } else if (slot_thd_id == thd_id) { + return &m_shared_block[slot_idx].block; + } else { + return nullptr; + } + } + + /** Given the THD identifier, try to release an acquired instance of a + * Block. In the event of success, slot will be marked as non-occupied. + * Assuming that Block is not empty, it will also be destroyed. + * + * Trying to release the Block/slot by using some other THD identifier is not + * possible and will therefore render this operation as failed. + * + * [in] thd_id THD identifier. + * @return Returns true (success) or false (failure). + * */ + bool try_release(size_t thd_id) { + auto slot_idx = thd_id & MODULO_MASK; + if (m_slot.load(slot_idx) == thd_id) { + auto &block = m_shared_block[slot_idx].block; + if (!block.is_empty()) block.destroy(); + m_slot.store(slot_idx, FREE_SLOT); + return true; + } + return false; + } +}; + +} // namespace temptable + +#endif /* TEMPTABLE_SHARED_BLOCK_POOL_H */ diff --git a/storage/temptable/include/temptable/table.h b/storage/temptable/include/temptable/table.h index 31c57b9bb2c7..30e818a6c92e 100644 --- a/storage/temptable/include/temptable/table.h +++ b/storage/temptable/include/temptable/table.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, 2018, Oracle and/or its affiliates. All Rights Reserved. +/* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the @@ -45,7 +45,8 @@ namespace temptable { class Table { public: - Table(TABLE *mysql_table, bool all_columns_are_fixed_size); + Table(TABLE *mysql_table, Block *shared_block, + bool all_columns_are_fixed_size); /* The `m_rows` member is too expensive to copy around. */ Table(const Table &) = delete; diff --git a/storage/temptable/src/allocator.cc b/storage/temptable/src/allocator.cc index a96eabcc83f0..a08ea3d4aaed 100644 --- a/storage/temptable/src/allocator.cc +++ b/storage/temptable/src/allocator.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All Rights Reserved. +/* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the @@ -25,28 +25,11 @@ TempTable custom allocator implementation. */ #include /* std::atomic */ #include /* size_t */ -#include /* uint8_t */ #include "storage/temptable/include/temptable/allocator.h" namespace temptable { -/** RAII-managed Allocator thread-resources cleanup class */ -struct End_thread { - ~End_thread() { - if (!shared_block.is_empty()) { - shared_block.destroy(); - } - } -}; - -/** Thread-local variable whose destruction will make sure that - * shared memory-block will be destroyed. */ -static thread_local End_thread end_thread; - -/* Global shared memory-block. */ -thread_local Block shared_block; - /* Initialization of MemoryMonitor static variables. */ std::atomic MemoryMonitor::ram(0); diff --git a/storage/temptable/src/handler.cc b/storage/temptable/src/handler.cc index f94c0f9e85c1..0f78b553733d 100644 --- a/storage/temptable/src/handler.cc +++ b/storage/temptable/src/handler.cc @@ -23,36 +23,40 @@ this program; if not, write to the Free Software Foundation, Inc., /** @file storage/temptable/src/handler.cc TempTable public handler API implementation. */ -#include -#include -#include - -#ifndef DBUG_OFF -#include -#endif - +#include "storage/temptable/include/temptable/handler.h" #include "my_base.h" #include "my_dbug.h" -#include "sql/handler.h" +#include "mysql/plugin.h" #include "sql/mysqld.h" +#include "sql/sql_thd_internal_api.h" #include "sql/system_variables.h" -#include "sql/table.h" -#include "storage/temptable/include/temptable/handler.h" #include "storage/temptable/include/temptable/row.h" -#include "storage/temptable/include/temptable/storage.h" -#include "storage/temptable/include/temptable/table.h" +#include "storage/temptable/include/temptable/sharded_kv_store.h" +#include "storage/temptable/include/temptable/shared_block_pool.h" namespace temptable { -/** A container for the list of the tables. Don't allocate memory for it from - * the Allocator because the Allocator keeps one block for reuse and it is - * only marked for reuse after all elements from it have been removed. This - * container, being a global variable may allocate some memory and never free - * it before its destructor is called at thread termination time. */ -using Tables = std::unordered_map; - -/** A list of the tables that currently exist for this OS thread. */ -static thread_local Tables tls_tables; +/** Key-value store containing all tables for all existing connections. + * See `Sharded_key_value_store` documentation for more details. + * */ +static Sharded_key_value_store kv_store_shard; + +/* Pool of shared-blocks, an external state to the custom `TempTable` memory + * allocator. See `Lock_free_shared_block_pool` documentation for more details. + * */ +static Lock_free_shared_block_pool shared_block_pool; + +/** Small helper function which debug-prints the miscelaneous statistics which + * key-value store has collected. + * */ +void kv_store_shards_debug_dump() { kv_store_shard.dbug_print(); } + +/** Small helper function which releases the slot (and memory occupied by the + * Block) in shared-block pool. + * */ +void shared_block_pool_release(THD *thd) { + shared_block_pool.try_release(thd_thread_id(thd)); +} #if defined(HAVE_WINNUMA) /** Page size used in memory allocation. */ @@ -64,6 +68,7 @@ DWORD win_page_size; Handler::Handler(handlerton *hton, TABLE_SHARE *table_share_arg) : ::handler(hton, table_share_arg), m_opened_table(), + m_shared_block(shared_block_pool.try_acquire(thd_thread_id(ha_thd()))), m_rnd_iterator(), m_rnd_iterator_is_positioned(), m_index_cursor(), @@ -71,16 +76,30 @@ Handler::Handler(handlerton *hton, TABLE_SHARE *table_share_arg) m_deleted_rows() { handler::ref_length = sizeof(Storage::Element *); + // Overriding `handlerton::` with a function-pointer in `TempTable`, or + // any other plugin, is not always sufficient for server to actually invoke + // that hook. This is the behavior which is in contrast to what one would + // expect and this is not how most handlerton interfaces work. However, there + // is a subset of handlerton interfaces which require special care and in + // addition to overriding the function-pointer, they require initializing + // `handlerton::ha_data` with some data which is not a `nullptr`. + // + // `handlerton::close_connection()` is one of such interfaces, and being + // relied upon by the `TempTable` implementation, we have to fill in the + // `ha_data` with some existing data. In this particular case, + // `&shared_block_pool` is selected but from `TempTable` PoV this does not + // have any semantic value, e.g. it could very well have been something else. + // + // This is a bit confusing and unexpected, and hence this comment to make life + // easier for readers of this part of the code. + thd_set_ha_data(ha_thd(), hton, &shared_block_pool); + #if defined(HAVE_WINNUMA) SYSTEM_INFO systemInfo; GetSystemInfo(&systemInfo); win_page_size = systemInfo.dwPageSize; #endif /* HAVE_WINNUMA */ - -#ifndef DBUG_OFF - m_owner = std::this_thread::get_id(); -#endif /* DBUG_OFF */ } Handler::~Handler() {} @@ -89,7 +108,6 @@ int Handler::create(const char *table_name, TABLE *mysql_table, HA_CREATE_INFO *, dd::Table *) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); DBUG_ASSERT(mysql_table != nullptr); DBUG_ASSERT(mysql_table->s != nullptr); DBUG_ASSERT(mysql_table->field != nullptr); @@ -113,9 +131,11 @@ int Handler::create(const char *table_name, TABLE *mysql_table, DBUG_EXECUTE_IF("temptable_create_return_non_result_type_exception", throw 42;); - const auto insert_result = tls_tables.emplace( - std::piecewise_construct, std::make_tuple(table_name), - std::forward_as_tuple(mysql_table, all_columns_are_fixed_size)); + auto &kv_store = kv_store_shard[thd_thread_id(ha_thd())]; + const auto insert_result = kv_store.emplace( + std::piecewise_construct, std::forward_as_tuple(table_name), + std::forward_as_tuple(mysql_table, m_shared_block, + all_columns_are_fixed_size)); ret = insert_result.second ? Result::OK : Result::TABLE_EXIST; @@ -131,17 +151,16 @@ int Handler::create(const char *table_name, TABLE *mysql_table, int Handler::delete_table(const char *table_name, const dd::Table *) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); DBUG_ASSERT(table_name != nullptr); Result ret; try { - const auto pos = tls_tables.find(table_name); - - if (pos != tls_tables.end()) { - if (&pos->second != m_opened_table) { - tls_tables.erase(pos); + auto &kv_store = kv_store_shard[thd_thread_id(ha_thd())]; + auto table = kv_store.find(table_name); + if (table) { + if (m_opened_table != table) { + kv_store.erase(table_name); ret = Result::OK; } else { /* Attempt to delete the currently opened table. */ @@ -165,7 +184,6 @@ int Handler::delete_table(const char *table_name, const dd::Table *) { int Handler::open(const char *table_name, int, uint, const dd::Table *) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); DBUG_ASSERT(m_opened_table == nullptr); DBUG_ASSERT(table_name != nullptr); DBUG_ASSERT(!m_rnd_iterator_is_positioned); @@ -175,13 +193,13 @@ int Handler::open(const char *table_name, int, uint, const dd::Table *) { Result ret; try { - Tables::iterator iter = tls_tables.find(table_name); - if (iter == tls_tables.end()) { - ret = Result::NO_SUCH_TABLE; - } else { - m_opened_table = &iter->second; - opened_table_validate(); + auto &kv_store = kv_store_shard[thd_thread_id(ha_thd())]; + m_opened_table = kv_store.find(table_name); + if (m_opened_table) { ret = Result::OK; + opened_table_validate(); + } else { + ret = Result::NO_SUCH_TABLE; } } catch (std::bad_alloc &) { ret = Result::OUT_OF_MEM; @@ -196,7 +214,6 @@ int Handler::open(const char *table_name, int, uint, const dd::Table *) { int Handler::close() { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); DBUG_ASSERT(m_opened_table != nullptr); m_opened_table = nullptr; @@ -217,8 +234,6 @@ int Handler::close() { int Handler::rnd_init(bool) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); - m_rnd_iterator_is_positioned = false; /* Marked unused - temporary fix for GCC 8 bug 82728. */ @@ -233,7 +248,6 @@ int Handler::rnd_init(bool) { int Handler::rnd_next(uchar *mysql_row) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_read_rnd_next_count); @@ -289,7 +303,6 @@ int Handler::rnd_next(uchar *mysql_row) { int Handler::rnd_pos(uchar *mysql_row, uchar *position) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_read_rnd_count); @@ -311,8 +324,6 @@ int Handler::rnd_pos(uchar *mysql_row, uchar *position) { int Handler::rnd_end() { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); - m_rnd_iterator_is_positioned = false; /* Marked unused - temporary fix for GCC 8 bug 82728. */ @@ -327,7 +338,6 @@ int Handler::rnd_end() { int Handler::index_init(uint index_no, bool) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); Result ret; @@ -350,7 +360,6 @@ int Handler::index_read(uchar *mysql_row, const uchar *mysql_search_cells, ha_rkey_function find_flag) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_read_key_count); @@ -459,7 +468,6 @@ int Handler::index_read(uchar *mysql_row, const uchar *mysql_search_cells, int Handler::index_next(uchar *mysql_row) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_read_next_count); @@ -472,7 +480,6 @@ int Handler::index_next(uchar *mysql_row) { int Handler::index_next_same(uchar *mysql_row, const uchar *, uint) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_read_next_count); @@ -557,7 +564,6 @@ int Handler::index_read_last(uchar *mysql_row, const uchar *mysql_search_cells, uint mysql_search_cells_len_bytes) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); const Result ret = static_cast( @@ -570,7 +576,6 @@ int Handler::index_read_last(uchar *mysql_row, const uchar *mysql_search_cells, int Handler::index_prev(uchar *mysql_row) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); DBUG_ASSERT(m_index_cursor.is_positioned()); @@ -605,8 +610,6 @@ int Handler::index_prev(uchar *mysql_row) { int Handler::index_end() { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); - handler::active_index = MAX_KEY; m_index_cursor.unposition(); @@ -623,8 +626,6 @@ int Handler::index_end() { void Handler::position(const uchar *) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); - Storage::Element *row; if (m_rnd_iterator_is_positioned) { @@ -643,7 +644,6 @@ void Handler::position(const uchar *) { int Handler::write_row(uchar *mysql_row) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_write_count); @@ -656,7 +656,6 @@ int Handler::write_row(uchar *mysql_row) { int Handler::update_row(const uchar *mysql_row_old, uchar *mysql_row_new) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); handler::ha_statistic_increment(&System_status_var::ha_update_count); @@ -680,7 +679,6 @@ int Handler::update_row(const uchar *mysql_row_old, uchar *mysql_row_new) { int Handler::delete_row(const uchar *mysql_row) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); DBUG_ASSERT(m_rnd_iterator_is_positioned); @@ -709,7 +707,6 @@ int Handler::delete_row(const uchar *mysql_row) { int Handler::truncate(dd::Table *) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); m_opened_table->truncate(); @@ -734,7 +731,6 @@ int Handler::delete_all_rows() { int Handler::info(uint) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); opened_table_validate(); stats.deleted = m_deleted_rows; @@ -903,7 +899,6 @@ double Handler::read_time(uint, uint, ha_rows rows) { int Handler::disable_indexes(uint mode) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); DBUG_ASSERT(m_opened_table != nullptr); Result ret; @@ -923,8 +918,6 @@ int Handler::disable_indexes(uint mode) { int Handler::enable_indexes(uint mode) { DBUG_TRACE; - DBUG_ASSERT(current_thread_is_creator()); - Result ret; if (mode == HA_KEY_SWITCH_ALL) { @@ -1043,10 +1036,4 @@ bool Handler::check_if_incompatible_data(HA_CREATE_INFO *, uint) { return false; } -#ifndef DBUG_OFF -bool Handler::current_thread_is_creator() const { - return m_owner == std::this_thread::get_id(); -} -#endif /* DBUG_OFF */ - } /* namespace temptable */ diff --git a/storage/temptable/src/plugin.cc b/storage/temptable/src/plugin.cc index fe7a7b9258f4..c8c49aeb3bcc 100644 --- a/storage/temptable/src/plugin.cc +++ b/storage/temptable/src/plugin.cc @@ -37,6 +37,13 @@ static handler *create_handler(handlerton *hton, TABLE_SHARE *table_share, bool, return new (mem_root) temptable::Handler(hton, table_share); } +static int close_connection(handlerton *hton, THD *thd) { + (void)hton; + temptable::kv_store_shards_debug_dump(); + temptable::shared_block_pool_release(thd); + return 0; +} + static st_mysql_storage_engine temptable_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION}; @@ -49,6 +56,7 @@ static int init(void *p) { h->flags = HTON_ALTER_NOT_SUPPORTED | HTON_CAN_RECREATE | HTON_HIDDEN | HTON_NOT_USER_SELECTABLE | HTON_NO_PARTITION | HTON_NO_BINLOG_ROW_OPT | HTON_SUPPORTS_EXTENDED_KEYS; + h->close_connection = close_connection; temptable::Allocator::init(); diff --git a/storage/temptable/src/table.cc b/storage/temptable/src/table.cc index ed02e8c8d22f..4251aa9070a2 100644 --- a/storage/temptable/src/table.cc +++ b/storage/temptable/src/table.cc @@ -46,8 +46,10 @@ TempTable Table implementation. */ namespace temptable { -Table::Table(TABLE *mysql_table, bool all_columns_are_fixed_size) - : m_rows(&m_allocator), +Table::Table(TABLE *mysql_table, Block *shared_block, + bool all_columns_are_fixed_size) + : m_allocator(shared_block), + m_rows(&m_allocator), m_all_columns_are_fixed_size(all_columns_are_fixed_size), m_indexes_are_enabled(true), m_mysql_row_length(mysql_table->s->rec_buff_length), diff --git a/unittest/gunit/temptable/allocator-t.cc b/unittest/gunit/temptable/allocator-t.cc index d827446f3a7c..f1e9a532b132 100644 --- a/unittest/gunit/temptable/allocator-t.cc +++ b/unittest/gunit/temptable/allocator-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -28,7 +28,7 @@ #include "allocator_helper.h" #include "storage/temptable/include/temptable/allocator.h" -#include "storage/temptable/src/allocator.cc" +#include "storage/temptable/include/temptable/block.h" namespace temptable_test { @@ -122,7 +122,8 @@ TEST(Allocator, BasicAlloc) { using Item = int; const int ITEM_COUNT = 100; - temptable::Allocator allocator; + temptable::Block shared_block; + temptable::Allocator allocator(&shared_block); std::vector item_pointers; item_pointers.assign(ITEM_COUNT, nullptr); @@ -165,7 +166,8 @@ TEST(Allocator, ZeroSize) { Allocator_helper::set_allocator_max_ram_default(); init_allocator_once(); - temptable::Allocator allocator; + temptable::Block shared_block; + temptable::Allocator allocator(&shared_block); int *item = nullptr; @@ -177,7 +179,8 @@ TEST(Allocator, ConstructDestroy) { Allocator_helper::set_allocator_max_ram_default(); init_allocator_once(); - temptable::Allocator allocator; + temptable::Block shared_block; + temptable::Allocator allocator(&shared_block); TestItem *item = nullptr; @@ -212,7 +215,8 @@ TEST(Allocator, Casts) { using ItemType2 = int; using ItemType3 = TestItem; - temptable::Allocator allocator1; + temptable::Block shared_block; + temptable::Allocator allocator1(&shared_block); temptable::Allocator allocator2(allocator1); temptable::Allocator allocator3(allocator2); diff --git a/unittest/gunit/temptable/temptable-handler-t.cc b/unittest/gunit/temptable/temptable-handler-t.cc index 0f9bdda734ce..e40c0f1d7f0b 100644 --- a/unittest/gunit/temptable/temptable-handler-t.cc +++ b/unittest/gunit/temptable/temptable-handler-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2018, 2020, Oracle and/or its affiliates. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -54,12 +54,22 @@ as my_error generated. */ // To use a test fixture, derive a class from testing::Test. class Handler_test : public testing::Test { protected: - virtual void SetUp() { - m_server_initializer.SetUp(); + void SetUp() override { + plugin_early_load_one( + nullptr, nullptr, + nullptr); // a hack which is needed to at least get + // LOCK_plugin_xxx mutexes initialized in order make this + // test-suite up and running again. init_handlerton(); + m_server_initializer.SetUp(); } - virtual void TearDown() { m_server_initializer.TearDown(); } + void TearDown() override { + m_server_initializer.TearDown(); + delete remove_hton2plugin(m_temptable_handlerton.slot); + plugin_shutdown(); // see a comment in SetUp() for a reason why is this + // needed + } THD *thd() { return m_server_initializer.thd(); } @@ -83,6 +93,7 @@ class Handler_test : public testing::Test { HTON_NOT_USER_SELECTABLE | HTON_NO_PARTITION | HTON_NO_BINLOG_ROW_OPT | HTON_SUPPORTS_EXTENDED_KEYS; + insert_hton2plugin(m_temptable_handlerton.slot, new st_plugin_int()); temptable::Allocator::init(); } }; diff --git a/unittest/gunit/temptable_allocator-t.cc b/unittest/gunit/temptable_allocator-t.cc index 43e88c18bbd3..21b016ab4bcb 100644 --- a/unittest/gunit/temptable_allocator-t.cc +++ b/unittest/gunit/temptable_allocator-t.cc @@ -24,12 +24,11 @@ this program; if not, write to the Free Software Foundation, Inc., #include "my_config.h" #include -#include /* std::array */ -#include /* std::bad_alloc */ -#include +#include -#include "storage/temptable/include/temptable/allocator.h" /* temptable::Allocator */ -#include "storage/temptable/include/temptable/constants.h" /* temptable::ALLOCATOR_MAX_BLOCK_BYTES */ +#include "storage/temptable/include/temptable/allocator.h" +#include "storage/temptable/include/temptable/block.h" +#include "storage/temptable/include/temptable/constants.h" namespace temptable_allocator_unittest { @@ -42,175 +41,178 @@ struct MemoryMonitorReadOnlyProbe : private temptable::MemoryMonitor { }; TEST(temptable_allocator, basic) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; - - constexpr size_t n_allocate = 128; - std::array a; - constexpr size_t n_elements = 16; - - for (size_t i = 0; i < n_allocate; ++i) { - a[i] = allocator.allocate(n_elements); - - for (size_t j = 0; j < n_elements; ++j) { - a[i][j] = 0xB; - } - } - - EXPECT_FALSE(temptable::shared_block.is_empty()); - - for (size_t i = 0; i < n_allocate; ++i) { - allocator.deallocate(a[i], n_elements); - } - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); + + constexpr size_t n_allocate = 128; + std::array a; + constexpr size_t n_elements = 16; + + for (size_t i = 0; i < n_allocate; ++i) { + a[i] = allocator.allocate(n_elements); + + for (size_t j = 0; j < n_elements; ++j) { + a[i][j] = 0xB; + } + } + + EXPECT_FALSE(shared_block.is_empty()); + + for (size_t i = 0; i < n_allocate; ++i) { + allocator.deallocate(a[i], n_elements); } + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } TEST(temptable_allocator, shared_block_is_kept_after_last_deallocation) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; - - uint8_t *ptr = allocator.allocate(16); - EXPECT_FALSE(temptable::shared_block.is_empty()); - - allocator.deallocate(ptr, 16); - EXPECT_FALSE(temptable::shared_block.is_empty()); - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); - } + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); + + uint8_t *ptr = allocator.allocate(16); + EXPECT_FALSE(shared_block.is_empty()); + + allocator.deallocate(ptr, 16); + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } TEST(temptable_allocator, rightmost_chunk_deallocated_reused_for_allocation) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; - - // Allocate first Chunk which is less than the 1MB - size_t first_chunk_size = 512 * 1024; - uint8_t *first_chunk = allocator.allocate(first_chunk_size); - - // Calculate and allocate second chunk in such a way that - // it lies within the block and fills it - size_t first_chunk_actual_size = - temptable::Chunk::size_hint(first_chunk_size); - size_t space_left_in_block = - temptable::shared_block.size() - - temptable::Block::size_hint(first_chunk_actual_size); - size_t second_chunk_size = - space_left_in_block - (first_chunk_actual_size - first_chunk_size); - uint8_t *second_chunk = allocator.allocate(second_chunk_size); - - // Make sure that pointers (Chunk's) are from same blocks - EXPECT_EQ(temptable::Block(temptable::Chunk(first_chunk)), - temptable::Block(temptable::Chunk(second_chunk))); - - EXPECT_FALSE(temptable::shared_block.can_accommodate(1)); - - // Deallocate Second Chunk - allocator.deallocate(second_chunk, second_chunk_size); - - // Allocate Second Chunk again - second_chunk = allocator.allocate(second_chunk_size); - - // Make sure that pointers (Chunk's) are from same blocks - EXPECT_EQ(temptable::Block(temptable::Chunk(first_chunk)), - temptable::Block(temptable::Chunk(second_chunk))); - - // Deallocate Second Chunk - allocator.deallocate(second_chunk, second_chunk_size); - - // Deallocate First Chunk - allocator.deallocate(first_chunk, first_chunk_size); - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); - } + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); + + // Allocate first Chunk which is less than the 1MB + size_t first_chunk_size = 512 * 1024; + uint8_t *first_chunk = allocator.allocate(first_chunk_size); + + // Calculate and allocate second chunk in such a way that + // it lies within the block and fills it + size_t first_chunk_actual_size = + temptable::Chunk::size_hint(first_chunk_size); + size_t space_left_in_block = + shared_block.size() - + temptable::Block::size_hint(first_chunk_actual_size); + size_t second_chunk_size = + space_left_in_block - (first_chunk_actual_size - first_chunk_size); + uint8_t *second_chunk = allocator.allocate(second_chunk_size); + + // Make sure that pointers (Chunk's) are from same blocks + EXPECT_EQ(temptable::Block(temptable::Chunk(first_chunk)), + temptable::Block(temptable::Chunk(second_chunk))); + + EXPECT_FALSE(shared_block.can_accommodate(1)); + + // Deallocate Second Chunk + allocator.deallocate(second_chunk, second_chunk_size); + + // Allocate Second Chunk again + second_chunk = allocator.allocate(second_chunk_size); + + // Make sure that pointers (Chunk's) are from same blocks + EXPECT_EQ(temptable::Block(temptable::Chunk(first_chunk)), + temptable::Block(temptable::Chunk(second_chunk))); + + // Deallocate Second Chunk + allocator.deallocate(second_chunk, second_chunk_size); + + // Deallocate First Chunk + allocator.deallocate(first_chunk, first_chunk_size); + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } TEST(temptable_allocator, ram_consumption_is_not_monitored_for_shared_blocks) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; - - // RAM consumption is 0 at the start - EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); - - // First allocation is fed from shared-block - size_t shared_block_n_elements = 1024 * 1024; - uint8_t *shared_block = allocator.allocate(shared_block_n_elements); - EXPECT_FALSE(temptable::shared_block.is_empty()); - - // RAM consumption is still 0 - EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); - - // Deallocate the shared-block - allocator.deallocate(shared_block, shared_block_n_elements); - EXPECT_FALSE(temptable::shared_block.is_empty()); - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); - } + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); + + // RAM consumption is 0 at the start + EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); + + // First allocation is fed from shared-block + size_t shared_block_n_elements = 1024 * 1024; + uint8_t *shared_block_ptr = allocator.allocate(shared_block_n_elements); + EXPECT_FALSE(shared_block.is_empty()); + + // RAM consumption is still 0 + EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); + + // Deallocate the shared-block + allocator.deallocate(shared_block_ptr, shared_block_n_elements); + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } TEST(temptable_allocator, ram_consumption_drops_to_zero_when_non_shared_block_is_destroyed) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; - - // RAM consumption is 0 at the start - EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); - - // Make sure we fill up the shared_block first - // nr of elements must be >= 1MiB in size - size_t shared_block_n_elements = 1024 * 1024 + 256; - uint8_t *shared_block = allocator.allocate(shared_block_n_elements); - EXPECT_FALSE(temptable::shared_block.is_empty()); - - // Not even 1-byte should be able to fit anymore - EXPECT_FALSE(temptable::shared_block.can_accommodate(1)); - - // Now our next allocation should result in new block allocation ... - size_t non_shared_block_n_elements = 2 * 1024; - uint8_t *non_shared_block = - allocator.allocate(non_shared_block_n_elements); - - // Make sure that pointers (Chunk's) are from different blocks - EXPECT_NE(temptable::Block(temptable::Chunk(non_shared_block)), - temptable::Block(temptable::Chunk(shared_block))); - - // RAM consumption should be greater or equal than - // non_shared_block_n_elements bytes at this point - EXPECT_GE(MemoryMonitorReadOnlyProbe::ram_consumption(), - non_shared_block_n_elements); - - // Deallocate the non-shared block - allocator.deallocate(non_shared_block, non_shared_block_n_elements); - - // RAM consumption must drop to 0 - EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); - - // Deallocate the shared-block - allocator.deallocate(shared_block, shared_block_n_elements); - EXPECT_FALSE(temptable::shared_block.is_empty()); - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); - } + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); + + // RAM consumption is 0 at the start + EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); + + // Make sure we fill up the shared_block first + // nr of elements must be >= 1MiB in size + size_t shared_block_n_elements = 1024 * 1024 + 256; + uint8_t *shared_block_ptr = allocator.allocate(shared_block_n_elements); + EXPECT_FALSE(shared_block.is_empty()); + + // Not even 1-byte should be able to fit anymore + EXPECT_FALSE(shared_block.can_accommodate(1)); + + // Now our next allocation should result in new block allocation ... + size_t non_shared_block_n_elements = 2 * 1024; + uint8_t *non_shared_block_ptr = + allocator.allocate(non_shared_block_n_elements); + + // Make sure that pointers (Chunk's) are from different blocks + EXPECT_NE(temptable::Block(temptable::Chunk(non_shared_block_ptr)), + temptable::Block(temptable::Chunk(shared_block_ptr))); + + // RAM consumption should be greater or equal than + // non_shared_block_n_elements bytes at this point + EXPECT_GE(MemoryMonitorReadOnlyProbe::ram_consumption(), + non_shared_block_n_elements); + + // Deallocate the non-shared block + allocator.deallocate(non_shared_block_ptr, non_shared_block_n_elements); + + // RAM consumption must drop to 0 + EXPECT_EQ(MemoryMonitorReadOnlyProbe::ram_consumption(), 0); + + // Deallocate the shared-block + allocator.deallocate(shared_block_ptr, shared_block_n_elements); + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } TEST(temptable_allocator, edge) { - temptable::Allocator allocator; + temptable::Block shared_block; + temptable::Allocator allocator(&shared_block); using namespace temptable; @@ -230,30 +232,31 @@ TEST(temptable_allocator, edge) { } TEST(temptable_allocator, block_size_cap) { - { - EXPECT_TRUE(temptable::shared_block.is_empty()); - std::thread t([]() { - temptable::Allocator allocator; + temptable::Block shared_block; + EXPECT_TRUE(shared_block.is_empty()); + temptable::Allocator allocator(&shared_block); - using namespace temptable; + using namespace temptable; - constexpr size_t alloc_size = 1_MiB; - constexpr size_t n_allocate = ALLOCATOR_MAX_BLOCK_BYTES / alloc_size + 10; - std::array a; + constexpr size_t alloc_size = 1_MiB; + constexpr size_t n_allocate = ALLOCATOR_MAX_BLOCK_BYTES / alloc_size + 10; + std::array a; - for (size_t i = 0; i < n_allocate; ++i) { - a[i] = allocator.allocate(alloc_size); - } + for (size_t i = 0; i < n_allocate; ++i) { + a[i] = allocator.allocate(alloc_size); + } - EXPECT_FALSE(temptable::shared_block.is_empty()); + EXPECT_FALSE(shared_block.is_empty()); - for (size_t i = 0; i < n_allocate; ++i) { - allocator.deallocate(a[i], alloc_size); - } - }); - t.join(); - EXPECT_TRUE(temptable::shared_block.is_empty()); + for (size_t i = 0; i < n_allocate; ++i) { + allocator.deallocate(a[i], alloc_size); } + + // Physically deallocate the shared-block (allocator keeps it alive + // intentionally) + EXPECT_FALSE(shared_block.is_empty()); + shared_block.destroy(); + EXPECT_TRUE(shared_block.is_empty()); } } /* namespace temptable_allocator_unittest */ diff --git a/unittest/gunit/temptable_storage-t.cc b/unittest/gunit/temptable_storage-t.cc index 8084ba3f3524..1d19c957cf45 100644 --- a/unittest/gunit/temptable_storage-t.cc +++ b/unittest/gunit/temptable_storage-t.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, 2019, Oracle and/or its affiliates. All Rights Reserved. +/* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, as published by the @@ -26,14 +26,16 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include -#include "storage/temptable/include/temptable/allocator.h" /* temptable::Allocator */ -#include "storage/temptable/include/temptable/storage.h" /* temptable::Storage */ +#include "storage/temptable/include/temptable/allocator.h" +#include "storage/temptable/include/temptable/block.h" +#include "storage/temptable/include/temptable/storage.h" namespace temptable_storage_unittest { TEST(StorageTest, Iterate) { std::thread t([]() { - temptable::Allocator allocator; + temptable::Block shared_block; + temptable::Allocator allocator(&shared_block); temptable::Storage storage(&allocator); storage.element_size(sizeof(uint64_t)); @@ -63,7 +65,8 @@ TEST(StorageTest, AllocatorRebind) { // Bug in VS2019 error C3409 if we do the same as above. // Turns out it is the rebind which confuses the compiler. auto thread_function = []() { - temptable::Allocator alloc; + temptable::Block shared_block; + temptable::Allocator alloc(&shared_block); uint8_t *shared_eater = alloc.allocate( 1048576); // Make sure to consume the initial shared block. uint8_t *ptr = alloc.allocate(100);