test_omnisci.py::test_binding triggers deadlock in omniscidb

[pearu]

omniscidb built with tsan reports:

==================
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=28479)
  Cycle in lock order graph: M473253342966390272 (0x000000000000) => M472971867989679264 (0x000000000000) => M473253342966390272

  Mutex M472971867989679264 acquired here while holding mutex M473253342966390272 in thread T5:
    #0 pthread_rwlock_rdlock ../../../../libsanitizer/tsan/tsan_interceptors.cc:1294 (libtsan.so.0+0x2ed2a)
    #1 std::shared_lock<std::shared_mutex>::shared_lock(std::shared_mutex&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/shared_mutex:71 (omnisci_server+0x16f81cf)
    #2 Executor::get_rt_udf_module(bool) const /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/Execute.h:469 (omnisci_server+0x16f81cf)
    #3 Executor::compileWorkUnit(std::vector<InputTableInfo, std::allocator<InputTableInfo> > const&, std::unordered_map<int, ColumnDescriptor const*, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, ColumnDescriptor const*> > > const&, RelAlgExecutionUnit const&, CompilationOptions const&, ExecutionOptions const&, CudaMgr_Namespace::CudaMgr const*, bool, std::shared_ptr<RowSetMemoryOwner>, unsigned long, signed char, bool, std::unordered_map<int, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > >, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > > > > >&, RenderInfo*) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/NativeCodegen.cpp:2671 (omnisci_server+0x16f81cf)
    #4 QueryCompilationDescriptor::compile(unsigned long, signed char, bool, RelAlgExecutionUnit const&, std::vector<InputTableInfo, std::allocator<InputTableInfo> > const&, std::unordered_map<int, ColumnDescriptor const*, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, ColumnDescriptor const*> > > const&, ColumnFetcher const&, CompilationOptions const&, ExecutionOptions const&, RenderInfo*, Executor*) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/Descriptors/QueryCompilationDescriptor.cpp:39 (omnisci_server+0x1baecd2)
    #5 Executor::executeWorkUnitImpl(unsigned long&, bool, bool, std::vector<InputTableInfo, std::allocator<InputTableInfo> > const&, RelAlgExecutionUnit const&, CompilationOptions const&, ExecutionOptions const&, Catalog_Namespace::Catalog const&, std::shared_ptr<RowSetMemoryOwner>, RenderInfo*, bool, std::unordered_map<int, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > >, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > > > > >&) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/Execute.cpp:1820 (omnisci_server+0x1552bd1)
    #6 Executor::executeWorkUnit(unsigned long&, bool, std::vector<InputTableInfo, std::allocator<InputTableInfo> > const&, RelAlgExecutionUnit const&, CompilationOptions const&, ExecutionOptions const&, Catalog_Namespace::Catalog const&, RenderInfo*, bool, std::unordered_map<int, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > >, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::unordered_map<int, std::shared_ptr<ColumnarResults const>, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::shared_ptr<ColumnarResults const> > > > > > >&) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/Execute.cpp:1746 (omnisci_server+0x1554409)
    #7 operator()<long unsigned int> /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:3361 (omnisci_server+0x17eb564)
    #8 RelAlgExecutor::executeWorkUnit(RelAlgExecutor::WorkUnit const&, std::vector<TargetMetaInfo, std::allocator<TargetMetaInfo> > const&, bool, CompilationOptions const&, ExecutionOptions const&, RenderInfo*, long, std::optional<unsigned long>) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:3399 (omnisci_server+0x1829a20)
    #9 RelAlgExecutor::executeProject(RelProject const*, CompilationOptions const&, ExecutionOptions const&, RenderInfo*, long, std::optional<unsigned long>) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:2046 (omnisci_server+0x182cfdd)
    #10 RelAlgExecutor::executeRelAlgStep(RaExecutionSequence const&, unsigned long, CompilationOptions const&, ExecutionOptions const&, RenderInfo*, long) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:990 (omnisci_server+0x182ef15)
    #11 RelAlgExecutor::executeRelAlgSeq(RaExecutionSequence const&, CompilationOptions const&, ExecutionOptions const&, RenderInfo*, long, bool) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:724 (omnisci_server+0x183186e)
    #12 RelAlgExecutor::executeRelAlgQueryNoRetry(CompilationOptions const&, ExecutionOptions const&, bool, RenderInfo*) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:520 (omnisci_server+0x18337ee)
    #13 RelAlgExecutor::executeRelAlgQuery(CompilationOptions const&, ExecutionOptions const&, bool, RenderInfo*) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/RelAlgExecutor.cpp:339 (omnisci_server+0x183556e)
    #14 operator() /home/pearu/git/omnisci-work/cgenstate-module-followup/ThriftHandler/DBHandler.cpp:6347 (omnisci_server+0xfe56f3)
    #15 execution<DBHandler::execute_rel_alg(ExecutionResult&, query_state::QueryStateProxy, const string&, bool, ExecutorDeviceType, int32_t, int32_t, bool, bool, const ExplainInfo&, std::optional<long unsigned int>) const::<lambda()> > /home/pearu/git/omnisci-work/cgenstate-module-followup/Shared/measure.h:34 (omnisci_server+0xfe56f3)
    #16 DBHandler::execute_rel_alg(ExecutionResult&, query_state::QueryStateProxy, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, ExecutorDeviceType, int, int, bool, bool, ExplainInfo const&, std::optional<unsigned long>) const /home/pearu/git/omnisci-work/cgenstate-module-followup/ThriftHandler/DBHandler.cpp:6346 (omnisci_server+0xfe56f3)
    #17 operator() /home/pearu/git/omnisci-work/cgenstate-module-followup/ThriftHandler/DBHandler.cpp:6780 (omnisci_server+0x1055dc0)
    #18 __invoke_impl<void, DBHandler::sql_execute_impl(ExecutionResult&, query_state::QueryStateProxy, bool, ExecutorDeviceType, int32_t, int32_t, bool)::<lambda(size_t)>&, long unsigned int> /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:60 (omnisci_server+0x1056b4e)
    #19 __invoke<DBHandler::sql_execute_impl(ExecutionResult&, query_state::QueryStateProxy, bool, ExecutorDeviceType, int32_t, int32_t, bool)::<lambda(size_t)>&, long unsigned int> /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:95 (omnisci_server+0x1056b4e)
    #20 operator() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:1421 (omnisci_server+0x1056b4e)
    #21 operator() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:1362 (omnisci_server+0x1056b4e)
    #22 _M_invoke /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/std_function.h:286 (omnisci_server+0x1056b4e)
    #23 std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>::operator()() const /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/std_function.h:688 (omnisci_server+0xf6b85e)
    #24 std::__future_base::_State_baseV2::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:561 (omnisci_server+0xf6b85e)
    #25 void std::__invoke_impl<void, void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::__invoke_memfun_deref, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:73 (omnisci_server+0xf674aa)
    #26 std::__invoke_result<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>::type std::__invoke<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:95 (omnisci_server+0xf674aa)
    #27 std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#1}::operator()() const /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/mutex:671 (omnisci_server+0xf674aa)
    #28 std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::operator()() const /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/mutex:676 (omnisci_server+0xf674aa)
    #29 std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&)::{lambda()#2}::_FUN() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/mutex:676 (omnisci_server+0xf674aa)
    #30 pthread_once ../../../../libsanitizer/tsan/tsan_interceptors.cc:1405 (libtsan.so.0+0x2f5a0)
    #31 __gthread_once /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/x86_64-conda-linux-gnu/bits/gthr-default.h:700 (omnisci_server+0xf7bba7)
    #32 void std::call_once<void (std::__future_base::_State_baseV2::*)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*>(std::once_flag&, void (std::__future_base::_State_baseV2::*&&)(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*, bool*), std::__future_base::_State_baseV2*&&, std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>*&&, bool*&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/mutex:683 (omnisci_server+0xf7bba7)
    #33 std::__future_base::_State_baseV2::_M_set_result(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>, bool) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:401 (omnisci_server+0xf7bd5e)
    #34 _M_run /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:1423 (omnisci_server+0xf7bd5e)
    #35 std::packaged_task<void (unsigned long)>::operator()(unsigned long) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/future:1551 (omnisci_server+0xfc2653)
    #36 QueryDispatchQueue::worker(unsigned long) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/QueryDispatchQueue.h:107 (omnisci_server+0xfc2653)
    #37 void std::__invoke_impl<void, void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long>(std::__invoke_memfun_deref, void (QueryDispatchQueue::*&&)(unsigned long), QueryDispatchQueue*&&, unsigned long&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:73 (omnisci_server+0xf6ce28)
    #38 std::__invoke_result<void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long>::type std::__invoke<void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long>(void (QueryDispatchQueue::*&&)(unsigned long), QueryDispatchQueue*&&, unsigned long&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:95 (omnisci_server+0xf6ce28)
    #39 void std::thread::_Invoker<std::tuple<void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long> >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:244 (omnisci_server+0xf6ce28)
    #40 std::thread::_Invoker<std::tuple<void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long> >::operator()() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:251 (omnisci_server+0xf6ce28)
    #41 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (QueryDispatchQueue::*)(unsigned long), QueryDispatchQueue*, unsigned long> > >::_M_run() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:195 (omnisci_server+0xf6ce28)
    #42  <null> (libstdc++.so.6+0xcc9d3)

    Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative warning message

  Mutex M473253342966390272 acquired here while holding mutex M472971867989679264 in thread T5746:
    #0 pthread_rwlock_wrlock ../../../../libsanitizer/tsan/tsan_interceptors.cc:1324 (libtsan.so.0+0x2ef9b)
    #1 __glibcxx_rwlock_wrlock /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/shared_mutex:73 (omnisci_server+0x102b9df)
    #2 std::__shared_mutex_pthread::lock() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/shared_mutex:186 (omnisci_server+0x102b9df)
    #3 std::shared_timed_mutex::lock() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/shared_mutex:459 (omnisci_server+0x102b9df)
    #4 std::unique_lock<std::shared_timed_mutex>::lock() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/unique_lock.h:141 (omnisci_server+0x102b9df)
    #5 std::unique_lock<std::shared_timed_mutex>::unique_lock(std::shared_timed_mutex&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/unique_lock.h:71 (omnisci_server+0x102b9df)
    #6 Executor::resetAllExecutors(bool) /home/pearu/git/omnisci-work/cgenstate-module-followup/QueryEngine/Execute.h:421 (omnisci_server+0x102b9df)
    #7 DBHandler::register_runtime_extension_functions(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<TUserDefinedFunction, std::allocator<TUserDefinedFunction> > const&, std::vector<TUserDefinedTableFunction, std::allocator<TUserDefinedTableFunction> > const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > const&) /home/pearu/git/omnisci-work/cgenstate-module-followup/ThriftHandler/DBHandler.cpp:7874 (omnisci_server+0x102b9df)
    #8 OmniSciProcessor::process_register_runtime_extension_functions(int, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, void*) /home/pearu/git/omnisci-work/cgenstate-module-followup/build-tsan/gen-cpp/OmniSci.cpp:35483 (omnisci_server+0xe94810)
    #9 OmniSciProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int, void*) /home/pearu/git/omnisci-work/cgenstate-module-followup/build-tsan/gen-cpp/OmniSci.cpp:29975 (omnisci_server+0xe8aefe)
    #10 apache::thrift::TDispatchProcessor::process(std::shared_ptr<apache::thrift::protocol::TProtocol>, std::shared_ptr<apache::thrift::protocol::TProtocol>, void*) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/include/thrift/TDispatchProcessor.h:121 (omnisci_server+0xca9e52)
    #11 TrackingProcessor::process(std::shared_ptr<apache::thrift::protocol::TProtocol>, std::shared_ptr<apache::thrift::protocol::TProtocol>, void*) /home/pearu/git/omnisci-work/cgenstate-module-followup/ThriftHandler/DBHandler.h:146 (omnisci_server+0xca9e52)
    #12 apache::thrift::server::TConnectedClient::run() /usr/local/src/conda/thrift-split-0.15.0/lib/cpp/src/thrift/server/TConnectedClient.cpp:61 (libthrift.so.0.15.0+0x6f530)

  Thread T5 (tid=28485, running) created by main thread at:
    #0 pthread_create ../../../../libsanitizer/tsan/tsan_interceptors.cc:964 (libtsan.so.0+0x2ddac)
    #1  <null> (libstdc++.so.6+0xcc92e)
    #2 std::shared_ptr<DBHandler>::shared_ptr<std::allocator<DBHandler>, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool>(std::_Sp_alloc_shared_tag<std::allocator<DBHandler> >, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/ext/new_allocator.h:146 (omnisci_server+0xcad7a4)
    #3 std::shared_ptr<DBHandler> std::allocate_shared<DBHandler, std::allocator<DBHandler>, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool>(std::allocator<DBHandler> const&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/shared_ptr.h:702 (omnisci_server+0xcad7a4)
    #4 std::shared_ptr<DBHandler> std::make_shared<DBHandler, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool>(std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::vector<LeafHostInfo, std::allocator<LeafHostInfo> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, bool&, unsigned int&, bool&, int&, unsigned long&, unsigned long&, unsigned long&, bool&, unsigned long&, AuthMetadata&, SystemParameters&, bool&, Licensing::License&, int&, int&, bool&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, File_Namespace::DiskCacheConfig&, bool&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/shared_ptr.h:718 (omnisci_server+0xcad7a4)
    #5 startMapdServer(CommandLineOptions&, bool) /home/pearu/git/omnisci-work/cgenstate-module-followup/MapDServer.cpp:503 (omnisci_server+0xcad7a4)
    #6 main /home/pearu/git/omnisci-work/cgenstate-module-followup/MapDServer.cpp:826 (omnisci_server+0xbced03)

  Thread T27 (tid=35477, running) created by thread T17 at:
    #0 pthread_create ../../../../libsanitizer/tsan/tsan_interceptors.cc:964 (libtsan.so.0+0x2ddac)
    #1  <null> (libstdc++.so.6+0xcc92e)
    #2 void std::__invoke_impl<void, void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int>(std::__invoke_other, void (*&&)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>&&, int&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:60 (omnisci_server+0xc90651)
    #3 std::__invoke_result<void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int>::type std::__invoke<void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int>(void (*&&)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>&&, int&&) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/bits/invoke.h:95 (omnisci_server+0xc90651)
    #4 void std::thread::_Invoker<std::tuple<void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int> >::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:244 (omnisci_server+0xc90651)
    #5 std::thread::_Invoker<std::tuple<void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int> >::operator()() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:251 (omnisci_server+0xc90651)
    #6 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(std::shared_ptr<apache::thrift::server::TThreadedServer>, int), std::shared_ptr<apache::thrift::server::TThreadedServer>, int> > >::_M_run() /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/thread:195 (omnisci_server+0xc90651)
    #7  <null> (libstdc++.so.6+0xcc9d3)

SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) /home/pearu/miniconda3/envs/omniscidb-llvm9-dev/x86_64-conda-linux-gnu/include/c++/9.4.0/shared_mutex:71 in std::shared_lock<std::shared_mutex>::shared_lock(std::shared_mutex&)

Using the current omniscidb-internal master.