From 52c5596138e439d4c0404288db666063a393b61a Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 31 Dec 2024 08:04:15 +0200 Subject: [PATCH] MDEV-35710 support for threadpool When client connections use threadpool, i.e. configuration has: thread_handling = pool-of-threads it turned out that during wsrep replication shutdown, not all client connections could be closed. Reason was that some client threads has stmt_da in state DA_EOF, and this state was earlier used to detect if client connection was issuing SHUTDOWN command. To fix this, the connection executing SHUTDOWN is now detected by looking at the actual command being executed: thd->get_command() == COM_SHUTDOWN During replication shutdown, all other connections but the SHUTDOWN executor, are terminated. This commit has new mtr test galera.galera_threadpool, which opens a number of threadpool client connections, and then restarts the node to verify that connections in threadpool are terminated during shutdown. --- .../suite/galera/r/galera_threadpool.result | 36 ++++++++++ .../suite/galera/t/galera_threadpool.cnf | 17 +++++ .../suite/galera/t/galera_threadpool.test | 67 +++++++++++++++++++ sql/wsrep_mysqld.cc | 19 ++++-- 4 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_threadpool.result create mode 100644 mysql-test/suite/galera/t/galera_threadpool.cnf create mode 100644 mysql-test/suite/galera/t/galera_threadpool.test diff --git a/mysql-test/suite/galera/r/galera_threadpool.result b/mysql-test/suite/galera/r/galera_threadpool.result new file mode 100644 index 0000000000000..e89e243621abe --- /dev/null +++ b/mysql-test/suite/galera/r/galera_threadpool.result @@ -0,0 +1,36 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_2c, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_2d, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_2e, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_2a; +CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY, f2 char) ENGINE=InnoDB; +INSERT INTO t1 VALUES (0,'a'); +INSERT INTO t1 VALUES (1,'a'); +connection node_2b; +SELECT * FROM t1; +f1 f2 +0 a +1 a +connection node_2c; +INSERT INTO t1 VALUES (2,'c'); +connection node_2d; +BEGIN; +SELECT * FROM t1; +f1 f2 +0 a +1 a +2 c +connection node_2e; +BEGIN; +UPDATE t1 SET f2='e' WHERE f1=0; +connection node_2; +connection node_1; +connection node_2; +connection node_1; +connection node_2; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_threadpool.cnf b/mysql-test/suite/galera/t/galera_threadpool.cnf new file mode 100644 index 0000000000000..c1a1e6a81aac9 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_threadpool.cnf @@ -0,0 +1,17 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +wsrep-node-name="node1" +log_bin=binlog +log_slave_updates=ON +wsrep_sst_method=rsync +thread_handling = pool-of-threads + +[mysqld.2] +wsrep-node-name="node2" +log_bin=binlog +log_slave_updates=ON +wsrep_sst_method=rsync +thread_handling = pool-of-threads + + diff --git a/mysql-test/suite/galera/t/galera_threadpool.test b/mysql-test/suite/galera/t/galera_threadpool.test new file mode 100644 index 0000000000000..78b26e6e89fa6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_threadpool.test @@ -0,0 +1,67 @@ +# +# Tests for threadpool support +# +--source include/galera_cluster.inc + +--let $node_1 = node_1 +--let $node_2 = node_2 + +--source ../galera/include/auto_increment_offset_save.inc + +# +# start connections in node 2, and execute some SQL statements +# leave also open transactions in the node +# +--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connect node_2c, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connect node_2d, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connect node_2e, 127.0.0.1, root, , test, $NODE_MYPORT_2 + +--connection node_2a +CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY, f2 char) ENGINE=InnoDB; +INSERT INTO t1 VALUES (0,'a'); +INSERT INTO t1 VALUES (1,'a'); + +--connection node_2b +SELECT * FROM t1; + +--connection node_2c +INSERT INTO t1 VALUES (2,'c'); + +--connection node_2d +BEGIN; +SELECT * FROM t1; + +--connection node_2e +BEGIN; +UPDATE t1 SET f2='e' WHERE f1=0; + +# +# Shut down node 2, all open connections should be closed +# +--connection node_2 +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +# And restart the node +--connection node_2 +let $restart_noprint=2; +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--source include/start_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--connection node_2 + +DROP TABLE t1; + +# +# Restore auto increment variables. +# +--source ../galera/include/auto_increment_offset_restore.inc diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 6c4fd8c950a16..dc9dbeacd2acb 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -2857,8 +2857,6 @@ static inline bool is_committing_connection(THD *thd) static my_bool have_client_connections(THD *thd, void*) { - DBUG_PRINT("quit",("Informing thread %lld that it's time to die", - (longlong) thd->thread_id)); if (is_client_connection(thd)) { if (thd->killed == KILL_CONNECTION || @@ -2912,13 +2910,18 @@ static my_bool kill_all_threads(THD *thd, THD *caller_thd) /* We skip slave threads & scheduler on this first loop through. */ if (is_client_connection(thd) && thd != caller_thd) { - if (thd->get_stmt_da()->is_eof()) + /* the connection executing SHUTDOWN, should do clean exit, + not aborting here */ + if (thd->get_command() == COM_SHUTDOWN) { + WSREP_DEBUG("leaving SHUTDOWN executing connection alive, thread: %lld", + (longlong) thd->thread_id); return 0; } - + /* replaying connection is killed by signal */ if (is_replaying_connection(thd)) { + WSREP_DEBUG("closing connection is replaying %lld", (longlong) thd->thread_id); thd->set_killed(KILL_CONNECTION_HARD); return 0; } @@ -2927,7 +2930,7 @@ static my_bool kill_all_threads(THD *thd, THD *caller_thd) { /* replicated transactions must be skipped */ WSREP_DEBUG("closing connection %lld", (longlong) thd->thread_id); - /* instead of wsrep_close_thread() we do now soft kill by THD::awake */ + /* instead of wsrep_close_thread() we do now hard kill by THD::awake */ thd->awake(KILL_CONNECTION_HARD); return 0; } @@ -2968,8 +2971,10 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD* except_caller_thd) */ server_threads.iterate(kill_remaining_threads, except_caller_thd); - DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)", THD_count::value())); - WSREP_DEBUG("waiting for client connections to close: %u", THD_count::value()); + DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)", + THD_count::value())); + WSREP_DEBUG("waiting for client connections to close: %u", + THD_count::value()); while (wait_to_end && server_threads.iterate(have_client_connections)) {