From 628da714429a8c3429bdb202ab93a3d7bad8ee18 Mon Sep 17 00:00:00 2001 From: Rahul Malik Date: Thu, 15 Feb 2024 16:05:27 +0100 Subject: [PATCH] https://github.com/rahulmalik87/pstress/issues/34 Improve the grammar SQL by adding DATE, DATETIME and TIMESTAMP Also added option sleep-after-create-table to add some sleep after a create table. Also while comparing result if compare betwen engine is used adding order by ipkey --- src/common.hpp | 1 + src/grammar.sql | 11 +- src/help.cpp | 5 + src/random_test.cpp | 259 ++++++++++++++++++++++---------------------- src/random_test.hpp | 77 +++++++++++++ 5 files changed, 221 insertions(+), 132 deletions(-) diff --git a/src/common.hpp b/src/common.hpp index 52a2f01..5407058 100644 --- a/src/common.hpp +++ b/src/common.hpp @@ -163,6 +163,7 @@ struct Option { NO_DATETIME, COMPARE_RESULT, SECONDARY_ENGINE, + SLEEP_AFTER_CREATE_TABLE, MAX } option; Option(Type t, Opt o, std::string n) diff --git a/src/grammar.sql b/src/grammar.sql index a1e8e8f..2e49314 100644 --- a/src/grammar.sql +++ b/src/grammar.sql @@ -1,4 +1,9 @@ -SELECT * FROM T1 JOIN T2 ON T1_INT_1 = T2_INT_1 ORDER BY T2_VARCHAR_1 DESC, T1_VARCHAR_1 -SELECT @@VERSION +SELECT count(*) FROM T1 JOIN T2 ON T1_INT_1 = T2_INT_1 ORDER BY T2_VARCHAR_1 DESC, T1_VARCHAR_1 +SELECT count(*) FROM T1 +SELECT sum(T1_INT_1) FROM T1 +SELECT avg(T1_INT_1) FROM T1 +SELECT max(T1_INT_1) FROM T1 +SELECT min(T1_INT_1) FROM T1 SELECT * FROM T1 -SELECT NOW() +SELECT DATE_ADD(T1_DATE_1, INTERVAL 1 DAY) FROM T1 +SELECT DATE_SUB(T1_DATE_1, INTERVAL 1 DAY) FROM T1 diff --git a/src/help.cpp b/src/help.cpp index dd87efd..9778059 100644 --- a/src/help.cpp +++ b/src/help.cpp @@ -152,6 +152,11 @@ void add_options() { opt->help = "Adding Ignore clause to update delete and insert "; opt->setInt(10); + opt = newOption(Option::INT, Option::SLEEP_AFTER_CREATE_TABLE, + "sleep-after-create-table"); + opt->help = "Sleep after create table"; + opt->setInt(0); + opt = newOption(Option::INT, Option::CALL_FUNCTION, "call-function-prob"); opt->help = "Probability of calling function "; opt->setInt(10); diff --git a/src/random_test.cpp b/src/random_test.cpp index 9a923ef..2b1ba8a 100644 --- a/src/random_test.cpp +++ b/src/random_test.cpp @@ -1232,6 +1232,8 @@ bool Table::load(Thd1 *thd) { run_query_failed = true; return false; } + std::this_thread::sleep_for(std::chrono::seconds( + options->at(Option::SLEEP_AFTER_CREATE_TABLE)->getInt())); /* load default data in table */ if (!options->at(Option::JUST_LOAD_DDL)->getBool()) { @@ -2149,7 +2151,7 @@ static void compare_between_engine(const std::string &sql, Thd1 *thd) { /* Get result set with forced */ if (options->at(Option::SECONDARY_ENGINE)->getString().size() > 0) execute_sql(" SET @@SESSION.USE_SECONDARY_ENGINE=FORCED ", thd); - if (!execute_sql(sql, thd) || thd->resul == nullptr) { + if (!execute_sql(sql, thd) || thd->result == nullptr) { print_and_log("Failed to execute query with forced " + sql, thd); return set_default(); }; @@ -2158,7 +2160,7 @@ static void compare_between_engine(const std::string &sql, Thd1 *thd) { /* Get result without forced */ if (options->at(Option::SECONDARY_ENGINE)->getString().size() > 0) execute_sql(" SET @@SESSION.USE_SECONDARY_ENGINE=OFF ", thd); - if (!execute_sql(sql, thd), thd->resul == nullptr) { + if (!execute_sql(sql, thd) || thd->result == nullptr) { print_and_log("Failed to execute query wit off " + sql, thd); return set_default(); } @@ -2870,6 +2872,7 @@ void Table::SelectRandomRow(Thd1 *thd) { "SELECT " + SelectColumn() + " FROM " + name_ + GetWherePrecise(); table_mutex.unlock(); if (options->at(Option::COMPARE_RESULT)->getBool()) { + sql += " order by ipkey"; compare_between_engine(sql, thd); } else { execute_sql(sql, thd); @@ -3279,157 +3282,155 @@ void alter_tablespace_rename(Thd1 *thd) { } } -/* load special sql from a file */ -static std::vector load_grammar_sql_from() { +/* load special sql from a file and return */ +static std::vector load_grammar_sql_from() { std::vector array; auto grammar_file = opt_string(GRAMMAR_FILE); - std::string sql, file; - if (grammar_file == "grammar.sql") - file = std::string(binary_fullpath) + "/" + std::string(grammar_file); - else - file = grammar_file; - - std::ifstream myfile(file); - if (myfile.is_open()) { - while (!myfile.eof()) { - getline(myfile, sql); - /* do not process any blank lines */ - if (sql.find_first_not_of("\t\n ") != std::string::npos) - array.push_back(sql); - } - myfile.close(); - } else - throw std::runtime_error("unable to open file " + file); - return array; + { + std::string sql, file; + if (grammar_file == "grammar.sql") + file = std::string(binary_fullpath) + "/" + std::string(grammar_file); + else + file = grammar_file; + + std::ifstream myfile(file); + if (myfile.is_open()) { + while (!myfile.eof()) { + getline(myfile, sql); + /* do not process any blank lines */ + if (sql.find_first_not_of("\t\n ") != std::string::npos) + array.push_back(sql); + } + myfile.close(); + } else + throw std::runtime_error("unable to open file " + file); + } + std::vector tables; + + for (auto &sql : array) { + /* Parse the grammar SQL and create a table object */ + std::vector sql_tables; + int tab_sql = 1; // start with 1 + do { // search for table + std::smatch match; + std::string tab_p = "T" + std::to_string(tab_sql++); // table pattern + + if (regex_search(sql, match, std::regex(tab_p))) { + + auto add_columns = [&](grammar_table::sql_col_types type) { + int col_sql = 1; + do { + std::string col_p = tab_p + "_" + + grammar_table::get_col_type(type) + "_" + + std::to_string(col_sql); + if (regex_search(sql, match, std::regex(col_p))) { + sql_tables.back().column_count.at(type)++; + col_sql++; + } else { + break; + } + } while (true); + }; + + sql_tables.emplace_back(tab_p); + + for (auto &type : grammar_table::get_vector_of_col_type()) { + add_columns(type); + } + } else + // if no more table found, + break; + } while (true); + tables.emplace_back(sql, sql_tables); + } + return tables; } /* return preformatted sql */ static void grammar_sql(std::vector *all_tables, Thd1 *thd) { - static std::vector all_sql = load_grammar_sql_from(); - enum sql_col_types { INT, VARCHAR }; + static auto all_tables_from_grammar = load_grammar_sql_from(); - if (all_sql.size() == 0) + if (all_tables_from_grammar.size() == 0) return; - struct table { - table(std::string n, std::vector i, std::vector v) - : name(n), int_col(i), varchar_col(v){}; - std::string name; - std::vector int_col; - std::vector varchar_col; - }; - - auto sql = all_sql[rand_int(all_sql.size() - 1)]; - - /* parse SQL in table */ - std::vector> sql_tables; - - int tab_sql = 1; // number of tables in sql - bool table_found; + auto currrent_table = + all_tables_from_grammar.at(rand_int(all_tables_from_grammar.size() - 1)); - do { // search for table - std::smatch match; - std::string tab_p = "T" + std::to_string(tab_sql); // table pattern + auto sql = currrent_table.sql; + auto &sql_tables = currrent_table.tables; - if (regex_search(sql, match, std::regex(tab_p))) { - table_found = true; - sql_tables.push_back({0, 0}); - - int col_sql = 1; - bool column_found; - - do { // search of int column - std::string col_p = tab_p + "_INT_" + std::to_string(col_sql); - if (regex_search(sql, match, std::regex(col_p))) { - column_found = true; - sql_tables.at(tab_sql - 1).at(INT)++; - col_sql++; - } else - column_found = false; - } while (column_found); + // Find the real table and columns + for (auto &table : sql_tables) { + int table_check = 100; // try to find table + do { + auto working_table = all_tables->at(rand_int(all_tables->size() - 1)); + working_table->table_mutex.lock(); + table.found_name = working_table->name_; - col_sql = 1; + auto columns = working_table->columns_; + int column_check = 20; // max number of times to find column do { - std::string col_p = tab_p + "_VARCHAR_" + std::to_string(col_sql); - if (regex_search(sql, match, std::regex(col_p))) { - column_found = true; - sql_tables.at(tab_sql - 1).at(VARCHAR)++; - col_sql++; - } else - column_found = false; - } while (column_found); - } else - table_found = false; - tab_sql++; - } while (table_found); + auto col = columns->at(rand_int(columns->size() - 1)); + auto col_type = + grammar_table::get_col_type(col->col_type_to_string(col->type_)); - std::vector
final_tables; + // if a valid column is not found in the table + if (col_type == grammar_table::MAX) + continue; - /* try at max 100 times */ - int table_check = 100; + if (table.column_count.at(col_type) > 0 && + table.column_count.at(col_type) != + (int)table.columns.at(col_type).size()) { + table.columns.at(col_type).emplace_back(col->name_); + } + } while (column_check-- > 0 && + table.total_column_count() != table.total_column_written()); - while (sql_tables.size() > 0 && table_check-- > 0) { + working_table->table_mutex.unlock(); - auto int_columns = sql_tables.back().at(INT); - auto varchar_columns = sql_tables.back().at(VARCHAR); - std::vector int_cols_str, var_cols_str; - int column_check = 20; - auto table = all_tables->at(rand_int(all_tables->size() - 1)); - table->table_mutex.lock(); - auto columns = table->columns_; + if (table.total_column_count() != table.total_column_written()) + table.reset_columns(); - // find columns in table // - do { - auto col = columns->at(rand_int(columns->size() - 1)); + } while (table.total_column_count() != table.total_column_written() && + table_check-- > 0); - if (int_columns > 0 && col->type_ == Column::INT) { - int_cols_str.push_back(col->name_); - int_columns--; - } - if (varchar_columns > 0 && col->type_ == Column::VARCHAR) { - var_cols_str.push_back(col->name_); - varchar_columns--; - } + if (table.total_column_count() != table.total_column_written()) { + std::cout << "NOT ABLE TO FIND any table for SQL in 100 iteration " << sql + << std::endl; + return; + } + } - if (int_columns == 0 && varchar_columns == 0) { - final_tables.emplace_back(table->name_, int_cols_str, var_cols_str); - sql_tables.pop_back(); + /* replace the found column and table */ + for (const auto &table : sql_tables) { + auto table_name = table.name; + for (size_t i = 0; i < table.columns.size(); i++) { + auto col = table.columns.at(i); + for (size_t j = 0; j < col.size(); j++) { + sql = + std::regex_replace(sql, + std::regex(table_name + "_" + + grammar_table::get_col_type( + (grammar_table::sql_col_types)i) + + "_" + std::to_string(j + 1)), + table_name + "." + col.at(j)); } - } while (!(int_columns == 0 && varchar_columns == 0) && column_check-- > 0); - - table->table_mutex.unlock(); + } + /* replace table "T1 " => tt_N T1 */ + sql = std::regex_replace(sql, std::regex(table_name + " "), + table.found_name + " " + table.name + " "); + /* replace table "T1$" => tt_N T1*/ + sql = std::regex_replace(sql, std::regex(table_name + "$"), + table.found_name + " " + table.name + ""); } - - if (sql_tables.size() == 0) { - - for (size_t i = 0; i < final_tables.size(); i++) { - auto table = final_tables.at(i); - auto table_name = "T" + std::to_string(i + 1); - - /* replace int column */ - for (size_t j = 0; j < table.int_col.size(); j++) - sql = std::regex_replace( - sql, std::regex(table_name + "_INT_" + std::to_string(j + 1)), - table_name + "." + table.int_col.at(j)); - - /* replace varchar column */ - for (size_t j = 0; j < table.varchar_col.size(); j++) - sql = std::regex_replace( - sql, std::regex(table_name + "_VARCHAR_" + std::to_string(j + 1)), - table_name + "." + table.varchar_col.at(j)); - - /* replace table "T1 " => tt_N T1 */ - sql = std::regex_replace(sql, std::regex(table_name + " "), - table.name + " " + table_name + " "); - /* replace table "T1$" => tt_N T1*/ - sql = std::regex_replace(sql, std::regex(table_name + "$"), - table.name + " " + table_name + ""); + if (options->at(Option::COMPARE_RESULT)->getBool()) { + compare_between_engine(sql, thd); + } else { + if (!execute_sql(sql, thd)) { + print_and_log("Grammar SQL failed " + sql, thd); } - - execute_sql(sql, thd); - } else - std::cout << "NOT ABLE TO FIND any SQL in special SQL" << std::endl; + } } /* save metadata to a file */ diff --git a/src/random_test.hpp b/src/random_test.hpp index c5988cb..f8dc626 100644 --- a/src/random_test.hpp +++ b/src/random_test.hpp @@ -473,4 +473,81 @@ void alter_database_encryption(Thd1 *thd); void create_in_memory_data(); void generate_metadata_for_tables(); void create_database_tablespace(Thd1 *thd); +/* Grammar table class used for parsing the grammar file */ +struct grammar_table { + grammar_table(std::string n) + : name(n), column_count(grammar_table::MAX, 0), + columns(grammar_table::MAX) {} + std::string name; + std::string found_name; + std::vector column_count; + std::vector> columns; + bool table_found = false; + enum sql_col_types { INT, VARCHAR, DATETIME, DATE, TIMESTAMP, MAX }; + static sql_col_types get_col_type(std::string type) { + if (type == "INT") + return INT; + if (type == "VARCHAR") + return VARCHAR; + if (type == "DATETIME") + return DATETIME; + if (type == "DATE") + return DATE; + if (type == "TIMESTAMP") + return TIMESTAMP; + return MAX; + } + static std::string get_col_type(sql_col_types type) { + switch (type) { + case INT: + return "INT"; + case VARCHAR: + return "VARCHAR"; + case DATETIME: + return "DATETIME"; + case DATE: + return "DATE"; + case TIMESTAMP: + return "TIMESTAMP"; + case MAX: + break; + } + return ""; + } + int total_column_count() { + int total = 0; + for (auto &i : column_count) + total += i; + return total; + } + int total_column_written() { + int total = 0; + for (auto &i : columns) + total += i.size(); + return total; + } + void reset_columns() { + for (auto &i : columns) { + i.clear(); + } + found_name = ""; + } + + static std::vector get_vector_of_col_type() { + std::vector v; + v.push_back(INT); + v.push_back(VARCHAR); + v.push_back(DATETIME); + v.push_back(DATE); + v.push_back(TIMESTAMP); + return v; + } +}; +struct grammar_tables { + /* use tables move constructor */ + grammar_tables(std::string sql_, std::vector tables_) + : sql(sql_), tables(tables_){}; + std::string sql; + std::vector tables; +}; #endif