Merge pull request #4216 from sysown/v2.x-track_slow_query_vars

Track some slow query variables

include/proxysql_structs.h

@@ -195,6 +195,7 @@ enum mysql_variable_name {
  SQL_LC_MESSAGES,
  SQL_LC_TIME_NAMES,
  SQL_LOCK_WAIT_TIMEOUT,
+ SQL_LOG_SLOW_FILTER,
  SQL_LONG_QUERY_TIME,
  SQL_MAX_EXECUTION_TIME,
  SQL_MAX_HEAP_TABLE_SIZE,
@@ -213,6 +214,7 @@ enum mysql_variable_name {
  SQL_GENERATE_INVISIBLE_PRIMARY_KEY,
  SQL_SQL_LOG_BIN,
  SQL_SQL_MODE,
+ SQL_QUOTE_SHOW_CREATE,
  SQL_REQUIRE_PRIMARY_KEY,
  SQL_SQL_SAFE_UPDATES,
  SQL_SQL_SELECT_LIMIT,
@@ -549,7 +551,7 @@ struct __SQP_query_parser_t {
 struct _PtrSize_t {
  unsigned int size;
  void *ptr;
-}; 
+};
 // struct for debugging module
 #ifdef DEBUG
 struct _debug_level {
@@ -600,7 +602,7 @@ struct mysql_protocol_events {
 
 // this struct define global variable entries, and how these are configured during startup
 struct _global_variable_entry_t {
- const char *group_name; // [group name] in proxysql.cnf 
+ const char *group_name; // [group name] in proxysql.cnf
  const char *key_name; // key name
  int dynamic; // if dynamic > 0 , reconfigurable
  //GOptionArg arg; // type of variable
@@ -762,7 +764,7 @@ EXTERN global_variables glovars;
 #ifndef GLOBAL_DEFINED_OPTS_ENTRIES
 #define GLOBAL_DEFINED_OPTS_ENTRIES
 ProxySQL_GlobalVariables GloVars {};
-#endif // GLOBAL_DEFINED_OPTS_ENTRIES 
+#endif // GLOBAL_DEFINED_OPTS_ENTRIES
 #ifndef GLOBAL_DEFINED_HOSTGROUP
 #define GLOBAL_DEFINED_HOSTGROUP
 MySQL_HostGroups_Manager *MyHGM;
@@ -1155,6 +1157,10 @@ mysql_variable_st mysql_tracked_variables[] {
  { SQL_LC_MESSAGES, SETTING_VARIABLE, true, false, false, false, (char *)"lc_messages", NULL, (char *)"" , false} ,
  { SQL_LC_TIME_NAMES, SETTING_VARIABLE, true, false, false, false, (char *)"lc_time_names", NULL, (char *)"" , false} ,
  { SQL_LOCK_WAIT_TIMEOUT, SETTING_VARIABLE, false, false, true, false, (char *)"lock_wait_timeout", NULL, (char *)"" , false} ,
+// log_queries_not_using_indexes is not enabled because in MySQL it is *only* a global variable, while in MariaDB is a global *and* session variable .
+// We believe it is not the time to create a lot of exceptions and complex logic for conflicting backend implementations
+// { SQL_LOG_QUERIES_NOT_USING_INDEXES, SETTING_VARIABLE, false, false, false, true, (char *)"log_queries_not_using_indexes", NULL, (char *)"OFF" , false} ,
+ { SQL_LOG_SLOW_FILTER, SETTING_VARIABLE, true, false, false, false, (char *)"log_slow_filter", NULL, (char *)"" , false} ,
  { SQL_LONG_QUERY_TIME, SETTING_VARIABLE, false, false, true, false, (char *)"long_query_time", NULL, (char *)"" , false} ,
  { SQL_MAX_EXECUTION_TIME, SETTING_VARIABLE, false, false, true, false, (char *)"max_execution_time", NULL, (char *)"" , false} ,
  { SQL_MAX_HEAP_TABLE_SIZE, SETTING_VARIABLE, false, false, true, false, (char *)"max_heap_table_size", NULL, (char *)"18446744073709547520" , false} ,
@@ -1173,7 +1179,8 @@ mysql_variable_st mysql_tracked_variables[] {
  { SQL_GENERATE_INVISIBLE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_generate_invisible_primary_key", NULL, (char *)"" , false} ,
  { SQL_SQL_LOG_BIN, SETTING_VARIABLE, false, false, false, true, (char *)"sql_log_bin", NULL, (char *)"ON" , false} ,
  { SQL_SQL_MODE, SETTING_VARIABLE, true, false, false, false, (char *)"sql_mode" , NULL, (char *)"" , false} ,
- { SQL_REQUIRE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_require_primary_key", NULL, (char *)"" , false} ,
+ { SQL_QUOTE_SHOW_CREATE, SETTING_VARIABLE, false, false, false, true, (char *)"sql_quote_show_create", NULL, (char *)"" , false} ,
+ { SQL_REQUIRE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_require_primary_key", NULL, (char *)"" , false} ,
  { SQL_SQL_SAFE_UPDATES, SETTING_VARIABLE, true, false, false, true, (char *)"sql_safe_updates", NULL, (char *)"OFF" , false} ,
  { SQL_SQL_SELECT_LIMIT, SETTING_VARIABLE, false, false, true, false, (char *)"sql_select_limit", NULL, (char *)"DEFAULT" , false} ,
  { SQL_TIME_ZONE, SETTING_VARIABLE, true, false, false, false, (char *)"time_zone", NULL, (char *)"SYSTEM" , false} ,

lib/MySQL_Session.cpp

@@ -80,6 +80,7 @@ static const std::set<std::string> mysql_variables_boolean = {
  "sql_big_selects",
  "sql_generate_invisible_primary_key",
  "sql_log_bin",
+ "sql_quote_show_create",
  "sql_require_primary_key",
  "sql_safe_updates",
  "unique_checks",
@@ -114,6 +115,7 @@ static const std::set<std::string> mysql_variables_strings = {
  "group_replication_consistency",
  "lc_messages",
  "lc_time_names",
+ "log_slow_filter",
  "optimizer_switch",
  "wsrep_osu_method",
 };
@@ -209,10 +211,10 @@ KillArgs::~KillArgs() {
 
 const char* KillArgs::get_host_address() const {
  const char* host_address = hostname;
- 
+
  if (ip_addr)
  host_address = ip_addr;
- 
+
  return host_address;
 }
 
@@ -812,7 +814,7 @@ void MySQL_Session::writeout() {
  if (client_myds) client_myds->array2buffer_full();
  if (mybe && mybe->server_myds && mybe->server_myds->myds_type==MYDS_BACKEND) {
  if (session_type==PROXYSQL_SESSION_MYSQL) {
- if (mybe->server_myds->net_failure==false) { 
+ if (mybe->server_myds->net_failure==false) {
  if (mybe->server_myds->poll_fds_idx>-1) { // NOTE: attempt to force writes
  mybe->server_myds->array2buffer_full();
  }
@@ -2819,7 +2821,7 @@ bool MySQL_Session::handler_again___status_CHANGING_SCHEMA(int *_rc) {
 }
 
 
-bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { 
+bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
  //fprintf(stderr,"CONNECTING_SERVER\n");
  unsigned long long curtime=monotonic_time();
  thread->atomic_curtime=curtime;
@@ -2880,7 +2882,7 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
  if (mirror) {
  PROXY_TRACE();
  NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA);
- } 
+ }
  }
 
  // NOTE-connect_retries_delay: This check alone is not enough for imposing
@@ -2966,7 +2968,7 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
  }
  if (mirror) {
  PROXY_TRACE();
- } 
+ }
  myds->destroy_MySQL_Connection_From_Pool(false);
  // NOTE-connect_retries_delay: In case of failure to connect, if
  // 'mysql_thread___connect_retries_delay' is set, we impose a delay in the session
@@ -4152,7 +4154,7 @@ int MySQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) {
  handler_ret = -1;
  return handler_ret;
  break;
- } 
+ }
  break;
  case FAST_FORWARD:
  mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size);
@@ -5222,7 +5224,7 @@ int MySQL_Session::handler() {
 
 
 __exit_DSS__STATE_NOT_INITIALIZED:
- 
+
 
  if (mybe && mybe->server_myds) {
  if (mybe->server_myds->DSS > STATE_MARIADB_BEGIN && mybe->server_myds->DSS < STATE_MARIADB_END) {
@@ -5362,7 +5364,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
  proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p . Returning\n", this, client_myds);
  return;
  }
- 
+
  if (
  (is_encrypted == false) && // the connection was encrypted
  (handshake_response_return == false) && // the authentication didn't complete
@@ -5375,16 +5377,16 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
  client_myds->wbio_ssl = BIO_new(BIO_s_mem());
  client_myds->ssl = GloVars.get_SSL_new();
  SSL_set_fd(client_myds->ssl, client_myds->fd);
- SSL_set_accept_state(client_myds->ssl); 
+ SSL_set_accept_state(client_myds->ssl);
  SSL_set_bio(client_myds->ssl, client_myds->rbio_ssl, client_myds->wbio_ssl);
  l_free(pkt->size,pkt->ptr);
  proxysql_keylog_attach_callback(GloVars.get_SSL_ctx());
  return;
  }
 
- if ( 
- //(client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size)==true) 
- (handshake_response_return == true) 
+ if (
+ //(client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size)==true)
+ (handshake_response_return == true)
  &&
  (
 #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP)
@@ -5570,7 +5572,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
  if (use_ssl == true && is_encrypted == false) {
  *wrong_pass=true;
  GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL);
- 
+
  char *_a=(char *)"ProxySQL Error: Access denied for user '%s' (using password: %s). SSL is required";
  char *_s=(char *)malloc(strlen(_a)+strlen(client_myds->myconn->userinfo->username)+32);
  sprintf(_s, _a, client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO"));
@@ -6321,7 +6323,7 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
  }
  if (mysql_variables.client_get_hash(this, idx) != var_value_int) {
  const MARIADB_CHARSET_INFO *ci = NULL;
- if (var == "character_set_results" || var == "character_set_connection" || 
+ if (var == "character_set_results" || var == "character_set_connection" ||
  var == "character_set_client" || var == "character_set_database") {
  ci = proxysql_find_charset_name(value1.c_str());
  }
@@ -6832,7 +6834,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
  l_free(pkt->size,pkt->ptr);
  client_myds->setDSS_STATE_QUERY_SENT_NET();
  client_myds->myprot.generate_statistics_response(true,NULL,NULL);
- client_myds->DSS=STATE_SLEEP; 
+ client_myds->DSS=STATE_SLEEP;
 }
 
 void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_CHANGE_USER(PtrSize_t *pkt, bool *wrong_pass) {
@@ -7317,7 +7319,7 @@ void MySQL_Session::SQLite3_to_MySQL(SQLite3_result *result, char *error, int af
  myds->DSS=STATE_SLEEP;
  free(l);
  free(p);
- 
+
  } else { // no result set
  if (error) {
  // there was an error
@@ -7462,7 +7464,7 @@ unsigned long long MySQL_Session::IdleTime() {
 
 
 // this is called either from RequestEnd(), or at the end of executing
-// prepared statements 
+// prepared statements
 void MySQL_Session::LogQuery(MySQL_Data_Stream *myds) {
  // we need to access statistics before calling CurrentQuery.end()
  // so we track the time here
@@ -7819,18 +7821,18 @@ bool MySQL_Session::known_query_for_locked_on_hostgroup(uint64_t digest) {
  case 5915334213354374281ULL: // "SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0"
  case 7837089204483965579ULL: // "SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO'"
  case 4312882378746554890ULL: // "SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0"
- case 4379922288366515816ULL: // "SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME... 
+ case 4379922288366515816ULL: // "SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME...
  case 12687634401278615449ULL: // "SET @rocksdb_enable_bulk_load = IF (@rocksdb_is_supported, 'SET SESSION rocksdb_bulk_load = 1', 'SET @rocksdb_dummy_bulk_load = 0')"
  case 15991633859978935883ULL: // "SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN"
  case 10636751085721966716ULL: // "SET @@GLOBAL.GTID_PURGED=?"
  case 15976043181199829579ULL: // "SET SQL_QUOTE_SHOW_CREATE=?"
  case 12094956190640701942ULL: // "SET SESSION information_schema_stats_expiry=0"
 /*
- case ULL: // 
- case ULL: // 
- case ULL: // 
- case ULL: // 
- case ULL: // 
+ case ULL: //
+ case ULL: //
+ case ULL: //
+ case ULL: //
+ case ULL: //
 */
  ret = true;
  break;

test/tap/tests/generate_set_session_csv.cpp

@@ -129,7 +129,7 @@ int main() {
  vars["sql_log_bin"]->add(bool_values);
  vars["sql_safe_updates"] = new variable("sql_safe_updates", true, false, true);
  vars["sql_safe_updates"]->add(bool_values);
- vars["sql_big_selects"] = new variable("sql_safe_updates", true, false, true);
+ vars["sql_big_selects"] = new variable("sql_big_selects", true, false, true);
  vars["sql_big_selects"]->add(bool_values);
 // vars["wsrep_sync_wait"] = new variable("wsrep_sync_wait", true, false);
 // vars["wsrep_sync_wait"]->add(bool_values);
@@ -231,8 +231,8 @@ int main() {
  vars["session_track_gtids"]->add("OWN_GTID");
 // vars["session_track_gtids"]->add("OFF");
 // vars["session_track_gtids"]->add("ALL_GTID");
- 
- 
+
+
  vars["optimizer_switch"] = new variable("optimizer_switch", true, false, false);
  vars["optimizer_switch"]->add(std::vector<std::string> {"'materialization=off'", "`materialization=on`", "\"materialization=off\""});
  vars["optimizer_switch"]->add(std::vector<std::string> {"'index_merge_union=off'", "`index_merge_union=on`", "\"index_merge_union=off\""});
@@ -252,6 +252,10 @@ int main() {
  vars["lc_messages"]->add(std::vector<std::string> {"es_ES", "'es_ES'", "`es_ES`", "\"es_ES\""});
  vars["lc_messages"]->add(std::vector<std::string> {"fr_FR", "'fr_FR'", "`fr_FR`", "\"fr_FR\""});
 
+ vars["log_slow_filter"] = new variable("log_slow_filter", true, false, false);
+ vars["log_slow_filter"]->add(std::vector<std::string> {"'not_using_index'", "`not_using_index`", "\"not_using_index\""});
+ vars["log_slow_filter"]->add(std::vector<std::string> {"'admin,filesort,filesort_on_disk,full_join'", "`admin,filesort,filesort_on_disk,full_join`", "\"admin,filesort,filesort_on_disk,full_join\""});
+
  vars["time_zone"] = new variable("time_zone", true, false, false);
  vars["time_zone"]->add(std::vector<std::string> {"'+01:00'", "`+02:15`", "\"+03:30\""});
  vars["time_zone"]->add(std::vector<std::string> {"'+04:45'", "`+05:00`", "\"+06:10\""});
@@ -281,6 +285,9 @@ int main() {
  vars["wsrep_osu_method"] = new variable("wsrep_osu_method", true, false, false);
  add_values_and_quotes("wsrep_osu_method", {"TOI","RSU"});
 
+ vars["sql_quote_show_create"] = new variable("sql_quote_show_create", true, false, true);
+ vars["sql_quote_show_create"]->add(bool_values);
+
  vars["sql_require_primary_key"] = new variable("sql_require_primary_key", true, false, true);
  vars["sql_require_primary_key"]->add(bool_values);
 

test/tap/tests/set_testing-240-t.cpp

@@ -47,7 +47,7 @@ char *schema=(char *)"information_schema";
 int silent = 0;
 int sysbench = 0;
 int local=0;
-int queries=3000;
+int queries=4000;
 int uniquequeries=0;
 int histograms=-1;
 

test/tap/tests/set_testing-240.h

@@ -34,6 +34,8 @@ const std::vector<std::string> possible_unknown_variables = {
  "optimizer_use_condition_selectivity",
  "max_statement_time",
  "sql_generate_invisible_primary_key",
+ "log_slow_filter",
+ "sql_quote_show_create",
  "sql_require_primary_key"
  };
 

test/tap/tests/set_testing.h

@@ -29,6 +29,8 @@ const std::vector<std::string> possible_unknown_variables = {
  "group_replication_consistency",
  "query_cache_type",
  "wsrep_osu_method",
+ "log_slow_filter",
+ "sql_quote_show_create",
  };
 
 int readTestCases(const std::string& fileName) {