MDEV-39261 MariaDB crash on startup in presence of indexed virtual columns

mysql-test/suite/gcol/r/innodb_virtual_basic.result

@@ -48,9 +48,9 @@ INSERT INTO t VALUES (1290, 212, DEFAULT, "xmx");
 ROLLBACK;
 SELECT c FROM t;
 c
-NULL
 13
 29
+NULL
 SELECT * FROM t;
 a	b	c	h
 10	3	13	mm
@@ -303,23 +303,23 @@ END|
 CALL UPDATE_t();
 SELECT c FROM t;
 c
-NULL
 19
-29
 2103
+29
+NULL
 CALL DELETE_insert_t();
 SELECT c FROM t;
 c
-NULL
 19
-29
 2103
+29
+NULL
 DROP INDEX idx ON t;
 CALL UPDATE_t();
 SELECT c FROM t;
 c
-2103
 19
+2103
 29
 NULL
 DROP PROCEDURE DELETE_insert_t;
@@ -523,10 +523,10 @@ UPDATE t SET h = "e" WHERE h="a";
 ROLLBACK;
 SELECT a, c, h FROM t;
 a	c	h
-NULL	NULL	d
 11	14	a
 18	19	b
 28	29	c
+NULL	NULL	d
 DROP TABLE t;
 CREATE TABLE `t1` (
 `col1` int(11) NOT NULL,

mysql-test/suite/gcol/t/innodb_virtual_basic.test

@@ -42,6 +42,7 @@ INSERT INTO t VALUES (128, 22, DEFAULT, "xx");
 INSERT INTO t VALUES (1290, 212, DEFAULT, "xmx");
 ROLLBACK;
 
+--sorted_result
 SELECT c FROM t;
 SELECT * FROM t;
 
@@ -356,13 +357,16 @@ END|
 delimiter ;|
 
 CALL UPDATE_t();
+--sorted_result
 SELECT c FROM t;
 
 CALL DELETE_insert_t();
+--sorted_result
 SELECT c FROM t;
 
 DROP INDEX idx ON t;
 CALL UPDATE_t();
+--sorted_result
 SELECT c FROM t;
 
 DROP PROCEDURE DELETE_insert_t;
@@ -537,6 +541,7 @@ START TRANSACTION;
 UPDATE t SET m =10 WHERE m = 1;
 UPDATE t SET h = "e" WHERE h="a";
 ROLLBACK;
+--sorted_result
 SELECT a, c, h FROM t;
 
 DROP TABLE t;

mysql-test/suite/vcol/r/races.result

@@ -14,3 +14,17 @@ disconnect con1;
 connection default;
 drop table t1;
 set debug_sync='reset';
+#
+# MDEV-39261 MariaDB crash on startup in presence of
+#        indexed virtual columns
+#
+# Create 33 tables with virtual index
+InnoDB		0 transactions not purged
+connect purge_control,localhost,root;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connection default;
+# Do update on all 33 tables
+# restart: --innodb_purge_threads=1 --debug_dbug=d,ib_purge_virtual_index_callback
+InnoDB		0 transactions not purged
+# Drop all 33 tables
+# restart

mysql-test/suite/vcol/t/races.test

@@ -20,3 +20,59 @@ disconnect con1;
 connection default;
 drop table t1;
 set debug_sync='reset';
+
+--echo #
+--echo # MDEV-39261 MariaDB crash on startup in presence of
+--echo #        indexed virtual columns
+--echo #
+# To make purge thread to work on multiple tables on the same batch,
+# we need 33 tables because there are 32 pre-existing purge_node exists.
+
+--echo # Create 33 tables with virtual index
+--disable_query_log
+let $i = 33;
+while ($i)
+{
+  eval CREATE TABLE t$i(
+    a INT PRIMARY KEY,
+    b INT DEFAULT 1, INDEX(b),
+    c INT GENERATED ALWAYS AS (a + b) VIRTUAL,
+    INDEX(c)
+  ) ENGINE=InnoDB;
+  eval INSERT INTO t$i(a) VALUES(1);
+  dec $i;
+}
+--enable_query_log
+--source ../../innodb/include/wait_all_purged.inc
+--connect purge_control,localhost,root
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+--connection default
+--echo # Do update on all 33 tables
+--disable_query_log
+let $i = 33;
+while ($i)
+{
+  eval UPDATE t$i SET b = 11 WHERE a = 1;
+  dec $i;
+}
+--enable_query_log
+
+let $shutdown_timeout=0;
+let $restart_parameters=--innodb_purge_threads=1 --debug_dbug=d,ib_purge_virtual_index_callback;
+--source include/restart_mysqld.inc
+--source ../../innodb/include/wait_all_purged.inc
+
+--echo # Drop all 33 tables
+--disable_query_log
+let $i = 33;
+while ($i)
+{
+  eval DROP TABLE t$i;
+  dec $i;
+}
+--enable_query_log
+
+let $restart_parameters=;
+let $shutdown_timeout=;
+--source include/restart_mysqld.inc

sql/sql_base.cc

@@ -820,7 +820,7 @@ static inline bool check_field_pointers(const TABLE *table)
     leave prelocked mode if needed.
 */
 
-int close_thread_tables(THD *thd)
+int close_thread_tables(THD *thd) noexcept
 {
   TABLE *table;
   int error= 0;

sql/sql_base.h

@@ -163,7 +163,7 @@ TABLE_LIST *find_table_in_list(TABLE_LIST *table,
                                TABLE_LIST *TABLE_LIST::*link,
                                const LEX_CSTRING *db_name,
                                const LEX_CSTRING *table_name);
-int close_thread_tables(THD *thd);
+int close_thread_tables(THD *thd) noexcept;
 void switch_to_nullable_trigger_fields(List<Item> &items, TABLE *);
 void switch_defaults_to_nullable_trigger_fields(TABLE *table);
 bool fill_record_n_invoke_before_triggers(THD *thd, TABLE *table,

sql/sql_class.cc

@@ -5024,10 +5024,10 @@ extern "C" const char *thd_priv_user(MYSQL_THD thd, size_t *length)
   have only one table open at any given time.
 */
 TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
-                        const char *tb, size_t tblen)
+                        const char *tb, size_t tblen,
+                        MDL_ticket *mdl_ticket) noexcept
 {
   DBUG_ENTER("open_purge_table");
-  DBUG_ASSERT(thd->open_tables == NULL);
   DBUG_ASSERT(thd->locked_tables_mode < LTM_PRELOCKED);
 
   /* Purge already hold the MDL for the table */
@@ -5038,6 +5038,7 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
 
   tl->init_one_table(&db_name, &table_name, 0, TL_READ);
   tl->i_s_requested_object= OPEN_TABLE_ONLY;
+  tl->mdl_request.ticket= mdl_ticket;
 
   bool error= open_table(thd, tl, &ot_ctx);
 
@@ -5050,11 +5051,9 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
   DBUG_RETURN(error ? NULL : tl->table);
 }
 
-TABLE *get_purge_table(THD *thd)
+MDL_ticket *get_mdl_ticket(TABLE *table)
 {
-  /* see above, at most one table can be opened */
-  DBUG_ASSERT(thd->open_tables == NULL || thd->open_tables->next == NULL);
-  return thd->open_tables;
+  return table->mdl_ticket;
 }
 
 /** Find an open table in the list of prelocked tabled
@@ -5217,10 +5216,13 @@ void destroy_background_thd(MYSQL_THD thd)
 
 void reset_thd(MYSQL_THD thd)
 {
+  const char *proc_info= thd->proc_info;
+  thd->proc_info="reset";
   close_thread_tables(thd);
   thd->release_transactional_locks();
   thd->free_items();
   free_root(thd->mem_root, MYF(MY_KEEP_PREALLOC));
+  thd->proc_info= proc_info;
 }
 
 /**

storage/innobase/dict/dict0dict.cc

@@ -626,13 +626,13 @@ bool dict_table_t::parse_name(char (&db_name)[NAME_LEN + 1],
     dict_sys.unfreeze();
 
   *db_name_len= filename_to_tablename(db_buf, db_name,
-                                      MAX_DATABASE_NAME_LEN + 1, true);
+                                      NAME_LEN + 1, true);
 
   if (is_temp)
     return false;
 
   *tbl_name_len= filename_to_tablename(tbl_buf, tbl_name,
-                                       MAX_TABLE_NAME_LEN + 1, true);
+                                       NAME_LEN + 1, true);
   return true;
 }
 

storage/innobase/dict/dict0stats_bg.cc

@@ -71,6 +71,7 @@ static bool			stats_initialised;
 
 static THD *dict_stats_thd;
 
+void reset_thd(MYSQL_THD thd);
 /*****************************************************************//**
 Free the resources occupied by the recalc pool, called once during
 thread de-initialization. */
@@ -393,7 +394,7 @@ static void dict_stats_func(void*)
   while (dict_stats_process_entry_from_recalc_pool(dict_stats_thd)) {}
   dict_defrag_process_entries_from_defrag_pool(dict_stats_thd);
 
-  innobase_reset_background_thd(dict_stats_thd);
+  reset_thd(dict_stats_thd);
   set_current_thd(nullptr);
   if (!is_recalc_pool_empty())
     dict_stats_schedule(MIN_RECALC_INTERVAL * 1000);

storage/innobase/handler/ha_innodb.cc

@@ -124,10 +124,11 @@ TABLE *find_fk_open_table(THD *thd, const char *db, size_t db_len,
 			  const char *table, size_t table_len);
 MYSQL_THD create_background_thd();
 void reset_thd(MYSQL_THD thd);
-TABLE *get_purge_table(THD *thd);
 TABLE *open_purge_table(THD *thd, const char *db, size_t dblen,
-			const char *tb, size_t tblen);
-void close_thread_tables(THD* thd);
+			const char *tb, size_t tblen,
+			MDL_ticket *mdl_ticket) noexcept;
+int close_thread_tables(THD* thd) noexcept;
+MDL_ticket *get_mdl_ticket(TABLE *table) noexcept;
 
 #ifdef MYSQL_DYNAMIC_PLUGIN
 #define tc_size  400
@@ -1772,26 +1773,6 @@ MYSQL_THD innobase_create_background_thd(const char* name)
 	return thd;
 }
 
-
-/** Close opened tables, free memory, delete items for a MYSQL_THD.
-@param[in]	thd	MYSQL_THD to reset */
-void
-innobase_reset_background_thd(MYSQL_THD thd)
-{
-	if (!thd) {
-		thd = current_thd;
-	}
-
-	ut_ad(thd);
-	ut_ad(THDVAR(thd, background_thread));
-
-	/* background purge thread */
-	const char *proc_info= thd_proc_info(thd, "reset");
-	reset_thd(thd);
-	thd_proc_info(thd, proc_info);
-}
-
-
 /******************************************************************//**
 Check if the transaction is an auto-commit transaction. TRUE also
 implies that it is a SELECT (read-only) transaction.
@@ -8536,7 +8517,7 @@ ATTRIBUTE_COLD bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &ta
 {
   char db_buf[NAME_LEN + 1];
   char tbl_buf[NAME_LEN + 1];
-  ulint db_buf_len, tbl_buf_len;
+  size_t db_buf_len, tbl_buf_len;
 
   if (!table.parse_name(db_buf, tbl_buf, &db_buf_len, &tbl_buf_len))
   {
@@ -20071,37 +20052,28 @@ ha_innobase::multi_range_read_explain_info(
 for purge thread */
 static TABLE* innodb_find_table_for_vc(THD* thd, dict_table_t* table)
 {
-	TABLE *mysql_table;
-	const bool  bg_thread = THDVAR(thd, background_thread);
-
-	if (bg_thread) {
-		if ((mysql_table = get_purge_table(thd))) {
-			return mysql_table;
-		}
-	} else {
-		if (table->vc_templ->mysql_table_query_id
-		    == thd_get_query_id(thd)) {
-			return table->vc_templ->mysql_table;
-		}
+	table->lock_mutex_lock();
+	TABLE *maria_table = table->vc_templ->mysql_table;
+	const uint64_t cached_id = table->vc_templ->mysql_table_query_id;
+	table->lock_mutex_unlock();
+	if (cached_id == thd_get_query_id(thd)) {
+		return maria_table;
 	}
 
+	TABLE *mysql_table;
 	char	db_buf[NAME_LEN + 1];
 	char	tbl_buf[NAME_LEN + 1];
-	ulint	db_buf_len, tbl_buf_len;
+	size_t	db_buf_len, tbl_buf_len;
 
 	if (!table->parse_name(db_buf, tbl_buf, &db_buf_len, &tbl_buf_len)) {
 		return NULL;
 	}
-
-	if (bg_thread) {
-		return open_purge_table(thd, db_buf, db_buf_len,
-					tbl_buf, tbl_buf_len);
-	}
-
 	mysql_table = find_fk_open_table(thd, db_buf, db_buf_len,
 					 tbl_buf, tbl_buf_len);
+	table->lock_mutex_lock();
 	table->vc_templ->mysql_table = mysql_table;
 	table->vc_templ->mysql_table_query_id = thd_get_query_id(thd);
+	table->lock_mutex_unlock();
 	return mysql_table;
 }
 

storage/innobase/include/dict0mem.h

@@ -2551,6 +2551,17 @@ struct dict_table_t {
   static dict_table_t *create(const span<const char> &name, fil_space_t *space,
                               ulint n_cols, ulint n_v_cols, ulint flags,
                               ulint flags2);
+
+  /** @return whether the table has any indexed virtual column */
+  bool has_virtual_index() const noexcept
+  {
+    if (UNIV_UNLIKELY(n_v_cols != 0))
+      for (dict_index_t *index = indexes.start;
+           index; index = UT_LIST_GET_NEXT(indexes, index))
+        if (index->has_virtual())
+          return true;
+   return false;
+  }
 };
 
 inline void dict_index_t::set_modified(mtr_t& mtr) const

storage/innobase/include/ha_prototypes.h

@@ -420,11 +420,6 @@ MYSQL_THD innobase_create_background_thd(const char* name);
 @param[in]	thd	MYSQL_THD to destroy */
 void destroy_background_thd(MYSQL_THD thd);
 
-/** Close opened tables, free memory, delete items for a MYSQL_THD.
-@param[in]	thd	MYSQL_THD to reset */
-void
-innobase_reset_background_thd(MYSQL_THD);
-
 /** Open a table based on a database and table name.
 @param db     schema name
 @param name   table name within the schema