From: vasil.dimov Date: August 2 2012 11:06am Subject: bzr push into mysql-5.6 branch (vasil.dimov:4095 to 4098) List-Archive: http://lists.mysql.com/commits/144516 Message-Id: <20120802110605.77D9D2E17A@mail.v5d.org> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 4098 Vasil Dimov 2012-08-02 Non-functional change: rename the record-long dict0stats_background.(cc|h) to dict0stats_bg.(cc|h) renamed: storage/innobase/dict/dict0stats_background.cc => storage/innobase/dict/dict0stats_bg.cc storage/innobase/include/dict0stats_background.h => storage/innobase/include/dict0stats_bg.h modified: storage/innobase/CMakeLists.txt storage/innobase/handler/ha_innodb.cc storage/innobase/handler/handler0alter.cc storage/innobase/row/row0mysql.cc storage/innobase/srv/srv0srv.cc storage/innobase/srv/srv0start.cc storage/innobase/dict/dict0stats_bg.cc storage/innobase/include/dict0stats_bg.h 4097 Vasil Dimov 2012-08-02 Non-functional whitespace change - indent global variables. modified: storage/innobase/dict/dict0stats_background.cc 4096 Vasil Dimov 2012-08-02 Fix Bug#14096230 USE A MORE EFFICIENT STRUCTURE FOR INNODB BACKGROUND STATS THREAD QUEUE Replace the home-made array with an STL vector for the stats auto recalc list. Approved by: Sunny (http://bur03.no.oracle.com/rb/r/1181/) modified: storage/innobase/dict/dict0stats_background.cc storage/innobase/include/dict0stats_background.h storage/innobase/row/row0mysql.cc 4095 Tor Didriksen 2012-08-02 Bug#14224928 RQG_MDL_STABILITY_VALGRIND FAILING WITH "STATUS_VALGRIND_FAILURE" ON DAILY TRUNK Post-push fix: don't set net.vio to NULL after closing it (we may leak memory) modified: sql/sql_class.cc === modified file 'storage/innobase/CMakeLists.txt' --- a/storage/innobase/CMakeLists.txt revid:tor.didriksen@stripped +++ b/storage/innobase/CMakeLists.txt revid:vasil.dimov@stripped @@ -276,7 +276,7 @@ SET(INNOBASE_SOURCES dict/dict0load.cc dict/dict0mem.cc dict/dict0stats.cc - dict/dict0stats_background.cc + dict/dict0stats_bg.cc dyn/dyn0dyn.cc eval/eval0eval.cc eval/eval0proc.cc === renamed file 'storage/innobase/dict/dict0stats_background.cc' => 'storage/innobase/dict/dict0stats_bg.cc' --- a/storage/innobase/dict/dict0stats_background.cc revid:tor.didriksen@stripped +++ b/storage/innobase/dict/dict0stats_bg.cc revid:vasil.dimov@stripped @@ -17,17 +17,20 @@ this program; if not, write to the Free *****************************************************************************/ /**************************************************//** -@file dict/dict0stats_background.cc +@file dict/dict0stats_bg.cc Code used for background table and index stats gathering. Created Apr 25, 2012 Vasil Dimov *******************************************************/ +#include +using namespace std; + #include "univ.i" #include "dict0dict.h" /* dict_table_open_on_id() */ #include "dict0stats.h" /* DICT_STATS_RECALC_PERSISTENT */ -#include "dict0stats_background.h" +#include "dict0stats_bg.h" #include "dict0mem.h" /* dict_table_struct */ #include "dict0types.h" /* table_id_t */ #include "ha0storage.h" /* ha_storage_* */ @@ -41,149 +44,139 @@ Created Apr 25, 2012 Vasil Dimov /** Minimum time interval between stats recalc for a given table */ #define MIN_RECALC_INTERVAL 10 /* seconds */ -#define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) +#define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) /** Event to wake up the stats thread */ -UNIV_INTERN os_event_t dict_stats_event = NULL; +UNIV_INTERN os_event_t dict_stats_event = NULL; -/** This mutex protects the following auto_recalc_* variables. */ -static ib_mutex_t auto_recalc_mutex; +/** This mutex protects the "recalc_pool" variable. */ +static ib_mutex_t recalc_pool_mutex; #ifdef HAVE_PSI_INTERFACE -static mysql_pfs_key_t auto_recalc_mutex_key; +static mysql_pfs_key_t recalc_pool_mutex_key; #endif /* HAVE_PSI_INTERFACE */ -/** The number of tables that can be queued before the queue is enlarged */ -#define AUTO_RECALC_LIST_INITIAL_SLOTS 128 -/** The autorecalc list - a simple array */ -static table_id_t* auto_recalc_list = NULL; -/** The size of the auto_recalc_list, in number of slots (not bytes) */ -static ulint auto_recalc_size; -/** The number of slots used in auto_recalc_list */ -static ulint auto_recalc_used; + +/** The number of tables that can be added to "recalc_pool" before +it is enlarged */ +#define RECALC_POOL_INITIAL_SLOTS 128 + +/** The multitude of tables whose stats are to be automatically +recalculated - an STL vector */ +typedef vector recalc_pool_t; +static recalc_pool_t recalc_pool; + +typedef recalc_pool_t::iterator recalc_pool_iterator_t; + +/*****************************************************************//** +Initialize the recalc pool, called once during thread initialization. */ +static +void +dict_stats_recalc_pool_init() +/*=========================*/ +{ + recalc_pool.reserve(RECALC_POOL_INITIAL_SLOTS); +} + +/*****************************************************************//** +Free the resources occupied by the recalc pool, called once during +thread de-initialization. */ +static +void +dict_stats_recalc_pool_deinit() +/*===========================*/ +{ + recalc_pool.clear(); +} /*****************************************************************//** -Add a table to the auto recalc list, which is processed by the +Add a table to the recalc pool, which is processed by the background stats gathering thread. Only the table id is added to the list, so the table can be closed after being enqueued and it will be opened when needed. If the table does not exist later (has been DROPped), -then it will be removed from the list and skipped. -dict_stats_enqueue_table_for_auto_recalc() @{ */ +then it will be removed from the pool and skipped. +dict_stats_recalc_pool_add() @{ */ UNIV_INTERN void -dict_stats_enqueue_table_for_auto_recalc( -/*=====================================*/ - const dict_table_t* table) /*!< in: table */ +dict_stats_recalc_pool_add( +/*=======================*/ + const dict_table_t* table) /*!< in: table to add */ { - mutex_enter(&auto_recalc_mutex); + mutex_enter(&recalc_pool_mutex); - ut_a(auto_recalc_used <= auto_recalc_size); + /* quit if already in the list */ + for (recalc_pool_iterator_t iter = recalc_pool.begin(); + iter != recalc_pool.end(); + ++iter) { - /* search if already in the list */ - for (ulint i = 0; i < auto_recalc_used; i++) { - if (auto_recalc_list[i] == table->id) { - /* already in the list, do not add */ - mutex_exit(&auto_recalc_mutex); + if (*iter == table->id) { + mutex_exit(&recalc_pool_mutex); return; } } - /* not in the list, need to be added */ + recalc_pool.push_back(table->id); - /* enlarge if no more space */ - if (auto_recalc_used == auto_recalc_size) { - table_id_t* p; - - p = reinterpret_cast( - ut_realloc(auto_recalc_list, - auto_recalc_size * 2 - * sizeof(auto_recalc_list[0]))); - - if (p == 0) { - /* auto_recalc_list is still valid, just quit without - adding the table to the list, maybe allocation will - succeed in the next enqueue operation */ - mutex_exit(&auto_recalc_mutex); - return; - } - - auto_recalc_list = p; - auto_recalc_size *= 2; - } - - auto_recalc_list[auto_recalc_used++] = table->id; - - mutex_exit(&auto_recalc_mutex); + mutex_exit(&recalc_pool_mutex); os_event_set(dict_stats_event); } /* @} */ /*****************************************************************//** -Pop a table from the auto recalc list. -dict_stats_dequeue_table_for_auto_recalc() @{ -@return true if list was non-empty and "id" was set, false otherwise */ +Get a table from the auto recalc pool. The returned table id is removed +from the pool. +dict_stats_recalc_pool_get() @{ +@return true if the pool was non-empty and "id" was set, false otherwise */ static bool -dict_stats_dequeue_table_for_auto_recalc( -/*=====================================*/ +dict_stats_recalc_pool_get( +/*=======================*/ table_id_t* id) /*!< out: table id, or unmodified if list is empty */ { - *id = UINT64_UNDEFINED; - - mutex_enter(&auto_recalc_mutex); - - if (auto_recalc_used > 0) { - - *id = auto_recalc_list[0]; + mutex_enter(&recalc_pool_mutex); - ut_a(auto_recalc_used > 0); + if (recalc_pool.empty()) { + mutex_exit(&recalc_pool_mutex); + return(false); + } - --auto_recalc_used; + *id = recalc_pool[0]; - ut_memmove( - &auto_recalc_list[0], - &auto_recalc_list[1], - auto_recalc_used * sizeof(auto_recalc_list[0])); - } + recalc_pool.erase(recalc_pool.begin()); - mutex_exit(&auto_recalc_mutex); + mutex_exit(&recalc_pool_mutex); - return(*id != UINT64_UNDEFINED); + return(true); } /* @} */ /*****************************************************************//** -Remove a table from the auto recalc list. -dict_stats_remove_table_from_auto_recalc() */ +Delete a given table from the auto recalc pool. +dict_stats_recalc_pool_del() */ UNIV_INTERN void -dict_stats_remove_table_from_auto_recalc( -/*=====================================*/ +dict_stats_recalc_pool_del( +/*=======================*/ const dict_table_t* table) /*!< in: table to remove */ { ut_ad(mutex_own(&dict_sys->mutex)); - mutex_enter(&auto_recalc_mutex); + mutex_enter(&recalc_pool_mutex); ut_ad(table->id > 0); - for (ulint i = 0; i < auto_recalc_used; ++i) { - - if (auto_recalc_list[i] == table->id) { - - ut_memmove( - &auto_recalc_list[i], - &auto_recalc_list[i + 1], - (auto_recalc_used - i - 1) - * sizeof(auto_recalc_list[0])); - - auto_recalc_used--; - + for (recalc_pool_iterator_t iter = recalc_pool.begin(); + iter != recalc_pool.end(); + ++iter) { + + if (*iter == table->id) { + /* erase() invalidates the iterator */ + recalc_pool.erase(iter); break; } } - mutex_exit(&auto_recalc_mutex); + mutex_exit(&recalc_pool_mutex); } /*****************************************************************//** @@ -232,7 +225,7 @@ dict_stats_thread_init() { dict_stats_event = os_event_create("dict_stats_event"); - /* The auto_recalc_mutex is acquired from: + /* The recalc_pool_mutex is acquired from: 1) the background stats gathering thread before any other latch and released without latching anything else in between (thus any level would do here) @@ -245,17 +238,10 @@ dict_stats_thread_init() and dict_operation_lock (SYNC_DICT_OPERATION) have been locked (thus a level mutex); - /* ut_time() could be expensive, pop_from_auto_recalc_list_and_recalc() + /* ut_time() could be expensive, the current function is called once every time a table has been changed more than 10% and on a system with lots of small tables, this could become hot. If we find out that this is a problem, then the check below could eventually @@ -331,7 +314,7 @@ pop_from_auto_recalc_list_and_recalc() too frequent stats updates we put back the table on the auto recalc list and do nothing. */ - dict_stats_enqueue_table_for_auto_recalc(table); + dict_stats_recalc_pool_add(table); } else { @@ -367,7 +350,7 @@ DECLARE_THREAD(dict_stats_thread)( /* Wake up periodically even if not signaled. This is because we may lose an event - if the below call to - pop_from_auto_recalc_list_and_recalc() puts the entry back + dict_stats_process_entry_from_recalc_pool() puts the entry back in the list, the os_event_set() will be lost by the subsequent os_event_reset(). */ os_event_wait_time( @@ -377,7 +360,7 @@ DECLARE_THREAD(dict_stats_thread)( break; } - pop_from_auto_recalc_list_and_recalc(); + dict_stats_process_entry_from_recalc_pool(); os_event_reset(dict_stats_event); } === modified file 'storage/innobase/handler/ha_innodb.cc' --- a/storage/innobase/handler/ha_innodb.cc revid:tor.didriksen@stripped +++ b/storage/innobase/handler/ha_innodb.cc revid:vasil.dimov@stripped @@ -74,7 +74,7 @@ this program; if not, write to the Free #include "row0merge.h" #include "dict0boot.h" #include "dict0stats.h" -#include "dict0stats_background.h" +#include "dict0stats_bg.h" #include "ha_prototypes.h" #include "ut0mem.h" #include "ibuf0ibuf.h" === modified file 'storage/innobase/handler/handler0alter.cc' --- a/storage/innobase/handler/handler0alter.cc revid:tor.didriksen@stripped +++ b/storage/innobase/handler/handler0alter.cc revid:vasil.dimov@stripped @@ -32,7 +32,7 @@ Smart ALTER TABLE #include "dict0crea.h" #include "dict0dict.h" #include "dict0stats.h" -#include "dict0stats_background.h" +#include "dict0stats_bg.h" #include "log0log.h" #include "row0log.h" #include "row0merge.h" === renamed file 'storage/innobase/include/dict0stats_background.h' => 'storage/innobase/include/dict0stats_bg.h' --- a/storage/innobase/include/dict0stats_background.h revid:tor.didriksen@stripped +++ b/storage/innobase/include/dict0stats_bg.h revid:vasil.dimov@stripped @@ -17,14 +17,14 @@ this program; if not, write to the Free *****************************************************************************/ /**************************************************//** -@file include/dict0stats_background.h +@file include/dict0stats_bg.h Code used for background table and index stats gathering. Created Apr 26, 2012 Vasil Dimov *******************************************************/ -#ifndef dict0stats_background_h -#define dict0stats_background_h +#ifndef dict0stats_bg_h +#define dict0stats_bg_h #include "univ.i" @@ -36,26 +36,26 @@ Created Apr 26, 2012 Vasil Dimov extern os_event_t dict_stats_event; /*****************************************************************//** -Add a table to the auto recalc list, which is processed by the +Add a table to the recalc pool, which is processed by the background stats gathering thread. Only the table id is added to the list, so the table can be closed after being enqueued and it will be opened when needed. If the table does not exist later (has been DROPped), -then it will be removed from the list and skipped. -dict_stats_enqueue_table_for_auto_recalc() @{ */ +then it will be removed from the pool and skipped. +dict_stats_recalc_pool_add() @{ */ UNIV_INTERN void -dict_stats_enqueue_table_for_auto_recalc( -/*=====================================*/ - const dict_table_t* table); /*!< in: table */ +dict_stats_recalc_pool_add( +/*=======================*/ + const dict_table_t* table); /*!< in: table to add */ /* @} */ /*****************************************************************//** -Remove a table from the auto recalc list. -dict_stats_remove_table_from_auto_recalc() @{ */ +Delete a given table from the auto recalc pool. +dict_stats_recalc_pool_del() */ UNIV_INTERN void -dict_stats_remove_table_from_auto_recalc( -/*=====================================*/ +dict_stats_recalc_pool_del( +/*=======================*/ const dict_table_t* table); /*!< in: table to remove */ /* @} */ @@ -113,4 +113,4 @@ DECLARE_THREAD(dict_stats_thread)( required by os_thread_create */ /* @} */ -#endif /* dict0stats_background_h */ +#endif /* dict0stats_bg_h */ === modified file 'storage/innobase/row/row0mysql.cc' --- a/storage/innobase/row/row0mysql.cc revid:tor.didriksen@stripped +++ b/storage/innobase/row/row0mysql.cc revid:vasil.dimov@stripped @@ -45,7 +45,7 @@ Created 9/17/2000 Heikki Tuuri #include "dict0load.h" #include "dict0boot.h" #include "dict0stats.h" -#include "dict0stats_background.h" +#include "dict0stats_bg.h" #include "trx0roll.h" #include "trx0purge.h" #include "trx0rec.h" @@ -1047,7 +1047,7 @@ row_update_statistics_if_needed( if (counter > n_rows / 10 /* 10% */ && dict_stats_auto_recalc_is_enabled(table)) { - dict_stats_enqueue_table_for_auto_recalc(table); + dict_stats_recalc_pool_add(table); table->stat_modified_counter = 0; } return; @@ -3725,7 +3725,7 @@ row_drop_table_for_mysql( fil_delete_link_file(name); } - dict_stats_remove_table_from_auto_recalc(table); + dict_stats_recalc_pool_del(table); /* Remove stats for this table and all of its indexes from the persistent storage if it exists and if there are stats for this === modified file 'storage/innobase/srv/srv0srv.cc' --- a/storage/innobase/srv/srv0srv.cc revid:tor.didriksen@stripped +++ b/storage/innobase/srv/srv0srv.cc revid:vasil.dimov@stripped @@ -59,7 +59,7 @@ Created 10/8/1995 Heikki Tuuri #include "btr0sea.h" #include "dict0load.h" #include "dict0boot.h" -#include "dict0stats_background.h" /* dict_stats_event */ +#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "row0mysql.h" #include "ha_prototypes.h" === modified file 'storage/innobase/srv/srv0start.cc' --- a/storage/innobase/srv/srv0start.cc revid:tor.didriksen@stripped +++ b/storage/innobase/srv/srv0start.cc revid:vasil.dimov@stripped @@ -71,7 +71,7 @@ Created 2/16/1996 Heikki Tuuri # include "buf0rea.h" # include "dict0boot.h" # include "dict0load.h" -# include "dict0stats_background.h" /* dict_stats_thread*(), dict_stats_event */ +# include "dict0stats_bg.h" /* dict_stats_thread*(), dict_stats_event */ # include "que0que.h" # include "usr0sess.h" # include "lock0lock.h" No bundle (reason: useless for push emails).