From: Andrei Elkin Date: December 1 2010 5:08pm Subject: bzr push into mysql-next-mr.crash-safe branch (andrei.elkin:3222 to 3223) WL#5569 List-Archive: http://lists.mysql.com/commits/125677 Message-Id: <201012011708.oB1H8wuf018740@mysql1000.dsl.inet.fi> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============1345990372==" --===============1345990372== MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Content-Disposition: inline 3223 Andrei Elkin 2010-12-01 wl#5569 MTS The limit conditions such as WQ len, total WQ:s size related changes. Also a new test file is added. @ mysql-test/suite/rpl/r/rpl_parallel_conf_limits.result new results file. @ mysql-test/suite/rpl/r/rpl_parallel_conflicts.result results updated. @ mysql-test/suite/rpl/t/rpl_parallel_conf_limits.test Testing two RAM usage by Workers limit parameters. @ mysql-test/suite/rpl/t/rpl_parallel_conflicts.test Converting an assert into wait for that condition. Todo: improve the test to let it run with slave_run_query_in_parallel. @ sql/log_event.cc limit condition (wq len, total wql sizes) related changes. fixing a compilation warn. @ sql/mysqld.cc renaming. @ sql/mysqld.h renaming. @ sql/rpl_rli.cc renaming. @ sql/rpl_rli.h s / slave_max_pending_jobs / opt_mts_slave_worker_queue_len_max / the new name is supposed to indicate the purpose of the entity more clearly. @ sql/rpl_slave.cc renaming. @ sql/sys_vars.cc renaming. added: mysql-test/suite/rpl/r/rpl_parallel_conf_limits.result mysql-test/suite/rpl/t/rpl_parallel_conf_limits.test modified: mysql-test/suite/rpl/r/rpl_parallel_conflicts.result mysql-test/suite/rpl/t/rpl_parallel_conflicts.test sql/log_event.cc sql/mysqld.cc sql/mysqld.h sql/rpl_rli.cc sql/rpl_rli.h sql/rpl_rli_pdb.cc sql/rpl_slave.cc sql/sys_vars.cc 3222 Andrei Elkin 2010-11-30 [merge] merging from from wl#5569 repo containing wl#5599 integration modified: sql/binlog.cc sql/log_event.cc sql/rpl_constants.h sql/rpl_info.cc sql/rpl_info.h sql/rpl_info_dummy.cc sql/rpl_info_dummy.h sql/rpl_info_factory.cc sql/rpl_info_factory.h sql/rpl_mi.cc sql/rpl_mi.h sql/rpl_rli.cc sql/rpl_rli.h sql/rpl_rli_pdb.cc sql/rpl_rli_pdb.h sql/rpl_slave.cc === added file 'mysql-test/suite/rpl/r/rpl_parallel_conf_limits.result' --- a/mysql-test/suite/rpl/r/rpl_parallel_conf_limits.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/r/rpl_parallel_conf_limits.result 2010-12-01 17:08:21 +0000 @@ -0,0 +1,47 @@ +stop slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +reset master; +reset slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +start slave; +create view coord_wait_list as SELECT id from Information_Schema.processlist where state like 'Waiting for Slave Worker%'; +include/stop_slave.inc +set @save.slave_exec_mode= @@global.slave_exec_mode; +set @@global.slave_exec_mode = 'Parallel'; +set @save_mts_slave_worker_queue_len_max= @@global.mts_slave_worker_queue_len_max; +set @@global.mts_slave_worker_queue_len_max= 5; +include/start_slave.inc +create database d1; +create table d1.t1 (a int auto_increment primary key) engine=innodb; +select sleep(2); +sleep(2) +0 +begin; +insert into d1.t1 set a=null; +begin; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +commit; +*** Coordinator must be waiting a for Worker to process its queue *** +rollback; +set @@global.mts_slave_worker_queue_len_max= @save_mts_slave_worker_queue_len_max; +include/stop_slave.inc +set @save_mts_pending_jobs_size_max = @@global.mts_pending_jobs_size_max; +set @@global.mts_pending_jobs_size_max= 1024; +include/start_slave.inc +create table d1.t2 (a int auto_increment primary key, b text null) engine=innodb; +begin; +insert into d1.t2 set a= 1; +begin; +commit; +*** Coordinator must be waiting for Workers have released pending events mem *** +rollback; +set @@global.mts_pending_jobs_size_max= @save_mts_pending_jobs_size_max; +drop database d1; +drop view coord_wait_list; +set @@global.slave_exec_mode= @save.slave_exec_mode; === modified file 'mysql-test/suite/rpl/r/rpl_parallel_conflicts.result' --- a/mysql-test/suite/rpl/r/rpl_parallel_conflicts.result 2010-11-26 21:08:30 +0000 +++ b/mysql-test/suite/rpl/r/rpl_parallel_conflicts.result 2010-12-01 17:08:21 +0000 @@ -29,6 +29,7 @@ insert into d3.t1 values (null); use d1; insert into d1.t1 values (null); commit; +*** Coordinator must be in waiting for a Worker to unassign from a partition *** rollback; select count(*) from d1.t1 into @d1; select count(*) from d2.t1 into @d2; === added file 'mysql-test/suite/rpl/t/rpl_parallel_conf_limits.test' --- a/mysql-test/suite/rpl/t/rpl_parallel_conf_limits.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/t/rpl_parallel_conf_limits.test 2010-12-01 17:08:21 +0000 @@ -0,0 +1,164 @@ +# +# WL#5569 MTS +# +# The test verifies correctness of MTS execution when system meets +# various limits due to configuration options. +# + +source include/master-slave.inc; +# no support for Query-log-event in this test +source include/have_binlog_format_row.inc; + +connection slave; + +create view coord_wait_list as SELECT id from Information_Schema.processlist where state like 'Waiting for Slave Worker%'; + +# restart in Parallel +source include/stop_slave.inc; +set @save.slave_exec_mode= @@global.slave_exec_mode; +set @@global.slave_exec_mode = 'Parallel'; + + +# max len of WQ +set @save_mts_slave_worker_queue_len_max= @@global.mts_slave_worker_queue_len_max; +set @@global.mts_slave_worker_queue_len_max= 5; +source include/start_slave.inc; + + +connection master; +create database d1; +create table d1.t1 (a int auto_increment primary key) engine=innodb; + + +connection slave; + +select sleep(2); +begin; +insert into d1.t1 set a=null; # lock a row that master has inserted into + + +connection master; + +begin; + +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; +insert into d1.t1 set a=null; + +commit; + +let $d1_t1_count=`select count(*) from d1.t1`; + +connection slave; + +--echo *** Coordinator must be waiting a for Worker to process its queue *** + +let $count= 1; +let $table= coord_wait_list; +source include/wait_until_rows_count.inc; + +rollback; + +let $count= $d1_t1_count; +let $table= d1.t1; +source include/wait_until_rows_count.inc; + +# cleanup of the max len +set @@global.mts_slave_worker_queue_len_max= @save_mts_slave_worker_queue_len_max; + + +# +# Max size of Worker queues +# + +--let $mts_max_q_size=1024 + +connection slave; + +source include/stop_slave.inc; +set @save_mts_pending_jobs_size_max = @@global.mts_pending_jobs_size_max; +eval set @@global.mts_pending_jobs_size_max= $mts_max_q_size; +source include/start_slave.inc; + +connection master; +create table d1.t2 (a int auto_increment primary key, b text null) engine=innodb; + +connection slave; + +# sync_slave_with_master +--disable_query_log +--disable_result_log +select sleep(2); +--enable_result_log +--enable_query_log + +begin; +insert into d1.t2 set a= 1; + +# master trans structure aims at testing C's wait loop + +let $i_loop= 10; + +connection master; + +begin; +--disable_query_log + eval insert into d1.t2 set a= 1, b= REPEAT('b', 1); + +while ($i_loop) +{ + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 1); + eval insert into d1.t2 set b= REPEAT('b', 2* $mts_max_q_size/3); + dec $i_loop; +} +--enable_query_log +commit; + +let $d1_t2_count=`select count(*) from d1.t2`; + +connection slave; + +--echo *** Coordinator must be waiting for Workers have released pending events mem *** + +let $count= 1; +let $table= coord_wait_list; +source include/wait_until_rows_count.inc; + +rollback; + +let $count= $d1_t2_count; +let $table= d1.t2; +source include/wait_until_rows_count.inc; + +# cleanup of the max len +set @@global.mts_pending_jobs_size_max= @save_mts_pending_jobs_size_max; + + +# +# cleanup +# + +connection master; + +drop database d1; + + +# sync_slave_with_master +connection slave; + +drop view coord_wait_list; + +--sleep 2 + +set @@global.slave_exec_mode= @save.slave_exec_mode; + + === modified file 'mysql-test/suite/rpl/t/rpl_parallel_conflicts.test' --- a/mysql-test/suite/rpl/t/rpl_parallel_conflicts.test 2010-11-26 21:08:30 +0000 +++ b/mysql-test/suite/rpl/t/rpl_parallel_conflicts.test 2010-12-01 17:08:21 +0000 @@ -32,6 +32,7 @@ source include/master-slave.inc; # TODO: convert this file into two tests for either value of # @@global.slave_run_query_in_parallel +source include/have_binlog_format_row.inc; connection slave; @@ -94,18 +95,13 @@ use d1; insert into d1.t1 values (null); # will be block at this point on Coord commit; ---sleep 4 - connection slave; -if (`SELECT @@global.binlog_format LIKE "row"`) -{ - if (`select COUNT(*) = 0 FROM coord_wait_list`) - { - SELECT * from Information_Schema.processlist; - --die Appologies, coodinator is supposed to be in the waiting state but it is not - } -} +--echo *** Coordinator must be in waiting for a Worker to unassign from a partition *** + +let $count= 1; +let $table= coord_wait_list; +source include/wait_until_rows_count.inc; # release the Worker rollback; === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2010-11-30 14:39:40 +0000 +++ b/sql/log_event.cc 2010-12-01 17:08:21 +0000 @@ -2210,15 +2210,10 @@ Slave_worker *Log_event::get_slave_worke // B or a DDL if ((is_b_event= starts_group()) || !rli->curr_group_seen_begin) { - // insert {NULL, W_s} to yield a new Group_cnt indexed element in GAQ - // Group_cnt= rli->gaq->e; - // rli->gaq->en_queue({NULL, W_s}); - g= { - log_pos, - NULL, - (ulong) -1, - const_cast(rli)->mts_total_groups++ - }; + g.master_log_pos= log_pos; + g.group_relay_log_name= NULL; + g.worker_id= (ulong) -1; + g.total_seqno= const_cast(rli)->mts_total_groups++; // the last occupied GAQ's array index rli->gaq->assigned_group_index= rli->gaq->en_queue((void *) &g); @@ -2411,6 +2406,9 @@ void append_item_to_jobs(slave_job_item Slave_worker *w, Relay_log_info *rli) { THD *thd= rli->info_thd; + int ret= -1; + ulonglong new_pend_size= rli->mts_pending_jobs_size + + ((Log_event*) (job_item->data))->data_written; DBUG_ASSERT(thd == current_thd); thd_proc_info(thd, "Feeding an event to a worker thread"); @@ -2418,44 +2416,30 @@ void append_item_to_jobs(slave_job_item mysql_mutex_lock(&rli->pending_jobs_lock); // C waits basing on *data* sizes in the queues - while (rli->mts_pending_jobs_size + - ((Log_event*) (job_item->data))->data_written - > rli->mts_pending_jobs_size_max) + while (new_pend_size > rli->mts_pending_jobs_size_max) { const char *old_msg; - rli->mts_wqs_oversize= TRUE; - rli->wait_jobs++; - old_msg= thd->enter_cond(&rli->pending_jobs_cond, - &rli->pending_jobs_lock, - "Waiting for Workers to unload queues"); - mysql_cond_wait(&rli->pending_jobs_cond, &rli->pending_jobs_lock); - thd->exit_cond(old_msg); - mysql_mutex_lock(&rli->pending_jobs_lock); - if (thd->killed) - return; - } - rli->mts_pending_jobs_size += ((Log_event*) (job_item->data))->data_written; + const char info_format[]= + "Waiting for Slave Workers to free pending events, requested size %lu"; + char wait_info[sizeof(info_format) + 4*sizeof(new_pend_size)]; -#if 0 - while (rli->pending_jobs >= rli->slave_pending_jobs_max) - { - const char *old_msg; - // Coordinator needs to wait to avoid the pending jobs queue overrun + sprintf(wait_info, info_format, new_pend_size); + rli->mts_wqs_oversize= TRUE; rli->wait_jobs++; - old_msg= thd->enter_cond(&rli->pending_jobs_cond, - &rli->pending_jobs_lock, - "Waiting for an event from sql thread"); + old_msg= thd->enter_cond(&rli->pending_jobs_cond, &rli->pending_jobs_lock, + wait_info); mysql_cond_wait(&rli->pending_jobs_cond, &rli->pending_jobs_lock); thd->exit_cond(old_msg); mysql_mutex_lock(&rli->pending_jobs_lock); if (thd->killed) return; + new_pend_size= rli->mts_pending_jobs_size + + ((Log_event*) (job_item->data))->data_written; } -#endif - + rli->mts_pending_jobs_size= new_pend_size; rli->stmt_jobs++; rli->pending_jobs++; - + mysql_mutex_unlock(&rli->pending_jobs_lock); // sleep while all queue lengths are gt Underrun @@ -2469,33 +2453,38 @@ void append_item_to_jobs(slave_job_item my_sleep(nap_weight * rli->mts_coordinator_basic_nap); } - if (!w->info_thd->killed) - { - int ret; + mysql_mutex_lock(&w->jobs_lock); + // possible WQ overfill + while (!w->info_thd->killed && !thd->killed && + (ret= en_queue(&w->jobs, job_item)) == -1) + { + const char *old_msg; + const char info_format[]= + "Waiting for Slave Worker %d queue: max len %lu, actual len %lu"; + char wait_info[sizeof(info_format) + 4*sizeof(w->id) + + 4*sizeof(w->jobs.s) + 4*sizeof(w->jobs.len)]; + + sprintf(wait_info, info_format, w->id, w->jobs.s, w->jobs.len); + old_msg= thd->enter_cond(&w->jobs_cond, &w->jobs_lock, wait_info); + w->jobs.overfill= TRUE; + w->jobs.waited_overfill++; + mysql_cond_wait(&w->jobs_cond, &w->jobs_lock); + thd->exit_cond(old_msg); + mysql_mutex_lock(&w->jobs_lock); - - // possible WQ overfill - while (!thd->killed && (ret= en_queue(&w->jobs, job_item)) == -1) - { - const char *old_msg; - old_msg= thd->enter_cond(&w->jobs_cond, &w->jobs_lock, - "Waiting for an event from sql thread"); - w->jobs.overfill= TRUE; - w->jobs.waited_overfill++; - mysql_cond_wait(&w->jobs_cond, &w->jobs_lock); - thd->exit_cond(old_msg); - - mysql_mutex_lock(&w->jobs_lock); - } + } + if (ret != -1) + { w->curr_jobs++; if (w->jobs.len == 1) mysql_cond_signal(&w->jobs_cond); - + mysql_mutex_unlock(&w->jobs_lock); } else { + mysql_mutex_unlock(&w->jobs_lock); mysql_mutex_lock(&rli->pending_jobs_lock); rli->pending_jobs--; // roll back of the prev incr mysql_mutex_unlock(&rli->pending_jobs_lock); @@ -2533,7 +2522,7 @@ int Log_event::apply_event(Relay_log_inf res= ev_begin->do_apply_event(rli); delete ev_begin; - /* B appears to be serial, reset parallel stautus of group + /* B appears to be serial, reset parallel status of group because the following T won't do that */ c_rli->curr_group_seen_begin= FALSE; @@ -2574,15 +2563,17 @@ int Log_event::apply_event(Relay_log_inf get_dynamic(&c_rli->curr_group_da, (uchar*) &da_item.data, i); append_item_to_jobs(&da_item, w, c_rli); } - if (rli->curr_group_da.elements > rli->curr_group_da.max_element) { + // reallocate to less mem + DBUG_ASSERT(rli->curr_group_da.max_element < rli->curr_group_da.elements); + c_rli->curr_group_da.elements= rli->curr_group_da.max_element; c_rli->curr_group_da.max_element= 0; freeze_size(&c_rli->curr_group_da); // restores max_element } - c_rli->curr_group_da.elements= 0; // to keep max_element-based allocation + c_rli->curr_group_da.elements= 0; } append_item_to_jobs(job_item, w, c_rli); === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2010-11-30 14:02:15 +0000 +++ b/sql/mysqld.cc 2010-12-01 17:08:21 +0000 @@ -461,7 +461,7 @@ uint slave_net_timeout; ulong slave_exec_mode_options; ulonglong slave_type_conversions_options; ulong slave_parallel_workers; -ulong slave_max_pending_jobs; +ulong opt_mts_slave_worker_queue_len_max; my_bool slave_local_timestamp_opt; my_bool opt_slave_run_query_in_parallel; ulong opt_mts_partition_hash_soft_max; === modified file 'sql/mysqld.h' --- a/sql/mysqld.h 2010-11-30 14:02:15 +0000 +++ b/sql/mysqld.h 2010-12-01 17:08:21 +0000 @@ -173,7 +173,7 @@ extern LEX_CSTRING reason_slave_blocked; extern ulong slave_trans_retries; extern uint slave_net_timeout; extern ulong slave_parallel_workers; -extern ulong slave_max_pending_jobs; +extern ulong opt_mts_slave_worker_queue_len_max; extern my_bool slave_local_timestamp_opt; extern my_bool opt_slave_run_query_in_parallel; extern ulong opt_mts_partition_hash_soft_max; === modified file 'sql/rpl_rli.cc' --- a/sql/rpl_rli.cc 2010-11-30 14:39:40 +0000 +++ b/sql/rpl_rli.cc 2010-12-01 17:08:21 +0000 @@ -93,7 +93,7 @@ Relay_log_info::Relay_log_info(bool is_s of non-processed yet jobs becomes bigger than the limit's value MHS_todo: consider a memory-size based param */ - slave_pending_jobs_max= ::slave_max_pending_jobs; + mts_slave_worker_queue_len_max= ::opt_mts_slave_worker_queue_len_max; // // TODO -- ANDREI --- You need to take care of possible failures related to === modified file 'sql/rpl_rli.h' --- a/sql/rpl_rli.h 2010-11-30 14:39:40 +0000 +++ b/sql/rpl_rli.h 2010-12-01 17:08:21 +0000 @@ -424,18 +424,18 @@ public: ulong trans_jobs, wait_jobs, stmt_jobs; // live time is one trans, statement (ndb epoch) mysql_mutex_t pending_jobs_lock; mysql_cond_t pending_jobs_cond; - ulong slave_pending_jobs_max; - ulonglong mts_pending_jobs_size; // actual mem usage by WQ:s - ulonglong mts_pending_jobs_size_max; // the max forcing to wait by C - bool mts_wqs_oversize; // C raises flag to wait some memory's released + ulong mts_slave_worker_queue_len_max; + ulonglong mts_pending_jobs_size; // actual mem usage by WQ:s + ulonglong mts_pending_jobs_size_max; // the max forcing to wait by C + bool mts_wqs_oversize; // C raises flag to wait some memory's released Slave_worker *last_assigned_worker; // a hint to partitioning func for some events Slave_committed_queue *gaq; DYNAMIC_ARRAY curr_group_assigned_parts; // CGAP - DYNAMIC_ARRAY curr_group_da; // deferred array to hold part-info-free events + DYNAMIC_ARRAY curr_group_da; // deferred array to hold partition-info-free events bool curr_group_seen_begin; // current group started with B-event or not bool run_query_in_parallel; // Query's default db not the actual db as part volatile ulong mts_wqs_underrun_w_id; // Id of a Worker whose queue is getting empty - volatile long mts_wqs_overrun; // W to incr and decr + volatile long mts_wqs_overrun; // W to incr and decr long mts_worker_underrun_level; // percent of WQ size at which Worker claims hungry ulong mts_coordinator_basic_nap; // C sleeps to avoid WQs overrun Slave_worker* get_current_worker() const; === modified file 'sql/rpl_rli_pdb.cc' --- a/sql/rpl_rli_pdb.cc 2010-11-30 14:39:40 +0000 +++ b/sql/rpl_rli_pdb.cc 2010-12-01 17:08:21 +0000 @@ -441,6 +441,9 @@ err: /** least_occupied in partition number sense. + This might be too coarse and computing based on assigned task + is a possibility. + Todo: combine two e.g by means of 2-index vector of weights. */ Slave_worker *get_least_occupied_worker(DYNAMIC_ARRAY *ws) { === modified file 'sql/rpl_slave.cc' --- a/sql/rpl_slave.cc 2010-11-30 14:39:40 +0000 +++ b/sql/rpl_slave.cc 2010-12-01 17:08:21 +0000 @@ -3563,7 +3563,7 @@ pthread_handler_t handle_slave_worker(vo mysql_mutex_lock(&w->jobs_lock); - DBUG_ASSERT(w->jobs.len == rli->slave_pending_jobs_max + 1); + DBUG_ASSERT(w->jobs.len == rli->mts_slave_worker_queue_len_max + 1); w->jobs.len= 0; mysql_cond_signal(&w->jobs_cond); // ready for duty @@ -3604,7 +3604,7 @@ pthread_handler_t handle_slave_worker(vo mysql_mutex_unlock(&rli->pending_jobs_lock); mysql_mutex_lock(&w->jobs_lock); - w->jobs.len= rli->slave_pending_jobs_max + 1; + w->jobs.len= rli->mts_slave_worker_queue_len_max + 1; mysql_cond_signal(&w->jobs_cond); // famous last goodbye mysql_mutex_unlock(&w->jobs_lock); @@ -3661,7 +3661,7 @@ int slave_start_single_worker(Relay_log_ w->usage_partition= 0; w->last_group_done_index= rli->gaq->s; // out of range - w->jobs.s= rli->slave_pending_jobs_max + 1; + w->jobs.s= rli->mts_slave_worker_queue_len_max + 1; my_init_dynamic_array(&w->jobs.Q, sizeof(Slave_job_item), w->jobs.s, 0); // todo: implement increment e.g n * 10; for (k= 0; k < w->jobs.s; k++) insert_dynamic(&w->jobs.Q, (uchar*) &empty); @@ -3670,7 +3670,7 @@ int slave_start_single_worker(Relay_log_ w->jobs.e= w->jobs.s; w->jobs.a= 0; - w->jobs.len= rli->slave_pending_jobs_max + 1; // to first handshake + w->jobs.len= rli->mts_slave_worker_queue_len_max + 1; // to first handshake w->jobs.overfill= FALSE; // todo: move into Slave_jobs_queue constructor w->jobs.waited_overfill= 0; w->wq_overrun_set= FALSE; @@ -3719,16 +3719,16 @@ int slave_start_workers(Relay_log_info * // GAQ queue holds seqno:s of scheduled groups. C polls workers in // @c lwm_checkpoint_period to update GAQ (see @c next_event()) - // The length of GAQ is derived from @c slave_max_pending_jobs to guarantee + // The length of GAQ is derived from @c mts_slave_worker_queue_len_max to guarantee // each assigned job being sent to a WQ will be represented by an item in GAQ. - // ::slave_max_pending_jobs is the worst case when all jobs contain + // ::mts_slave_worker_queue_len_max is the worst case when all jobs contain // one event and map to one worker. rli->gaq= new Slave_committed_queue(rli->get_group_master_log_name(), sizeof(Slave_job_group), - ::slave_max_pending_jobs, n); + ::opt_mts_slave_worker_queue_len_max, n); // size of WQ stays fixed in one slave session - rli->slave_pending_jobs_max= ::slave_max_pending_jobs; + rli->mts_slave_worker_queue_len_max= ::opt_mts_slave_worker_queue_len_max; rli->mts_pending_jobs_size= 0; rli->mts_pending_jobs_size_max= ::opt_mts_pending_jobs_size_max; rli->mts_wqs_underrun_w_id= (ulong) -1; @@ -3771,7 +3771,7 @@ void slave_stop_workers(Relay_log_info * mysql_mutex_lock(&w->jobs_lock); - if (w->jobs.len == rli->slave_pending_jobs_max + 1) + if (w->jobs.len == rli->mts_slave_worker_queue_len_max + 1) { mysql_mutex_unlock(&w->jobs_lock); continue; @@ -3793,7 +3793,7 @@ void slave_stop_workers(Relay_log_info * w->end_info(); mysql_mutex_lock(&w->jobs_lock); - while (w->jobs.len != rli->slave_pending_jobs_max + 1) + while (w->jobs.len != rli->mts_slave_worker_queue_len_max + 1) { const char *save_proc_info; save_proc_info= thd->enter_cond(&w->jobs_cond, &w->jobs_lock, === modified file 'sql/sys_vars.cc' --- a/sql/sys_vars.cc 2010-11-30 14:02:15 +0000 +++ b/sql/sys_vars.cc 2010-12-01 17:08:21 +0000 @@ -3109,15 +3109,14 @@ static Sys_var_ulong Sys_slave_parallel_ "Number of worker threads for executing events in parallel ", GLOBAL_VAR(slave_parallel_workers), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, ULONG_MAX), DEFAULT(4), BLOCK_SIZE(1)); - -// TODO: redefine slave_max_pending_jobs - -static Sys_var_ulong Sys_slave_max_pending_jobs( - "slave_max_pending_jobs", - "Number of replication events read out of Relay log and still not applied. " - "The coordinator thread suspends further jobs assigning until " - "conditions have been improved ", - GLOBAL_VAR(slave_max_pending_jobs), CMD_LINE(REQUIRED_ARG), +static Sys_var_ulong Sys_mts_slave_worker_queue_len_max( + "mts_slave_worker_queue_len_max", + "Max length of one MTS Worker queue. Presence in the queue indicates " + "a replication event was read out of Relay log and not yet applied. " + "Notice the max size of event data in all queues are governed by " + "mts_pending_jobs_size_max. Whichever limit is reached Coordinator thread" + "suspends further jobs assigning until conditions have been improved.", + GLOBAL_VAR(opt_mts_slave_worker_queue_len_max), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, ULONG_MAX - 1), DEFAULT(40000), BLOCK_SIZE(1)); static Sys_var_mybool Sys_slave_local_timestamp( "slave_local_timestamp", "if enabled slave computes the event appying " @@ -3137,7 +3136,7 @@ static Sys_var_ulong Sys_mts_partition_h GLOBAL_VAR(opt_mts_partition_hash_soft_max), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, ULONG_MAX), DEFAULT(16), BLOCK_SIZE(1)); static Sys_var_ulonglong Sys_mts_pending_jobs_size_max( - "opt_mts_pending_jobs_size_max", + "mts_pending_jobs_size_max", "Max size of Slave Worker queues holding yet not applied events." "The least possible value must be not less than the master size " "max_allowed_packet.", --===============1345990372== MIME-Version: 1.0 Content-Type: text/bzr-bundle; charset="us-ascii"; name="bzr/andrei.elkin@stripped" Content-Transfer-Encoding: 7bit Content-Disposition: inline # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: andrei.elkin@stripped # target_branch: file:///home/andrei/MySQL/BZR/2a-23May/WL/mysql-next-\ # mr-wl5569/ # testament_sha1: b00fa827fe409c6ce2723b48fa580bcdedd3d8c8 # timestamp: 2010-12-01 19:08:58 +0200 # base_revision_id: andrei.elkin@stripped\ # k9kypuuu6ysyau6w # # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWcf6JX8ADrv/gF6QgCB///// fqf/qv////pgG0c06d9ujvPQHO+2NecuezsNFdudq2Yir2Afe92em33Ouy7e3duze7jt173rW57O gDhDptnbuLYa9enVefYbvbO1mzp9PQSRInqaqf6Gqe0ap+kzVP0mTT1NSem1R5PVPUz0ptJoA2po 9Q0DNQSUAE0AQNEaSbSZT1D0m1ABo0yNDQNAGgGpkyaTJTQ0aaaANNAAAAAAAAAASIiNNJiU9JPS eNQof6hJ5DT0Sbap5IND1BoPKaekGnkEUggkw0TBBoNTJpmoamyU9qamJoNGg0AAASJBACAGpqnp inqn6g2RGoDIyAaAAaAaJgiwL72EIdcqLP2c0MSDJmGYxselj8uPVqT80OVueIaWJ3x/FpMeQfVy DhUd7kx+fv99ctCTFDVu6H5I6OnuM9vP7P8HumphYvHWSudMx4wq6wrP54gu+t72qNNt7uMBPuf1 GWfRx+UwfW5n6MZnbp/1Jsz1beBi+T4Ujwc3hH0McH92azfDfHD21buDAkxSq5enHGI3GDjCxy6B q5G+ZKd3Z+P8kY2rlF+x5M3fKyG0ChiSq9qEIO0yOTTiGANNt7buA0F8o0h1NdqzraSY6iXDNAzp jKiCqaRNSVDjA1JaQFJwcpAKYAhDZMHI3xQ/O02oZvbf8nl/ysnVEC13+nGvPh5QSXdYhJUAMmkF GIulrBLVYhSTA5hsuAgSkATpkA9Gzi8nxjNFEzM2RssHFZuwG4uCpMOB1HxJf4KRFqjzH8iwgs+w TVo3G83C3egLmEchDG2wBtsbbENptsBs0t/1Idj1bLWlFd9bPdXc6QxlNo5BzmbMH0lGQIArLAK0 KH+Dd1abtXl4obMiYNnLM8ayXdHac5xYbgxtLYrOzImCHZy7zeaPFmMxi8LFcdVfGmKs8mrsm2VM K40UWzz2Jxvhk8HLyjKcqOIeb8OZw6ToOmSaFQMMJhjyOODAxp45tmaQWq0pzzmQww8uKjk18k4E vSPDL0U0umNbOdtfhvd7Y3n+R40k1pgsdC73c0g55PpsUbVvzzjz2PfQeQC5TaYMRdOjnEGuGLgn x+XdWpydOeP2BCiZ9d5UApxkSESlIKeuB6Ycby67uE5AdNnard5fWAKrXRDQYDfLfrLJWMCt1o16 wTvqMpHOt1MoNYJTGblx0AtILqUSE2f3HN0bIrWuYRGsJ/XR141SodqcoWH5Ia5ZvbHeawe1+R5M 53ibp4e4FTi/OeHHMw7sbrUc9viYy38bij943GQLIgrm0gHMfgMnJnH78CcfBzSuxrwvwQuBx+60 TZwp8SWLF7gDTxMYcUp9Ga7tC4jI5M1KNIBQudq8p9ZsAucNjrkwmWZVsmug3Pw/xhHCYeA7cc53 SzFsPt39vPtuuc5fQY3mXc6GAaK3CZksRPcpQXpa7XX538w/pjvdfmp8n11OTfPSeT2CSW2vMHfN YGzdALGsz57rqc0N7WnnuQjN8CftO0aUQMmEAzdiFeZK3vmh0wJvID6yCbBCQaALPvImFAOTyz0O fX6eGZk7APs7dT8obj2XDiMHc++fX2QVZugI25Kz8Xlw+sloPn3+nNxDRJfouO+i+ebRY1PU2KG9 5IENsoqlG9QKHocWQtHBZhczDM077wYFVX7DJ2KnOHcBJusKsJPxUvhqIQnwXyorKTrOo6jUVpHr j4bCDwzjKSR+tChOhHo0GGAO1H1DALka2ScB1geRIEoxLCoO0SGUiRBkcFSqkwvcvGMLgbdNttj7 blYBb4clbkoWwOOshVa5W1AqErS1T77W4jJxl4RltCrwss6OMOP3r6KahTXRUJUpSshCPOQhEXwk Gm5RgrQ4IWKWFwUWGlBQYxlxASFW+95BNjZ/SPtZ+hjZkwW/mkSzb5NQEHLSCImSbFMCQSUIz7ev l3GIIm0LsYUYXw9EWQrOECt02gFPWhRsQgisJzIQiM0LBhQkIEKBQGReUaFIvhCGGxiepzRRqRom TBk1IkoCcrG53J2PMuKuE1PoCUyRQoXImqoMQKMWETWbmsjE9RiQfAYWVwxMCotBVFj4EMgIYkZs takzgMnbQydMMACsYh2nUbQVKCsCWmogJ8kpAH8Prt9D0GadiMqUXMuf59hYujydcK2qUE4DRiQ5 S2cICT7rbSwhIQE4LncmMA22+GXnTrxdx02621rLJxiaE5CV+AyAyioJZnBIcBdj7+FaHBYWLoXw fUbHhfIlLs0QEdTIaqHs5kWWOB+e/DyeZ3U44yezfFByJaWkuliqOE16nuBQSB7TA3lpIyKnU7Dc EwwMyJ4diKS6lS18syGaNH5OayrB+7GmhjGLa4wEXHvCbhlHoJsEJNEBI4cbxXL3UbEEnjdkyWhT gWIBQemMgcTmabwTJSdxORfcQAzGZvIaDiM0LKbvYtOlm1yGgxOKkdhtAsJFYHAkdFwaZmhU24HW 8gQAOgQIkh40iMGmLxcReQOstkVLS85HrBavAgDO+0+1qYza83F0NuoadBEBbE29MpO2m0uBQ/Tk cjwTmCuOOx3o1SNdzTtog12GQoasoNmrlrksjBQUJpm3Ui6sYnV1cDDoMiuyoti8C1WqhBPQXJBp mqki0OucLMAuYScXAYmR2IGJQzxKmmXI19WQHUVEwLzuXqWZ3qfIlu4SIjhrsettS2JykQ5N0ovp 7Dk4hQComrkwS8lZ7DcvJMnSBMc3oOETbmGQykwNXiPRTomXLkVJSGMu2koKrUkgaMys1VPjfHSh a1crKFDDQvyPG0NkBehPgwYZjl0LjUOxA+wrEMT4wNzMXmr1orHK0aujm+LoWdDcetFImaeziRfF qYTqVuaTIjtDAkpWdoE5KuAb8NcPznrzaFZ4jMKhgx5LZtku0zI9R7gSmZUpIcYYnMwaDleXTEnj u0MwxJqyRiMs0MjqHmvjWR7kywEsV7Vs2Unpto3ScstKVm6uHN7ECDJEjqLQyGqZQzYazA2KkDI/ 8ObQgmEInErhMZmjMeS6rzgM0KmhQ3TzEeQkonQGyqJXJ40osCo4wIDyhqPMiMvWdzirxgHcDoYK AM1jsyl0nJ58nnuh88SqDu+PGogLiKFjh9fs3g8YQLjMUjNdwND4Zl1TNzSZf5ibGJr70THtyHea jChI4qt1wWjCsypgVi048dUjfosnHd/lse1oiTTlrAkxgMROYzMXiJKPWYjGVNz0rxXNesEnGCou i4LY0Vjp0ZfPGOWb3vdLSxVUFIKUcqtZ06PeRN58CAJYFmkjidVbxMCxYbOtW0u3zBJphsbl7spD E8us0EpxxkVnAYTfG/jkRLrED1rIrEqA49CkTLhhyOwt1lfc2DtmXsgyjnlHRagaSHlli5hz5xrE aNaqTKdjElYDnI0OMCqCDshkyxUkPcCuLHEhDkYRmpc+eW1yHbnAvNSUciizKhuB5Ea14Aa8mE+D 9RJrcGIZZI0iyj5QBPQuUo3DeEYJzRKCQ2aHVeH0SKE0cHTlAmm96m9Uc50JElSwNeV1Y/Nh4RWd YLRC0bOblg+bCLjxXdrXT4EWzd+JoVKGZt1Hzde/14HB7mH2NLfWeORFJ743RPbQey5ko168pW2t IYVWBrfhZxY7EcC7DCG9pK3sgYfBflOfo96FuaAPJQPP8P84dbSxaPUhYZsDB6+KEVtCvd76PumG AdptjabGFAWsBKhMY0NsQM3KpAaAHe9AKv6vKj0MJIgkH8CDAIQ4Lo+z+/r+HguYCW3pmpIMx+DC gBtMb/D/1sAZ+FpQT7dff7wM/Ma+4aB5Q/uxywih7pzOFe37/6j+55B33/vtxOeYE/SPSzCky99o AueQ+p//Wg01hmOFiOAq7QOTd2Jj/D5d6xS21GOJT6LjoOs8CTHofkNGkm4DG4OkSGRu+lm3nSlf EwGReXUUlxSXEDCfhaEy2A+TIBqin3CKJdhqlZS2y/9l9puSyGNjY22hjbGmJjHiv1iP0okMqMhg 1stLrDAtUEXXsFpAQTGqNAZQSUzN5Kz0K+8aiCGNBQL19LHBBIKZEqElhjLFBJSFDthzUz0wAlXQ TgA5wL9XYWov9AOHG3svRvE/AhJLlEqRgXUkfSfQSPzLxJnzG2pEsPJlckgyLz5ihhkYFiJJJduI wZj1gHQQi8eZlBzq3mJodbCVUlPE9qQusGDKq5fcB3DPryCBzKQnJ0CpqEnZLKTh8kfznQjAMvTK 32kNpN1wZQ+zLBAphFqQny0e1x54iivQEVWeoc6tYTjqMITQpjmoNUFhYMKjNScsEitaT2xMvSp8 IF4UDBMXlCBJEKSAhANrcr8iFQnig+Nx+7yA8S1kPaZy4h7VsLxeKiIy7gLlEvETOMTAlOK/9KXg rBAzBwgcEKFx7CRRMHkALxn5QK5TMZcR1O3Z9JLMgcZg2DkmduhFPa8ZkbEw+HWHjwQJafVWvAWb QFBdkvM1GM9pbQ7CC32ns41LR7qwxqzBZiP1NLFx4m3bXow0MFofxSRlU7jkcTDzBV1KGQfaajAN jwT6T0RseX6B/JYXQYIe0VLQuAIwESNwcsjaJAQJECTBxswUpfBDN5pliAwhIMwzNvvvXimBiPEZ Ds8oP8tS5UCVYB+iXgCmFojvOw8TIQGz0xmPDYO55GC7mB5DhoxGBWA5GYliaG+hNgGYlfQ4FLHc WBEvEKqKyA8UutwzibdAYHxBA7hqX6+B6XDdmmJQevxKdDuARLj0jwLybJM0YzUZDmJm4amZHUsl HawHYZDpZqJzjB17Sqrp7lx/i/ODuDUkcx4z8w27IyJTCYfbPLGYKrOr4xgtJ+fGw5YS9C95J3F8 zIKmcVzjDSCAvspcYw7UMQjbCFOQLRXA0lqLYbnAzM6DMXDlxYSkVr+Q82y6QvGHWN4sY6doY1jw zFBWUHmaDy8wuHEyvU82Go5FObuSZq1Jo1MDw2Q5LIGOcOwxQ6oAlcDaqcSuIVuFohQZA1AGnivB loh5sTgMPxiPFNejIFCsfKBWEbEczxTSPGaTZGJBTOM54zOZYYBw4+V6zVqn5dbaqUc/MB9jaIGC QjjTZVR45VZ6PBet0QRU+cYHVHuUTQ7Yh5ioW0hZ3SlAVIADu6W1L+oQJhQ2l4NVaGCyKSSnMSzF iiLy+MDm8rpCkx8roQSbElQorB1pHV1CUUBLSPfSm++l/IncHE5ungCkuJRW8r1yBmk7CUIQfVEK SG3TO4UNQHXGuPasBssVwDAHZg2iWu0mVhBxbAzGbE8FggeEDaXDvV7Qr0mY9XeeDwPUEgoRPUeJ 1AoOKkiRQmVPanA+JgI9p6i64qGBkeFSpw4XI4FtTWj7AJMsnoD0XxuBkpIHqYnysrBOx5gMTNAo L1hEBfM73sqBSLzpQsQiPCADRDvKST6AwaNBKAMPud77DBvRbDUbAJB5CEMw5AMDSYRU3tLO0Cj3 5dLK8Cc1N7SRPjUb5LEyH6SQHtIAiNtbyDUUglyvzmFtfgWOLve4a+PGxxjvVTkqBvRZCJCSORn5 NHqO52FtBxkSrt5aBgeJqdw9kKsVzYuQLuwgyI5PfowC/xdoLjP5OQfDSFqM4RDVDc4woXIZTtIr A8oEhE0517wCi7BjGBMKJLlIytCtUUGfiKhrl5MMUwO5+9qiYCyPoDIAvAYxbxglj6eVMEg6saEx tWeh0uWrG8l07kl2IOBLcNt1hQEiBB5gPmvBQIuzTVlowzHO+S8wSzyUGg4CDZFTzfWozYyNN7ic mwzyOdGRAREEQIaXzHQ+16XL0o6nWdI8bYBUNHGYgM4KYBNwdKiZ1e1wTzApPx4roApBEcKy55FG dEZyEJPcQEEwrqRzNuo3DZqITvqgYiYIzhZpAbkTAcamtewNQQFFkFchGRylFg9aQDBAbREgG8mk lSBmF4F7VIQZbDvggiCCDYKye0DmhJZXmqMCucdDB0NjCgsRo02/qzEIlI8iMoAI3g1MQp8Yg4Z3 O40KOHna81SpV2sOYgUSvGcy9dRc4Y5HzeQA29zqdKwwgkHiykagymY0QPNejRCcQkChbriEZrcn KDfSKSSQdYQkfB2AqqIcBUROtKoUZAVG9l0dRDV8a9Zcx52hwbyjCgpzpWQPU8AwrJJM8vQQKq8p kiqo7sFxhukSDLcjYoIG3ukdxOENgOZvZI641CbWF3AvYlLayktsiwGYiQyFZTkFuEAySZRU4E+K aJDaBJWYyJMlGQdVBEnE6JvBxZKezizxkffKINOj0G5ExoH3kmTYAwabQlmaBJM7VZn8ovkONxaC YZc+ARYD1L4gMdRlkhMGGom7jAYkwU0goUGUgWGIULj8QBVo2bj4vY8kcH4aqWwozgTPCWLoe4Dz K0AqEcPB9whCKRqCFLAuE9CmlHUOJtA/KQpm+OjrPw3nEpgomqF2uWATuGFZiz3SRMgMA0YT7sLa hRZpXdVsgsYQoCL8/ghtvokUSb+gQhpKLBOz60NyrAURyT9C2yCQ/F55hf99wBmTWDqogNz9XM5I BMqyJ2Q2mpxxkQgD04EhYy5WEai0vYDDORmRI5IBgWGFKzdZkRiQ1Y80MEGFPWBLVIRnRSHNoek9 hZTAeFww5GDyvW5QLKfQB9jrfQ7nXA9IHB0vW6yzQoGNgPcC05nYBYXJTBlBArBoeajs0qTnMgms oIGiBlR4/cISOwZRMSTqgSugQu3xjWyAbFYidpaaoxgWnGaVEoZYahypeozP0fhhvtLBkV+KH9cq zQs11TLX0Fh0N5P5VwBIytvU/ykkb6o9LjAvpil8wMgbcwa14tY170gzUS6ken4oiHDY4iIcNyR6 FMQohscmq24b0Qxpx5HMpCaHRkKd+fG3hK4A8WNGOkQ+/NVMImIRvgDq3lIjaB25sIweiBIctLBF BCINcOda2VIuqWtQKBDARwOxz0i6odSQtH2eBJQmIQqzYMrnNAPiDl+bm0hlhI/odTZy823WI44y 0AtQIULXn3XeRuSQnTDjB6Mg41eKjDA54Ff2HyCnY6doOK3GwsQWCbhIJABUpx0WhsvOE0BcAXM+ ERERERERHT6C4HFFEmkHMA29gwghPmWi56qcMmMTRWchZZBmQ3KsQTR8AuSUZjHBqEQHQZj8C1Vw Dx82+srZJjSW5iQaBJ5WGjB1flK030iJaQpxdY8jdE6odZWqVlpVm/kQUJlBjHfvIEaFQ1EGwdBo X+xwOAOoz4l7lAWYheQhmxp5JEE6EwYRymxamYB0BaFiXNUMw/amLgGKVGFQWIeJXgJL1jcQUyiF 3yENikeOO3EGg3BiCRPYwIzvPY85wMuJSC0Oo/ny+NE99hwZ8IF3NueYpeJyyKboAk1TvQ2k5yIG daq3wXUtlkMQmhN3B3qMEYCq4SVVM29IlN9OvRJBQmdKXG2OfAhGzQBAlRUVt3Ssypifi7nOBaBA HM/Aeklp0Z8QBlaQgH7/ekiFQyloGgQ+GfZG7ok2AZGRXcASkl04IZ1TcWAsuYzt0L4PmTkJ7FKX EKAcQHge06i3VDUhgGSCxoN644EErpnsbhNDCTawz9b0NAqrlfyDPGUW7ZTf5zZxB2SkSmNjYazO zG8FJ4PMMQS50bIAZ1cA9iZGlqJU98u2xJpBLXrMcphnT2O59r0Ave1aKGmFmC2PQSZiNCAgHze9 umHQ0ex2HqKosmx8HeHGnFLVFCdgMn64aaKGoHHQUHsA+og8AE8wOpxeD2nIZx8G40FWpm4GtuE5 CBAmwBxN2N5XK7W5074I/+LuSKcKEhj/RK/g --===============1345990372==--