From: Andrei Elkin Date: November 27 2010 5:18pm Subject: bzr commit into mysql-next-mr.crash-safe branch (andrei.elkin:3219) List-Archive: http://lists.mysql.com/commits/125229 Message-Id: <201011271718.oARHIJZT002980@mysql1000.dsl.inet.fi> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============0538220478==" --===============0538220478== MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Content-Disposition: inline #At file:///home/andrei/MySQL/BZR/2a-23May/WL/wl5569-merge/ based on revid:andrei.elkin@stripped 3219 Andrei Elkin 2010-11-27 [merge] merging from from wl5569 repo added: mysql-test/suite/rpl/r/rpl_parallel_conflicts.result mysql-test/suite/rpl/t/rpl_parallel_conflicts.test modified: sql/log_event.cc sql/log_event.h sql/mysqld.cc sql/mysqld.h sql/rpl_rli.h sql/rpl_rli_pdb.cc sql/rpl_rli_pdb.h sql/rpl_slave.cc sql/sys_vars.cc === added file 'mysql-test/suite/rpl/r/rpl_parallel_conflicts.result' --- a/mysql-test/suite/rpl/r/rpl_parallel_conflicts.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/r/rpl_parallel_conflicts.result 2010-11-26 21:08:30 +0000 @@ -0,0 +1,78 @@ +stop slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +reset master; +reset slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +start slave; +create view coord_wait_list as SELECT id from Information_Schema.processlist where state like 'Waiting for Slave Worker%'; +include/stop_slave.inc +set @save.slave_exec_mode= @@global.slave_exec_mode; +set @@global.slave_exec_mode = 'Parallel'; +include/start_slave.inc +create database d1; +create database d2; +create database d3; +create table d1.t1 (a int auto_increment primary key) engine=innodb; +create table d2.t1 (a int auto_increment primary key) engine=innodb; +create table d3.t1 (a int auto_increment primary key) engine=innodb; +begin; +insert into d2.t1 values (1); +begin; +use d1; +insert into d1.t1 values (null); +use d2; +insert into d2.t1 values (1); +commit; +begin; +use d3; +insert into d3.t1 values (null); +use d1; +insert into d1.t1 values (null); +commit; +rollback; +select count(*) from d1.t1 into @d1; +select count(*) from d2.t1 into @d2; +select count(*) from d3.t1 into @d3; +use d1; +create table `exists_only_on_slave` (a int); +begin; +insert into d1.t1 values (null); +insert into d2.t1 values (null); +insert into d3.t1 values (null); +begin; +use d1; +insert into d1.t1 values (null); +commit; +begin; +use d2; +insert into d2.t1 values (null); +commit; +begin; +use d3; +insert into d3.t1 values (null); +commit; +use d1; +drop table if exists `exists_only_on_slave`; +select sleep(1); +sleep(1) +0 +select count(*) - @d1 as 'zero' from d1.t1; +zero +0 +select count(*) - @d2 as 'zero' from d2.t1; +zero +0 +select count(*) - @d3 as 'zero' from d3.t1; +zero +0 +use d1; +select count(*) as 'zero' from `exists_only_on_slave`; +zero +0 +rollback; +drop database d1; +drop database d2; +drop database d3; +drop view coord_wait_list; +set @@global.slave_exec_mode= @save.slave_exec_mode; +*** End of the tests *** === added file 'mysql-test/suite/rpl/t/rpl_parallel_conflicts.test' --- a/mysql-test/suite/rpl/t/rpl_parallel_conflicts.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/t/rpl_parallel_conflicts.test 2010-11-26 21:08:30 +0000 @@ -0,0 +1,224 @@ +# +# WL#5569 MTS +# +# The test checks cases of hashing conflicts forcing a special hanling. +# The cases include +# +# I. two Worker jobs conflict to each other +# +# a. two multi-statement transactions containing more than one partition +# in which one is common are mapped to different Workers. +# b. similarly two autocommit queries or ddl:s +# +# Handling of the cases is carried out as the following: +# when Coordinator hits to an occupied by not the currenly assigned Worker +# partition it marks the partition and goes to wait till the Worker-owner +# has released it and signaled. +# +# II. An event requires the sequential execution +# +# Coordinator does not schedule the event and is waiting till all workers have +# released their partitions and signalled. + +source include/master-slave.inc; + +# +# Testing with the statement format requires +# @@global.slave_run_query_in_parallel = 1. +# Notice, parallelization for Query-log-event is limitted +# to the default dababase. That's why 'use db'. +# With the default @@global.slave_run_query_in_parallel == 0 +# the tests in stmt format still run to prove switching to the sequential. + +# TODO: convert this file into two tests for either value of +# @@global.slave_run_query_in_parallel + +connection slave; + +--disable_query_log +--disable_result_log +call mtr.add_suppression('Error reading slave worker configuration'); +--enable_query_log +--enable_result_log + +create view coord_wait_list as SELECT id from Information_Schema.processlist where state like 'Waiting for Slave Worker%'; + +source include/stop_slave.inc; + +set @save.slave_exec_mode= @@global.slave_exec_mode; +set @@global.slave_exec_mode = 'Parallel'; +source include/start_slave.inc; + + +connection master; + +create database d1; +create database d2; +create database d3; +create table d1.t1 (a int auto_increment primary key) engine=innodb; +create table d2.t1 (a int auto_increment primary key) engine=innodb; +create table d3.t1 (a int auto_increment primary key) engine=innodb; + +# +# I. Two parallel jobs conflict +# +# two conflicting jobs to follow + +# sync_slave_with_master + +# TODO: remove once `sync_slave_with_master' got fixed + +--sleep 3 + +# To be really conflicting slave needs to block commit of the first. +connection slave; + +begin; +insert into d2.t1 values (1); + +connection master; + +# Job_1 +begin; +use d1; +insert into d1.t1 values (null); +use d2; +insert into d2.t1 values (1); # will be block at this point on Worker +commit; + +# Job_2 +begin; +use d3; +insert into d3.t1 values (null); +use d1; +insert into d1.t1 values (null); # will be block at this point on Coord +commit; + +--sleep 4 + +connection slave; + +if (`SELECT @@global.binlog_format LIKE "row"`) +{ + if (`select COUNT(*) = 0 FROM coord_wait_list`) + { + SELECT * from Information_Schema.processlist; + --die Appologies, coodinator is supposed to be in the waiting state but it is not + } +} + +# release the Worker +rollback; + +let $count= 2; +let $table= d1.t1; +source include/wait_until_rows_count.inc; + + +# +# II. The only-sequential conflicts with ongoing parallel applying +# + +# a. DDL waits for all workers have processed their earlier scheduled assignments + +connection slave1; + +# fix the tables status. Tables are supposed to exist, possibly with data left +# after previous part. + +select count(*) from d1.t1 into @d1; +select count(*) from d2.t1 into @d2; +select count(*) from d3.t1 into @d3; +use d1; +create table `exists_only_on_slave` (a int); + +connection slave; + +# put in the way of workers blocking load + +begin; +insert into d1.t1 values (null); +insert into d2.t1 values (null); +insert into d3.t1 values (null); + +connection master; + +# Job_1 +begin; +use d1; +insert into d1.t1 values (null); +commit; + +# Job_2 +begin; +use d2; +insert into d2.t1 values (null); +commit; + + +# Job_3 +begin; +use d3; +insert into d3.t1 values (null); +commit; + +--disable_warnings +use d1; +drop table if exists `exists_only_on_slave`; +--enable_warnings + + +connection slave1; + +select sleep(1); # give Workers a little time to process (but they won't) + +select count(*) - @d1 as 'zero' from d1.t1; +select count(*) - @d2 as 'zero' from d2.t1; +select count(*) - @d3 as 'zero' from d3.t1; + +# proof the master DDL has not got through +use d1; +select count(*) as 'zero' from `exists_only_on_slave`; + +connection slave; + +rollback; # release workers + +connection slave1; + +# to finish up with getting all committed. + +let $count= `select @d1 + 1`; +let $table= d1.t1; +source include/wait_until_rows_count.inc; + +let $count= `select @d2 + 1`; +let $table= d2.t1; +source include/wait_until_rows_count.inc; + +let $count= `select @d3 + 1`; +let $table= d3.t1; +source include/wait_until_rows_count.inc; +connection slave; + + +# +# cleanup +# + +connection master; + +drop database d1; +drop database d2; +drop database d3; + +--sleep 4 + +connection slave; +#sync_slave_with_master; + +drop view coord_wait_list; +set @@global.slave_exec_mode= @save.slave_exec_mode; + +--echo *** End of the tests *** + === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2010-11-27 09:17:41 +0000 +++ b/sql/log_event.cc 2010-11-27 17:17:41 +0000 @@ -2106,7 +2106,7 @@ bool Log_event::contains_partition_info( r - a mini-group internal "regular" event that follows its g-parent (Write, Update, Delete -rows) S - sequentially applied event (may not be a part of any group). - Events of this type are determined via @c only_serial_exec() + Events of this type are determined via @c only_sequential_exec() earlier and don't cause calling this method . T - terminator of the group (XID, COMMIT, ROLLBACK) @@ -2140,6 +2140,7 @@ Slave_worker *Log_event::get_slave_worke // rli->gaq->en_queue({NULL, W_s}); g= { log_pos, + NULL, (ulong) -1, const_cast(rli)->mts_total_groups++ }; @@ -2205,12 +2206,35 @@ Slave_worker *Log_event::get_slave_worke if (ends_group() || !rli->curr_group_seen_begin) { uint i; - // assert (\exists P_k . W_s \in CGAP) if P_k is present in ev mts_group_cnt= rli->gaq->assigned_group_index; - + + if (!worker->relay_log_change_notified) + { + /* + Prior this event, C rotated the relay log to drop each + Worker's notified flag. + Now group terminating event initiates the new name + delivery through the current group relaylog slot in GAQ. + */ + + Slave_job_group *ptr_g= + (Slave_job_group *) + dynamic_array_ptr(&rli->gaq->Q, rli->gaq->assigned_group_index); + + DBUG_ASSERT(ptr_g->group_relay_log_name == NULL); + + ptr_g->group_relay_log_name= (char *) + my_malloc(strlen(const_cast(rli)->get_group_relay_log_name()) + 1, MYF(MY_WME)); + strcpy(ptr_g->group_relay_log_name, const_cast(rli)->get_group_relay_log_name()); + worker->relay_log_change_notified= TRUE; + } + DBUG_ASSERT(worker == rli->last_assigned_worker); + if (!worker) { + DBUG_ASSERT(0); + // a very special case of the empty group: {B, T} DBUG_ASSERT(rli->curr_group_assigned_parts.elements == 0 && rli->curr_group_da.elements == 1); @@ -2225,6 +2249,8 @@ Slave_worker *Log_event::get_slave_worke // reset the B-group marker const_cast(rli)->curr_group_seen_begin= FALSE; + + const_cast(rli)->curr_group_is_parallel= TRUE; // mark for Coord's T-event delete } return worker; @@ -2370,10 +2396,50 @@ int Log_event::apply_event(Relay_log_inf Slave_worker *w= NULL; Slave_job_item item= {NULL}, *job_item= &item; Relay_log_info *c_rli= const_cast(rli); // constless alias + bool parallel; + + if (!(parallel= rli->is_parallel_exec()) || + only_sequential_exec(rli->run_query_in_parallel, rli->curr_group_seen_begin)) + { + if (parallel) + { + // This case relates to Query parallel apply which breaks into + // DDL and {B, Q, T} group, where Q owns g-parallel property. - if (!rli->is_parallel_exec() || only_serial_exec() /* || wait(APH.N == 0) */) + // Apply possibly deferred B + if (rli->curr_group_da.elements > 0) + { + int res; + Log_event *ev_begin= * (Log_event**) pop_dynamic(&c_rli->curr_group_da); + + DBUG_ASSERT(rli->curr_group_da.elements == 0); + DBUG_ASSERT(rli->curr_group_seen_begin); + + res= ev_begin->do_apply_event(rli); + delete ev_begin; + /* B appears to be serial, reset parallel stautus of group + because the following T won't do that */ + c_rli->curr_group_seen_begin= FALSE; + + if (res) + DBUG_RETURN(res); + } + + DBUG_ASSERT(!rli->curr_group_seen_begin); + c_rli->curr_group_is_parallel= FALSE; // Coord will destruct all the rest of events + if (!parallel_exec_by_coordinator(::server_id)) + (void) wait_for_workers_to_finish(rli); + } DBUG_RETURN(do_apply_event(rli)); - + } + + if (get_type_code() == ROWS_QUERY_LOG_EVENT) + { + rli->report(ERROR_LEVEL, 0, + "No parallel support for ROWS_QUERY_LOG_EVENT"); + DBUG_RETURN(1); + } + if ((!(w= get_slave_worker_id(rli)) || DBUG_EVALUATE_IF("fault_injection_get_slave_worker", 1, 0))) DBUG_RETURN(rli->curr_group_assigned_parts.elements == 0 ? FALSE : TRUE); === modified file 'sql/log_event.h' --- a/sql/log_event.h 2010-11-27 09:17:41 +0000 +++ b/sql/log_event.h 2010-11-27 17:17:41 +0000 @@ -1153,40 +1153,62 @@ public: public: /** - mst-II: to execute serially due to - technical or conceptual limitation + MST: to execute serially due to technical or conceptual limitation - @return TRUE for all but {Query,Rand,User_var,Intvar,Rows}_log_event + @return TRUE if despite permanent parallel execution mode an event + needs applying in a real isolation that is sequentially. */ - bool only_serial_exec() + bool only_sequential_exec(bool query_in_parallel, bool group_term_in_parallel) { return - // todo: the 4 types below are limitly parallel-supported (the default - // session db not the actual db) + /* + the 4 types below are limitly parallel-supported (the default + session db not the actual db). + Decision on BEGIN is deferred till the following event. + Decision on Commit or Xid is forced by the one for BEGIN. + */ - // get_type_code() == QUERY_EVENT || - // get_type_code() == INTVAR_EVENT || - // get_type_code() == USER_VAR_EVENT || - // get_type_code() == RAND_EVENT || - - get_type_code() == STOP_EVENT || - get_type_code() == ROTATE_EVENT || - get_type_code() == LOAD_EVENT || - get_type_code() == SLAVE_EVENT || - get_type_code() == CREATE_FILE_EVENT || - get_type_code() == APPEND_BLOCK_EVENT || - get_type_code() == EXEC_LOAD_EVENT || - get_type_code() == DELETE_FILE_EVENT || - get_type_code() == NEW_LOAD_EVENT || - get_type_code() == FORMAT_DESCRIPTION_EVENT || - get_type_code() == BEGIN_LOAD_QUERY_EVENT || - get_type_code() == EXECUTE_LOAD_QUERY_EVENT || + (!query_in_parallel && + ((get_type_code() == QUERY_EVENT + && !starts_group() && !ends_group()) || + get_type_code() == INTVAR_EVENT || + get_type_code() == USER_VAR_EVENT || + get_type_code() == RAND_EVENT)) || + + (!group_term_in_parallel && ends_group()) || + + get_type_code() == START_EVENT_V3 || + get_type_code() == STOP_EVENT || + get_type_code() == ROTATE_EVENT || + get_type_code() == LOAD_EVENT || + get_type_code() == SLAVE_EVENT || + get_type_code() == CREATE_FILE_EVENT || + get_type_code() == APPEND_BLOCK_EVENT || + get_type_code() == EXEC_LOAD_EVENT || + get_type_code() == DELETE_FILE_EVENT || + get_type_code() == NEW_LOAD_EVENT || + get_type_code() == FORMAT_DESCRIPTION_EVENT|| + get_type_code() == BEGIN_LOAD_QUERY_EVENT || + get_type_code() == EXECUTE_LOAD_QUERY_EVENT|| /* todo: make parallel */ get_type_code() == PRE_GA_WRITE_ROWS_EVENT || - get_type_code() == PRE_GA_UPDATE_ROWS_EVENT || - get_type_code() == PRE_GA_DELETE_ROWS_EVENT || + get_type_code() == PRE_GA_UPDATE_ROWS_EVENT|| + get_type_code() == PRE_GA_DELETE_ROWS_EVENT|| get_type_code() == INCIDENT_EVENT; } - + + /** + MST: some events can be applied by Coordinator concurrently with Workers. + + @return TRUE if that's the case, + FALSE otherwise. + */ + bool parallel_exec_by_coordinator(ulong slave_server_id) + { + return + get_type_code() == FORMAT_DESCRIPTION_EVENT && + (server_id == (uint32) ::server_id); + } + /** Events of a cetain type carry partitioning data such as db names. */ === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2010-11-27 09:17:41 +0000 +++ b/sql/mysqld.cc 2010-11-27 17:17:41 +0000 @@ -459,6 +459,7 @@ ulonglong slave_type_conversions_options ulong slave_parallel_workers; ulong slave_max_pending_jobs; my_bool slave_local_timestamp_opt; +my_bool opt_slave_run_query_in_parallel; ulong thread_cache_size=0; ulong binlog_cache_size=0; ulonglong max_binlog_cache_size=0; === modified file 'sql/mysqld.h' --- a/sql/mysqld.h 2010-11-27 09:17:41 +0000 +++ b/sql/mysqld.h 2010-11-27 17:17:41 +0000 @@ -176,6 +176,7 @@ extern uint slave_net_timeout; extern ulong slave_parallel_workers; extern ulong slave_max_pending_jobs; extern my_bool slave_local_timestamp_opt; +extern my_bool opt_slave_run_query_in_parallel; extern uint max_user_connections; extern ulong what_to_log,flush_time; extern ulong max_prepared_stmt_count, prepared_stmt_count; === modified file 'sql/rpl_rli.h' --- a/sql/rpl_rli.h 2010-11-27 09:17:41 +0000 +++ b/sql/rpl_rli.h 2010-11-27 17:17:41 +0000 @@ -438,15 +438,16 @@ public: Slave_worker *last_assigned_worker; // a hint to partitioning func for some events Slave_committed_queue *gaq; DYNAMIC_ARRAY curr_group_assigned_parts; // CGAP - DYNAMIC_ARRAY curr_group_da; // deferred array to hold part-info-free events - bool curr_group_seen_begin; // current group started with B-event or not - volatile Slave_worker* slave_worker_is_error; + DYNAMIC_ARRAY curr_group_da; // deferred array to hold part-info-free events + bool curr_group_seen_begin; // current group started with B-event or not + bool run_query_in_parallel; // Query's default db not the actual db as part Slave_worker* get_current_worker() const; Slave_worker* set_this_worker(Slave_worker *w) { return this_worker= w; } Slave_worker* this_worker; // used by w_rli. The cental rli has it as NULL. ulonglong mts_total_groups; // total event groups distributed in current session - + volatile Slave_worker* slave_worker_is_error; + bool curr_group_is_parallel; // a mark for Coord to indicate on T-event of the curr group at delete /* A sorted array of Worker current assignements number to provide approximate view on Workers loading. === modified file 'sql/rpl_rli_pdb.cc' --- a/sql/rpl_rli_pdb.cc 2010-11-25 08:47:39 +0000 +++ b/sql/rpl_rli_pdb.cc 2010-11-27 15:36:50 +0000 @@ -138,6 +138,7 @@ bool Slave_worker::write_info(Rpl_info_h */ if (to->prepare_info_for_write() || + to->set_info(partitions) || to->set_info(group_relay_log_name) || to->set_info((ulong)group_relay_log_pos) || to->set_info(group_master_log_name) || @@ -284,12 +285,6 @@ Slave_worker *get_slave_worker(const cha insert_dynamic(&rli->curr_group_assigned_parts, (uchar*) key); DBUG_PRINT("info", ("Searching for %s, %d", dbname, dblength)); - /* - The database name was not found which means that a worker never - processed events from that database. In such case, we need to - map the database to a worker my inserting an entry into the - hash map. - */ hash_value= my_calc_hash(&mapping_db_to_worker, (uchar*) dbname, dblength); @@ -301,6 +296,12 @@ Slave_worker *get_slave_worker(const cha (uchar*) dbname, dblength); if (!entry) { + /* + The database name was not found which means that a worker never + processed events from that database. In such case, we need to + map the database to a worker my inserting an entry into the + hash map. + */ my_bool ret; mysql_mutex_unlock(&slave_worker_hash_lock); @@ -326,6 +327,7 @@ Slave_worker *get_slave_worker(const cha */ entry->worker= !rli->last_assigned_worker ? get_least_occupied_worker(workers) : rli->last_assigned_worker; + entry->worker->usage_partition++; mysql_mutex_lock(&slave_worker_hash_lock); ret= my_hash_insert(&mapping_db_to_worker, (uchar*) entry); @@ -346,7 +348,9 @@ Slave_worker *get_slave_worker(const cha { entry->worker= !rli->last_assigned_worker ? get_least_occupied_worker(workers) : rli->last_assigned_worker; + entry->worker->usage_partition++; entry->usage++; + my_hash_update(&mapping_db_to_worker, (uchar*) entry, (uchar*) dbname, dblength); } @@ -360,20 +364,37 @@ Slave_worker *get_slave_worker(const cha my_hash_update(&mapping_db_to_worker, (uchar*) entry, (uchar*) dbname, dblength); } - else // may be the hashing conflict + else { - DBUG_ASSERT(rli->last_assigned_worker == NULL || - rli->curr_group_assigned_parts.elements > 1); + // The case APH contains a W_d != W_c != NULL assigned to + // D-partition represents + // the hashing conflict and is handled as the following: + + THD *thd= rli->info_thd; + const char *proc_info; + const char info_format[]= + "Waiting for Slave Worker %d to release partition `%s`"; + char wait_info[sizeof(info_format) + 4*sizeof(entry->worker->id) + + NAME_LEN + 1]; - DBUG_ASSERT(0); // ... TODO ... *not* ready yet + DBUG_ASSERT(rli->last_assigned_worker != NULL && + rli->curr_group_assigned_parts.elements > 1); // future assignenment and marking at the same time entry->worker= rli->last_assigned_worker; - wait(); + sprintf(wait_info, info_format, entry->worker->id, entry->db); + + proc_info= thd->enter_cond(&slave_worker_hash_cond, &slave_worker_hash_lock, + wait_info); + mysql_cond_wait(&slave_worker_hash_cond, &slave_worker_hash_lock); + thd->exit_cond(proc_info); + mysql_mutex_lock(&slave_worker_hash_lock); DBUG_ASSERT(entry->usage == 0); + entry->usage= 1; + entry->worker->usage_partition++; } mysql_mutex_unlock(&slave_worker_hash_lock); @@ -407,10 +428,6 @@ Slave_worker *get_least_occupied_worker( DBUG_ASSERT(worker != NULL); - worker->usage_partition++; - - DBUG_ASSERT(worker->usage_partition != 0); - return(worker); } @@ -431,7 +448,23 @@ void Slave_worker::slave_worker_ends_gro uint i; if (!error) + { + Slave_job_group *ptr_g= + (Slave_job_group *) + dynamic_array_ptr(&c_rli->gaq->Q, c_rli->gaq->assigned_group_index); + if (ptr_g->group_relay_log_name != NULL) + { + // memorizing a new relay-log file name + + DBUG_ASSERT(strlen(ptr_g->group_relay_log_name) + 1 + <= sizeof(group_relay_log_name)); + + strcpy(group_relay_log_name, ptr_g->group_relay_log_name); + delete ptr_g->group_relay_log_name; // C allocated + ptr_g->group_relay_log_name= NULL; // mark freed + } last_group_done_index = gaq_idx; + } for (i= curr_group_exec_parts.elements; i > 0; i--) { @@ -448,7 +481,7 @@ void Slave_worker::slave_worker_ends_gro my_hash_search_using_hash_value(&mapping_db_to_worker, hash_value, (uchar*) key + 1, key[0]); - DBUG_ASSERT(entry && entry->usage != 0 && entry->worker == this); + DBUG_ASSERT(entry && entry->usage != 0); DBUG_ASSERT(strlen(key + 1) == (uchar) key[0]); @@ -457,7 +490,11 @@ void Slave_worker::slave_worker_ends_gro (uchar*) key + 1, key[0]); if (entry->usage == 0) + { usage_partition--; + if (entry->worker != this) // Coordinator is waiting + mysql_cond_signal(&slave_worker_hash_cond); + } else DBUG_ASSERT(usage_partition != 0); /* @@ -644,3 +681,45 @@ ulong Slave_committed_queue::move_queue_ return cnt; } + + +int wait_for_workers_to_finish(Relay_log_info const *rli) +{ + uint ret= 0; + HASH *hash= &mapping_db_to_worker; + for (uint i= 0, ret= 0; i < hash->records; i++) + { + db_worker *entry; + THD *thd= rli->info_thd; + const char *proc_info; + const char info_format[]= + "Waiting for Slave Worker %d to release partition `%s`"; + char wait_info[sizeof(info_format) + 4*sizeof(entry->worker->id) + + NAME_LEN + 1]; + + mysql_mutex_lock(&slave_worker_hash_lock); + + entry= (db_worker*) my_hash_element(hash, i); + + DBUG_ASSERT(entry); + + if (entry->usage > 0) + { + sprintf(wait_info, info_format, entry->worker->id, entry->db); + entry->worker= NULL; + + proc_info= thd->enter_cond(&slave_worker_hash_cond, &slave_worker_hash_lock, + wait_info); + mysql_cond_wait(&slave_worker_hash_cond, &slave_worker_hash_lock); + thd->exit_cond(proc_info); + ret++; + + DBUG_ASSERT(entry->usage == 0); + } + else + { + mysql_mutex_unlock(&slave_worker_hash_lock); + } + } + return ret; +} === modified file 'sql/rpl_rli_pdb.h' --- a/sql/rpl_rli_pdb.h 2010-11-25 09:03:54 +0000 +++ b/sql/rpl_rli_pdb.h 2010-11-27 15:36:50 +0000 @@ -24,6 +24,7 @@ bool init_hash_workers(ulong slave_paral void destroy_hash_workers(); Slave_worker *get_slave_worker(const char *dbname, Relay_log_info *rli); Slave_worker *get_least_occupied_worker(DYNAMIC_ARRAY *workers); +int wait_for_workers_to_finish(Relay_log_info const *rli); #define SLAVE_WORKER_QUEUE_SIZE 8096 #define SLAVE_INIT_DBS_IN_GROUP 4 // initial allocation for CGEP dynarray @@ -90,9 +91,19 @@ public: typedef struct st_slave_job_group { - //struct event_coordinates coord; - my_off_t pos; // filename in Slave_committed_queue::current_binlog[] + my_off_t master_log_pos; + /* + When RL name changes C allocates and fill in a new name of RL, + otherwise it fills in NULL. + C keeps track of each Worker has been notified on the updating + to make sure the routine runs once per change. + + W checks the value at commit and memoriezes a not-NULL + with prior freeing old one's allocation. The memorized value + plays its role at commit until a new has arrived. + */ + char *group_relay_log_name; ulong worker_id; ulonglong total_seqno; } Slave_job_group; @@ -106,9 +117,6 @@ class Slave_committed_queue : public cir { public: - /* Allocation of file_name that is common for all Slave_assigned_job_group:s */ - char current_binlog[FN_REFLEN]; - /* master's Rot-ev exec */ void update_current_binlog(const char *post_rotate); @@ -128,7 +136,6 @@ public: : circular_buffer_queue(el_size, max, inc) { uint k; - strmake(current_binlog, log, sizeof(current_binlog) - 1); my_init_dynamic_array(&last_done, sizeof(s), n, 0); for (k= 0; k < n; k++) insert_dynamic(&last_done, (uchar*) &s); // empty for each Worker @@ -187,6 +194,7 @@ public: ulong trans_jobs; // how many jobs per trns volatile int curr_jobs; // the current assignments ulong usage_partition; // number of different partitions handled by this worker + volatile bool relay_log_change_notified; // Coord sets and resets, W can read /* We need to make this a dynamic field. /Alfranio === modified file 'sql/rpl_slave.cc' --- a/sql/rpl_slave.cc 2010-11-27 09:17:41 +0000 +++ b/sql/rpl_slave.cc 2010-11-27 17:17:41 +0000 @@ -2625,7 +2625,7 @@ int apply_event_and_update_pos(Log_event if (!rli->is_in_group() && rli->slave_exec_mode != slave_exec_mode_options) rli->slave_exec_mode= slave_exec_mode_options; - int reason= ev->shall_skip(rli); + int reason= ev->shall_skip(rli); // TODO: MTS skip handling if (reason == Log_event::EVENT_SKIP_COUNT) { sql_slave_skip_counter= --rli->slave_skip_counter; @@ -2687,7 +2687,7 @@ int apply_event_and_update_pos(Log_event // It was served so inside apply_event() above. // The main RLI table is safe to update now. -// if (!rli->is_parallel_exec() || ev->only_serial_exec()) +// if (!rli->is_parallel_exec() || ev->only_sequential_exec()) error= ev->update_pos(rli); #if 1 @@ -2888,8 +2888,9 @@ static int exec_relay_log_event(THD* thd if (thd->variables.binlog_rows_query_log_events) handle_rows_query_log_event(ev, rli); - if (!rli->is_parallel_exec() && !ev->only_serial_exec() && - ev->get_type_code() != ROWS_QUERY_LOG_EVENT) // mts todo: check this case + if ((!rli->is_parallel_exec() || + ev->only_sequential_exec(rli->run_query_in_parallel, rli->curr_group_is_parallel)) + && ev->get_type_code() != ROWS_QUERY_LOG_EVENT) // mts TODO: check this case { DBUG_PRINT("info", ("Deleting the event after it has been executed")); @@ -3608,11 +3609,9 @@ err: mysql_mutex_unlock(&LOCK_thread_count); } - delete w->w_rli; // fixme: experimenting - my_thread_end(); pthread_exit(0); - DBUG_RETURN(0); + DBUG_RETURN(0); } /** @@ -3636,11 +3635,13 @@ int slave_start_single_worker(Relay_log_ // fixme: experimenting to make Workers to run ev->update_pos(w->w_rli) // fixme: a real hack! part of Rpl_info_factory::create_rli(RLI_REPOSITORY_FILE, FALSE); w->w_rli= new Relay_log_info(FALSE); - Rpl_info_dummy *dummy_handler= new Rpl_info_dummy(FALSE); + Rpl_info_dummy *dummy_handler= new Rpl_info_dummy(TRUE); w->w_rli->set_rpl_info_handler(dummy_handler); ulong key_worker_idx[]= { server_id, w->id }; w->init_info(key_worker_idx, NUMBER_OF_FIELDS_TO_IDENTIFY_WORKER); + w->relay_log_change_notified= FALSE; // the 1st group to contain relaylog name + // ALFRANIO --> The recovery procedure must be introduced here. w->w_rli->workers= rli->workers; // shallow copying is sufficient @@ -3720,7 +3721,7 @@ int slave_start_workers(Relay_log_info * rli->mts_total_groups= 0; rli->slave_worker_is_error= NULL; rli->curr_group_seen_begin= NULL; - + rli->run_query_in_parallel= opt_slave_run_query_in_parallel; for (i= 0; i < n; i++) { if ((error= slave_start_single_worker(rli, i))) @@ -3793,6 +3794,8 @@ void slave_stop_workers(Relay_log_info * delete_dynamic(&w->jobs.Q); delete_dynamic(&w->curr_group_exec_parts); // GCEP delete_dynamic_element(&rli->workers, i); + delete w->w_rli; + delete w; } @@ -5360,6 +5363,13 @@ static Log_event* next_event(Relay_log_i rli->flush_info(key_info_idx, key_info_size); } + /* Reset the relay-log-change-notified status of Slave Workers */ + for (uint i; i < rli->workers.elements; i++) + { + Slave_worker *w= (Slave_worker *) dynamic_array_ptr(&rli->workers, i); + w->relay_log_change_notified= FALSE; + } + /* Now we want to open this next log. To know if it's a hot log (the one being written by the I/O thread now) or a cold log, we can use === modified file 'sql/sys_vars.cc' --- a/sql/sys_vars.cc 2010-11-27 09:17:41 +0000 +++ b/sql/sys_vars.cc 2010-11-27 17:17:41 +0000 @@ -3052,6 +3052,12 @@ static Sys_var_mybool Sys_slave_local_ti "time value to implicitly affected timestamp columms. Otherwise (default) " "installs prescribed by the master value.", GLOBAL_VAR(slave_local_timestamp_opt), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); +static Sys_var_mybool Sys_slave_run_query_in_parallel( + "slave_run_query_in_parallel", + "The default not an actual database name is used as partition info " + "for parallel execution of Query_log_event ", + GLOBAL_VAR(opt_slave_run_query_in_parallel), CMD_LINE(OPT_ARG), + DEFAULT(FALSE)); #endif static bool check_locale(sys_var *self, THD *thd, set_var *var) --===============0538220478== MIME-Version: 1.0 Content-Type: text/bzr-bundle; charset="us-ascii"; name="bzr/andrei.elkin@stripped" Content-Transfer-Encoding: 7bit Content-Disposition: inline # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: andrei.elkin@stripped # target_branch: file:///home/andrei/MySQL/BZR/2a-23May/WL/wl5569-\ # merge/ # testament_sha1: 2b0ab3fbf924055f6d3bee99c0c9b93bbefc0f4f # timestamp: 2010-11-27 19:18:19 +0200 # source_branch: file:///home/andrei/MySQL/BZR/2a-23May/mysql-next-mr/ # base_revision_id: andrei.elkin@stripped\ # uxb9pgwaja1c4489 # # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWfJW3c4AHiX/gH/1oAp///// f///6v////5gMZ3KZZfPbvj333PfdDPvnW97271uRkXnvru2rtKXt4PZdC82l7N1ixAkaxZpbfWK FOgaGy9dO+CynWl1VK5xu9bN3Jr0N69ue7BN9u2997kNtHcz73B15aZOJtndhy25zg2NrDZhWjLT uo75ely3u997vrt3ZTfYrqTfCSQmQBNMTQJiGghpqeqeFJ+km9NMoTTJ6n6o0AHqaGT9UH6oJQQa CAiamE0m1MlPKaMIM0g0YEyYQaNNpNMQ0aZMNT0CGppFNpoU2mmU08po9R5Q9J5Ieo0BoAADQA2o BoJCRCJkwmkZqT9TR6qeGpptJtERowgaGmgAAAxNBoIlEAmgIAATQMo0ymxNJpmTVP0p+qNPFNMa mgZGRoAJEgTQCmEyJNqeqfpKb1PJiNTU8KPU0HqeoNDQHqPUB6gBkbAF+sYhCNDb7P+td/6jyXym Twu/n64zyH1Wl7rHr5qtSmXJVRpBuQE/jl4WpH+JU0+8iz7L/H0JWvNr7nPubD86TV2Ry3Rah9MD w5843R19Z62w/dem9to0KfyUPdqRvIjc9FN/TbMW4yazBpEbJw8JK+bvRvV9HsH0f3nVwqf54Ns8 R1FR1MtVfxQTXA3+FsDh4Y2TUivC6LwOg40SilGmzdggImG7B4HfzDVk5MqmnAYHEadqcaPLu2PJ y3/o78Sna+ov2P6SUDnB7BaxdejCA7u7xeZgnYnbS/wP120K2qeg8W2RvlB6k1wxn93y1cYSbWTz 0Ma86+7XNnPgqJP3PjwZDsiAdN9ITf74UVE0343sIVQcuxJeu1FuFNXAh5dQM2Tvju+nq47OXo5p 6t03PHo7cY7Se19a4ritI3jJoGsWV7O2m9bJHFt8fO1yXYeounvZDpUTNtUNF4jxmlOv2yyNryPJ Zi9mI5FtpZcrYJJOyVJqQcJkyO2ghTYGTKtasFkzIaNWi7mwjCEFWSLQClqYbu9cTDDhoc3q0TSb N2HyyBQ2IBUBVXm+/HBs4KqJJA0d5cK29OGd1hmqtX4qtbi5yAvLXla1EF0iyKpAs6RLizPKTU2G BixIl7Z3KCwMJC+Dx11bAkbbIxNhywpJIl2niqc0Xw4UB2nAdNaKIMYeLCdX88ujr24IoOyCoMiE EDOLWKdkRzgieCOEVPHBIQKQEpEkL6cUqyRi3JaS8kRCcOKzmm84U9hzQbM/QwoMG0WvCMrbdTo8 IeY6ra9FzM8CJ6bfZU318PODFw/WBqQwappSPaJ+spMkEnEBg66k9F8hyh1u3HJ87asvxsWHP7iu +H+Bt8nVYBn6I8srKQZ84DvdIp4RSJGISSSEIMgySQhIkIoSEIEkaQNz0DXB7me8kvU0I82vs9D9 FbeypidGoNQGLJBByxjzrzalsKtMFjRiNuLLnZkZGdrrQoivUvCkDNXVnld8ofFhBHwTYMrC3oK1 naaiVzmzIQhR0wZeJtAm1qWNgWNDcVVGSCGBCZ9jAFYyqKmSiLIw6LjEuFNXKSbqgYgkXIYgXhC5 V8hds9AtRjaUkjI0pJoxRM1QEqQXWGQqiLK4VCq5zQF4gMJS01My9bXhkEIzMhWEKdZR5TO8GO1U fj6OQ/1uWst4fE642SI0hzxdB3RnB53+c8ZIQkIXKMp6uytPm9WvyZY6sj18TRhkY5N+v3zns+Dr vhNnatyBdb0xRO37SkjSfkfPvJF2XZwg6HvV+i9vsjFznsszTnXTmRUghYb+/ZKy10seExUZzNKL hnM7CZryD98oT2q/Fy3MP14RIGy8atPeaGxjg8ocT1tO4Imlg9q3z+pjDrXI7YgRIrRCbau503wI R9lEBz3dJ3HedhiTfSRc2251h4N/2RFlTqv0kFYQnHvqeAwm/PGd2Ur/uynpqSlZkdYL6B8Tv8FJ bve909xd+jk2c3Wf09JeIfnwIldP14GPT01c9xUBfcK4gMoB06EMoTp0Bm7kN4xpjzwtH0OqGspT ZI16FgHRCqYzSyLbx8rvAjpgUxrCkMnd2ArryraviR57mi6qNMKqu8QAkPR/F5SkHrXyJ5WAucYq pFaFAcccS/3HIYxjGY36PLptF7Z750ebW+6GBVjrqpmKJcrP77JVK+Wf2lgxoPyGrob31g6o0ydw 0kd+fyWtcqRCXXb4MDsPZhl7RbMpl8YqiRXOHN/W2ou8ppDLPhXlrWdK/QuJTNiskaLzifx8cxTu NXyCB1kukRjIvfQ/a1VnsoOyQqn1QBQjSbzxHQx8B1xVATy02nXrWJRE1Be3a6uSNLDXRpa4eCgh YO0zM/jiFdd15zh8Nbzxc/VGNZehafVl3nTQ9mEfsAmzrXIN7mrljEzu8cVrqJzFr6vKs50oZzO2 WoinSW6PEe9dDBhfP+eeU15xE5z859QRJyQ70n8Ue0iuu4+Oaum1p6fcQfNr9naaCMu1y9MEdund 2WpstK9l8iBqVMWdZEAXB4xFmSfd1AFF52+OXhwKfjmdMMtljfdy8+y4eLUa1i0sLxLPsYit7PLB n//B54opcv79mSvGj627PbZSTKynN3pTh+1bPY1YWz7XaveuPG+G6Y61/rzamJrquvzhEOZSOSxJ Xxc9xYvNfh/hfCj8LBzPiM1mUWtM+Wh5vjtr8I+/iZ6obybTRv9JH2sBibf0Hwm9/B8U+C58XCY2 7LWvYw9K7i+QxXw2IocVDYWpsbYNtttp/bXSzQ0ByBqh8kkE4SNtvZW/VJ8rLIac6LdlRPJvbNW9 1O/dr8wCP0mnDDR+7+j9n4eukBEGB0REpFQ/Sg1gJIkiTSfLCtQPGQuPiA4P6WPn2jmEMHkBL6+W //OxmnnDp9PZT3FJFBg+jfnxkfDgq4GR9Cxt1w097EiAgiHgLNgP7KYxkb0Xvb+n3vObOq/M0jSM 1PMmeY3QmGiFJB879W3UPopofh81oK8Z4sXBezn8mWWEdJsMfYq6enCPV5fm+bryH9U4tf3fL+jP 5LFXsYHlLc0Fp74sJDXnEwMCVJOODYTM99TKrqHf7RtVve74Ozs9tkR9Xm2Va64Y/G2nCiNenR+M NqD52uX1+3beTQ2JsKlZEui+wPIcDx8omuRCQa2tkMrXt8BT9kDcwP2xf0EF8JsMeNqHA3Ztt1bD YEaJa/3ft2bR1T6Cct9EHYkflF1FzBTKHhYdp7TiPsPee08xzDu+P7uwS4TL3kn/pGiyk9Q8ixGh UhVKhEgET7hSgwUStm0eEU8uEA60uLI9Js4g8IHv4gUBAT1c/5DQF0zUqCmohadn9I29I86RNFi/ QHSjA5O0UPXdPFs4xDgDUnhNQdu5fYrsbmAUTjOZ4wx4Nn5XpkkFDFUCRnSsQReI5SLrkc7rKNV7 zfbQ3b39/8uqUq5zTj+ar+OaJwRb9X1XhYkfUWft79cF/U0hvVbK0zVXd1lJGgFU7QtApKkutVR6 wcMzHmUBvwpjWpVRKIiKFKqsimMbwWBmLDGNHeNJ5BjIMGORxxkIRNxjkkue4B7qSPc93t+58FUd rjBjaYmxsBdPHVJEiSIkiWpN+3zwoo7m62/2b0OSYupJCDcmIYF4Fjm59HZXcjfJha3sbEYgJdjS y6oh+e2UsF2JtIjEszFNpYu7RJgTdmIJYQsiBhlJH1DKCQXglLQxEYlS5OghYQLBYpogJoEYgLUR zAggtHEteg/s9N2su11EbwySvuGKrJ1yZOsTKZC0OBqsvnsGxfHHQ3JIzRQ0Phobl9ILl7aXHlk3 UubFJTuKpMuWuuKAo7FSYkGJYyJlTCQpoC4h2ChOghamqxACFNVQcBlzTU1wCxMKmQyc1cdSR1Eg mWVwWBqa9WZBX5BFARhMWqzlKgNmNEAO1P1Gmc65JNNg7kJsh94n2b2kiz0iJsh25Uk7ElupJUrB YETSdTUTtnvAvBQTAIn11AFcJvcFl/rmvEkg2OcIVNUQEUBvAriYEDUHEWJQqKXk4qqBRegEscYI gAzJbJOF1NQ2CW1n2SjXoiilyYGQxFWCRDAFKXQJfyb+Wu74jpjSO0DcVmvTFYx66ndTYPqudEW3 R1M8s0RG6pY4jI7ovJcAArHq5oWuIlNo9gdQxxKDyMWhcJAjoIvuE2uROFvY1kWZQRwylKzEHJWS 5yIEJ3aDiBoIcOPNNVTwNF0oYM5/A9fQbS+d2b2wmmab0Sz0HGXwj2krS4N6hCg7DU7JBVg5+DPM JQMGl3AK6JnChftOZYR1qeYgBZQYDwPCdEkqGPRoWB+cEYaUfHsPUdEkS6MIc+g1nntKwvkzOu1q rE69oVMqrvBlOr8rK9RonSUbmjQikYHralMzwQBhG/QJyLDgnB3nFc1yWEQJdJzpfyNZFh1GwrJf oOhZ5GET4EyBZEiRNTQ3jyJpyNTHc1Oh1Eg1z30yIywKkJHIgdiVXYDnEjiUEpcSIpcZYnfepJ2i giQNsEhmO2djIotidC2ZxqOQAmJDRzKncbkpYByNdibN0F3kMqUXBXKFJJQsYzlssbiBgWjETa1t kbYFKQEgLWJxFg7A0GgqLj6ivLjHk9U0kwri4sNJEDwSihkLEnxEEBI9KF0I660KIagsafPLhM/U Cc7RNJ173xP/uaFF9eGFZ6QPxcar6lic7Zdievf8Aoe1CtkE8864Uu4iBqZXgYUZNRBJzGCZVC9K hrpZySqKT1gZZmg86sxkyrZQoNRHatJm5jx8coULEt74IMm6SJ0aKesltjAnkM6FzqO8LfhmQddf ho6GjIIZ3kX3M5Ecg4UKWEKaPCoRqMDSGK2LNaXsTp6CYosFiuDzbkXFx3kmbQUELQn0lJIeJFpW QOIIeWECoeVkTIiVZLCsgXF1FBNaniRyFsFwE4SKBIOkVyFki+mnIzhnrR342VerJ0ZrROJVW12l Jq0a8V3c8iBJIWVRTJAkunXJUAPQFY0ZQRNsm2uxPJ/SUjwPe+NTc3sPCCaeyc9NKOKIkXQ2sQLa V5HI0q3whZZpd4jFQbEFLjvpgUYyZj2FzRCr5+ULsd2l1SvnyU13MQkq6FTQ2fGbFsVIPWcFOiFr ExXEY0FRKm+N11GGF9qKCYcYlRSYGCDgu8GGJjEErJHANBoMjVJHRA1MXkSo5JXvrIjyI8rMx8+I yrIwI1E7F1xpJH5hI4ceSWc2TVeoowwsfpnOGJeC3erspTdoZhiSxdDlfQqyHkY0W4IaGw8UHb6e hkG9McTHhVG26FZzC1rKJHZYuHW4zPiIMbtnr3ViS1RI4M2EkynxCRmbYYWiGKS2mBQhcHjtEh5S XlBSUjKcp/BdQOLB5iQBcWNjGZsRAoNNY7ku3PXSh2ZTRc/F5cSiWgbncTkWMxy7dTAoMfOhYptz qZVOpjmTtW4hXFZeYoZyQi60lW42mRURLzMmLS4vJDGONBBBkbY04JFzU0PKgqDAtWFONiZ7hA+R CVhP2NJgUjG+fUHu+fKp0nA5XmWtE4wrKdidLm12Viy2d23RqoKMC4udwENrcahcQYaCTgLExHKY lnOnJzMmCRqMw2LWIPrQrYaMfXrI0E18WCc+EE0qTOBMxwW1brlL64cUEJFyzl7EjhTwIrVpwvIp c2K90Ro9TZeqQTN6zQ31LBtjZgodCHJjXn4hygiOToXIuOSEgwRdFNDiS2J2KYGzKmJFhkvQlBWw 35eOqOxiR3CQJhjUzz0ON1Oyy6RyXlXikUrOKTcpWiu8ijrKspYTJcEarTEEYyx/H3kGBCgesQxt faeWf1CwIgbCBg3o5axNn+JKymjTnyNk38PK3edhIQ6iIb6ZM7Gpqc7RCrWARK9ebD6k/KLblIJa EEsuRiXNQzuSfbsVzreUMdog2p0FOrGld4FyzgKzGCRqZM6UQqeSIRjrqrKL75N8k3oAF5qJBfBU kXkTNkomhAxdKDIxqcj5Wrr2FJjEcMN8EdguMkaoPSiI670axcstmnVbHVsfnPNKDUmR6ik3ecuj LUky8GELhEFhywMmBkfBzk8bsaadrwvfkyWmaq5CTfTUklnhxp0qAhKTDHQ+MIH3NRkxggkW23Gr M7yZIsWpkcoSNplNRxps5GzzJMsyopw5iwyqsMjy0ItPwFZDfnY23txG4rCG5vUkbmqU1BSidwMT 0qcyaosbkpbHoFDJIm3jAnwib6OQ+0uJS1nei6rDUSdUKZKELBTrDOaVC9NCNsag1iYlgyxCqF1Z hDBsWuelMVUoMFjQYu9qmOuO2JdTBBRjHBoVhhAyCQFioglOxrS/KXNiowogmCeEyTJmg0nLHp6W L16jdeRxiD5MSE8A1HYPI4HDuOxYySOiWOON5hHEEEjsnNCZkeDckdCuuRQMsCk7C5wjng30XY35 2c7bVhoByS8oWZqLmHw5bYKsC5IESoDcghIZ1SLBUHki1KJKYJVV3dkMJgBZIgLhBcn3cK8yoJZj UFQ+p7UlL0AOrBgrlDDLc7of9vwb6/c4Pk2WUu8pOU0NxvbMcloHHHKi+PJV15yZ58WOpnt8fF4L bWi0IPEQbrb2McbhxaNodz64ZsZpgNOBtGNZEKNUCYo5PsOm4qKTCwTBtE89tIDClCmCZbFAopJq iLDJhTxUIXstOuW2FfZF92c8NW+o6z99qcHtBLhuZvbA91/L55NN/O4trINIICjp0s9E6boi5J7K FIlZUC3ZhbAf3eRe4XEHeNMbBjFjg0htJFlBKiUCLGEIxUIZqMHtD7QGp730An0hFVqvugh/MTQ9 cAgkFEkf6ywOf0p+A/R8X8Lj7/h9x89D99RK933/vM/m0ZGv69MYH8JztisjEgERJAkk3VdWaOmo b0LY/uLE/GV0QTcT/v8D+0hzkLK/to/qmt1upK8v+bOSh/ac/0/pfOztKGxvv6+YP1GvivGwSKHS YhfyUEYoGiRKU2AhWaR500sRLhymR8dO/P/n/G+Qcbhynrc8gp9IjaJlhjK4bkMXsYczmOe6JaPQ S/fE1s1H+ElLeFpAdgx4F42QMhmffAvhFR+jt5aG+1V2FYStLZ030+m6aiad6StYT2R4QDuZXE3k +/NGaWtDC0SL4bCBkwF268rKC05HDloc4aJ+JO7cAyAy50kmSbkzBomsSgUSvZEF3QZAYhtDg+VX ThOomAur+aY6RGix7COoccgNCci09G92uAdol0CI7W92BASHqN9upw3caadS6MGas7Fgqo0Fb4oo 6HLWygDTI/FBzQoIJ9GOtT47IKFyQpR5sppwJnlpwMJQEa2FEajHlpQO/kPtPcfU/BHDl6xGOUz8 nybGtRweeVO64SZh4TdLCwtl5WpUVCw4th6nTo7c8d34u/gH95JKfAsNlX4p6lT3lT4p/Qi/vufa F5BfQi1JlWgVhLERYLhOBYiPMGDKB8CJcQcYkimCmoIpYAxGw98Qxgri0Miq2IUEAzQxoi4kE0f4 myUYkiDdKuAmDZiLAwS5dY6RIRSMuEhTX8QkWqqgEyuYlBa6JgfxsSMmFAVRYIIzAW7AVAdG3dhd Yigr99rLcwDhZKbYFYEMF0JJbCS6I7QhaeIv77DxMO7zVQQ+0cAWgF9v3/o2mM71uVMhhZ26gJf5 CH71/IfjhPymSh4lTr8swoCjvXWD5v98kjP2hMmCx+Y+ZOii2Sh8xwU6mpCD7SmxfTPgCYwUFGuV DAmL7zQTlOsAhoHAG/qGZvQ/ErnNJUN13Lr6bQZJIXAsYOqKpzNzjL8ZETjkNyZXlpse6rWdBXVh eINKAGWzLX3MdySO8GDcJhGBlx/1Bk5ztso4xBAhoPgly+4+IcI/hrz64wow5iANZhrY9iD+V6I7 4XZRypkFUCqfsmkTUawYpRUMgmiJMgDSmpdevARmQnrjKM7s4fdZxhzWhuh5YUTgBPqICl4wFZiG WHIWaHjFLmHJjTQsRkYiqIvkMcJtXOoJ4wQQbjhCEVkgqrHymiYLTmSMhjsNuEUII8aAfJmEzGCG TEZN9aUjAHejHprzBbdTHBuAuOobNK104jIr1hEANKTdnQViyd+i0ObBSwkXngOIaznmeRHSRIHa XzBA6Bec9R6jSDEuPAgp+QsZ9UQJEiaMi4SoQewywHuWPeIkl7RDViuRZdAmbWSJoDnIsM2LuGJe aCbURJYFxqxNBaLv7tt5dkYH06LinrF6hJk7rGzMzQVmZ3In+y0BUBYqdqJC9cwpxj1jHol0iHV/ BeQLrHtTerj2jQ4YZnJpkncVtVF3dpP8IqRyQnaemMgWiKSTxc5iYkdZvOI4EPKWjuGFcPYYCD0E HOo1TsXHPee0c2LlCvK70Lk1llhac90isvtNaWKtMhMI/4/Z+p2+S41G0s0GJsG2Xb6hLoT7wv3I 1lhzBIYpMiJaTIsN6VNCwj6EnFm83kPbqRuLxxPRULiTFpeZnUiRdWw6kHQzM0cHHUwft5OXifev I3hmxkrhzhFoPBzxyVHD59a7BK7FS86ABfLYVnYJN4JfC8FsTQMLCUW3a2+agu0mcvICoSoP/BPB 1nIu+dC8K6gDWPHxag5KUFhCMTjIDKKweGnVNjYm2jCqPB2GpZ9kruYOjEQ5EEJCdkjT6WDmLKbE T5iGNrVDjUCshRsDHqDqAUBFABP1dx4LQAZZBa8AgazmUD1BJTV9xE7SYCI+GgB6ghUHaSN9JMBE vLQVtbzPjdOYJlBySHSxA8jS5vJDuQRL1OuQyYN8gA04GAQfByCDkajyC52jBmoIUFEGM0P2qg0E S8xJs4kcB6A9jC6BwkSOii+PaKrnzrLRU5mQ7aOIGCV1xgRLtY+/eUAecB59+PXkkJkE44B8jx8z 0j5/Z6NjuNjEUuJzNjbjqss5C0oCfwduBKresWWNy0TrIR4zVJUUHU56SBdgnMKXaxXMVhgJBUin 7hBSTnYfn/Kh2/dGJ6Tx9o319k+39feX4FChObtxoc+2C4NkHpkr5/H59baTIpYJjCRmAMYDGBMU iGjIchm1agJ0PDEjMkmFryRHKyoti5ElpMbu1Y0pShFKFuC4UZRBXDpS5qYLwmATd2ADUDWkyqrl kTDuycSjCuWiSKIk8XGJ2EXZggmkkJoGApUjAdJ4nIo9Ax+Xxm64eEStMd/YhRIxJeIM/mN1wcxx hhI0VIkg6tCJpd2ORdmdsmddVBkHGYuMj3GxrUzOJeUhzOhsQIsXJmRhxMI9fr4rR9zUwihQwfMH MAoAt4urOueRRiG02BHDrVMRMFAiZIJQLiqlCCPGcBvbzxZDzqXQuDBfUQtclBATkj3CTjc4E6ou 6KNX6i1ZZgi3aJMyFzhUnzjxs38R4SiN5YTGnhEi0ry1ffOtnj3PckHQg0W82RHwmAXWxLzHoEbq 4zs9vhzbwhUES2fVtImkMuPZfAMKsO7YLWLLsLyRxICQxWWTG0XaOIHZBfIqtSRgJAswYSsMznxR IEFteOc4c8+jVANlFDTo8QmFUgVyS0oYA++1yW0EenpWaU6LXpALPyM15uyoTTieD7e2JsLiD3Py 8/+kXHoZDRQonTUaQlYQIgxBQdCyRKYL57NpoZv8q98Uv1iBp4AB5kbCu8YJ/OAkE4KBqox09wiG pzmky1yh0b4URJmQmYntJBJvuggFgDiqPbHAyS6yWjWIKq8LiVhIxsENhiXFNgLdiR8eOWnHHE2P ZHCtsnSCbKwEB4OopxfgkmXlCuQ0TB/ZWETRYDFIgaSNJDChXCHWla4c/BfCNcH3R722yUIQBgct 4oRNwQTM1NpcAwT1j2FlzAR9HNzru0cAcZRWJIQIMiHdEwO9JL1nze2q8j0AyhWQ+495UPbQ8it7 HUtUBWIPYWufKqlJBUzwshaZegwWFVSmBUx/KjK5QNl5Hy6lDY4EDYwaUsavfg+D2ZE4FOcKKaud BSC0pHGlzHsKTRgSIz+uYEpxnpxIlffndrQlkJFyUA3un7QGjgdQDFKCOkngDZR50mAJ5ANop3mk bEAtTzxHdEyCMhAgQzC1r5gMkdq+SwU18+AZ2iGvgTRIxiBIDEiMhI5gUMcLOcE6R9QlS68iUgA3 +QNfrTmTz600uJAKxaRAIkZnMHgA+hh80RA1GeAX05JIuLGp+wI8EHiJEqNcAYxkB1FKFNUJ81Go +u0uMu8cu0U7x6BLgPQu728CNjJ6KKbKY6svroIhiQ7hLpFCTVQBLiJlcuthcQxgMO9um66FDAXr odFVCskSRJAZAkBkXtxc2EqUer2SIVcJLUU1XUdKVYo4iQTIl7QPkFC6I6CxsPkQQvOpvFLAPCFk CSEFa2KA0AsWlU6kyWryXXkFK7IDe6YdEJCSSMPOibabfbkJrhy34LGwYCVt+k/jOJ4iEOcCC8wN Alr1G1WDVo49w1sW68vbn2BmCtnE959MJRDurUdCLgBrQz9SaT86WpywtAU8NOFrRmsaIY4DocmA YRc9qpYCDwimepe1C83MzCmFFADDqa6BVUzxkC1JgO0eGwppPgYeWjdNtaXE4jb4UME2FFT4Y0Pp drODoTkmYrYfUcIme5LA5H4ihsEiCXiWjuT4Dx0HaBkktmj2B5wnDUjuA7IJCAkgSQIQFjAg4nJ6 eM4zf2E1uojCLCEjOkel5XkNadBpRDrx5w7GDEVQNb2xiKIkEy28NYWVgPOw26TIDIAi6rS8Vleg RynfDEvR5PRrAUzIc6IIuDoBOxNQ4LZQT2CHenKm12DYniD08zqJIAbB1pvawsDzGo6RNOkUyRWi 2pvKgtkAYZnAR1R2TzxiQvpKJc8spDWExoiRDKJQ6ZKFGU1TlN0pHB7wapioQMkDzoMhQkQidMCV AOzVeEQC8ODlG4OiC+YnHBNDBMyEkGGZViCagyonP12eC92vuTYqECKwIoVQDDEOmtxoAM/EWaO8 upQYAwI0071TjVRgDpKlwVUBhQJJCvlZVQLe8QBG4ZbSek8hdwEwrwPfWC91AkBmCKnWIKy9ls5F 6ZTWCq/WnAgcGskkY7whsCnCEAQiWWBVv8inAnA8hpaUYdPjYPGF7C+yqA+MiiUtYEadGzk1sMfO JBuSfAzAkkbSAOmEYIFBq5tLb6tjQe1KewCkkkzYSAOA0YLVenmcZ/NI+xh3ipD6QEMqJdGlzx58 gNGh4uAQYCokbtbfiALAXBMi/vVqMTwDyCmhE0tRTWXkA8YJfXSOgMMwXQZimwDUpyYv16X5kdNk nZSnzu+hHv26V5nxNKN8GMxlG3YzdzBFId2VYasc8TcHDriC17Uh3G4WhBD/dFGruGMgwogQTYBx FzlCCkCYIE1YTCCRcAgqXonlXOucYmxpDAXlDKMAiuYvSFkBIkV1kDOyyC6AGmolqTBAsCZNtsXi JZQ21j+DKBb3S5jAeoclVCLoA3mIpSIYCZjWp1Ad94WFjwl6NF9FN1TG4A2QWo7R1i5akuFPYOYp fgbdGdZLFSIZpQYEIEIBJCQsFo0Iq0CMIgVJlMcZEikeMlJJME2CTaAPoEGliYK3o9SMEv1lmEcZ Cs9GN/WyKECAjZ+RxJ+mCyAuIjxBM8okXIJssDFBhPP/8n1pRTEQo0xGBtoNBbnkHIPmVIBusLvR p5MlC6BxJy4JmifeQVPOYZOYJkmkglp1Z8Wsd9MfY1BI4goyBeR/GKHE4pAzgAUYcJEDvG+irvDL S8WB4hAi7EcJ0HUT+pv3QRFRKiYBrkkrokhgyupDS7rFshEZ/C0cwJt2bmeKqu1ZlioHakkKkAQ7 oBn3GAz31zjGDGNNSEQIQIRLkTVcDpAuxBon0AdLzQ4YNJns/XNfgtAteeGvUKUEHmXfxa7JbGnw BNRosoHlv+NgaktVDLuHtZYsrKSBO6UIFjZSyZhZqVHfFDuHpOW9Moowjq3dgUhEtjCB32ggwI0L 46F2AwEwAYK9lS2xTXhaKc9QX8wDTZDAzSzM1CH6BSi+kRlhfodRoSMvDQr2tDmobCGtmSPV1R+V g81MBTC8vXMm+fceUrJ8GcQRw+ROHjXa5N0mJkYV+1OIxX7BQ8Qu+HjFO1dd4vq8gH0+4dQocKi8 YEO7kFLAd3D4YNCKVNKRdhA6CQY4m0EkGzrZ4KJYkGmyS1QbTVEIHtKEMWg0sY7P7TOoQw2CDNEZ MbhMlaCGSWBZABCggHIG8jZjeEVUkMRNJpWsDHFIkahg1zAPfirn3v5xIGqMuEtkdotELxluMcgk AZAArdHUdZT28PurJQh4cydtX3espdPGGAUBdfgGGuLfXe/OkTDL8PLZGYoTISTbTvyBmVtbhw5h 3J58bRlDtuYS7uklkagzoQzYJGlR2O+JFArwVtBgG8EQq3oTkaQjidy9ogT5iB8Zw6lN8kcbqHrR UTTViEGMmNUr7QDaGxbkKhUfdsxijTGjwy5QkMkkGxsiGgJN2cyS4g1I7C7L4IqYiiUSToCEEjsg 4v1PedLAMKMQUKsiZ/ZJd9J01KxWvSWNoJtFqYPlC8qHNigJDJUGnK17mnQ4sFvrUrWUZjAzSBzM 0IwraoeURHQ4tm27IF6sspmyX4iNWmFQKpfShe0MOdNpYhnOYxQA8Lg1/KTWma7Use4OMzTg4iNp CCQIECJDCAUgUhBaG9jvF+cPQAXz8cwqh/3OA88USR6ZzMl/niLEMPH0HoJT/cw4oYBtkDbYA2lm ArY0OBKlkKjJeSIhUB+Us/I0DSzmE54siEgOiloE03EL0iI9jNdIs15wUgsjFY2NKen1gd4CUa8E uQBbk1zEUUhCQhIor6/zGYp1DehAGxQ5zIaJQDQco1/TCpJ2QQqhRZTJsNdSfvYaThUwSIhyqaIA 1POVVIyYCUTIgh6KZ3KDd6MgHHL+GR8MMsW9nqy2VmGwC0olBbBQdXcxdcx44LJGBcSyjLSjztW2 QrvKEFyEmhRxSgZImEy2XnzbLBBIgiiG5Zku89CcgV8TclWlWUgi7VJkxEhTSQwVWCt5BXJZxKkQ CXSR4iA5jEf9BRh4pM8hIGe2LfAeWxAx64mZBrFUasKxQCcSNxsAHWXYBC5UuCBTBA1A/rCZNgO6 LFrCe+lkSkoNSdyhEo7UaQhnsr7eWXTlaHF2y4o/gxMFEcVeyXviEc7vgMDOhslmAEsCh508ewTF SNyHaaTuKnyxXxQ8cRDjwhAL2CRrkE9Qq0nIl0QwJokW5tekDhPDpeqDiMydYQLSqVBQyDEUHB0q IiqVO4As4gT1pz5aS+aNNJBgEFIh7YBSMnFxBrGbSlnB4ExdaxAMQdz4oJeWBZIqMZaHfrdqtKjd MkqzCt1xOABZK4m7hhUMc5TnTQuM1qk+wfWCYCnrGnAm3ZjyiVEIw96P+QERJUU3ThDhNUuMQ38T gEPCFeaKth0pVaiDLlECRnIKSSJNEMQEZTpFojTMk+m8IiQxBvWLkCPRdURW3oKKVBwXAZin1I9Q 9YnSPOjaNHMR31vS0cB3polKUFNEFWvCmKocAMD88v4gsu7gO4d4dd7AmAob6e0agnMWm2fUCcFu ASJILe+YeGtC8++GsYPr9bB5BHqByN4h44OfX1of1jAKAR2BzFaj3wwfkKM3y8wT8IZCFxftaiZA 97siobQrBCBprMj0o8SIjw9JZHGPjEirZ0J941RoBxAegUPOjYgvwG9PIjV7XHynfGywqYjbZyzd SqRmyth5SwqSWAngw32qg+WAQxMxUKQjFDaQQoUTQUnBNyFh8gdjoQama/qLDEV6+NPQlwvejFNI oX7bArluTs4kwUqgwqlB9AegDH6SFgnhajD2I3hEEbkVgfpOosYcjsHvP/gF1Jxnwp+3S1o8Va3D RPQquQ1gkBUJIJDf+LuSKcKEh5K27nA= --===============0538220478==--