From: Frazer Clement Date: June 21 2011 3:21pm Subject: bzr commit into mysql-5.1-telco-7.1 branch (frazer.clement:4254) List-Archive: http://lists.mysql.com/commits/139600 Message-Id: <201106211521.p5LFLu5e011802@acsmt358.oracle.com> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="===============0096834017494391638==" --===============0096834017494391638== MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Content-Disposition: inline #At file:///home/frazer/bzr/mysql-5.1-telco-7.1-reflect-push/ based on revid:magnus.blaudd@stripped 4254 Frazer Clement 2011-06-21 Commit for club run added: mysql-test/suite/ndb_rpl/r/ndb_rpl_circular_2ch_rep_status.result mysql-test/suite/ndb_rpl/r/ndb_rpl_init_rep_status.result mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.cnf mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.test mysql-test/suite/ndb_rpl/t/ndb_rpl_init_rep_status.test sql/ndb_mi.cc sql/ndb_mi.h modified: mysql-test/suite/ndb/r/ndb_basic.result sql/Makefile.am sql/ha_ndbcluster.cc sql/ha_ndbcluster.h sql/ha_ndbcluster_binlog.cc === modified file 'mysql-test/suite/ndb/r/ndb_basic.result' --- a/mysql-test/suite/ndb/r/ndb_basic.result 2011-06-16 18:16:01 +0000 +++ b/mysql-test/suite/ndb/r/ndb_basic.result 2011-06-21 15:21:44 +0000 @@ -76,6 +76,7 @@ Ndb_number_of_data_nodes # Ndb_number_of_ready_data_nodes # Ndb_pruned_scan_count # Ndb_scan_count # +Ndb_slave_max_replicated_epoch # SHOW GLOBAL VARIABLES LIKE 'ndb\_%'; Variable_name Value ndb_autoincrement_prefetch_sz # === added file 'mysql-test/suite/ndb_rpl/r/ndb_rpl_circular_2ch_rep_status.result' --- a/mysql-test/suite/ndb_rpl/r/ndb_rpl_circular_2ch_rep_status.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_rpl/r/ndb_rpl_circular_2ch_rep_status.result 2011-06-21 15:21:44 +0000 @@ -0,0 +1,144 @@ +include/rpl_init.inc [topology=1->2,4->3] +include/rpl_connect.inc [creating master] +include/rpl_connect.inc [creating master1] +include/rpl_connect.inc [creating slave] +include/rpl_connect.inc [creating slave1] +include/rpl_start_slaves.inc +Cluster A servers have no epoch replication info +select count(1) from mysql.ndb_apply_status; +count(1) +0 +Cluster A servers have no max replicated epoch value +select variable_name, variable_value from information_schema.global_status +where variable_name='Ndb_slave_max_replicated_epoch'; +variable_name variable_value +NDB_SLAVE_MAX_REPLICATED_EPOCH 0 +select variable_name, variable_value from information_schema.global_status +where variable_name='Ndb_slave_max_replicated_epoch'; +variable_name variable_value +NDB_SLAVE_MAX_REPLICATED_EPOCH 0 +Make a change originating at Cluster A +use test; +create table t1 (a int primary key, b varchar(100)) engine=ndb; +insert into t1 values (1, "Venice"); +Allow it to propagate to Cluster B +Originate a second unrelated change at Cluster B, to allow us to wait for +reverse propagation in the testcase +insert into t1 values (2, "Death"); +Allow it to propagate to Cluster A +Observe new entry in ndb_apply_status on Cluster A +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +4 +Non-slave server on Cluster A will have no value for Max Replicated Epoch +select variable_name, variable_value from information_schema.global_status +where variable_name='Ndb_slave_max_replicated_epoch'; +variable_name variable_value +NDB_SLAVE_MAX_REPLICATED_EPOCH 0 +Slave server on Cluster A has current value for Max Replicated Epoch +select +count(1) +from +information_schema.global_status, +mysql.ndb_apply_status +where +server_id = 1 +and +variable_name='Ndb_slave_max_replicated_epoch' + and +variable_value = epoch; +count(1) +1 +Now swap replication channels around +include/rpl_stop_slaves.inc +include/rpl_change_topology.inc [new topology=2->1,3->4] +Get current master status on Cluster A new master (next pos in Binlog) +Setup slave on Cluster B to use it +Get current master status on Cluster B new master (next pos in Binlog) +Setup slave on Cluster A to use it +Show that Cluster A Slave server (old master) has no Max replicated epoch before receiving data +select variable_name, variable_value from information_schema.global_status +where variable_name='Ndb_slave_max_replicated_epoch'; +variable_name variable_value +NDB_SLAVE_MAX_REPLICATED_EPOCH 0 +Cluster A Master server (old slave) has old Max replicated epoch +select +count(1) +from +information_schema.global_status, +mysql.ndb_apply_status +where +server_id = 1 +and +variable_name='Ndb_slave_max_replicated_epoch' + and +variable_value = epoch; +count(1) +1 +Now start slaves up +include/rpl_start_slaves.inc +Show that applying something from Cluster B causes the +old Max Rep Epoch to be loaded from ndb_apply_status +There is no new Max Rep Epoch from Cluster A as it has not changed +anything yet +insert into test.t1 values (3, "From the Sea"); +Allow to propagate to Cluster A +New Slave server on Cluster A has loaded old Max-Replicated-Epoch +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +2 +4 +select +count(1) +from +information_schema.global_status, +mysql.ndb_apply_status +where +server_id = 1 +and +variable_name='Ndb_slave_max_replicated_epoch' + and +variable_value = epoch; +count(1) +1 +Now make a new Cluster A change and see that the Max Replicated Epoch advances +once it has propagated +insert into test.t1 values (4, "Brooke"); +Propagate to Cluster B +Make change on Cluster B to allow waiting for reverse propagation +insert into test.t1 values (5, "Rupert"); +Wait for propagation back to Cluster A +Show that Cluster A now has 2 different server_id entries in ndb_apply_status +Those from the new master (server_id 3) are highest. +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +2 +3 +4 +select +count(1) +from +information_schema.global_status, +mysql.ndb_apply_status +where +server_id = 3 +and +variable_name='Ndb_slave_max_replicated_epoch' + and +variable_value = epoch; +count(1) +1 +local_server_with_max_epoch +3 +Done +drop table t1; +include/rpl_stop_slaves.inc +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +include/rpl_start_slaves.inc +include/rpl_end.inc === added file 'mysql-test/suite/ndb_rpl/r/ndb_rpl_init_rep_status.result' --- a/mysql-test/suite/ndb_rpl/r/ndb_rpl_init_rep_status.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_rpl/r/ndb_rpl_init_rep_status.result 2011-06-21 15:21:44 +0000 @@ -0,0 +1,78 @@ +include/master-slave.inc +[connection master] +reset master; +stop slave; +Generate something in the Masters Binlog +use test; +create table t1 (a int primary key, b int) engine=ndb; +insert into t1 values (1,1); +Initial state +select * from mysql.ndb_apply_status; +server_id epoch log_name start_pos end_pos +select variable_value from information_schema.global_status +where variable_name like '%Ndb_slave_max_replicated_epoch%'; +variable_value +0 +select @slave_server_id:=(variable_value+0) from information_schema.global_variables +where variable_name like 'server_id'; +@slave_server_id:=(variable_value+0) +2 +Default, no data, max replicated epoch will be 0. +reset slave; +start slave; +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +select variable_value from information_schema.global_status +where variable_name like 'Ndb_slave_max_replicated_epoch'; +variable_value +0 +Default, load of own serverid from ndb_apply_status, should be 111 +drop table test.t1; +stop slave; +reset slave; +insert into mysql.ndb_apply_status values (@slave_server_id, 111, 'Fictional log', 222, 333); +start slave; +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +2 +select variable_value from information_schema.global_status +where variable_name like 'Ndb_slave_max_replicated_epoch'; +variable_value +111 +drop table test.t1; +Check that reset slave resets Ndb_slave_max_replicated_epoch +stop slave; +select variable_value from information_schema.global_status +where variable_name like 'Ndb_slave_max_replicated_epoch'; +variable_value +111 +reset slave; +select variable_value from information_schema.global_status +where variable_name like 'Ndb_slave_max_replicated_epoch'; +variable_value +0 +Multiple-channel, load highest of configured serverids, should be 222 +set @other_local_server_id=@slave_server_id+1; +set @other_remote_server_id=@slave_server_id+2; +insert into mysql.ndb_apply_status values (@slave_server_id, 111, 'Fictional log', 222, 333); +insert into mysql.ndb_apply_status values (@other_local_server_id, 222, 'Fictional log', 222, 333); +insert into mysql.ndb_apply_status values (@other_remote_server_id, 444, 'Fictional log', 222, 333); +CHANGE MASTER TO IGNORE_SERVER_IDS=(3);; +start slave; +select server_id from mysql.ndb_apply_status order by server_id; +server_id +1 +2 +3 +4 +select variable_value from information_schema.global_status +where variable_name like 'Ndb_slave_max_replicated_epoch'; +variable_value +222 +stop slave; +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +start slave; +drop table test.t1; +include/rpl_end.inc === added file 'mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.cnf' --- a/mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.cnf 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.cnf 2011-06-21 15:21:44 +0000 @@ -0,0 +1,14 @@ +!include ndb_rpl_circular_2ch.cnf + +[mysqld.1.1] +ndb-log-apply-status + +[mysqld.2.1] +ndb-log-apply-status + +[mysqld.1.slave] +ndb-log-apply-status + +[mysqld.2.slave] +ndb-log-apply-status + === added file 'mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.test' --- a/mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.test 2011-06-21 15:21:44 +0000 @@ -0,0 +1,219 @@ +--source include/have_ndb.inc +--source suite/ndb_rpl/ndb_master-slave_2ch.inc +--source include/have_binlog_format_mixed_or_row.inc + +# +# Test that the Maximum replicated epoch is maintained +# as expected in a circular, 2 channel configuration. +# The channels are swapped, and replication is restarted +# The MaxReplicatedEpoch is reloaded from ndb_apply_status +# for the Servers considered local (IGNORE_SERVER_IDS) +# +--connection master +--echo Cluster A servers have no epoch replication info +select count(1) from mysql.ndb_apply_status; + +--echo Cluster A servers have no max replicated epoch value +select variable_name, variable_value from information_schema.global_status + where variable_name='Ndb_slave_max_replicated_epoch'; +--connection master1 +select variable_name, variable_value from information_schema.global_status + where variable_name='Ndb_slave_max_replicated_epoch'; + +--echo Make a change originating at Cluster A +--connection master +use test; +create table t1 (a int primary key, b varchar(100)) engine=ndb; +insert into t1 values (1, "Venice"); + +--echo Allow it to propagate to Cluster B +--sync_slave_with_master slave + +--echo Originate a second unrelated change at Cluster B, to allow us to wait for +--echo reverse propagation in the testcase +--connection slave1 +insert into t1 values (2, "Death"); + +--echo Allow it to propagate to Cluster A +--sync_slave_with_master master1 + +--echo Observe new entry in ndb_apply_status on Cluster A +--connection master +select server_id from mysql.ndb_apply_status order by server_id; + +--echo Non-slave server on Cluster A will have no value for Max Replicated Epoch +select variable_name, variable_value from information_schema.global_status + where variable_name='Ndb_slave_max_replicated_epoch'; + +--echo Slave server on Cluster A has current value for Max Replicated Epoch +--connection master1 +# Here we join the max rep epoch with ndb_apply_status for server id 1 +# (Our site's current master server) +select + count(1) + from + information_schema.global_status, + mysql.ndb_apply_status + where + server_id = 1 + and + variable_name='Ndb_slave_max_replicated_epoch' + and + variable_value = epoch; + + +--echo Now swap replication channels around +--source include/rpl_stop_slaves.inc +--let $rpl_topology= 2->1,3->4 +--source include/rpl_change_topology.inc + +# We've changed the direction, but need to set binlog filenames +# and positions + +# +# 'Normally' we should use the ndb_apply_status max applied epoch, +# then lookup ndb_binlog_index etc. +# However, in this case (and probably in lots of real cases), no changes +# have been made after the last applied epoch, so there is no epoch +# after the current one, and therefore no entry in ndb_binlog_index +# to get the correct position from. +# We could just re-apply the last epoch applied, but that's imprecise, +# and causes us to create an ndb_apply_status entry for Server 3 when +# it has not really been master for those changes. +# So we just look at the Master status instead. +# +#--echo Get max applied epochs from a server on each cluster +#--connection slave +#let $max_applied_cluster_a_epoch = query_get_value("SELECT MAX(epoch) AS epoch FROM mysql.ndb_apply_status WHERE server_id IN (1,3)", epoch, 1); +#--connection master +#let $max_applied_cluster_b_epoch = query_get_value("SELECT MAX(epoch) AS epoch FROM mysql.ndb_apply_status WHERE server_id IN (2,4)", epoch, 1); +# +#--echo Get corresponding Binlog filename + pos from new Master servers +#--connection master1 +#eval select * from mysql.ndb_binlog_index where epoch > $max_applied_cluster_a_epoch ; +#let $cluster_a_master_log_file = query_get_value("SELECT SUBSTRING_INDEX(File, '/', -1) as File from mysql.ndb_binlog_index WHERE epoch >= $max_applied_cluster_a_epoch", File, 1); +#let $cluster_a_master_log_pos = query_get_value("SELECT Position from mysql.ndb_binlog_index WHERE epoch >= $max_applied_cluster_a_epoch", Position, 1); +#--connection slave +#eval select * from mysql.ndb_binlog_index where epoch > $max_applied_cluster_b_epoch; +#let $cluster_b_master_log_file = query_get_value("SELECT SUBSTRING_INDEX(File, '/', -1) as File from mysql.ndb_binlog_index WHERE epoch >= $max_applied_cluster_b_epoch", File, 1); +#let $cluster_b_master_log_pos = query_get_value("SELECT Position from mysql.ndb_binlog_index WHERE epoch >= $max_applied_cluster_b_epoch", Position, 1); +#--echo Now change new Slave servers to new Master file + pos +#--connection master +#--echo Changing master to $cluster_b_master_log_file, $cluster_b_master_log_pos +#eval CHANGE MASTER TO MASTER_LOG_FILE="$cluster_b_master_log_file", MASTER_LOG_POS=$cluster_b_master_log_pos; +#--connection slave1 +#--echo Changing master to $cluster_a_master_log_file, $cluster_a_master_log_pos +#eval CHANGE MASTER TO MASTER_LOG_FILE="$cluster_a_master_log_file", MASTER_LOG_POS=$cluster_a_master_log_pos; + +--echo Get current master status on Cluster A new master (next pos in Binlog) +--connection master1 +let $cluster_a_master_log_file = query_get_value("SHOW MASTER STATUS", "File", 1); +let $cluster_a_master_log_pos = query_get_value("SHOW MASTER STATUS", "Position", 1); +--echo Setup slave on Cluster B to use it +--connection slave1 +--disable_query_log +eval CHANGE MASTER TO MASTER_LOG_FILE="$cluster_a_master_log_file", MASTER_LOG_POS=$cluster_a_master_log_pos; +--enable_query_log + +--echo Get current master status on Cluster B new master (next pos in Binlog) +--connection slave +let $cluster_b_master_log_file = query_get_value("SHOW MASTER STATUS", "File", 1); +let $cluster_b_master_log_pos = query_get_value("SHOW MASTER STATUS", "Position", 1); +--echo Setup slave on Cluster A to use it +--connection master +--disable_query_log +eval CHANGE MASTER TO MASTER_LOG_FILE="$cluster_b_master_log_file", MASTER_LOG_POS=$cluster_b_master_log_pos; +--enable_query_log + +--connection master +--echo Show that Cluster A Slave server (old master) has no Max replicated epoch before receiving data +select variable_name, variable_value from information_schema.global_status + where variable_name='Ndb_slave_max_replicated_epoch'; + +--connection master1 +--echo Cluster A Master server (old slave) has old Max replicated epoch +select + count(1) + from + information_schema.global_status, + mysql.ndb_apply_status + where + server_id = 1 + and + variable_name='Ndb_slave_max_replicated_epoch' + and + variable_value = epoch; + +--echo Now start slaves up +--source include/rpl_start_slaves.inc + +--echo Show that applying something from Cluster B causes the +--echo old Max Rep Epoch to be loaded from ndb_apply_status +--echo There is no new Max Rep Epoch from Cluster A as it has not changed +--echo anything yet + +--connection slave +insert into test.t1 values (3, "From the Sea"); + +--echo Allow to propagate to Cluster A +--sync_slave_with_master master + +--connection master +--echo New Slave server on Cluster A has loaded old Max-Replicated-Epoch +select server_id from mysql.ndb_apply_status order by server_id; +select + count(1) + from + information_schema.global_status, + mysql.ndb_apply_status + where + server_id = 1 + and + variable_name='Ndb_slave_max_replicated_epoch' + and + variable_value = epoch; + +--echo Now make a new Cluster A change and see that the Max Replicated Epoch advances +--echo once it has propagated + +--connection master1 +insert into test.t1 values (4, "Brooke"); + +--echo Propagate to Cluster B +--sync_slave_with_master slave1 + +--echo Make change on Cluster B to allow waiting for reverse propagation +--connection slave +insert into test.t1 values (5, "Rupert"); + +--echo Wait for propagation back to Cluster A +--sync_slave_with_master master + +--echo Show that Cluster A now has 2 different server_id entries in ndb_apply_status +--echo Those from the new master (server_id 3) are highest. +select server_id from mysql.ndb_apply_status order by server_id; +select + count(1) + from + information_schema.global_status, + mysql.ndb_apply_status + where + server_id = 3 + and + variable_name='Ndb_slave_max_replicated_epoch' + and + variable_value = epoch; + +let $max_epoch = query_get_value("select max(epoch) as epoch from mysql.ndb_apply_status where server_id in (1,3)","epoch", 1); +--disable_query_log +eval select server_id as local_server_with_max_epoch from mysql.ndb_apply_status where epoch=$max_epoch; +--enable_query_log + +--echo Done + +--connection master1 +drop table t1; +--sync_slave_with_master slave1 + +--source suite/ndb_rpl/ndb_master-slave_2ch_end.inc + === added file 'mysql-test/suite/ndb_rpl/t/ndb_rpl_init_rep_status.test' --- a/mysql-test/suite/ndb_rpl/t/ndb_rpl_init_rep_status.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_rpl/t/ndb_rpl_init_rep_status.test 2011-06-21 15:21:44 +0000 @@ -0,0 +1,89 @@ +--source include/have_ndb.inc +--source include/have_binlog_format_mixed_or_row.inc +--source suite/ndb_rpl/ndb_master-slave.inc + +# Test Slave initialisation of Ndb_slave_max_replicated_epoch status var + +--connection slave +reset master; +stop slave; + +--connection master +--echo Generate something in the Masters Binlog +use test; +create table t1 (a int primary key, b int) engine=ndb; + +insert into t1 values (1,1); + +--connection slave +--echo Initial state +select * from mysql.ndb_apply_status; +select variable_value from information_schema.global_status + where variable_name like '%Ndb_slave_max_replicated_epoch%'; +select @slave_server_id:=(variable_value+0) from information_schema.global_variables + where variable_name like 'server_id'; + +--echo Default, no data, max replicated epoch will be 0. +reset slave; +start slave; +--connection master +--sync_slave_with_master +--connection slave +--replace_column 3 # 4 # 5 # +select server_id from mysql.ndb_apply_status order by server_id; +select variable_value from information_schema.global_status + where variable_name like 'Ndb_slave_max_replicated_epoch'; + +--echo Default, load of own serverid from ndb_apply_status, should be 111 +drop table test.t1; +stop slave; +reset slave; +insert into mysql.ndb_apply_status values (@slave_server_id, 111, 'Fictional log', 222, 333); +start slave; +--connection master +--sync_slave_with_master +--connection slave +--replace_column 3 # 4 # 5 # +select server_id from mysql.ndb_apply_status order by server_id; +select variable_value from information_schema.global_status + where variable_name like 'Ndb_slave_max_replicated_epoch'; + +drop table test.t1; + +--echo Check that reset slave resets Ndb_slave_max_replicated_epoch +stop slave; +select variable_value from information_schema.global_status + where variable_name like 'Ndb_slave_max_replicated_epoch'; +reset slave; +select variable_value from information_schema.global_status + where variable_name like 'Ndb_slave_max_replicated_epoch'; + +--echo Multiple-channel, load highest of configured serverids, should be 222 +set @other_local_server_id=@slave_server_id+1; +set @other_remote_server_id=@slave_server_id+2; +insert into mysql.ndb_apply_status values (@slave_server_id, 111, 'Fictional log', 222, 333); +insert into mysql.ndb_apply_status values (@other_local_server_id, 222, 'Fictional log', 222, 333); +insert into mysql.ndb_apply_status values (@other_remote_server_id, 444, 'Fictional log', 222, 333); + +let $local_server_ids = `select @other_local_server_id`; + +--eval CHANGE MASTER TO IGNORE_SERVER_IDS=($local_server_ids); +start slave; +--connection master +--sync_slave_with_master +--connection slave +--replace_column 3 # 4 # 5 # +select server_id from mysql.ndb_apply_status order by server_id; +select variable_value from information_schema.global_status + where variable_name like 'Ndb_slave_max_replicated_epoch'; + +# Clean up +stop slave; +CHANGE MASTER TO IGNORE_SERVER_IDS= (); +start slave; +--connection master +drop table test.t1; +sync_slave_with_master; + +--source include/rpl_end.inc + === modified file 'sql/Makefile.am' --- a/sql/Makefile.am 2011-06-15 10:37:56 +0000 +++ b/sql/Makefile.am 2011-06-21 15:21:44 +0000 @@ -63,6 +63,7 @@ noinst_HEADERS = item.h item_func.h item ha_ndbcluster_lock_ext.h ha_ndbinfo.h \ ha_ndbcluster_glue.h \ ha_ndb_index_stat.h \ + ndb_mi.h \ ha_partition.h rpl_constants.h \ debug_sync.h \ opt_range.h protocol.h rpl_tblmap.h rpl_utility.h \ @@ -140,6 +141,8 @@ libndb_la_SOURCES= ha_ndbcluster.cc \ ha_ndb_index_stat.cc \ ha_ndbinfo.cc +libndb_extra_DIST= ndb_mi.cc + gen_lex_hash_SOURCES = gen_lex_hash.cc gen_lex_hash_LDFLAGS = @NOINST_LDFLAGS@ @@ -159,7 +162,7 @@ BUILT_SOURCES = $(BUILT_MAINT_SRC) lex_ EXTRA_DIST = udf_example.c udf_example.def $(BUILT_MAINT_SRC) \ nt_servc.cc nt_servc.h \ message.mc message.h message.rc MSG00001.bin \ - CMakeLists.txt + CMakeLists.txt $(libndb_extra_DIST) CLEANFILES = lex_hash.h sql_yacc.output link_sources DISTCLEANFILES = $(EXTRA_PROGRAMS) === modified file 'sql/ha_ndbcluster.cc' --- a/sql/ha_ndbcluster.cc 2011-06-21 13:50:33 +0000 +++ b/sql/ha_ndbcluster.cc 2011-06-21 15:21:44 +0000 @@ -30,8 +30,6 @@ #include "ha_ndbcluster_glue.h" -#include "rpl_mi.h" - #ifdef WITH_NDBCLUSTER_STORAGE_ENGINE #include "ha_ndbcluster.h" #include @@ -46,6 +44,7 @@ #include #include +#include "ndb_mi.h" #ifdef ndb_dynamite #undef assert @@ -459,7 +458,11 @@ update_slave_api_stats(Ndb* ndb) st_ndb_slave_state g_ndb_slave_state; st_ndb_slave_state::st_ndb_slave_state() - : current_conflict_defined_op_count(0) + : current_conflict_defined_op_count(0), + current_master_server_epoch(0), + current_max_rep_epoch(0), + max_rep_epoch(0), + sql_run_id(~Uint32(0)) { memset(current_violation_count, 0, sizeof(current_violation_count)); memset(total_violation_count, 0, sizeof(total_violation_count)); @@ -471,6 +474,7 @@ st_ndb_slave_state::atTransactionAbort() /* Reset current-transaction counters + state */ memset(current_violation_count, 0, sizeof(current_violation_count)); current_conflict_defined_op_count = 0; + current_max_rep_epoch = 0; } void @@ -485,8 +489,196 @@ st_ndb_slave_state::atTransactionCommit( current_violation_count[i] = 0; } current_conflict_defined_op_count = 0; + if (current_max_rep_epoch > max_rep_epoch) + { + DBUG_PRINT("info", ("Max replicated epoch increases from %llu to %llu", + max_rep_epoch, + current_max_rep_epoch)); + + max_rep_epoch = current_max_rep_epoch; + } + current_max_rep_epoch = 0; +} + +void +st_ndb_slave_state::atApplyStatusWrite(Uint32 master_server_id, + Uint32 row_server_id, + Uint64 row_epoch, + bool is_row_server_id_local) +{ + if (row_server_id == master_server_id) + { + /* + WRITE_ROW to ndb_apply_status injected by MySQLD + immediately upstream of us. + Record epoch + */ + current_master_server_epoch = row_epoch; + assert(! is_row_server_id_local); + } + else if (is_row_server_id_local) + { + DBUG_PRINT("info", ("Recording application of local server %u epoch %llu " + " which is %s.", + row_server_id, row_epoch, + (row_epoch > g_ndb_slave_state.current_max_rep_epoch)? + " new highest." : " older than previously applied")); + if (row_epoch > current_max_rep_epoch) + { + /* + Store new highest epoch in thdvar. If we commit successfully + then this can become the new global max + */ + current_max_rep_epoch = row_epoch; + } + } +} + +void +st_ndb_slave_state::atResetSlave() +{ + /* Reset the Maximum replicated epoch vars + * on slave reset + * No need to touch the sql_run_id as that + * will increment if the slave is started + * again. + */ + current_max_rep_epoch = 0; + max_rep_epoch = 0; +} + +int +ha_ndbcluster::check_slave_state(THD* thd) +{ + DBUG_ENTER("check_slave_state"); + +#ifdef HAVE_NDB_BINLOG + if (!thd->slave_thread) + DBUG_RETURN(0); + + Uint32 runId = ndb_mi_get_slave_run_id(); + DBUG_PRINT("info", ("Slave SQL thread run id is %u", + runId)); + if (unlikely(runId != g_ndb_slave_state.sql_run_id)) + { + DBUG_PRINT("info", ("Slave run id changed from %u, " + "treating as Slave restart", + g_ndb_slave_state.sql_run_id)); + g_ndb_slave_state.sql_run_id = runId; + + /* Always try to load the Max Replicated Epoch info + * first. + * Could be made optional if it's a problem + */ + { + /* + Load highest replicated epoch from a local + MySQLD from the cluster. + */ + DBUG_PRINT("info", ("Loading applied epoch information from %s", + NDB_APPLY_TABLE)); + NdbError ndb_error; + Uint64 highestAppliedEpoch = 0; + do + { + Ndb* ndb= check_ndb_in_thd(thd); + NDBDICT* dict= ndb->getDictionary(); + NdbTransaction* trans= NULL; + ndb->setDatabaseName(NDB_REP_DB); + Ndb_table_guard ndbtab_g(dict, NDB_APPLY_TABLE); + + const NDBTAB* ndbtab= ndbtab_g.get_table(); + if (unlikely(ndbtab == NULL)) + { + ndb_error = dict->getNdbError(); + break; + } + + trans= ndb->startTransaction(); + if (unlikely(trans == NULL)) + { + ndb_error = ndb->getNdbError(); + break; + } + + do + { + NdbScanOperation* sop = trans->getNdbScanOperation(ndbtab); + if (unlikely(sop == NULL)) + { + ndb_error = trans->getNdbError(); + break; + } + + const Uint32 server_id_col_num = 0; + const Uint32 epoch_col_num = 1; + NdbRecAttr* server_id_ra; + NdbRecAttr* epoch_ra; + + if (unlikely((sop->readTuples(NdbOperation::LM_CommittedRead) != 0) || + ((server_id_ra = sop->getValue(server_id_col_num)) == NULL) || + ((epoch_ra = sop->getValue(epoch_col_num)) == NULL))) + { + ndb_error = sop->getNdbError(); + break; + } + + if (trans->execute(NdbTransaction::Commit)) + { + ndb_error = trans->getNdbError(); + break; + } + + int rc = 0; + while (0 == (rc= sop->nextResult(true))) + { + Uint32 serverid = server_id_ra->u_32_value(); + Uint64 epoch = epoch_ra->u_64_value(); + + if ((serverid == ::server_id) || + (ndb_mi_get_ignore_server_id(serverid))) + { + highestAppliedEpoch = MAX(epoch, highestAppliedEpoch); + } + } + + if (rc != 1) + { + ndb_error = sop->getNdbError(); + break; + } + } while (0); + + trans->close(); + } while(0); + + if (ndb_error.code != 0) + { + sql_print_warning("NDB Slave : Could not determine maximum replicated epoch from %s.%s " + "at Slave start, error %u %s", + NDB_REP_DB, + NDB_APPLY_TABLE, + ndb_error.code, ndb_error.message); + } + + /* + Set Global status variable to the Highest Applied Epoch from + the Cluster DB. + If none was found, this will be zero. + */ + g_ndb_slave_state.max_rep_epoch = highestAppliedEpoch; + sql_print_information("NDB Slave : MaxReplicatedEpoch set to %llu (%u/%u) at Slave start", + g_ndb_slave_state.max_rep_epoch, + (Uint32)(g_ndb_slave_state.max_rep_epoch >> 32), + (Uint32)(g_ndb_slave_state.max_rep_epoch & 0xffffffff)); + } // Load highest replicated epoch + } // New Slave SQL thread run id +#endif + + DBUG_RETURN(0); } + static int update_status_variables(Thd_ndb *thd_ndb, st_ndb_status *ns, Ndb_cluster_connection *c) @@ -616,6 +808,7 @@ SHOW_VAR ndb_status_injector_variables[] SHOW_VAR ndb_status_slave_variables[]= { NDBAPI_COUNTERS("_slave", &g_slave_api_client_stats), + {"slave_max_replicated_epoch", (char*) &g_ndb_slave_state.max_rep_epoch, SHOW_LONGLONG}, {NullS, NullS, SHOW_LONG} }; @@ -708,7 +901,8 @@ static int ndb_to_mysql_error(const NdbE } #ifdef HAVE_NDB_BINLOG -extern Master_info *active_mi; +#include "ndb_mi.cc" + /* Write conflicting row to exceptions table. */ static int write_conflict_row(NDB_SHARE *share, NdbTransaction *trans, @@ -736,8 +930,8 @@ static int write_conflict_row(NDB_SHARE } { uint32 server_id= (uint32)::server_id; - uint32 master_server_id= (uint32)active_mi->master_id; - uint64 master_epoch= (uint64)active_mi->master_epoch; + uint32 master_server_id= (uint32) ndb_mi_get_master_server_id(); + uint64 master_epoch= (uint64) g_ndb_slave_state.current_master_server_epoch; uint32 count= (uint32)++(cfn_share->m_count); if (ex_op->setValue((Uint32)0, (const char *)&(server_id)) || ex_op->setValue((Uint32)1, (const char *)&(master_server_id)) || @@ -3910,7 +4104,7 @@ bool ha_ndbcluster::isManualBinlogExec(T #ifndef EMBEDDED_LIBRARY return thd ? ( thd->rli_fake? - thd->rli_fake->get_flag(Relay_log_info::IN_STMT) : false) + ndb_mi_get_in_relay_log_statement(thd->rli_fake) : false) : false; #else /* For Embedded library, we can't determine if we're @@ -4166,25 +4360,42 @@ handle_conflict_op_error(Thd_ndb* thd_nd #endif /* HAVE_NDB_BINLOG */ +#ifdef HAVE_NDB_BINLOG +/* + is_serverid_local +*/ +static bool is_serverid_local(Uint32 serverid) +{ + /* + If it's not our serverid, check the + IGNORE_SERVER_IDS setting to check if + it's local. + */ + return ((serverid == ::server_id) || + ndb_mi_get_ignore_server_id(serverid)); +} +#endif + int ha_ndbcluster::write_row(uchar *record) { DBUG_ENTER("ha_ndbcluster::write_row"); #ifdef HAVE_NDB_BINLOG if (m_share == ndb_apply_status_share && table->in_use->slave_thread) { - uint32 sid, master_server_id= active_mi->master_id; - memcpy(&sid, table->field[0]->ptr + (record - table->record[0]), sizeof(sid)); - if (sid == master_server_id) - { - uint64 master_epoch; - memcpy(&master_epoch, table->field[1]->ptr + (record - table->record[0]), - sizeof(master_epoch)); - active_mi->master_epoch= master_epoch; - } + uint32 row_server_id, master_server_id= ndb_mi_get_master_server_id(); + uint64 row_epoch; + memcpy(&row_server_id, table->field[0]->ptr + (record - table->record[0]), + sizeof(row_server_id)); + memcpy(&row_epoch, table->field[1]->ptr + (record - table->record[0]), + sizeof(row_epoch)); + g_ndb_slave_state.atApplyStatusWrite(master_server_id, + row_server_id, + row_epoch, + is_serverid_local(row_server_id)); } #endif /* HAVE_NDB_BINLOG */ DBUG_RETURN(ndb_write_row(record, FALSE, FALSE)); -} +}; /** Insert one record into NDB @@ -4204,6 +4415,10 @@ int ha_ndbcluster::ndb_write_row(uchar * Uint32 num_sets= 0; DBUG_ENTER("ha_ndbcluster::ndb_write_row"); + error = check_slave_state(thd); + if (unlikely(error)) + DBUG_RETURN(error); + has_auto_increment= (table->next_number_field && record == table->record[0]); if (has_auto_increment && table_share->primary_key != MAX_KEY) @@ -4748,6 +4963,11 @@ int ha_ndbcluster::ndb_update_row(const DBUG_ENTER("ndb_update_row"); DBUG_ASSERT(trans); + + error = check_slave_state(thd); + if (unlikely(error)) + DBUG_RETURN(error); + /* * If IGNORE the ignore constraint violations on primary and unique keys, * but check that it is not part of INSERT ... ON DUPLICATE KEY UPDATE @@ -5082,6 +5302,10 @@ int ha_ndbcluster::ndb_delete_row(const DBUG_ENTER("ndb_delete_row"); DBUG_ASSERT(trans); + error = check_slave_state(thd); + if (unlikely(error)) + DBUG_RETURN(error); + ha_statistic_increment(&SSV::ha_delete_count); m_rows_changed++; @@ -6543,30 +6767,15 @@ static int ndbcluster_update_apply_statu r|= op->setValue(1u, (Uint64)0); DBUG_ASSERT(r == 0); } -#if MYSQL_VERSION_ID < 50600 - const char* group_master_log_name = - active_mi->rli.group_master_log_name; - const Uint64 group_master_log_pos = - (Uint64)active_mi->rli.group_master_log_pos; - const Uint64 future_event_relay_log_pos = - (Uint64)active_mi->rli.future_event_relay_log_pos; - const Uint64 group_relay_log_pos = - (Uint64)active_mi->rli.group_relay_log_pos; -#else - /* - - Master_info's rli member returns Relay_log_info* - - Relay_log_info members are protected and must be accessed - using accessor functions - */ const char* group_master_log_name = - active_mi->rli->get_group_master_log_name(); + ndb_mi_get_group_master_log_name(); const Uint64 group_master_log_pos = - (Uint64)active_mi->rli->get_group_master_log_pos(); + ndb_mi_get_group_master_log_pos(); const Uint64 future_event_relay_log_pos = - (Uint64)active_mi->rli->get_future_event_relay_log_pos(); + ndb_mi_get_future_event_relay_log_pos(); const Uint64 group_relay_log_pos = - (Uint64)active_mi->rli->get_group_relay_log_pos(); -#endif + ndb_mi_get_group_relay_log_pos(); + // log_name char tmp_buf[FN_REFLEN]; ndb_pack_varchar(ndbtab->getColumn(2u), tmp_buf, === modified file 'sql/ha_ndbcluster.h' --- a/sql/ha_ndbcluster.h 2011-06-17 07:14:20 +0000 +++ b/sql/ha_ndbcluster.h 2011-06-21 15:21:44 +0000 @@ -345,13 +345,23 @@ struct st_ndb_slave_state /* Counter values for current slave transaction */ Uint32 current_conflict_defined_op_count; Uint32 current_violation_count[CFT_NUMBER_OF_CFTS]; + Uint64 current_master_server_epoch; + Uint64 current_max_rep_epoch; /* Cumulative counter values */ Uint64 total_violation_count[CFT_NUMBER_OF_CFTS]; + Uint64 max_rep_epoch; + Uint32 sql_run_id; /* Methods */ void atTransactionCommit(); void atTransactionAbort(); + void atResetSlave(); + + void atApplyStatusWrite(Uint32 master_server_id, + Uint32 row_server_id, + Uint64 row_epoch, + bool is_row_server_id_local); st_ndb_slave_state(); }; @@ -886,6 +896,8 @@ private: int start_statement(THD *thd, Thd_ndb *thd_ndb, uint table_count); int init_handler_for_statement(THD *thd); + int check_slave_state(THD* thd); + Thd_ndb *m_thd_ndb; NdbScanOperation *m_active_cursor; const NdbDictionary::Table *m_table; === modified file 'sql/ha_ndbcluster_binlog.cc' --- a/sql/ha_ndbcluster_binlog.cc 2011-06-16 18:16:01 +0000 +++ b/sql/ha_ndbcluster_binlog.cc 2011-06-21 15:21:44 +0000 @@ -47,6 +47,7 @@ extern my_bool opt_ndb_log_updated_only; extern my_bool opt_ndb_log_binlog_index; extern my_bool opt_ndb_log_apply_status; extern ulong opt_ndb_extra_logging; +extern st_ndb_slave_state g_ndb_slave_state; bool ndb_log_empty_epochs(void); @@ -892,6 +893,9 @@ static void ndbcluster_reset_slave(THD * thd_stmt_da(thd)->reset_diagnostics_area(); } + /* Reset Ndb_slave_max_replicated_epoch */ + g_ndb_slave_state.atResetSlave(); + DBUG_VOID_RETURN; } === added file 'sql/ndb_mi.cc' --- a/sql/ndb_mi.cc 1970-01-01 00:00:00 +0000 +++ b/sql/ndb_mi.cc 2011-06-21 15:21:44 +0000 @@ -0,0 +1,78 @@ +/* + Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "ndb_mi.h" +#include "rpl_mi.h" + +extern Master_info *active_mi; + + +Uint32 ndb_mi_get_master_server_id() +{ + return (Uint32) active_mi->master_id; +} + +const char* ndb_mi_get_group_master_log_name() +{ +#if MYSQL_VERSION_ID < 50600 + return active_mi->rli.group_master_log_name; +#else + return active_mi->rli->get_group_master_log_name(); +#endif +} + +Uint64 ndb_mi_get_group_master_log_pos() +{ +#if MYSQL_VERSION_ID < 50600 + return (Uint64) active_mi->rli.group_master_log_pos; +#else + return (Uint64) active_mi->rli->get_group_master_log_pos(); +#endif +} + +Uint64 ndb_mi_get_future_event_relay_log_pos() +{ +#if MYSQL_VERSION_ID < 50600 + return (Uint64) active_mi->rli.future_event_relay_log_pos; +#else + return (Uint64) active_mi->rli->get_future_event_relay_log_pos(); +#endif +} + +Uint64 ndb_mi_get_group_relay_log_pos() +{ +#if MYSQL_VERSION_ID < 50600 + return (Uint64) active_mi->rli.group_relay_log_pos; +#else + return (Uint64) active_mi->rli->get_group_relay_log_pos(); +#endif +} + +bool ndb_mi_get_ignore_server_id(Uint32 server_id) +{ + return (active_mi->shall_ignore_server_id(server_id) != 0); +} + +Uint32 ndb_mi_get_slave_run_id() +{ + return active_mi->rli.slave_run_id; +} + +bool ndb_mi_get_in_relay_log_statement(Relay_log_info* rli) +{ + return (rli->get_flag(Relay_log_info::IN_STMT) != 0); +} === added file 'sql/ndb_mi.h' --- a/sql/ndb_mi.h 1970-01-01 00:00:00 +0000 +++ b/sql/ndb_mi.h 2011-06-21 15:21:44 +0000 @@ -0,0 +1,46 @@ +/* + Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef NDB_MI_H +#define NDB_MI_H + +/* + This file defines methods for interacting with the + Master Info structure on a Slave MySQLD. + These methods are only valid when running in an + active slave thread. +*/ + +/* + Accessors +*/ +Uint32 ndb_mi_get_master_server_id(); +const char* ndb_mi_get_group_master_log_name(); +Uint64 ndb_mi_get_group_master_log_pos(); +Uint64 ndb_mi_get_future_event_relay_log_pos(); +Uint64 ndb_mi_get_group_relay_log_pos(); +bool ndb_mi_get_ignore_server_id(Uint32 server_id); +Uint32 ndb_mi_get_slave_run_id(); + + +/* + Relay log info related functions +*/ +bool ndb_mi_get_in_relay_log_statement(struct Relay_log_info* rli); + +// #ifndef NDB_MI_H +#endif --===============0096834017494391638== MIME-Version: 1.0 Content-Type: text/bzr-bundle; charset="us-ascii"; name="bzr/frazer.clement@stripped" Content-Transfer-Encoding: 7bit Content-Disposition: inline # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: frazer.clement@stripped\ # 35xe45y8ys1f4v3c # target_branch: file:///home/frazer/bzr/mysql-5.1-telco-7.1-reflect-\ # push/ # testament_sha1: d001346be47d202f4505e2af44e021216fe5d602 # timestamp: 2011-06-21 16:21:49 +0100 # base_revision_id: magnus.blaudd@stripped\ # k6tjvt4uuwvoqejp # # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWQhU5rgAFiB/gH/0HUB///// /+f/7v////9gLQ7s7tjp03zvefQ29uPQAdKFK63B77zn2+uxu+ee89eWh0Gd7nQAB33mnyyyvm+5 663t1AK6uuFdVkCLdN6LbS2ejrVxtZUGaDZWoLLe3dMtlstGjS9m6bLbt0Oy2bJt7uXajXuxSqpB JEUwIaDTKp7T0Kn+o1GU09qQ9I00empo0Bo9QHpqZAekBpoCUIATQJkCAm1TCMgozakZpqaAPUDQ GgAAeoYaCaVPTU2SYjRoHqNAGgA0AAAAAAAAAJNRIIJoEwgYppkp7aqep4p5T0NTTwp6ZqjelMmB kjEDJggRKRkIEwjUyemiZU9PSmjwp6E/SaTRo01PKYHqTENqYAeoTaBUkQERiACAaBG0RqZlNIye TSDTR6gaZNPU0AADKkAulhAGHD1cwP5vHHqRx8p1SM7Uuf6xqYHYZQV9JMV7zppSP9Y/9/qrfwr/ zKpMleCHNWl5bKA5p0LlmHHVIZqAsRIMPHqVMjNlYksjVVBoAqrXmh4/O8gzR3QMXm6+9nP3y+62 vBDLzaceavQRllmeWpihy/0ZmVoUKjvAT3hq0kFYFEIDB4AvB8urAcIGOiBtuw0c3pD/xctitRVV 2zrYMSr+8zqKPJjI6/3vKUInOf/DijNAWj+l6QdGdBlUM6DK6zyT6u4sORpAPDZxvmqLQtGeDVC/ qJOVhJVqbZO+aga5mZX5t+0vlbmD/5JVoFbNF+ieSMMzW9fJETAMqGXbOY/hL3TgrxndXDp8rQU5 u9HWVfgtmsO+ChlVzgkS4ahjqvGDeMwlf7jhY9PPSqUCRu5OL2qi8XtPDGMoA5zAqj1JDInBEKWD nsrmJqbEYsxRd8ymCwFMLeENCm8rY5sZWQYFYzRytzNnN6C/QK5HOGTAHU1UfRuypCes0T1Mo57/ 9ObQ4LkGG2LI0I4k34U5mPZZqqqqkGCs4c95OaByIAWTcCFSTTjqrTq/tmL55OZ96ahX8HfeM+FF o+K16u17nkJyINUe+DPkrlirJWm3MF1i7vcccfnBwhVa8Tg0neMbQlBXmqv9oXCPVWkkKQsaFFpD iQHKC0cEQPlUiN6nIfrE6FYVVgUx6TjiMiYus51Dy12YWmqvK4brmhSUdLjknMTCe0o+WuHchEM4 SPfSIlkoGdpFkRiHQgiTIuFSMCIkiJIYqg5rmzD57sfQ4lKHYW1bywrevWuZwd3XE2cZh3JNMnC0 Bssm6jNHhmYCCiHmp1chTs7ISBUP64MMoifNR6J1BY6FqwUyAW38ZNw8V4oGhMFTNdYljoaCr1Hl B6U8aRkkEJGRZHSGJq7C5fye2iKGCgMimdOD8UQh5WHBEf6/c5zveM5zSfNwEupkP8CFiIhb4XtO QvpCqctCkM19WADpqkFSC4Oaw/wR1+0XIKvBWWPCD/hgVlOTElQeVTD84PmWLrKkLKl3UV0VkpZZ TZViKNJpOfIV/XY7+PE8fxB16ysROxrdaRQYeO+naQBx4uRIgw6gRNwxuqvNPtxrdfiJoXEH9pBT 95Y2ec7n1Y4C6D0+4HiPsxpPPnjpglrWp0v/GCXYLFGCyXdW5x+/W80laWgZiJ+pYZDMMOTFJsGA c6xUox9szyN+gGROxjTxEmfggf7B/eCVBweB2sYhYbr+oLh1UYzMwcjEhRuijvtbbFsC4nMg7hPu USYuR6jUeP8ROFftRChHEoFBiRq1UPyHs3w0yIvR53TDkXBLDPeRTNWMf3HKG4wZZmZ5CdvVu3j4 jzWbkqfrIqiqLMiVlywCZJ8DkwqmJH5DNy9+Di5cFIrXczsLAYh97y4YncODIVkdLHEx20KlE6gm n4xcO4hqKdcTuLj3aL+FdSCPx6uVJ5azMfIz1CRCiqWgwOY8/pknxa/UoTSj/DUfY1t3BsP3LOFU 540PkD3GUQozJMVD5V6C6AUzbR8M086lt+rWBNysRweLQcMQHecvOT0sHg3RslKCvsCHQgCPGLj5 KrEEI/PAg/nWfdtLl3YTGPIJmyKKbZB5GJJQsNTnd5nxU46j1Qr9hi8vcUF48nCpycPLGMgIVV5F uwiEKGHmHzmeIITvQGoi2bA/RZnCYG7G4QLHYuQL6gwBqHVixrGbiVwK5JOFzAegjCLBDgVIPRHP Fbly8kzwtPDMgO076xJXP83UTYKJ+Am6eO7hVOzLZdgru4beNqo5DXxuyLaO7tseguSkkNPL2CuN zRGZQUU7mMFGIHcb3FiReKZl7uIF2NgUwGGGQbSNnnuIhLL7mehPyMI2brFuXVNqU2NVLOaRm21g UJR9cFDJmZRVOZ0LxKSFRu9BLlc0E6HA5dTVV4pfBWd6uG5AqEi6KI9vaaIiPqdARbYtYWdheoT0 29uoe4hIqINyBsh1iA9PVGIKioooKqqMpOQ4UxyQbKfoOOm5ghm2l4pIVBwiiYg3P2rsEonYJs26 Jg9amxwbsMG7NseB4y1FJimyniXVmc8G6NYiStEgeutHEDWP+pVjDo7NUVQidiZ11x0+p2+Cqmdk XwsLEjr3pyfQmMbYNM6NTLiZcGrF9BNYeZWaTEUKTmKSjP5jqULmBO6v/f/HCQewuMiHj+wXMiHE 3f1TQ1GGMlKO0ML82F4X5ZyKbNxc/JzXc7+/L4G5/FRyU0wpmnjgn8uO/fQ00W6tgfsioqf+r+1A EXnmemnF1Gfpn8oOndfiubmct/nquvuX+J8kaPa0WUU8s4oOfsHkvUsqlCJR/u2w86AVBJhEQjX4 bCc/4/PDOIkSSMZIKQZ3iGQlCFDowKd57hUngZPwOaJkB79mDvAoxPEwX9qRZYG0BVVVBJ19DKKq pp01ggn4PphemN5+sorG88kqqqUNuz4ykMZWimcaS7D3OLhM8esxI0HEjogvtJbNKW13BnPkr+8g BXE9Uw37HU9hiPF6eD61QRSYqTEZRWQSyQOSariRzIYITyawfGplQwmD4Kig993+7iRQl9bhwdaZ rRm+pkJ/mfiv63vM8jM8viJ8TMf9PKQuJAlqVK3H2HEopkVFq0/iRKxfdXvf9vpo0D8hxf1dPHHN 4aatrt5mHV16c6GYnLW1+dD52HMhiZ67gwDDdKm27zxKZgMHHVjxTV59wLyXuNqzxgD1MZId0h2e gaMO4vJP5VgUUyLVYH7eqbgqcuAoZrEd1gz3jCnQo6YenoeBM6Cn2ng1MlmSmKfU2OpmwjjOqmQ0 Fn1MHpZJgoNUYrKvwCTiBKM9BxD7CnB4/gE2VdyIGRt4icZ1HqPCEWCICgIgKKRZJHtfZ3ysXpd7 Fi3uJ7n6IyGDFzrFKXJM3gJKdKcHhXL0enlY5eN1ul6nJfhVVT2JXerJeHgzta3Ks+pj0H5bP0SS FlHK+Cd2/kmPEYrKc48IpU7VOwPRCgjwZA75chEjdPE56FTmYy3ocFXJdNskwbI5Skp4yUgxMeG+ Ymz1wK8yJw2LVjTjIvhvoVdsUCtusVotZCGR9FuGV5M3VaXNobgwlyZaBEztjc69erFyBs/AL6Xy +U8n2vQ/YGllJJ2c++Ac05Sow6gqAVD0c3jD59wr57u98hft1O5h3J3MkpCKV8O3yY9amNggWZHl qbjd8mLiZDjWXcrPNS7YpnCS63FTG6yJU0H1HlDK1XsWEyEf9P55nwShngWQgTgFR4gJuXFGuSLw Ij5lHFAsOmjh4pUeSIlBFQqkv7WCknW0lIkSlTZTMtNJj2tuxRAjFnd9KICXVkVHVRlpF5B7NCx8 gMTW+MyxEEtgJzn0QbiH/FyEhQxGB7DSfH3hbjgYFjeQ48TaLcbMFR3eI4+7e6Zv66BkZiKjz2JA zpcb62StEEQ6lsIGZMfSKtl4yfi/pKRo/PHocHlPqwEcRR+olOFkYyjtnBJx13K6IyOpjJCAhGVE DOMSZBhiUT1D18yZAOJc0JWh1cNpoajp4+UiLjQ3OWB9xwPOSh2FTEEwnmduuJyNiJ1RNhzzoOPG Ah4HX4AiSOZ3n/3SPFDzRTuabZUhDRI5QzWKPAUUQfazhPUTEF7S7eIicytAICoiCefm/gYG47YS bHBJFiTyNvK5IlnNSM3Y1eKr783G2qY4Ty7HoWGJFCxKg89446ZkDM+gx2HZnqjaaInYb+XQgYH3 glDMsbnkRNi5Ud8TU7fXBITE7j5TwG8deh4izm5yYqi6P4vVz5vPmIAdkYaD1O/z8Do/mYEsRwox 8qIgryJoSPM8PB2KBa1QPBd843LwdJp7I4mZEBlyGwcojhiJ6iu4IjjunFfEoXuUhMR3zQSTyMms jtgUlAuhIVRnIELjgLSrhMhaWYj2icVo6YVpAcu9YcpO08S002V34ok6eRiKdVXnMa+El19whhp0 3L4uwqK0wcnjp1UHTcLUpIWRuCcx0F5ONnAurDr2myzt7CoWeI1zTWIIkMODDHhEfCY3JIYmQIlJ ySbjBkiODAzPSxiQoZNNVUsVMEE573MxjMwuYe+kTYgTkeZ84jHENlOQ2r0gSqLDRdK+jElDJBrC qRQe1zBNJNZcQH16WaQwTRbDELDnj3IQnWaCkHKIIYjziTiLTf0EeRJpoScRcK6EBXZ43G0BsalZ iJIJkSHImmJAsMMMbmhmM8ap8kyRGBqewl43BEgCJr7eO9oNDKjPlLPfQsa1BsBhxQFk6RYZMJGR mOdSAiIbnlcJGFzogJ7eRMwjbBz1dZitp4FB3BxF1W8vKBEy1kORRxzygCoJboMQNzqEYGIKJ1TQ 57JAXngueb1Vb7LOKySxsMuq80uYOmozCpBPETHF5coMY0mRzycEzpOBKUyM2mbEAaKQW6DmwKnA 58715YCJibCjDzAUuHX6zsO5KmXByye6rUZmS4kBQOBKRE3GNoTIm0RksPFTqVBo8KbQN2RIzFke /AePW5sZ5FhtLyFJiqvQm/lALGffsXJGtD2AiQvjixI3ezNgrRzXTgOKFyEJ5lShExmTNx/hjQUv KQxZ8jMkRUV8EjxzxMcoyoOJjh+5LfVFe4xKhA1YeakLkDyETVDLGDB9PitCUjZNeVVKCl2Z7CY7 7Ni5gkRxrcLZpjMTBGyuNpw+57sHRA2x+CbKUDTdqDMPgDKf5PRzINCbC0Fw9QYBmDMQPVSLUkEc bhmuAqTMZLqg4LN1B1OGw31MsOuMEyG6qBjIPQGMslDwFnCWUa8g1NllLKomhlhfzymsvcBL2xpK dxwP+jIv1h6AZIHy+gPmHv0c35w0hF70yFh4K6EkycdVZa1myB6WfMHTpykKEVA5wmtxgioMREJw DhYBR7M3tkMjE9v1kwctT3pUV11COrm3TYcIk+lpKU6Cx13cikf1qROxPvSf0LvjPS/U/6YNQJ/6 GwMw2GJMg3KqcJOwoPUID9jQ5cIkUkIkhS81Ef2Fwj2YIcT/AfSGQ3yByn0YDZvaxGRWSRQ7TRQN tQm0n6ipWg+v1lIssA+bI1Ibg6OQ14luq/YE5CcU8gh00nYXTixKKXg7HKn/hkikYDkZR1MEwujj ehTzKSlKd8nEXT5xRzO5ZHiQs8hwhOshgnlwGBCiFydSKpjuDdFJdwdyU54vOo6MGaT+rhkP+zxr MXUuqOakKCGEmMJpXZDF04kZvQSirMQQVSsC8XEDkgs1pWgvkU4YgL2/m64ghKYGGW+4ZWzM10sA IsluryHihkdGP+F3FTy3sDU3nthOFinMQUHHFRwlBGIBDTD2+wS//Qyvdeof5nYQSDh6i3stVVW8 nWdSOhTceZ2oxT5EyrXXzPTeX32dRpKOw1dUtGLBTtVNKTzQ7GbKDRQMVLxNpmTx9ibS1DIxCEBe 7We3XybJIp9d+xZ4jFzNr5eMqTmKWWYpvRRpzlyz4T7wcx4zAbgJn6SznEna+JVVUN6LKYckqTim O9608cc3fe3Fg45c3FBehv6lOZexpBobUwC8wKprizXtOio5GlTUXDxTiQKC0bIYQ4vwjEbzu//O nIMuNpRSyyCsCwRoFWeBxxsR5bW20+E7GknM5TtPVM3Ya2KKn5/r/m93yCjfzFtSWM/KdsN/YfSf B6QQ9E5w8ZCxA93UbJTCH+RdaLyLGXfOo3pPaPlglkjGcxQH5AnigfQuJ5sB+aqwmWqZPgojrjZj 7RLzM81oJM8oZqqitVKpxndENpE+v7CG9ELLIT4JCwmWcA+JBSAiKRYKshFkFjcLkZGFgFHEw01A wy8SZSTfJlDUiQYgKoKiqKIqjOEDWPClRF4iVVVQ0DVNbAMDIawkMSbEKSEs22CkiCApA9bdhEdC GYbpjBSQlmRNNchHPYoiq75vDgkYGVUqyD1XwWNBNjZKK0STE4UXBooKpRoMCXGWWUWFQzmwCaQA jNQxX5RLtBCSBYMRTU6wuEiEESSICYghUyNN+kwkgYkPZ9dzBiiHA5fcvLig+YItS8bz5xXW6A3C 6CA/T4Dc/qPYfaYjxnL8ch2ystEtY4xu6emOMsDuiWcRAiUF1e8bDcQoAc0GCJrfE0UTn3nFnXcb FQs8ZHj/b+tNFUeHw869OR4HlZxjV9jmdrructcocHJJNp0O6uiX872qfQk5ExfQyU7qWb598cSZ +GHhwmOq9gpJL3eXlM9bNuUJPuVJG0y/ZLOKE91JDl13VxWaUwatP5x+Io0d1DVQqWoNEbrcrRe4 D50iA0LC4dVB6boKnXoZFPDImCRFrqW3RHScurKUplShYDL5Z189VOfiNgE8IgisUJFJILW2qxU8 9ktRe3n66q13nPq4N7fbrWPILxBOWTjEkUkDsxEyNt1qIaBmZUVO5vYblKWSTQE2HkMpYOFKSilQ kpQgQiC+g+rDlUjEiQIiBC9Q8Hv6BXp5TphTkU9BwOQvO4h3HxPoPOOBUySQRDzlEsosigkzavZx KBllOp3vmao1aQWVlE8xoSJrTkHlSWp9n5dPYeD0/c/fPtb4zNh4lpJ63TSdapHi4xFZpdKxMYnz O6XPTkymRPgYu12vE8i7yeVsfPi9nHsWbPW9DB0ubJsRtWycFeUjpn8TjWc8BU2tXdDjcG1i7vzc /zeW2L73OOdU3tj0xt6C+AybnrYN3MoOUUwg5RQDwjlDAuVKRjXmQ8hF4EdRHrjy8tA8wrtUTP6y /MJfrgQ0UWgodCL+u2xRh4lxD+QkCCUbHnhesRGuW6y0wjlI/wqJf7u9Y+RTNKmfxMIXe5vqw2he kk8bCBzjBE50gCLUDFQmPCkTIKDI4eLTGHXgcp5TC+EBk7O4ndnHA+WkuQphgBOklFBUKmSXMqkC gjEcHALsG6TDgsylCUWXVol0Zd/s3ZJ9x0S1Woa1eKkMFGl13yyUyamI7mLV3eGl7C8+0MvafQKV B5WkCBE/MaQJj7iJkGAibEzDuZNZz0kjac0mzoc64xUgqSprYxPb3uN87sMz8lSSklIgqBQVSMVR yudTN0NjpeU9lF6q1kjpZtjGNGx+H07ucm0CXCCOA5FCx+qce0iSJ9iCMajr4P+ZJjGP4SHt9/cB zbxDzFZno74cf+HIGEZoSxaZ4yGWLmH32LI88e9JyNK2epNVtBgbNVEY7aWnObBgYL+QoXGJxFdM 2X7VlgMqMCXCIoIMMVexXp3DXCVgYsGrJvScJbhnbfv2L7aVYXUswuxMGColVrlbg4l7uKZcaYmK yaFnYTB2NJCEsSgiGKt6Qw1VZCO11RCtlZcIuejLGFqzGmZjlZjQnFzbALDgmQzaIQE2CCxMotKg 9bxvOyjBw5t2Hf3/S0c7V1zzs8831EnDip2VO/49dr83zO98rg5HQ5G91Oiu20R2vykuF6QwQtA3 HHjkuhzDtR6VbxUoIkMqJYwihdUpUkYMIRZH4nK6HyaT8+tnnqGtJ9io4eTyaNmJ3yIrWR6eVdlS tqFqQdB7vNZBh3cJfoWk+WulxrSL0lTDxy/XU1uUf1IpZYlBhhRVAUQCtjuoBmdT5e6SezBdOh6n KeNOt2Pe8Z6XseW2c0nM2uZ3cm54T5JzTCX8Oue766Gh7QivHqNSg3Zn0h3W3AePgK+IpjVopLGD iV3K2TxTJ+A/XrHHJ+cbj8zpG9vRTIiwF1IVGgvE4HMFygRYLAiocXFnKRFAUiMQWRZFI97CB/dG BcaqmIoCClDOOQNqCCKATYnaES2MiEpESkUC4SMhbSSC5VSJFUhXPFAopJZXOPDMIh+jCnYeHEXF MswaB0DArUgNgTOSRD68VBVR+CDs6O/qXwIx8bms2KxrPawky5OmdVIIM3m8x9bB1KiAIOMipGZp 2Hq3vFCUHInlgiwONEBjCQ6zpd5MTMNsJ29vxNUrR1p6siOpPxSonEPRneOeE1avW9JscO7xzY+6 MIdYglSvjpoqAtYo94hpOQqe6e46WDYne+l8cGhJ8XwYPwr6MVKfQ976Fvg9jTNZ2+K5gU2NrqbG DidcSyphJK+PG9vzJ5ngm7eV/uSybZ5iYJNq8t870vOzx15U4LSTRDCUxfVgC/ofe+MSiY78ZN5S BzNr0J+BxtPbm3H5NGg3ilSVSShWzG8isFoH14P2/uz+ntrX78wVcyBnQiBALuCmgkZGRkV0xTSi VHXaorwqkHEPMZhtkGFKBSqCvPWs85PfNIZEkKM6CFIKiBUKKiLQIoFAYvKI+Hykh0QpdWqB+kg9 gsHSrerEOIUED4okiJ01wIBiwV7ayCRSWKKlKd6PrL8yGqFGwFkjgokLyXGK0PUMGuRtIZIJePrX eAQIsoefw4lawRuO8UTO+kxbqWPtiwuCDfVog8b7h68r6TH9Q+3y9FJ5FrSNJUXXskuq2dAFsFiy As7GQJUQN6UXiEPMWcN01Qfwp3+r27oZ9TVfKFgmkkTlKN59HgCcQcI3zFIwWQzKLUsVJJ5R8rM5 H6eOfciTaMJ0s03h3B3wJyh+zMPMsFixRPhVwA19mQZrozDBtPKlGkgJ0BeFDwO19RDqO5N5YekS zG2TU/FPnQYTcELWHYn6Z76MiEiEiEgzpN16J3+r05ch5CIrJIERgibvMUPMpv6M5zEJBgxmiif+ 1ezCGZW4PLYdZxhRKRD/6U8K8psjzIdwhe4lQMDA2o+gKF4ur7mLcTlMDIXLwZmhe925dU0nnd2Y b18bQ7xYOJDAY3AZg+M7VdLrPwm89zcb8biTpg60U4lQEKKhOYwdMlshDfLDV096pOdriZSFqGw0 GsLlElFKmuyRwaT0G2R8UoUKVKqUAuRsTIBFHaFl+DwSdIyTZCJFkQ3lSwgIOS30BiCnqxUHTddZ FqXGHl8ckkhCabUKQ4IyilQAUZ3QjnKxLSBcYJRqAPqKIURDp84NGyHOh5f8kkldqJFh+M23ZOAV Ck5yropFT+Woh4I2snwdlvIj5G+cyow2CdY2oQOYXIFqIJnacUBiob7dUw7dvLhYXPEpIm/aYVDj UScZSSVaWjC91LxDq3vNgwzSTlekqdTi5y50ykKlVEB+RKgLD5aAhU9hM4QGB5AMK/FRb8N0LTaf Ngq2QB6uByQhyJCHtbDBAL05OGZqt9c1bUJutFntetN2fvbI7JsTabzfuRxNQyHzlHzZiv1bg8gZ xA9gcPWmnVmNWmISjplBRAuimo0ZMnLGGeGoRQKMMoqX0FkmeUMDFyxAKTsQDE0DISF5RFZtFq3I eBKQF1AwApcVRfxmJ8Ql5aYRJrYUqSXWyLmEktc0fXXKbH95fwhgkSJIIhzc5dgp98/xD2PzZHeb fRPo3FPnZTn5UUUWbN1GjJrzQ1DVVwylZghM2E9Dmc40k6ZZSj0JCmGTDBMJiKwQdeId3nMQsPTE AKEeCBkEIc+frquBPyMzbhaI1vlZIWTDcHsipPBKeZdPWxaJ92rWRvG0i/z1Cy9VJUUZhUsqglIi qiKkioqkMXSbDCexg2dT2qfjB+UlEYgyHQMgUClIkRGB+JotZMLWtUk31GCn5MJOzXe73tTH6X2K FZWBVEfBBGjJxsePtDa4i7VJO2VEkySo+B7+2RxGz5qPOVvjCakhRCqbT8NqJ6TgGChAkESQXeQ5 yKLpv0YGf5zor7MTlXskNVIvvV88lLcG4WFk8t5cOtVo5GzVyuieBmjFaTvKXqSidSppW18CBnVz pkDOkuydyVOe5ELGvfgwENjt/YXQF/dLImAXQNJaH6C1g3UxZuBn9gQAHCBw1SN869HPduLJdLJm iWfo0bWb5JyO6LepRcNYpOjdB+ZvYaJGaFpaG2wfjsSi4o3HafNwpV/W5Fu4fmOQUNDpLUAAzDoI hAqH2sYMixQgjFRgooqJAZFSRAyTXf2a4JEihDafLBgRGMUHNgZhuFlUsQU12t3l533HSQ983nOB 8cJ0ODs5SD6Xs9j7Zkme+RzJh8Wxs6+qicdFIsh7Q1ASBEWCEWMYwZLoKF6VYFIKJLKlExcUouGx tACxjUj+pVuNJZajBFEkiygxTJ5GBC+Kwdg7O4sB+d8RYNMr9ZF97llVNR2qaWkkmoUIJRUoZRi6 LpkhnRgmIe0HbwJEx8RFUGDhgRMr3wJGQZHlDwIboiLdgqJAU2yAxWC1vVcGgtBAgHvEIMKw+GKC /DuteJ4WqRGzmNlklnuZO11tqfhfrnjLHR0CbTEUNHMMOcSRmv19o1v1GyD0KgeEge8c3rAqnwEE 5FHSdohY2u5/5/RRuYXT5b6g/CQSAQfkov0Zjn5wGdRvE95YJWXwXugJcJCEiEu53vOR2T6pyCQw UUPIfAe6bNrNwLRnAHj7GdaQwgri0cYMI4si1HVwXIzYzwUZ6MGqtX7bs+WbGeOa7Oy2V73bPdt0 NLGDNmYVnfC68phWz4LMkMUoHAlZmNU6blBMuBtCYN+RBOEOBeJmVCiwM+QL2l2SxxNFG2cS91lO UVze+VNIZSzZSl9hlvLRhhrkz6DJ3os/0seK9ge7Mjf725kk4GcnhfxS0qr4IdiyLY7F16EQiRug aKg0MRqIHDhHZqDhkHvpKaMhOMcgbbDNgsczycpVWvKBiD9R5TnDpjZQ1EYRF9JQwTwRVu+RSgpr gliM0k5PYxTJ0sHwbpDdJ6FVVemydoSHq+vk2+SisiHXwNh4WWsKY1xosPCyQ08ZXOD8ZYMF6dH3 ZF8KWhMeQskL1KKIpURgz5zPBSIDBGLEJEirFB4GIFnUkt2sN6l6UOEmlEBSxTmJBO/tg2GcIBjj raE9swGeWC0Ty4Aum6sYcpqVZlylzS9BGGDNmjLoI0Y3jOaZ4MNNhjeMMKZsMFEzaRSpRTXLXMxm CZsyLMnomhfOlspBhL+o0yyDaJebqCGlBOMh4pFFmUkPvnkaIO2wY6b0W4KOpCOujqhW1ECgMWRR UjFlMbpxJmJlMDGxMci56otxk3wJYtZAQ2N32rj72I8pBdJJShRisY6bc5lhuUN9QVmsddIOSjSM ySPgoTGkiN+7kcbBo6GAdNKihK9zY2mSiYhz08EfptJ6KJFc8LNx51osrR4VmtfITdrnK6u0qWId HXUlxhUIIwlFUjRRaqqB5bQvXMqJ0tk2idQqFEpHVCYp9znIieEEdYuUOO6wBp+5QmikGCG5mbbk t5C0Spb75A3cRojvAB8I9ORz+maikKUkfMZNU9bFJ3V9Sdjk02crcZpEWqbPPBYi7zpO910rZC+9 zyNqLMDaU/ucz0ynFOKDPSTtX4hVlwqd/kLofe+3jyw/L12kZyWVE2AMFWrBSagoDQPMEdrz4hv/ CbkFPawUbac6cacvzTFd2z7MCTzDf7EUoUUQojxvda1TmKSPIWLCk95csXE4uBTegd8Oogj5IqhA YIMKGgDg2klqEFGTGHHkwC7g2sn2zR9LtnvXY/WzCcIwhBgwGQeJ6ygDnDteweW58VTVc9U+Bh0s E5ioqoqKqiivupJqSnQWNBcXOvXFIEJBIRBDYWOoFv8B81nrYdDNyOfAHQo9Dp9C0s3KqTbm9LEm ONS6rVJmWsedazgSXnyy8wg1QtBOZ9CzmE2nUbrGkwMfpBeIo9JxmwpXoB8EKCszVgeM+E+B609n YDb/QQKWsWC5SWqYX0pK2/TX3f9i7kinChIBCpzXAA== --===============0096834017494391638==--