List:Commits« Previous MessageNext Message »
From:Andrei Elkin Date:May 26 2011 5:03pm
Subject:bzr commit into mysql-next-mr-wl5569 branch (andrei.elkin:3282) WL#5569
WL#5754
View as plain text  
#At file:///home/andrei/MySQL/BZR/2a-23May/WL/mysql-next-mr-wl5569/ based on revid:andrei.elkin@stripped

 3282 Andrei Elkin	2011-05-26
      wl#5569 MTS
      wl#5754 Query event parallel applying
      
      More cleanup is done;
      Fixing temp tables manipulation.
      Asserting an impossible to support use case of group of events
      not wrapped with BEGIN/COMMIT.
      
      Todo: recognize old master binlog to refuse to run in parallel.
     @ mysql-test/extra/rpl_tests/rpl_parallel_load_innodb.test
        Elaborated version of rpl_parallel_load generator still narrowed down to
        test performance with Innodb.
     @ mysql-test/suite/rpl/t/disabled.def
        Disabling few tests that triggers the assert installed in log_event.cc of this commit.
     @ mysql-test/suite/rpl/t/rpl_parallel_innodb-master.opt
        new test opt file is added.
     @ mysql-test/suite/rpl/t/rpl_parallel_innodb-slave.opt
        new test opt file is added.
     @ mysql-test/suite/rpl/t/rpl_parallel_innodb.test
        Elaborated version of rpl_parallel narrowed down to
        test performance with Innodb.
     @ sql/field.cc
        Old master binlog events can't be run in parallel 
        for few reasons. Therefore that paticular branch of code
        is irrelevant for MTS.
     @ sql/log_event.cc
        Asserting a not-implemented support of group of events not braced
        with BEGIN/COMMIT(Xid).
        Such groups are possible in stored routine logging and when an old
        server binlog file is adopted by MTS-aware slave.
     @ sql/rpl_rli_pdb.cc
        Removing redundant my_hash_update;
        cleanup;
        Fixing temp tables related issue of leaving wait_for_worker
        without all entries of APH given out their temp tables.
     @ sql/sql_base.cc
        Func is renamed.
        Removing all traces of previous idea to return value out of
        modify_slave_open_temp_tables.

    added:
      mysql-test/extra/rpl_tests/rpl_parallel_load_innodb.test
      mysql-test/suite/rpl/t/rpl_parallel_innodb-master.opt
      mysql-test/suite/rpl/t/rpl_parallel_innodb-slave.opt
      mysql-test/suite/rpl/t/rpl_parallel_innodb.test
    modified:
      mysql-test/suite/rpl/t/disabled.def
      sql/field.cc
      sql/log_event.cc
      sql/rpl_rli_pdb.cc
      sql/sql_base.cc
=== added file 'mysql-test/extra/rpl_tests/rpl_parallel_load_innodb.test'
--- a/mysql-test/extra/rpl_tests/rpl_parallel_load_innodb.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/extra/rpl_tests/rpl_parallel_load_innodb.test	2011-05-26 17:03:08 +0000
@@ -0,0 +1,333 @@
+#
+# This is a load generator to call from rpl_parallel and rpl_sequential tests
+
+#
+#
+# load volume parameter
+#
+
+let $iter = 100;
+let $init_rows= 50;
+
+#
+# Distribution of queries within an iteration:
+# legends:
+# auto = auto_increment=1, trans = inside BEGIN-COMMIT, 
+# del = Delete, ins =- Insert, upd = Update
+#
+let $ins_auto_wk= 1;
+let $ins_auto_nk= 1;
+let $ins_trans_wk= 1;
+let $ins_trans_nk= 1;
+let $upd_trans_nk= 0;
+let $upd_trans_wk= 1;
+let $del_trans_nk= 0;
+let $del_trans_wk= 1;
+
+# windows run on PB2 is too slow to time out
+disable_query_log;
+if (`select convert(@@version_compile_os using latin1) IN ("Win32","Win64","Windows") as "TRUE"`)
+{
+  let $iter = 100;
+}
+enable_query_log;
+
+let $databases = 16;
+
+connection slave;
+
+call mtr.add_suppression('Slave: Error dropping database'); ## todo: fix
+
+source include/stop_slave.inc;
+start slave;
+
+connection master;
+
+--disable_query_log
+--disable_result_log
+
+
+let $i = $databases + 1;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+  
+  eval create database test$i1;
+  eval use test$i1;
+  create table ti_nk (a int, b int, c text) engine=innodb;
+  create table ti_wk (a int auto_increment primary key, b int, c text) engine=innodb;
+  let $l1= $init_rows;
+  while($l1)
+  {
+       	eval insert into ti_nk values($l1, $i1, uuid());
+	dec $l1;
+  }
+  
+  if (`select @@global.binlog_format like 'statement'`)
+  {
+     create view v_ti_nk as select a,b from ti_nk;
+     create view v_ti_wk as select a,b from ti_wk;
+  }
+  
+  if (`select @@global.binlog_format not like 'statement'`)
+  {
+     create view v_ti_nk as select a,b,c from ti_nk;
+     create view v_ti_wk as select a,b,c from ti_wk;
+  }
+
+  # this table is special - just for timing. It's more special on test0 db
+  # where it contains master timing of the load as well.
+  create table benchmark (state text) engine=myisam; # timestamp keep on the slave side
+
+  dec $i;
+}
+
+--enable_result_log
+--enable_query_log
+
+
+sync_slave_with_master;
+#connection slave;
+
+--disable_query_log
+--disable_result_log
+
+let $i = $databases + 1;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+  eval use test$i1;
+  alter table benchmark add ts timestamp not null default current_timestamp;
+
+  dec $i;
+}
+--enable_result_log
+--enable_query_log
+
+
+# not gather events into relay log w/o executing yet
+stop slave sql_thread;
+
+##call p1(1);
+
+connection master;
+
+--disable_query_log
+--disable_result_log
+
+#
+# Load producer
+#
+
+# initial timestamp to record
+
+# the extra ts col on slave is effective only with the STMT format (todo: bug-report)
+set @save.binlog_format= @@session.binlog_format;
+set @@session.binlog_format=STATEMENT;
+let $i = $databases + 1;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+  eval use test$i1;
+
+  insert into benchmark set state='slave takes on load';
+
+  dec $i;
+}
+set @@session.binlog_format= @save.binlog_format;
+
+connection slave;
+
+use test0;
+insert into benchmark set state='master started load';
+
+
+connection master;
+
+while ($iter)
+{
+    let $i = $databases + 1;
+
+    while ($i)
+    {
+	let $i1 = $i;
+	dec $i1;
+
+	eval use test$i1;
+
+        let $ins= $ins_auto_nk;
+	while ($ins)
+	{
+	    eval insert into ti_nk values($iter, $i1, uuid());
+	    dec $ins;
+	}
+
+        let $ins= $ins_auto_wk;
+	while ($ins)
+	{
+	    eval insert into ti_wk values(null,  $i1, uuid());
+	    dec $ins;
+	}
+
+	begin;
+
+        let $ins= $ins_trans_nk;
+	while ($ins)
+	{
+	    eval insert into ti_nk values($iter, $i1, uuid());
+	    dec $ins;
+	}
+
+        let $ins= $ins_trans_wk;
+	while ($ins)
+	{
+	    eval insert into ti_wk values(null,  $i1, repeat('a', round(rand()*10)));
+	    dec $ins;
+	}
+
+	let $min=`select min(a) from ti_nk`;
+        let $del= $del_trans_nk;
+	while ($del)
+	{
+	    eval delete from ti_nk where a= $min + $del;
+	    dec $del;
+	}
+
+	let $min=`select min(a) from ti_nk`;
+        let $del= $del_trans_wk;
+	while ($del)
+	{
+	    eval delete from ti_wk where a= $min + $del;
+	    dec $del;
+	}
+
+        let $upd= $upd_trans_nk;
+	while ($upd)
+	{
+	    update ti_nk set c= uuid();
+	    dec $upd;
+	}
+
+        let $upd= $upd_trans_wk;
+	while ($upd)
+	{
+	    update ti_wk set c= uuid();
+	    dec $upd;
+	}
+
+	commit;
+	dec $i;
+    }
+
+    dec $iter;
+}
+
+connection slave;
+
+use test0;
+insert into benchmark set state='master ends load';
+
+connection master;
+
+# terminal timestamp to record
+
+let $i = $databases + 1;
+set @save.binlog_format= @@session.binlog_format;
+set @@session.binlog_format=STATEMENT;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+  eval use test$i1;
+
+  insert into benchmark set state='slave is supposed to finish with load';
+
+  dec $i;
+}
+set @@session.binlog_format= @save.binlog_format;
+
+--enable_result_log
+--enable_query_log
+
+connection slave;
+
+## todo: record start and end time of appying to compare times of
+#  parallel and sequential execution.
+
+--disable_query_log
+--disable_result_log
+
+insert into test0.benchmark set state='slave is processing load';
+
+# To force filling timestamp cols with the slave local clock values
+# to implement benchmarking.
+
+set @save.mts_exp_slave_local_timestamp=@@global.mts_exp_slave_local_timestamp;
+set @@global.mts_exp_slave_local_timestamp=1;
+start slave sql_thread;
+
+let $wait_timeout= 600;
+let $wait_condition= SELECT count(*)+sleep(1) = 5 FROM test0.benchmark;
+source include/wait_condition.inc;
+
+use test0;
+insert into benchmark set state='slave ends load';
+
+use test;
+select * from test0.benchmark into outfile 'benchmark.out';
+select ts from test0.benchmark where state like 'master started load' into @m_0;
+select ts from test0.benchmark where state like 'master ends load' into @m_1;
+select ts from test0.benchmark where state like 'slave takes on load' into @s_m0;
+select ts from test0.benchmark where state like 'slave is supposed to finish with load' into @s_m1;
+
+select ts from test0.benchmark where state like 'slave ends load' into @s_1;
+select ts from test0.benchmark where state like 'slave is processing load' into @s_0;
+select time_to_sec(@m_1) - time_to_sec(@m_0) as 'delta_m', 
+       time_to_sec(@s_1) - time_to_sec(@s_0) as 'delta_s',
+       time_to_sec(@s_m1) - time_to_sec(@s_m0) as 'delta_sm'  into outfile 'delta.out';
+
+
+let $i = $databases + 1;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+
+  let $diff_tables=master:test$i1.v_ti_nk, slave:test$i1.v_ti_nk;
+  source include/diff_tables.inc;
+
+  let $diff_tables=master:test$i1.v_ti_wk, slave:test$i1.v_ti_wk;
+  source include/diff_tables.inc;
+
+  dec $i;
+}
+--enable_result_log
+--enable_query_log
+
+
+connection master;
+
+--disable_query_log
+--disable_result_log
+
+let $i = $databases + 1;
+while($i)
+{
+  let $i1 = $i;
+  dec $i1;
+
+  eval drop database test$i1;
+  dec $i;
+}
+
+--enable_result_log
+--enable_query_log
+
+sync_slave_with_master;
+#connection slave;
+set @@global.mts_exp_slave_local_timestamp= @save.mts_exp_slave_local_timestamp;
+
+# End of the tests

=== modified file 'mysql-test/suite/rpl/t/disabled.def'
--- a/mysql-test/suite/rpl/t/disabled.def	2011-01-11 23:01:02 +0000
+++ b/mysql-test/suite/rpl/t/disabled.def	2011-05-26 17:03:08 +0000
@@ -15,3 +15,7 @@ rpl_spec_variables        : BUG#47661 20
 rpl_row_event_max_size    : Bug#55675 2010-10-25 andrei mysql_binlog_send attempts to read events partly
 rpl_delayed_slave         : Bug#57514 2010-11-09 andrei rpl_delayed_slave fails sporadically in pb
 rpl_log_pos               : BUG#55675 2010-09-10 alfranio rpl.rpl_log_pos fails sporadically with error binlog truncated in the middle
+
+rpl_sp_effects	  	  : bug@MTS SELECT with sf() logging does not follow the correct pattern (no BEGIN/COMMIT) Thu May 26 15:48:06 EEST 2011 Andrei
+rpl_auto_increment_bug33029  : same as rpl_sp_effects
+rpl.rpl_cross_version	  : same as rpl_sp_effects

=== added file 'mysql-test/suite/rpl/t/rpl_parallel_innodb-master.opt'
--- a/mysql-test/suite/rpl/t/rpl_parallel_innodb-master.opt	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/rpl/t/rpl_parallel_innodb-master.opt	2011-05-26 17:03:08 +0000
@@ -0,0 +1 @@
+--log-warnings=0

=== added file 'mysql-test/suite/rpl/t/rpl_parallel_innodb-slave.opt'
--- a/mysql-test/suite/rpl/t/rpl_parallel_innodb-slave.opt	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/rpl/t/rpl_parallel_innodb-slave.opt	2011-05-26 17:03:08 +0000
@@ -0,0 +1,5 @@
+--log-warnings=0 --slave-transaction-=0 --innodb_flush_log_at_trx_commit=0  --skip-log-bin --skip-log-slave-updates --sync_binlog=0
+
+
+
+

=== added file 'mysql-test/suite/rpl/t/rpl_parallel_innodb.test'
--- a/mysql-test/suite/rpl/t/rpl_parallel_innodb.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/rpl/t/rpl_parallel_innodb.test	2011-05-26 17:03:08 +0000
@@ -0,0 +1,43 @@
+#
+# WL#5563 Prototype for Parallel Slave with db name partitioning.
+#
+# The test checks correctness of replication and is designed for 
+# benchmarking and comparision with results of its sequential
+# counterpart rpl_sequential.test.
+# Both tests leave mysqld.2/data/test/delta.out file
+# that contains a row with two columns.
+#  1. the duration (in seconds) of execution on the master
+#  2. the duration of execution on the slave
+#  The 2nd column of the rpl_parallel can be compared with the 2nd of rpl_sequential.test.
+#
+#  The duration recorded in the file accounts the SQL thread/workers work.
+#  That is benchmarking on the slave side is effectively started with
+#  `start slave sql_thread'.
+#  NOTICE, there is set @@global.slave_local_timestamp=1;
+#  
+#
+# of load that rpl_parallel_load.test represents.
+# See there how to tune load and concurrency parameters.
+#
+# Example of usage.
+# To gather a collection of figures:
+#  mysql-test$ export slave;
+#  mysql-test$ slave=parallel; for n in `seq 1 10`; 
+#  do ./mtr  --vardir=/dev/shm/var1 --mtr-build-thread=765 rpl_$slave 
+#             --mysqld=--binlog-format=statement; 
+#     find /dev/shm/var1 -name delta.out -exec cat {} \; | cat >> delta.$slave.log; 
+#  done
+#
+# mysql-test$ slave=sequential; ...
+#
+# In the end there will be mysql-test/delta.{parallel,sequential}.log files.
+#
+
+let $rpl_skip_reset_master_and_slave= 1;
+
+--source include/master-slave.inc
+
+connection master;
+source extra/rpl_tests/rpl_parallel_load_innodb.test;
+
+--source include/rpl_end.inc

=== modified file 'sql/field.cc'
--- a/sql/field.cc	2011-05-25 16:02:13 +0000
+++ b/sql/field.cc	2011-05-26 17:03:08 +0000
@@ -6517,7 +6517,7 @@ Field_string::compatible_field_size(uint
 {
 #ifdef HAVE_REPLICATION
   const Check_field_param check_param = { this };
-  if (rpl_master_has_bug(rli_arg, 37426, TRUE,
+  if (!mts_is_worker(rli_arg->info_thd) && rpl_master_has_bug(rli_arg, 37426, TRUE,
                          check_field_for_37426, &check_param))
     return FALSE;                        // Not compatible field sizes
 #endif

=== modified file 'sql/log_event.cc'
--- a/sql/log_event.cc	2011-05-25 16:02:13 +0000
+++ b/sql/log_event.cc	2011-05-26 17:03:08 +0000
@@ -2527,11 +2527,21 @@ Slave_worker *Log_event::get_slave_worke
       DBUG_ASSERT(get_type_code() == INTVAR_EVENT ||
                   get_type_code() == RAND_EVENT ||
                   get_type_code() == USER_VAR_EVENT ||
-                  get_type_code() ==  ROWS_QUERY_LOG_EVENT);
+                  get_type_code() == ROWS_QUERY_LOG_EVENT);
 
       insert_dynamic(&rli->curr_group_da, (uchar*) &ptr_curr_ev);
       
-      DBUG_ASSERT(rli->curr_group_da.elements > 0);
+      if (!rli->curr_group_seen_begin)
+      {
+        // TODO: fix the master side to wrap with B/T cases like
+        // `set @user_var, select f()' that are logged w/o B-event
+        // Notice, while the select-f() can be mended in the current
+        // master version, the old server binlogs can't brought to MTS because
+        // of not following B|T-bracing rule for DML events.
+        DBUG_ASSERT(0);
+      }
+
+      DBUG_ASSERT(rli->curr_group_da.elements > 1);
     }
 
   // the group terminal event (Commit, Xid or a DDL query)
@@ -4212,6 +4222,7 @@ void Query_log_event::attach_temp_tables
     {
       mts_move_temp_tables_to_thd(thd,
                                   mts_assigned_partitions[i]->temporary_tables);
+      mts_assigned_partitions[i]->temporary_tables= NULL;
     }
   }
 }
@@ -4251,7 +4262,7 @@ void Query_log_event::detach_temp_tables
   */
   for (int i= 0; i < mts_accessed_dbs; i++)
   {
-    mts_assigned_partitions[i]->temporary_tables= 0;
+    mts_assigned_partitions[i]->temporary_tables= NULL;
   }
 
   for (TABLE *table= thd->temporary_tables; table;)
@@ -4270,7 +4281,15 @@ void Query_log_event::detach_temp_tables
     // table pointer is shifted inside the function
     table= mts_move_temp_table_to_entry(table, thd, mts_assigned_partitions[i]);
   }
+
   DBUG_ASSERT(!thd->temporary_tables);
+#ifndef DBUG_OFF
+  for (int i= 0; i < mts_accessed_dbs; i++)
+  {
+    DBUG_ASSERT(!mts_assigned_partitions[i]->temporary_tables ||
+                !mts_assigned_partitions[i]->temporary_tables->prev);
+  }
+#endif
 }
 
 int Query_log_event::do_apply_event(Relay_log_info const *rli)

=== modified file 'sql/rpl_rli_pdb.cc'
--- a/sql/rpl_rli_pdb.cc	2011-05-25 16:02:13 +0000
+++ b/sql/rpl_rli_pdb.cc	2011-05-26 17:03:08 +0000
@@ -313,7 +313,7 @@ void destroy_hash_workers(Relay_log_info
 }
 
 /**
-   Relocating temporary table reference into @c entry location.
+   Relocating temporary table reference into @c entry's table list head.
    Sources can be the coordinator's and the Worker's thd->temporary_tables.
 
    @param table   TABLE instance pointer
@@ -573,7 +573,9 @@ Slave_worker *get_slave_worker(const cha
     }
 
     ret= my_hash_insert(&mapping_db_to_worker, (uchar*) entry);
+
     mysql_mutex_unlock(&slave_worker_hash_lock);
+
     if (ret)
     {
       my_free(db);
@@ -592,9 +594,6 @@ Slave_worker *get_slave_worker(const cha
         get_least_occupied_worker(workers) : rli->last_assigned_worker;
       entry->worker->usage_partition++;
       entry->usage++;
-
-      my_hash_update(&mapping_db_to_worker, (uchar*) entry,
-                     (uchar*) dbname, dblength);
     }
     else if (entry->worker == rli->last_assigned_worker ||
              !rli->last_assigned_worker)
@@ -603,8 +602,6 @@ Slave_worker *get_slave_worker(const cha
       DBUG_ASSERT(entry->worker);
 
       entry->usage++;
-      my_hash_update(&mapping_db_to_worker, (uchar*) entry,
-                     (uchar*) dbname, dblength);
     }
     else
     {
@@ -771,8 +768,6 @@ void Slave_worker::slave_worker_ends_gro
     DBUG_ASSERT(strlen(key + 1) == (uchar) key[0]);
 
     entry->usage--;
-    my_hash_update(&mapping_db_to_worker, (uchar*) entry,
-                   (uchar*) key + 1, key[0]);
 
     if (entry->usage == 0)
     {
@@ -783,6 +778,8 @@ void Slave_worker::slave_worker_ends_gro
       */
 
       DBUG_ASSERT(this->info_thd->temporary_tables == 0);
+      DBUG_ASSERT(!entry->temporary_tables ||
+                  !entry->temporary_tables->prev);
 
       usage_partition--;
       if (entry->worker != this) // Coordinator is waiting
@@ -1108,11 +1105,11 @@ int wait_for_workers_to_finish(Relay_log
     }
     else
     {
-      // resources relocation
-      mts_move_temp_tables_to_thd(thd, entry->temporary_tables);
-      entry->temporary_tables= NULL;
       mysql_mutex_unlock(&slave_worker_hash_lock);
     }
+    // resources relocation
+    mts_move_temp_tables_to_thd(thd, entry->temporary_tables);
+    entry->temporary_tables= NULL;
   }
   return ret;
 }

=== modified file 'sql/sql_base.cc'
--- a/sql/sql_base.cc	2011-05-25 16:02:13 +0000
+++ b/sql/sql_base.cc	2011-05-26 17:03:08 +0000
@@ -125,21 +125,18 @@ static void init_tdc_psi_keys(void)
 }
 #endif /* HAVE_PSI_INTERFACE */
 
-static void incr_slave_open_temp_tables(THD *thd, int inc)
+static void modify_slave_open_temp_tables(THD *thd, int inc)
 {
-  int32 ret;
-  
   if (thd->system_thread == SYSTEM_THREAD_SLAVE_WORKER)
   {
     my_atomic_rwlock_wrlock(&slave_open_temp_tables_lock);
-    ret= my_atomic_add32(&slave_open_temp_tables, inc);
+    my_atomic_add32(&slave_open_temp_tables, inc);
     my_atomic_rwlock_wrlock(&slave_open_temp_tables_unlock);
-    ret += inc;
   }
   else
-    ret= (slave_open_temp_tables += inc);
-
-  return;
+  {
+    slave_open_temp_tables += inc;
+  }
 }
 
 /**
@@ -2149,7 +2146,7 @@ void close_temporary_table(THD *thd, TAB
   {
     /* natural invariant of temporary_tables */
     DBUG_ASSERT(slave_open_temp_tables || !thd->temporary_tables);
-    incr_slave_open_temp_tables(thd, -1);
+    modify_slave_open_temp_tables(thd, -1);
   }
   close_temporary(table, free_share, delete_table);
   DBUG_VOID_RETURN;
@@ -5874,7 +5871,7 @@ TABLE *open_table_uncached(THD *thd, con
     thd->temporary_tables= tmp_table;
     thd->temporary_tables->prev= 0;
     if (thd->slave_thread)
-      incr_slave_open_temp_tables(thd, 1);
+      modify_slave_open_temp_tables(thd, 1);
   }
   tmp_table->pos_in_table_list= 0;
   DBUG_PRINT("tmptable", ("opened table: '%s'.'%s' 0x%lx", tmp_table->s->db.str,


Attachment: [text/bzr-bundle] bzr/andrei.elkin@oracle.com-20110526170308-cs6mwxf6kuhp3tow.bundle
Thread
bzr commit into mysql-next-mr-wl5569 branch (andrei.elkin:3282) WL#5569WL#5754Andrei Elkin26 May