From: Bjorn Munch Date: November 7 2010 2:57pm Subject: bzr push into mysql-5.1-mtr branch (bjorn.munch:2947 to 2948) Bug#57840 List-Archive: http://lists.mysql.com/commits/123035 X-Bug: 57840 Message-Id: <201011071457.oA7Evmcu006255@khepri15.norway.sun.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 2948 Bjorn Munch 2010-11-05 Bug #57840 MTR: parallel execution breaks with smart ordering of test cases There were actually more problems in this area: Slaves (if any) were unconditionally restarted, this appears unnecessary. Sort criteria were suboptimal, included the test name. Added logic to "reserve" a sequence of tests with same config for one thread Got rid of sort_criteria hash, put it into the test case itself Adds little sanity check that expected worker picks up test Fixed some tests that may fail if starting on running server Some of these fail only if *same* test is repeated. Finally, special sorting of tests that do --force-restart added: mysql-test/suite/binlog/t/binlog_index-master.opt mysql-test/suite/rpl/t/rpl_current_user-master.opt mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt modified: mysql-test/lib/mtr_cases.pm mysql-test/mysql-test-run.pl mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt mysql-test/suite/rpl/r/rpl_ignore_table.result mysql-test/suite/rpl/t/rpl_cross_version-master.opt mysql-test/suite/rpl/t/rpl_ignore_table.test mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh mysql-test/t/key_cache-master.opt mysql-test/t/mysqlbinlog-master.opt 2947 Bjorn Munch 2010-10-21 Follow-up to Bug #55582 which allows checking strings in if Simplified cases where a select was used to compare variable against '' modified: mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test mysql-test/extra/rpl_tests/rpl_loaddata.test mysql-test/include/check_concurrent_insert.inc mysql-test/include/check_no_concurrent_insert.inc mysql-test/include/get_relay_log_pos.inc mysql-test/include/kill_query.inc mysql-test/include/kill_query_and_diff_master_slave.inc mysql-test/include/setup_fake_relay_log.inc mysql-test/include/show_binlog_events.inc mysql-test/include/show_rpl_debug_info.inc mysql-test/include/wait_for_slave_io_error.inc mysql-test/include/wait_for_slave_param.inc mysql-test/include/wait_for_slave_sql_error.inc mysql-test/include/wait_for_status_var.inc mysql-test/suite/rpl/t/rpl_killed_ddl.test === modified file 'mysql-test/lib/mtr_cases.pm' --- a/mysql-test/lib/mtr_cases.pm 2010-08-30 09:25:10 +0000 +++ b/mysql-test/lib/mtr_cases.pm 2010-11-05 14:26:38 +0000 @@ -170,8 +170,6 @@ sub collect_test_cases ($$$) { if ( $opt_reorder && !$quick_collect) { # Reorder the test cases in an order that will make them faster to run - my %sort_criteria; - # Make a mapping of test name to a string that represents how that test # should be sorted among the other tests. Put the most important criterion # first, then a sub-criterion, then sub-sub-criterion, etc. @@ -183,24 +181,31 @@ sub collect_test_cases ($$$) { # Append the criteria for sorting, in order of importance. # push(@criteria, "ndb=" . ($tinfo->{'ndb_test'} ? "A" : "B")); + push(@criteria, $tinfo->{template_path}); # Group test with equal options together. # Ending with "~" makes empty sort later than filled my $opts= $tinfo->{'master_opt'} ? $tinfo->{'master_opt'} : []; push(@criteria, join("!", sort @{$opts}) . "~"); + # Add slave opts if any + if ($tinfo->{'slave_opt'}) + { + push(@criteria, join("!", sort @{$tinfo->{'slave_opt'}})); + } + # This sorts tests with force-restart *before* identical tests + push(@criteria, $tinfo->{force_restart} ? "force-restart" : "no-restart"); - $sort_criteria{$tinfo->{name}} = join(" ", @criteria); + $tinfo->{criteria}= join(" ", @criteria); } - @$cases = sort { - $sort_criteria{$a->{'name'}} . $a->{'name'} cmp - $sort_criteria{$b->{'name'}} . $b->{'name'}; } @$cases; + @$cases = sort {$a->{criteria} cmp $b->{criteria}; } @$cases; # For debugging the sort-order # foreach my $tinfo (@$cases) # { - # print("$sort_criteria{$tinfo->{'name'}} -> \t$tinfo->{'name'}\n"); + # my $tname= $tinfo->{name} . ' ' . $tinfo->{combination}; + # my $crit= $tinfo->{criteria}; + # print("$tname\n\t$crit\n"); # } - } if (defined $print_testcases){ === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2010-10-19 12:01:14 +0000 +++ b/mysql-test/mysql-test-run.pl 2010-11-05 14:26:38 +0000 @@ -662,22 +662,40 @@ sub run_test_server ($$$) { next; } - # Prefer same configuration, or just use next if --noreorder - if (!$opt_reorder or (defined $result and - $result->{template_path} eq $t->{template_path})) - { - #mtr_report("Test uses same config => good match"); - # Test uses same config => good match - $next= splice(@$tests, $i, 1); - last; - } - # Second best choice is the first that does not fulfill # any of the above conditions if (!defined $second_best){ #mtr_report("Setting second_best to $i"); $second_best= $i; } + + # Smart allocation of next test within this thread. + + if ($opt_reorder and $opt_parallel > 1 and defined $result) + { + my $wid= $result->{worker}; + # Reserved for other thread, try next + next if (defined $t->{reserved} and $t->{reserved} != $wid); + if (! defined $t->{reserved}) + { + # Force-restart not relevant when comparing *next* test + $t->{criteria} =~ s/force-restart$/no-restart/; + my $criteria= $t->{criteria}; + # Reserve similar tests for this worker, but not too many + my $maxres= (@$tests - $i) / $opt_parallel + 1; + for (my $j= $i+1; $j <= $i + $maxres; $j++) + { + my $tt= $tests->[$j]; + last unless defined $tt; + last if $tt->{criteria} ne $criteria; + $tt->{reserved}= $wid; + } + } + } + + # At this point we have found next suitable test + $next= splice(@$tests, $i, 1); + last; } # Use second best choice if no other test has been found @@ -686,10 +704,12 @@ sub run_test_server ($$$) { mtr_error("Internal error, second best too large($second_best)") if $second_best > $#$tests; $next= splice(@$tests, $second_best, 1); + delete $next->{reserved}; } if ($next) { - #$next->print_test(); + # We don't need this any more + delete $next->{criteria}; $next->write_test($sock, 'TESTCASE'); $running{$next->key()}= $next; $num_ndb_tests++ if ($next->{ndb_test}); @@ -772,6 +792,11 @@ sub run_worker ($) { delete($test->{'comment'}); delete($test->{'logfile'}); + # A sanity check. Should this happen often we need to look at it. + if (defined $test->{reserved} && $test->{reserved} != $thread_num) { + my $tres= $test->{reserved}; + mtr_warning("Test reserved for w$tres picked up by w$thread_num"); + } $test->{worker} = $thread_num if $opt_parallel > 1; run_testcase($test); @@ -4575,17 +4600,6 @@ sub server_need_restart { } } - # Temporary re-enable the "always restart slave" hack - # this should be removed asap, but will require that each rpl - # testcase cleanup better after itself - ie. stop and reset - # replication - # Use the "#!use-slave-opt" marker to detect that this is a "slave" - # server - if ( $server->option("#!use-slave-opt") ){ - mtr_verbose_restart($server, "Always restart slave(s)"); - return 1; - } - my $is_mysqld= grep ($server eq $_, mysqlds()); if ($is_mysqld) { === added file 'mysql-test/suite/binlog/t/binlog_index-master.opt' --- a/mysql-test/suite/binlog/t/binlog_index-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/binlog/t/binlog_index-master.opt 2010-11-05 14:26:38 +0000 @@ -0,0 +1 @@ +--force-restart === modified file 'mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt' --- a/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt 2007-06-27 12:28:02 +0000 +++ b/mysql-test/suite/binlog/t/binlog_stm_binlog-master.opt 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ -O max_binlog_size=4096 +--force-restart === modified file 'mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt' --- a/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt 2009-09-24 14:52:52 +0000 +++ b/mysql-test/suite/binlog/t/binlog_stm_do_db-master.opt 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ --binlog-do-db=b42829 +--force-restart === modified file 'mysql-test/suite/rpl/r/rpl_ignore_table.result' --- a/mysql-test/suite/rpl/r/rpl_ignore_table.result 2008-11-13 19:19:00 +0000 +++ b/mysql-test/suite/rpl/r/rpl_ignore_table.result 2010-11-05 14:26:38 +0000 @@ -141,3 +141,4 @@ HEX(word) SELECT * FROM tmptbl504451f4258$1; ERROR 42S02: Table 'test.tmptbl504451f4258$1' doesn't exist DROP TABLE t5; +call mtr.force_restart(); === modified file 'mysql-test/suite/rpl/t/rpl_cross_version-master.opt' --- a/mysql-test/suite/rpl/t/rpl_cross_version-master.opt 2009-01-27 11:33:30 +0000 +++ b/mysql-test/suite/rpl/t/rpl_cross_version-master.opt 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ --replicate-same-server-id --relay-log=slave-relay-bin --secure-file-priv=$MYSQL_TMP_DIR +--force-restart === added file 'mysql-test/suite/rpl/t/rpl_current_user-master.opt' --- a/mysql-test/suite/rpl/t/rpl_current_user-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/t/rpl_current_user-master.opt 2010-11-05 14:26:38 +0000 @@ -0,0 +1 @@ +--force-restart === modified file 'mysql-test/suite/rpl/t/rpl_ignore_table.test' --- a/mysql-test/suite/rpl/t/rpl_ignore_table.test 2008-11-13 19:19:00 +0000 +++ b/mysql-test/suite/rpl/t/rpl_ignore_table.test 2010-11-05 14:26:38 +0000 @@ -174,3 +174,5 @@ SELECT * FROM tmptbl504451f4258$1; connection master; DROP TABLE t5; sync_slave_with_master; + +call mtr.force_restart(); === modified file 'mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh' --- a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh 2009-11-28 04:43:16 +0000 +++ b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-master.sh 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ +rm -f $MYSQLTEST_VARDIR/std_data_master_link ln -s $MYSQLTEST_VARDIR/std_data $MYSQLTEST_VARDIR/std_data_master_link === modified file 'mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh' --- a/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh 2009-11-28 04:43:16 +0000 +++ b/mysql-test/suite/rpl/t/rpl_loaddata_symlink-slave.sh 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ +rm -f $MYSQLTEST_VARDIR/std_data_slave_link ln -s $MYSQLTEST_VARDIR/std_data $MYSQLTEST_VARDIR/std_data_slave_link === added file 'mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt' --- a/mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/rpl/t/rpl_slave_load_tmpdir_not_exist-master.opt 2010-11-05 14:26:38 +0000 @@ -0,0 +1 @@ +--force-restart === modified file 'mysql-test/t/key_cache-master.opt' --- a/mysql-test/t/key_cache-master.opt 2003-07-06 16:09:57 +0000 +++ b/mysql-test/t/key_cache-master.opt 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ --key_buffer_size=2M --small.key_buffer_size=256K --small.key_buffer_size=128K +--force-restart === modified file 'mysql-test/t/mysqlbinlog-master.opt' --- a/mysql-test/t/mysqlbinlog-master.opt 2003-09-24 19:25:58 +0000 +++ b/mysql-test/t/mysqlbinlog-master.opt 2010-11-05 14:26:38 +0000 @@ -1 +1,2 @@ --max-binlog-size=4096 +--force-restart No bundle (reason: useless for push emails).