Hi,
There are some suggestions, you can have a look!
Dao-Gang.Qu@stripped wrote:
> #At file:///home/daogangq/mysql/bzrwork/bug45214/5.1-bt/ based on
> revid:davi.arnaut@stripped
>
> 2941 Dao-Gang.Qu@stripped 2009-07-02
> Bug #45214 get_master_version_and_clock does not report error when queries
> fail
>
> The "get_master_version_and_clock(...)" function in sql/slave.cc ignores
> error and passes directly when queries fail, or queries succeed
> but the result retrieved is empty.
>
> The "get_master_version_and_clock(...)" function should try to reconnect
> master
> if queries fail because of transient network problems, and fail otherwise.
> The I/O thread should print a warning if the some system variables do not
> exist on master (very old master)
> @ mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test
> Added test file for bug #45214
> @ mysql-test/suite/rpl/r/rpl_get_master_version_and_clock.result
> Added test result for bug #45214
> @ mysql-test/suite/rpl/t/rpl_get_master_version_and_clock.test
> Added test file for bug #45214
> @ sql/slave.cc
> The 'is_network_error()' function is added for checking if the error is
> caused by network.
> Added a new return value (2) to 'get_master_version_and_clock()' function
> result set
> to indicate transient network errors when queries fail, and the caller should
>
> try to reconnect in this case.
>
> added:
> mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test
> mysql-test/suite/rpl/r/rpl_get_master_version_and_clock.result
> mysql-test/suite/rpl/t/rpl_get_master_version_and_clock.test
> modified:
> sql/slave.cc
> === added file 'mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test'
> --- a/mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test 1970-01-01
> 00:00:00 +0000
> +++ b/mysql-test/extra/rpl_tests/rpl_get_master_version_and_clock.test 2009-07-02
> 10:55:59 +0000
> @@ -0,0 +1,63 @@
> +#
> +# BUG#45214
> +# The common part of the "rpl_get_master_version_and_clock" test.
> +# Restart slave under network disconnection between slave and master
> +# following the steps:
> +# 1 - Got DBUG_SYNC_POINT lock
> +# 2 - Set DBUG_SYNC_POINT before call mysql_real_query(...) function in
> get_master_version_and_clock(...) function and hang here
> +# 3 - shutdown master server for simulating network disconnection
> +# 4 - Release DBUG_SYNC_POINT lock
> +# 5 - Check if the slave I/O thread tries to reconnect to master.
> +#
> +# Note: Please make sure initialize the $debug_lock when call the test script.
> +#
> +connection slave;
> +if (`SELECT '$debug_lock' = ''`)
> +{
> + --die Cannot continue. Please set value for $debug_lock.
> +}
> +
> +# Restart slave
> +--disable_warnings
> +stop slave;
> +source include/wait_for_slave_to_stop.inc;
> +start slave;
> +source include/wait_for_slave_to_start.inc;
> +connection master;
> +# Write file to make mysql-test-run.pl expect the "crash", but don't start
> +# it until it's told to
> +--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
> +wait
> +EOF
> +
> +# Send shutdown to the connected server and give
> +# it 10 seconds to die before zapping it
> +shutdown_server 10;
> +
> +connection slave;
> +eval SELECT RELEASE_LOCK($debug_lock);
> +
> +# Write file to make mysql-test-run.pl start up the server again
> +--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
> +restart
> +EOF
> +
> +connection master;
> +# Turn on reconnect
> +--enable_reconnect
> +
> +# Call script that will poll the server waiting for it to be back online again
> +--source include/wait_until_connected_again.inc
> +
> +# Turn off reconnect again
> +--disable_reconnect
> +
> +connection slave;
> +source include/wait_for_slave_to_start.inc;
> +
> +# Show slave last IO errno
> +let $last_io_errno= query_get_value("show slave status", Last_IO_Errno, 1);
> +echo Slave_IO_Errno= $last_io_errno;
> +
> +
> +# End of tests
>
> === added file 'mysql-test/suite/rpl/r/rpl_get_master_version_and_clock.result'
> --- a/mysql-test/suite/rpl/r/rpl_get_master_version_and_clock.result 1970-01-01
> 00:00:00 +0000
> +++ b/mysql-test/suite/rpl/r/rpl_get_master_version_and_clock.result 2009-07-02
> 10:55:59 +0000
> @@ -0,0 +1,40 @@
> +stop slave;
> +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
> +reset master;
> +reset slave;
> +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
> +start slave;
> +SELECT IS_FREE_LOCK("debug_lock.before_get_UNIX_TIMESTAMP");
> +IS_FREE_LOCK("debug_lock.before_get_UNIX_TIMESTAMP")
> +1
> +SELECT GET_LOCK("debug_lock.before_get_UNIX_TIMESTAMP", 1000);
> +GET_LOCK("debug_lock.before_get_UNIX_TIMESTAMP", 1000)
> +1
> +set global debug= 'd,debug_lock.before_get_UNIX_TIMESTAMP';
> +stop slave;
> +start slave;
> +SELECT RELEASE_LOCK("debug_lock.before_get_UNIX_TIMESTAMP");
> +RELEASE_LOCK("debug_lock.before_get_UNIX_TIMESTAMP")
> +1
> +Slave_IO_Errno= 2013
> +SELECT IS_FREE_LOCK("debug_lock.before_get_SERVER_ID");
> +IS_FREE_LOCK("debug_lock.before_get_SERVER_ID")
> +1
> +SELECT GET_LOCK("debug_lock.before_get_SERVER_ID", 1000);
> +GET_LOCK("debug_lock.before_get_SERVER_ID", 1000)
> +1
> +set global debug= 'd,debug_lock.before_get_SERVER_ID';
> +stop slave;
> +start slave;
> +SELECT RELEASE_LOCK("debug_lock.before_get_SERVER_ID");
> +RELEASE_LOCK("debug_lock.before_get_SERVER_ID")
> +1
> +Slave_IO_Errno= 2013
> +set global debug= '';
> +reset master;
> +include/stop_slave.inc
> +change master to master_port=SLAVE_PORT;
> +start slave;
> +*** must be having the replicate-same-server-id IO thread error ***
> +Slave_IO_Errno= 1593
> +Slave_IO_Error= Fatal error: The slave I/O thread stops because master and slave
> have equal MySQL server ids; these ids must be different for replication to work (or the
> --replicate-same-server-id option must be used on slave but this does not always make
> sense; please check the manual before using it).
>
> === added file 'mysql-test/suite/rpl/t/rpl_get_master_version_and_clock.test'
> --- a/mysql-test/suite/rpl/t/rpl_get_master_version_and_clock.test 1970-01-01
> 00:00:00 +0000
> +++ b/mysql-test/suite/rpl/t/rpl_get_master_version_and_clock.test 2009-07-02
> 10:55:59 +0000
> @@ -0,0 +1,60 @@
> +#
> +# BUG#45214
> +# This test verifies if the slave I/O tread tries to reconnect to
> +# master when it tries to get the values of the UNIX_TIMESTAMP, SERVER_ID,
> +# COLLATION_SERVER and TIME_ZONE from master under network disconnection.
> +# The COLLATION_SERVER and TIME_ZONE are got only on master server version 4.
> +# So they can't be verified by test case here.
> +# Finish the following tests by calling its common test script:
> +# extra/rpl_tests/rpl_get_master_version_and_clock.test.
> +# And meanwhile this test checks that the slave I/O thread refuses to start if
> slave
> +# and master have the same server id (because this is a useless setup,
> +# and otherwise SHOW SLAVE STATUS shows progress but all queries are
> +# ignored, which has caught our customers), unless
> +# --replicate-same-server-id.
> +#
> +
> +source include/master-slave.inc;
> +
> +#Test case 1: Try to get the value of the UNIX_TIMESTAMP from master under network
> disconnection
> +connection slave;
> +let $debug_saved= `select @@global.debug`;
> +
> +let $debug_lock= "debug_lock.before_get_UNIX_TIMESTAMP";
> +eval SELECT IS_FREE_LOCK($debug_lock);
> +eval SELECT GET_LOCK($debug_lock, 1000);
> +
> +set global debug= 'd,debug_lock.before_get_UNIX_TIMESTAMP';
> +source extra/rpl_tests/rpl_get_master_version_and_clock.test;
> +
> +#Test case 2: Try to get the value of the SERVER_ID from master under network
> disconnection
> +connection slave;
> +let $debug_lock= "debug_lock.before_get_SERVER_ID";
> +eval SELECT IS_FREE_LOCK($debug_lock);
> +eval SELECT GET_LOCK($debug_lock, 1000);
> +
> +set global debug= 'd,debug_lock.before_get_SERVER_ID';
> +source extra/rpl_tests/rpl_get_master_version_and_clock.test;
> +
> +eval set global debug= '$debug_saved';
> +
> +#Test case 3: This test checks that the slave I/O thread refuses to start
> +#if slave and master have the same server id.
> +connection slave;
> +reset master;
> +# replicate ourselves
> +source include/stop_slave.inc;
> +--replace_result $SLAVE_MYPORT SLAVE_PORT
> +eval change master to master_port=$SLAVE_MYPORT;
> +start slave;
> +
> +let $slave_param= Last_IO_Errno;
> +let $slave_param_value= 1593;
> +source include/wait_for_slave_param.inc;
> +--echo *** must be having the replicate-same-server-id IO thread error ***
> +let $last_io_errno= query_get_value("show slave status", Last_IO_Errno, 1);
> +let $last_io_error= query_get_value("show slave status", Last_IO_Error, 1);
> +echo Slave_IO_Errno= $last_io_errno;
> +echo Slave_IO_Error= $last_io_error;
> +
> +# End of tests
>
> === modified file 'sql/slave.cc'
> --- a/sql/slave.cc 2009-06-09 16:44:26 +0000
> +++ b/sql/slave.cc 2009-07-02 10:55:59 +0000
> @@ -38,6 +38,7 @@
> #include <my_dir.h>
> #include <sql_common.h>
> #include <errmsg.h>
> +#include <mysqld_error.h>
> #include <mysys_err.h>
>
> #ifdef HAVE_REPLICATION
> @@ -842,6 +843,29 @@ int init_intvar_from_file(int* var, IO_C
> DBUG_RETURN(1);
> }
>
> +
> +/*
> + Check if the error is caused by network.
> + @param[in] errorno Number of the error.
> + RETURNS:
> + TRUE network error
> + FALSE not network error
> +*/
> +
> +bool is_network_error(uint errorno)
> +{
> + if (errorno == CR_CONNECTION_ERROR ||
> + errorno == CR_CONN_HOST_ERROR ||
> + errorno == CR_SERVER_GONE_ERROR ||
> + errorno == CR_SERVER_LOST ||
> + errorno == ER_CON_COUNT_ERROR ||
> + errorno == ER_SERVER_SHUTDOWN)
> + return TRUE;
> +
> + return FALSE;
> +}
> +
> +
> /*
> Note that we rely on the master's version (3.23, 4.0.14 etc) instead of
> relying on the binlog's version. This is not perfect: imagine an upgrade
> @@ -854,6 +878,7 @@ int init_intvar_from_file(int* var, IO_C
> RETURNS
> 0 ok
> 1 error
> + 2 transient network problem, the caller should try to reconnect
> */
>
> static int get_master_version_and_clock(MYSQL* mysql, Master_info* mi)
> @@ -864,6 +889,7 @@ static int get_master_version_and_clock(
> MYSQL_RES *master_res= 0;
> MYSQL_ROW master_row;
> DBUG_ENTER("get_master_version_and_clock");
> + int query_re= 0;
>
> /*
> Free old description_event_for_queue (that is needed if we are in
> @@ -938,21 +964,37 @@ static int get_master_version_and_clock(
> Compare the master and slave's clock. Do not die if master's clock is
> unavailable (very old master not supporting UNIX_TIMESTAMP()?).
> */
> + DBUG_SYNC_POINT("debug_lock.before_get_UNIX_TIMESTAMP", 10);
> + query_re= mysql_real_query(mysql, STRING_WITH_LEN("SELECT UNIX_TIMESTAMP()"));
> + if ((query_re || !(master_res= mysql_store_result(mysql))) &&
> + is_network_error(mysql_errno(mysql)))
> + {
master_res= NULL;
if ((!mysql_real_query(...) &&
(master_res= mysql_store_result(...)) &&
(master_row= mysql_fetch_row(...)))
{
// Query is OK
}
else if (is_network_error(...))
{
// Network error
}
else
{
// Fatal error
}
This can reduce merge problem and can make it more clear to understand.
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master clock failed with error: %s", mysql_error(mysql));
> + DBUG_RETURN(2);
> + }
>
> - if (!mysql_real_query(mysql, STRING_WITH_LEN("SELECT UNIX_TIMESTAMP()"))
> &&
> - (master_res= mysql_store_result(mysql)) &&
> - (master_row= mysql_fetch_row(master_res)))
> + if ((master_row= mysql_fetch_row(master_res)))
> {
> mi->clock_diff_with_master=
> (long) (time((time_t*) 0) - strtoul(master_row[0], 0, 10));
> }
> + else if (is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master clock failed with error: %s", mysql_error(mysql));
> + if (master_res)
> + mysql_free_result(master_res);
> + DBUG_RETURN(2);
> + }
> else if (!check_io_slave_killed(mi->io_thd, mi, NULL))
move this out of the function, check_io_slave_killed() in the caller.
> {
> mi->clock_diff_with_master= 0; /* The "most sensible" value */
> - sql_print_warning("\"SELECT UNIX_TIMESTAMP()\" failed on master, "
> - "do not trust column Seconds_Behind_Master of SHOW "
> - "SLAVE STATUS. Error: %s (%d)",
> - mysql_error(mysql), mysql_errno(mysql));
> + mi->report(ERROR_LEVEL, ER_UNKNOWN_ERROR,
> + "\"SELECT UNIX_TIMESTAMP()\" failed on master, "
> + "do not trust column Seconds_Behind_Master of SHOW "
> + "SLAVE STATUS. Error: %s (%d)",
> + mysql_error(mysql), mysql_errno(mysql));
This should be a warning.
> }
> if (master_res)
> mysql_free_result(master_res);
> @@ -967,26 +1009,54 @@ static int get_master_version_and_clock(
> Note: we could have put a @@SERVER_ID in the previous SELECT
> UNIX_TIMESTAMP() instead, but this would not have worked on 3.23 masters.
> */
> - if (!mysql_real_query(mysql,
> - STRING_WITH_LEN("SHOW VARIABLES LIKE 'SERVER_ID'"))
> &&
> - (master_res= mysql_store_result(mysql)))
> - {
> - if ((master_row= mysql_fetch_row(master_res)) &&
> - (::server_id == strtoul(master_row[1], 0, 10)) &&
> - !mi->rli.replicate_same_server_id)
> + DBUG_SYNC_POINT("debug_lock.before_get_SERVER_ID", 10);
> + query_re= mysql_real_query(mysql, STRING_WITH_LEN("SHOW VARIABLES LIKE
> 'SERVER_ID'"));
> + if (query_re || !(master_res= mysql_store_result(mysql)))
> + {
> + if (is_network_error(mysql_errno(mysql)))
> {
> - errmsg= "The slave I/O thread stops because master and slave have equal \
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master SERVER_ID failed with error: %s", mysql_error(mysql));
> + DBUG_RETURN(2);
> + }
> + /* Fatal error */
> + errmsg= "The slave I/O thread stops because a fatal error is encountered \
> +when it try to get the value of SERVER_ID variable from master.";
> + err_code= ER_SLAVE_FATAL_ERROR;
> + sprintf(err_buff, ER(err_code), errmsg);
> + goto err;
> + }
> +
> + master_row= mysql_fetch_row(master_res);
> + if (master_row && (::server_id == strtoul(master_row[1], 0, 10))
> &&
> + !mi->rli.replicate_same_server_id)
> + {
> + errmsg= "The slave I/O thread stops because master and slave have equal \
> MySQL server ids; these ids must be different for replication to work (or \
> the --replicate-same-server-id option must be used on slave but this does \
> not always make sense; please check the manual before using it).";
> - err_code= ER_SLAVE_FATAL_ERROR;
> - sprintf(err_buff, ER(err_code), errmsg);
> + err_code= ER_SLAVE_FATAL_ERROR;
> + sprintf(err_buff, ER(err_code), errmsg);
> + }
> + else if (!master_row)
> + {
> + if (is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master SERVER_ID failed with error: %s", mysql_error(mysql));
> + mysql_free_result(master_res);
> + DBUG_RETURN(2);
> }
> - mysql_free_result(master_res);
> - if (errmsg)
> - goto err;
> + else
> + mi->report(WARNING_LEVEL, ER_UNKNOWN_SYSTEM_VARIABLE,
> + "Unknown system variable 'SERVER_ID' on master, \
> +maybe it is a very old master.");
> }
> + mysql_free_result(master_res);
>
> + if (errmsg)
> + goto err;
put the goto where errmsg was set.
> +
> /*
> Check that the master's global character_set_server and ours are the same.
> Not fatal if query fails (old master?).
> @@ -1008,23 +1078,55 @@ not always make sense; please check the
> if (*mysql->server_version == '3')
> goto err;
>
> - if ((*mysql->server_version == '4') &&
> - !mysql_real_query(mysql,
> - STRING_WITH_LEN("SELECT @@GLOBAL.COLLATION_SERVER"))
> &&
> - (master_res= mysql_store_result(mysql)))
> + if (*mysql->server_version == '4')
> {
> - if ((master_row= mysql_fetch_row(master_res)) &&
> - strcmp(master_row[0], global_system_variables.collation_server->name))
> + query_re= mysql_real_query(mysql, STRING_WITH_LEN("SELECT
> @@GLOBAL.COLLATION_SERVER"));
> + if (query_re || !(master_res= mysql_store_result(mysql)))
> {
> - errmsg= "The slave I/O thread stops because master and slave have \
> + if (is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master COLLATION_SERVER failed with error: %s",
> mysql_error(mysql));
> + DBUG_RETURN(2);
> + }
> +
> + if (mysql_errno(mysql) != ER_UNKNOWN_SYSTEM_VARIABLE)
> + {
> + /* Fatal error */
> + errmsg= "The slave I/O thread stops because a fatal error is encountered \
> +when it try to get the value of COLLATION_SERVER global variable from master.";
> + err_code= ER_SLAVE_FATAL_ERROR;
> + sprintf(err_buff, ER(err_code), errmsg);
> + goto err;
> + }
> + else
> + mi->report(WARNING_LEVEL, ER_UNKNOWN_SYSTEM_VARIABLE,
> + "Unknown system variable 'COLLATION_SERVER' on master, \
> +maybe it is a very old master.");
> + }
> + else
> + {
> + master_row= mysql_fetch_row(master_res);
> + if (master_row && strcmp(master_row[0],
> + global_system_variables.collation_server->name))
> + {
> + errmsg= "The slave I/O thread stops because master and slave have \
> different values for the COLLATION_SERVER global variable. The values must \
> be equal for replication to work";
> - err_code= ER_SLAVE_FATAL_ERROR;
> - sprintf(err_buff, ER(err_code), errmsg);
> + err_code= ER_SLAVE_FATAL_ERROR;
> + sprintf(err_buff, ER(err_code), errmsg);
> + }
> + else if (!master_row && is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master COLLATION_SERVER failed with error: %s",
> mysql_error(mysql));
> + mysql_free_result(master_res);
> + DBUG_RETURN(2);
> + }
> + mysql_free_result(master_res);
> + if (errmsg)
> + goto err;
> }
> - mysql_free_result(master_res);
> - if (errmsg)
> - goto err;
> }
>
> /*
> @@ -1042,13 +1144,28 @@ be equal for replication to work";
> This check is only necessary for 4.x masters (and < 5.0.4 masters but
> those were alpha).
> */
> - if ((*mysql->server_version == '4') &&
> - !mysql_real_query(mysql, STRING_WITH_LEN("SELECT @@GLOBAL.TIME_ZONE"))
> &&
> - (master_res= mysql_store_result(mysql)))
> - {
> - if ((master_row= mysql_fetch_row(master_res)) &&
> - strcmp(master_row[0],
> - global_system_variables.time_zone->get_name()->ptr()))
> + if (*mysql->server_version == '4')
> + {
> + query_re= mysql_real_query(mysql, STRING_WITH_LEN("SELECT
> @@GLOBAL.TIME_ZONE"));
> + if (query_re || !(master_res= mysql_store_result(mysql)))
> + {
> + if (is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master TIME_ZONE failed with error: %s",
> mysql_error(mysql));
> + DBUG_RETURN(2);
> + }
> + /* Fatal error */
> + errmsg= "The slave I/O thread stops because a fatal error is encountered \
> +when it try to get the value of TIME_ZONE global variable from master.";
> + err_code= ER_SLAVE_FATAL_ERROR;
> + sprintf(err_buff, ER(err_code), errmsg);
> + goto err;
> + }
> +
> + master_row= mysql_fetch_row(master_res);
> + if (master_row && strcmp(master_row[0],
> +
> global_system_variables.time_zone->get_name()->ptr()))
> {
> errmsg= "The slave I/O thread stops because master and slave have \
> different values for the TIME_ZONE global variable. The values must \
> @@ -1056,6 +1173,13 @@ be equal for replication to work";
> err_code= ER_SLAVE_FATAL_ERROR;
> sprintf(err_buff, ER(err_code), errmsg);
> }
> + else if (!master_row && is_network_error(mysql_errno(mysql)))
> + {
> + mi->report(WARNING_LEVEL, mysql_errno(mysql),
> + "Get master TIME_ZONE failed with error: %s", mysql_error(mysql));
> + mysql_free_result(master_res);
> + DBUG_RETURN(2);
> + }
> mysql_free_result(master_res);
>
> if (errmsg)
> @@ -2372,6 +2496,7 @@ pthread_handler_t handle_slave_io(void *
> char llbuff[22];
> uint retry_count;
> bool suppress_warnings;
> + int ret;
> #ifndef DBUG_OFF
> uint retry_count_reg= 0, retry_count_dump= 0, retry_count_event= 0;
> #endif
> @@ -2451,8 +2576,20 @@ connected:
> mi->slave_running= MYSQL_SLAVE_RUN_CONNECT;
> thd->slave_net = &mysql->net;
> thd_proc_info(thd, "Checking master version");
> - if (get_master_version_and_clock(mysql, mi))
> + ret= get_master_version_and_clock(mysql, mi);
> + if (ret == 1)
> + /* Fatal error */
> goto err;
> +
> + if (ret == 2)
> + {
> + suppress_warnings= FALSE;
check_io_slave_killed() here.
> + /* Try to reconnect because the error was caused by a transient network problem
> */
> + if (try_to_reconnect(thd, mysql, mi, &retry_count, suppress_warnings,
> + reconnect_messages[SLAVE_RECON_ACT_REG]))
> + goto err;
> + goto connected;
> + }
>
> if (mi->rli.relay_log.description_event_for_queue->binlog_version > 1)
> {