List:Commits« Previous MessageNext Message »
From:Libing Song Date:February 17 2011 2:40am
Subject:bzr commit into mysql-trunk branch (anders.song:3653) Bug#11747577
View as plain text  
#At file:///home/songlibing/bzrwork/wt3/mysql-trunk/ based on revid:dmitry.lenev@stripped

 3653 Libing Song	2011-02-17
      BUG#11747577 Not able to recover binary/blob data correctly using mysqlbinlog
            
      The queries generated by mysqlbinlog could not be executed by 'mysql' client
      if there was any 0x00 in it. Or the queries recovered diverged data if there
      was any 0x0D0A sequence in it. The reason is that 'mysql' client always
      translated '0x0D0A'('\r\n') to '0x0A'('\n') and treated '0x00' as the end
      of a query if it is in non-interactive mode(receiving queries from a file
      or a pipe).
            
      This patch added the option 'binary-mode' in 'mysql' program.  'mysql' will
      not translated '0x0D0A' to '0x0A' and not treat '0x00' as the end of a query
      if binary-mode is set 1 and it is in non-interactive mode.  Meanwhile all
      mysql client commands except 'delimiter' and '\C' are disabled. This will
      improve the performance when mysql applies huge logs.

    added:
      mysql-test/r/mysql_binary_mode.result
      mysql-test/t/mysql_binary_mode.test
    modified:
      client/client_priv.h
      client/my_readline.h
      client/mysql.cc
      client/readline.cc
=== modified file 'client/client_priv.h'
--- a/client/client_priv.h	2011-01-16 04:02:29 +0000
+++ b/client/client_priv.h	2011-02-17 02:39:58 +0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 2001-2006 MySQL AB, 2009 Sun Microsystems, Inc
+/* Copyright 2000, 2011, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -86,6 +86,7 @@ enum options_client
   OPT_DEFAULT_PLUGIN,
   OPT_RAW_OUTPUT, OPT_WAIT_SERVER_ID, OPT_STOP_NEVER,
   OPT_BINLOG_ROWS_EVENT_MAX_SIZE,
+  OPT_BINARY_MODE,
   OPT_MAX_CLIENT_OPTION
 };
 

=== modified file 'client/my_readline.h'
--- a/client/my_readline.h	2011-02-05 05:04:15 +0000
+++ b/client/my_readline.h	2011-02-17 02:39:58 +0000
@@ -1,7 +1,7 @@
 #ifndef CLIENT_MY_READLINE_INCLUDED
 #define CLIENT_MY_READLINE_INCLUDED
 
-/* Copyright (C) 2000 MySQL AB
+/* Copyright 2000, 2011, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -34,7 +34,7 @@ typedef struct st_line_buffer
 
 extern LINE_BUFFER *batch_readline_init(ulong max_size,FILE *file);
 extern LINE_BUFFER *batch_readline_command(LINE_BUFFER *buffer, char * str);
-extern char *batch_readline(LINE_BUFFER *buffer);
+extern char *batch_readline(LINE_BUFFER *buffer, bool binary_mode);
 extern void batch_readline_end(LINE_BUFFER *buffer);
 
 #endif /* CLIENT_MY_READLINE_INCLUDED */

=== modified file 'client/mysql.cc'
--- a/client/mysql.cc	2011-02-15 12:38:39 +0000
+++ b/client/mysql.cc	2011-02-17 02:39:58 +0000
@@ -1,4 +1,4 @@
-/* Copyright 2000, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright 2000, 2011, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -150,6 +150,7 @@ static uint my_end_arg;
 static char * opt_mysql_unix_port=0;
 static char *opt_bind_addr = NULL;
 static int connect_flag=CLIENT_INTERACTIVE;
+static my_bool opt_binary_mode= FALSE;
 static char *current_host,*current_db,*current_user=0,*opt_password=0,
             *current_prompt=0, *delimiter_str= 0,
             *default_charset= (char*) MYSQL_AUTODETECT_CHARSET_NAME,
@@ -1044,9 +1045,10 @@ static void initialize_readline (char *n
 static void fix_history(String *final_command);
 #endif
 
-static COMMANDS *find_command(char *name,char cmd_name);
-static bool add_line(String &buffer,char *line,char *in_string,
-                     bool *ml_comment, bool truncated);
+static COMMANDS *find_command(char *name);
+static COMMANDS *find_command(char cmd_name);
+static bool add_line(String &buffer, char *line, ulong line_length,
+                     char *in_string, bool *ml_comment, bool truncated);
 static void remove_cntrl(String &buffer);
 static void print_table_data(MYSQL_RES *result);
 static void print_table_data_html(MYSQL_RES *result);
@@ -1064,6 +1066,30 @@ extern "C" sig_handler handle_sigint(int
 static sig_handler window_resize(int sig);
 #endif
 
+const char DELIMITER_NAME[]= "delimiter";
+const uint DELIMITER_NAME_LEN= sizeof(DELIMITER_NAME) - 1;
+inline bool is_delimiter_command(char *name, ulong len)
+{
+  return  (len == DELIMITER_NAME_LEN &&
+           !my_strnncoll(charset_info, (uchar*)name, DELIMITER_NAME_LEN,
+                         (uchar *)DELIMITER_NAME, DELIMITER_NAME_LEN));
+}
+
+inline int get_command_index(char cmd_char)
+{
+  /*
+    All commands are in the first part of commands array and have a function
+    to implemente it.
+  */
+  for (uint i=0; *commands[i].func; i++)
+    if (commands[i].cmd_char == cmd_char)
+      return i;
+  return -1;
+}
+
+static int delimiter_index= -1;
+static int charset_index= -1;
+static bool real_binary_mode= FALSE;
 
 int main(int argc,char *argv[])
 {
@@ -1072,7 +1098,9 @@ int main(int argc,char *argv[])
   MY_INIT(argv[0]);
   DBUG_ENTER("main");
   DBUG_PROCESS(argv[0]);
-  
+
+  charset_index= get_command_index('C');
+  delimiter_index= get_command_index('d');
   delimiter_str= delimiter;
   default_prompt = my_strdup(getenv("MYSQL_PS1") ? 
 			     getenv("MYSQL_PS1") : 
@@ -1574,6 +1602,11 @@ static struct my_option my_long_options[
     "Default authentication client-side plugin to use.",
    (uchar**) &opt_default_auth, (uchar**) &opt_default_auth, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+  {"binary-mode", OPT_BINARY_MODE,
+   "ASCII '0x00' is allowed in queries and '0x0D0A'('\\r\\n') sequence "
+   "is not translated to '0x0A'('\\n') if 'mysql' is non-interactive and "
+   "--binary-mode is set.",
+   &opt_binary_mode, &opt_binary_mode, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -1845,13 +1878,36 @@ static int read_and_execute(bool interac
   ulong line_number=0;
   bool ml_comment= 0;  
   COMMANDS *com;
+  ulong line_length= 0;
   status.exit_status=1;
 
   for (;;)
   {
+
+    real_binary_mode= !interactive && opt_binary_mode;
     if (!interactive)
     {
-      line=batch_readline(status.line_buff);
+      line=batch_readline(status.line_buff, real_binary_mode);
+      line_length= status.line_buff->read_length;
+      /*
+        ASCII 0x00 is not allowed appearing in queries if it is not in binary
+        mode.
+      */
+      if (!real_binary_mode && line && strlen(line) != line_length)
+      {
+        status.exit_status= 1;
+        String msg;
+        msg.append("ASCII '0x00' appeared in the query.  It is disallowed unless "
+                   "option --binary-mode is set and 'mysql' is non-interactive.  "
+                   "Please set --binary-mode 1 if the ASCII '0x00' is expected.  "
+                   "Query: '");
+        msg.append(glob_buffer);
+        msg.append(line);
+        msg.append("'.");
+        put_info(msg.c_ptr(), INFO_ERROR);
+        break;
+      }
+
       /*
         Skip UTF8 Byte Order Marker (BOM) 0xEFBBBF.
         Editors like "notepad" put this marker in
@@ -1913,6 +1969,8 @@ static int read_and_execute(bool interac
       */
       if (opt_outfile && line)
 	fprintf(OUTFILE, "%s\n", line);
+
+      line_length= line ? strlen(line) : 0;
     }
     // End of file or system error
     if (!line)
@@ -1929,7 +1987,7 @@ static int read_and_execute(bool interac
       (We want to allow help, print and clear anywhere at line start
     */
     if ((named_cmds || glob_buffer.is_empty())
-	&& !ml_comment && !in_string && (com=find_command(line,0)))
+	&& !ml_comment && !in_string && (com=find_command(line)))
     {
       if ((*com->func)(&glob_buffer,line) > 0)
 	break;
@@ -1941,7 +1999,7 @@ static int read_and_execute(bool interac
 #endif
       continue;
     }
-    if (add_line(glob_buffer, line, &in_string, &ml_comment,
+    if (add_line(glob_buffer, line, line_length, &in_string, &ml_comment,
                  status.line_buff ? status.line_buff->truncated : 0))
       break;
   }
@@ -1962,71 +2020,102 @@ static int read_and_execute(bool interac
   buffer.free();
   tmpbuf.free();
 #endif
-
+  real_binary_mode= FALSE;
   return status.exit_status;
 }
 
+static COMMANDS *find_command(char cmd_char)
+{
+  DBUG_ENTER("find_command");
+  DBUG_PRINT("enter", ("cmd_char: %d", cmd_char));
+
+  int index= -1;
+  if (real_binary_mode)
+  {
+    if (cmd_char == 'C')
+      index= charset_index;
+  }
+  else
+    index= get_command_index(cmd_char);
+
+  if (index >= 0)
+  {
+    DBUG_PRINT("exit",("found command: %s", commands[index].name));
+    DBUG_RETURN(&commands[index]);
+  }
+  else
+    DBUG_RETURN((COMMANDS *) 0);
+}
 
-static COMMANDS *find_command(char *name,char cmd_char)
+static COMMANDS *find_command(char *name)
 {
   uint len;
   char *end;
   DBUG_ENTER("find_command");
-  DBUG_PRINT("enter",("name: '%s'  char: %d", name ? name : "NULL", cmd_char));
 
-  if (!name)
+  DBUG_ASSERT(name != NULL);
+  DBUG_PRINT("enter", ("name: '%s'", name));
+
+  while (my_isspace(charset_info,*name))
+    name++;
+  /*
+    If there is an \\g in the row or if the row has a delimiter but
+    this is not a delimiter command, let add_line() take care of
+    parsing the row and calling find_command()
+  */
+  if ((!real_binary_mode && strstr(name, "\\g")) ||
+      (strstr(name, delimiter) &&
+       !is_delimiter_command(name, DELIMITER_NAME_LEN)))
+      DBUG_RETURN((COMMANDS *) 0);
+
+  if ((end=strcont(name," \t")))
   {
-    len=0;
-    end=0;
+    len=(uint) (end - name);
+    while (my_isspace(charset_info,*end))
+      end++;
+    if (!*end)
+      end=0;					// no arguments to function
+  }
+  else
+    len=(uint) strlen(name);
+
+  int index= -1;
+  if (real_binary_mode)
+  {
+    if (is_delimiter_command(name, len))
+      index= delimiter_index;
   }
   else
   {
-    while (my_isspace(charset_info,*name))
-      name++;
     /*
-      If there is an \\g in the row or if the row has a delimiter but
-      this is not a delimiter command, let add_line() take care of
-      parsing the row and calling find_command()
+      All commands are in the first part of commands array and have a function
+      to implemente it.
     */
-    if (strstr(name, "\\g") || (strstr(name, delimiter) &&
-                                !(strlen(name) >= 9 &&
-                                  !my_strnncoll(&my_charset_latin1,
-                                                (uchar*) name, 9,
-                                                (const uchar*) "delimiter",
-                                                9))))
-      DBUG_RETURN((COMMANDS *) 0);
-    if ((end=strcont(name," \t")))
+    for (uint i= 0; commands[i].func; i++)
     {
-      len=(uint) (end - name);
-      while (my_isspace(charset_info,*end))
-	end++;
-      if (!*end)
-	end=0;					// no arguments to function
+      if (commands[i].name[len] == '\0' &&
+          !my_strnncoll(&my_charset_latin1, (uchar*)name, len,
+                        (uchar*)commands[i].name,len) &&
+          (!end || commands[i].takes_params))
+      {
+        index= i;
+        break;
+      }
     }
-    else
-      len=(uint) strlen(name);
   }
 
-  for (uint i= 0; commands[i].name; i++)
+  if (index >= 0)
   {
-    if (commands[i].func &&
-	((name &&
-	  !my_strnncoll(&my_charset_latin1, (uchar*)name, len,
-				     (uchar*)commands[i].name,len) &&
-	  !commands[i].name[len] &&
-	  (!end || (end && commands[i].takes_params))) ||
-	 (!name && commands[i].cmd_char == cmd_char)))
-    {
-      DBUG_PRINT("exit",("found command: %s", commands[i].name));
-      DBUG_RETURN(&commands[i]);
-    }
+    DBUG_PRINT("exit",("found command: %s", commands[index].name));
+    DBUG_RETURN(&commands[index]);
   }
+
   DBUG_RETURN((COMMANDS *) 0);
 }
 
 
-static bool add_line(String &buffer,char *line,char *in_string,
-                     bool *ml_comment, bool truncated)
+static bool add_line(String &buffer, char *line, ulong line_length,
+                     char *in_string, bool *ml_comment, bool truncated)
 {
   uchar inchar;
   char buff[80], *pos, *out;
@@ -2041,10 +2130,11 @@ static bool add_line(String &buffer,char
   if (status.add_to_history && line[0] && not_in_history(line))
     add_history(line);
 #endif
-  char *end_of_line=line+(uint) strlen(line);
+  char *end_of_line= line + line_length;
 
-  for (pos=out=line ; (inchar= (uchar) *pos) ; pos++)
+  for (pos=out=line; pos < end_of_line; pos++)
   {
+    inchar= (uchar) *pos;
     if (!preserve_comments)
     {
       // Skip spaces at the beginning of a statement
@@ -2084,7 +2174,7 @@ static bool add_line(String &buffer,char
 	*out++= (char) inchar;
 	continue;
       }
-      if ((com=find_command(NullS,(char) inchar)))
+      if ((com=find_command((char) inchar)))
       {
         // Flush previously accepted characters
         if (out != line)
@@ -2160,7 +2250,7 @@ static bool add_line(String &buffer,char
 
       pos--;
 
-      if ((com= find_command(buffer.c_ptr(), 0)))
+      if ((com= find_command(buffer.c_ptr())))
       {
           
         if ((*com->func)(&buffer, buffer.c_ptr()) > 0)
@@ -2275,9 +2365,7 @@ static bool add_line(String &buffer,char
     uint length=(uint) (out-line);
 
     if (!truncated &&
-        (length < 9 || 
-         my_strnncoll (charset_info, 
-                       (uchar *)line, 9, (const uchar *) "delimiter", 9)))
+        (length < 9 || !is_delimiter_command(line, DELIMITER_NAME_LEN)))
     {
       /* 
         Don't add a new line in case there's a DELIMITER command to be 

=== modified file 'client/readline.cc'
--- a/client/readline.cc	2011-02-09 11:16:33 +0000
+++ b/client/readline.cc	2011-02-17 02:39:58 +0000
@@ -1,4 +1,4 @@
-/* Copyright (C) 2000 MySQL AB
+/* Copyright 2000, 2011, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -52,7 +52,7 @@ LINE_BUFFER *batch_readline_init(ulong m
 }
 
 
-char *batch_readline(LINE_BUFFER *line_buff)
+char *batch_readline(LINE_BUFFER *line_buff, bool binary_mode)
 {
   char *pos;
   ulong out_length;
@@ -60,10 +60,14 @@ char *batch_readline(LINE_BUFFER *line_b
   if (!(pos=intern_read_line(line_buff, &out_length)))
     return 0;
   if (out_length && pos[out_length-1] == '\n')
-    if (--out_length && pos[out_length-1] == '\r')  /* Remove '\n' */
+  {
+    out_length--;                                   /* Remove '\n' */
+    if (!binary_mode && pos[out_length-1] == '\r')
       out_length--;                                 /* Remove '\r' */
+  }
   line_buff->read_length=out_length;
   pos[out_length]=0;
+  DBUG_DUMP("Query: ", (unsigned char *)pos, out_length);
   return pos;
 }
 
@@ -223,7 +227,7 @@ char *intern_read_line(LINE_BUFFER *buff
   for (;;)
   {
     pos=buffer->end_of_line;
-    while (*pos != '\n' && *pos)
+    while (*pos != '\n' && pos != buffer->end)
       pos++;
     if (pos == buffer->end)
     {
@@ -249,6 +253,7 @@ char *intern_read_line(LINE_BUFFER *buff
       buffer->truncated= 0;
     buffer->end_of_line=pos+1;
     *out_length=(ulong) (pos + 1 - buffer->eof - buffer->start_of_line);
+    DBUG_DUMP("Query: ", (unsigned char *)buffer->start_of_line, *out_length);
     DBUG_RETURN(buffer->start_of_line);
   }
 }

=== added file 'mysql-test/r/mysql_binary_mode.result'
Binary files a/mysql-test/r/mysql_binary_mode.result	1970-01-01 00:00:00 +0000 and b/mysql-test/r/mysql_binary_mode.result	2011-02-17 02:39:58 +0000 differ

=== added file 'mysql-test/t/mysql_binary_mode.test'
--- a/mysql-test/t/mysql_binary_mode.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/mysql_binary_mode.test	2011-02-17 02:39:58 +0000
@@ -0,0 +1,45 @@
+source include/have_binlog_format_statement.inc;
+
+--echo # Bug#33048 Not able to recover binary/blob data correctly using mysqlbinlog
+--echo # --------------------------------------------------------------------------
+--echo # The test verify that 0x00 and 0x0D0A sequence can be handled correctly by
+--echo # mysql
+--echo
+let $table_name= `SELECT 0x410D0A42`;
+eval CREATE TABLE `$table_name` (c1 CHAR(100));
+
+let $char= `SELECT 0x410042`;
+eval INSERT INTO `$table_name` VALUES("$char");
+
+let $char= `SELECT 0x410D0A42`;
+eval INSERT INTO `$table_name` VALUES("$char");
+
+eval SELECT HEX(c1) FROM `$table_name`;
+
+--echo
+FLUSH LOGS;
+eval DROP TABLE `$table_name`;
+
+--echo
+let $MYSQLD_DATADIR= `SELECT @@datadir`;
+--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 > $MYSQLTEST_VARDIR/tmp/my.sql
+
+--echo # '--exec mysql ...' without --binary-mode option
+--echo # It creates the table with a wrong table name and generates an error.
+--error 1
+--exec $MYSQL test < $MYSQLTEST_VARDIR/tmp/my.sql 2>&1
+
+--echo
+--echo # It is not in binary_mode, so table name '0x410D0A42' is translated to
+--echo # '0x410A42' by mysql.
+let $table_name1= `SELECT 0x410A42`;
+eval DROP TABLE `$table_name1`;
+
+--echo
+--echo # In binary_mode, table name '0x410D0A42' and string '0x410042' can be
+--echo # handled correctly.
+--exec $MYSQL --binary-mode test < $MYSQLTEST_VARDIR/tmp/my.sql
+eval SELECT HEX(c1) FROM `$table_name`;
+
+--echo
+eval DROP TABLE `$table_name`;


Attachment: [text/bzr-bundle] bzr/anders.song@greatopensource.com-20110217023958-xfnrk5d08ujv9ixk.bundle
Thread
bzr commit into mysql-trunk branch (anders.song:3653) Bug#11747577Libing Song17 Feb
  • Re: bzr commit into mysql-trunk branch (anders.song:3653) Bug#11747577Sven Sandberg17 Feb