bk commit into 5.0 tree (gshchepa:1.2610) BUG#35993

[<gshchepa]

Below is the list of changes that have just been committed into a local
5.0 repository of gshchepa.  When gshchepa does a push these changes
will be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-04-23 00:30:26+05:00, gshchepa@stripped +4 -0
  Fixed bug #35993: memory corruption and crash with multibyte conversion.
  
  Grouping of long BLOB/TEXT values combined with the call to CONVERT
  function casting to GBK or BIG5 charsets crashes the server:
  
    SELECT CONVERT(blob_column USING big5) FROM t GROUP BY 1.
  
  
  MySQL server uses sorting (the filesort procedure) in the temporary
  table to evaluate the GROUP BY clause in case of lack of suitable index.
  That procedure takes into account only first @max_sort_length bytes
  (system variable, usually 1024) of TEXT/BLOB sorting key string.
  The my_strnxfrm_gbk and my_strnxfrm_big5 fill temporary keys
  with data of whole blob length instead of @max_sort_length bytes
  length. That buffer overrun has been fixed.

  mysql-test/r/ctype_gbk.result@stripped, 2008-04-22 23:39:42+05:00, gshchepa@stripped +5 -0
    Added test case for bug #35993.

  mysql-test/t/ctype_gbk.test@stripped, 2008-04-22 23:39:44+05:00, gshchepa@stripped +12 -0
    Added test case for bug #35993.

  strings/ctype-big5.c@stripped, 2008-04-22 23:39:46+05:00, gshchepa@stripped +4 -2
    Fixed bug #35993: memory corruption and crash with multibyte conversion.
    
    Buffer overrun has been fixed in the my_strnxfrm_big5 function.

  strings/ctype-gbk.c@stripped, 2008-04-22 23:39:47+05:00, gshchepa@stripped +4 -2
    Fixed bug #35993: memory corruption and crash with multibyte conversion.
    
    Buffer overrun has been fixed in the my_strnxfrm_gbk function.

diff -Nrup a/mysql-test/r/ctype_gbk.result b/mysql-test/r/ctype_gbk.result
--- a/mysql-test/r/ctype_gbk.result	2008-03-26 12:33:53 +04:00
+++ b/mysql-test/r/ctype_gbk.result	2008-04-22 23:39:42 +05:00
@@ -247,4 +247,9 @@ t1	CREATE TABLE `t1` (
   `c2` text NOT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=gbk
 drop table t1;
+CREATE TABLE t1(a MEDIUMTEXT);
+INSERT INTO t1 VALUES (REPEAT(0x1125,200000)), (''), ('');
+SELECT CONVERT(a USING gbk)  FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+SELECT CONVERT(a USING big5) FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+DROP TABLES t1;
 End of 5.0 tests
diff -Nrup a/mysql-test/t/ctype_gbk.test b/mysql-test/t/ctype_gbk.test
--- a/mysql-test/t/ctype_gbk.test	2008-02-04 11:10:37 +04:00
+++ b/mysql-test/t/ctype_gbk.test	2008-04-22 23:39:44 +05:00
@@ -53,4 +53,16 @@ alter table t1 change c1 c1 mediumtext  
 show create table t1;
 drop table t1;
 
+#
+# Bug#35993: severe memory corruption and crash with multibyte conversion
+#
+
+CREATE TABLE t1(a MEDIUMTEXT);
+INSERT INTO t1 VALUES (REPEAT(0x1125,200000)), (''), ('');
+
+SELECT CONVERT(a USING gbk)  FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+SELECT CONVERT(a USING big5) FROM t1 GROUP BY 1 LIMIT 1 INTO @nullll;
+
+DROP TABLES t1;
+
 --echo End of 5.0 tests
diff -Nrup a/strings/ctype-big5.c b/strings/ctype-big5.c
--- a/strings/ctype-big5.c	2007-10-04 10:54:48 +05:00
+++ b/strings/ctype-big5.c	2008-04-22 23:39:46 +05:00
@@ -307,15 +307,17 @@ static int my_strnxfrm_big5(CHARSET_INFO
 {
   uint16 e;
   uint dstlen= len;
+  uchar *dest_end= dest + dstlen;
 
   len = srclen;
-  while (len--)
+  while (len-- && dest < dest_end)
   {
     if ((len > 0) && isbig5code(*src, *(src+1)))
     {
       e = big5strokexfrm((uint16) big5code(*src, *(src+1)));
       *dest++ = big5head(e);
-      *dest++ = big5tail(e);
+      if (dest < dest_end)
+        *dest++ = big5tail(e);
       src +=2;
       len--;
     } else
diff -Nrup a/strings/ctype-gbk.c b/strings/ctype-gbk.c
--- a/strings/ctype-gbk.c	2007-06-07 13:16:48 +05:00
+++ b/strings/ctype-gbk.c	2008-04-22 23:39:47 +05:00
@@ -2668,15 +2668,17 @@ static int my_strnxfrm_gbk(CHARSET_INFO 
 {
   uint16 e;
   uint dstlen= len;
+  uchar *dest_end= dest + dstlen;
 
   len = srclen;
-  while (len--)
+  while (len-- && dest < dest_end)
   {
     if ((len > 0) && isgbkcode(*src, *(src+1)))
     {
       e = gbksortorder((uint16) gbkcode(*src, *(src+1)));
       *dest++ = gbkhead(e);
-      *dest++ = gbktail(e);
+      if (dest < dest_end)
+        *dest++ = gbktail(e);
       src+=2;
       len--;
     } else