bzr push into mysql-5.1 branch (norvald.ryeng:3594 to 3595) Bug#11765254

[Norvald H. Ryeng]

 3595 Norvald H. Ryeng	2011-08-30
      Bug#11765254 - 58200: ASSERTION FAILED: PARAM.SORT_LENGTH WHEN
      GROUPING BY FUNCTIONS.... (PART
      
      The bug was introduced in a patch for bug 49897.
      
      Problem: The assertion inserted by the original patch to guard against
      zero-lenght sort keys during merge phase triggers also when the whole
      set fits in memory.
      
      Fix: Move assert so that it does not trigger if the whole set is in
      memory.
     @ mysql-test/r/group_by.result
        Add test for bug#11765254
     @ mysql-test/t/group_by.test
        Add test for bug#11765254
     @ sql/filesort.cc
        Move assertion

    modified:
      mysql-test/r/group_by.result
      mysql-test/t/group_by.test
      sql/filesort.cc
 3594 Tor Didriksen	2011-08-29
      BUG#12911710 - VALGRIND FAILURE IN ROW-DEBUG:PERFSCHEMA.SOCKET_SUMMARY_BY_INSTANCE_FUNC 
      
      Converting the number zero to binary and back yielded the number zero,
      but with no digits, i.e. zero precision.
      This made the multiply algorithm go haywire in various ways.
     @ include/decimal.h
        Document struct st_decimal_t
     @ mysql-test/r/type_newdecimal.result
        New test case (valgrind warnings)
     @ mysql-test/t/type_newdecimal.test
        New test case (valgrind warnings)
     @ sql/my_decimal.h
        Remove the HAVE_purify enabled/disabled code.
     @ strings/decimal.c
        Make a proper zero, with non-zero precision.

    modified:
      include/decimal.h
      mysql-test/r/type_newdecimal.result
      mysql-test/t/type_newdecimal.test
      sql/my_decimal.h
      strings/decimal.c
=== modified file 'mysql-test/r/group_by.result'
--- a/mysql-test/r/group_by.result	2011-02-18 10:50:06 +0000
+++ b/mysql-test/r/group_by.result	2011-08-30 07:56:07 +0000
@@ -1891,4 +1891,36 @@ a	AVG(t1.b)	t11c	t12c
 1	4.0000	6	6
 2	2.0000	7	7
 DROP TABLE t1;
+#
+# Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+# by functions
+#
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+1
+1
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+1
+1
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+Warning	1292	Truncated incorrect INTEGER value: ''
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+1
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+Warning	1292	Truncated incorrect INTEGER value: 'K'
+Warning	1292	Truncated incorrect INTEGER value: 'jxW<'
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 # End of 5.1 tests

=== modified file 'mysql-test/t/group_by.test'
--- a/mysql-test/t/group_by.test	2011-02-18 10:50:06 +0000
+++ b/mysql-test/t/group_by.test	2011-08-30 07:56:07 +0000
@@ -1283,5 +1283,19 @@ FROM t1 GROUP BY a;
 
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11765254 (58200): Assertion failed: param.sort_length when grouping
+--echo # by functions
+--echo #
+
+SET SQL_BIG_TABLES=1;
+CREATE TABLE t1(a INT);
+INSERT INTO t1 VALUES (0),(0);
+SELECT 1 FROM t1 GROUP BY IF(`a`,'','');
+SELECT 1 FROM t1 GROUP BY TRIM(LEADING RAND() FROM '');
+SELECT 1 FROM t1 GROUP BY SUBSTRING('',SLEEP(0),'');
+SELECT 1 FROM t1 GROUP BY SUBSTRING(SYSDATE() FROM 'K' FOR 'jxW<');
+DROP TABLE t1;
+SET SQL_BIG_TABLES=0;
 
 --echo # End of 5.1 tests

=== modified file 'sql/filesort.cc'
--- a/sql/filesort.cc	2011-08-25 08:38:07 +0000
+++ b/sql/filesort.cc	2011-08-30 07:56:07 +0000
@@ -144,8 +144,6 @@ ha_rows filesort(THD *thd, TABLE *table,
   error= 1;
   bzero((char*) &param,sizeof(param));
   param.sort_length= sortlength(thd, sortorder, s_length, &multi_byte_charset);
-  /* filesort cannot handle zero-length records. */
-  DBUG_ASSERT(param.sort_length);
   param.ref_length= table->file->ref_length;
   param.addon_field= 0;
   param.addon_length= 0;
@@ -257,6 +255,9 @@ ha_rows filesort(THD *thd, TABLE *table,
   }
   else
   {
+    /* filesort cannot handle zero-length records during merge. */
+    DBUG_ASSERT(param.sort_length != 0);
+
     if (table_sort.buffpek && table_sort.buffpek_len < maxbuffer)
     {
       x_free(table_sort.buffpek);

No bundle (reason: useless for push emails).