bk commit into 5.0 tree (evgen:1.2595) BUG#33675

[eugene]

Below is the list of changes that have just been committed into a local
5.0 repository of evgen. When evgen does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-01-09 00:40:41+03:00, evgen@stripped +3 -0
  Bug#33675: Usage of an uninitialized memory by filesort in a subquery caused
  server crash.
  
  The filesort implementation has an optimization for subquery execution which
  consists of reusing previously allocated buffers. In particular the call to
  the read_buffpek_from_file function might be skipped when a big enough buffer
  for buffer descriptors (buffpeks) is already allocated. Beside allocating
  memory for buffpeks this function fills allocated buffer with data read from
  disk. Skipping it might led to using an arbitrary memory as fields' data and
  finally to a crash.
  
  Now the read_buffpek_from_file function is always called. It allocates
  new buffer only when necessary, but always fill it with correct data.

  mysql-test/r/subselect.result@stripped, 2008-01-09 00:39:19+03:00, evgen@stripped +9
-0
    Added a test case for the bug#33675: Usage of an uninitialized memory by
    filesort in a subquery caused server crash.

  mysql-test/t/subselect.test@stripped, 2008-01-09 00:39:53+03:00, evgen@stripped +22
-0
    Added a test case for the bug#33675: Usage of an uninitialized memory by
    filesort in a subquery caused server crash.

  sql/filesort.cc@stripped, 2008-01-09 00:39:02+03:00, evgen@stripped +13 -9
    Bug#33675: Usage of an uninitialized memory by filesort in a subquery caused
    server crash.Now the read_buffpek_from_file function is always called. It allocates
    new buffer only when necessary, but always fill it with correct data.

diff -Nrup a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
--- a/mysql-test/r/subselect.result	2007-12-02 03:48:40 +03:00
+++ b/mysql-test/r/subselect.result	2008-01-09 00:39:19 +03:00
@@ -4383,4 +4383,13 @@ SELECT 2 FROM t1 WHERE EXISTS ((SELECT 1
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to
your MySQL server version for the right syntax to use near 'UNION 
 (SELECT 1 FROM t2 WHERE t1.a = t2.a))' at line 2
 DROP TABLE t1,t2;
+create table t1(f11 int, f12 int);
+create table t2(f21 int unsigned not null, f22 int, f23 varchar(10));
+insert into t1 values(1,1),(2,2), (3, 3);
+set session sort_buffer_size= 33*1024;
+select count(*) from t1 where f12 = 
+(select f22 from t2 where f22 = f12 order by f21 desc, f22, f23 limit 1);
+count(*)
+3
+drop table t1,t2;
 End of 5.0 tests.
diff -Nrup a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
--- a/mysql-test/t/subselect.test	2007-12-13 13:49:11 +03:00
+++ b/mysql-test/t/subselect.test	2008-01-09 00:39:53 +03:00
@@ -3230,4 +3230,26 @@ SELECT 2 FROM t1 WHERE EXISTS ((SELECT 1
 DROP TABLE t1,t2;
 
 
+#
+# Bug#33675: Usage of an uninitialized memory by filesort in a subquery
+#            caused server crash.
+#
+create table t1(f11 int, f12 int);
+create table t2(f21 int unsigned not null, f22 int, f23 varchar(10));
+insert into t1 values(1,1),(2,2), (3, 3);
+let $i=10000;
+--disable_query_log
+--disable_warnings
+while ($i)
+{
+  eval insert into t2 values (-1 , $i/5000 + 1, '$i');
+  dec $i;
+}
+--enable_warnings
+--enable_query_log
+set session sort_buffer_size= 33*1024;
+select count(*) from t1 where f12 = 
+(select f22 from t2 where f22 = f12 order by f21 desc, f22, f23 limit 1);
+
+drop table t1,t2;
 --echo End of 5.0 tests.
diff -Nrup a/sql/filesort.cc b/sql/filesort.cc
--- a/sql/filesort.cc	2007-10-22 16:10:03 +04:00
+++ b/sql/filesort.cc	2008-01-09 00:39:02 +03:00
@@ -37,7 +37,8 @@ if (my_b_write((file),(byte*) (from),par
 
 static char **make_char_array(char **old_pos, register uint fields,
                               uint length, myf my_flag);
-static BUFFPEK *read_buffpek_from_file(IO_CACHE *buffer_file, uint count);
+static byte *read_buffpek_from_file(IO_CACHE *buffer_file, uint count,
+                                    byte *buf);
 static ha_rows find_all_keys(SORTPARAM *param,SQL_SELECT *select,
 			     uchar * *sort_keys, IO_CACHE *buffer_file,
 			     IO_CACHE *tempfile,IO_CACHE *indexfile);
@@ -238,9 +239,10 @@ ha_rows filesort(THD *thd, TABLE *table,
   }
   else
   {
-    if (!table_sort.buffpek && table_sort.buffpek_len < maxbuffer &&
-        !(table_sort.buffpek=
-          (byte *) read_buffpek_from_file(&buffpek_pointers, maxbuffer)))
+    if (!(table_sort.buffpek=
+          read_buffpek_from_file(&buffpek_pointers, maxbuffer,
+                                 (table_sort.buffpek_len < maxbuffer ?
+                                  NULL : table_sort.buffpek))))
       goto err;
     buffpek= (BUFFPEK *) table_sort.buffpek;
     table_sort.buffpek_len= maxbuffer;
@@ -368,18 +370,20 @@ static char **make_char_array(char **old
 
 /* Read 'count' number of buffer pointers into memory */
 
-static BUFFPEK *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count)
+static byte *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count,
+                                    byte *buf)
 {
-  ulong length;
-  BUFFPEK *tmp;
+  ulong length= sizeof(BUFFPEK)*count;
+  byte *tmp= buf;
   DBUG_ENTER("read_buffpek_from_file");
   if (count > UINT_MAX/sizeof(BUFFPEK))
     return 0; /* sizeof(BUFFPEK)*count will overflow */
-  tmp=(BUFFPEK*) my_malloc(length=sizeof(BUFFPEK)*count, MYF(MY_WME));
+  if (!tmp)
+    tmp= (byte *)my_malloc(length, MYF(MY_WME));
   if (tmp)
   {
     if (reinit_io_cache(buffpek_pointers,READ_CACHE,0L,0,0) ||
-	my_b_read(buffpek_pointers, (byte*) tmp, length))
+	my_b_read(buffpek_pointers, tmp, length))
     {
       my_free((char*) tmp, MYF(0));
       tmp=0;