bk commit into 5.0 tree (dkatz:1.2436) BUG#27119

[damien]

Below is the list of changes that have just been committed into a local
5.0 repository of dkatz. When dkatz does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2007-05-01 15:03:47-04:00, dkatz@stripped +2 -0
  Bug #27119  	server crash with integer division by zero during filesort on huge result
  

  myisam/sort.c@stripped, 2007-05-01 15:03:45-04:00, dkatz@stripped +2 -1
    Replaced a break statement with a goto statement so that a failure will instead break
out of the higher level while-loop, instead of just the nested for-loop.

  sql/filesort.cc@stripped, 2007-05-01 15:03:45-04:00, dkatz@stripped +12
-6
    Fixed an allocation routine to detect integer overflow.
    
    Fixed several unchecked error codes.
    
    Changed an index variable from int to uint to the match the type of the variable it's
being compared with.
    
    Replaced a break statement with a goto statement so that a failure will instead break
out of the higher level while-loop, instead of just the nested for-loop.

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	dkatz
# Host:	damien-katzs-computer.local
# Root:	/Users/dkatz/50a

--- 1.60/myisam/sort.c	2007-01-22 07:10:36 -05:00
+++ 1.61/myisam/sort.c	2007-05-01 15:03:45 -04:00
@@ -773,7 +773,7 @@ static int NEAR_F merge_many_buff(MI_SOR
     {
       if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                         buffpek+i,buffpek+i+MERGEBUFF-1))
-        break; /* purecov: inspected */
+        goto cleanup;
     }
     if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                       buffpek+i,buffpek+ *maxbuffer))
@@ -783,6 +783,7 @@ static int NEAR_F merge_many_buff(MI_SOR
     temp=from_file; from_file=to_file; to_file=temp;
     *maxbuffer= (int) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);                   /* This holds old result */
   if (to_file == t_file)
     *t_file=t_file2;                            /* Copy result file */

--- 1.117/sql/filesort.cc	2007-03-07 07:51:40 -05:00
+++ 1.118/sql/filesort.cc	2007-05-01 15:03:45 -04:00
@@ -249,7 +249,8 @@ ha_rows filesort(THD *thd, TABLE *table,
 	open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER,
 			  MYF(MY_WME)))
       goto err;
-    reinit_io_cache(outfile,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(outfile,WRITE_CACHE,0L,0,0))
+      goto err;
 
     /*
       Use also the space previously used by string pointers in sort_buffer
@@ -369,6 +370,8 @@ static BUFFPEK *read_buffpek_from_file(I
   ulong length;
   BUFFPEK *tmp;
   DBUG_ENTER("read_buffpek_from_file");
+  if (count > ULONG_MAX/sizeof(BUFFPEK))
+    return 0; /* sizeof(BUFFPEK)*count will overflow */
   tmp=(BUFFPEK*) my_malloc(length=sizeof(BUFFPEK)*count, MYF(MY_WME));
   if (tmp)
   {
@@ -907,7 +910,7 @@ static bool save_index(SORTPARAM *param,
 int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
 		    BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file)
 {
-  register int i;
+  register uint i;
   IO_CACHE t_file2,*from_file,*to_file,*temp;
   BUFFPEK *lastbuff;
   DBUG_ENTER("merge_many_buff");
@@ -922,14 +925,16 @@ int merge_many_buff(SORTPARAM *param, uc
   from_file= t_file ; to_file= &t_file2;
   while (*maxbuffer >= MERGEBUFF2)
   {
-    reinit_io_cache(from_file,READ_CACHE,0L,0,0);
-    reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(from_file,READ_CACHE,0L,0,0))
+      goto cleanup;
+    if (reinit_io_cache(to_file,WRITE_CACHE,0L,0,0))
+      goto cleanup;
     lastbuff=buffpek;
-    for (i=0 ; i <= (int) *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
+    for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
     {
       if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 			buffpek+i,buffpek+i+MERGEBUFF-1,0))
-	break;					/* purecov: inspected */
+      goto cleanup;
     }
     if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 		      buffpek+i,buffpek+ *maxbuffer,0))
@@ -941,6 +946,7 @@ int merge_many_buff(SORTPARAM *param, uc
     setup_io_cache(to_file);
     *maxbuffer= (uint) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);			// This holds old result
   if (to_file == t_file)
   {