bk commit into 5.0 tree (dkatz:1.2485) BUG#27119

[damien]

Below is the list of changes that have just been committed into a local
5.0 repository of dkatz. When dkatz does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2007-05-16 16:14:13-04:00, dkatz@stripped +2 -0
  Bug #27119  	server crash with integer division by zero during filesort on huge result
  
  Added checks to detect integer overflow and fixed other bugs on the error path.

  myisam/sort.c@stripped, 2007-05-16 16:14:08-04:00, dkatz@stripped +2 -1
    Replaced a break statement with a goto statement so that a failure will instead break

  sql/filesort.cc@stripped, 2007-05-16 16:14:09-04:00, dkatz@stripped +15 -6
    Fixed an allocation routine to detect integer overflow, and as an optimization a check that prevents the number of buffpeks being larger than can possibly fit into memory.
        
    Fixed several unchecked error codes.
        
    Changed an index variable from int to uint to the match the type of the variable it's
    being compared with.
        
        Replaced a break statement with a goto statement so that a failure will instead break
    out of the higher level while-loop, instead of just the nested for-loop.

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	dkatz
# Host:	damien-katzs-computer.local
# Root:	/Users/dkatz/50_div_zero

--- 1.61/myisam/sort.c	2007-05-04 19:04:07 -04:00
+++ 1.62/myisam/sort.c	2007-05-16 16:14:08 -04:00
@@ -773,7 +773,7 @@ static int NEAR_F merge_many_buff(MI_SOR
     {
       if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                         buffpek+i,buffpek+i+MERGEBUFF-1))
-        break; /* purecov: inspected */
+        goto cleanup;
     }
     if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++,
                       buffpek+i,buffpek+ *maxbuffer))
@@ -783,6 +783,7 @@ static int NEAR_F merge_many_buff(MI_SOR
     temp=from_file; from_file=to_file; to_file=temp;
     *maxbuffer= (int) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);                   /* This holds old result */
   if (to_file == t_file)
     *t_file=t_file2;                            /* Copy result file */

--- 1.117/sql/filesort.cc	2007-03-07 07:51:40 -05:00
+++ 1.118/sql/filesort.cc	2007-05-16 16:14:09 -04:00
@@ -249,7 +249,8 @@ ha_rows filesort(THD *thd, TABLE *table,
 	open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER,
 			  MYF(MY_WME)))
       goto err;
-    reinit_io_cache(outfile,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(outfile,WRITE_CACHE,0L,0,0))
+      goto err;
 
     /*
       Use also the space previously used by string pointers in sort_buffer
@@ -369,6 +370,8 @@ static BUFFPEK *read_buffpek_from_file(I
   ulong length;
   BUFFPEK *tmp;
   DBUG_ENTER("read_buffpek_from_file");
+  if (count > ULONG_MAX/sizeof(BUFFPEK))
+    return 0; /* sizeof(BUFFPEK)*count will overflow */
   tmp=(BUFFPEK*) my_malloc(length=sizeof(BUFFPEK)*count, MYF(MY_WME));
   if (tmp)
   {
@@ -600,6 +603,9 @@ write_keys(SORTPARAM *param, register uc
       open_cached_file(tempfile, mysql_tmpdir, TEMP_PREFIX, DISK_BUFFER_SIZE,
                        MYF(MY_WME)))
     goto err;                                   /* purecov: inspected */
+  /* check we won't have more buffpeks than we can possibly keep in memory */
+  if (my_b_tell(buffpek_pointers) + sizeof(BUFFPEK) > (ulonglong)ULONG_MAX)
+    goto err;
   buffpek.file_pos= my_b_tell(tempfile);
   if ((ha_rows) count > param->max_rows)
     count=(uint) param->max_rows;               /* purecov: inspected */
@@ -907,7 +913,7 @@ static bool save_index(SORTPARAM *param,
 int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
 		    BUFFPEK *buffpek, uint *maxbuffer, IO_CACHE *t_file)
 {
-  register int i;
+  register uint i;
   IO_CACHE t_file2,*from_file,*to_file,*temp;
   BUFFPEK *lastbuff;
   DBUG_ENTER("merge_many_buff");
@@ -922,14 +928,16 @@ int merge_many_buff(SORTPARAM *param, uc
   from_file= t_file ; to_file= &t_file2;
   while (*maxbuffer >= MERGEBUFF2)
   {
-    reinit_io_cache(from_file,READ_CACHE,0L,0,0);
-    reinit_io_cache(to_file,WRITE_CACHE,0L,0,0);
+    if (reinit_io_cache(from_file,READ_CACHE,0L,0,0))
+      goto cleanup;
+    if (reinit_io_cache(to_file,WRITE_CACHE,0L,0,0))
+      goto cleanup;
     lastbuff=buffpek;
-    for (i=0 ; i <= (int) *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
+    for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF)
     {
       if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 			buffpek+i,buffpek+i+MERGEBUFF-1,0))
-	break;					/* purecov: inspected */
+      goto cleanup;
     }
     if (merge_buffers(param,from_file,to_file,sort_buffer,lastbuff++,
 		      buffpek+i,buffpek+ *maxbuffer,0))
@@ -941,6 +949,7 @@ int merge_many_buff(SORTPARAM *param, uc
     setup_io_cache(to_file);
     *maxbuffer= (uint) (lastbuff-buffpek)-1;
   }
+cleanup:
   close_cached_file(to_file);			// This holds old result
   if (to_file == t_file)
   {