bk commit into 4.1 tree (ingo:1.2186) BUG#9487

[ingo]

Below is the list of changes that have just been committed into a local
4.1 repository of mydev. When mydev does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2186 05/04/15 19:26:38 ingo@stripped +1 -0
  Bug#9487 - myisampack segmentation violation and bus error
  Fixed some casts for 64-bit systems.
  Fixed a possible buffer overflow.

  myisam/myisampack.c
    1.38 05/04/15 19:25:32 ingo@stripped +16 -3
    Bug#9487 - myisampack segmentation violation and bus error
    Added or changed proper casts for the argument of flush_buffer().
    Added a check for the counterstrike of the trick to have
    file_buffer.end some bytes below the real end of the buffer.

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	ingo
# Host:	chilla.local
# Root:	/home/mydev/mysql-4.1-bug9487

--- 1.37/myisam/myisampack.c	Tue Nov  2 07:58:42 2004
+++ 1.38/myisam/myisampack.c	Fri Apr 15 19:25:32 2005
@@ -1709,7 +1709,7 @@
     ulong tot_blob_length=0;
     if (! error)
     {
-      if (flush_buffer(max_calc_length+max_pack_length))
+      if (flush_buffer((ulong) max_calc_length + (ulong) max_pack_length))
 	break;
       record_pos=file_buffer.pos;
       file_buffer.pos+=max_pack_length;
@@ -1930,7 +1930,20 @@
 static int flush_buffer(ulong neaded_length)
 {
   ulong length;
-  if ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length)
+
+  /*
+    file_buffer.end is 8 bytes lower than the real end of the buffer.
+    This is done so that the end-of-buffer condition does not need to be
+    checked for every byte (see write_bits()). Consequently,
+    file_buffer.pos can become greater than file_buffer.end. The
+    algorithms in the other functions ensure that there will never be
+    more than 8 bytes written to the buffer without an end-of-buffer
+    check. So the buffer cannot be overrun. But we need to check for the
+    near-to-buffer-end condition to avoid a negative result, which is
+    casted to unsigned and thus becomes giant.
+  */
+  if ((file_buffer.pos < file_buffer.end) &&
+      ((ulong) (file_buffer.end - file_buffer.pos) > neaded_length))
     return 0;
   length=(ulong) (file_buffer.pos-file_buffer.buffer);
   file_buffer.pos=file_buffer.buffer;
@@ -2002,7 +2015,7 @@
     }
 #endif
     if (file_buffer.pos >= file_buffer.end)
-      VOID(flush_buffer((uint) ~0));
+      VOID(flush_buffer(~ (ulong) 0));
     file_buffer.bits=(int) (BITS_SAVED - bits);
     file_buffer.current_byte=(uint) (value << (BITS_SAVED - bits));
   }