bk commit into 4.0 tree (ingo:1.2093) BUG#8321

[ingo]

Below is the list of changes that have just been committed into a local
4.0 repository of mydev. When mydev does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2093 05/04/28 13:21:48 ingo@stripped +1 -0
  Bug#8321 - myisampack bug in compression algorithm
  This is the second of three changesets. It contains the pure bug fix.
  It also contains the second after-review fixes.
  The problem was that with gcc on x86, shifts are done modulo word size. 
  'value' is 32 bits wide and shifting it by 32 bits is a no-op.
  This was triggered by an evil distribution of character incidences. 
  A distribution of 2917027827 characters made of 202 distinct values led to
  34 occurrences of 32-bit Huffman codes.
  This might have been the first time ever that write_bits() had to write
  32-bit values. Since it can be expected that one day even 32 bits might
  be insufficient, the third changeset suggests to enlarge some variables
  to 64 bits.

  myisam/myisampack.c
    1.35 05/04/28 13:21:23 ingo@stripped +6 -2
    Bug#8321 - myisampack bug in compression algorithm
    This is the second of three changesets. It contains the pure bug fix.
    It also contains the second after-review fixes.
    The problem was that with gcc on x86, shifts are done modulo word size. 
    'value' is 32 bits wide and shifting it by 32 bits is a no-op.
    This was triggered by an evil distribution of character incidences. 
    A distribution of 2917027827 characters made of 202 distinct values led to
    34 occurrences of 32-bit Huffman codes.
    This might have been the first time ever that write_bits() had to write
    32-bit values. Since it can be expected that one day even 32 bits might
    be insufficient, the third changeset suggests to enlarge some variables
    to 64 bits.

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	ingo
# Host:	chilla.local
# Root:	/home/mydev/mysql-4.0-4000

--- 1.34/myisam/myisampack.c	Wed Oct 27 21:51:24 2004
+++ 1.35/myisam/myisampack.c	Thu Apr 28 13:21:23 2005
@@ -1975,7 +1975,9 @@
   {
     reg3 uint byte_buff;
     bits= (uint) -file_buffer.bits;
-    byte_buff=file_buffer.byte | (uint) (value >> bits);
+    DBUG_ASSERT(bits <= 8 * sizeof(value));
+    byte_buff= (file_buffer.byte |
+               ((bits != 8 * sizeof(value)) ? (uint) (value >> bits) : 0));
 #if BITS_SAVED == 32
     *file_buffer.pos++= (byte) (byte_buff >> 24) ;
     *file_buffer.pos++= (byte) (byte_buff >> 16) ;
@@ -1983,7 +1985,9 @@
     *file_buffer.pos++= (byte) (byte_buff >> 8) ;
     *file_buffer.pos++= (byte) byte_buff;
 
-    value&=(1 << bits)-1;
+    DBUG_ASSERT(bits <= 8 * sizeof(ulong));
+    if (bits != 8 * sizeof(value))
+      value&= (((ulong) 1) << bits) - 1;
 #if BITS_SAVED == 16
     if (bits >= sizeof(uint))
     {