bk commit into 5.1 tree (davi:1.2570) BUG#35272

[Davi Arnaut]

Below is the list of changes that have just been committed into a local
5.1 repository of davi.  When davi does a push these changes
will be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-03-25 15:53:57-03:00, davi@stripped +3 -0
  Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash
  
  When trying to get the requested amount of memory for the keybuffer,
  the out of memory could be signaled if one of the tentative allocations
  fail. Later the server would crash (debug assert) when trying to send
  a ok packet with a error set.
  
  The solution is only to signal the error if all tentative allocations
  for the keybuffer fail.

  mysql-test/r/key_cache.result@stripped, 2008-03-25 15:53:54-03:00, davi@stripped +5 -0
    Add test case result for Bug#35272

  mysql-test/t/key_cache.test@stripped, 2008-03-25 15:53:55-03:00, davi@stripped +25 -0
    Add test case for Bug#35272

  mysys/mf_keycache.c@stripped, 2008-03-25 15:53:55-03:00, davi@stripped +3 -1
    Don't set error on my_large_malloc if allocation fails.
    Set the error if all tentative allocations failed.

diff -Nrup a/mysql-test/r/key_cache.result b/mysql-test/r/key_cache.result
--- a/mysql-test/r/key_cache.result	2007-11-30 11:52:49 -02:00
+++ b/mysql-test/r/key_cache.result	2008-03-25 15:53:54 -03:00
@@ -368,3 +368,8 @@ Variable_name	Value
 key_cache_block_size	1536
 SET GLOBAL key_cache_block_size= @bug28478_key_cache_block_size;
 DROP TABLE t1;
+SET @save_key_buffer = @@global.key_buffer_size;
+SET @@global.key_buffer_size = 4294967295;
+SET @@global.key_buffer_size = 9223372036854775807;
+SET @@global.key_buffer_size = @save_key_buffer;
+End of 5.1 tests
diff -Nrup a/mysql-test/t/key_cache.test b/mysql-test/t/key_cache.test
--- a/mysql-test/t/key_cache.test	2007-05-31 15:04:51 -03:00
+++ b/mysql-test/t/key_cache.test	2008-03-25 15:53:55 -03:00
@@ -247,3 +247,28 @@ SET GLOBAL key_cache_block_size= @bug284
 DROP TABLE t1;
 
 # End of 4.1 tests
+
+#
+# Bug#35272: @@global.key_buffer_size = 4294967295 let the server crash
+#
+
+SET @save_key_buffer = @@global.key_buffer_size;
+
+# Wee try to force Out Of Memory here. key_buffer_size is ULL, so
+# on a 32 bit machine, 4GB is the most we can ask for before the
+# server complains about value/variable mismatch. At the off chance
+# of one of our 64-bit machines actually offering us 4GB, we also
+# accept "no error" (in addition to the expected "out of memory").
+--error 0,ER_OUTOFMEMORY
+SET @@global.key_buffer_size = 4294967295;
+
+# on 32-bit, we get "out of range", on 64-bit, "out of memory".
+--error 0,ER_WRONG_ARGUMENTS,ER_OUTOFMEMORY
+--disable_warnings
+SET @@global.key_buffer_size = 9223372036854775807;
+--enable_warnings
+
+# restore normal value, just in case we got the 4GB or something.
+SET @@global.key_buffer_size = @save_key_buffer;
+
+--echo End of 5.1 tests
diff -Nrup a/mysys/mf_keycache.c b/mysys/mf_keycache.c
--- a/mysys/mf_keycache.c	2007-10-18 08:32:38 -02:00
+++ b/mysys/mf_keycache.c	2008-03-25 15:53:55 -03:00
@@ -102,6 +102,7 @@
 */
 
 #include "mysys_priv.h"
+#include "mysys_err.h"
 #include <keycache.h>
 #include "my_static.h"
 #include <m_string.h>
@@ -430,7 +431,7 @@ int init_key_cache(KEY_CACHE *keycache, 
       /* Allocate memory for cache page buffers */
       if ((keycache->block_mem=
 	   my_large_malloc((size_t) blocks * keycache->key_cache_block_size,
-			  MYF(MY_WME))))
+			  MYF(0))))
       {
         /*
 	  Allocate memory for blocks, hash_links and hash entries;
@@ -445,6 +446,7 @@ int init_key_cache(KEY_CACHE *keycache, 
       if (blocks < 8)
       {
         my_errno= ENOMEM;
+        my_error(EE_OUTOFMEMORY, MYF(0), blocks * keycache->key_cache_block_size);
         goto err;
       }
       blocks= blocks / 4*3;