bzr commit into mysql-5.0 branch (marc.alff:2656) Bug#38296

[Marc Alff]

#At file:///home/malff/BZR-TREE/mysql-5.0-38296/

 2656 Marc Alff	2008-08-08
      Bug#38296 (low memory crash with many conditions in a query)
      
      This fix is for 5.0/5.1 only
      
      Before this fix, a query using many an arbitrary number of conditions in a
      query could consume arbitrary amount of memory in the server, and cause
      a crash in the parser on Out Of Memory condition, due to an optimization
      implemented for OR / XOR / AND operators.
      
      With this fix, the parser now gracefully returns an ER_OUT_OF_RESOURCES
      error to the client when OOM occurs when parsing OR / XOR / AND.
      
      No test scripts are provided, since automating OOM failures is not
      instrumented in the server.
      Tested under the debugger, to verify that an error in alloc_root cause the
      thread to returns gracefully all the way to the client application, with
      an ER_OUT_OF_RESOURCES error.
modified:
  mysys/my_alloc.c
  sql/field.h
  sql/item.h
  sql/sp_head.cc
  sql/sql_lex.h
  sql/sql_list.h
  sql/sql_string.h
  sql/sql_yacc.yy
  sql/thr_malloc.cc

=== modified file 'mysys/my_alloc.c'
--- a/mysys/my_alloc.c	2007-04-18 20:50:32 +0000
+++ b/mysys/my_alloc.c	2008-08-08 16:59:58 +0000
@@ -202,7 +202,7 @@ gptr alloc_root(MEM_ROOT *mem_root,unsig
     {
       if (mem_root->error_handler)
 	(*mem_root->error_handler)();
-      return((gptr) 0);				/* purecov: inspected */
+      DBUG_RETURN((gptr) 0);			/* purecov: inspected */
     }
     mem_root->block_num++;
     next->next= *prev;

=== modified file 'sql/field.h'
--- a/sql/field.h	2008-05-06 16:43:46 +0000
+++ b/sql/field.h	2008-08-08 16:59:58 +0000
@@ -48,7 +48,8 @@ class Field
   Field(const Item &);				/* Prevent use of these */
   void operator=(Field &);
 public:
-  static void *operator new(size_t size) {return (void*) sql_alloc((uint) size); }
+  static void *operator new(size_t size) throw ()
+  { return (void*) sql_alloc((uint) size); }
   static void operator delete(void *ptr_arg, size_t size) { TRASH(ptr_arg, size); }
 
   char		*ptr;			// Position to field in record

=== modified file 'sql/item.h'
--- a/sql/item.h	2008-03-12 07:59:15 +0000
+++ b/sql/item.h	2008-08-08 16:59:58 +0000
@@ -439,9 +439,9 @@ class Item {
   Item(const Item &);			/* Prevent use of these */
   void operator=(Item &);
 public:
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   { return (void*) sql_alloc((uint) size); }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr,size_t size) { TRASH(ptr, size); }
   static void operator delete(void *ptr, MEM_ROOT *mem_root) {}

=== modified file 'sql/sp_head.cc'
--- a/sql/sp_head.cc	2008-07-14 21:41:30 +0000
+++ b/sql/sp_head.cc	2008-08-08 16:59:58 +0000
@@ -446,7 +446,7 @@ sp_head::operator new(size_t size) throw
   init_sql_alloc(&own_root, MEM_ROOT_BLOCK_SIZE, MEM_ROOT_PREALLOC);
   sp= (sp_head *) alloc_root(&own_root, size);
   if (sp == NULL)
-    return NULL;
+    DBUG_RETURN(NULL);
   sp->main_mem_root= own_root;
   DBUG_PRINT("info", ("mem_root 0x%lx", (ulong) &sp->mem_root));
   DBUG_RETURN(sp);

=== modified file 'sql/sql_lex.h'
--- a/sql/sql_lex.h	2008-07-14 21:41:30 +0000
+++ b/sql/sql_lex.h	2008-08-08 16:59:58 +0000
@@ -331,11 +331,11 @@ public:
   bool no_table_names_allowed; /* used for global order by */
   bool no_error; /* suppress error message (convert it to warnings) */
 
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   {
     return (void*) sql_alloc((uint) size);
   }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr,size_t size) { TRASH(ptr, size); }
   static void operator delete(void *ptr, MEM_ROOT *mem_root) {}

=== modified file 'sql/sql_list.h'
--- a/sql/sql_list.h	2007-03-29 06:35:28 +0000
+++ b/sql/sql_list.h	2008-08-08 16:59:58 +0000
@@ -27,7 +27,7 @@ public:
   {
     return (void*) sql_alloc((uint) size);
   }
-  static void *operator new[](size_t size)
+  static void *operator new[](size_t size) throw ()
   {
     return (void*) sql_alloc((uint) size);
   }
@@ -466,7 +466,7 @@ public:
 struct ilink
 {
   struct ilink **prev,*next;
-  static void *operator new(size_t size)
+  static void *operator new(size_t size) throw ()
   {
     return (void*)my_malloc((uint)size, MYF(MY_WME | MY_FAE));
   }

=== modified file 'sql/sql_string.h'
--- a/sql/sql_string.h	2007-01-22 12:10:46 +0000
+++ b/sql/sql_string.h	2008-08-08 16:59:58 +0000
@@ -78,7 +78,7 @@ public:
     Alloced_length=str.Alloced_length; alloced=0; 
     str_charset=str.str_charset;
   }
-  static void *operator new(size_t size, MEM_ROOT *mem_root)
+  static void *operator new(size_t size, MEM_ROOT *mem_root) throw ()
   { return (void*) alloc_root(mem_root, (uint) size); }
   static void operator delete(void *ptr_arg,size_t size)
   { TRASH(ptr_arg, size); }

=== modified file 'sql/sql_yacc.yy'
--- a/sql/sql_yacc.yy	2008-07-14 21:41:30 +0000
+++ b/sql/sql_yacc.yy	2008-08-08 16:59:58 +0000
@@ -4577,12 +4577,16 @@ expr:
             {
               /* X OR Y */
               $$ = new (YYTHD->mem_root) Item_cond_or($1, $3);
+              if ($$ == NULL)
+                MYSQL_YYABORT;
             }
           }
         | expr XOR expr %prec XOR
           {
             /* XOR is a proprietary extension */
             $$ = new (YYTHD->mem_root) Item_cond_xor($1, $3);
+            if ($$ == NULL)
+              MYSQL_YYABORT;
           }
         | expr and expr %prec AND_SYM
           {
@@ -4623,6 +4627,8 @@ expr:
             {
               /* X AND Y */
               $$ = new (YYTHD->mem_root) Item_cond_and($1, $3);
+              if ($$ == NULL)
+                MYSQL_YYABORT;
             }
           }
 	| NOT_SYM expr %prec NOT_SYM

=== modified file 'sql/thr_malloc.cc'
--- a/sql/thr_malloc.cc	2006-12-31 00:02:27 +0000
+++ b/sql/thr_malloc.cc	2008-08-08 16:59:58 +0000
@@ -21,10 +21,35 @@
 extern "C" {
   void sql_alloc_error_handler(void)
   {
-    THD *thd=current_thd;
-    if (thd)					// QQ;  To be removed
-      thd->fatal_error();			/* purecov: inspected */
     sql_print_error(ER(ER_OUT_OF_RESOURCES));
+
+    THD *thd=current_thd;
+    if (thd)
+    {
+      /*
+        This thread is Out Of Memory.
+        An OOM condition is a fatal error.
+        It should not be caught by error handlers in stored procedures.
+        Also, recording that SQL condition in the condition area could
+        cause more memory allocations, which in turn could raise more
+        OOM conditions, causing recursion in the error handling code itself.
+        As a result, my_error() should not be invoked, and the
+        thread diagnostics area is set to an error status directly.
+        The visible result for a client application will be:
+        - a query fails with an ER_OUT_OF_RESOURCES error,
+        returned in the error packet.
+        - SHOW ERROR/SHOW WARNINGS may be empty.
+      */
+
+      NET *net= &thd->net;
+      thd->fatal_error();
+      if (!net->last_error[0])                  // Return only first message
+      {
+        strmake(net->last_error, ER(ER_OUT_OF_RESOURCES),
+                sizeof(net->last_error)-1);
+        net->last_errno= ER_OUT_OF_RESOURCES;
+      }
+    }
   }
 }