bzr commit into mysql-5.0-bugteam branch (gshchepa:2728) Bug#42037

[Gleb Shchepa]

#At file:///work/bzr/5.0-42037/ based on revid:azundris@stripped

 2728 Gleb Shchepa	2009-01-13
      Bug #42037: Queries containing a subquery with DISTINCT and
                  ORDER BY could cause a server crash
      
      Dependent subqueries like
      
        SELECT COUNT(*) FROM t1, t2 WHERE t2.b
         IN (SELECT DISTINCT t2.b FROM t2 WHERE t2.b = t1.a)
      
      hogged memory or even caused server crash on tables with
      significant number of rows (for example, this query failed 
      on 1000 of records in every table and 50M of free
      virtual memory).
      
      Subselect engine resets JOIN structure of dependent inner
      query on every row of outer table. In this case restore_tmp()
      reset tmp_join->join_tab_reexec and tmp_join->table_reexec
      methods to NULL pointers and the next call to make_simple_join()
      reallocated them again and again.
      
      The JOIN::restore_tmp method has been modified to reuse
      previously allocated memory of join_tab_reexec and table_reexec.
      
      NOTE: this patch doesn't include standard test case (this is
      "out of memory" bug). See bug #42037 page for test cases.
modified:
  sql/sql_select.cc
  sql/sql_select.h

per-file messages:
  sql/sql_select.cc
    Bug #42037: Queries containing a subquery with DISTINCT and
                ORDER BY could cause a server crash
                
    1. JOIN::init_tmp() has been added to use instead of restore_tmp()
       for the first call.
    2. JOIN::restore_tmp() has been extended to save and restore
       join_tab_reexec table_reexec fields.
  sql/sql_select.h
     #42037: Queries containing a subquery with DISTINCT and
                ORDER BY could cause a server crash
    
    The JOIN::init_tmp method has been added.
=== modified file 'sql/sql_select.cc'
--- a/sql/sql_select.cc	2008-12-24 15:24:11 +0000
+++ b/sql/sql_select.cc	2009-01-13 18:20:37 +0000
@@ -1480,11 +1480,31 @@ JOIN::optimize()
 
 
 /*
+  Init values in temporary join
+*/
+void JOIN::init_tmp()
+{
+  memcpy(tmp_join, this, (size_t) sizeof(JOIN));
+}
+
+
+/*
   Restore values in temporary join
 */
 void JOIN::restore_tmp()
 {
-  memcpy(tmp_join, this, (size_t) sizeof(JOIN));
+  /*
+    Save join_tab_reexec and table_reexec values to prevent their
+    unnecessary multiple allocation (memory hog) in make_simple_join()
+    after tmp_join->join_free() in a subquery like
+    SELECT COUNT(*) FROM t1, t2 WHERE t2.b
+      IN (SELECT DISTINCT t2.b FROM t2 WHERE t2.b = t1.a)
+  */
+  JOIN_TAB *save_join_tab_reexec= tmp_join->join_tab_reexec;
+  TABLE **save_table_reexec= tmp_join->table_reexec;
+  init_tmp();
+  tmp_join->join_tab_reexec= save_join_tab_reexec;
+  tmp_join->table_reexec= save_table_reexec;
 }
 
 
@@ -1550,7 +1570,7 @@ JOIN::init_save_join_tab()
   if (!(tmp_join= (JOIN*)thd->alloc(sizeof(JOIN))))
     return 1;                                  /* purecov: inspected */
   error= 0;				       // Ensure that tmp_join.error= 0
-  restore_tmp();
+  init_tmp();
   return 0;
 }
 

=== modified file 'sql/sql_select.h'
--- a/sql/sql_select.h	2008-12-24 15:24:11 +0000
+++ b/sql/sql_select.h	2009-01-13 18:20:37 +0000
@@ -430,7 +430,10 @@ public:
   int reinit();
   void exec();
   int destroy();
+private:
+  void init_tmp();
   void restore_tmp();
+public:
   bool alloc_func_list();
   bool make_sum_func_list(List<Item> &all_fields, List<Item> &send_fields,
 			  bool before_group_by, bool recompute= FALSE);