From: Tor Didriksen
Date: January 10 2011 9:57am
Subject: bzr push into mysql-trunk branch (tor.didriksen:3471 to 3472) Bug#59331
List-Archive: http://lists.mysql.com/commits/128260
X-Bug: 59331
Message-Id: <20110110095717.3E0D23127@atum07.norway.sun.com>
MIME-Version: 1.0
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit

 3472 Tor Didriksen	2011-01-06
      Bug#59331 filesort with priority queue: handling of out-of-memory
      
      Verifies that we have an error if init_queue() fails.
     @ mysql-test/r/filesort_debug.result
        New test case.
     @ mysql-test/t/filesort_debug.test
        New test case.
     @ sql/bounded_queue.h
        Add DBUG code to simulate out-of-memory.
     @ sql/filesort.cc
        Terminate filesort if pq.init() fails.
        Also: don't allocate more memory than we actually need for the 'strip-off-addon-fields' case.

    added:
      mysql-test/r/filesort_debug.result
      mysql-test/t/filesort_debug.test
    modified:
      sql/bounded_queue.h
      sql/filesort.cc
 3471 Jorgen Loland	2011-01-10
      BUG#58837: ICP crash or valgrind error due to uninitialized 
                 value in innobase_index_cond
            
      The problem was that ICP pushed a condition to an InnoDB 
      table accessed through the primary key, and handler::end_range 
      was not initialized in this case. This problem emerged now 
      because ICP on InnoDB primary index has recently been enabled.
     @ mysql-test/include/icp_tests.inc
        Added test case for BUG#58837
     @ mysql-test/r/innodb_icp.result
        Added test case for BUG#58837
     @ mysql-test/r/innodb_icp_none.result
        Added test case for BUG#58837
     @ mysql-test/r/myisam_icp.result
        Added test case for BUG#58837
     @ mysql-test/r/myisam_icp_none.result
        Added test case for BUG#58837
     @ sql/handler.h
        Initialize handler::end_range in ha_rnd_init() and ha_rnd_end()

    modified:
      mysql-test/include/icp_tests.inc
      mysql-test/r/innodb_icp.result
      mysql-test/r/innodb_icp_none.result
      mysql-test/r/myisam_icp.result
      mysql-test/r/myisam_icp_none.result
      sql/handler.h
=== added file 'mysql-test/r/filesort_debug.result'
--- a/mysql-test/r/filesort_debug.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/filesort_debug.result	2011-01-06 14:38:38 +0000
@@ -0,0 +1,11 @@
+#
+# Bug#59331 filesort with priority queue: handling of out-of-memory
+#
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10");
+set debug= '+d,bounded_queue_init_fail';
+SELECT * FROM t1 ORDER BY f1 ASC, f0 LIMIT 1;
+ERROR HY000: Out of memory (Needed 24 bytes)
+DROP TABLE t1;

=== added file 'mysql-test/t/filesort_debug.test'
--- a/mysql-test/t/filesort_debug.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/filesort_debug.test	2011-01-06 14:38:38 +0000
@@ -0,0 +1,16 @@
+--source include/have_debug.inc
+
+--echo #
+--echo # Bug#59331 filesort with priority queue: handling of out-of-memory
+--echo #
+
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10");
+
+set debug= '+d,bounded_queue_init_fail';
+--error 5
+SELECT * FROM t1 ORDER BY f1 ASC, f0 LIMIT 1;
+
+DROP TABLE t1;

=== modified file 'sql/bounded_queue.h'
--- a/sql/bounded_queue.h	2010-12-17 09:41:21 +0000
+++ b/sql/bounded_queue.h	2011-01-06 14:38:38 +0000
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 
+/* Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved. 
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -166,6 +166,10 @@ int Bounded_queue<Element_type, Key_type
   if (compare == NULL)
     compare=
       reinterpret_cast<compare_function>(get_ptr_compare(compare_length));
+
+  DBUG_EXECUTE_IF("bounded_queue_init_fail",
+                  DBUG_SET("+d,simulate_out_of_memory"););
+
   // We allocate space for one extra element, for replace when queue is full.
   return init_queue(&m_queue, (uint) max_elements + 1,
                     0, max_at_top,

=== modified file 'sql/filesort.cc'
--- a/sql/filesort.cc	2011-01-04 15:23:59 +0000
+++ b/sql/filesort.cc	2011-01-06 14:38:38 +0000
@@ -226,10 +226,15 @@ ha_rows filesort(THD *thd, TABLE *table,
                 compare_length,
                 &make_sortkey, &param, table_sort.sort_keys))
     {
-      // If failed to init pq, fall back to merge-sort.
-      DBUG_PRINT("info", ("failed to allocate PQ, fallback to sort-merge"));
+      /*
+       If we fail to init pq, we have to give up:
+       out of memory means my_malloc() will call my_error().
+      */
+      DBUG_PRINT("info", ("failed to allocate PQ"));
       my_free(table_sort.sort_keys);
       table_sort.sort_keys= NULL;
+      DBUG_ASSERT(thd->is_error());
+      goto err;
     }
   }
   else
@@ -1222,7 +1227,8 @@ bool check_if_pq_applicable(Sort_param *
         DBUG_RETURN(false);
 
       make_char_array(filesort_info,
-                      param->max_keys_per_buffer, param->rec_length);
+                      param->max_keys_per_buffer,
+                      param->sort_length + param->ref_length);
       if (filesort_info->sort_keys)
       {
         // Make attached data to be references instead of fields.

No bundle (reason: useless for push emails).