bzr commit into mysql-5.1 branch (kgeorge:2686) Bug#37627

[Georgi Kodinov]

#At file:///home/kgeorge/mysql/bzr/B37627-5.1-bugteam/

 2686 Georgi Kodinov	2008-07-04
      Bug#37627: Killing query with sum(exists()) or avg(exists()) reproducibly crashes
server
      
      On error executing an IN Subquery Item_in_exists::val_str and
Item_in_exists::val_decimal
      were returning a NULL pointer without setting the NULL flag.
      Since IN subquery predicates are not supposed to return NULL values we must return 
      0 in such cases (and count on the subquery code actually setting the error).
modified:
  mysql-test/r/subselect.result
  mysql-test/t/subselect.test
  sql/item_subselect.cc

per-file messages:
  mysql-test/r/subselect.result
    Bug#37627: test case
  mysql-test/t/subselect.test
    Bug#37627: test case
  sql/item_subselect.cc
    Bug #37627: fixed IN predicate to return 0 on error instead of a NULL pointer
=== modified file 'mysql-test/r/subselect.result'
--- a/mysql-test/r/subselect.result	2008-06-19 03:08:41 +0000
+++ b/mysql-test/r/subselect.result	2008-07-04 13:25:10 +0000
@@ -4398,3 +4398,16 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 a	b
 DROP TABLE t1,t2;
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SUM(EXISTS(SELECT RAND() FROM t1))
+0
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+REVERSE(EXISTS(SELECT RAND() FROM t1))
+0
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;

=== modified file 'mysql-test/t/subselect.test'
--- a/mysql-test/t/subselect.test	2008-06-25 17:01:17 +0000
+++ b/mysql-test/t/subselect.test	2008-07-04 13:25:10 +0000
@@ -3298,3 +3298,19 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 
 DROP TABLE t1,t2;
+
+#
+# Bug #37627: Killing query with sum(exists()) or avg(exists()) reproducibly
+# crashes server
+#
+
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+# Setup the mysqld to crash at certain point
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;

=== modified file 'sql/item_subselect.cc'
--- a/sql/item_subselect.cc	2008-03-28 15:09:14 +0000
+++ b/sql/item_subselect.cc	2008-07-04 13:25:10 +0000
@@ -254,6 +254,11 @@ bool Item_subselect::exec()
   if (thd->is_error())
   /* Do not execute subselect in case of a fatal error */
     return 1;
+  /*
+    Simulate a failure in sub-query execution. Used to test e.g.
+    out of memory or query being killed conditions.
+  */
+  DBUG_EXECUTE_IF("subselect_exec_fail", return 1;);
 
   res= engine->exec();
 
@@ -719,27 +724,48 @@ longlong Item_exists_subselect::val_int(
   return value;
 }
 
+
+/**
+  Return the result of EXISTS as a string value
+
+  Converts the true/false result into a string value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    buffer to hold the resulting string value
+  @retval                      Pointer to the converted string.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 String *Item_exists_subselect::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   str->set((ulonglong)value,&my_charset_bin);
   return str;
 }
 
 
+/**
+  Return the result of EXISTS as a decimal value
+
+  Converts the true/false result into a decimal value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    Buffer to hold the resulting decimal value
+  @retval                      Pointer to the converted decimal.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 my_decimal *Item_exists_subselect::val_decimal(my_decimal *decimal_value)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   int2my_decimal(E_DEC_FATAL_ERROR, value, 0, decimal_value);
   return decimal_value;
 }