bzr commit into mysql-5.1 branch (kgeorge:2686) Bug#37627

[Georgi Kodinov]

#At file:///home/kgeorge/mysql/bzr/B37627-5.1-bugteam/

 2686 Georgi Kodinov	2008-07-04
      Bug#37627: Killing query with sum(exists()) or avg(exists()) reproducibly crashes server
      
      When there is an error executing EXISTS predicates they return NULL as their string
      or decimal value but don't set the NULL value flag.
      Fixed by returning 0 (as a decimal or a string) on error exectuting the subquery.
      Note that we can't return NULL as EXISTS is not supposed to return NULL.
modified:
  mysql-test/r/subselect.result
  mysql-test/t/subselect.test
  sql/item_subselect.cc

per-file messages:
  mysql-test/r/subselect.result
    Bug#37627: test case
  mysql-test/t/subselect.test
    Bug#37627: test case
  sql/item_subselect.cc
    Bug#37627: return decimal (or string) 0 isntead of a NULL pointer on
    error calculating an EXISTS predicate.
=== modified file 'mysql-test/r/subselect.result'
--- a/mysql-test/r/subselect.result	2008-06-19 03:08:41 +0000
+++ b/mysql-test/r/subselect.result	2008-07-04 14:02:17 +0000
@@ -4398,3 +4398,16 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 a	b
 DROP TABLE t1,t2;
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SUM(EXISTS(SELECT RAND() FROM t1))
+0
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+REVERSE(EXISTS(SELECT RAND() FROM t1))
+0
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;

=== modified file 'mysql-test/t/subselect.test'
--- a/mysql-test/t/subselect.test	2008-06-25 17:01:17 +0000
+++ b/mysql-test/t/subselect.test	2008-07-04 14:02:17 +0000
@@ -3298,3 +3298,19 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 
 DROP TABLE t1,t2;
+
+#
+# Bug #37627: Killing query with sum(exists()) or avg(exists()) reproducibly
+# crashes server
+#
+
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+# Setup the mysqld to crash at certain point
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;

=== modified file 'sql/item_subselect.cc'
--- a/sql/item_subselect.cc	2008-03-28 15:09:14 +0000
+++ b/sql/item_subselect.cc	2008-07-04 14:02:17 +0000
@@ -254,6 +254,11 @@ bool Item_subselect::exec()
   if (thd->is_error())
   /* Do not execute subselect in case of a fatal error */
     return 1;
+  /*
+    Simulate a failure in sub-query execution. Used to test e.g.
+    out of memory or query being killed conditions.
+  */
+  DBUG_EXECUTE_IF("subselect_exec_fail", return 1;);
 
   res= engine->exec();
 
@@ -719,27 +724,48 @@ longlong Item_exists_subselect::val_int(
   return value;
 }
 
+
+/**
+  Return the result of EXISTS as a string value
+
+  Converts the true/false result into a string value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    buffer to hold the resulting string value
+  @retval                      Pointer to the converted string.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 String *Item_exists_subselect::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   str->set((ulonglong)value,&my_charset_bin);
   return str;
 }
 
 
+/**
+  Return the result of EXISTS as a decimal value
+
+  Converts the true/false result into a decimal value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    Buffer to hold the resulting decimal value
+  @retval                      Pointer to the converted decimal.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 my_decimal *Item_exists_subselect::val_decimal(my_decimal *decimal_value)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   int2my_decimal(E_DEC_FATAL_ERROR, value, 0, decimal_value);
   return decimal_value;
 }