bk commit into 6.0 tree (sergefp:1.2623) BUG#36137

[Sergey Petrunia]

Below is the list of changes that have just been committed into a local
6.0 repository of sergefp.  When sergefp does a push these changes
will be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-04-23 09:26:34+04:00, sergefp@stripped +3 -0
  BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
  - Call join->flatten_subqueries() before we call join->optimize()
    If we don't do this, subquery predicates which were slated for conversion
    into semi-join nests will remain half-converted and will fire an assert
    when we attempt to evaluate them. 

  mysql-test/r/subselect_sj2.result@stripped, 2008-04-23 09:26:25+04:00, sergefp@stripped +7 -0
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Testcase

  mysql-test/t/subselect_sj2.test@stripped, 2008-04-23 09:26:25+04:00, sergefp@stripped +10 -1
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Testcase

  sql/item_subselect.cc@stripped, 2008-04-23 09:26:25+04:00, sergefp@stripped +2 -1
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Call join->flatten_subqueries() before we call join->optimize()

diff -Nrup a/mysql-test/r/subselect_sj2.result b/mysql-test/r/subselect_sj2.result
--- a/mysql-test/r/subselect_sj2.result	2008-04-06 03:19:17 +04:00
+++ b/mysql-test/r/subselect_sj2.result	2008-04-23 09:26:25 +04:00
@@ -438,3 +438,10 @@ from t1 
 where fid IN (select fid from t2 where (id between 7502 and 8420) order by fid );
 ERROR 42S22: Unknown column 'fid' in 'IN/ALL/ANY subquery'
 drop table t1, t2;
+create table t1 (a int, b int, key (a), key (b));
+insert into t1 values (2,4),(2,4),(2,4);
+select t1.a from t1 
+where 
+t1.a in (select 1 from t1 where t1.a in (select 1 from t1) group by  t1.a);
+a
+drop table t1;
diff -Nrup a/mysql-test/t/subselect_sj2.test b/mysql-test/t/subselect_sj2.test
--- a/mysql-test/t/subselect_sj2.test	2008-04-06 03:19:17 +04:00
+++ b/mysql-test/t/subselect_sj2.test	2008-04-23 09:26:25 +04:00
@@ -590,5 +590,14 @@ from t1 
 where fid IN (select fid from t2 where (id between 7502 and 8420) order by fid );
 
 drop table t1, t2;
-
+ 
+#
+# BUG#36137 "virtual longlong Item_in_subselect::val_int(): Assertion `0' failed."
+#
+create table t1 (a int, b int, key (a), key (b));
+insert into t1 values (2,4),(2,4),(2,4);
+select t1.a from t1 
+where 
+  t1.a in (select 1 from t1 where t1.a in (select 1 from t1) group by  t1.a);
+drop table t1;
 
diff -Nrup a/sql/item_subselect.cc b/sql/item_subselect.cc
--- a/sql/item_subselect.cc	2008-04-14 14:09:58 +04:00
+++ b/sql/item_subselect.cc	2008-04-23 09:26:25 +04:00
@@ -3171,7 +3171,8 @@ int subselect_hash_sj_engine::exec()
     int res= 0;
     SELECT_LEX *save_select= thd->lex->current_select;
     thd->lex->current_select= materialize_engine->select_lex;
-    if ((res= materialize_join->optimize()))
+    if ((res= materialize_join->flatten_subqueries()) || 
+        (res= materialize_join->optimize()))
       goto err;
     materialize_join->exec();
     if ((res= test(materialize_join->error || thd->is_fatal_error)))