bk commit into 6.0 tree (sergefp:1.2625) BUG#36137

[Sergey Petrunia]

Below is the list of changes that have just been committed into a local
6.0 repository of sergefp.  When sergefp does a push these changes
will be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-04-23 06:56:36+04:00, sergefp@stripped +3 -0
  BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
  - Call join->flatten_subqueries() before we call join->optimize(). 
    If we don't do this, subquery predicates which were slated for conversion
    it semi-join nests will remain in the WHERE clause and will fire an assert
    when we attempt to evaluate them (this happens because these predicates
    have not undergone the mandatory IN->EXISTS rewrite)

  mysql-test/r/subselect_sj2.result@stripped, 2008-04-23 06:56:29+04:00, sergefp@stripped +8
-0
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Testcase

  mysql-test/t/subselect_sj2.test@stripped, 2008-04-23 06:56:29+04:00, sergefp@stripped +10 -0
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Testcase

  sql/item_subselect.cc@stripped, 2008-04-23 06:56:30+04:00, sergefp@stripped +4 -0
    BUG#36137: virtual longlong Item_in_subselect::val_int(): Assertion `0' failed.
    - Call join->flatten_subqueries() before we call join->optimize(). 
      If we don't do this, subquery predicates which were slated for conversion
      it semi-join nests will remain in the WHERE clause and will fire an assert
      when we attempt to evaluate them (this happens because these predicates
      have not undergone the mandatory IN->EXISTS rewrite)

diff -Nrup a/mysql-test/r/subselect_sj2.result b/mysql-test/r/subselect_sj2.result
--- a/mysql-test/r/subselect_sj2.result	2008-04-06 03:19:17 +04:00
+++ b/mysql-test/r/subselect_sj2.result	2008-04-23 06:56:29 +04:00
@@ -438,3 +438,11 @@ from t1 
 where fid IN (select fid from t2 where (id between 7502 and 8420) order by fid );
 ERROR 42S22: Unknown column 'fid' in 'IN/ALL/ANY subquery'
 drop table t1, t2;
+create table t1 (a int, b int, key (a), key (b));
+insert into t1 values (2,4),(2,4),(2,4);
+select t1.a from t1 
+where 
+t1.a not in (select 1 from t1) and
+t1.a in (select 1 from t1 where t1.a in (select 1 from t1) group by  t1.a);
+a
+drop table t1;
diff -Nrup a/mysql-test/t/subselect_sj2.test b/mysql-test/t/subselect_sj2.test
--- a/mysql-test/t/subselect_sj2.test	2008-04-06 03:19:17 +04:00
+++ b/mysql-test/t/subselect_sj2.test	2008-04-23 06:56:29 +04:00
@@ -591,4 +591,14 @@ where fid IN (select fid from t2 where (
 
 drop table t1, t2;
 
+#
+# BUG#36137 "virtual longlong Item_in_subselect::val_int(): Assertion `0' failed."
+#
+create table t1 (a int, b int, key (a), key (b));
+insert into t1 values (2,4),(2,4),(2,4);
+select t1.a from t1 
+where 
+  t1.a not in (select 1 from t1) and
+  t1.a in (select 1 from t1 where t1.a in (select 1 from t1) group by  t1.a);
+drop table t1;
 
diff -Nrup a/sql/item_subselect.cc b/sql/item_subselect.cc
--- a/sql/item_subselect.cc	2008-03-20 01:47:53 +03:00
+++ b/sql/item_subselect.cc	2008-04-23 06:56:30 +04:00
@@ -3162,6 +3162,10 @@ int subselect_hash_sj_engine::exec()
     int res= 0;
     SELECT_LEX *save_select= thd->lex->current_select;
     thd->lex->current_select= materialize_engine->select_lex;
+    /* dump_TABLE_LIST_graph(select_lex, select_lex->leaf_tables); */
+    if (materialize_join->flatten_subqueries())
+      goto err;
+    /* dump_TABLE_LIST_struct(select_lex, select_lex->leaf_tables); */
     if ((res= materialize_join->optimize()))
       goto err;
     materialize_join->exec();