bzr commit into mysql-5.1 branch (sergey.glukhov:3580) Bug#59387

[Sergey Glukhov]

#At file:///home/gluh/MySQL/mysql-5.1/ based on revid:ole.john.aske@stripped

 3580 Sergey Glukhov	2011-02-07
      Bug#59387 Failing assertion: cursor->pos_state == 1997660512 (BTR_PCUR_IS_POSITIONED)
      The crash happened due to wrong calculation
      of key length during creation of reference for
      sort order index. The problem is that
      keyuse->used_tables has OUTER_REF_TABLE_BIT enabled
      and it breaks correct key length calculation.
     @ mysql-test/suite/innodb/r/innodb_mysql.result
        test case
     @ mysql-test/suite/innodb/t/innodb_mysql.test
        test case
     @ sql/sql_select.cc
        The crash happened due to wrong calculation
        of key length during creation of reference for
        sort order index. The problem is that
        keyuse->used_tables has OUTER_REF_TABLE_BIT enabled
        and it breaks correct key length calculation.

    modified:
      mysql-test/suite/innodb/r/innodb_mysql.result
      mysql-test/suite/innodb/t/innodb_mysql.test
      sql/sql_select.cc
=== modified file 'mysql-test/suite/innodb/r/innodb_mysql.result'
--- a/mysql-test/suite/innodb/r/innodb_mysql.result	2010-11-23 10:18:47 +0000
+++ b/mysql-test/suite/innodb/r/innodb_mysql.result	2011-02-07 12:43:21 +0000
@@ -2639,6 +2639,23 @@ COUNT(*)
 1537
 SET SESSION sort_buffer_size = DEFAULT;
 DROP TABLE t1;
+#
+# Bug#59387 Failing assertion: cursor->pos_state == 1997660512 (BTR_PCUR_IS_POSITIONED)
+#
+CREATE TABLE t1 (a INT) ENGINE=INNODB;
+INSERT INTO t1 VALUES (0);
+CREATE TABLE t2 (d BINARY(2), PRIMARY KEY (d(1)), UNIQUE KEY (d)) ENGINE=INNODB;
+SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+1
+1
+EXPLAIN SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	1	Using where
+2	DEPENDENT SUBQUERY	t2	eq_ref	PRIMARY,d	d	2	func	1	Using where
+3	DEPENDENT SUBQUERY	t2	index	NULL	d	2	NULL	1	Using where; Using index
+DROP TABLE t1, t2;
 End of 5.1 tests
 #
 # Test for bug #39932 "create table fails if column for FK is in different

=== modified file 'mysql-test/suite/innodb/t/innodb_mysql.test'
--- a/mysql-test/suite/innodb/t/innodb_mysql.test	2010-12-20 11:58:33 +0000
+++ b/mysql-test/suite/innodb/t/innodb_mysql.test	2011-02-07 12:43:21 +0000
@@ -868,6 +868,22 @@ SET SESSION sort_buffer_size = DEFAULT;
 DROP TABLE t1;
 
 
+--echo #
+--echo # Bug#59387 Failing assertion: cursor->pos_state == 1997660512 (BTR_PCUR_IS_POSITIONED)
+--echo #
+
+CREATE TABLE t1 (a INT) ENGINE=INNODB;
+INSERT INTO t1 VALUES (0);
+CREATE TABLE t2 (d BINARY(2), PRIMARY KEY (d(1)), UNIQUE KEY (d)) ENGINE=INNODB;
+
+SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+
+EXPLAIN SELECT 1 FROM t1 WHERE NOT EXISTS
+(SELECT 1 FROM t2 WHERE d = (SELECT d FROM t2 WHERE a >= 1) ORDER BY d);
+
+DROP TABLE t1, t2;
+
 --echo End of 5.1 tests
 
 

=== modified file 'sql/sql_select.cc'
--- a/sql/sql_select.cc	2011-02-07 09:36:21 +0000
+++ b/sql/sql_select.cc	2011-02-07 12:43:21 +0000
@@ -5763,7 +5763,7 @@ static bool create_ref_for_key(JOIN *joi
     */
     do
     {
-      if (!(~used_tables & keyuse->used_tables))
+      if (!(~used_tables & (keyuse->used_tables & ~OUTER_REF_TABLE_BIT)))
       {
 	if (keyparts == keyuse->keypart &&
 	    !(found_part_ref_or_null & keyuse->optimize))
@@ -5816,7 +5816,7 @@ static bool create_ref_for_key(JOIN *joi
     for (i=0 ; i < keyparts ; keyuse++,i++)
     {
       while (keyuse->keypart != i ||
-	     ((~used_tables) & keyuse->used_tables))
+	     ((~used_tables) & (keyuse->used_tables & ~OUTER_REF_TABLE_BIT)))
 	keyuse++;				/* Skip other parts */
 
       uint maybe_null= test(keyinfo->key_part[i].null_bit);


Attachment: [text/bzr-bundle] bzr/[e-mail]