bzr push into mysql-trunk branch (sergey.glukhov:3733 to 3734) Bug#13723054

[Sergey Glukhov]

 3734 Sergey Glukhov	2012-04-24
      Bug#13723054 CRASH WITH MIN/MAX AFTER QUICK_GROUP_MIN_MAX_SELECT::NEXT_MIN
      The crash is caused by arbitrary memory area owerwriting in case of
      BLOB fields during attempt to copy BLOB field key image into record
      buffer(record buffer is too small to get BLOB key part image).
      note:
      QUICK_GROUP_MIN_MAX_SELECT can not work with BLOB fields
      because it uses record buffer as temporary buffer for key values
      however this case is filtered out by covering_keys() check
      in get_best_group_min_max() as BLOBs always require key length
      modificator in the key declaration and if the key has a BLOB
      then it can not be covered key.
      The fix is to use 'max_used_key_length' key length instead of 0.
     @ mysql-test/r/func_group_innodb.result
        test case
     @ mysql-test/t/func_group_innodb.test
        test case
     @ sql/opt_range.cc
        use 'max_used_key_length' key length instead of 0
        as 0 means the total length of the key and it could
        lead to arbitrary memory area owerwriting in case of
        BLOB fields.

    modified:
      mysql-test/r/func_group_innodb.result
      mysql-test/t/func_group_innodb.test
      sql/opt_range.cc
 3733 Sergey Glukhov	2012-04-24
      Bug#13011553 CRASH IN SORTCMP OR CACHED_ITEM_STR::CMP IF GROUP BY SUBQUERY WITH ROLLUP
      During of comparing of grouped values
      DATETIME values are compared as STRING
      rather than INT. It leads to crash in the
      case of NULL value comparison because val_str()
      method could return zero pointer to the string
      and further comparison fails. The fix is to
      compare DATETIME grouped values for ROLLAUP as INT.
     @ mysql-test/r/olap.result
        test case
     @ mysql-test/t/olap.test
        test case
     @ sql/item.h
        added new class Cached_item_temporal
     @ sql/item_buff.cc
        use Cached_item_temporal object if item is DATE/TIME/DATETIME
     @ sql/item_func.h
        oveloaded field_type() method for Item_func_rollup_const class

    modified:
      mysql-test/r/olap.result
      mysql-test/t/olap.test
      sql/item.h
      sql/item_buff.cc
      sql/item_func.h
=== modified file 'mysql-test/r/func_group_innodb.result'
--- a/mysql-test/r/func_group_innodb.result	2012-04-18 06:07:33 +0000
+++ b/mysql-test/r/func_group_innodb.result	2012-04-24 08:27:22 +0000
@@ -192,3 +192,17 @@ BIT_XOR(t1.b)
 0
 DROP TABLE t1, t2;
 End of 5.5 tests
+#
+# Bug#13723054 CRASH WITH MIN/MAX AFTER QUICK_GROUP_MIN_MAX_SELECT::NEXT_MIN
+#
+CREATE TABLE t1(a BLOB, b VARCHAR(255) CHARSET LATIN1, c INT,
+KEY(b, c, a(765))) ENGINE=INNODB;
+INSERT INTO t1(a, b, c) VALUES ('', 'a', 0), ('', 'a', null), ('', 'a', 0);
+SELECT MIN(c) FROM t1 GROUP BY b;
+MIN(c)
+0
+EXPLAIN SELECT MIN(c) FROM t1 GROUP BY b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	NULL	b	263	NULL	4	Using index for group-by
+DROP TABLE t1;
+End of 5.6 tests

=== modified file 'mysql-test/t/func_group_innodb.test'
--- a/mysql-test/t/func_group_innodb.test	2012-04-18 06:04:36 +0000
+++ b/mysql-test/t/func_group_innodb.test	2012-04-24 08:27:22 +0000
@@ -133,3 +133,19 @@ DROP TABLE t1, t2;
 
 
 --echo End of 5.5 tests
+
+--echo #
+--echo # Bug#13723054 CRASH WITH MIN/MAX AFTER QUICK_GROUP_MIN_MAX_SELECT::NEXT_MIN
+--echo #
+
+CREATE TABLE t1(a BLOB, b VARCHAR(255) CHARSET LATIN1, c INT,
+                KEY(b, c, a(765))) ENGINE=INNODB;
+INSERT INTO t1(a, b, c) VALUES ('', 'a', 0), ('', 'a', null), ('', 'a', 0);
+
+SELECT MIN(c) FROM t1 GROUP BY b;
+EXPLAIN SELECT MIN(c) FROM t1 GROUP BY b;
+
+DROP TABLE t1;
+
+
+--echo End of 5.6 tests

=== modified file 'sql/opt_range.cc'
--- a/sql/opt_range.cc	2012-03-21 14:27:34 +0000
+++ b/sql/opt_range.cc	2012-04-24 08:27:22 +0000
@@ -12420,7 +12420,7 @@ int QUICK_GROUP_MIN_MAX_SELECT::next_min
     if (min_max_arg_part && min_max_arg_part->field->is_null())
     {
       /* Find the first subsequent record without NULL in the MIN/MAX field. */
-      key_copy(tmp_record, record, index_info, 0);
+      key_copy(tmp_record, record, index_info, max_used_key_length);
       result= head->file->ha_index_read_map(record, tmp_record,
                                             make_keypart_map(real_key_parts),
                                             HA_READ_AFTER_KEY);

No bundle (reason: useless for push emails).