bzr push into mysql-trunk branch (norvald.ryeng:3528 to 3529) Bug#11764818

[Norvald H. Ryeng]

 3529 Norvald H. Ryeng	2011-10-27
      Bug #11764818 57692: CRASH IN ITEM_FUNC_IN::VAL_INT() WITH ZEROFILL
      
      Problem: During optimization, ZEROFILL values may be converted to
      string constants. However, the IN function does not handle switching
      datatypes after planning, leading to IN finding a null pointer instead
      of its argument.
      
      Item_func_in creates a table of cmp_items, one for each datatype used
      in the comparison. This table is created during query planning by
      fix_length_and_dec(). During optimization,
      Item_field::equal_fields_propagator() converts ZEROFILL numbers to
      strings, but the comparison table in Item_func_in is not
      updated. During execution, the position in the comparison table is
      found by examining the result types of the fields to be
      compared. Since the result type of the fields have changed, the wrong
      position in the comparison table is inferred, and this position
      contains a null pointer.
      
      Fix: Set the cmp_context of arguments to Item_func_in so that
      Item_field::equal_fields_propagator() will not convert them to
      strings.
     @ mysql-test/r/compare.result
        Add test for bug#11764818
     @ mysql-test/t/compare.test
        Add test for bug#11764818
     @ sql/item_cmpfunc.cc
        Set cmp_context of arguments to Item_func_in

    modified:
      mysql-test/r/compare.result
      mysql-test/t/compare.test
      sql/item_cmpfunc.cc
 3528 Alexander Nozdrin	2011-10-27 [merge]
      Null merge from mysql-5.5.

=== modified file 'mysql-test/r/compare.result'
--- a/mysql-test/r/compare.result	2011-07-19 15:11:15 +0000
+++ b/mysql-test/r/compare.result	2011-10-27 07:03:49 +0000
@@ -96,3 +96,7 @@ SELECT * FROM t1 WHERE a > '2008-01-01'
 a
 DROP TABLE t1;
 End of 5.0 tests
+CREATE TABLE t1(a INT ZEROFILL);
+SELECT 1 FROM t1 WHERE t1.a IN (1, t1.a) AND t1.a=2;
+1
+DROP TABLE t1;

=== modified file 'mysql-test/t/compare.test'
--- a/mysql-test/t/compare.test	2008-09-18 12:55:36 +0000
+++ b/mysql-test/t/compare.test	2011-10-27 07:03:49 +0000
@@ -86,3 +86,11 @@ SELECT * FROM t1 WHERE a > '2008-01-01'
 DROP TABLE t1;
 
 --echo End of 5.0 tests
+
+#
+# Bug #11764818 57692: Crash in item_func_in::val_int() with ZEROFILL
+#
+
+CREATE TABLE t1(a INT ZEROFILL);
+SELECT 1 FROM t1 WHERE t1.a IN (1, t1.a) AND t1.a=2;
+DROP TABLE t1;

=== modified file 'sql/item_cmpfunc.cc'
--- a/sql/item_cmpfunc.cc	2011-10-18 13:45:29 +0000
+++ b/sql/item_cmpfunc.cc	2011-10-27 07:03:49 +0000
@@ -4242,6 +4242,16 @@ void Item_func_in::fix_length_and_dec()
       }
     }
   }
+  /*
+    Set cmp_context of all arguments. This prevents
+    Item_field::equal_fields_propagator() from transforming a zerofill integer
+    argument into a string constant. Such a change would require rebuilding
+    cmp_itmes.
+   */
+  for (arg= args + 1, arg_end= args + arg_count; arg != arg_end ; arg++)
+  {
+    arg[0]->cmp_context= item_cmp_type(left_result_type, arg[0]->result_type());
+  }
   max_length= 1;
 }
 

No bundle (reason: useless for push emails).