bzr commit into mysql-5.0-bugteam branch (gshchepa:2748) Bug#40761

[Gleb Shchepa]

#At file:///work/bzr/mysql-5.0-bugteam/ based on revid:gshchepa@stripped

 2748 Gleb Shchepa	2008-12-12
      Bug #40761: Assert on sum function on
                  IF(..., CAST(longtext AS UNSIGNED), signed_val)
                  (was: LEFT JOIN on inline view crashes server)
      
      Select from a LONGTEXT column wrapped with an expression
      like "IF(..., CAST(longtext_column AS UNSIGNED), smth_signed)"
      failed an assertion or crashed the server. IFNULL function was
      affected too.
      
      LONGTEXT column item has a maximum length of 32^2-1 bytes,
      at the same time this is a maximum possible length of any
      MySQL item. CAST(longtext_column AS UNSIGNED) returns some
      unsigned numeric result of length 32^2-1, so the result of
      IF/IFNULL function of this number and some other signed number
      will have text length of (32^2-1)+1=32^2 (one byte for the
      minus sign) - there is integer overflow, and the length is
      equal to zero. That caused assert/crash.
      
      CAST AS UNSIGNED function has been modified to limit maximal
      length of resulting number to 67 (maximal length of DECIMAL
      and two characters for minus sign and dot).
modified:
  mysql-test/r/func_if.result
  mysql-test/t/func_if.test
  sql/item_func.h

per-file messages:
  mysql-test/r/func_if.result
    Added test case for bug #40761.
  mysql-test/t/func_if.test
    Added test case for bug #40761.
  sql/item_func.h
    Bug #40761: Assert on sum function on
                IF(..., CAST(longtext AS UNSIGNED), signed_val)
    
    CAST AS UNSIGNED function has been modified to limit maximal
    length of resulting number to 67 (maximal length of DECIMAL
    and two characters for minus sign and dot).
=== modified file 'mysql-test/r/func_if.result'
--- a/mysql-test/r/func_if.result	2008-12-12 10:59:10 +0000
+++ b/mysql-test/r/func_if.result	2008-12-12 13:16:25 +0000
@@ -176,4 +176,13 @@ IF((ROUND(t1.a,2)=1), 2,
 IF((ROUND(t1.a,2)=1), 2,
 IF((R
 DROP TABLE t1;
+CREATE TABLE t1 (c LONGTEXT);
+INSERT INTO t1 VALUES(1), (2), (3), (4), ('12345678901234567890');
+SELECT * FROM (SELECT MAX(IF(1, CAST(c AS UNSIGNED), 0)) FROM t1) AS te;
+MAX(IF(1, CAST(c AS UNSIGNED), 0))
+12345678901234567890
+SELECT * FROM (SELECT MAX(IFNULL(CAST(c AS UNSIGNED), 0)) FROM t1) AS te;
+MAX(IFNULL(CAST(c AS UNSIGNED), 0))
+12345678901234567890
+DROP TABLE t1;
 End of 5.0 tests

=== modified file 'mysql-test/t/func_if.test'
--- a/mysql-test/t/func_if.test	2008-12-12 10:59:10 +0000
+++ b/mysql-test/t/func_if.test	2008-12-12 13:16:25 +0000
@@ -150,4 +150,18 @@ FROM t1;
 
 DROP TABLE t1;
 
+#
+# Bug #40761: Assert on sum func on IF(..., CAST(longtext AS UNSIGNED), signed)
+#             (was: LEFT JOIN on inline view crashes server)
+#
+
+CREATE TABLE t1 (c LONGTEXT);
+INSERT INTO t1 VALUES(1), (2), (3), (4), ('12345678901234567890');
+
+SELECT * FROM (SELECT MAX(IF(1, CAST(c AS UNSIGNED), 0)) FROM t1) AS te;
+SELECT * FROM (SELECT MAX(IFNULL(CAST(c AS UNSIGNED), 0)) FROM t1) AS te;
+
+DROP TABLE t1;
+
+
 --echo End of 5.0 tests

=== modified file 'sql/item_func.h'
--- a/sql/item_func.h	2008-12-12 10:59:10 +0000
+++ b/sql/item_func.h	2008-12-12 13:16:25 +0000
@@ -351,7 +351,10 @@ public:
   Item_func_unsigned(Item *a) :Item_func_signed(a) {}
   const char *func_name() const { return "cast_as_unsigned"; }
   void fix_length_and_dec()
-  { max_length=args[0]->max_length; unsigned_flag=1; }
+  {
+    max_length= min(args[0]->max_length, DECIMAL_MAX_PRECISION + 2);
+    unsigned_flag=1;
+  }
   longlong val_int();
   void print(String *str);
 };