bzr commit into mysql-6.0 branch (alexeyk:2922) Bug#8457

[Alexey Kopytov]

#At file:///Users/kaa/src/bzr/bug8457/my60-bug8457/

 2922 Alexey Kopytov	2008-12-11
      Fix for bug #8457: Precision math: DIV returns incorrect result with large decimal
                         value
                  
      For the DIV operator, neither operands nor result were checked for integer
      overflows.
      
      This patch changes the DIV behavior for non-integer operands as follows:
      if either of the operands has a non-integer type, convert both operands to the
      DECIMAL type, then calculate the division using DECIMAL arithmetics. Convert the 
      resulting DECIMAL value into BIGINT [UNSIGNED] if it fits into the corresponding 
      range, or throw an 'out of range' error otherwise.
modified:
  mysql-test/r/func_math.result
  mysql-test/r/type_varchar.result
  mysql-test/t/func_math.test
  sql/item_func.cc

per-file messages:
  mysql-test/r/func_math.result
    Added a test case for bug #8457.
    Fixed results for a test case depending on the wrong behavior.
  mysql-test/r/type_varchar.result
    Fixed results for a test case depending on the wrong behavior.
  mysql-test/t/func_math.test
    Added a test case for bug #8457.
  sql/item_func.cc
    If either of the operands has a non-integer type, convert both operands to the
    DECIMAL type, then calculate the division using DECIMAL arithmetics. Convert the 
    resulting DECIMAL value into BIGING [UNSIGNED] if it fits into the corresponding 
    range, or throw an 'out of range' error otherwise.
=== modified file 'mysql-test/r/func_math.result'
--- a/mysql-test/r/func_math.result	2008-07-14 12:49:19 +0000
+++ b/mysql-test/r/func_math.result	2008-12-11 15:27:05 +0000
@@ -382,6 +382,9 @@ y
 SELECT b DIV 900 y FROM t1 GROUP BY y;
 y
 0
+Warnings:
+Warning	1366	Incorrect decimal value: '' for column '' at row -1
+Warning	1366	Incorrect decimal value: '' for column '' at row -1
 SELECT c DIV 900 y FROM t1 GROUP BY y;
 y
 0
@@ -457,3 +460,8 @@ SELECT POW(10, 309);
 POW(10, 309)
 NULL
 End of 5.1 tests
+select 123456789012345678901234567890.123456789012345678901234567890 div 1 as x;
+ERROR 22003: Out of range value for column 'x' at row 1
+select "123456789012345678901234567890.123456789012345678901234567890" div 1 as x;
+ERROR 22003: Out of range value for column 'x' at row 1
+End of 6.0 tests

=== modified file 'mysql-test/r/type_varchar.result'
--- a/mysql-test/r/type_varchar.result	2006-12-05 09:19:56 +0000
+++ b/mysql-test/r/type_varchar.result	2008-12-11 15:27:05 +0000
@@ -475,8 +475,9 @@ a	(a DIV 2)
 60	30
 t	0
 Warnings:
-Warning	1292	Truncated incorrect INTEGER value: '1a'
-Warning	1292	Truncated incorrect INTEGER value: 't '
+Warning	1292	Truncated incorrect DECIMAL value: '1a'
+Warning	1366	Incorrect decimal value: '' for column '' at row -1
+Warning	1292	Truncated incorrect DECIMAL value: 't '
 SELECT a,CAST(a AS SIGNED) FROM t1 ORDER BY a;
 a	CAST(a AS SIGNED)
 10	10

=== modified file 'mysql-test/t/func_math.test'
--- a/mysql-test/t/func_math.test	2008-04-07 15:57:54 +0000
+++ b/mysql-test/t/func_math.test	2008-12-11 15:27:05 +0000
@@ -283,3 +283,14 @@ SELECT EXP(750);
 SELECT POW(10, 309);
 
 --echo End of 5.1 tests
+
+#
+# Bug #8457: Precision math: DIV returns incorrect result with large decimal value
+#
+
+--error ER_WARN_DATA_OUT_OF_RANGE
+select 123456789012345678901234567890.123456789012345678901234567890 div 1 as x;
+--error ER_WARN_DATA_OUT_OF_RANGE
+select "123456789012345678901234567890.123456789012345678901234567890" div 1 as x; 
+
+--echo End of 6.0 tests

=== modified file 'sql/item_func.cc'
--- a/sql/item_func.cc	2008-11-06 18:39:27 +0000
+++ b/sql/item_func.cc	2008-12-11 15:27:05 +0000
@@ -1366,6 +1366,34 @@ void Item_func_div::fix_length_and_dec()
 longlong Item_func_int_div::val_int()
 {
   DBUG_ASSERT(fixed == 1);
+
+  if (args[0]->result_type() != INT_RESULT ||
+      args[1]->result_type() != INT_RESULT)
+  {
+    my_decimal value0, value1, tmp;
+    my_decimal *val0, *val1;
+    longlong res;
+    int err;
+
+    val0= args[0]->val_decimal(&value0);
+    val1= args[1]->val_decimal(&value1);
+    if ((null_value= (args[0]->null_value || args[1]->null_value)))
+      return 0;
+
+    if ((err= my_decimal_div(E_DEC_FATAL_ERROR & ~E_DEC_DIV_ZERO, &tmp,
+                             val0, val1, 0)) > 3)
+    {
+      if (err == E_DEC_DIV_ZERO)
+        signal_divide_by_null();
+      return 0;
+    }
+
+    if (my_decimal2int(E_DEC_FATAL_ERROR, &tmp, unsigned_flag, &res) &
+        E_DEC_OVERFLOW)
+      my_error(ER_WARN_DATA_OUT_OF_RANGE, MYF(0), name, 1);
+    return res;
+  }
+  
   longlong value=args[0]->val_int();
   longlong val2=args[1]->val_int();
   if ((null_value= (args[0]->null_value || args[1]->null_value)))