MySQL Lists are EOL. Please join:

List:Commits« Previous MessageNext Message »
From:Alexey Botchkov Date:November 9 2008 1:19pm
Subject:bzr commit into mysql-5.0-bugteam branch (holyfoot:2709) Bug#10633
View as plain text  
#At file:///home/hf/work/mysql_common/10633/

 2709 Alexey Botchkov	2008-11-09
      Bug#10633 Precision math: round(x,d) causes truncation if d > 30 
              result of the 'DECIMAL' operation was limited with MAX_SCALE.
              But the MAX_SCAL only limits fields of tables, not the 'decimal' engine itself.
              So the limit was changed to MAX_PRECISION.
      
      per-file comments:
        mysql-test/r/func_math.result
      Bug#10633 Precision math: round(x,d) causes truncation if d > 30 
         test result fixed
        mysql-test/r/type_decimal.result
      Bug#10633 Precision math: round(x,d) causes truncation if d > 30 
         test result fixed
        mysql-test/r/type_newdecimal.result
      Bug#10633 Precision math: round(x,d) causes truncation if d > 30 
         test result fixed
        sql/item_func.cc
      Bug#10633 Precision math: round(x,d) causes truncation if d > 30 
         scale limit for decimal operation changed with MAX_PRECISION
modified:
  mysql-test/r/func_math.result
  mysql-test/r/type_decimal.result
  mysql-test/r/type_newdecimal.result
  sql/item_func.cc

=== modified file 'mysql-test/r/func_math.result'
--- a/mysql-test/r/func_math.result	2007-10-01 09:51:59 +0000
+++ b/mysql-test/r/func_math.result	2008-11-09 13:15:56 +0000
@@ -285,28 +285,28 @@ round(1.12e1, 4294967296)	truncate(1.12e
 11.2	11.2
 select round(1.5, 2147483640), truncate(1.5, 2147483640);
 round(1.5, 2147483640)	truncate(1.5, 2147483640)
-1.500000000000000000000000000000	1.500000000000000000000000000000
+1.50000000000000000000000000000000000000000000000000000000000000000	1.50000000000000000000000000000000000000000000000000000000000000000
 select round(1.5, -2147483649), round(1.5, 2147483648);
 round(1.5, -2147483649)	round(1.5, 2147483648)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select truncate(1.5, -2147483649), truncate(1.5, 2147483648);
 truncate(1.5, -2147483649)	truncate(1.5, 2147483648)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select round(1.5, -4294967296), round(1.5, 4294967296);
 round(1.5, -4294967296)	round(1.5, 4294967296)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select truncate(1.5, -4294967296), truncate(1.5, 4294967296);
 truncate(1.5, -4294967296)	truncate(1.5, 4294967296)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select round(1.5, -9223372036854775808), round(1.5, 9223372036854775808);
 round(1.5, -9223372036854775808)	round(1.5, 9223372036854775808)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select truncate(1.5, -9223372036854775808), truncate(1.5, 9223372036854775808);
 truncate(1.5, -9223372036854775808)	truncate(1.5, 9223372036854775808)
-0	1.500000000000000000000000000000
+0	1.50000000000000000000000000000000000000000000000000000000000000000
 select round(1.5, 18446744073709551615), truncate(1.5, 18446744073709551615);
 round(1.5, 18446744073709551615)	truncate(1.5, 18446744073709551615)
-1.500000000000000000000000000000	1.500000000000000000000000000000
+1.50000000000000000000000000000000000000000000000000000000000000000	1.50000000000000000000000000000000000000000000000000000000000000000
 select round(18446744073709551614, -1), truncate(18446744073709551614, -1);
 round(18446744073709551614, -1)	truncate(18446744073709551614, -1)
 18446744073709551610	18446744073709551610

=== modified file 'mysql-test/r/type_decimal.result'
--- a/mysql-test/r/type_decimal.result	2008-04-24 20:39:37 +0000
+++ b/mysql-test/r/type_decimal.result	2008-11-09 13:15:56 +0000
@@ -832,8 +832,11 @@ c
 123456
 SELECT ROUND( a, 100 ) AS c FROM t1 ORDER BY c;
 c
-123456.000000000000000000000000000000
-123456.000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
 CREATE TABLE t2( a NUMERIC, b INT );
 INSERT INTO t2 VALUES (123456, 100);
 SELECT TRUNCATE( a, b ) AS c FROM t2 ORDER BY c;
@@ -854,8 +857,11 @@ c
 123456
 SELECT ROUND( a, 100 ) AS c FROM t3 ORDER BY c;
 c
-123456.000000000000000000000000000000
-123456.000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
 CREATE TABLE t4( a DECIMAL, b INT );
 INSERT INTO t4 VALUES (123456, 40), (123456, 40);
 SELECT TRUNCATE( a, b ) AS c FROM t4 ORDER BY c;
@@ -868,8 +874,11 @@ c
 123456
 SELECT ROUND( a, 100 ) AS c FROM t4 ORDER BY c;
 c
-123456.000000000000000000000000000000
-123456.000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+123456.00000000000000000000000000000000000000000000000000000000000000000
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
 delete from t1;
 INSERT INTO t1 VALUES (1234567890, 20), (999.99, 5);
 Warnings:

=== modified file 'mysql-test/r/type_newdecimal.result'
--- a/mysql-test/r/type_newdecimal.result	2008-08-15 19:46:21 +0000
+++ b/mysql-test/r/type_newdecimal.result	2008-11-09 13:15:56 +0000
@@ -760,10 +760,10 @@ round(-99999999999999999.999,3)
 -99999999999999999.999
 select truncate(99999999999999999999999999999999999999,31);
 truncate(99999999999999999999999999999999999999,31)
-99999999999999999999999999999999999999.000000000000000000000000000000
+99999999999999999999999999999999999999.0000000000000000000000000000000
 select truncate(99.999999999999999999999999999999999999,31);
 truncate(99.999999999999999999999999999999999999,31)
-99.999999999999999999999999999999
+99.9999999999999999999999999999999
 select truncate(99999999999999999999999999999999999999,-31);
 truncate(99999999999999999999999999999999999999,-31)
 99999990000000000000000000000000000000

=== modified file 'sql/item_func.cc'
--- a/sql/item_func.cc	2008-07-30 11:07:37 +0000
+++ b/sql/item_func.cc	2008-11-09 13:15:56 +0000
@@ -2004,13 +2004,13 @@ void Item_func_round::fix_length_and_dec
   case DECIMAL_RESULT:
   {
     hybrid_type= DECIMAL_RESULT;
-    decimals_to_set= min(DECIMAL_MAX_SCALE, decimals_to_set);
+    decimals_to_set= min(DECIMAL_MAX_PRECISION, decimals_to_set);
     int decimals_delta= args[0]->decimals - decimals_to_set;
     int precision= args[0]->decimal_precision();
     int length_increase= ((decimals_delta <= 0) || truncate) ? 0:1;
 
     precision-= decimals_delta - length_increase;
-    decimals= min(decimals_to_set, DECIMAL_MAX_SCALE);
+    decimals= min(decimals_to_set, DECIMAL_MAX_PRECISION);
     max_length= my_decimal_precision_to_length(precision, decimals,
                                                unsigned_flag);
     break;

Thread
bzr commit into mysql-5.0-bugteam branch (holyfoot:2709) Bug#10633Alexey Botchkov9 Nov