bk commit into 4.1 tree (gluh:1.2344) BUG#9854

[gluh]

Below is the list of changes that have just been committed into a local
4.1 repository of gluh. When gluh does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2344 05/07/13 14:23:09 gluh@stripped +3 -0
  Bug #9854 hex() and out of range handling
    added out of range handling

  sql/item_strfunc.cc
    1.231 05/07/13 14:22:28 gluh@stripped +14 -2
    Bug #9854 hex() and out of range handling
      added out of range handling

  mysql-test/t/func_str.test
    1.76 05/07/13 14:22:28 gluh@stripped +5 -0
    Bug #9854 hex() and out of range handling
      added out of range handling

  mysql-test/r/func_str.result
    1.95 05/07/13 14:22:28 gluh@stripped +3 -0
    Bug #9854 hex() and out of range handling
      added out of range handling

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	gluh
# Host:	eagle.intranet.mysql.r18.ru
# Root:	/home/gluh/MySQL/Merge/4.1

--- 1.230/sql/item_strfunc.cc	Tue Jul 12 20:34:46 2005
+++ 1.231/sql/item_strfunc.cc	Wed Jul 13 14:22:28 2005
@@ -2348,9 +2348,21 @@
   DBUG_ASSERT(fixed == 1);
   if (args[0]->result_type() != STRING_RESULT)
   {
-    /* Return hex of unsigned longlong value */
-    longlong dec= args[0]->val_int();
+    ulonglong dec;
     char ans[65],*ptr;
+    /* Return hex of unsigned longlong value */
+    if (args[0]->result_type() == REAL_RESULT)
+    {
+      double val= args[0]->val();
+      if ((val <= (double) LONGLONG_MIN) || 
+          (val >= (double) (ulonglong) ULONGLONG_MAX))
+        dec=  ~(longlong) 0;
+      else
+        dec= (ulonglong) (val + (val > 0 ? 0.5 : -0.5));
+    }
+    else
+      dec= (ulonglong) args[0]->val_int();
+
     if ((null_value= args[0]->null_value))
       return 0;
     ptr= longlong2str(dec,ans,16);

--- 1.94/mysql-test/r/func_str.result	Wed Jul  6 13:36:25 2005
+++ 1.95/mysql-test/r/func_str.result	Wed Jul 13 14:22:28 2005
@@ -865,3 +865,6 @@
 abcd	abcd	d	cd	bcd	abcd	
 abcde	abcde	e	de	cde	bcde	abcde
 drop table t1;
+select hex(29223372036854775809), hex(-29223372036854775809);
+hex(29223372036854775809)	hex(-29223372036854775809)
+FFFFFFFFFFFFFFFF	FFFFFFFFFFFFFFFF

--- 1.75/mysql-test/t/func_str.test	Wed Jul  6 13:36:25 2005
+++ 1.76/mysql-test/t/func_str.test	Wed Jul 13 14:22:28 2005
@@ -606,3 +606,8 @@
 select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1;
 select * from (select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1) t;
 drop table t1;
+
+#
+# Bug #9854  hex() and out of range handling
+#
+select hex(29223372036854775809), hex(-29223372036854775809);