bk commit into 5.0 tree (gunnar:1.2050) BUG#10963

[Gunnar von Boehn]

Below is the list of changes that have just been committed into a local
5.0 repository of gvb. When gvb does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2050 06/02/17 19:40:59 gunnar@stripped. +4 -0
  item_strfunc.cc:
    fix for bug #10963 LEFT/RIGHT/SUBSTR/.. string function on large length parameter
  
    The following string functions did suffer from overflows when called with looong
length parameters
    LEFT(), RIGHT(), SUBSTR(), INSERT(), RPAD(), LPAD(), LOCATE(), REPEAT(), SPACE()
  
    The problems was that the length parameter inside these functions got assigned to INT
or LONG variables.
    While none of the functions supports strings longer than 2^32,
    calling these functions with longer length parameter should work and not result in
overflows.
    
    As LONG has different ranges on some platforms some of these errors were not visible
on certain platforms.
    E.g LLP systems like 64-bit windows or 32-bit systems see overflows that are invisible
on 64-bit Linux.
  
    The fix handles the length as longlong now. Length longer than MAX_INT32 are handled
as MAX_INT32.

  mysql-test/r/func_str.result
    1.110 06/02/17 19:27:46 gvb@titan. +45 -0

  mysql-test/t/func_str.test
    1.84 06/02/17 19:27:45 gvb@titan. +28 -0

  sql/item_func.cc
    1.274 06/02/17 19:27:44 gvb@titan. +5 -3

  sql/item_strfunc.cc
    1.263 06/02/17 19:26:39 gvb@titan. +34 -10
    fix for bug #10963 LEFT/RIGHT/SUBSTR/.. string functiont on large length parameter

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	gunnar
# Host:	titan.
# Root:	/data/BK/mysql-5.0_10963

--- 1.273/sql/item_func.cc	2006-02-07 13:21:41 +01:00
+++ 1.274/sql/item_func.cc	2006-02-17 19:27:44 +01:00
@@ -2216,13 +2216,15 @@
     return 0; /* purecov: inspected */
   }
   null_value=0;
-  uint start=0;
-  uint start0=0;
+  longlong start=0;
+  longlong start0=0;
   my_match_t match;
 
   if (arg_count == 3)
   {
-    start0= start =(uint) args[2]->val_int()-1;
+    start0= start =(longlong) args[2]->val_int()-1;
+    unsigned_flag  =args[2]->unsigned_flag;
+    if( unsigned_flag || start > INT_MAX32) start=INT_MAX32;
     start=a->charpos(start);
     
     if (start > a->length() || start+b->length() > a->length())

--- 1.262/sql/item_strfunc.cc	2006-01-11 15:34:38 +01:00
+++ 1.263/sql/item_strfunc.cc	2006-02-17 19:26:39 +01:00
@@ -905,13 +905,18 @@
 {
   DBUG_ASSERT(fixed == 1);
   String *res,*res2;
-  uint start,length;
 
   null_value=0;
   res=args[0]->val_str(str);
   res2=args[3]->val_str(&tmp_value);
-  start=(uint) args[1]->val_int()-1;
-  length=(uint) args[2]->val_int();
+  longlong start=(longlong) args[1]->val_int()-1;
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || start > INT_MAX32) start=INT_MAX32;
+
+  longlong length=(longlong) args[2]->val_int();
+  unsigned_flag  =args[2]->unsigned_flag;
+  if( unsigned_flag || length > INT_MAX32) length=INT_MAX32;
+
   if (args[0]->null_value || args[1]->null_value || args[2]->null_value ||
       args[3]->null_value)
     goto null; /* purecov: inspected */
@@ -998,7 +1003,9 @@
 {
   DBUG_ASSERT(fixed == 1);
   String *res  =args[0]->val_str(str);
-  long length  =(long) args[1]->val_int();
+  longlong length  =(longlong) args[1]->val_int();
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || length > UINT_MAX32) length=UINT_MAX32;	   
   uint char_pos;
 
   if ((null_value=args[0]->null_value))
@@ -1039,8 +1046,10 @@
 {
   DBUG_ASSERT(fixed == 1);
   String *res  =args[0]->val_str(str);
-  long length  =(long) args[1]->val_int();
-
+  longlong length  =(longlong) args[1]->val_int();
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || length > UINT_MAX32) length=UINT_MAX32;
+  
   if ((null_value=args[0]->null_value))
     return 0; /* purecov: inspected */
   if (length <= 0)
@@ -1069,7 +1078,13 @@
   DBUG_ASSERT(fixed == 1);
   String *res  = args[0]->val_str(str);
   int32 start	= (int32) args[1]->val_int();
-  int32 length	= arg_count == 3 ? (int32) args[2]->val_int() : INT_MAX32;
+  longlong length = INT_MAX32;
+  if(arg_count == 3)
+  {
+    length  =(longlong) args[2]->val_int();
+    unsigned_flag  =args[2]->unsigned_flag;
+    if( unsigned_flag || length > INT_MAX32) length=INT_MAX32;
+  }
   int32 tmp_length;
 
   if ((null_value=(args[0]->null_value || args[1]->null_value ||
@@ -2075,7 +2090,11 @@
   DBUG_ASSERT(fixed == 1);
   uint length,tot_length;
   char *to;
-  long count= (long) args[1]->val_int();
+  longlong count= (longlong) args[1]->val_int();
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || count > INT_MAX32) count=INT_MAX32;
+
+
   String *res =args[0]->val_str(str);
 
   if (args[0]->null_value || args[1]->null_value)
@@ -2148,7 +2167,9 @@
   uint32 res_byte_length,res_char_length,pad_char_length,pad_byte_length;
   char *to;
   const char *ptr_pad;
-  int32 count= (int32) args[1]->val_int();
+  longlong count= (longlong) args[1]->val_int();
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || count > INT_MAX32) count=INT_MAX32;
   int32 byte_count= count * collation.collation->mbmaxlen;
   String *res =args[0]->val_str(str);
   String *rpad = args[2]->val_str(&rpad_str);
@@ -2233,7 +2254,10 @@
 {
   DBUG_ASSERT(fixed == 1);
   uint32 res_char_length,pad_char_length;
-  ulong count= (long) args[1]->val_int(), byte_count;
+  ulong byte_count;
+  longlong count= (longlong) args[1]->val_int();
+  unsigned_flag  =args[1]->unsigned_flag;
+  if( unsigned_flag || count > INT_MAX32) count=INT_MAX32; 
   String *res= args[0]->val_str(&tmp_value);
   String *pad= args[2]->val_str(&lpad_str);
 

--- 1.109/mysql-test/r/func_str.result	2005-10-11 23:58:17 +02:00
+++ 1.110/mysql-test/r/func_str.result	2006-02-17 19:27:46 +01:00
@@ -1023,3 +1023,48 @@
 format(d, 2)
 NULL
 drop table t1;
+SELECT LEFT('MYSQL',11000111000111000111);
+LEFT('MYSQL',11000111000111000111)
+MYSQL
+SELECT LEFT('MYSQL',256*256*256*256);
+LEFT('MYSQL',256*256*256*256)
+MYSQL
+SELECT RIGHT('MYSQL',11000111000111000111);
+RIGHT('MYSQL',11000111000111000111)
+MYSQL
+SELECT RIGHT('MYSQL',256*256*256*256);
+RIGHT('MYSQL',256*256*256*256)
+MYSQL
+SELECT SUBSTR('MYSQL',1,11000111000111000111);
+SUBSTR('MYSQL',1,11000111000111000111)
+MYSQL
+SELECT SUBSTR('MYSQL',1,256*256*256*256);
+SUBSTR('MYSQL',1,256*256*256*256)
+MYSQL
+SELECT INSERT('MYSQL',11000111000111000111,1,'DB');
+INSERT('MYSQL',11000111000111000111,1,'DB')
+MYSQL
+SELECT INSERT('MYSQL',256*256*256*256,1,'DB');
+INSERT('MYSQL',256*256*256*256,1,'DB')
+MYSQL
+SELECT INSERT('MYSQL',1,11000111000111000111,'DB');
+INSERT('MYSQL',1,11000111000111000111,'DB')
+DB
+SELECT INSERT('MYSQL',1,256*256*256*256,'DB');
+INSERT('MYSQL',1,256*256*256*256,'DB')
+DB
+SELECT LOCATE('SQL','MYSQL',256*256*256*256+1);
+LOCATE('SQL','MYSQL',256*256*256*256+1)
+0
+SELECT LPAD('MY',256*256*256*256,'SQL');
+LPAD('MY',256*256*256*256,'SQL')
+NULL
+SELECT RPAD('MY',256*256*256*256,'SQL');
+RPAD('MY',256*256*256*256,'SQL')
+NULL
+SELECT REPEAT('MYSQL',256*256*256*256);
+REPEAT('MYSQL',256*256*256*256)
+NULL
+SELECT SPACE(256*256*256*256);
+SPACE(256*256*256*256)
+NULL

--- 1.83/mysql-test/t/func_str.test	2005-10-11 23:58:17 +02:00
+++ 1.84/mysql-test/t/func_str.test	2006-02-17 19:27:45 +01:00
@@ -676,4 +676,32 @@
 select format(d, 2) from t1;
 drop table t1;
 
+#
+# Bug #10963: LOCATE/LEFT/RIGHT/SUBSTR/INSERT/LPAD/RPAD/REPEAT/SPACE/
+# string functions returns wrong result with large length/offset parameter
+#
+
+SELECT LEFT('MYSQL',11000111000111000111);
+SELECT LEFT('MYSQL',256*256*256*256);
+
+SELECT RIGHT('MYSQL',11000111000111000111);
+SELECT RIGHT('MYSQL',256*256*256*256);
+
+SELECT SUBSTR('MYSQL',1,11000111000111000111);
+SELECT SUBSTR('MYSQL',1,256*256*256*256);
+
+SELECT INSERT('MYSQL',11000111000111000111,1,'DB');
+SELECT INSERT('MYSQL',256*256*256*256,1,'DB');
+SELECT INSERT('MYSQL',1,11000111000111000111,'DB');
+SELECT INSERT('MYSQL',1,256*256*256*256,'DB');
+SELECT LOCATE('SQL','MYSQL',256*256*256*256+1);
+
+--disable_warnings
+SELECT LPAD('MY',256*256*256*256,'SQL');
+SELECT RPAD('MY',256*256*256*256,'SQL');
+SELECT REPEAT('MYSQL',256*256*256*256);
+SELECT SPACE(256*256*256*256);
+--enable_warnings
+
+
 # End of 5.0 tests