bzr commit into mysql-trunk branch (alexander.barkov:3723) Bug#59648 Bug#11766519

[Alexander Barkov]

#At file:///home/bar/mysql-bzr/mysql-trunk.b59648/ based on revid:alexander.barkov@stripped

 3723 Alexander Barkov	2011-03-03
      BUG#11766519 (bug#59648): MY_STRTOLL10_MB2: ASSERTION `(*ENDPTR - S) % 2 == 0' FAILED
      
      Problem: wrong character set pointer was passed to my_strtoll10_mb2,
      which led to DBUG_ASSERT failure in some cases.
      
        @ mysql-test/r/func_encrypt_ucs2.result
        @ mysql-test/t/func_encrypt_ucs2.test
        @ mysql-test/r/ctype_ucs.result
        @ mysql-test/t/ctype_ucs.test
        Adding tests
      
        @ sql/item_func.cc
        "cs" initialization was wrong (res does not necessary point to &str_value)
      
        @ sql/item_strfunc.cc
        Item_func_dec_encrypt::val_str() and Item_func_des_descrypt::val_str()
        did not set character set for tmp_value the returned value, so the old value
        copied from args[1]->val_str() were incorrectly returned with tmp_value.

    added:
      mysql-test/r/func_encrypt_ucs2.result
      mysql-test/t/func_encrypt_ucs2.test
    modified:
      mysql-test/r/ctype_ucs.result
      mysql-test/t/ctype_ucs.test
      sql/item_func.cc
      sql/item_strfunc.cc
=== modified file 'mysql-test/r/ctype_ucs.result'
--- a/mysql-test/r/ctype_ucs.result	2011-02-10 08:47:05 +0000
+++ b/mysql-test/r/ctype_ucs.result	2011-03-03 11:23:58 +0000
@@ -1304,6 +1304,12 @@ CREATE VIEW v1 AS SELECT 1 from t1
 WHERE t1.b <=> (SELECT a FROM t1 WHERE a < SOME(SELECT '1'));
 DROP VIEW v1;
 DROP TABLE t1;
+#
+# Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+#
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850))
+00
 End of 5.0 tests
 #
 # Start of 5.5 tests

=== added file 'mysql-test/r/func_encrypt_ucs2.result'
--- a/mysql-test/r/func_encrypt_ucs2.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/func_encrypt_ucs2.result	2011-03-03 11:23:58 +0000
@@ -0,0 +1,16 @@
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)))
+9
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '\xFFT\xDCiK\x92j\xE6\xFC'
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)))
+4
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);
+CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'test'

=== modified file 'mysql-test/t/ctype_ucs.test'
--- a/mysql-test/t/ctype_ucs.test	2011-01-18 07:16:49 +0000
+++ b/mysql-test/t/ctype_ucs.test	2011-03-03 11:23:58 +0000
@@ -699,6 +699,11 @@ WHERE t1.b <=> (SELECT a FROM t1 WHERE a
 DROP VIEW v1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+
 --echo End of 5.0 tests
 
 

=== added file 'mysql-test/t/func_encrypt_ucs2.test'
--- a/mysql-test/t/func_encrypt_ucs2.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/func_encrypt_ucs2.test	2011-03-03 11:23:58 +0000
@@ -0,0 +1,12 @@
+-- source include/have_ssl_crypto_functs.inc
+-- source include/have_ucs2.inc
+
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);

=== modified file 'sql/item_func.cc'
--- a/sql/item_func.cc	2011-03-01 14:47:01 +0000
+++ b/sql/item_func.cc	2011-03-03 11:23:58 +0000
@@ -886,7 +886,7 @@ longlong Item_func_numhybrid::val_int()
       return 0;
 
     char *end= (char*) res->ptr() + res->length();
-    CHARSET_INFO *cs= str_value.charset();
+    CHARSET_INFO *cs= res->charset();
     return (*(cs->cset->strtoll10))(cs, res->ptr(), &end, &err_not_used);
   }
   default:

=== modified file 'sql/item_strfunc.cc'
--- a/sql/item_strfunc.cc	2011-02-10 08:47:05 +0000
+++ b/sql/item_strfunc.cc	2011-03-03 11:23:58 +0000
@@ -796,6 +796,7 @@ String *Item_func_des_encrypt::val_str(S
   tmp_arg[res_length-1]=tail;                   // save extra length
   tmp_value.realloc(res_length+1);
   tmp_value.length(res_length+1);
+  tmp_value.set_charset(&my_charset_bin);
   tmp_value[0]=(char) (128 | key_number);
   // Real encryption
   bzero((char*) &ivec,sizeof(ivec));
@@ -883,6 +884,7 @@ String *Item_func_des_decrypt::val_str(S
   if ((tail=(uint) (uchar) tmp_value[length-2]) > 8)
     goto wrong_key;				     // Wrong key
   tmp_value.length(length-1-tail);
+  tmp_value.set_charset(&my_charset_bin);
   return &tmp_value;
 
 error:


Attachment: [text/bzr-bundle] bzr/[e-mail]