bzr push into mysql-5.1 branch (alexander.barkov:3607 to 3608) Bug#59648 Bug#11766519

[Alexander Barkov]

 3608 Alexander Barkov	2011-03-03
      BUG#11766519 (bug#59648): MY_STRTOLL10_MB2: ASSERTION `(*ENDPTR - S) % 2 == 0' FAILED
            
      Problem: wrong character set pointer was passed to my_strtoll10_mb2,
      which led to DBUG_ASSERT failure in some cases.
      
        @ mysql-test/r/func_encrypt_ucs2.result
        @ mysql-test/t/func_encrypt_ucs2.test
        @ mysql-test/r/ctype_ucs.result
        @ mysql-test/t/ctype_ucs.test
        Adding tests
      
        @ sql/item_func.cc
        "cs" initialization was wrong (res does not necessarily point to &str_value)
      
        @ sql/item_strfunc.cc
        Item_func_dec_encrypt::val_str() and Item_func_des_descrypt::val_str()
        did not set character set for tmp_value (the returned value),
        so the old value, which was previously copied from args[1]->val_str(),
        was incorrectly returned with tmp_value.

    added:
      mysql-test/r/func_encrypt_ucs2.result
      mysql-test/t/func_encrypt_ucs2.test
    modified:
      mysql-test/r/ctype_ucs.result
      mysql-test/t/ctype_ucs.test
      sql/item_func.cc
      sql/item_strfunc.cc
 3607 Alexander Barkov	2011-03-03
      An after-fix for
      Bug#11765108 - Bug#58036: CLIENT UTF32, UTF16, UCS2 SHOULD BE DISALLOWED, THEY CRASH SERVER 
      
      Fixing wrong usage of DBUG_ASSERT.
      In non-debug version thd_init_client_charset
      was not executed at all.

    modified:
      sql/sql_parse.cc
=== modified file 'mysql-test/r/ctype_ucs.result'
--- a/mysql-test/r/ctype_ucs.result	2010-04-06 07:26:59 +0000
+++ b/mysql-test/r/ctype_ucs.result	2011-03-03 12:04:04 +0000
@@ -1238,4 +1238,10 @@ CREATE VIEW v1 AS SELECT 1 from t1
 WHERE t1.b <=> (SELECT a FROM t1 WHERE a < SOME(SELECT '1'));
 DROP VIEW v1;
 DROP TABLE t1;
+#
+# Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+#
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850))
+00
 End of 5.0 tests

=== added file 'mysql-test/r/func_encrypt_ucs2.result'
--- a/mysql-test/r/func_encrypt_ucs2.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/func_encrypt_ucs2.result	2011-03-03 12:04:04 +0000
@@ -0,0 +1,19 @@
+#
+# Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+#
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)))
+9
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '?T?iK?j??'
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)))
+4
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);
+CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED)
+0
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: 'test'

=== modified file 'mysql-test/t/ctype_ucs.test'
--- a/mysql-test/t/ctype_ucs.test	2010-04-06 07:26:59 +0000
+++ b/mysql-test/t/ctype_ucs.test	2011-03-03 12:04:04 +0000
@@ -741,4 +741,9 @@ WHERE t1.b <=> (SELECT a FROM t1 WHERE a
 DROP VIEW v1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+SELECT HEX(CHAR(COALESCE(NULL, CHAR(COUNT('%s') USING ucs2), 1, @@global.license, NULL) USING cp850));
+
 --echo End of 5.0 tests

=== added file 'mysql-test/t/func_encrypt_ucs2.test'
--- a/mysql-test/t/func_encrypt_ucs2.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/func_encrypt_ucs2.test	2011-03-03 12:04:04 +0000
@@ -0,0 +1,12 @@
+-- source include/have_ssl.inc
+-- source include/have_ucs2.inc
+
+--echo #
+--echo # Bug#59648 my_strtoll10_mb2: Assertion `(*endptr - s) % 2 == 0' failed.
+--echo #
+
+SELECT CHAR_LENGTH(DES_ENCRYPT(0, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_ENCRYPT(0, CHAR('1' USING ucs2)),UNSIGNED);
+
+SELECT CHAR_LENGTH(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' USING ucs2)));
+SELECT CONVERT(DES_DECRYPT(0xFF0DC9FC9537CA75F4, CHAR('1' using ucs2)), UNSIGNED);

=== modified file 'sql/item_func.cc'
--- a/sql/item_func.cc	2011-02-22 21:03:32 +0000
+++ b/sql/item_func.cc	2011-03-03 12:04:04 +0000
@@ -840,7 +840,7 @@ longlong Item_func_numhybrid::val_int()
       return 0;
 
     char *end= (char*) res->ptr() + res->length();
-    CHARSET_INFO *cs= str_value.charset();
+    CHARSET_INFO *cs= res->charset();
     return (*(cs->cset->strtoll10))(cs, res->ptr(), &end, &err_not_used);
   }
   default:

=== modified file 'sql/item_strfunc.cc'
--- a/sql/item_strfunc.cc	2011-01-17 12:11:33 +0000
+++ b/sql/item_strfunc.cc	2011-03-03 12:04:04 +0000
@@ -519,6 +519,7 @@ String *Item_func_des_encrypt::val_str(S
   tmp_arg[res_length-1]=tail;                   // save extra length
   tmp_value.realloc(res_length+1);
   tmp_value.length(res_length+1);
+  tmp_value.set_charset(&my_charset_bin);
   tmp_value[0]=(char) (128 | key_number);
   // Real encryption
   bzero((char*) &ivec,sizeof(ivec));
@@ -606,6 +607,7 @@ String *Item_func_des_decrypt::val_str(S
   if ((tail=(uint) (uchar) tmp_value[length-2]) > 8)
     goto wrong_key;				     // Wrong key
   tmp_value.length(length-1-tail);
+  tmp_value.set_charset(&my_charset_bin);
   return &tmp_value;
 
 error:

No bundle (reason: useless for push emails).