commit into mysql-6.0-rpl branch (bar:2655) Bug#34391

[Alexander Barkov]

#At file:///home/bar/mysql-bzr/mysql-6.0.b34391/

 2655 Alexander Barkov	2008-06-07
      Bug#34391 Character sets: crash if char(), utf32, innodb
      Problem: mbminlen in dict_col_t and dtype_t uses 2 bits,
      which cannot store the value of "4" required for UTF32.
      Fix: adding macros to pack/unpack the range 1..4 to 0..3.
added:
  mysql-test/r/innodb-utf32.result
  mysql-test/t/innodb-utf32.test
modified:
  storage/innobase/data/data0type.c
  storage/innobase/dict/dict0mem.c
  storage/innobase/handler/ha_innodb.cc
  storage/innobase/include/data0type.h
  storage/innobase/include/data0type.ic
  storage/innobase/include/dict0dict.ic
  storage/innobase/include/dict0mem.h
  storage/innobase/row/row0ins.c
  storage/innobase/row/row0row.c
  storage/innobase/row/row0sel.c
  storage/innobase/row/row0upd.c

=== added file 'mysql-test/r/innodb-utf32.result'
--- a/mysql-test/r/innodb-utf32.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/innodb-utf32.result	2008-06-07 08:28:10 +0000
@@ -0,0 +1,10 @@
+drop table if exists t1;
+create table t1 (s1 char(1) character set utf32) engine=innodb;
+insert into t1 values ('a'),('b'),('c');
+select s1, hex(s1) from t1 order by s1;
+s1	hex(s1)
+a	00000061
+b	00000062
+c	00000063
+drop table t1;
+End of 6.0 tests

=== added file 'mysql-test/t/innodb-utf32.test'
--- a/mysql-test/t/innodb-utf32.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/innodb-utf32.test	2008-06-07 08:28:10 +0000
@@ -0,0 +1,16 @@
+-- source include/have_innodb.inc
+-- source include/have_utf32.inc
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+#
+# Bug#34391 Character sets: crash if char(), utf32, innodb
+#
+create table t1 (s1 char(1) character set utf32) engine=innodb;
+insert into t1 values ('a'),('b'),('c');
+select s1, hex(s1) from t1 order by s1;
+drop table t1;
+
+--echo End of 6.0 tests

=== modified file 'storage/innobase/data/data0type.c'
--- a/storage/innobase/data/data0type.c	2007-07-10 11:37:43 +0000
+++ b/storage/innobase/data/data0type.c	2008-06-07 08:28:10 +0000
@@ -197,7 +197,7 @@
 		ut_a((type->prtype & DATA_MYSQL_TYPE_MASK) < DATA_N_SYS_COLS);
 	}
 
-	ut_a(type->mbminlen <= type->mbmaxlen);
+	ut_a(dtype_unpack_mbminlen(type) <= dtype_unpack_mbmaxlen(type));
 
 	return(TRUE);
 }

=== modified file 'storage/innobase/dict/dict0mem.c'
--- a/storage/innobase/dict/dict0mem.c	2007-08-28 00:18:14 +0000
+++ b/storage/innobase/dict/dict0mem.c	2008-06-07 08:28:10 +0000
@@ -218,8 +218,8 @@
 
 	dtype_get_mblen(mtype, prtype, &mbminlen, &mbmaxlen);
 
-	col->mbminlen = (unsigned int) mbminlen;
-	col->mbmaxlen = (unsigned int) mbmaxlen;
+	dcol_pack_mbminlen(col, (unsigned int) mbminlen);
+	dcol_pack_mbmaxlen(col, (unsigned int) mbmaxlen);
 }
 
 /**************************************************************************

=== modified file 'storage/innobase/handler/ha_innodb.cc'
--- a/storage/innobase/handler/ha_innodb.cc	2008-04-01 13:19:15 +0000
+++ b/storage/innobase/handler/ha_innodb.cc	2008-06-07 08:28:10 +0000
@@ -3235,8 +3235,8 @@
 
 		templ->charset = dtype_get_charset_coll_noninline(
 				index->table->cols[i].prtype);
-		templ->mbminlen = index->table->cols[i].mbminlen;
-		templ->mbmaxlen = index->table->cols[i].mbmaxlen;
+		templ->mbminlen = dcol_unpack_mbminlen(&(index->table->cols[i]));
+		templ->mbmaxlen = dcol_unpack_mbmaxlen(&(index->table->cols[i]));
 		templ->is_unsigned = index->table->cols[i].prtype
 							& DATA_UNSIGNED;
 		if (templ->type == DATA_BLOB) {

=== modified file 'storage/innobase/include/data0type.h'
--- a/storage/innobase/include/data0type.h	2006-09-21 09:48:26 +0000
+++ b/storage/innobase/include/data0type.h	2008-06-07 08:28:10 +0000
@@ -437,12 +437,18 @@
 					the string, MySQL uses 1 or 2
 					bytes to store the string length) */
 
-	unsigned	mbminlen:2;	/* minimum length of a
+	unsigned	mb_minlen:2;	/* minimum length of a
 					character, in bytes */
-	unsigned	mbmaxlen:3;	/* maximum length of a
+	unsigned	mb_maxlen:3;	/* maximum length of a
 					character, in bytes */
 };
 
+#define dtype_unpack_mbminlen(x) ((x)->mb_minlen + 1)
+#define dtype_unpack_mbmaxlen(x) ((x)->mb_maxlen + 1)
+#define dtype_pack_mbminlen(x,l) ((x)->mb_minlen= l - 1)
+#define dtype_pack_mbmaxlen(x,l) ((x)->mb_maxlen= l - 1)
+
+
 #ifndef UNIV_NONINL
 #include "data0type.ic"
 #endif

=== modified file 'storage/innobase/include/data0type.ic'
--- a/storage/innobase/include/data0type.ic	2007-01-05 02:51:34 +0000
+++ b/storage/innobase/include/data0type.ic	2008-06-07 08:28:10 +0000
@@ -88,8 +88,9 @@
 	ulint	mbmaxlen;
 
 	dtype_get_mblen(type->mtype, type->prtype, &mbminlen, &mbmaxlen);
-	type->mbminlen = mbminlen;
-	type->mbmaxlen = mbmaxlen;
+	ut_ad(mbminlen > 0 && mbmaxlen > 0);
+	dtype_pack_mbminlen(type, mbminlen);
+	dtype_pack_mbmaxlen(type, mbmaxlen);
 
 	ut_ad(dtype_validate(type));
 }
@@ -179,7 +180,7 @@
 	const dtype_t*	type)	/* in: type */
 {
 	ut_ad(type);
-	return(type->mbminlen);
+	return(dtype_unpack_mbminlen(type));
 }
 /*************************************************************************
 Gets the maximum length of a character, in bytes. */
@@ -192,7 +193,7 @@
 	const dtype_t*	type)	/* in: type */
 {
 	ut_ad(type);
-	return(type->mbmaxlen);
+	return(dtype_unpack_mbmaxlen(type));
 }
 
 /*************************************************************************
@@ -558,5 +559,6 @@
 	const dtype_t*	type)	/* in: type */
 {
 	return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len,
-					type->mbminlen, type->mbmaxlen) > 0);
+					dtype_unpack_mbminlen(type),
+					dtype_unpack_mbmaxlen(type)) > 0);
 }

=== modified file 'storage/innobase/include/dict0dict.ic'
--- a/storage/innobase/include/dict0dict.ic	2007-07-10 14:34:21 +0000
+++ b/storage/innobase/include/dict0dict.ic	2008-06-07 08:28:10 +0000
@@ -26,8 +26,8 @@
 	type->mtype = col->mtype;
 	type->prtype = col->prtype;
 	type->len = col->len;
-	type->mbminlen = col->mbminlen;
-	type->mbmaxlen = col->mbmaxlen;
+	dtype_pack_mbminlen(type, dcol_unpack_mbminlen(col));
+	dtype_pack_mbmaxlen(type, dcol_unpack_mbmaxlen(col));
 }
 
 #ifdef UNIV_DEBUG
@@ -64,7 +64,8 @@
 	const dict_col_t*	col)	/* in: column */
 {
 	return(dtype_get_min_size_low(col->mtype, col->prtype, col->len,
-				      col->mbminlen, col->mbmaxlen));
+				      dcol_unpack_mbminlen(col),
+				      dcol_unpack_mbmaxlen(col)));
 }
 /***************************************************************************
 Returns the maximum size of the column. */
@@ -87,7 +88,8 @@
 	const dict_col_t*	col)	/* in: column */
 {
 	return(dtype_get_fixed_size_low(col->mtype, col->prtype, col->len,
-					col->mbminlen, col->mbmaxlen));
+					dcol_unpack_mbminlen(col),
+					dcol_unpack_mbmaxlen(col)));
 }
 /***************************************************************************
 Returns the ROW_FORMAT=REDUNDANT stored SQL NULL size of a column.

=== modified file 'storage/innobase/include/dict0mem.h'
--- a/storage/innobase/include/dict0mem.h	2008-03-27 01:56:18 +0000
+++ b/storage/innobase/include/dict0mem.h	2008-06-07 08:28:10 +0000
@@ -145,9 +145,9 @@
 					the string, MySQL uses 1 or 2
 					bytes to store the string length) */
 
-	unsigned	mbminlen:2;	/* minimum length of a
+	unsigned	mb_minlen:2;	/* minimum length of a
 					character, in bytes */
-	unsigned	mbmaxlen:3;	/* maximum length of a
+	unsigned	mb_maxlen:3;	/* maximum length of a
 					character, in bytes */
 	/*----------------------*/
 	/* End of definitions copied from dtype_t */
@@ -159,6 +159,11 @@
 					of an index */
 };
 
+#define dcol_unpack_mbminlen(x) ((x)->mb_minlen + 1)
+#define dcol_unpack_mbmaxlen(x) ((x)->mb_maxlen + 1)
+#define dcol_pack_mbminlen(x,l) ((x)->mb_minlen= l - 1)
+#define dcol_pack_mbmaxlen(x,l) ((x)->mb_maxlen= l - 1)
+
 /* DICT_MAX_INDEX_COL_LEN is measured in bytes and is the maximum
 indexed column length (or indexed prefix length). It is set to 3*256,
 so that one can create a column prefix index on 256 characters of a

=== modified file 'storage/innobase/row/row0ins.c'
--- a/storage/innobase/row/row0ins.c	2007-07-10 11:37:43 +0000
+++ b/storage/innobase/row/row0ins.c	2008-06-07 08:28:10 +0000
@@ -500,7 +500,8 @@
 				if (ufield->new_val.len != UNIV_SQL_NULL
 				    && dtype_get_at_most_n_mbchars(
 					col->prtype,
-					col->mbminlen, col->mbmaxlen,
+					dcol_unpack_mbminlen(col),
+					dcol_unpack_mbmaxlen(col),
 					col->len,
 					ufield->new_val.len,
 					ufield->new_val.data)
@@ -535,7 +536,7 @@
 						  parent_ufield->new_val.data,
 						  parent_ufield->new_val.len);
 
-					switch (UNIV_EXPECT(col->mbminlen,1)) {
+					switch (UNIV_EXPECT(dcol_unpack_mbminlen(col),1)) {
 					default:
 						ut_error;
 					case 1:
@@ -2226,7 +2227,9 @@
 				= dict_field_get_col(ind_field);
 
 			field->len = dtype_get_at_most_n_mbchars(
-				col->prtype, col->mbminlen, col->mbmaxlen,
+				col->prtype,
+				dcol_unpack_mbminlen(col),
+				dcol_unpack_mbmaxlen(col),
 				ind_field->prefix_len,
 				row_field->len, row_field->data);
 		} else {

=== modified file 'storage/innobase/row/row0row.c'
--- a/storage/innobase/row/row0row.c	2007-07-10 11:37:43 +0000
+++ b/storage/innobase/row/row0row.c	2008-06-07 08:28:10 +0000
@@ -146,7 +146,9 @@
 		    && dfield_get_len(dfield2) != UNIV_SQL_NULL) {
 
 			storage_len = dtype_get_at_most_n_mbchars(
-				col->prtype, col->mbminlen, col->mbmaxlen,
+				col->prtype,
+				dcol_unpack_mbminlen(col),
+				dcol_unpack_mbmaxlen(col),
 				ind_field->prefix_len,
 				dfield_get_len(dfield2), dfield2->data);
 
@@ -426,8 +428,8 @@
 				dfield_set_len(dfield,
 					       dtype_get_at_most_n_mbchars(
 						       dtype->prtype,
-						       dtype->mbminlen,
-						       dtype->mbmaxlen,
+						       dtype_unpack_mbminlen(dtype),
+						       dtype_unpack_mbmaxlen(dtype),
 						       clust_col_prefix_len,
 						       len, (char*) field));
 			}
@@ -530,8 +532,8 @@
 				dfield_set_len(dfield,
 					       dtype_get_at_most_n_mbchars(
 						       dtype->prtype,
-						       dtype->mbminlen,
-						       dtype->mbmaxlen,
+						       dtype_unpack_mbminlen(dtype),
+						       dtype_unpack_mbmaxlen(dtype),
 						       clust_col_prefix_len,
 						       len, (char*) field));
 			}
@@ -591,7 +593,9 @@
 		    && dfield->len != UNIV_SQL_NULL) {
 
 			dfield->len = dtype_get_at_most_n_mbchars(
-				col->prtype, col->mbminlen, col->mbmaxlen,
+				col->prtype,
+				dcol_unpack_mbminlen(col),
+				dcol_unpack_mbmaxlen(col),
 				field->prefix_len, dfield->len, dfield->data);
 		}
 	}

=== modified file 'storage/innobase/row/row0sel.c'
--- a/storage/innobase/row/row0sel.c	2008-04-01 13:19:15 +0000
+++ b/storage/innobase/row/row0sel.c	2008-06-07 08:28:10 +0000
@@ -107,7 +107,9 @@
 		if (ifield->prefix_len > 0 && clust_len != UNIV_SQL_NULL) {
 
 			clust_len = dtype_get_at_most_n_mbchars(
-				col->prtype, col->mbminlen, col->mbmaxlen,
+				col->prtype,
+				dcol_unpack_mbminlen(col),
+				dcol_unpack_mbmaxlen(col),
 				ifield->prefix_len,
 				clust_len, (char*) clust_field);
 		}

=== modified file 'storage/innobase/row/row0upd.c'
--- a/storage/innobase/row/row0upd.c	2007-01-05 02:51:34 +0000
+++ b/storage/innobase/row/row0upd.c	2008-06-07 08:28:10 +0000
@@ -929,8 +929,8 @@
 					dfield->len
 						= dtype_get_at_most_n_mbchars(
 							col->prtype,
-							col->mbminlen,
-							col->mbmaxlen,
+							dcol_unpack_mbminlen(col),
+							dcol_unpack_mbmaxlen(col),
 							field->prefix_len,
 							new_val->len,
 							new_val->data);
@@ -1005,8 +1005,8 @@
 					dfield->len
 						= dtype_get_at_most_n_mbchars(
 							col->prtype,
-							col->mbminlen,
-							col->mbmaxlen,
+							dcol_unpack_mbminlen(col),
+							dcol_unpack_mbmaxlen(col),
 							field->prefix_len,
 							new_val->len,
 							new_val->data);