List:Maria Storage Engine« Previous MessageNext Message »
From:Michael Widenius Date:June 28 2008 12:45pm
Subject:bzr commit into MySQL/Maria:mysql-maria branch (monty:2647) Bug#37007
View as plain text  
#At bzr+ssh://bk-internal.mysql.com/bzrroot/server/mysql-maria/

 2647 Michael Widenius	2008-06-28
      Fix for Bug #37007 Maria: different checksum for MyISAM table depending on CHECKSUM=0|1
      This also adds a check that MyISAM tables with incompatible checksums are detected by CHECK TABLE ... [FOR UPGRADE] and thus also by mysql_upgrade.
      The tables that are incomatible are MyISAM tables with ROW_FORMAT=fixed and has VARCHAR fields and have CHECKSUM enabled.
      Before these tables gave different checksum if you used CHECK TABLE with or without EXTENDED
added:
  mysql-test/r/row-checksum-old.result
  mysql-test/r/row-checksum.result
  mysql-test/t/row-checksum-old-master.opt
  mysql-test/t/row-checksum-old.test
  mysql-test/t/row-checksum.test
modified:
  mysql-test/r/old-mode.result
  mysql-test/t/old-mode.test
  sql/ha_partition.cc
  sql/handler.cc
  sql/handler.h
  sql/sql_show.cc
  sql/sql_table.cc
  storage/maria/ha_maria.cc
  storage/myisam/ha_myisam.cc
  storage/myisam/ha_myisam.h
  storage/myisam/mi_open.c
  storage/myisam/myisamdef.h

per-file messages:
  mysql-test/r/old-mode.result
    Now we get same results with and without EXTENDED
  mysql-test/r/row-checksum-old.result
    Initial results
  mysql-test/r/row-checksum.result
    Initial results
  mysql-test/t/old-mode.test
    Added test with QUICK to show that the live checksum is not used when running with --old
  mysql-test/t/row-checksum-old-master.opt
    Start mysqld with --old mode to enable old checksum code
  mysql-test/t/row-checksum-old.test
    Run row-checksum test under mysqld --old
  mysql-test/t/row-checksum.test
    Verify that checksum are calculated the same way with and without EXTENDED
    We run this with several storage engines to ensure results are the same over storage engines
  sql/ha_partition.cc
    Use new HA_HAS_xxx_CHECKSUM flags
  sql/handler.cc
    Use new HA_HAS_xxx_CHECKSUM flags
  sql/handler.h
    Split HA_HAS_CHECKSUM into HA_HAS_NEW_CHECKSUM and HA_HAS_OLD_CHECKSUM flags.
    This is a safe API change as only MyISAM and Maria should use these handler flags.
  sql/sql_show.cc
    Use new HA_HAS_xxx_CHECKSUM flags
  sql/sql_table.cc
    Use file->checksum() for live checksums if the life checksum method corresponds to the mysqld --old flag
  storage/maria/ha_maria.cc
    Use new HA_HAS_xxx_CHECKSUM flags
  storage/myisam/ha_myisam.cc
    Set HA_HAS_OLD_CHECKSUM and/or HA_HAS_NEW_CHECKSUM flags depending on if this is a new myisam or old myisam file
    Add method check_for_upgrade() to detect if the table is of old version with a checksum that is incompatible with CHECK TABLE ... EXTENDED
  storage/myisam/ha_myisam.h
    Added check_for_upgrade()
  storage/myisam/mi_open.c
    Removed not neede cast
    Initialize share->has_null_fields and share->has_varchar_fields variables
  storage/myisam/myisamdef.h
    Added share->has_null_fields and share->has_varchar_fields
=== modified file 'mysql-test/r/old-mode.result'
--- a/mysql-test/r/old-mode.result	2007-12-04 21:23:42 +0000
+++ b/mysql-test/r/old-mode.result	2008-06-28 12:45:15 +0000
@@ -5,8 +5,12 @@ insert t1 values (1, "aaa", "bbb"), (NUL
 insert t2 select * from t1;
 checksum table t1, t2;
 Table	Checksum
-test.t1	3442722830
+test.t1	2948697075
 test.t2	2948697075
+checksum table t1, t2 quick;
+Table	Checksum
+test.t1	NULL
+test.t2	NULL
 checksum table t1, t2 extended;
 Table	Checksum
 test.t1	2948697075

=== added file 'mysql-test/r/row-checksum-old.result'
--- a/mysql-test/r/row-checksum-old.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/row-checksum-old.result	2008-06-28 12:45:15 +0000
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	452555338
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	452555338
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	4108368782
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	4108368782
+drop table t1;

=== added file 'mysql-test/r/row-checksum.result'
--- a/mysql-test/r/row-checksum.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/r/row-checksum.result	2008-06-28 12:45:15 +0000
@@ -0,0 +1,85 @@
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	229851577
+checksum table t1 quick;
+Table	Checksum
+test.t1	229851577
+checksum table t1 extended;
+Table	Checksum
+test.t1	229851577
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+Table	Checksum
+test.t1	3885665021
+checksum table t1 quick;
+Table	Checksum
+test.t1	NULL
+checksum table t1 extended;
+Table	Checksum
+test.t1	3885665021
+drop table t1;

=== modified file 'mysql-test/t/old-mode.test'
--- a/mysql-test/t/old-mode.test	2007-12-04 21:23:42 +0000
+++ b/mysql-test/t/old-mode.test	2008-06-28 12:45:15 +0000
@@ -12,5 +12,6 @@ create table t2 (a int, b varchar(200), 
 insert t1 values (1, "aaa", "bbb"), (NULL, "", "ccccc"), (0, NULL, "");
 insert t2 select * from t1;
 checksum table t1, t2;
+checksum table t1, t2 quick;
 checksum table t1, t2 extended;
 drop table t1,t2;

=== added file 'mysql-test/t/row-checksum-old-master.opt'
--- a/mysql-test/t/row-checksum-old-master.opt	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/row-checksum-old-master.opt	2008-06-28 12:45:15 +0000
@@ -0,0 +1 @@
+--old

=== added file 'mysql-test/t/row-checksum-old.test'
--- a/mysql-test/t/row-checksum-old.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/row-checksum-old.test	2008-06-28 12:45:15 +0000
@@ -0,0 +1,4 @@
+#
+# Run row-checksum.test with old mode
+#
+--source t/row-checksum.test

=== added file 'mysql-test/t/row-checksum.test'
--- a/mysql-test/t/row-checksum.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/t/row-checksum.test	2008-06-28 12:45:15 +0000
@@ -0,0 +1,62 @@
+#
+# Test checksum
+#
+
+-- source include/have_innodb.inc
+-- source include/have_maria.inc
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+
+create table t1 (a int null, v varchar(100)) engine=maria checksum=0;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+create table t1 (a int null, v varchar(100)) engine=maria checksum=1;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;
+
+
+#
+# These checksums will be different prefixes fixed sizes rows with one extra
+# flag byte
+#
+create table t1 (a int null, v varchar(100)) engine=myisam checksum=1 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table if exists t1;
+
+create table t1 (a int null, v varchar(100)) engine=innodb checksum=0 row_format=fixed;
+insert into t1 values(null, null), (1, "hello");
+checksum table t1;
+checksum table t1 quick;
+checksum table t1 extended;
+drop table t1;

=== modified file 'sql/ha_partition.cc'
--- a/sql/ha_partition.cc	2008-05-29 18:39:25 +0000
+++ b/sql/ha_partition.cc	2008-06-28 12:45:15 +0000
@@ -4615,7 +4615,7 @@ void ha_partition::get_dynamic_partition
   stat_info->update_time=          file->stats.update_time;
   stat_info->check_time=           file->stats.check_time;
   stat_info->check_sum= 0;
-  if (file->ha_table_flags() & HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
     stat_info->check_sum= file->checksum();
   return;
 }

=== modified file 'sql/handler.cc'
--- a/sql/handler.cc	2008-05-29 18:39:25 +0000
+++ b/sql/handler.cc	2008-06-28 12:45:15 +0000
@@ -3435,7 +3435,7 @@ void handler::get_dynamic_partition_info
   stat_info->update_time=          stats.update_time;
   stat_info->check_time=           stats.check_time;
   stat_info->check_sum=            0;
-  if (table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_OLD_CHECKSUM))
     stat_info->check_sum= checksum();
   return;
 }

=== modified file 'sql/handler.h'
--- a/sql/handler.h	2008-05-29 18:39:25 +0000
+++ b/sql/handler.h	2008-06-28 12:45:15 +0000
@@ -107,7 +107,8 @@
 #define HA_CAN_FULLTEXT        (1 << 21)
 #define HA_CAN_SQL_HANDLER     (1 << 22)
 #define HA_NO_AUTO_INCREMENT   (1 << 23)
-#define HA_HAS_CHECKSUM        (1 << 24)
+/* Has automatic checksums and uses the old checksum format */
+#define HA_HAS_OLD_CHECKSUM    (1 << 24)
 /* Table data are stored in separate files (for lower_case_table_names) */
 #define HA_FILE_BASED	       (1 << 26)
 #define HA_NO_VARCHAR	       (1 << 27)
@@ -124,6 +125,8 @@
 */
 #define HA_BINLOG_ROW_CAPABLE  (LL(1) << 34)
 #define HA_BINLOG_STMT_CAPABLE (LL(1) << 35)
+/* Has automatic checksums and uses the new checksum format */
+#define HA_HAS_NEW_CHECKSUM    (LL(1) << 36)
 
 /*
   Set of all binlog flags. Currently only contain the capabilities

=== modified file 'sql/sql_show.cc'
--- a/sql/sql_show.cc	2008-06-03 19:35:39 +0000
+++ b/sql/sql_show.cc	2008-06-28 12:45:15 +0000
@@ -3613,7 +3613,7 @@ static int get_schema_tables_record(THD 
         table->field[16]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
         table->field[16]->set_notnull();
       }
-      if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+      if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
       {
         table->field[18]->store((longlong) file->checksum(), TRUE);
         table->field[18]->set_notnull();
@@ -4670,7 +4670,7 @@ static void store_schema_partitions_reco
     table->field[20]->store_time(&time, MYSQL_TIMESTAMP_DATETIME);
     table->field[20]->set_notnull();
   }
-  if (file->ha_table_flags() & (ulong) HA_HAS_CHECKSUM)
+  if (file->ha_table_flags() & (HA_HAS_OLD_CHECKSUM | HA_HAS_NEW_CHECKSUM))
   {
     table->field[21]->store((longlong) stat_info.check_sum, TRUE);
     table->field[21]->set_notnull();

=== modified file 'sql/sql_table.cc'
--- a/sql/sql_table.cc	2008-06-11 20:45:14 +0000
+++ b/sql/sql_table.cc	2008-06-28 12:45:15 +0000
@@ -7287,11 +7287,14 @@ bool mysql_checksum_table(THD *thd, TABL
     }
     else
     {
-      if (t->file->ha_table_flags() & HA_HAS_CHECKSUM &&
-	  !(check_opt->flags & T_EXTEND))
+      /* Call ->checksum() if the table checksum matches 'old_mode' settings */
+      if (!(check_opt->flags & T_EXTEND) &&
+          (((t->file->ha_table_flags() & HA_HAS_OLD_CHECKSUM) &&
+            thd->variables.old_mode) ||
+           ((t->file->ha_table_flags() & HA_HAS_NEW_CHECKSUM) &&
+            !thd->variables.old_mode)))
 	protocol->store((ulonglong)t->file->checksum());
-      else if (!(t->file->ha_table_flags() & HA_HAS_CHECKSUM) &&
-	       (check_opt->flags & T_QUICK))
+      else if (check_opt->flags & T_QUICK)
 	protocol->store_null();
       else
       {

=== modified file 'storage/maria/ha_maria.cc'
--- a/storage/maria/ha_maria.cc	2008-06-28 08:27:14 +0000
+++ b/storage/maria/ha_maria.cc	2008-06-28 12:45:15 +0000
@@ -923,7 +923,7 @@ int ha_maria::open(const char *name, int
     int_table_flags|= HA_CAN_INSERT_DELAYED;
   }
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
-    int_table_flags |= HA_HAS_CHECKSUM;
+    int_table_flags |= HA_HAS_NEW_CHECKSUM;
 
   for (i= 0; i < table->s->keys; i++)
   {

=== modified file 'storage/myisam/ha_myisam.cc'
--- a/storage/myisam/ha_myisam.cc	2008-06-28 08:27:14 +0000
+++ b/storage/myisam/ha_myisam.cc	2008-06-28 12:45:15 +0000
@@ -690,7 +690,19 @@ int ha_myisam::open(const char *name, in
   if (!table->s->db_record_offset)
     int_table_flags|=HA_REC_NOT_IN_SEQ;
   if (file->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
-    int_table_flags|=HA_HAS_CHECKSUM;
+  {
+    /*
+      Set which type of automatic checksum we have
+      The old checksum and new checksum are identical if there is no
+      null fields.
+      Files with new checksum has the HA_OPTION_NULL_FIELDS bit set.
+    */      
+    if ((file->s->options & HA_OPTION_NULL_FIELDS) ||
+        !file->s->has_null_fields)
+      int_table_flags|= HA_HAS_NEW_CHECKSUM;
+    if (!(file->s->options & HA_OPTION_NULL_FIELDS))
+      int_table_flags|= HA_HAS_OLD_CHECKSUM;
+  }
 
   for (i= 0; i < table->s->keys; i++)
   {
@@ -2042,6 +2054,27 @@ bool ha_myisam::check_if_incompatible_da
   return COMPATIBLE_DATA_YES;
 }
 
+
+/**
+  Check if a table is incompatible with the current version.
+
+  The cases are:
+  - Table has checksum, varchars and are not of dynamic record type
+*/
+
+int ha_myisam::check_for_upgrade(HA_CHECK_OPT *check_opt)
+{
+  if (!(file->s->options & HA_OPTION_NULL_FIELDS) &&
+      !(file->s->options & HA_OPTION_PACK_RECORD) &&
+      file->s->has_varchar_fields)
+  {
+    /* We need alter there to get the HA_OPTION_NULL_FIELDS flag to be set */
+    return HA_ADMIN_NEEDS_ALTER;
+  }
+  return HA_ADMIN_OK;
+}
+
+
 extern int mi_panic(enum ha_panic_function flag);
 int myisam_panic(handlerton *hton, ha_panic_function flag)
 {

=== modified file 'storage/myisam/ha_myisam.h'
--- a/storage/myisam/ha_myisam.h	2008-05-29 15:33:33 +0000
+++ b/storage/myisam/ha_myisam.h	2008-06-28 12:45:15 +0000
@@ -119,6 +119,7 @@ class ha_myisam: public handler
                                   ulonglong *nb_reserved_values);
   int rename_table(const char * from, const char * to);
   int delete_table(const char *name);
+  int check_for_upgrade(HA_CHECK_OPT *check_opt);
   int check(THD* thd, HA_CHECK_OPT* check_opt);
   int analyze(THD* thd,HA_CHECK_OPT* check_opt);
   int repair(THD* thd, HA_CHECK_OPT* check_opt);

=== modified file 'storage/myisam/mi_open.c'
--- a/storage/myisam/mi_open.c	2008-04-28 16:24:05 +0000
+++ b/storage/myisam/mi_open.c	2008-06-28 12:45:15 +0000
@@ -19,6 +19,7 @@
 #include "sp_defs.h"
 #include "rt_index.h"
 #include <m_ctype.h>
+#include <mysql_version.h>
 
 #if defined(MSDOS) || defined(__WIN__)
 #ifdef __WIN__
@@ -453,13 +454,20 @@ MI_INFO *mi_open(const char *name, int m
       share->rec[i].pack_type=0;
       share->rec[i].huff_tree=0;
       share->rec[i].offset=offset;
-      if (share->rec[i].type == (int) FIELD_BLOB)
+      if (share->rec[i].type == FIELD_BLOB)
       {
 	share->blobs[j].pack_length=
 	  share->rec[i].length - portable_sizeof_char_ptr;
 	share->blobs[j].offset=offset;
 	j++;
       }
+#if MYSQL_VERSION_ID <= 60100
+      /* This is to detect old checksum option */
+      if (share->rec[i].null_bit)
+        share->has_null_fields= 1;
+      if (share->rec[i].type == FIELD_VARCHAR)
+        share->has_varchar_fields= 1;
+#endif
       offset+=share->rec[i].length;
     }
     share->rec[i].type=(int) FIELD_LAST;	/* End marker */

=== modified file 'storage/myisam/myisamdef.h'
--- a/storage/myisam/myisamdef.h	2008-05-29 18:39:25 +0000
+++ b/storage/myisam/myisamdef.h	2008-06-28 12:45:15 +0000
@@ -210,6 +210,9 @@ typedef struct st_mi_isam_share
   enum data_file_type data_file_type;
   /* Below flag is needed to make log tables work with concurrent insert */
   my_bool is_log_table;
+  /* This is 1 if they table checksum is of old type */
+  my_bool has_null_fields;
+  my_bool has_varchar_fields;
 
   my_bool changed,                      /* If changed since lock */
     global_changed,                     /* If changed since open */

Thread
bzr commit into MySQL/Maria:mysql-maria branch (monty:2647) Bug#37007Michael Widenius28 Jun