List:Commits« Previous MessageNext Message »
From:John David Duncan Date:September 28 2011 9:58pm
Subject:bzr push into mysql-5.5-cluster branch (john.duncan:3563 to 3568)
View as plain text  
 3568 John David Duncan	2011-09-28
      any line in compiler_warnings.supp must start with .* to be treated as a regex

    modified:
      support-files/compiler_warnings.supp
 3567 John David Duncan	2011-09-28
      small fix

    modified:
      storage/ndb/memcache/src/Record.cc
 3566 John David Duncan	2011-09-28
      Part 2 of alignment change: float, double, and datetime are now aligned within the Record buffer.
      Call NdbDictionary::Column::getSize() on any non-string column, and if size is 2,4,8, or 16, add padding to align it to its size.

    modified:
      storage/ndb/memcache/include/DataTypeHandler.h
      storage/ndb/memcache/src/DataTypeHandler.cc
      storage/ndb/memcache/src/Record.cc
 3565 John David Duncan	2011-09-28
      This is a new approach to alignment issues.
      Rather than reading and writing unaligned values, add padding in Record so that integer columns are always aligned.
      In this patch, integers are always aligned to their sizes. 
      Doubles and floats are not padded in the record and are treated by DataTypeHandler as potentially unaligned.
      Time and Date types may not be correctly handled here (they may be unpadded, yet read & written as ints), so another patch is needed to address them. 

    modified:
      storage/ndb/memcache/include/DataTypeHandler.h
      storage/ndb/memcache/include/Record.h
      storage/ndb/memcache/sandbox.sh.in
      storage/ndb/memcache/src/DataTypeHandler.cc
      storage/ndb/memcache/src/Record.cc
 3564 John David Duncan	2011-09-27
      Solaris printf() segfaults on null string pointers :(

    modified:
      storage/ndb/memcache/src/ClusterConnectionPool.cc
 3563 John David Duncan	2011-09-27
      Removed errant assertion (the Ndb's custom data pointer holds an NdbInstance, not a workitem)

    modified:
      storage/ndb/memcache/src/ndb_worker.cc
=== modified file 'storage/ndb/memcache/include/DataTypeHandler.h'
--- a/storage/ndb/memcache/include/DataTypeHandler.h	2011-09-27 07:53:52 +0000
+++ b/storage/ndb/memcache/include/DataTypeHandler.h	2011-09-28 21:43:28 +0000
@@ -26,28 +26,37 @@
 
 #include "ndbmemcache_global.h"
 
-// FOR INTEGER TYPES: x86 allows unaligned access, but most other machines do not.
-// FOR FLOATING POINT TYPES: access must be aligned on all architectures
-#define LOAD_UNALIGNED(Type, x, buf) \
+/* Assign x := *buf, assuming correct alignment */
+#define LOAD_ALIGNED_DATA(Type, x, buf) \
 Type x = *((Type *) buf);
 
-#define STORE_UNALIGNED(Type, x, buf) \
+/* Assign *buf := x, assuming correct alignment */
+#define STORE_ALIGNED_DATA(Type, x, buf) \
 *((Type *) buf) = (Type) x;
 
-#define LOAD_ALIGNED(Type, x, buf) \
+/* Assign x := *buf */
+#define ALIGN_AND_LOAD(Type, x, buf) \
 Type x; \
 memcpy(&x, buf, sizeof(x));
 
-#define STORE_ALIGNED(Type, x, buf) \
+/* Assign *buf := x */
+#define ALIGN_AND_STORE(Type, x, buf) \
 Type tmp_value = (Type) x; \
 memcpy(buf, &tmp_value, sizeof(tmp_value));
 
+/* FOR INTEGER TYPES, x86 allows unaligned access, but most other machines do not.
+   (FOR FLOATING POINT TYPES: access must be aligned on all architectures).
+   Wherever in the code there is a LOAD_ALIGNED_DATA macro, we assume the record  
+   has been laid out with necessary padding for alignment.  But if you ever get 
+   an alignment error (e.g. Bus Error on Sparc), you can replace LOAD_ALIGNED_DATA
+   with LOAD_FOR ARCHITECTURE.
+*/
 #if defined(__i386) || defined(__x86_64)
-#define LOAD_FOR_ARCHITECTURE LOAD_UNALIGNED
-#define STORE_FOR_ARCHITECTURE STORE_UNALIGNED 
+#define LOAD_FOR_ARCHITECTURE LOAD_ALIGNED_DATA
+#define STORE_FOR_ARCHITECTURE STORE_ALIGNED_DATA
 #else
-#define LOAD_FOR_ARCHITECTURE LOAD_ALIGNED
-#define STORE_FOR_ARCHITECTURE STORE_ALIGNED
+#define LOAD_FOR_ARCHITECTURE ALIGN_AND_LOAD
+#define STORE_FOR_ARCHITECTURE ALIGN_AND_STORE
 #endif
 
 

=== modified file 'storage/ndb/memcache/include/Record.h'
--- a/storage/ndb/memcache/include/Record.h	2011-09-12 10:05:07 +0000
+++ b/storage/ndb/memcache/include/Record.h	2011-09-28 20:28:34 +0000
@@ -99,6 +99,7 @@ class Record {
   /* Private methods */
   const Record & operator=(const Record &) const;
   void build_null_bitmap();
+  void pad_offset_for_alignment();
 };
 
 

=== modified file 'storage/ndb/memcache/sandbox.sh.in'
--- a/storage/ndb/memcache/sandbox.sh.in	2011-09-23 01:44:59 +0000
+++ b/storage/ndb/memcache/sandbox.sh.in	2011-09-28 20:28:34 +0000
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# FIXME:  Solaris doesn't have whoami and "kill" doesn't work as used here.
+
 prefix=@CMAKE_INSTALL_PREFIX@
 bindir=@INSTALL_BINDIR@
 libexecdir=@INSTALL_SBINDIR@

=== modified file 'storage/ndb/memcache/src/ClusterConnectionPool.cc'
--- a/storage/ndb/memcache/src/ClusterConnectionPool.cc	2011-09-22 18:27:10 +0000
+++ b/storage/ndb/memcache/src/ClusterConnectionPool.cc	2011-09-28 04:55:30 +0000
@@ -96,6 +96,7 @@ ClusterConnectionPool::ClusterConnection
 Ndb_cluster_connection * ClusterConnectionPool::connect(const char *connectstring) {
   DEBUG_ENTER_METHOD("ClusterConnectionPool::connect");
   int conn_retries = 0;
+  if(connectstring == 0) connectstring = "";
   Ndb_cluster_connection *c = new Ndb_cluster_connection(connectstring);
   
   /* Set name that appears in the cluster log file */

=== modified file 'storage/ndb/memcache/src/DataTypeHandler.cc'
--- a/storage/ndb/memcache/src/DataTypeHandler.cc	2011-09-27 07:53:52 +0000
+++ b/storage/ndb/memcache/src/DataTypeHandler.cc	2011-09-28 21:43:28 +0000
@@ -133,12 +133,13 @@ impl_writeToNdb dth_encode_decimal;
 NumericHandler dth_native_int8   = { dth_read32<Int8>,  dth_write32<Int8>  };
 NumericHandler dth_native_int16  = { dth_read32<Int16>, dth_write32<Int16> };
 NumericHandler dth_native_int32  = { dth_read32<Int32>, dth_write32<Int32> };
-NumericHandler dth_native_uint8  = { dth_read32<Uint8>,  dth_write32<Uint8>  };
-NumericHandler dth_native_uint16 = { dth_read32<Uint16>, dth_write32<Uint16> };
-NumericHandler dth_native_uint32 = { dth_read32<Uint32>, dth_write32<Uint32> };
+NumericHandler dth_native_uint8  = { dth_read32<Uint8>,  dth_write32<Uint8> };
+NumericHandler dth_native_uint16 = { dth_read32<Uint16>, dth_write32<Uint16>};
+NumericHandler dth_native_uint32 = { dth_read32<Uint32>, dth_write32<Uint32>};
 NumericHandler dth_native_year   = { dth_read32_year, dth_write32_year };
 NumericHandler dth_native_medium = { dth_read32_medium, dth_write32_medium };
-NumericHandler dth_native_medium_unsigned = { dth_read32_medium_unsigned, dth_write32_medium_unsigned };
+NumericHandler dth_native_medium_unsigned = 
+                 { dth_read32_medium_unsigned, dth_write32_medium_unsigned };
 
 
 /***** Singleton Handlers *****/
@@ -218,7 +219,7 @@ DataTypeHandler Handler_Bigint = {
   dth_decode_bigint,
   dth_length_s<Int64>,
   dth_encode_bigint,
-  0,
+  0, 
   false
 };
 
@@ -544,7 +545,7 @@ int dth_encode_char(const NdbDictionary:
 
 template<typename INTTYPE> size_t dth_length_s(const NdbDictionary::Column *,
                                                const void *buf) {
-  LOAD_FOR_ARCHITECTURE(INTTYPE, i, buf);
+  LOAD_ALIGNED_DATA(INTTYPE, i, buf);
   size_t len = (i < 0) ? 2 : 1;  
   for( ; i > 0 ; len++) i = i / 10;
   return len;  
@@ -552,7 +553,7 @@ template<typename INTTYPE> size_t dth_le
 
 template<typename INTTYPE> size_t dth_length_u(const NdbDictionary::Column *,
                                                const void *buf) {
-  LOAD_FOR_ARCHITECTURE(INTTYPE, i, buf);
+  LOAD_ALIGNED_DATA(INTTYPE, i, buf);
   size_t len = 1;
   for( ; i > 0 ; len++) i = i / 10;
   return len;  
@@ -560,14 +561,14 @@ template<typename INTTYPE> size_t dth_le
 
 /* read32: read the value from the buffer into an int32 */
 template<typename INTTYPE> int dth_read32(Int32 &result, const void * const buf) {
-  LOAD_FOR_ARCHITECTURE(INTTYPE, i, buf);
+  LOAD_ALIGNED_DATA(INTTYPE, i, buf);
   result = (Int32) i;
   return 1;
 }
 
 /* write32: write an int32 into the buffer */
 template<typename INTTYPE> int dth_write32(Int32 value, void *buf) {
-  STORE_FOR_ARCHITECTURE(INTTYPE, value, buf);
+  STORE_ALIGNED_DATA(INTTYPE, value, buf);
   return 1;
 }
 
@@ -628,7 +629,7 @@ int dth_encode_tiny_unsigned(const NdbDi
 /***** SMALLINT ******/
 int dth_decode_smallint(const NdbDictionary::Column *col,
                         char * &str, const void *buf) {  
-  LOAD_FOR_ARCHITECTURE(Int16, shortval, buf);
+  LOAD_ALIGNED_DATA(Int16, shortval, buf);
   return sprintf(str, "%hd", shortval);
 }
 
@@ -640,7 +641,7 @@ int dth_encode_smallint(const NdbDiction
   if((! safe_strtol(copy_buff, &intval)) || intval > 32767 || intval < -32768) {
     return DTH_NUMERIC_OVERFLOW;
   }
-  STORE_FOR_ARCHITECTURE(Int16, intval, buf);
+  STORE_ALIGNED_DATA(Int16, intval, buf);
   
   return len;
 }
@@ -649,7 +650,7 @@ int dth_encode_smallint(const NdbDiction
 /***** SMALL UNSIGNED ******/
 int dth_decode_small_unsigned(const NdbDictionary::Column *col,
                               char * &str, const void *buf) {
-  LOAD_FOR_ARCHITECTURE(Uint16, shortval, buf);
+  LOAD_ALIGNED_DATA(Uint16, shortval, buf);
   return sprintf(str, "%hu", shortval);
 }
 
@@ -661,7 +662,7 @@ int dth_encode_small_unsigned(const NdbD
   if((! safe_strtoul(copy_buff, &intval)) || intval > 65535) {
     return DTH_NUMERIC_OVERFLOW;
   }
-  STORE_FOR_ARCHITECTURE(Uint16, intval, buf);
+  STORE_ALIGNED_DATA(Uint16, intval, buf);
   return len;
 }
 
@@ -765,7 +766,7 @@ int dth_write32_medium_unsigned(Int32 va
 /***** INT *****/
 int dth_decode_int(const NdbDictionary::Column *col,
                    char * &str, const void *buf) {  
-  LOAD_FOR_ARCHITECTURE(int, i, buf);
+  LOAD_ALIGNED_DATA(int, i, buf);
   return sprintf(str, "%d", i);
 }
 
@@ -777,7 +778,7 @@ int dth_encode_int(const NdbDictionary::
   if(! safe_strtol(copy_buff, &intval))
     return DTH_NUMERIC_OVERFLOW;
 
-  STORE_FOR_ARCHITECTURE(int, intval, buf);
+  STORE_ALIGNED_DATA(int, intval, buf);
   return len;
 }                                 
 
@@ -785,7 +786,7 @@ int dth_encode_int(const NdbDictionary::
 /***** INT UNSIGNED *****/
 int dth_decode_unsigned(const NdbDictionary::Column *col,
                         char * &str, const void *buf) {  
-  LOAD_FOR_ARCHITECTURE(Uint32, i, buf)
+  LOAD_ALIGNED_DATA(Uint32, i, buf)
   return sprintf(str, "%du", i);
 }
 
@@ -797,7 +798,7 @@ int dth_encode_unsigned(const NdbDiction
   if(! safe_strtoul(copy_buff, &uintval))
     return DTH_NUMERIC_OVERFLOW;
 
-  STORE_FOR_ARCHITECTURE(Uint32, uintval, buf);
+  STORE_ALIGNED_DATA(Uint32, uintval, buf);
   return len;
 }                                 
 
@@ -805,7 +806,7 @@ int dth_encode_unsigned(const NdbDiction
 /***** BIGINT *****/
 int dth_decode_bigint(const NdbDictionary::Column *col,  
                       char * &str, const void *buf) {
-  LOAD_FOR_ARCHITECTURE(Int64, int64val, buf);
+  LOAD_ALIGNED_DATA(Int64, int64val, buf);
   return sprintf(str, "%"PRId64, int64val);
 }
 
@@ -817,7 +818,7 @@ int dth_encode_bigint(const NdbDictionar
   if(! safe_strtoll(copy_buff, &int64val)) 
     return DTH_NUMERIC_OVERFLOW;
 
-  STORE_FOR_ARCHITECTURE(int64_t, int64val, buf);
+  STORE_ALIGNED_DATA(int64_t, int64val, buf);
   return len;
 }
 
@@ -825,7 +826,7 @@ int dth_encode_bigint(const NdbDictionar
 /***** BIGINT UNSIGNED *****/
 int dth_decode_ubigint(const NdbDictionary::Column *col,  
                        char * &str, const void *buf) {
-  LOAD_FOR_ARCHITECTURE(Uint64, uint64val, buf);
+  LOAD_ALIGNED_DATA(Uint64, uint64val, buf);
   return sprintf(str, "%"PRIu64, uint64val);
 }
 
@@ -837,7 +838,7 @@ int dth_encode_ubigint(const NdbDictiona
   if(! safe_strtoull(copy_buff, &uint64val))  
     return DTH_NUMERIC_OVERFLOW;
 
-  STORE_FOR_ARCHITECTURE(uint64_t, uint64val, buf);
+  STORE_ALIGNED_DATA(uint64_t, uint64val, buf);
   return len;
 }                                 
 
@@ -1040,8 +1041,8 @@ int dth_decode_datetime(const NdbDiction
   Int32 int_date, int_time;
   time_helper tm = { 0,0,0,0,0,0,0, false };
   
-  /* Read the datetime from the buffer */
-  LOAD_FOR_ARCHITECTURE(Uint64, int_datetime, buf);
+  /* Read the datetime from the buffer. */
+  LOAD_ALIGNED_DATA(Uint64, int_datetime, buf);
   
   /* Factor it out */
   int_date = int_datetime / 1000000;
@@ -1070,7 +1071,7 @@ int dth_encode_datetime(const NdbDiction
   if(! safe_strtoull(copybuff.ptr, &int_datetime)) return DTH_NUMERIC_OVERFLOW;
   
   /* Store it */
-  STORE_FOR_ARCHITECTURE(Uint64, int_datetime, buf);
+  STORE_ALIGNED_DATA(Uint64, int_datetime, buf);
   
   return 1;
 }
@@ -1085,7 +1086,7 @@ int dth_encode_datetime(const NdbDiction
 */ 
 int dth_decode_float(const NdbDictionary::Column *col, 
                      char * &str, const void *buf) {
-  LOAD_ALIGNED(float, fval, buf);
+  LOAD_ALIGNED_DATA(float, fval, buf);
 
   double dval = fval;
   return sprintf(str, "%G", dval);
@@ -1094,21 +1095,21 @@ int dth_decode_float(const NdbDictionary
 size_t dth_length_float(const NdbDictionary::Column *col,
                         const void *buf) {
   char stack_copy[16];
-  LOAD_ALIGNED(float, fval, buf);
+  LOAD_ALIGNED_DATA(float, fval, buf);
   double dval = fval;
   return snprintf(stack_copy, 16, "%G", dval);
 }
 
 int dth_decode_double(const NdbDictionary::Column *col, 
                      char * &str, const void *buf) {
-  LOAD_ALIGNED(double, dval, buf);
+  LOAD_ALIGNED_DATA(double, dval, buf);
   return sprintf(str, "%.10F", dval);
 }
 
 size_t dth_length_double(const NdbDictionary::Column *col,
                          const void *buf) {
   char stack_copy[30];
-  LOAD_ALIGNED(double, dval, buf);
+  LOAD_ALIGNED_DATA(double, dval, buf);
   return snprintf(stack_copy, 30, "%.10F", dval);
 }
 
@@ -1122,7 +1123,7 @@ template <typename FPTYPE> int dth_encod
     return DTH_NUMERIC_OVERFLOW;
   }
   
-  STORE_ALIGNED(FPTYPE, dval, buf);
+  STORE_ALIGNED_DATA(FPTYPE, dval, buf);
   return len;
 }
 

=== modified file 'storage/ndb/memcache/src/Record.cc'
--- a/storage/ndb/memcache/src/Record.cc	2011-09-28 03:42:51 +0000
+++ b/storage/ndb/memcache/src/Record.cc	2011-09-28 21:48:01 +0000
@@ -49,7 +49,11 @@ Record::~Record() {
 void Record::addColumn(short col_type, const NdbDictionary::Column *column) {
   assert(index < ncolumns);
 
-  /* place the index correctly into the columns array */
+  /* The "column map" is an array that maps a specifier like 
+     "COL_STORE_VALUE + 1" (the second value column) or 
+     "COL_STORE_CAS" (the cas column) 
+      to that column's index in the record. 
+  */
   switch(col_type) {
     case COL_STORE_KEY:
       map[COL_STORE_KEY + nkeys++] = index;
@@ -68,17 +72,21 @@ void Record::addColumn(short col_type, c
     default:
       assert("Bad column type" == 0);
   }
-
-  /* Build the Record Specification */
-  specs[index].column = column;
-  
-  /* The CAS is manipulated using Record::getPointer() and must be aligned */
-  if(col_type == COL_STORE_CAS)  {
-    int bad_offset = rec_size % 8;
-    if(bad_offset) rec_size += (8 - bad_offset);
-  }
   
-  specs[index].offset = rec_size;  /* use the current record size */
+  /* Link to the Dictionary Column */
+  specs[index].column = column;
+
+  /* Link to the correct DataTypeHandler */
+  handlers[index] = getDataTypeHandlerForColumn(column);
+    
+  /* If the data type requires alignment, insert some padding.
+     This call will alter rec_size if needed */
+  pad_offset_for_alignment();
+
+  /* The current record size is the offset of this column */
+  specs[index].offset = rec_size;  
+
+  /* Set nullbits in the record specification */
   if(column->getNullable()) {
     specs[index].nullbit_byte_offset = n_nullable / 8;
     specs[index].nullbit_bit_in_byte = n_nullable % 8;
@@ -89,9 +97,6 @@ void Record::addColumn(short col_type, c
     specs[index].nullbit_bit_in_byte = 0;
   }
 
-  /* Link in the correct DataTypeHandler */
-  handlers[index] = getDataTypeHandlerForColumn(column);
-
   /* Increment the counter and record size */
   index += 1;
   rec_size += getColumnRecordSize(column);
@@ -281,3 +286,25 @@ size_t Record::getStringifiedLength(char
   return total;
 }
 
+
+void Record::pad_offset_for_alignment() {
+  int alignment = 1;
+  int bad_offset = 0;
+  
+  if(index == map[COL_STORE_CAS]) {  // CAS column requires 8-byte alignment
+    alignment = 8;
+  }
+  else if(! handlers[index]->contains_string) {
+    alignment = specs[index].column->getSize();
+  }
+
+  switch(alignment) {
+    case 2: case 4: case 8: case 16:  /* insert padding */
+      bad_offset = rec_size % alignment;
+      if(bad_offset) 
+        rec_size += (alignment - bad_offset);
+      break;
+    default:
+      break;
+  }
+}

=== modified file 'support-files/compiler_warnings.supp'
--- a/support-files/compiler_warnings.supp	2011-09-22 08:23:05 +0000
+++ b/support-files/compiler_warnings.supp	2011-09-28 21:56:55 +0000
@@ -110,5 +110,5 @@ client/mysqldump.c : .*may be used unini
 # 
 # Bundled memcached and libevent
 #
-storage/ndb/extra/.* : .*
+.*/ndb/memcache/extra/.* : .*
 

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.5-cluster branch (john.duncan:3563 to 3568) John David Duncan2 Oct