List:Commits« Previous MessageNext Message »
From:John David Duncan Date:September 24 2011 5:47am
Subject:bzr push into mysql-5.5-cluster branch (john.duncan:3542 to 3544)
View as plain text  
 3544 John David Duncan	2011-09-23
      Some refactoring in ndb/memcache DataTypeHandler.
      type_char test should pass now.

    modified:
      storage/ndb/memcache/include/DataTypeHandler.h
      storage/ndb/memcache/src/DataTypeHandler.cc
      storage/ndb/memcache/src/Record.cc
 3543 John David Duncan	2011-09-23
      This fixes a crashing bug when appending \r\n to a CHAR value so that it can 
      be sent directly from the NDB buffer without a copy.

    modified:
      storage/ndb/memcache/src/DataTypeHandler.cc
      storage/ndb/memcache/src/ndb_worker.cc
 3542 John David Duncan	2011-09-23
      More portability fixes for ndb/memcache/unit/test_workqueue

    modified:
      storage/ndb/memcache/unit/CMakeLists.txt
=== modified file 'storage/ndb/memcache/include/DataTypeHandler.h'
--- a/storage/ndb/memcache/include/DataTypeHandler.h	2011-09-12 10:05:07 +0000
+++ b/storage/ndb/memcache/include/DataTypeHandler.h	2011-09-24 05:45:06 +0000
@@ -64,12 +64,13 @@ typedef struct {
  
   // String Writer.  Returns length written or < 0 for error.
   int (*writeToNdb)(const NdbDictionary::Column *col, size_t len, 
-                    size_t offset, const char *str, void * const buf);
+                    const char *str, void * const buf);
 
   // NumericHandler.  
   NumericHandler * native_handler;
   
   // Will readFromNdb() return a pointer to a string inside of buf?
+  // 1 = CHAR; 2 = VARCHAR; 3 = LONGVARCHAR
   int contains_string;
 } DataTypeHandler;
 

=== modified file 'storage/ndb/memcache/src/DataTypeHandler.cc'
--- a/storage/ndb/memcache/src/DataTypeHandler.cc	2011-09-22 18:27:10 +0000
+++ b/storage/ndb/memcache/src/DataTypeHandler.cc	2011-09-24 05:45:06 +0000
@@ -42,7 +42,7 @@ extern EXTENSION_LOGGER_DESCRIPTOR *logg
 
 #define DECODE_ARGS const NdbDictionary::Column *, char * &, const void * const
 #define SFDLEN_ARGS const NdbDictionary::Column *, const void * const
-#define ENCODE_ARGS const NdbDictionary::Column *, size_t, size_t, const char *, void * const
+#define ENCODE_ARGS const NdbDictionary::Column *, size_t, const char *, void * const
 
 typedef int    impl_readFromNdb(DECODE_ARGS);
 typedef size_t impl_getStringifiedLength(SFDLEN_ARGS);
@@ -154,7 +154,7 @@ DataTypeHandler Handler_Varchar =  {
   dth_length_varchar,      // getStringifiedLength()
   dth_encode_varchar,      // writeToNdb()
   0,                       // native_handler
-  true                     // contains_string
+  2                        // contains_string + 1-byte length
 };
 
 DataTypeHandler Handler_LongVarchar = {
@@ -162,7 +162,7 @@ DataTypeHandler Handler_LongVarchar = {
   dth_length_longvarchar,
   dth_encode_longvarchar,
   0,
-  true
+  3                       // contains string + 2 length bytes
 };
 
 DataTypeHandler Handler_Char =  {
@@ -170,7 +170,7 @@ DataTypeHandler Handler_Char =  {
   dth_length_char,        // getStringifiedLength()
   dth_encode_char,        // writeToNdb()
   0,                      // native_handler
-  true                    // contains_string
+  1                       // contains_string (no length bytes)
 };
 
 DataTypeHandler Handler_enum = {   /* NDB sees ENUM columns as CHAR(1) */
@@ -427,7 +427,7 @@ size_t dth_length_unsupported(const NdbD
   return 0;
 }
 
-int dth_encode_unsupported(const NdbDictionary::Column *col, size_t, size_t, 
+int dth_encode_unsupported(const NdbDictionary::Column *col, size_t,
                            const char *, void *) {
   logger->log(LOG_WARNING, 0, "Unsupported column type: %s\n", col->getName());
   return DTH_NOT_SUPPORTED;
@@ -451,16 +451,15 @@ size_t dth_length_varchar(const NdbDicti
 
 
 int dth_encode_varchar(const NdbDictionary::Column *col, 
-                       size_t len, size_t offset, const char *str, void *buf) {
-  size_t total_len = len + offset;
+                       size_t len, const char *str, void *buf) {
   uint8_t * length_byte = (uint8_t *) buf;
-  char *char_buffer = ((char *) buf) + 1 + offset;
+  char *char_buffer = ((char *) buf) + 1;
     
-  if(total_len > col->getLength())
+  if(len > col->getLength())
     return DTH_VALUE_TOO_LONG;
   
   /* Set the length byte */
-  *length_byte = (uint8_t) total_len;
+  *length_byte = (uint8_t) len;
   
   /* Copy string value into buffer */
   strncpy(char_buffer, str, len);
@@ -486,12 +485,11 @@ size_t dth_length_longvarchar(const NdbD
   return (size_t) ( *length_byte_1 + (*length_byte_2 << 8));
 }
 
-int dth_encode_longvarchar(const NdbDictionary::Column *col, 
-                           size_t len, size_t offset,
+int dth_encode_longvarchar(const NdbDictionary::Column *col, size_t len, 
                            const char *str, void *buf) {
   char *cbuf = ((char *) buf);
-  char *dest = cbuf + 2 + offset;
-  unsigned short total_len = len + offset;
+  char *dest = cbuf + 2;
+  unsigned short total_len = len;
   const unsigned short short_lo = 255;
   const unsigned short short_hi = 65535 ^ 255; 
   
@@ -520,17 +518,18 @@ size_t dth_length_char(const NdbDictiona
   return col->getLength();
 }
 
-int dth_encode_char(const NdbDictionary::Column *col, 
-                    size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_char(const NdbDictionary::Column *col, size_t len,
+                    const char *str, void *buf) {
   char *cbuf = ((char *) buf);
-  char *dest = cbuf + offset;
-  if(len > col->getLength()) return DTH_VALUE_TOO_LONG;
+  char *dest = cbuf;
+  if(len > col->getLength()) 
+    return DTH_VALUE_TOO_LONG;
 
   /* copy string into buffer */
   memcpy(dest, str, len);
 
   /* right-pad with spaces */
-  for(char *s = dest+len ; len < col->getLength() ; len++) {
+  for(char *s = dest+len ; len <= col->getLength() ; len++) {
     *(s++) = ' ';
   }
 
@@ -573,7 +572,6 @@ template<typename INTTYPE> int dth_write
 /* Make a safe copy of the text representation of a number, discarding 
    any terminal junk characters that may be in the buffer */
 #define MAKE_COPY_BUFFER(sz) \
-  assert(offset == 0); \
   char copy_buff[sz]; \
   if(len >= sz) return DTH_VALUE_TOO_LONG; \
   strncpy(copy_buff, str, len); \
@@ -588,8 +586,8 @@ int dth_decode_tinyint(const NdbDictiona
   return sprintf(str, "%d", (int) i) + 1;  // +1 for null terminator
 }
 
-int dth_encode_tinyint(const NdbDictionary::Column *col, 
-                       size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_tinyint(const NdbDictionary::Column *col, size_t len,
+                       const char *str, void *buf) {
   MAKE_COPY_BUFFER(8);
   int intval = 0;
 
@@ -610,7 +608,7 @@ int dth_decode_tiny_unsigned(const NdbDi
 }
 
 int dth_encode_tiny_unsigned(const NdbDictionary::Column *, size_t len, 
-                             size_t offset, const char *str, void *buf) {
+                             const char *str, void *buf) {
   MAKE_COPY_BUFFER(8);
   Uint32 intval = 0;
   
@@ -630,8 +628,8 @@ int dth_decode_smallint(const NdbDiction
   return sprintf(str, "%hd",* (short *) buf) + 1;  // +1 for null terminator
 }
 
-int dth_encode_smallint(const NdbDictionary::Column *col, 
-                        size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_smallint(const NdbDictionary::Column *col, size_t len, 
+                        const char *str, void *buf) {
   MAKE_COPY_BUFFER(8);
   int intval = 0;
   
@@ -650,8 +648,8 @@ int dth_decode_small_unsigned(const NdbD
   return sprintf(str, "%hu",* (short *) buf) + 1;  // +1 for null terminator
 }
 
-int dth_encode_small_unsigned(const NdbDictionary::Column *col, 
-                              size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_small_unsigned(const NdbDictionary::Column *col, size_t len, 
+                              const char *str, void *buf) {
   MAKE_COPY_BUFFER(8);
   Uint32 intval = 0;
   
@@ -680,8 +678,8 @@ size_t dth_length_mediumint(const NdbDic
   return len;  
 }
 
-int dth_encode_mediumint(const NdbDictionary::Column *col, 
-                         size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_mediumint(const NdbDictionary::Column *col, size_t len,
+                         const char *str, void *buf) {
   int intval = 0;
   const int MAXVAL = 8388607;
   const int MINVAL = -8388608;  
@@ -730,7 +728,7 @@ size_t dth_length_medium_unsigned(const 
 }
 
 int dth_encode_medium_unsigned(const NdbDictionary::Column *, size_t len,
-                               size_t offset, const char *str, void *buf) {
+                               const char *str, void *buf) {
   MAKE_COPY_BUFFER(16);
   Uint32 intval = 0;
   
@@ -766,8 +764,8 @@ int dth_decode_int(const NdbDictionary::
   return sprintf(str, "%d",* (int *) buf) + 1;  // +1 for null terminator
 }
 
-int dth_encode_int(const NdbDictionary::Column *col, 
-                   size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_int(const NdbDictionary::Column *col, size_t len, 
+                   const char *str, void *buf) {
   MAKE_COPY_BUFFER(32);
 
   int *ibuf = (int *) buf;
@@ -784,8 +782,8 @@ int dth_decode_unsigned(const NdbDiction
   return sprintf(str, "%du",* (Uint32 *) buf) + 1;  // +1 for null terminator
 }
 
-int dth_encode_unsigned(const NdbDictionary::Column *col, 
-                        size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_unsigned(const NdbDictionary::Column *col, size_t len,
+                        const char *str, void *buf) {
   MAKE_COPY_BUFFER(32);
   
   Uint32 *ibuf = (Uint32 *) buf;
@@ -802,8 +800,8 @@ int dth_decode_bigint(const NdbDictionar
   return sprintf(str, "%"PRId64,* (Int64 *) buf) + 1;  // +1 for null
 }
 
-int dth_encode_bigint(const NdbDictionary::Column *col, 
-                      size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_bigint(const NdbDictionary::Column *col, size_t len,
+                      const char *str, void *buf) {
   MAKE_COPY_BUFFER(32);
   
   int64_t *ibuf = (int64_t *) buf;
@@ -820,8 +818,8 @@ int dth_decode_ubigint(const NdbDictiona
   return sprintf(str, "%"PRIu64,* (Uint64 *) buf) + 1;  // +1 for null
 }
 
-int dth_encode_ubigint(const NdbDictionary::Column *col, 
-                       size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_ubigint(const NdbDictionary::Column *col, size_t len,
+                       const char *str, void *buf) {
   MAKE_COPY_BUFFER(32);
 
   uint64_t *ibuf = (uint64_t *) buf;
@@ -843,9 +841,8 @@ size_t dth_length_enum(const NdbDictiona
   return 1;
 }
 
-int dth_encode_enum(const NdbDictionary::Column *col, 
-                    size_t len, size_t offset, const char *str, void *buf) {
-  assert(offset == 0);
+int dth_encode_enum(const NdbDictionary::Column *col, size_t len,
+                    const char *str, void *buf) {
   char *cbuf = (char *) buf;
   *cbuf = *str;  
   return 1;
@@ -864,7 +861,7 @@ size_t dth_length_year(const NdbDictiona
 }
 
 int dth_encode_year(const NdbDictionary::Column *, size_t len, 
-                    size_t offset, const char *str, void *buf) {
+                    const char *str, void *buf) {
   MAKE_COPY_BUFFER(8);
   Uint32 intval = 0;
   
@@ -965,7 +962,7 @@ size_t dth_length_date(const NdbDictiona
 }
 
 int dth_encode_date(const NdbDictionary::Column *, size_t len, 
-                    size_t offset, const char *str, void *buf) {
+                    const char *str, void *buf) {
   Int32  int_date;
   Uint32  encoded_date; 
   time_helper tm = { 0,0,0,0,0,0,0, false };
@@ -1010,7 +1007,7 @@ size_t dth_length_time(const NdbDictiona
 }
 
 int dth_encode_time(const NdbDictionary::Column *, size_t len, 
-                    size_t offset, const char *str, void *buf) {
+                    const char *str, void *buf) {
   Int32  int_time;
   
   /* Make a safe (null-terminated) copy */
@@ -1051,7 +1048,7 @@ size_t dth_length_datetime(const NdbDict
 }
 
 int dth_encode_datetime(const NdbDictionary::Column *, size_t len, 
-                        size_t offset, const char *str, void *buf) {
+                        const char *str, void *buf) {
   uint64_t int_datetime;
   
   /* Make a safe (null-terminated) copy */
@@ -1102,8 +1099,7 @@ size_t dth_length_double(const NdbDictio
 }
 
 template <typename T> int dth_encode_fp(const NdbDictionary::Column *col, 
-                                        size_t len, size_t offset, 
-                                        const char *str, void *buf) {
+                                        size_t len, const char *str, void *buf) {
   MAKE_COPY_BUFFER(64);
   errno = 0;
   double d = strtod(copy_buff, NULL);
@@ -1131,8 +1127,7 @@ size_t dth_length_decimal(const NdbDicti
   return col->getScale() + col->getPrecision() + 3; // sign, point, and terminator
 }
 
-int dth_encode_decimal(const NdbDictionary::Column *col, 
-                       size_t len, size_t offset, 
+int dth_encode_decimal(const NdbDictionary::Column *col, size_t len, 
                        const char *str, void *buf) {
   MAKE_COPY_BUFFER(64);
   int scale = col->getScale();

=== modified file 'storage/ndb/memcache/src/Record.cc'
--- a/storage/ndb/memcache/src/Record.cc	2011-09-22 18:27:10 +0000
+++ b/storage/ndb/memcache/src/Record.cc	2011-09-24 05:45:06 +0000
@@ -153,11 +153,13 @@ bool Record::complete(NdbDictionary::Dic
 */
 bool Record::appendCRLF(int id, size_t len, char *buffer) const {
   int idx = map[id];
-  if(   handlers[idx]->contains_string 
-     && handlers[idx]->writeToNdb(specs[idx].column, 2, len, "\r\n", buffer) 
-     >= 0)
-  {
-    return true; 
+  int length_bytes = handlers[idx]->contains_string;
+
+if(length_bytes) {
+    size_t offset = len + length_bytes - 1;  /* see DataTypeHandler.h */
+    buffer[offset]   = '\r';
+    buffer[offset+1] = '\n';
+    return true;
   }
   return false;
 }
@@ -256,7 +258,7 @@ bool Record::setUint64Value(int id, Uint
 
 int Record::encode(int id, const char *key, int nkey,
                    char *buffer) const {
-  return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, 0, key, 
+  return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, key, 
                                        buffer + specs[map[id]].offset);
 }
 

=== modified file 'storage/ndb/memcache/src/ndb_worker.cc'
--- a/storage/ndb/memcache/src/ndb_worker.cc	2011-09-23 18:29:30 +0000
+++ b/storage/ndb/memcache/src/ndb_worker.cc	2011-09-23 23:56:04 +0000
@@ -335,9 +335,10 @@ op_status_t worker_do_read(workitem *wqi
   const char *dbkey = workitem_get_key_suffix(wqitem);
 
   /* Use the workitem's inline buffer as a key buffer; 
-     allocate a new result buffer large enough for the result */
+     allocate a new result buffer large enough for the result.
+     Add 2 bytes to hold potential \r\n in a no-copy result. */
   Operation op(plan, OP_READ, wqitem->ndb_key_buffer);
-  workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer());  
+  workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer() + 2);
   op.buffer = wqitem->row_buffer_1;
 
   /* Copy the key into the key buffer, ecnoding it for NDB */
@@ -517,6 +518,9 @@ op_status_t worker_do_math(workitem *wqi
 
 void DB_callback(int result, NdbTransaction *tx, void *itemptr) {
   workitem *wqitem = (workitem *) itemptr;
+
+  assert(tx->getNdb()->getCustomData() == wqitem);
+
   ndb_pipeline * & pipeline = wqitem->pipeline;
   status_block * return_status;
   bool tx_did_match = false;
@@ -761,7 +765,7 @@ void incr_callback(int result, NdbTransa
 
 
 bool finalize_read(workitem *wqitem) {
-  DEBUG_ENTER();
+  DEBUG_PRINT("%d.%d",wqitem->pipeline->id, wqitem->id);
   
   bool need_hash_item;
   Operation op(wqitem->plan, OP_READ);

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.5-cluster branch (john.duncan:3542 to 3544) John David Duncan24 Sep