From: John David Duncan Date: September 24 2011 5:47am Subject: bzr push into mysql-5.5-cluster branch (john.duncan:3542 to 3544) List-Archive: http://lists.mysql.com/commits/141118 Message-Id: <201109240547.p8O5lxGc027010@acsmt358.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 3544 John David Duncan 2011-09-23 Some refactoring in ndb/memcache DataTypeHandler. type_char test should pass now. modified: storage/ndb/memcache/include/DataTypeHandler.h storage/ndb/memcache/src/DataTypeHandler.cc storage/ndb/memcache/src/Record.cc 3543 John David Duncan 2011-09-23 This fixes a crashing bug when appending \r\n to a CHAR value so that it can be sent directly from the NDB buffer without a copy. modified: storage/ndb/memcache/src/DataTypeHandler.cc storage/ndb/memcache/src/ndb_worker.cc 3542 John David Duncan 2011-09-23 More portability fixes for ndb/memcache/unit/test_workqueue modified: storage/ndb/memcache/unit/CMakeLists.txt === modified file 'storage/ndb/memcache/include/DataTypeHandler.h' --- a/storage/ndb/memcache/include/DataTypeHandler.h 2011-09-12 10:05:07 +0000 +++ b/storage/ndb/memcache/include/DataTypeHandler.h 2011-09-24 05:45:06 +0000 @@ -64,12 +64,13 @@ typedef struct { // String Writer. Returns length written or < 0 for error. int (*writeToNdb)(const NdbDictionary::Column *col, size_t len, - size_t offset, const char *str, void * const buf); + const char *str, void * const buf); // NumericHandler. NumericHandler * native_handler; // Will readFromNdb() return a pointer to a string inside of buf? + // 1 = CHAR; 2 = VARCHAR; 3 = LONGVARCHAR int contains_string; } DataTypeHandler; === modified file 'storage/ndb/memcache/src/DataTypeHandler.cc' --- a/storage/ndb/memcache/src/DataTypeHandler.cc 2011-09-22 18:27:10 +0000 +++ b/storage/ndb/memcache/src/DataTypeHandler.cc 2011-09-24 05:45:06 +0000 @@ -42,7 +42,7 @@ extern EXTENSION_LOGGER_DESCRIPTOR *logg #define DECODE_ARGS const NdbDictionary::Column *, char * &, const void * const #define SFDLEN_ARGS const NdbDictionary::Column *, const void * const -#define ENCODE_ARGS const NdbDictionary::Column *, size_t, size_t, const char *, void * const +#define ENCODE_ARGS const NdbDictionary::Column *, size_t, const char *, void * const typedef int impl_readFromNdb(DECODE_ARGS); typedef size_t impl_getStringifiedLength(SFDLEN_ARGS); @@ -154,7 +154,7 @@ DataTypeHandler Handler_Varchar = { dth_length_varchar, // getStringifiedLength() dth_encode_varchar, // writeToNdb() 0, // native_handler - true // contains_string + 2 // contains_string + 1-byte length }; DataTypeHandler Handler_LongVarchar = { @@ -162,7 +162,7 @@ DataTypeHandler Handler_LongVarchar = { dth_length_longvarchar, dth_encode_longvarchar, 0, - true + 3 // contains string + 2 length bytes }; DataTypeHandler Handler_Char = { @@ -170,7 +170,7 @@ DataTypeHandler Handler_Char = { dth_length_char, // getStringifiedLength() dth_encode_char, // writeToNdb() 0, // native_handler - true // contains_string + 1 // contains_string (no length bytes) }; DataTypeHandler Handler_enum = { /* NDB sees ENUM columns as CHAR(1) */ @@ -427,7 +427,7 @@ size_t dth_length_unsupported(const NdbD return 0; } -int dth_encode_unsupported(const NdbDictionary::Column *col, size_t, size_t, +int dth_encode_unsupported(const NdbDictionary::Column *col, size_t, const char *, void *) { logger->log(LOG_WARNING, 0, "Unsupported column type: %s\n", col->getName()); return DTH_NOT_SUPPORTED; @@ -451,16 +451,15 @@ size_t dth_length_varchar(const NdbDicti int dth_encode_varchar(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { - size_t total_len = len + offset; + size_t len, const char *str, void *buf) { uint8_t * length_byte = (uint8_t *) buf; - char *char_buffer = ((char *) buf) + 1 + offset; + char *char_buffer = ((char *) buf) + 1; - if(total_len > col->getLength()) + if(len > col->getLength()) return DTH_VALUE_TOO_LONG; /* Set the length byte */ - *length_byte = (uint8_t) total_len; + *length_byte = (uint8_t) len; /* Copy string value into buffer */ strncpy(char_buffer, str, len); @@ -486,12 +485,11 @@ size_t dth_length_longvarchar(const NdbD return (size_t) ( *length_byte_1 + (*length_byte_2 << 8)); } -int dth_encode_longvarchar(const NdbDictionary::Column *col, - size_t len, size_t offset, +int dth_encode_longvarchar(const NdbDictionary::Column *col, size_t len, const char *str, void *buf) { char *cbuf = ((char *) buf); - char *dest = cbuf + 2 + offset; - unsigned short total_len = len + offset; + char *dest = cbuf + 2; + unsigned short total_len = len; const unsigned short short_lo = 255; const unsigned short short_hi = 65535 ^ 255; @@ -520,17 +518,18 @@ size_t dth_length_char(const NdbDictiona return col->getLength(); } -int dth_encode_char(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_char(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { char *cbuf = ((char *) buf); - char *dest = cbuf + offset; - if(len > col->getLength()) return DTH_VALUE_TOO_LONG; + char *dest = cbuf; + if(len > col->getLength()) + return DTH_VALUE_TOO_LONG; /* copy string into buffer */ memcpy(dest, str, len); /* right-pad with spaces */ - for(char *s = dest+len ; len < col->getLength() ; len++) { + for(char *s = dest+len ; len <= col->getLength() ; len++) { *(s++) = ' '; } @@ -573,7 +572,6 @@ template int dth_write /* Make a safe copy of the text representation of a number, discarding any terminal junk characters that may be in the buffer */ #define MAKE_COPY_BUFFER(sz) \ - assert(offset == 0); \ char copy_buff[sz]; \ if(len >= sz) return DTH_VALUE_TOO_LONG; \ strncpy(copy_buff, str, len); \ @@ -588,8 +586,8 @@ int dth_decode_tinyint(const NdbDictiona return sprintf(str, "%d", (int) i) + 1; // +1 for null terminator } -int dth_encode_tinyint(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_tinyint(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(8); int intval = 0; @@ -610,7 +608,7 @@ int dth_decode_tiny_unsigned(const NdbDi } int dth_encode_tiny_unsigned(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { MAKE_COPY_BUFFER(8); Uint32 intval = 0; @@ -630,8 +628,8 @@ int dth_decode_smallint(const NdbDiction return sprintf(str, "%hd",* (short *) buf) + 1; // +1 for null terminator } -int dth_encode_smallint(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_smallint(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(8); int intval = 0; @@ -650,8 +648,8 @@ int dth_decode_small_unsigned(const NdbD return sprintf(str, "%hu",* (short *) buf) + 1; // +1 for null terminator } -int dth_encode_small_unsigned(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_small_unsigned(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(8); Uint32 intval = 0; @@ -680,8 +678,8 @@ size_t dth_length_mediumint(const NdbDic return len; } -int dth_encode_mediumint(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_mediumint(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { int intval = 0; const int MAXVAL = 8388607; const int MINVAL = -8388608; @@ -730,7 +728,7 @@ size_t dth_length_medium_unsigned(const } int dth_encode_medium_unsigned(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { MAKE_COPY_BUFFER(16); Uint32 intval = 0; @@ -766,8 +764,8 @@ int dth_decode_int(const NdbDictionary:: return sprintf(str, "%d",* (int *) buf) + 1; // +1 for null terminator } -int dth_encode_int(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_int(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(32); int *ibuf = (int *) buf; @@ -784,8 +782,8 @@ int dth_decode_unsigned(const NdbDiction return sprintf(str, "%du",* (Uint32 *) buf) + 1; // +1 for null terminator } -int dth_encode_unsigned(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_unsigned(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(32); Uint32 *ibuf = (Uint32 *) buf; @@ -802,8 +800,8 @@ int dth_decode_bigint(const NdbDictionar return sprintf(str, "%"PRId64,* (Int64 *) buf) + 1; // +1 for null } -int dth_encode_bigint(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_bigint(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(32); int64_t *ibuf = (int64_t *) buf; @@ -820,8 +818,8 @@ int dth_decode_ubigint(const NdbDictiona return sprintf(str, "%"PRIu64,* (Uint64 *) buf) + 1; // +1 for null } -int dth_encode_ubigint(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { +int dth_encode_ubigint(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { MAKE_COPY_BUFFER(32); uint64_t *ibuf = (uint64_t *) buf; @@ -843,9 +841,8 @@ size_t dth_length_enum(const NdbDictiona return 1; } -int dth_encode_enum(const NdbDictionary::Column *col, - size_t len, size_t offset, const char *str, void *buf) { - assert(offset == 0); +int dth_encode_enum(const NdbDictionary::Column *col, size_t len, + const char *str, void *buf) { char *cbuf = (char *) buf; *cbuf = *str; return 1; @@ -864,7 +861,7 @@ size_t dth_length_year(const NdbDictiona } int dth_encode_year(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { MAKE_COPY_BUFFER(8); Uint32 intval = 0; @@ -965,7 +962,7 @@ size_t dth_length_date(const NdbDictiona } int dth_encode_date(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { Int32 int_date; Uint32 encoded_date; time_helper tm = { 0,0,0,0,0,0,0, false }; @@ -1010,7 +1007,7 @@ size_t dth_length_time(const NdbDictiona } int dth_encode_time(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { Int32 int_time; /* Make a safe (null-terminated) copy */ @@ -1051,7 +1048,7 @@ size_t dth_length_datetime(const NdbDict } int dth_encode_datetime(const NdbDictionary::Column *, size_t len, - size_t offset, const char *str, void *buf) { + const char *str, void *buf) { uint64_t int_datetime; /* Make a safe (null-terminated) copy */ @@ -1102,8 +1099,7 @@ size_t dth_length_double(const NdbDictio } template int dth_encode_fp(const NdbDictionary::Column *col, - size_t len, size_t offset, - const char *str, void *buf) { + size_t len, const char *str, void *buf) { MAKE_COPY_BUFFER(64); errno = 0; double d = strtod(copy_buff, NULL); @@ -1131,8 +1127,7 @@ size_t dth_length_decimal(const NdbDicti return col->getScale() + col->getPrecision() + 3; // sign, point, and terminator } -int dth_encode_decimal(const NdbDictionary::Column *col, - size_t len, size_t offset, +int dth_encode_decimal(const NdbDictionary::Column *col, size_t len, const char *str, void *buf) { MAKE_COPY_BUFFER(64); int scale = col->getScale(); === modified file 'storage/ndb/memcache/src/Record.cc' --- a/storage/ndb/memcache/src/Record.cc 2011-09-22 18:27:10 +0000 +++ b/storage/ndb/memcache/src/Record.cc 2011-09-24 05:45:06 +0000 @@ -153,11 +153,13 @@ bool Record::complete(NdbDictionary::Dic */ bool Record::appendCRLF(int id, size_t len, char *buffer) const { int idx = map[id]; - if( handlers[idx]->contains_string - && handlers[idx]->writeToNdb(specs[idx].column, 2, len, "\r\n", buffer) - >= 0) - { - return true; + int length_bytes = handlers[idx]->contains_string; + +if(length_bytes) { + size_t offset = len + length_bytes - 1; /* see DataTypeHandler.h */ + buffer[offset] = '\r'; + buffer[offset+1] = '\n'; + return true; } return false; } @@ -256,7 +258,7 @@ bool Record::setUint64Value(int id, Uint int Record::encode(int id, const char *key, int nkey, char *buffer) const { - return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, 0, key, + return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, key, buffer + specs[map[id]].offset); } === modified file 'storage/ndb/memcache/src/ndb_worker.cc' --- a/storage/ndb/memcache/src/ndb_worker.cc 2011-09-23 18:29:30 +0000 +++ b/storage/ndb/memcache/src/ndb_worker.cc 2011-09-23 23:56:04 +0000 @@ -335,9 +335,10 @@ op_status_t worker_do_read(workitem *wqi const char *dbkey = workitem_get_key_suffix(wqitem); /* Use the workitem's inline buffer as a key buffer; - allocate a new result buffer large enough for the result */ + allocate a new result buffer large enough for the result. + Add 2 bytes to hold potential \r\n in a no-copy result. */ Operation op(plan, OP_READ, wqitem->ndb_key_buffer); - workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer()); + workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer() + 2); op.buffer = wqitem->row_buffer_1; /* Copy the key into the key buffer, ecnoding it for NDB */ @@ -517,6 +518,9 @@ op_status_t worker_do_math(workitem *wqi void DB_callback(int result, NdbTransaction *tx, void *itemptr) { workitem *wqitem = (workitem *) itemptr; + + assert(tx->getNdb()->getCustomData() == wqitem); + ndb_pipeline * & pipeline = wqitem->pipeline; status_block * return_status; bool tx_did_match = false; @@ -761,7 +765,7 @@ void incr_callback(int result, NdbTransa bool finalize_read(workitem *wqitem) { - DEBUG_ENTER(); + DEBUG_PRINT("%d.%d",wqitem->pipeline->id, wqitem->id); bool need_hash_item; Operation op(wqitem->plan, OP_READ); No bundle (reason: useless for push emails).