3544 John David Duncan 2011-09-23
Some refactoring in ndb/memcache DataTypeHandler.
type_char test should pass now.
modified:
storage/ndb/memcache/include/DataTypeHandler.h
storage/ndb/memcache/src/DataTypeHandler.cc
storage/ndb/memcache/src/Record.cc
3543 John David Duncan 2011-09-23
This fixes a crashing bug when appending \r\n to a CHAR value so that it can
be sent directly from the NDB buffer without a copy.
modified:
storage/ndb/memcache/src/DataTypeHandler.cc
storage/ndb/memcache/src/ndb_worker.cc
3542 John David Duncan 2011-09-23
More portability fixes for ndb/memcache/unit/test_workqueue
modified:
storage/ndb/memcache/unit/CMakeLists.txt
=== modified file 'storage/ndb/memcache/include/DataTypeHandler.h'
--- a/storage/ndb/memcache/include/DataTypeHandler.h 2011-09-12 10:05:07 +0000
+++ b/storage/ndb/memcache/include/DataTypeHandler.h 2011-09-24 05:45:06 +0000
@@ -64,12 +64,13 @@ typedef struct {
// String Writer. Returns length written or < 0 for error.
int (*writeToNdb)(const NdbDictionary::Column *col, size_t len,
- size_t offset, const char *str, void * const buf);
+ const char *str, void * const buf);
// NumericHandler.
NumericHandler * native_handler;
// Will readFromNdb() return a pointer to a string inside of buf?
+ // 1 = CHAR; 2 = VARCHAR; 3 = LONGVARCHAR
int contains_string;
} DataTypeHandler;
=== modified file 'storage/ndb/memcache/src/DataTypeHandler.cc'
--- a/storage/ndb/memcache/src/DataTypeHandler.cc 2011-09-22 18:27:10 +0000
+++ b/storage/ndb/memcache/src/DataTypeHandler.cc 2011-09-24 05:45:06 +0000
@@ -42,7 +42,7 @@ extern EXTENSION_LOGGER_DESCRIPTOR *logg
#define DECODE_ARGS const NdbDictionary::Column *, char * &, const void * const
#define SFDLEN_ARGS const NdbDictionary::Column *, const void * const
-#define ENCODE_ARGS const NdbDictionary::Column *, size_t, size_t, const char *, void * const
+#define ENCODE_ARGS const NdbDictionary::Column *, size_t, const char *, void * const
typedef int impl_readFromNdb(DECODE_ARGS);
typedef size_t impl_getStringifiedLength(SFDLEN_ARGS);
@@ -154,7 +154,7 @@ DataTypeHandler Handler_Varchar = {
dth_length_varchar, // getStringifiedLength()
dth_encode_varchar, // writeToNdb()
0, // native_handler
- true // contains_string
+ 2 // contains_string + 1-byte length
};
DataTypeHandler Handler_LongVarchar = {
@@ -162,7 +162,7 @@ DataTypeHandler Handler_LongVarchar = {
dth_length_longvarchar,
dth_encode_longvarchar,
0,
- true
+ 3 // contains string + 2 length bytes
};
DataTypeHandler Handler_Char = {
@@ -170,7 +170,7 @@ DataTypeHandler Handler_Char = {
dth_length_char, // getStringifiedLength()
dth_encode_char, // writeToNdb()
0, // native_handler
- true // contains_string
+ 1 // contains_string (no length bytes)
};
DataTypeHandler Handler_enum = { /* NDB sees ENUM columns as CHAR(1) */
@@ -427,7 +427,7 @@ size_t dth_length_unsupported(const NdbD
return 0;
}
-int dth_encode_unsupported(const NdbDictionary::Column *col, size_t, size_t,
+int dth_encode_unsupported(const NdbDictionary::Column *col, size_t,
const char *, void *) {
logger->log(LOG_WARNING, 0, "Unsupported column type: %s\n", col->getName());
return DTH_NOT_SUPPORTED;
@@ -451,16 +451,15 @@ size_t dth_length_varchar(const NdbDicti
int dth_encode_varchar(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
- size_t total_len = len + offset;
+ size_t len, const char *str, void *buf) {
uint8_t * length_byte = (uint8_t *) buf;
- char *char_buffer = ((char *) buf) + 1 + offset;
+ char *char_buffer = ((char *) buf) + 1;
- if(total_len > col->getLength())
+ if(len > col->getLength())
return DTH_VALUE_TOO_LONG;
/* Set the length byte */
- *length_byte = (uint8_t) total_len;
+ *length_byte = (uint8_t) len;
/* Copy string value into buffer */
strncpy(char_buffer, str, len);
@@ -486,12 +485,11 @@ size_t dth_length_longvarchar(const NdbD
return (size_t) ( *length_byte_1 + (*length_byte_2 << 8));
}
-int dth_encode_longvarchar(const NdbDictionary::Column *col,
- size_t len, size_t offset,
+int dth_encode_longvarchar(const NdbDictionary::Column *col, size_t len,
const char *str, void *buf) {
char *cbuf = ((char *) buf);
- char *dest = cbuf + 2 + offset;
- unsigned short total_len = len + offset;
+ char *dest = cbuf + 2;
+ unsigned short total_len = len;
const unsigned short short_lo = 255;
const unsigned short short_hi = 65535 ^ 255;
@@ -520,17 +518,18 @@ size_t dth_length_char(const NdbDictiona
return col->getLength();
}
-int dth_encode_char(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_char(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
char *cbuf = ((char *) buf);
- char *dest = cbuf + offset;
- if(len > col->getLength()) return DTH_VALUE_TOO_LONG;
+ char *dest = cbuf;
+ if(len > col->getLength())
+ return DTH_VALUE_TOO_LONG;
/* copy string into buffer */
memcpy(dest, str, len);
/* right-pad with spaces */
- for(char *s = dest+len ; len < col->getLength() ; len++) {
+ for(char *s = dest+len ; len <= col->getLength() ; len++) {
*(s++) = ' ';
}
@@ -573,7 +572,6 @@ template<typename INTTYPE> int dth_write
/* Make a safe copy of the text representation of a number, discarding
any terminal junk characters that may be in the buffer */
#define MAKE_COPY_BUFFER(sz) \
- assert(offset == 0); \
char copy_buff[sz]; \
if(len >= sz) return DTH_VALUE_TOO_LONG; \
strncpy(copy_buff, str, len); \
@@ -588,8 +586,8 @@ int dth_decode_tinyint(const NdbDictiona
return sprintf(str, "%d", (int) i) + 1; // +1 for null terminator
}
-int dth_encode_tinyint(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_tinyint(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(8);
int intval = 0;
@@ -610,7 +608,7 @@ int dth_decode_tiny_unsigned(const NdbDi
}
int dth_encode_tiny_unsigned(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(8);
Uint32 intval = 0;
@@ -630,8 +628,8 @@ int dth_decode_smallint(const NdbDiction
return sprintf(str, "%hd",* (short *) buf) + 1; // +1 for null terminator
}
-int dth_encode_smallint(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_smallint(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(8);
int intval = 0;
@@ -650,8 +648,8 @@ int dth_decode_small_unsigned(const NdbD
return sprintf(str, "%hu",* (short *) buf) + 1; // +1 for null terminator
}
-int dth_encode_small_unsigned(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_small_unsigned(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(8);
Uint32 intval = 0;
@@ -680,8 +678,8 @@ size_t dth_length_mediumint(const NdbDic
return len;
}
-int dth_encode_mediumint(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_mediumint(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
int intval = 0;
const int MAXVAL = 8388607;
const int MINVAL = -8388608;
@@ -730,7 +728,7 @@ size_t dth_length_medium_unsigned(const
}
int dth_encode_medium_unsigned(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(16);
Uint32 intval = 0;
@@ -766,8 +764,8 @@ int dth_decode_int(const NdbDictionary::
return sprintf(str, "%d",* (int *) buf) + 1; // +1 for null terminator
}
-int dth_encode_int(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_int(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(32);
int *ibuf = (int *) buf;
@@ -784,8 +782,8 @@ int dth_decode_unsigned(const NdbDiction
return sprintf(str, "%du",* (Uint32 *) buf) + 1; // +1 for null terminator
}
-int dth_encode_unsigned(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_unsigned(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(32);
Uint32 *ibuf = (Uint32 *) buf;
@@ -802,8 +800,8 @@ int dth_decode_bigint(const NdbDictionar
return sprintf(str, "%"PRId64,* (Int64 *) buf) + 1; // +1 for null
}
-int dth_encode_bigint(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_bigint(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(32);
int64_t *ibuf = (int64_t *) buf;
@@ -820,8 +818,8 @@ int dth_decode_ubigint(const NdbDictiona
return sprintf(str, "%"PRIu64,* (Uint64 *) buf) + 1; // +1 for null
}
-int dth_encode_ubigint(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
+int dth_encode_ubigint(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(32);
uint64_t *ibuf = (uint64_t *) buf;
@@ -843,9 +841,8 @@ size_t dth_length_enum(const NdbDictiona
return 1;
}
-int dth_encode_enum(const NdbDictionary::Column *col,
- size_t len, size_t offset, const char *str, void *buf) {
- assert(offset == 0);
+int dth_encode_enum(const NdbDictionary::Column *col, size_t len,
+ const char *str, void *buf) {
char *cbuf = (char *) buf;
*cbuf = *str;
return 1;
@@ -864,7 +861,7 @@ size_t dth_length_year(const NdbDictiona
}
int dth_encode_year(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
MAKE_COPY_BUFFER(8);
Uint32 intval = 0;
@@ -965,7 +962,7 @@ size_t dth_length_date(const NdbDictiona
}
int dth_encode_date(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
Int32 int_date;
Uint32 encoded_date;
time_helper tm = { 0,0,0,0,0,0,0, false };
@@ -1010,7 +1007,7 @@ size_t dth_length_time(const NdbDictiona
}
int dth_encode_time(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
Int32 int_time;
/* Make a safe (null-terminated) copy */
@@ -1051,7 +1048,7 @@ size_t dth_length_datetime(const NdbDict
}
int dth_encode_datetime(const NdbDictionary::Column *, size_t len,
- size_t offset, const char *str, void *buf) {
+ const char *str, void *buf) {
uint64_t int_datetime;
/* Make a safe (null-terminated) copy */
@@ -1102,8 +1099,7 @@ size_t dth_length_double(const NdbDictio
}
template <typename T> int dth_encode_fp(const NdbDictionary::Column *col,
- size_t len, size_t offset,
- const char *str, void *buf) {
+ size_t len, const char *str, void *buf) {
MAKE_COPY_BUFFER(64);
errno = 0;
double d = strtod(copy_buff, NULL);
@@ -1131,8 +1127,7 @@ size_t dth_length_decimal(const NdbDicti
return col->getScale() + col->getPrecision() + 3; // sign, point, and terminator
}
-int dth_encode_decimal(const NdbDictionary::Column *col,
- size_t len, size_t offset,
+int dth_encode_decimal(const NdbDictionary::Column *col, size_t len,
const char *str, void *buf) {
MAKE_COPY_BUFFER(64);
int scale = col->getScale();
=== modified file 'storage/ndb/memcache/src/Record.cc'
--- a/storage/ndb/memcache/src/Record.cc 2011-09-22 18:27:10 +0000
+++ b/storage/ndb/memcache/src/Record.cc 2011-09-24 05:45:06 +0000
@@ -153,11 +153,13 @@ bool Record::complete(NdbDictionary::Dic
*/
bool Record::appendCRLF(int id, size_t len, char *buffer) const {
int idx = map[id];
- if( handlers[idx]->contains_string
- && handlers[idx]->writeToNdb(specs[idx].column, 2, len, "\r\n", buffer)
- >= 0)
- {
- return true;
+ int length_bytes = handlers[idx]->contains_string;
+
+if(length_bytes) {
+ size_t offset = len + length_bytes - 1; /* see DataTypeHandler.h */
+ buffer[offset] = '\r';
+ buffer[offset+1] = '\n';
+ return true;
}
return false;
}
@@ -256,7 +258,7 @@ bool Record::setUint64Value(int id, Uint
int Record::encode(int id, const char *key, int nkey,
char *buffer) const {
- return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, 0, key,
+ return handlers[map[id]]->writeToNdb(specs[map[id]].column, nkey, key,
buffer + specs[map[id]].offset);
}
=== modified file 'storage/ndb/memcache/src/ndb_worker.cc'
--- a/storage/ndb/memcache/src/ndb_worker.cc 2011-09-23 18:29:30 +0000
+++ b/storage/ndb/memcache/src/ndb_worker.cc 2011-09-23 23:56:04 +0000
@@ -335,9 +335,10 @@ op_status_t worker_do_read(workitem *wqi
const char *dbkey = workitem_get_key_suffix(wqitem);
/* Use the workitem's inline buffer as a key buffer;
- allocate a new result buffer large enough for the result */
+ allocate a new result buffer large enough for the result.
+ Add 2 bytes to hold potential \r\n in a no-copy result. */
Operation op(plan, OP_READ, wqitem->ndb_key_buffer);
- workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer());
+ workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer() + 2);
op.buffer = wqitem->row_buffer_1;
/* Copy the key into the key buffer, ecnoding it for NDB */
@@ -517,6 +518,9 @@ op_status_t worker_do_math(workitem *wqi
void DB_callback(int result, NdbTransaction *tx, void *itemptr) {
workitem *wqitem = (workitem *) itemptr;
+
+ assert(tx->getNdb()->getCustomData() == wqitem);
+
ndb_pipeline * & pipeline = wqitem->pipeline;
status_block * return_status;
bool tx_did_match = false;
@@ -761,7 +765,7 @@ void incr_callback(int result, NdbTransa
bool finalize_read(workitem *wqitem) {
- DEBUG_ENTER();
+ DEBUG_PRINT("%d.%d",wqitem->pipeline->id, wqitem->id);
bool need_hash_item;
Operation op(wqitem->plan, OP_READ);
No bundle (reason: useless for push emails).
| Thread |
|---|
| • bzr push into mysql-5.5-cluster branch (john.duncan:3542 to 3544) | John David Duncan | 24 Sep |
