From: John David Duncan Date: July 19 2012 11:50pm Subject: bzr push into mysql-5.5-cluster-7.2 branch (john.duncan:3968 to 3970) List-Archive: http://lists.mysql.com/commits/144469 Message-Id: <20120719235027.332.24318.3970@dhcp-whq-twvpn-1-vpnpool-10-159-217-87.vpn.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 3970 John David Duncan 2012-07-17 Fix bugs in NDB+Memcached where multi-part keys were properly supported. added: mysql-test/suite/ndb_memcache/r/mpart_key.result mysql-test/suite/ndb_memcache/t/mpart_key.test modified: storage/ndb/memcache/include/Operation.h storage/ndb/memcache/src/ExternalValue.cc storage/ndb/memcache/src/Operation.cc storage/ndb/memcache/src/ndb_worker.cc 3969 John David Duncan 2012-07-17 Fix so that memcached no honors the NDB_CONNECTSTRING variable. Connectstring is: (1st choice) connecstring specified on the command line (2nd choice) NDB_CONNECTSTRING if set (3rd choice) default of localhost:1186 modified: storage/ndb/memcache/src/ndb_engine.c 3968 Mauritz Sundell 2012-07-12 [merge] Merge 7.1 -> 7.2 modified: storage/ndb/test/run-test/conf-ndb07.cnf === added file 'mysql-test/suite/ndb_memcache/r/mpart_key.result' --- a/mysql-test/suite/ndb_memcache/r/mpart_key.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_memcache/r/mpart_key.result 2012-07-18 06:54:14 +0000 @@ -0,0 +1,5 @@ +SELECT * FROM ndbmemcache.hashtags; +hashtag tweet_id author +mysql 1 frederick +oscon 1 fred +oscon 2 freddy === added file 'mysql-test/suite/ndb_memcache/t/mpart_key.test' --- a/mysql-test/suite/ndb_memcache/t/mpart_key.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_memcache/t/mpart_key.test 2012-07-18 06:54:14 +0000 @@ -0,0 +1,69 @@ +# CHAR + +--source suite/ndb_memcache/include/have_memcache.inc +--source suite/ndb_memcache/include/memcached_wait_for_ready.inc + + +# +# Configuration change for this test +# + +--disable_query_log +--disable_result_log +USE ndbmemcache; + +CREATE TABLE hashtags ( + hashtag varchar(20), + tweet_id int, + author varchar(15), + primary key(hashtag, tweet_id)) +ENGINE=ndb; + +INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) + VALUES("test_mkey", "ndbmemcache", "hashtags", "hashtag,tweet_id", "author"); + +INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) + VALUES(0, "hashtag:", "ndb-only", "test_mkey"); + +UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0; +--enable_query_log +--enable_result_log + +# +# Memcache operations for this test +# + +--perl + +use strict; +use Carp; +use lib "lib"; +use My::Memcache; + +my $port = $ENV{NDB_MEMCACHED_1_PORT} or die "Need NDB_MEMCACHED_1_PORT"; + +# Use a binary protocol connection (so keys can contain spaces) +my $mc = My::Memcache::Binary->new(); +my $r = $mc->connect("localhost",$port); + +my $cf_gen = $mc->wait_for_reconf(); + +if($cf_gen == 0) { + Carp::confess("FAILED WAIT_FOR_RECONF"); +} + + +$mc->set("hashtag:oscon\t1","fred") || Carp::confess("FAILED # 01 (SET)"); +$mc->add("hashtag:mysql\t1","frederick") || Carp::confess("FAILED # 02 (SET)"); +$mc->set("hashtag:oscon\t2","freddy") || Carp::confess("FAILED # 03 (SET)"); + +($mc->get("hashtag:oscon\t1") == "fred") || Carp::confess("FAILED # 04 (GET)"); +($mc->get("hashtag:mysql\t1") == "frederick") || Carp::confess("FAILED # 05 (GET)"); +($mc->get("hashtag:oscon\t2") == "freddy") || Carp::confess("FAILED # 06 (GET)"); + + +EOF + + +--sorted_result +SELECT * FROM ndbmemcache.hashtags; === modified file 'storage/ndb/memcache/include/Operation.h' --- a/storage/ndb/memcache/include/Operation.h 2011-12-18 23:21:21 +0000 +++ b/storage/ndb/memcache/include/Operation.h 2012-07-18 06:54:14 +0000 @@ -71,14 +71,16 @@ public: // Methods for writing to the key record size_t requiredKeyBuffer(); void clearKeyNullBits(); + bool setKey(int nparts, const char *key_str, size_t key_str_len); bool setKeyPart(int id, const char *strval, size_t strlen); bool setKeyPartInt(int id, int value); void setKeyPartNull(int id); - - // Methods for writing to the row + + // Methods for writing to the row size_t requiredBuffer(); void setNullBits(); void clearNullBits(); + bool setKeyFieldsInRow(int nparts, const char *key_str, size_t key_str_len); bool setColumn(int id, const char *strval, size_t strlen); bool setColumnInt(int id, int value); bool setColumnBigUnsigned(int id, Uint64 value); === modified file 'storage/ndb/memcache/src/ExternalValue.cc' --- a/storage/ndb/memcache/src/ExternalValue.cc 2012-05-18 00:07:06 +0000 +++ b/storage/ndb/memcache/src/ExternalValue.cc 2012-07-18 06:54:14 +0000 @@ -28,6 +28,7 @@ #include "status_block.h" #include "ExpireTime.h" #include "ExternalValue.h" +#include "TabSeparatedValues.h" /* Externs */ extern EXTENSION_LOGGER_DESCRIPTOR *logger; @@ -108,11 +109,12 @@ int ExternalValue::do_delete(memory_pool } -inline bool ExternalValue::setupKey(workitem *item, Operation &op) { +bool ExternalValue::setupKey(workitem *item, Operation &op) { + const TableSpec & spec = * (item->plan->spec); op.key_buffer = item->ndb_key_buffer; - op.clearKeyNullBits(); const char *dbkey = workitem_get_key_suffix(item); - return op.setKeyPart(COL_STORE_KEY, dbkey, item->base.nsuffix); + + return op.setKey(spec.nkeycols, dbkey, item->base.nsuffix); } @@ -561,11 +563,6 @@ bool ExternalValue::updatePart(int id, i void ExternalValue::setMiscColumns(Operation & op) const { - //fixme: the key is not "misc" ! - /* Set the key column in the header row */ - const char *dbkey = workitem_get_key_suffix(wqitem); - op.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); - /* Set the CAS value in the header row */ if(do_server_cas) op.setColumnBigUnsigned(COL_STORE_CAS, * wqitem->cas); @@ -587,6 +584,9 @@ void ExternalValue::setMiscColumns(Opera void ExternalValue::setValueColumns(Operation & op) const { + const char *dbkey = workitem_get_key_suffix(wqitem); + op.setKeyFieldsInRow(wqitem->plan->spec->nkeycols, dbkey, wqitem->base.nsuffix); + if(shouldExternalize(new_hdr.length)) { /* Long value */ DEBUG_PRINT("[long]"); === modified file 'storage/ndb/memcache/src/Operation.cc' --- a/storage/ndb/memcache/src/Operation.cc 2011-12-09 08:51:24 +0000 +++ b/storage/ndb/memcache/src/Operation.cc 2012-07-18 06:54:14 +0000 @@ -21,6 +21,7 @@ #include "Operation.h" +#include "TabSeparatedValues.h" /* @@ -108,3 +109,53 @@ NdbIndexScanOperation * Operation::scanI sizeof(opts)); } + +bool Operation::setKey(int nparts, const char *dbkey, size_t key_len ) { + + clearKeyNullBits(); + + if(nparts > 1) { + TabSeparatedValues tsv(dbkey, nparts, key_len); + int idx = 0; + do { + if(tsv.getLength()) { + DEBUG_PRINT("Set key part %d [%.*s]", idx, tsv.getLength(), tsv.getPointer()); + if(! setKeyPart(COL_STORE_KEY+idx, tsv.getPointer(), tsv.getLength())) + return false; + } + else { + DEBUG_PRINT("Set key part NULL: %d ", idx); + setKeyPartNull(COL_STORE_KEY+idx); + } + idx++; + } while (tsv.advance()); + } + else { + return setKeyPart(COL_STORE_KEY, dbkey, key_len); + } +} + + +bool Operation::setKeyFieldsInRow(int nparts, const char *dbkey, size_t key_len ) { + if(nparts > 1) { + TabSeparatedValues tsv(dbkey, nparts, key_len); + int idx = 0; + do { + if(tsv.getLength()) { + DEBUG_PRINT("Set key part %d [%.*s]", idx, tsv.getLength(), tsv.getPointer()); + if(! setColumn(COL_STORE_KEY+idx, tsv.getPointer(), tsv.getLength())) + return false; + } + else { + DEBUG_PRINT("Set key part NULL: %d ", idx); + setColumnNull(COL_STORE_KEY+idx); + } + idx++; + } while (tsv.advance()); + } + else { + return setColumn(COL_STORE_KEY, dbkey, key_len); + } +} + + === modified file 'storage/ndb/memcache/src/ndb_engine.c' --- a/storage/ndb/memcache/src/ndb_engine.c 2012-05-02 03:01:21 +0000 +++ b/storage/ndb/memcache/src/ndb_engine.c 2012-07-18 04:04:00 +0000 @@ -131,6 +131,12 @@ ENGINE_ERROR_CODE create_instance(uint64 ndb_eng->startup_options.debug_enable = false; ndb_eng->startup_options.reconf_enable = true; + /* Now let NDB_CONNECTSRING environment variable override the default */ + const char * env_connectstring = getenv("NDB_CONNECTSTRING"); + if(env_connectstring) + ndb_eng->startup_options.connectstring = env_connectstring; + + /* Set engine informational structure */ ndb_eng->info.info.description = "NDB Memcache " VERSION; ndb_eng->info.info.num_features = 3; ndb_eng->info.info.features[0].feature = ENGINE_FEATURE_CAS; === modified file 'storage/ndb/memcache/src/ndb_worker.cc' --- a/storage/ndb/memcache/src/ndb_worker.cc 2012-05-18 00:07:06 +0000 +++ b/storage/ndb/memcache/src/ndb_worker.cc 2012-07-18 06:54:14 +0000 @@ -88,7 +88,7 @@ private: QueryPlan * &plan; /* Private methods*/ - void setKeyForReading(Operation &op); + bool setKeyForReading(Operation &op); }; @@ -278,9 +278,9 @@ op_status_t WorkerStep1::do_delete() { Operation op(plan, OP_DELETE); op.key_buffer = wqitem->ndb_key_buffer; - op.clearKeyNullBits(); const char *dbkey = workitem_get_key_suffix(wqitem); - op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + if(! op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix)) + return op_overflow; tx = op.startTransaction(wqitem->ndb_instance->db); @@ -326,8 +326,7 @@ op_status_t WorkerStep1::do_write() { bool op_ok; /* Set the key */ - op.clearKeyNullBits(); - op_ok = op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + op_ok = op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix); if(! op_ok) return op_overflow; /* Allocate and encode the buffer for the row */ @@ -336,7 +335,7 @@ op_status_t WorkerStep1::do_write() { /* Set the row */ op.setNullBits(); - op.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + op.setKeyFieldsInRow(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix); if(plan->spec->nvaluecols > 1) { /* Multiple Value Columns */ @@ -457,7 +456,7 @@ op_status_t WorkerStep1::do_read() { DEBUG_ENTER(); Operation op(plan, OP_READ); - setKeyForReading(op); + if(! setKeyForReading(op)) return op_overflow; NdbOperation::LockMode lockmode; NdbTransaction::ExecType commitflag; @@ -492,7 +491,7 @@ op_status_t WorkerStep1::do_append() { return op_not_supported; Operation op(plan, OP_READ); - setKeyForReading(op); + if(! setKeyForReading(op)) return op_overflow; /* Read with an exculsive lock */ if(! op.readTuple(tx, NdbOperation::LM_Exclusive)) { @@ -508,7 +507,7 @@ op_status_t WorkerStep1::do_append() { } -void WorkerStep1::setKeyForReading(Operation &op) { +bool WorkerStep1::setKeyForReading(Operation &op) { DEBUG_ENTER(); /* Use the workitem's inline key buffer */ @@ -519,14 +518,16 @@ void WorkerStep1::setKeyForReading(Opera workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer() + 2); op.buffer = wqitem->row_buffer_1; - /* Copy the key into the key buffer, ecnoding it for NDB */ + /* set the key */ op.clearKeyNullBits(); const char *dbkey = workitem_get_key_suffix(wqitem); - op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + if(! op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix)) + return false; /* Start a transaction */ tx = op.startTransaction(wqitem->ndb_instance->db); DEBUG_ASSERT(tx); + return true; } @@ -583,12 +584,11 @@ op_status_t WorkerStep1::do_math() { op3.buffer = wqitem->row_buffer_2; /* The two items share a key buffer, so we encode the key just once */ - op1.clearKeyNullBits(); - op1.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + op1.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix); /* The insert operation also needs the key written into the row */ op2.clearNullBits(); - op2.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix); + op2.setKeyFieldsInRow(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix); /* CAS */ if(server_cas) { @@ -942,7 +942,8 @@ void worker_append(NdbTransaction *tx, w /* Set the row */ op.setNullBits(); - op.setColumn(COL_STORE_KEY, workitem_get_key_suffix(item), item->base.nsuffix); + op.setKeyFieldsInRow(item->plan->spec->nkeycols, + workitem_get_key_suffix(item), item->base.nsuffix); op.setColumn(COL_STORE_VALUE, current_val, total_len); if(item->prefix_info.has_cas_col) op.setColumnBigUnsigned(COL_STORE_CAS, * item->cas); No bundle (reason: useless for push emails).