List:Commits« Previous MessageNext Message »
From:John David Duncan Date:July 19 2012 11:50pm
Subject:bzr push into mysql-5.5-cluster-7.2 branch (john.duncan:3968 to 3970)
View as plain text  
 3970 John David Duncan	2012-07-17
      Fix bugs in NDB+Memcached where multi-part keys were properly supported.

    added:
      mysql-test/suite/ndb_memcache/r/mpart_key.result
      mysql-test/suite/ndb_memcache/t/mpart_key.test
    modified:
      storage/ndb/memcache/include/Operation.h
      storage/ndb/memcache/src/ExternalValue.cc
      storage/ndb/memcache/src/Operation.cc
      storage/ndb/memcache/src/ndb_worker.cc
 3969 John David Duncan	2012-07-17
      Fix so that memcached no honors the NDB_CONNECTSTRING variable.
      Connectstring is: 
        (1st choice) connecstring specified on the command line
        (2nd choice) NDB_CONNECTSTRING if set
        (3rd choice) default of localhost:1186

    modified:
      storage/ndb/memcache/src/ndb_engine.c
 3968 Mauritz Sundell	2012-07-12 [merge]
      Merge 7.1 -> 7.2

    modified:
      storage/ndb/test/run-test/conf-ndb07.cnf
=== added file 'mysql-test/suite/ndb_memcache/r/mpart_key.result'
--- a/mysql-test/suite/ndb_memcache/r/mpart_key.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb_memcache/r/mpart_key.result	2012-07-18 06:54:14 +0000
@@ -0,0 +1,5 @@
+SELECT * FROM ndbmemcache.hashtags;
+hashtag	tweet_id	author
+mysql	1	frederick
+oscon	1	fred
+oscon	2	freddy

=== added file 'mysql-test/suite/ndb_memcache/t/mpart_key.test'
--- a/mysql-test/suite/ndb_memcache/t/mpart_key.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb_memcache/t/mpart_key.test	2012-07-18 06:54:14 +0000
@@ -0,0 +1,69 @@
+# CHAR
+
+--source suite/ndb_memcache/include/have_memcache.inc
+--source suite/ndb_memcache/include/memcached_wait_for_ready.inc
+
+
+# 
+# Configuration change for this test
+#
+
+--disable_query_log
+--disable_result_log
+USE ndbmemcache;
+
+CREATE TABLE hashtags (
+  hashtag varchar(20),
+  tweet_id int,
+  author varchar(15),
+  primary key(hashtag, tweet_id))
+ENGINE=ndb;
+
+INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns)
+  VALUES("test_mkey", "ndbmemcache", "hashtags", "hashtag,tweet_id", "author");
+
+INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) 
+  VALUES(0, "hashtag:", "ndb-only", "test_mkey");
+
+UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0;
+--enable_query_log
+--enable_result_log
+
+# 
+#  Memcache operations for this test
+#
+
+--perl
+
+use strict;
+use Carp;
+use lib "lib";
+use My::Memcache;
+
+my $port = $ENV{NDB_MEMCACHED_1_PORT} or die "Need NDB_MEMCACHED_1_PORT";
+
+# Use a binary protocol connection (so keys can contain spaces)
+my $mc = My::Memcache::Binary->new();  
+my $r = $mc->connect("localhost",$port);
+
+my $cf_gen = $mc->wait_for_reconf();
+
+if($cf_gen == 0) {
+  Carp::confess("FAILED WAIT_FOR_RECONF");
+}
+
+
+$mc->set("hashtag:oscon\t1","fred")      || Carp::confess("FAILED # 01 (SET)");
+$mc->add("hashtag:mysql\t1","frederick") || Carp::confess("FAILED # 02 (SET)");
+$mc->set("hashtag:oscon\t2","freddy")    || Carp::confess("FAILED # 03 (SET)");
+  
+($mc->get("hashtag:oscon\t1") == "fred")      || Carp::confess("FAILED # 04 (GET)");
+($mc->get("hashtag:mysql\t1") == "frederick") || Carp::confess("FAILED # 05 (GET)");
+($mc->get("hashtag:oscon\t2") == "freddy")    || Carp::confess("FAILED # 06 (GET)");
+
+
+EOF
+
+
+--sorted_result
+SELECT * FROM ndbmemcache.hashtags;

=== modified file 'storage/ndb/memcache/include/Operation.h'
--- a/storage/ndb/memcache/include/Operation.h	2011-12-18 23:21:21 +0000
+++ b/storage/ndb/memcache/include/Operation.h	2012-07-18 06:54:14 +0000
@@ -71,14 +71,16 @@ public: 
   // Methods for writing to the key record
   size_t requiredKeyBuffer();
   void clearKeyNullBits();
+  bool setKey(int nparts, const char *key_str, size_t key_str_len);
   bool setKeyPart(int id, const char *strval, size_t strlen);
   bool setKeyPartInt(int id, int value);
   void setKeyPartNull(int id);
-
-  // Methods for writing to the row   
+  
+  // Methods for writing to the row 
   size_t requiredBuffer();
   void setNullBits();
   void clearNullBits();
+  bool setKeyFieldsInRow(int nparts, const char *key_str, size_t key_str_len);
   bool setColumn(int id, const char *strval, size_t strlen);
   bool setColumnInt(int id, int value);
   bool setColumnBigUnsigned(int id, Uint64 value);

=== modified file 'storage/ndb/memcache/src/ExternalValue.cc'
--- a/storage/ndb/memcache/src/ExternalValue.cc	2012-05-18 00:07:06 +0000
+++ b/storage/ndb/memcache/src/ExternalValue.cc	2012-07-18 06:54:14 +0000
@@ -28,6 +28,7 @@
 #include "status_block.h"
 #include "ExpireTime.h"
 #include "ExternalValue.h"
+#include "TabSeparatedValues.h"
 
 /* Externs */
 extern EXTENSION_LOGGER_DESCRIPTOR *logger;
@@ -108,11 +109,12 @@ int ExternalValue::do_delete(memory_pool
 }
 
 
-inline bool ExternalValue::setupKey(workitem *item, Operation &op) {
+bool ExternalValue::setupKey(workitem *item, Operation &op) { 
+  const TableSpec & spec = * (item->plan->spec);
   op.key_buffer = item->ndb_key_buffer;
-  op.clearKeyNullBits();
   const char *dbkey = workitem_get_key_suffix(item);
-  return op.setKeyPart(COL_STORE_KEY, dbkey, item->base.nsuffix);
+  
+  return op.setKey(spec.nkeycols, dbkey, item->base.nsuffix);
 }
 
 
@@ -561,11 +563,6 @@ bool ExternalValue::updatePart(int id, i
 
 
 void ExternalValue::setMiscColumns(Operation & op) const {
-  //fixme: the key is not "misc" !
-  /* Set the key column in the header row */
-  const char *dbkey = workitem_get_key_suffix(wqitem);
-  op.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);
-
   /* Set the CAS value in the header row */
   if(do_server_cas) 
     op.setColumnBigUnsigned(COL_STORE_CAS, * wqitem->cas);  
@@ -587,6 +584,9 @@ void ExternalValue::setMiscColumns(Opera
 
 
 void ExternalValue::setValueColumns(Operation & op) const {
+  const char *dbkey = workitem_get_key_suffix(wqitem);
+  op.setKeyFieldsInRow(wqitem->plan->spec->nkeycols, dbkey, wqitem->base.nsuffix);
+  
   if(shouldExternalize(new_hdr.length)) {
     /* Long value */
     DEBUG_PRINT("[long]");

=== modified file 'storage/ndb/memcache/src/Operation.cc'
--- a/storage/ndb/memcache/src/Operation.cc	2011-12-09 08:51:24 +0000
+++ b/storage/ndb/memcache/src/Operation.cc	2012-07-18 06:54:14 +0000
@@ -21,6 +21,7 @@
 
 
 #include "Operation.h"
+#include "TabSeparatedValues.h"
 
 
 /* 
@@ -108,3 +109,53 @@ NdbIndexScanOperation * Operation::scanI
                        sizeof(opts));
 }
 
+
+bool Operation::setKey(int nparts, const char *dbkey, size_t key_len ) {
+  
+  clearKeyNullBits();
+
+  if(nparts > 1) {
+    TabSeparatedValues tsv(dbkey, nparts, key_len);
+    int idx = 0;
+    do {
+      if(tsv.getLength()) {
+        DEBUG_PRINT("Set key part %d [%.*s]", idx, tsv.getLength(), tsv.getPointer());
+        if(! setKeyPart(COL_STORE_KEY+idx, tsv.getPointer(), tsv.getLength()))
+          return false;
+      }
+      else {
+        DEBUG_PRINT("Set key part NULL: %d ", idx);
+        setKeyPartNull(COL_STORE_KEY+idx);
+      }
+      idx++;
+    } while (tsv.advance());
+  }
+  else { 
+    return setKeyPart(COL_STORE_KEY, dbkey, key_len);
+  }
+}
+
+
+bool Operation::setKeyFieldsInRow(int nparts, const char *dbkey, size_t key_len ) {
+  if(nparts > 1) {
+    TabSeparatedValues tsv(dbkey, nparts, key_len);
+    int idx = 0;
+    do {
+      if(tsv.getLength()) {
+        DEBUG_PRINT("Set key part %d [%.*s]", idx, tsv.getLength(), tsv.getPointer());
+        if(! setColumn(COL_STORE_KEY+idx, tsv.getPointer(), tsv.getLength()))
+          return false;
+      }
+      else {
+        DEBUG_PRINT("Set key part NULL: %d ", idx);
+        setColumnNull(COL_STORE_KEY+idx);
+      }
+      idx++;
+    } while (tsv.advance());
+  }
+  else {
+    return setColumn(COL_STORE_KEY, dbkey, key_len);
+  }
+}
+
+

=== modified file 'storage/ndb/memcache/src/ndb_engine.c'
--- a/storage/ndb/memcache/src/ndb_engine.c	2012-05-02 03:01:21 +0000
+++ b/storage/ndb/memcache/src/ndb_engine.c	2012-07-18 04:04:00 +0000
@@ -131,6 +131,12 @@ ENGINE_ERROR_CODE create_instance(uint64
   ndb_eng->startup_options.debug_enable  = false;
   ndb_eng->startup_options.reconf_enable = true;
 
+  /* Now let NDB_CONNECTSRING environment variable override the default */
+  const char * env_connectstring = getenv("NDB_CONNECTSTRING");
+  if(env_connectstring)
+    ndb_eng->startup_options.connectstring = env_connectstring;
+
+  /* Set engine informational structure */
   ndb_eng->info.info.description = "NDB Memcache " VERSION;
   ndb_eng->info.info.num_features = 3;
   ndb_eng->info.info.features[0].feature = ENGINE_FEATURE_CAS;

=== modified file 'storage/ndb/memcache/src/ndb_worker.cc'
--- a/storage/ndb/memcache/src/ndb_worker.cc	2012-05-18 00:07:06 +0000
+++ b/storage/ndb/memcache/src/ndb_worker.cc	2012-07-18 06:54:14 +0000
@@ -88,7 +88,7 @@ private:
   QueryPlan * &plan;
 
   /* Private methods*/
-  void setKeyForReading(Operation &op);
+  bool setKeyForReading(Operation &op);
 };
 
 
@@ -278,9 +278,9 @@ op_status_t WorkerStep1::do_delete() {
   Operation op(plan, OP_DELETE);
   
   op.key_buffer = wqitem->ndb_key_buffer;
-  op.clearKeyNullBits();
   const char *dbkey = workitem_get_key_suffix(wqitem);
-  op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);  
+  if(! op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix))
+    return op_overflow;
   
   tx = op.startTransaction(wqitem->ndb_instance->db);
   
@@ -326,8 +326,7 @@ op_status_t WorkerStep1::do_write() {
   bool op_ok;
   
   /* Set the key */
-  op.clearKeyNullBits();
-  op_ok = op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);
+  op_ok = op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix);
   if(! op_ok) return op_overflow;
   
   /* Allocate and encode the buffer for the row */ 
@@ -336,7 +335,7 @@ op_status_t WorkerStep1::do_write() {
   
   /* Set the row */
   op.setNullBits();
-  op.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);
+  op.setKeyFieldsInRow(plan->spec->nkeycols,  dbkey, wqitem->base.nsuffix);
   
   if(plan->spec->nvaluecols > 1) {
     /* Multiple Value Columns */
@@ -457,7 +456,7 @@ op_status_t WorkerStep1::do_read() {
   DEBUG_ENTER();
   
   Operation op(plan, OP_READ);
-  setKeyForReading(op);
+  if(! setKeyForReading(op)) return op_overflow;
   
   NdbOperation::LockMode lockmode;
   NdbTransaction::ExecType commitflag;
@@ -492,7 +491,7 @@ op_status_t WorkerStep1::do_append() {
     return op_not_supported;
   
   Operation op(plan, OP_READ);
-  setKeyForReading(op);
+  if(! setKeyForReading(op)) return op_overflow;
   
   /* Read with an exculsive lock */
   if(! op.readTuple(tx, NdbOperation::LM_Exclusive)) {
@@ -508,7 +507,7 @@ op_status_t WorkerStep1::do_append() {
 }
 
 
-void WorkerStep1::setKeyForReading(Operation &op) {
+bool WorkerStep1::setKeyForReading(Operation &op) {
   DEBUG_ENTER();
   
   /* Use the workitem's inline key buffer */
@@ -519,14 +518,16 @@ void WorkerStep1::setKeyForReading(Opera
   workitem_allocate_rowbuffer_1(wqitem, op.requiredBuffer() + 2);
   op.buffer = wqitem->row_buffer_1;
   
-  /* Copy the key into the key buffer, ecnoding it for NDB */
+  /* set the key */
   op.clearKeyNullBits();
   const char *dbkey = workitem_get_key_suffix(wqitem);
-  op.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);  
+  if(! op.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix))
+    return false;
   
   /* Start a transaction */
   tx = op.startTransaction(wqitem->ndb_instance->db);
   DEBUG_ASSERT(tx);
+  return true;
 }
 
 
@@ -583,12 +584,11 @@ op_status_t WorkerStep1::do_math() {
     op3.buffer = wqitem->row_buffer_2;
     
     /* The two items share a key buffer, so we encode the key just once */
-    op1.clearKeyNullBits();
-    op1.setKeyPart(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);  
+    op1.setKey(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix);
     
     /* The insert operation also needs the key written into the row */
     op2.clearNullBits();
-    op2.setColumn(COL_STORE_KEY, dbkey, wqitem->base.nsuffix);
+    op2.setKeyFieldsInRow(plan->spec->nkeycols, dbkey, wqitem->base.nsuffix);
     
     /* CAS */
     if(server_cas) {
@@ -942,7 +942,8 @@ void worker_append(NdbTransaction *tx, w
   
   /* Set the row */
   op.setNullBits();
-  op.setColumn(COL_STORE_KEY, workitem_get_key_suffix(item), item->base.nsuffix);
+  op.setKeyFieldsInRow(item->plan->spec->nkeycols, 
+                       workitem_get_key_suffix(item), item->base.nsuffix);
   op.setColumn(COL_STORE_VALUE, current_val, total_len);
   if(item->prefix_info.has_cas_col) 
     op.setColumnBigUnsigned(COL_STORE_CAS, * item->cas);

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.5-cluster-7.2 branch (john.duncan:3968 to 3970) John David Duncan20 Jul