From: John David Duncan Date: October 13 2012 5:05am Subject: bzr push into mysql-5.5-cluster-7.2 branch (john.duncan:4029 to 4030) List-Archive: http://lists.mysql.com/commits/145028 Message-Id: <20121013050524.4452.30931.4030@dhcp-whq-twvpn-1-vpnpool-10-159-222-129.vpn.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 4030 John David Duncan 2012-10-12 [merge] merge added: mysql-test/suite/ndb_memcache/include/misc_tables.inc modified: mysql-test/lib/My/Memcache.pm mysql-test/suite/ndb_memcache/include/datatypes_tables.inc mysql-test/suite/ndb_memcache/r/external_values.result mysql-test/suite/ndb_memcache/t/external_values.test mysql-test/suite/ndb_memcache/t/math3.test mysql-test/suite/ndb_memcache/t/mpart_key.test mysql-test/suite/ndb_memcache/t/type_char.test mysql-test/suite/ndb_memcache/t/unique_idx.test 4029 magnus.blaudd@stripped 2012-10-12 ndb - keep WITH_NDB_JAVA set to OFF a few more days modified: storage/ndb/CMakeLists.txt === modified file 'mysql-test/lib/My/Memcache.pm' --- a/mysql-test/lib/My/Memcache.pm 2012-10-12 04:54:41 +0000 +++ b/mysql-test/lib/My/Memcache.pm 2012-10-12 21:48:24 +0000 @@ -34,7 +34,7 @@ ### $mc->delete(key) returns 1 on success, 0 on failure ### $mc->stats(stat_key) get stats; returns a hash ### $mc->incr(key, amount, [initial]) returns the new value or undef -### $mc->decr(key, amount, [initial]) like incr. The third argument is used +### $mc->decr(key, amount, [initial]) like incr. The third argument is used ### in the Binary protocol ONLY. ### $mc->flush() flush_all ### @@ -53,14 +53,10 @@ ### TO DO: REPLACE with CAS ID. use strict; -use lib 'lib'; use IO::Socket::INET; use IO::File; use Carp; use Time::HiRes; -use mtr_report; # for main::mtr_verbose() - -require "mtr_process.pl"; # for mtr_ping_port() package My::Memcache; @@ -68,19 +64,21 @@ sub new { my $pkg = shift; # min/max wait refer to msec. wait during temporary errors. Both powers of 2. bless { "created" => 1 , "error" => "" , "cf_gen" => 0, - "exptime" => 0 , "flags" => 0 , - "minWait" => 4, "maxWait" => 8192, + "req_id" => 0, "minWait" => 4, "maxWait" => 8192, "temp_errors" => 0 , "total_wait" => 0 }, $pkg; } + +# Common code to ASCII and BINARY protocols: + sub fail { my $self = shift; my $msg = - "error: " . $self->{error} . "\n" . - "bin_req_id: " . $self->{bin_req_id} . "\n" . - "temp_errors: " . $self->{temp_errors} . "\n". - "total_wait: " . $self->{total_wait} . "\n"; + "error: " . $self->{error} . "\n" . + "req_id: " . $self->{req_id} . "\n" . + "temp_errors: " . $self->{temp_errors} . "\n". + "total_wait: " . $self->{total_wait} . "\n"; while(my $extra = shift) { $msg .= $extra . "\n"; } @@ -91,21 +89,22 @@ sub connect { my $self = shift; my $host = shift; my $port = shift; + my $conn; # Wait for memcached to be ready, up to ten seconds. my $retries = 100; - while($retries && (::mtr_ping_port($port) == 0)) - { - Time::HiRes::usleep(100 * 1000); - $retries--; - } + do { + $conn = IO::Socket::INET->new(PeerAddr => "$host:$port", Proto => "tcp"); + if(! $conn) { + Time::HiRes::usleep(100 * 1000); + $retries--; + } + } while($retries && !$conn); - my $conn = IO::Socket::INET->new(PeerAddr => "$host:$port", Proto => "tcp"); if($conn) { $self->{connection} = $conn; $self->{connected} = 1; $self->{server} = "$host:$port"; - $self->{bin_req_id} = 0; return 1; } $self->{error} = "CONNECTION_FAILED"; @@ -117,20 +116,6 @@ sub DESTROY { $self->{connection}->close(); } -sub note_config_version { - my $self = shift; - - my $vardir = $ENV{MYSQLTEST_VARDIR}; - # Fetch the memcached current config generation number and save it - my %stats = $self->stats("reconf"); - my $F = IO::File->new("$vardir/tmp/memcache_cf_gen", "w") or die; - my $ver = $stats{"Running"}; - print $F "$ver\n"; - $F->close(); - - $self->{cf_gen} = $ver; -} - sub set_expires { my $self = shift; my $delta = shift; @@ -145,6 +130,32 @@ sub set_flags { $self->{flags} = $flags; } +# Some member variables are per-request. +# Clear them in preparation for a new request, and increment the request counter. +sub new_request { + my $self = shift; + $self->{error} = undef; + $self->{exptime} = 0; + $self->{flags} = 0; + $self->{req_id}++; +} + + +# note_config_version and wait_for_reconf are only for use by mysql-test-run +sub note_config_version { + my $self = shift; + + my $vardir = $ENV{MYSQLTEST_VARDIR}; + # Fetch the memcached current config generation number and save it + my %stats = $self->stats("reconf"); + my $F = IO::File->new("$vardir/tmp/memcache_cf_gen", "w") or die; + my $ver = $stats{"Running"}; + print $F "$ver\n"; + $F->close(); + + $self->{cf_gen} = $ver; +} + sub wait_for_reconf { my $self = shift; @@ -174,7 +185,6 @@ sub wait_for_reconf { return $new_gen; } - # wait_for_config_generation($cf_gen) # Wait until memcached is running config generation >= to $cf_gen # Returns 0 on error/timeout, or the actual running generation number @@ -198,17 +208,23 @@ sub wait_for_config_generation { return $ready; } +# ----------------------------------------------------------------------- +# ------------------ ASCII PROTOCOL -------------------- +# ----------------------------------------------------------------------- + sub ascii_command { my $self = shift; my $packet = shift; my $sock = $self->{connection}; my $waitTime = $self->{minWait}; my $maxWait = $self->{maxWait}; + my $reply; do { + $self->new_request(); $sock->print($packet) || Carp::confess("send error: ". $packet); - $self->{error} = $sock->getline(); - $self->normalize_error(); + $reply = $sock->getline(); + $self->normalize_error($reply); if($self->{error} eq "SERVER_TEMPORARY_ERROR") { if($waitTime < $maxWait) { $self->{temp_errors} += 1; @@ -221,7 +237,7 @@ sub ascii_command { } } while($self->{error} eq "SERVER_TEMPORARY_ERROR" && $waitTime <= $maxWait); - return $self->{error}; + return $reply; } @@ -229,8 +245,7 @@ sub delete { my $self = shift; my $key = shift; - $self->ascii_command("delete $key\r\n"); - return ($self->{error} =~ "^DELETED"); + return ($self->ascii_command("delete $key\r\n") =~ "^DELETED"); } @@ -306,11 +321,8 @@ sub _txt_math { my ($self, $cmd, $key, $delta) = @_; my $response = $self->ascii_command("$cmd $key $delta \r\n"); - if ($response =~ "^NOT_FOUND") { - $self->{error} = "NOT_FOUND"; - return undef; - } - elsif ($response =~ "ERROR") { + if ($response =~ "^NOT_FOUND" || $response =~ "ERROR") { + $self->normalize_error($response); return undef; } @@ -336,6 +348,7 @@ sub stats { my $key = shift; my $sock = $self->{connection}; + $self->new_request(); $sock->print("stats $key\r\n") || Carp::confess "send error"; $self->{error} = "OK"; @@ -355,16 +368,19 @@ sub flush { my $self = shift; my $key = shift; my $result = $self->ascii_command("flush_all\r\n"); - return ($self->{error} =~ "^OK"); + return ($self->{error} eq "OK"); } # Try to provide consistent error messagees across ascii & binary protocols sub normalize_error { my $self = shift; + my $reply = shift; my %error_message = ( "STORED\r\n" => "OK", "EXISTS\r\n" => "KEY_EXISTS", + "NOT_FOUND\r\n" => "NOT_FOUND", + "NOT_STORED\r\n" => "NOT_STORED", "CLIENT_ERROR value too big\r\n" => "VALUE_TOO_LARGE", "SERVER_ERROR object too large for cache\r\n" => "VALUE_TOO_LARGE", "CLIENT_ERROR invalid arguments\r\n" => "INVALID_ARGUMENTS", @@ -374,8 +390,7 @@ sub normalize_error { "SERVER_ERROR internal\r\n" => "INTERNAL_ERROR", "SERVER_ERROR temporary failure\r\n" => "SERVER_TEMPORARY_ERROR" ); - my $norm_error = $error_message{$self->{error}}; - $self->{error} = $norm_error if(defined($norm_error)); + $self->{error} = $error_message{$reply}; return 0; } @@ -433,11 +448,11 @@ sub send_binary_request { my $cas_hi = 0; my $cas_lo = 0; - $self->{bin_req_id}++; + $self->new_request(); my $header = pack(BINARY_HEADER_FMT, BINARY_REQUEST, $cmd, $key_len, $extra_len, 0, 0, $total_len, - $self->{bin_req_id}, $cas_hi, $cas_lo); + $self->{req_id}, $cas_hi, $cas_lo); my $packet = $header . $extra_header . $key . $val; $sock->send($packet) || Carp::confess "send failed"; @@ -448,7 +463,7 @@ sub get_binary_response { my $self = shift; my $sock = $self->{connection}; my $header_len = length(pack(BINARY_HEADER_FMT)); - my $expected = $self->{bin_req_id}; + my $expected = $self->{req_id}; my $header; my $body=""; === modified file 'mysql-test/suite/ndb_memcache/include/datatypes_tables.inc' --- a/mysql-test/suite/ndb_memcache/include/datatypes_tables.inc 2011-09-30 20:46:27 +0000 +++ b/mysql-test/suite/ndb_memcache/include/datatypes_tables.inc 2012-10-12 21:50:46 +0000 @@ -45,6 +45,17 @@ INSERT INTO key_prefixes (key_prefix, po SELECT concat(table_name,":"), "ndb-only", table_name FROM tv_tablist; +CREATE TABLE test_char_key (mkey char(40) PRIMARY KEY, val varchar(200)); +CREATE TABLE test_char_val (mkey varchar(20) PRIMARY KEY, val char(200)); + +INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) + VALUES("tt_char_key", "ndbmemcache", "test_char_key", "mkey", "val"), + ("tt_char_val", "ndbmemcache", "test_char_val", "mkey", "val"); + +INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) + VALUES(0, "tck:", "ndb-only", "tt_char_key"), + (0, "tcv:", "ndb-only", "tt_char_val"); + UPDATE memcache_server_roles set update_timestamp = now() WHERE role_id = 0; === added file 'mysql-test/suite/ndb_memcache/include/misc_tables.inc' --- a/mysql-test/suite/ndb_memcache/include/misc_tables.inc 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/ndb_memcache/include/misc_tables.inc 2012-10-13 04:48:45 +0000 @@ -0,0 +1,77 @@ + +--disable_query_log +--disable_result_log + +--source suite/ndb_memcache/include/memcached_wait_for_ready.inc + +use ndbmemcache; +let $done = query_get_value (select key_prefix k from key_prefixes where container = 'test_mkey', k, 1); +if ($done != "hashtag:") +{ + +set default_storage_engine = ndbcluster; + +CREATE TABLE hashtags ( + hashtag varchar(20), + tweet_id int, + author varchar(15), + primary key(hashtag, tweet_id)); + +INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) + VALUES("test_mkey", "ndbmemcache", "hashtags", "hashtag,tweet_id", "author"); + +INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) + VALUES(0, "hashtag:", "ndb-only", "test_mkey"); + + +CREATE TABLE test_unique_idx (pkey int PRIMARY KEY, + mkey char(40), + val varchar(200), + UNIQUE INDEX (mkey) + ); +INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) + VALUES("tt_uidx_pk", "ndbmemcache", "test_unique_idx", "pkey", "mkey,val"), + ("tt_uidx_rd", "ndbmemcache", "test_unique_idx", "mkey", "pkey,val"), + ("tt_uidx_uk", "ndbmemcache", "test_unique_idx", "mkey", "val"); + +INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) + VALUES(0, "tup:", "ndb-only", "tt_uidx_pk"), + (0, "tur:", "ndb-only", "tt_uidx_rd"), + (0, "tui:", "ndb-only", "tt_uidx_uk"); + +SET ndb_table_no_logging = 1; +CREATE TABLE IF NOT EXISTS large_nolog ( + `mkey` VARCHAR(250) NOT NULL, + `cas_value` BIGINT UNSIGNED, + `string_value` VARBINARY(2000), + `ext_id` INT UNSIGNED, + `ext_size` INT UNSIGNED, + PRIMARY KEY USING HASH (mkey) +); + +CREATE TABLE `external_nolog` ( + `id` INT UNSIGNED AUTO_INCREMENT NOT NULL, + `part` SMALLINT NOT NULL, + `content` VARBINARY(13950) NOT NULL, + PRIMARY KEY (id,part) + ); +SET ndb_table_no_logging = 0; + +INSERT INTO containers + SET name = "ext_no_log", db_schema = "ndbmemcache", + db_table = "large_nolog", + key_columns = "mkey", value_columns = "string_value", + cas_column = "cas_value", + large_values_table = "ndbmemcache.external_nolog"; + +INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) + VALUES(0, "bxx:", "ndb-test", "ext_no_log"); + +UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0; + +--source suite/ndb_memcache/include/wait_for_reconf.inc +} + +--enable_query_log +--enable_result_log + === modified file 'mysql-test/suite/ndb_memcache/r/external_values.result' --- a/mysql-test/suite/ndb_memcache/r/external_values.result 2011-12-09 08:51:24 +0000 +++ b/mysql-test/suite/ndb_memcache/r/external_values.result 2012-10-13 04:48:45 +0000 @@ -4,8 +4,13 @@ FROM demo_table_large t1 JOIN external_values t2 ON (t1.ext_id = t2.id) GROUP BY mkey, string_value, ext_size; mkey string_value ext_size total_parts -t10 NULL 15000 15000 -t9 NULL 15000 15000 test8 NULL 160153 160153 test8c NULL 13951 13951 test_set NULL 13949 13949 +SELECT mkey, string_value, ext_size, SUM(length(content)) AS total_parts +FROM large_nolog t1 +JOIN external_nolog t2 ON (t1.ext_id = t2.id) +GROUP BY mkey, string_value, ext_size; +mkey string_value ext_size total_parts +t10 NULL 15000 15000 +t9 NULL 15000 15000 === modified file 'mysql-test/suite/ndb_memcache/t/external_values.test' --- a/mysql-test/suite/ndb_memcache/t/external_values.test 2012-04-26 11:58:05 +0000 +++ b/mysql-test/suite/ndb_memcache/t/external_values.test 2012-10-13 04:48:45 +0000 @@ -1,4 +1,5 @@ --source suite/ndb_memcache/include/have_memcache.inc +--source suite/ndb_memcache/include/misc_tables.inc --perl @@ -154,19 +155,19 @@ sub run_test() { my $r2 = $mc->get("b:test8c"); ($r2 == $val_13949 . "!?") || Carp::confess("results unexpected"); - # APPEND stress test - $mc->add("b:t9", $val_50); + # APPEND stress test. This uses non-logging tables. + $mc->add("bxx:t9", $val_50); for my $i (2 .. 300) { - $mc->append("b:t9", $val_50); - my $r = $mc->get("b:t9"); + $mc->append("bxx:t9", $val_50); + my $r = $mc->get("bxx:t9"); ($r == $val_50 x $i) || Carp::confess("results unexpected"); } - # PREPEND stress test - $mc->add("b:t10", $val_50); + # PREPEND stress test. This uses non-logging tables. + $mc->add("bxx:t10", $val_50); for my $i (2 .. 300) { - $mc->prepend("b:t10", $val_50); - my $r = $mc->get("b:t10"); + $mc->prepend("bxx:t10", $val_50); + my $r = $mc->get("bxx:t10"); ($r == $val_50 x $i) || Carp::confess("results unexpected"); } @@ -184,3 +185,7 @@ SELECT mkey, string_value, ext_size, SUM FROM demo_table_large t1 JOIN external_values t2 ON (t1.ext_id = t2.id) GROUP BY mkey, string_value, ext_size; +SELECT mkey, string_value, ext_size, SUM(length(content)) AS total_parts + FROM large_nolog t1 + JOIN external_nolog t2 ON (t1.ext_id = t2.id) + GROUP BY mkey, string_value, ext_size; === modified file 'mysql-test/suite/ndb_memcache/t/math3.test' --- a/mysql-test/suite/ndb_memcache/t/math3.test 2012-04-26 11:58:05 +0000 +++ b/mysql-test/suite/ndb_memcache/t/math3.test 2012-10-12 21:50:46 +0000 @@ -20,6 +20,10 @@ if($r == 0) { } else { my $r; + + $mc->delete("math_test_3a"); + $mc->delete("math_test_3b"); + $r = $mc->incr("math_test_3a", 1, 1500); # initialize at 1500 ($r == 1500) || Carp::confess("wrong result"); === modified file 'mysql-test/suite/ndb_memcache/t/mpart_key.test' --- a/mysql-test/suite/ndb_memcache/t/mpart_key.test 2012-07-18 06:54:14 +0000 +++ b/mysql-test/suite/ndb_memcache/t/mpart_key.test 2012-10-12 21:50:46 +0000 @@ -1,37 +1,7 @@ -# CHAR --source suite/ndb_memcache/include/have_memcache.inc --source suite/ndb_memcache/include/memcached_wait_for_ready.inc - - -# -# Configuration change for this test -# - ---disable_query_log ---disable_result_log -USE ndbmemcache; - -CREATE TABLE hashtags ( - hashtag varchar(20), - tweet_id int, - author varchar(15), - primary key(hashtag, tweet_id)) -ENGINE=ndb; - -INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) - VALUES("test_mkey", "ndbmemcache", "hashtags", "hashtag,tweet_id", "author"); - -INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) - VALUES(0, "hashtag:", "ndb-only", "test_mkey"); - -UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0; ---enable_query_log ---enable_result_log - -# -# Memcache operations for this test -# +--source suite/ndb_memcache/include/misc_tables.inc --perl @@ -46,12 +16,9 @@ my $port = $ENV{NDB_MEMCACHED_1_PORT} or my $mc = My::Memcache::Binary->new(); my $r = $mc->connect("localhost",$port); -my $cf_gen = $mc->wait_for_reconf(); - -if($cf_gen == 0) { - Carp::confess("FAILED WAIT_FOR_RECONF"); -} - +$mc->delete("hashtag:oscon\t1"); +$mc->delete("hashtag:mysql\t1"); +$mc->delete("hashtag:oscon\t2"); $mc->set("hashtag:oscon\t1","fred") || Carp::confess("FAILED # 01 (SET)"); $mc->add("hashtag:mysql\t1","frederick") || Carp::confess("FAILED # 02 (SET)"); @@ -61,7 +28,6 @@ $mc->set("hashtag:oscon\t2","freddy") ($mc->get("hashtag:mysql\t1") == "frederick") || Carp::confess("FAILED # 05 (GET)"); ($mc->get("hashtag:oscon\t2") == "freddy") || Carp::confess("FAILED # 06 (GET)"); - EOF === modified file 'mysql-test/suite/ndb_memcache/t/type_char.test' --- a/mysql-test/suite/ndb_memcache/t/type_char.test 2012-04-26 11:58:05 +0000 +++ b/mysql-test/suite/ndb_memcache/t/type_char.test 2012-10-12 21:50:46 +0000 @@ -2,35 +2,7 @@ --source suite/ndb_memcache/include/have_memcache.inc --source suite/ndb_memcache/include/memcached_wait_for_ready.inc - - -# -# Configuration change for this test -# - ---disable_query_log ---disable_result_log -USE ndbmemcache; -CREATE TABLE test_char_key (mkey char(40) PRIMARY KEY, val varchar(200)) - ENGINE=ndbcluster; -CREATE TABLE test_char_val (mkey varchar(20) PRIMARY KEY, val char(200)) - ENGINE=ndbcluster; - -INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) - VALUES("tt_char_key", "ndbmemcache", "test_char_key", "mkey", "val"), - ("tt_char_val", "ndbmemcache", "test_char_val", "mkey", "val"); - -INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) - VALUES(0, "tck:", "ndb-only", "tt_char_key"), - (0, "tcv:", "ndb-only", "tt_char_val"); - -UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0; ---enable_query_log ---enable_result_log - -# -# Memcache operations for this test -# +--source suite/ndb_memcache/include/datatypes_tables.inc --perl @@ -45,12 +17,6 @@ my $port = $ENV{NDB_MEMCACHED_1_PORT} or my $mc = My::Memcache::Binary->new(); my $r = $mc->connect("localhost",$port); -my $cf_gen = $mc->wait_for_reconf(); - -if($cf_gen == 0) { - Carp::confess("FAILED WAIT_FOR_RECONF"); -} - # test CHAR key with VARCHAR value $mc->set("tck:a","fred") || Carp::confess("FAILED # 01 (SET)"); $mc->set("tck:1","frederick") || Carp::confess("FAILED # 02 (SET)"); === modified file 'mysql-test/suite/ndb_memcache/t/unique_idx.test' --- a/mysql-test/suite/ndb_memcache/t/unique_idx.test 2012-04-26 11:58:05 +0000 +++ b/mysql-test/suite/ndb_memcache/t/unique_idx.test 2012-10-12 21:50:46 +0000 @@ -11,38 +11,7 @@ --source suite/ndb_memcache/include/have_memcache.inc --source suite/ndb_memcache/include/memcached_wait_for_ready.inc - - -# -# Configuration change for this test -# - ---disable_query_log ---disable_result_log -USE ndbmemcache; -CREATE TABLE test_unique_idx (pkey int PRIMARY KEY, - mkey char(40), - val varchar(200), - UNIQUE INDEX (mkey) - ) ENGINE=ndbcluster; - -INSERT INTO containers (name, db_schema, db_table, key_columns, value_columns) - VALUES("tt_uidx_pk", "ndbmemcache", "test_unique_idx", "pkey", "mkey,val"), - ("tt_uidx_rd", "ndbmemcache", "test_unique_idx", "mkey", "pkey,val"), - ("tt_uidx_uk", "ndbmemcache", "test_unique_idx", "mkey", "val"); - -INSERT INTO key_prefixes(server_role_id, key_prefix, policy, container) - VALUES(0, "tup:", "ndb-only", "tt_uidx_pk"), - (0, "tur:", "ndb-only", "tt_uidx_rd"), - (0, "tui:", "ndb-only", "tt_uidx_uk"); - -UPDATE memcache_server_roles set update_timestamp = NOW() where role_id = 0; ---enable_query_log ---enable_result_log - -# -# Memcache operations for this test -# +--source suite/ndb_memcache/include/misc_tables.inc --perl @@ -56,12 +25,6 @@ my $port = $ENV{NDB_MEMCACHED_1_PORT} or my $mc = My::Memcache->new(); my $r = $mc->connect("localhost",$port); -my $cf_gen = $mc->wait_for_reconf(); - -if($cf_gen == 0) { - Carp::confess("FAILED WAIT_FOR_RECONF"); -} - # 1: SET on primary key $mc->set("tup:1","key1\tSuperbe!") || Carp::confess("Failed test 1.A"); $mc->set("tup:2","key2\tIncroyable!") || Carp::confess("Failed test 1.B"); No bundle (reason: useless for push emails).