List:Commits« Previous MessageNext Message »
From:Jorgen Austvik Date:July 30 2010 9:03am
Subject:bzr commit into mysql-5.1-telco-6.3 branch (jorgen.austvik:3237)
View as plain text  
#At file:///devel/mysql/5.1-telco-6.3-bigmem/ based on revid:frazer@stripped

 3237 Jorgen Austvik	2010-07-30
      Test for big memory

    added:
      mysql-test/suite/ndb/include/
      mysql-test/suite/ndb/include/memory_usage.inc
      mysql-test/suite/ndb/include/ndb_info.inc
      mysql-test/suite/ndb/include/restart_cluster.inc
      mysql-test/suite/ndb/include/restart_cluster_rolling.inc
      mysql-test/suite/ndb/include/restart_node.inc
      mysql-test/suite/ndb/include/restart_random_node.inc
      mysql-test/suite/ndb_big/smoke.test
      storage/ndb/test/run-test/atrt-mysqltest
      storage/ndb/test/run-test/release-bigmem-tests.txt
    modified:
      mysql-test/mysql-test-run.pl
      mysql-test/suite/ndb_big/my.cnf
      storage/ndb/test/run-test/Makefile.am
      storage/ndb/test/run-test/db.cpp
      storage/ndb/test/run-test/main.cpp
      storage/ndb/test/run-test/setup.cpp
=== modified file 'mysql-test/mysql-test-run.pl'

=== modified file 'mysql-test/mysql-test-run.pl'
--- a/mysql-test/mysql-test-run.pl	2010-06-16 12:04:53 +0000
+++ b/mysql-test/mysql-test-run.pl	2010-07-30 09:03:35 +0000
@@ -1968,6 +1968,11 @@
 		  ["storage/ndb/src/mgmclient", "bin"],
 		  "ndb_mgm");
 
+    $ENV{'NDB_NDBD'}=
+      my_find_bin($basedir,
+		  ["storage/ndb/src/kernel", "bin", "libexec"],
+		  ["ndbmtd", "ndbd"]);
+
     $ENV{'NDB_TOOLS_DIR'}=
       my_find_dir($basedir,
 		  ["storage/ndb/tools", "bin"]);

=== added directory 'mysql-test/suite/ndb/include'
=== added file 'mysql-test/suite/ndb/include/memory_usage.inc'
--- a/mysql-test/suite/ndb/include/memory_usage.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/memory_usage.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,95 @@
+--perl
+##
+## NDB Memory Usage
+##
+## Include script that sets two variables that can be used to evaluate actual memory
+## usage in NDB data nodes
+##
+## Parameters:
+##   $ndb_node_id - The node ID to get the memory usage for, can be a number of the node, or 'ALL',
+##                  which returns the average useage over all nodes. ALL is the default value.
+##
+## Returned variables:
+##   $MTR_NDB_DATA_USAGE_PERCENTAGE  - The percentage of data memory used on all nodes
+##   $MTR_NDB_INDEX_USAGE_PERCENTAGE - The percentage of index memory used on all nodes
+##
+## Values are read from ndb_mgm REPORT MemoryUsage
+##
+## Example output from ALL REPORT MemoryUsage
+## ndb_mgm -e "ALL REPORT MemoryUsage"
+## Connected to Management Server at: localhost:16660
+## Node 1: Data usage is 85%(548 32K pages of total 640)
+## Node 1: Index usage is 86%(138 8K pages of total 160)
+## Node 2: Data usage is 85%(548 32K pages of total 640)
+## Node 2: Index usage is 86%(138 8K pages of total 160)
+
+use strict;
+
+use IO::File;
+
+#
+# Set up paths
+#
+my $vardir = $ENV{MYSQLTEST_VARDIR} or die "Need MYSQLTEST_VARDIR";
+my $ndb_connectstring = $ENV{NDB_CONNECTSTRING} or die "Need NDB_CONNECTSTRING";
+my $ndb_mgm = $ENV{NDB_MGM} or die "Need NDB_MGM";
+
+# Input parameters
+my $ndb_node_id = $ENV{ndb_node_id} || 'ALL';
+
+my $cmd = "$ndb_mgm --ndb-connectstring='$ndb_connectstring' -e \"$ndb_node_id REPORT MemoryUsage\"";
+# print "Calling ndb_mgm: '$cmd'\n";
+my $output = `$cmd`;
+
+my $memory = {};
+foreach my $line (split("\n", $output)) {
+    # Skip empty lines
+    next if ($line =~ m/^,+$/g);
+
+    if ($line =~ m/Node (\d+): (Index|Data) usage is \d+%\((\d+) (\d+)K pages of total (\d+)\)/g) {
+        my $node_id = $1;
+        my $type = lc($2);
+        my $used = $3;
+        my $size = $4;
+        my $total = $5;
+
+        if (!defined $memory->{$type}) {
+            $memory->{$type} = {}
+        }
+        if (!defined $memory->{$type}->{$node_id}) {
+            $memory->{$type}->{$node_id} = {};
+        }
+        $memory->{$type}->{$node_id}->{'used'} = $used;
+        $memory->{$type}->{$node_id}->{'total'} = $total;
+        $memory->{$type}->{$node_id}->{'size'} = $size;
+    }
+}
+
+sub usage_percentage {
+    my $type = lc(shift);
+    my $mem  = shift;
+
+    my $used = 0;
+    my $total = 0;
+
+    foreach my $node (keys %{$mem->{$type}}) {
+        $used  += $mem->{$type}->{$node}->{'used'};
+        $total += $mem->{$type}->{$node}->{'total'};
+    }
+    die "Parsing error - not able to detect total number of pages, output: '$output'" if ($total == 0);
+    return sprintf("%.2f", $used * 100 / $total);
+}
+
+my $num_nodes = scalar(keys %{$memory->{Data}});
+my $data_usage_percentage = usage_percentage('Data', $memory);
+my $index_usage_percentage = usage_percentage('Index', $memory);
+
+my $file_name = "$vardir/tmp/ndb_memory_usage_result.inc";
+my $F = IO::File->new($file_name, 'w') or die "Could not open '$file_name' for writing";
+print $F "--let \$MTR_NDB_DATA_USAGE_PERCENTAGE= $data_usage_percentage\n";
+print $F "--let \$MTR_NDB_INDEX_USAGE_PERCENTAGE= $index_usage_percentage\n";
+$F->close();
+
+EOF
+
+--source $MYSQLTEST_VARDIR/tmp/ndb_memory_usage_result.inc

=== added file 'mysql-test/suite/ndb/include/ndb_info.inc'
--- a/mysql-test/suite/ndb/include/ndb_info.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/ndb_info.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,101 @@
+--perl
+use strict;
+
+use File::Basename;
+use IO::File;
+use lib "lib/";
+use My::Find;
+
+#
+# Include file that sets the following environment variables:
+# MTR_NDB_NO_OF_NODES - Number of ndb[mt]d data nodes
+# MTR_NDB_NO_OF_REPLICAS - Number of replicas
+# MTR_NDB_MAX_NO_OF_TABLES - Max number of tables
+# MTR_NDB_MAX_NO_OF_ORDERED_INDEXES - Max number of ordered indexes
+# MTR_NDB_DATA_MEMORY - Data memory per node
+# MTR_NDB_INDEX_MEMORY - Index memory per node
+# MTR_NDB_USABLE_DATA_MEMORY - Data memory available for user (#nodes * data_memory / #replicas)
+# MTR_NDB_USABLE_INDEX_MEMORY - Index memory available for user (#nodes * index_memory / #replicas)
+#
+
+#
+# Set up paths
+#
+my $vardir = $ENV{MYSQLTEST_VARDIR} or die "Need MYSQLTEST_VARDIR";
+my $mysql_test_dir = $ENV{MYSQL_TEST_DIR} or die "Need MYSQL_TEST_DIR";
+my $basedir = dirname($mysql_test_dir);
+my $ndb_connectstring = $ENV{NDB_CONNECTSTRING} or die "Need NDB_CONNECTSTRING";
+
+#
+# Check if the needed jars are available
+#
+my $ndb_config = my_find_file($basedir,
+                              ["storage/ndb/tools", "bin"],
+                              ["ndb_config", "ndb_config.exe"], NOT_REQUIRED);
+
+my $fields = "NoOfReplicas,MaxNoOfTables,MaxNoOfOrderedIndexes,DataMemory,IndexMemory";
+my $cmd = "$ndb_config --ndb-connectstring='$ndb_connectstring' -q '$fields' -f ',' -r '\\n'";
+print "Calling ndb_config: '$cmd'\n";
+my $output = `$cmd`;
+
+my $no_of_nodes = 0;
+my $no_of_replicas = 0;
+my $max_no_of_tables = 0;
+my $max_no_of_ordered_indexes = 0;
+my $data_memory = 0;
+my $index_memory = 0;
+
+foreach my $line (split("\n", $output)) {
+    # Skip empty lines
+    next if ($line =~ m/^,+$/g);
+
+    # One line per node
+    $no_of_nodes++;
+
+    # Same on each line (node)
+    ($no_of_replicas, $max_no_of_tables, $max_no_of_ordered_indexes, $data_memory, $index_memory) = split(',', $line);
+}
+
+my $usable_data_memory = 0;
+my $usable_index_memory = 0;
+
+if ($no_of_replicas > 0) {
+    $usable_data_memory = $no_of_nodes * $data_memory / $no_of_replicas;
+    $usable_index_memory = $no_of_nodes * $index_memory / $no_of_replicas;
+} else {
+    die "Failed to parse ndb_config output - could not get number of replicas";
+}
+
+sub get_first_last {
+  my $type = shift;
+
+  my $cmd = "$ndb_config --ndb-connectstring='$ndb_connectstring' -q 'NodeId' -f '' -r ',' --type=$type";
+  my $output = `$cmd`;
+  chomp($output);
+  my @nums = split(',', $output);
+  my @sorted = sort(@nums);
+  return ($sorted[0], $sorted[scalar(@sorted) - 1]);
+}
+
+my ($first_ndbd_nodeid, $last_ndbd_nodeid) = get_first_last('ndbd');
+my ($first_mgmd_nodeid, $last_mgmd_nodeid) = get_first_last('ndb_mgmd');
+
+my $file_name = "$vardir/tmp/ndb_info_result.inc";
+my $F = IO::File->new($file_name, 'w') or die "Could not open '$file_name' for writing";
+print $F "--let \$MTR_NDB_NO_OF_NODES= $no_of_nodes\n";
+print $F "--let \$MTR_NDB_NO_OF_REPLICAS= $no_of_replicas\n";
+print $F "--let \$MTR_NDB_MAX_NO_OF_TABLES= $max_no_of_tables\n";
+print $F "--let \$MTR_NDB_MAX_NO_OF_ORDERED_INDEXES= $max_no_of_ordered_indexes\n";
+print $F "--let \$MTR_NDB_DATA_MEMORY= $data_memory\n";
+print $F "--let \$MTR_NDB_INDEX_MEMORY= $index_memory\n";
+print $F "--let \$MTR_NDB_USABLE_DATA_MEMORY= $usable_data_memory\n";
+print $F "--let \$MTR_NDB_USABLE_INDEX_MEMORY= $usable_index_memory\n";
+print $F "--let \$MTR_NDB_FIRST_NDBD_NODEID= $first_ndbd_nodeid\n";
+print $F "--let \$MTR_NDB_LAST_NDBD_NODEID= $last_ndbd_nodeid\n";
+print $F "--let \$MTR_NDB_FIRST_MGMD_NODEID= $first_mgmd_nodeid\n";
+print $F "--let \$MTR_NDB_LAST_MGMD_NODEID= $last_mgmd_nodeid\n";
+$F->close();
+
+EOF
+
+--source $MYSQLTEST_VARDIR/tmp/ndb_info_result.inc

=== added file 'mysql-test/suite/ndb/include/restart_cluster.inc'
--- a/mysql-test/suite/ndb/include/restart_cluster.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/restart_cluster.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,31 @@
+##
+## Restarts a NDB cluster
+##
+## Parameters:
+##   $waiter_timeout_start      - How long wo tait for a cluster to start, default 600 seconds
+##   $waiter_timeout_stop       - How long to wait for a cluster to stop, default 30 seconds
+##
+## In addition, $NDB_CONNECTSTRING, $NDB_MGM, $NDB_TOOLS_DIR and $NDB_TOOLS_OUTPUT
+## must be set (they are normally proivided by MTR).
+##
+
+# Default values
+if (`SELECT LENGTH('$waiter_timeout_start') = 0`) {
+  let $waiter_timeout_start= 600;
+}
+if (`SELECT LENGTH('$waiter_timeout_stop') = 0`) {
+  let $waiter_timeout_stop= 30;
+}
+
+--echo # System restart:
+--append_file $NDB_TOOLS_OUTPUT
+# System restart:
+EOF
+
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "all restart -n" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_waiter --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" --not-started --timeout=$waiter_timeout_stop >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "all start" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_waiter --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" --timeout=$waiter_timeout_start >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT

=== added file 'mysql-test/suite/ndb/include/restart_cluster_rolling.inc'
--- a/mysql-test/suite/ndb/include/restart_cluster_rolling.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/restart_cluster_rolling.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,32 @@
+##
+## Restarts a NDB cluster by rolling node restart
+##
+## Parameters:
+##   $waiter_timeout_start      - How long wo tait for a single node to start, default from restart_node.inc
+##   $waiter_timeout_stop       - How long to wait for a single node to stop, default from restart_node.inc
+##   $MTR_NDB_FIRST_NDBD_NODEID - The first node id of the NDBD data nodes
+##   $MTR_NDB_LAST_NDBD_NODEID  - The last node id of the NDBD data nodes
+##
+## This include file has as a prerequisite that the NDBD node ids is in a numbered range without
+## holes which starts with MTR_NDB_FIRST_NDBD_NODEID and ends with $MTR_NDB_LAST_NDBD_NODEID.
+##
+## The last two parameters are normally provided by
+##   --source suite/ndb/include/ndb_info.inc
+##
+## In addition, $NDB_CONNECTSTRING, $NDB_MGM, $NDB_TOOLS_DIR and $NDB_TOOLS_OUTPUT
+## must be set (they are normally proivided by MTR).
+##
+
+--echo # Rolling system restart from node $MTR_NDB_FIRST_NDBD_NODEID to node $MTR_NDB_LAST_NDBD_NODEID:
+--append_file $NDB_TOOLS_OUTPUT
+# Rolling system restart:
+EOF
+
+--let $ndb_node_id= $MTR_NDB_LAST_NDBD_NODEID
+while (`SELECT $ndb_node_id >= $MTR_NDB_FIRST_NDBD_NODEID`) {
+  --echo # Restarting node $ndb_node_id as part of rolling system restart
+  --source suite/ndb/include/restart_node.inc
+  --dec $ndb_node_id
+}
+--let $ndb_node_id=
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT

=== added file 'mysql-test/suite/ndb/include/restart_node.inc'
--- a/mysql-test/suite/ndb/include/restart_node.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/restart_node.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,36 @@
+##
+## Restarts a single NDB data node
+##
+## Parameters:
+##   $ndb_node_id          - The node to be restarted
+##   $waiter_timeout_start - How long wo tait for a single node to start, default 120 seconds
+##   $waiter_timeout_stop  - How long to wait for a single node to stop, default 60 seconds
+##
+## In addition, $NDB_CONNECTSTRING, $NDB_MGM, $NDB_TOOLS_DIR and $NDB_TOOLS_OUTPUT
+## must be set (they are normally proivided by MTR).
+##
+
+# Default values
+if (`SELECT LENGTH('$waiter_timeout_start') = 0`) {
+  let $waiter_timeout_start= 120;
+}
+if (`SELECT LENGTH('$waiter_timeout_stop') = 0`) {
+  let $waiter_timeout_stop= 60;
+}
+
+--echo # Restarting node $ndb_node_id, timeouts: start: $waiter_timeout_start seconds, stop: $waiter_timeout_stop seconds:
+
+--append_file $NDB_TOOLS_OUTPUT
+# Restarting random node, stopping part:
+EOF
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "$ndb_node_id restart -n" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_waiter --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" --wait-nodes=$ndb_node_id --not-started --timeout=$waiter_timeout_stop >> $NDB_TOOLS_OUTPUT
+
+--append_file $NDB_TOOLS_OUTPUT
+# Restarting random node, starting part:
+EOF
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "$ndb_node_id start" >> $NDB_TOOLS_OUTPUT
+--exec $NDB_TOOLS_DIR/ndb_waiter --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" --timeout=$waiter_timeout_start >> $NDB_TOOLS_OUTPUT
+--exec $NDB_MGM -e "show" >> $NDB_TOOLS_OUTPUT

=== added file 'mysql-test/suite/ndb/include/restart_random_node.inc'
--- a/mysql-test/suite/ndb/include/restart_random_node.inc	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb/include/restart_random_node.inc	2010-07-30 09:03:35 +0000
@@ -0,0 +1,23 @@
+##
+## Restarts a single random NDB data node
+##
+## Parameters:
+##   $waiter_timeout_start      - How long wo tait for a single node to start, default from restart_node.inc
+##   $waiter_timeout_stop       - How long to wait for a single node to stop, default from restart_node.inc
+##   $MTR_NDB_FIRST_NDBD_NODEID - The first node id of the NDBD data nodes
+##   $MTR_NDB_NO_OF_NODES       - The number of NDBD data nodes
+##
+## This include file has as a prerequisite that the NDBD node ids is in a numbered range without
+## holes which starts with MTR_NDB_FIRST_NDBD_NODEID and has no more than MTR_NDB_NO_OF_NODES nodes.
+##
+## The last two parameters are normally provided by
+##   --source suite/ndb/include/ndb_info.inc
+##
+## In addition, $NDB_CONNECTSTRING, $NDB_MGM, $NDB_TOOLS_DIR and $NDB_TOOLS_OUTPUT
+## must be set (they are normally proivided by MTR).
+##
+
+--let $ndb_node_id = `SELECT FLOOR($MTR_NDB_FIRST_NDBD_NODEID + (RAND() * $MTR_NDB_NO_OF_NODES))`
+--echo # Restarting random node $ndb_node_id
+--source suite/ndb/include/restart_node.inc
+--let $ndb_node_id=

=== modified file 'mysql-test/suite/ndb_big/my.cnf'
--- a/mysql-test/suite/ndb_big/my.cnf	2009-12-06 18:57:17 +0000
+++ b/mysql-test/suite/ndb_big/my.cnf	2010-07-30 09:03:35 +0000
@@ -68,3 +68,6 @@
 engine-condition-pushdown=1
 ndb-wait-connected=600
 ndb-wait-setup=300
+
+[ENV]
+NDB_CONNECTSTRING=             @mysql_cluster.1.ndb_connectstring
\ No newline at end of file

=== added file 'mysql-test/suite/ndb_big/smoke.test'
--- a/mysql-test/suite/ndb_big/smoke.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/ndb_big/smoke.test	2010-07-30 09:03:35 +0000
@@ -0,0 +1,311 @@
+--source suite/ndb_big/suite.inc
+--source include/have_ndb.inc
+--source suite/ndb/include/ndb_info.inc
+# --call mtr.force_restart()
+set new=on;
+
+select version();
+
+#
+# Table with many columns will fill up to 15% of memory
+#
+
+--let $num_cols= 125
+--let $fill_percentage= 15
+--echo # Table with $num_cols columns will fill up to $fill_percentage% of memory
+
+--let $sql= CREATE TABLE manycol (pri int(11) NOT NULL
+--let $c= $num_cols
+while ($c)
+{
+  --let $sql=$sql, c$c CHAR(2) NULL
+  --dec $c
+}
+--let $sql= $sql, PRIMARY KEY (pri)) ENGINE=NDB DEFAULT CHARSET=utf8 PARTITION BY HASH (pri)
+--source suite/ndb_big/run_query_with_retry.inc
+
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage before inserts: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Inserting rows into the many columns table
+--source suite/ndb_big/start_timer.inc
+--disable_query_log
+--let $row= 0
+while (`SELECT $MTR_NDB_DATA_USAGE_PERCENTAGE < $fill_percentage`)
+{
+  --let $sql= INSERT INTO manycol (pri
+  --let $c= $num_cols
+  while ($c)
+  {
+    --let $sql=$sql, c$c
+    --dec $c
+  }
+  --let $sql= $sql) VALUES
+
+  --let $separator=
+  --let $batch_size= 50
+  while ($batch_size)
+  {
+    --inc $row
+    --let $sql= $sql$separator($row
+    --let $c= $num_cols
+    while ($c)
+    {
+      --let $sql=$sql, 'ab'
+      --dec $c
+    }
+    --let $sql= $sql)
+    --dec $batch_size
+    --let $separator= ,
+  }
+  --source suite/ndb_big/run_query_with_retry.inc
+
+  --source suite/ndb/include/memory_usage.inc
+
+  # Workaround for bug#52464, since MTR caches --source file content in while loops
+  --eval SET @content = LOAD_FILE('$MYSQLTEST_VARDIR/tmp/ndb_memory_usage_result.inc')
+  --let $start= `SELECT LOCATE('=', @content) + 1`
+  --let $end = `SELECT LOCATE(CHAR(10), @content)`
+  --let $MTR_NDB_DATA_USAGE_PERCENTAGE= `SELECT CAST(TRIM(SUBSTR(@content, $start, $end - $start)) AS DECIMAL(10,2))`
+}
+--enable_query_log
+--source suite/ndb_big/end_timer.inc
+
+SELECT COUNT(*) AS 'Rows inserted into manycol table' FROM manycol;
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after inserts into manycol table: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+
+#
+# Empty table
+#
+--echo # Empty table
+CREATE TABLE empty (pri int(11) NOT NULL) ENGINE=NDB;
+
+
+#
+# Table with many data types
+#
+
+--let $fill_percentage= 25
+--echo # Table with many column types will fill up to $fill_percentage% of memory
+
+CREATE TABLE types (
+  pri int(11) NOT NULL,
+  ti TINYINT,
+  si SMALLINT,
+  mi MEDIUMINT,
+  bi BIGINT,
+  ft FLOAT(20),
+  tt FLOAT(40),
+  du DOUBLE,
+  nu NUMERIC(10, 5),
+  bt BIT(16),
+  da DATE,
+  tm TIME,
+  dt DATETIME,
+  ts TIMESTAMP,
+  ye YEAR,
+  ch CHAR(2),
+  bn BINARY,
+  vc VARCHAR(10),
+  vb VARBINARY(10),
+  en ENUM('one', 'two', 'three'),
+  se SET('one', 'two', 'three'),
+  PRIMARY KEY (pri)
+) ENGINE=NDB DEFAULT CHARSET=utf8 PARTITION BY HASH (pri);
+
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage before inserts: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Inserting rows into the column types table
+--source suite/ndb_big/start_timer.inc
+--disable_query_log
+--let $row= 0
+while (`SELECT $MTR_NDB_DATA_USAGE_PERCENTAGE < $fill_percentage`)
+{
+  --let $sql= INSERT INTO types (pri, ti, si, mi, bi, ft, tt, du, nu, bt, da, tm, dt, ts, ye, ch, bn, vc, vb, en, se) VALUES
+
+  --let $batch_size= 50
+  --let $separator=
+  while ($batch_size)
+  {
+    --inc $row
+    --let $sql= $sql$separator($row, 2, 3, 4, 5, 1.2, 3.14, 2.71, 42.42, b'1101', '1977-05-13', '01:23:45', '1980-01-02 01:34:54', NOW(), 2010, 'cd', b'10101', 'xyz', 'abc', 'one', 'one,two')
+    --dec $batch_size
+    --let $separator= ,
+  }
+  --source suite/ndb_big/run_query_with_retry.inc
+
+  --source suite/ndb/include/memory_usage.inc
+
+  # Workaround for bug#52464, since MTR caches --source file content in while loops
+  --eval SET @content = LOAD_FILE('$MYSQLTEST_VARDIR/tmp/ndb_memory_usage_result.inc')
+  --let $start= `SELECT LOCATE('=', @content) + 1`
+  --let $end = `SELECT LOCATE(CHAR(10), @content)`
+  --let $MTR_NDB_DATA_USAGE_PERCENTAGE= `SELECT CAST(TRIM(SUBSTR(@content, $start, $end - $start)) AS DECIMAL(10,2))`
+}
+--enable_query_log
+--source suite/ndb_big/end_timer.inc
+
+SELECT COUNT(*) AS 'Rows inserted into types table' FROM types;
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after inserts into types table: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after inserts: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+
+#
+# Create extra indexes
+#
+
+--echo # Creating indexes
+
+CREATE UNIQUE INDEX types_ints ON types (pri, ti, si, mi, bi);
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after create index 1: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after create index 1: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+CREATE UNIQUE INDEX types_dec ON types (ft, tt, du, nu, pri);
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after create index 2: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after create index 2: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+CREATE UNIQUE INDEX types_date ON types (ts, da, tm, dt, pri);
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after create index 3: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after create index 3: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+
+--source suite/ndb/include/memory_usage.inc
+--echo # Index memory usage after creating secondary indexes: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+
+#
+# Create BLOB (text) table
+#
+
+--echo # Table with BLOB(TEXT) columns
+CREATE TABLE blobs (pri int(11) NOT NULL, data1 TEXT, data2 TEXT, PRIMARY KEY (pri)) ENGINE=NDB DEFAULT CHARSET=utf8 PARTITION BY HASH (pri);
+
+--let $fill_percentage= 75
+--echo # Inserting rows into the blobs table until $fill_percentage% memory fill
+--source suite/ndb_big/start_timer.inc
+--disable_query_log
+--let $row= 0
+while (`SELECT $MTR_NDB_DATA_USAGE_PERCENTAGE < $fill_percentage`)
+{
+  --let $sql= INSERT INTO blobs(pri, data1, data2) VALUES
+
+  --let $batch_size= 50
+  --let $separator=
+  while ($batch_size)
+  {
+    --inc $row
+    --let $sql= $sql$separator($row, REPEAT('dt01', 10240 * RAND()), REPEAT('dt02', 1024 * RAND()))
+    --dec $batch_size
+    --let $separator= ,
+  }
+  --source suite/ndb_big/run_query_with_retry.inc
+
+  --source suite/ndb/include/memory_usage.inc
+  # Workaround for bug#52464, since MTR caches --source file content in while loops
+  --eval SET @content = LOAD_FILE('$MYSQLTEST_VARDIR/tmp/ndb_memory_usage_result.inc')
+  --let $start= `SELECT LOCATE('=', @content) + 1`
+  --let $end = `SELECT LOCATE(CHAR(10), @content)`
+  --let $MTR_NDB_DATA_USAGE_PERCENTAGE= `SELECT CAST(TRIM(SUBSTR(@content, $start, $end - $start)) AS DECIMAL(10,2))`
+}
+--enable_query_log
+--source suite/ndb_big/end_timer.inc
+
+SELECT COUNT(*) AS 'Rows inserted into blobs table' FROM blobs;
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after inserts into blobs table: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after inserts into blobs table: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+
+#
+# Fill up with many tables, as long as memory/index usage < 95%
+#
+--let $fill_percentage= 95
+--echo # Creating tables until $fill_percentage% memory/index fill is reached
+
+--source suite/ndb_big/start_timer.inc
+--disable_query_log
+--let $table_id= 0
+while (`SELECT $MTR_NDB_DATA_USAGE_PERCENTAGE < $fill_percentage AND $MTR_NDB_INDEX_USAGE_PERCENTAGE < $fill_percentage AND $table_id < 1500`)
+{
+  --inc $table_id
+  --let sql=CREATE TABLE t$table_id (c1 int(11) NOT NULL, c3 text, PRIMARY KEY (c1)) ENGINE=NDB DEFAULT CHARSET=utf8 PARTITION BY HASH (c1)
+  --source suite/ndb_big/run_query_with_retry.inc
+
+  --let $rows= 100
+  --let $batch_size= 50
+  --let $i= `SELECT $rows / $batch_size`
+  while ($i)
+  {
+    --let $batch= $batch_size
+    let $separator= ;
+    --let $sql= INSERT t$table_id (c1, c3) VALUES
+    while($batch)
+    {
+      --let $sql=$sql$separator(($i*$batch_size)+$batch, REPEAT('ks87', 128*RAND()))
+      --dec $batch
+      --let $separator = ,
+    }
+    --source suite/ndb_big/run_query_with_retry.inc
+    --dec $i
+  }
+
+  --source suite/ndb/include/memory_usage.inc
+  # Workaround for bug#52464, since MTR caches --source file content in while loops
+  --eval SET @content = LOAD_FILE('$MYSQLTEST_VARDIR/tmp/ndb_memory_usage_result.inc')
+  --let $start= `SELECT LOCATE('=', @content) + 1`
+  --let $end = `SELECT LOCATE(CHAR(10), @content)`
+  --let $MTR_NDB_DATA_USAGE_PERCENTAGE= `SELECT CAST(TRIM(SUBSTR(@content, $start, $end - $start)) AS DECIMAL(10,2))`
+  --let $index_start= `SELECT LOCATE('=', @content, $start) + 1`
+  --let $index_end = `SELECT LOCATE(CHAR(10), @content, $end + 1)`
+  --let $MTR_NDB_INDEX_USAGE_PERCENTAGE= `SELECT CAST(TRIM(SUBSTR(@content, $index_start, $index_end - $index_start)) AS DECIMAL(10,2))`
+#  --echo # After table t$table_id: data: $MTR_NDB_DATA_USAGE_PERCENTAGE%, index: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+}
+
+--echo # Num tables: $table_id
+--source suite/ndb/include/memory_usage.inc
+--echo # Memory usage after inserts into many tables table: $MTR_NDB_DATA_USAGE_PERCENTAGE%
+--echo # Index memory usage after inserts into many tables table: $MTR_NDB_INDEX_USAGE_PERCENTAGE%
+
+--append_file $NDB_TOOLS_OUTPUT
+# ndb_mgm -e "ALL REPORT MemoryUsage"
+EOF
+--exec $NDB_MGM -e "ALL REPORT MemoryUsage" >> $NDB_TOOLS_OUTPUT
+
+## Backup
+--echo # Starting backup:
+--source suite/ndb_big/start_timer.inc
+--source include/ndb_backup.inc
+--source suite/ndb_big/end_timer.inc
+
+## Random node restart
+--source suite/ndb_big/start_timer.inc
+--source suite/ndb/include/restart_random_node.inc
+--source suite/ndb_big/end_timer.inc
+
+## System restart
+--source suite/ndb_big/start_timer.inc
+--source suite/ndb/include/restart_cluster.inc
+--source suite/ndb_big/end_timer.inc
+
+## Rolling system restart
+--source suite/ndb_big/start_timer.inc
+--source suite/ndb/include/restart_cluster_rolling.inc
+--source suite/ndb_big/end_timer.inc
+
+## Clean up
+--echo # Dropping tables:
+--let $t= $table_id
+--source suite/ndb_big/start_timer.inc
+while ($t)
+{
+  --let $sql=drop table t$t
+  --source suite/ndb_big/run_query_with_retry.inc
+  --dec $t
+}
+--source suite/ndb_big/end_timer.inc
+
+DROP TABLE blobs;
+DROP TABLE types;
+DROP TABLE manycol;
+DROP TABLE empty;
+
+## Test suceeded
+exit;

=== modified file 'storage/ndb/test/run-test/Makefile.am'
--- a/storage/ndb/test/run-test/Makefile.am	2009-12-17 15:43:35 +0000
+++ b/storage/ndb/test/run-test/Makefile.am	2010-07-30 09:03:35 +0000
@@ -23,7 +23,7 @@
 include $(top_srcdir)/storage/ndb/config/type_ndbapitools.mk.am
 
 test_PROGRAMS = atrt
-test_DATA=daily-basic-tests.txt daily-devel-tests.txt 16node-tests.txt \
+test_DATA=daily-basic-tests.txt daily-devel-tests.txt release-bigmem-tests.txt 16node-tests.txt \
           conf-ndbmaster.cnf \
           conf-fimafeng08.cnf conf-dl145a.cnf test-tests.txt conf-test.cnf db.sql \
           conf-upgrade.cnf upgrade-tests.txt \
@@ -51,7 +51,7 @@
 atrt_LDFLAGS = -static @ndb_bin_am_ldflags@
 
 wrappersdir=$(prefix)/bin
-wrappers_SCRIPTS=atrt-testBackup atrt-mysql-test-run
+wrappers_SCRIPTS=atrt-testBackup atrt-mysql-test-run atrt-mysqltest
 
 EXTRA_DIST = $(test_DATA) $(test_SCRIPTS) $(wrappers_SCRIPTS) README.ATRT atrt.hpp
 

=== added file 'storage/ndb/test/run-test/atrt-mysqltest'
--- a/storage/ndb/test/run-test/atrt-mysqltest	1970-01-01 00:00:00 +0000
+++ b/storage/ndb/test/run-test/atrt-mysqltest	2010-07-30 09:03:35 +0000
@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# Use like this in the ATRT test file:
+#
+# max-time: 180
+# cmd: atrt-mysqltest
+# args: --test-file=suite/ndb/t/smoke.test
+#
+set -x
+RUN_DIR=`pwd`
+
+RESULT_FILE="$RUN_DIR/mysqltest.out"
+MYSQL_TEST_DIR=$MYSQL_BASE_DIR/mysql-test
+RUN_LOG="$RUN_DIR/ndb_testrun.log"
+
+# Expected subdirectory by tests written for MTR.
+mkdir -p $RUN_DIR/tmp
+
+$MYSQL_BASE_DIR/bin/ndb_waiter -c $NDB_CONNECTSTRING > $RESULT_FILE
+
+cd $MYSQL_BASE_DIR/mysql-test
+
+NDB_MGM="$MYSQL_BASE_DIR/storage/ndb/src/mgmclient/ndb_mgm"
+if [ -f "$MYSQL_BASE_DIR/bin/ndb_mgm" ]; then
+  NDB_MGM="$MYSQL_BASE_DIR/bin/ndb_mgm"
+fi
+
+NDB_NDBD="$MYSQL_BASE_DIR/bin/ndbd"
+if [ -f "$MYSQL_BASE_DIR/bin/ndbmtd" ]; then
+  NDB_NDBD="$MYSQL_BASE_DIR/bin/ndbmtd"
+fi
+if [ -f "$MYSQL_BASE_DIR/storage/ndb/src/kernel/ndbd" ]; then
+  NDB_NDBD="$MYSQL_BASE_DIR/storage/ndb/src/kernel/ndbd"
+fi
+if [ -f "$MYSQL_BASE_DIR/storage/ndb/src/kernel/ndbmtd" ]; then
+  NDB_NDBD="$MYSQL_BASE_DIR/storage/ndb/src/kernel/ndbmtd"
+fi
+
+NDB_TOOLS_DIR="$MYSQL_BASE_DIR/bin"
+if [ -e "$MYSQL_BASE_DIR/storage/ndb/tools" ]; then
+  NDB_TOOLS_DIR="$MYSQL_BASE_DIR/storage/ndb/tools"
+fi
+
+NDB_EXAMPLES_DIR="$MYSQL_BASE_DIR/bin"
+if [ -e "$MYSQL_BASE_DIR/storage/ndb/ndbapi-examples" ]; then
+  NDB_EXAMPLES_DIR="$MYSQL_BASE_DIR/storage/ndb/ndbapi-examples"
+fi
+
+NDB_EXAMPLES_BINARY="$MYSQL_BASE_DIR/bin/ndbapi_simple"
+if [ -f "$MYSQL_BASE_DIR/storage/ndb/ndbapi-examples/ndbapi_simple/ndbapi_simple" ]; then
+  NDB_EXAMPLES_BINARY="$MYSQL_BASE_DIR/storage/ndb/ndbapi-examples/ndbapi_simple/ndbapi_simple"
+fi
+
+echo "Default group suffix: $MYSQL_GROUP_SUFFIX"
+
+# MYSQL_GROUP_SUFFIX: This is a MySQL client, started as a NDB API client by ATRT.
+NDB_CONNECTSTRING=$NDB_CONNECTSTRING MYSQL_TEST_DIR=$MYSQL_BASE_DIR/mysql-test \
+NDB_TOOLS_OUTPUT=$RUN_LOG NDB_EXAMPLES_OUTPUT=$RUN_LOG NDB_MGM=$NDB_MGM NDB_NDBD=NDB_NDBD \
+NDB_TOOLS_DIR=$NDB_TOOLS_DIR NDB_EXAMPLES_DIR=$NDB_EXAMPLES_DIR NDB_EXAMPLES_BINARY=$NDB_EXAMPLES_BINARY \
+MYSQLTEST_VARDIR=$RUN_DIR MYSQL_GROUP_SUFFIX=.1.4node \
+                              $MYSQL_BASE_DIR/bin/mysqltest --defaults-file=$MYSQL_HOME/my.cnf \
+                              --tmpdir=$RUN_DIR \
+                              --logdir=$RUN_DIR \
+                              --basedir=$MYSQL_BASE_DIR/mysql-test/ \
+                              --ps-protocol \
+                              --tail-lines=20 \
+                              --database=test \
+                              --nowarnings \
+                              --skip-safemalloc $* | tee -a $RESULT_FILE
+
+# Could be added, but would then also need the mtr database:
+#                              --include=include/mtr_check.sql \
+# We could also run the bootstrap scripts
+
+echo "Run complete, result code: '$?'"
+
+r=$?
+f=`grep -c '\[ fail \]' $RESULT_FILE`
+o=`grep -c '\[ pass \]' $RESULT_FILE`
+
+if [ ( $r -eq 0 ) -o ( $o -gt 0 -a $f -eq 0 ) ]
+then
+    echo "NDBT_ProgramExit: OK"
+    exit 0
+fi
+
+echo "NDBT_ProgramExit: Failed"
+exit 1

=== modified file 'storage/ndb/test/run-test/db.cpp'
--- a/storage/ndb/test/run-test/db.cpp	2010-02-22 14:02:11 +0000
+++ b/storage/ndb/test/run-test/db.cpp	2010-07-30 09:03:35 +0000
@@ -180,12 +180,13 @@
   }
 
   BaseString tmp;
-  tmp.assfmt(". ./env.sh ; %s", sql);
+  tmp.assfmt(". %s/env.sh ; %s", proc->m_proc.m_cwd.c_str(), sql);
 
   g_logger.debug("%s-system(%s)", proc->m_proc.m_cwd.c_str(), tmp.c_str());
-  if (system(tmp.c_str()) != 0)
+  int sres = system(tmp.c_str());
+  if (sres != 0)
   {
-    g_logger.error("Failed to run sql: %s", sql);
+    g_logger.error("Failed to run sql: %s, return: %d, error %d: '%s' (command: '%s', directory: '%s')", sres, sql, errno, strerror(errno), tmp.c_str(), proc->m_proc.m_cwd.c_str());
     res = false;
     abort();
   }

=== modified file 'storage/ndb/test/run-test/main.cpp'
--- a/storage/ndb/test/run-test/main.cpp	2010-04-16 08:50:13 +0000
+++ b/storage/ndb/test/run-test/main.cpp	2010-07-30 09:03:35 +0000
@@ -431,6 +431,7 @@
       {
         g_logger.debug("Timed out");
 	result = ERR_MAX_TIME_ELAPSED;
+        g_logger.info("Timeout '%s' after %d seconds", test_case.m_name, test_case.m_max_time);
 	break;
       }
       NdbSleep_SecSleep(1);

=== added file 'storage/ndb/test/run-test/release-bigmem-tests.txt'
--- a/storage/ndb/test/run-test/release-bigmem-tests.txt	1970-01-01 00:00:00 +0000
+++ b/storage/ndb/test/run-test/release-bigmem-tests.txt	2010-07-30 09:03:35 +0000
@@ -0,0 +1,3 @@
+max-time: 14400
+cmd: atrt-mysqltest
+args: --test-file=suite/ndb/t/smoke.test

=== modified file 'storage/ndb/test/run-test/setup.cpp'
--- a/storage/ndb/test/run-test/setup.cpp	2010-03-17 09:41:21 +0000
+++ b/storage/ndb/test/run-test/setup.cpp	2010-07-30 09:03:35 +0000
@@ -341,6 +341,13 @@
     proc.m_proc.m_args.append(" --core-file");
     if (g_fix_nodeid)
       proc.m_proc.m_args.appfmt(" --ndb-nodeid=%d", proc.m_nodeid);
+
+    // Add ndb connect string
+    const char * val;
+    if (cluster.m_options.m_loaded.get(ndbcs, &val)) {
+      proc.m_proc.m_args.appfmt(" %s=%s", ndbcs, val);
+    }
+
     proc.m_proc.m_cwd.appfmt("%smysqld.%u", dir.c_str(), proc.m_index);
     proc.m_proc.m_shutdown_options = "SIGKILL"; // not nice
     proc.m_proc.m_env.appfmt(" MYSQL_GROUP_SUFFIX=.%u%s", 
@@ -623,6 +630,7 @@
   {
     if (cluster.m_processes[i]->m_type == atrt_process::AP_NDB_MGMD ||
 	cluster.m_processes[i]->m_type == atrt_process::AP_NDB_API ||
+	cluster.m_processes[i]->m_type == atrt_process::AP_MYSQLD ||
 	cluster.m_processes[i]->m_type == atrt_process::AP_NDBD)
     {
       features |= atrt_options::AO_NDBCLUSTER;


Attachment: [text/bzr-bundle] bzr/jorgen.austvik@sun.com-20100730090335-ao71gwl44ijnumik.bundle
Thread
bzr commit into mysql-5.1-telco-6.3 branch (jorgen.austvik:3237) Jorgen Austvik30 Jul