MySQL Lists are EOL. Please join:

List:Commits« Previous MessageNext Message »
From:John H. Embretsen Date:February 10 2009 10:08pm
Subject:bzr commit into mysql-6.0-falcon-team branch (john.embretsen:3012) Bug#41870
Bug#42202
View as plain text  
#At file:///export/home/tmp/je159969/mysql/bzr-repos/fteam/ based on revid:olav@stripped

 3012 John H. Embretsen	2009-02-10
      New test related to Bug#41870 and Bug#42202, verifying that scheduled scavenging of old record versions in Falcon works with BLOBs, i.e. that the tablespace does not grow indefinitely when updating a BLOB multiple times.
      
      This test is perhaps more of a functional test of timing (schedule) based scavenging 
      related to BLOB updates than a real regression test for bug 41870 (Unbounded 
      tablespace growth when updating BLOB record). This is because in order to be as 
      timing-independent as possible, the test requires a fixed non-default value of 
      the option falcon_scavenge_schedule (currently set to 'every 5 seconds').
      
      On a standard disk based system the test may take a long time (~2 min on a relatively
      fast server with backround load) to run, and requires 30-50 MB of space, so this is
      a --big-test. In memory (--mem) the test should normally run in about 6-7 seconds.
added:
  mysql-test/suite/falcon/r/falcon_blob_space-big.result
  mysql-test/suite/falcon/t/falcon_blob_space-big.cnf
  mysql-test/suite/falcon/t/falcon_blob_space-big.test

per-file messages:
  mysql-test/suite/falcon/r/falcon_blob_space-big.result
    Result file for new test for tablespace growth control in relation to BLOB updates.
  mysql-test/suite/falcon/t/falcon_blob_space-big.cnf
    Configuration file telling the Falcon scavenger to run every 5 seconds.
    A .cnf file is used due to the difficulty of handling spaces and asterisks in the option's value when using other solutions.
  mysql-test/suite/falcon/t/falcon_blob_space-big.test
    New test for controlled tablespace growth in relation to BLOB updates (bug 41870).
=== added file 'mysql-test/suite/falcon/r/falcon_blob_space-big.result'
--- a/mysql-test/suite/falcon/r/falcon_blob_space-big.result	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/falcon/r/falcon_blob_space-big.result	2009-02-10 22:06:35 +0000
@@ -0,0 +1,138 @@
+*** Bug #41870 ***
+SET @@storage_engine = 'Falcon';
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (
+pk INT AUTO_INCREMENT PRIMARY KEY, 
+myblob LONGBLOB
+);
+
+** Load BLOB data into table
+
+INSERT INTO t1 (myblob) VALUES (repeat('a', 1*1024*1024));
+SELECT pk, LENGTH(myblob) FROM t1;
+pk	LENGTH(myblob)
+1	1048576
+** Recording initial size of falcon_user.fts tablespace file
+
+** Verify that falcon_scavenge_schedule is as expected:
+
+SHOW VARIABLES LIKE '%falcon_scavenge_schedule%';
+Variable_name	Value
+falcon_scavenge_schedule	1,6,11,16,21,26,31,36,41,46,51,56 * * * * *
+
+** Updating BLOB data 50 times
+
+50 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+49 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+48 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+47 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+46 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+45 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+44 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+43 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+42 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+41 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+40 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+39 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+38 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+37 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+36 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+35 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+34 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+33 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+32 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+31 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+30 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+29 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+28 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+27 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+26 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+25 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+24 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+23 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+22 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+21 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+20 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+19 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+18 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+17 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+16 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+15 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+14 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+13 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+12 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+11 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+10 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+9 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+8 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+7 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+6 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+5 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+4 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+3 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+2 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+1 updates left
+UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+
+** Checking file size of default tablespace file (falcon_user.fts)...
+** Tablespace growth threshold: 52428800 bytes
+** File size growth below test threshold. Test OK.
+
+** Final checks and cleanup...
+
+SELECT count(*) FROM t1;
+count(*)
+1
+SELECT pk, LENGTH(myblob) FROM t1;
+pk	LENGTH(myblob)
+1	1048576
+DROP TABLE t1;

=== added file 'mysql-test/suite/falcon/t/falcon_blob_space-big.cnf'
--- a/mysql-test/suite/falcon/t/falcon_blob_space-big.cnf	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/falcon/t/falcon_blob_space-big.cnf	2009-02-10 22:06:35 +0000
@@ -0,0 +1,2 @@
+[mysqld.1]
+falcon_scavenge_schedule=1,6,11,16,21,26,31,36,41,46,51,56 * * * * *

=== added file 'mysql-test/suite/falcon/t/falcon_blob_space-big.test'
--- a/mysql-test/suite/falcon/t/falcon_blob_space-big.test	1970-01-01 00:00:00 +0000
+++ b/mysql-test/suite/falcon/t/falcon_blob_space-big.test	2009-02-10 22:06:35 +0000
@@ -0,0 +1,267 @@
+--source include/have_falcon.inc
+--source include/big_test.inc
+
+#
+# Bug #41870: Unbounded tablespace growth when updating BLOB record
+#
+# This test inserts a 1 MB BLOB into a table and updates it
+# i times (i = 50) with the same data. The test measures the size
+# of the falcon_user.fts tablespace file before and after updating
+# BLOB data. If the file size growth is "too large" (i.e. close to
+# "i" times the BLOB data), we probably have introduced a regression.
+#
+# See also Bug#42202 (Unacceptable tablespace growth when doing rapid BLOB
+# updates).
+#
+# This test requires a few megabytes of storage (depending on how many updates
+# we do and how well the Falcon scavenger performs). Also, with disk-based
+# storage Falcon currently needs some time to process this kind of data in the
+# database, so it's a --big-test to avoid timeouts on slow machines.
+#
+# This test depends to some degree on the timing of the Falcon scavenger,
+# because with such large blobs and no/few other smaller record allocations
+# scavenging relies on time-based triggering, not load-based (as long as a fix
+# for Bug#42202 (BLOB-specific triggering of scavenger) remains unimplemented).
+#
+# Currently, load-based scavenging does not trigger unless a certain 
+# percentage (2%) of the record cache has been filled (this is checked
+# every 64 record allocations). Time-based scavenging is scheduled from the 
+# value of the --falcon-scavenge-schedule option.
+#
+# Hence the sleeps, to allow the test to succeed even in main memory (--mem). 
+# If scavenger timing changes, one might have to modify this test (e.g. number 
+# of BLOB updates, sleeps, size of BLOB, etc.).
+#
+# NOTE: This test currently requires a non-default falcon_scavenge_schedule, to
+#       avoid timing dependencies.
+#       This option is set in this test's .cnf file because the value contains
+#       spaces and asterisks, which do not apply nicely in all unix shells and 
+#       Windows command lines (even with -master.opt file).
+#       The falcon_scavenge_schedule option is static, and must be set upon
+#       Falcon initialization. This means that the server normally should not 
+#       bootstrap with Falcon enabled.
+#
+
+--echo *** Bug #41870 ***
+
+# ----------------------------------------------------- #
+# --- Initialisation                                --- #
+# ----------------------------------------------------- #
+let $engine = 'Falcon';
+eval SET @@storage_engine = $engine;
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (
+  pk INT AUTO_INCREMENT PRIMARY KEY, 
+  myblob LONGBLOB
+);
+
+
+--echo
+--echo ** Load BLOB data into table
+--echo
+INSERT INTO t1 (myblob) VALUES (repeat('a', 1*1024*1024));
+
+## Alternatively, load from a file...
+#eval INSERT INTO t1 (myblob) VALUES (LOAD_FILE('$MYSQL_TEST_DIR/include/UnicodeData.txt'));
+
+#
+# Check that the data load went ok. 
+# Otherwise we have NULL in the myblob column.
+#
+SELECT pk, LENGTH(myblob) FROM t1;
+
+#
+# We need to know the server's datadir in order to check Falcon's
+# tablespace files. Store the datadir path as an environment
+# variable so that we can read it from Perl later in the test.
+# (Using Perl to check and compare file sizes).
+#
+let MYSQLD_DATADIR= `SELECT @@datadir`;
+
+#
+# Check and make a note of the size of the falcon tablespace 
+# before updating data. This is to allow the test to run e.g. 
+# several times in a row, or after other tests using the same 
+# server instance, without failing.
+#
+# Storing the file size value in a file, as we would have a 
+# hard time retreiving it later (in a different perl section
+# or in the test script itself) otherwise.
+#
+perl;
+  #
+  # Get hold of path to current datadir
+  #
+  $datadir= $ENV{'MYSQLD_DATADIR'};
+  if (length($datadir) < 1)
+  {
+    print "ERROR: Unable to get path to datadir (MYSQLD_DATADIR)\n";
+    exit;
+  }
+
+  #
+  # Measure the size of the default Falcon tablespace file:
+  #
+  $filesize_before = -s "$datadir/falcon_user.fts";
+  
+  #
+  # Note: Forward slashes inside file path on Perl/Windows should work, but
+  #       if it does not, consider adding logic for handling this, e.g.
+  #       similar to http://search.cpan.org/~duncand/File-VirtualPath-1.011/lib/File/VirtualPath.pm
+  #
+  
+  if ($filesize_before < 1)
+  {
+    print "ERROR: Unable to read tablespace file falcon_user.fts\n";
+    print "       Looked in datadir: $datadir\n";
+    print "       Larger than INT_MAX bytes? (value: $filesize_before)\n";
+    exit;
+  }
+
+  #
+  # Using a file in MYSQL_TMP_DIR as temporary storage,
+  # to be able to read the value later in the test.
+  #
+
+  print "** Recording initial size of falcon_user.fts tablespace file\n";
+  $tmp_file= "$ENV{'MYSQL_TMP_DIR'}/falcon_filesize.tmp";
+
+  open(OUTFILE, ">$tmp_file")
+    or die("Unable to write to file $tmp_file: $!\n");
+  print OUTFILE "$filesize_before\n";
+  close(OUTFILE);
+
+EOF
+
+#
+# Test depends on the value of the falcon_scavenge_schedule option.
+#
+--echo
+--echo ** Verify that falcon_scavenge_schedule is as expected:
+--echo
+SHOW VARIABLES LIKE '%falcon_scavenge_schedule%';
+
+
+# ----------------------------------------------------- #
+# --- Test                                          --- #
+# ----------------------------------------------------- #
+
+#
+# i = number of BLOB updates to do in this test before
+#     measuring tablespace file size.
+#
+let $i=50;
+
+--echo
+--echo ** Updating BLOB data $i times
+--echo
+
+while ($i)
+{
+  --echo $i updates left
+  UPDATE t1 SET myblob = (repeat('b', 1*1024*1024)) WHERE pk = 1;
+  
+  # Sleep to allow the scavenger to catch up (e.g. if test is run with --mem).
+  # This is not really needed on slower storage systems/disks if
+  # $i is large enough. Adjustable at runtime by setting --sleep option of MTR.
+  # Depends on the value of the option falcon_scavenge_schedule.
+  --sleep 0.1
+  
+  dec $i;
+}
+
+## Check tablespace file size after updates
+--echo
+--echo ** Checking file size of default tablespace file (falcon_user.fts)...
+
+perl;
+  #
+  # Get hold of path to current datadir, previously stored as env var.
+  #
+  $datadir= $ENV{'MYSQLD_DATADIR'};
+  if (length($datadir) < 1)
+  {
+    print "ERROR: Unable to get path to datadir\n";
+    exit;
+  }
+
+  #
+  # Measure current file size
+  #
+  $filesize_after = -s "$datadir/falcon_user.fts";
+  
+  #
+  # Now, read the value (file size) recorded at the beginning of the test.
+  # We assume that the file $tmp_file  contains a single line with a 
+  # single number representing the original file size in bytes.
+  #
+
+  $tmp_file= "$ENV{'MYSQL_TMP_DIR'}/falcon_filesize.tmp";
+  open(FILE, "<", $tmp_file) or die("Unable to read $tmp_file!: $!\n");
+  $filesize_before=<FILE>;
+  $filesize_before =~ s/\s+$//;  # remove line endings from number
+  close(FILE);
+
+  if ( !($filesize_before =~ /^[1-9][0-9]*$/) || ($filesize_before < 1) )
+  {
+    print "ERROR: Unable to read original file size.\n";
+    print "       Expected to find a single positive number in the \n";
+    print "       file $tmp_file, but did not.\n";
+    print "       File contents: $filesize_before\n";
+    print "       Tablespace file larger than INT_MAX bytes?\n";
+    exit;
+  }
+
+  #
+  # Calculate file size growth.
+  #
+  $growth= $filesize_after - $filesize_before;
+  
+  #
+  # threshold = tablespace file size growth threshold. If the 
+  #             tablespace file size exceeds this threshold, 
+  #             we call it a test failure. 
+  #
+  #   Current threshold: $i * approx. BLOB size (rounded down)
+  #
+
+  $threshold= 50 * 1 * 1024 * 1024;
+  print "** Tablespace growth threshold: $threshold bytes\n";
+  
+  if ($growth > $threshold)
+  {
+    print "FAILURE: Tablespace filesize growth exceeded our threshold!\n";
+    print "         Size of tablespace file before: $filesize_before bytes\n";
+    print "         Size of tablespace file after:  $filesize_after bytes\n";
+    print "         Growth during BLOB updates was: $growth bytes\n";
+    print "\n";
+    print "         Please investigate if this is a regression or if the\n";
+    print "         test case needs adjustments (see bug 41870).\n";
+  }
+  else
+  {
+    print "** File size growth below test threshold. Test OK.\n";
+  }
+EOF
+
+--echo
+--echo ** Final checks and cleanup...
+--echo
+
+# ----------------------------------------------------- #
+# --- Check                                         --- #
+# ----------------------------------------------------- #
+SELECT count(*) FROM t1;
+SELECT pk, LENGTH(myblob) FROM t1;
+
+# ----------------------------------------------------- #
+# --- Final cleanup                                 --- #
+# ----------------------------------------------------- #
+DROP TABLE t1;
+# Unset temporarily used environment variable
+let MYSQLD_DATADIR=;
+

Thread
bzr commit into mysql-6.0-falcon-team branch (john.embretsen:3012) Bug#41870Bug#42202John H. Embretsen10 Feb