bzr commit into mysql-5.1-bugteam branch (anurag.shekhar:2768) Bug#39918

[Anurag Shekhar]

#At file:///home/anurag/mysqlsrc/mysql-5.1-bugteam/ based on revid:patrick.crews@stripped

 2768 Anurag Shekhar	2009-02-04
      Bug #39918 memory (heap) engine crashing with b-tree index and DELETE with seg fault
      
      While evaluating self join with delete server uses two instance of handler 
      (with both holding reference to same file). During query evaluation last_pos 
      is cached in handler to improve search performance. But in the mean while 
      second handler is used to delete the node last_pos is holding. This makes 
      the last_pos an invalid and when accessed results in segmentation fault.
      
      To fix this I have introduced a new variable in HP_INFO structure (deleted) 
      which remembers the number of deleted (initialized while opening the file) 
      nodes since last_pos is updated. If this value differs from the value of 
      deleted nodes in shared structure last_pos is invalidated (set to zero).
modified:
  include/heap.h
  mysql-test/r/heap_btree.result
  mysql-test/t/heap_btree.test
  storage/heap/hp_open.c
  storage/heap/hp_rnext.c

per-file messages:
  include/heap.h
    Added new field in HP_INFO to remember number of deleted nodes since last_pos was last updated.
  mysql-test/r/heap_btree.result
    Updated result for newly introduced test.
  mysql-test/t/heap_btree.test
    Added new case to test self join with delete.
  storage/heap/hp_open.c
    Initializes HP_INFO's deleted field to shared->deleted (value while opening the file)
  storage/heap/hp_rnext.c
    Introduced new test to check if any more node has been deleted since last_pos was updated. If it has then invalidate last_pos (by setting it to zero).
=== modified file 'include/heap.h'
--- a/include/heap.h	2007-07-24 22:58:12 +0000
+++ b/include/heap.h	2009-02-04 08:36:11 +0000
@@ -179,6 +179,9 @@ typedef struct st_heap_info
   THR_LOCK_DATA lock;
 #endif
   LIST open_list;
+  int deleted;                    /* Record number of deleted keys in tree
+                                     If this value isn't same as s->deleted 
+                                     last_pos should be reinitialized */
 } HP_INFO;
 
 

=== modified file 'mysql-test/r/heap_btree.result'
--- a/mysql-test/r/heap_btree.result	2007-10-19 20:12:14 +0000
+++ b/mysql-test/r/heap_btree.result	2009-02-04 08:36:11 +0000
@@ -337,3 +337,7 @@ NULL	NULL
 NULL	1
 drop table t1;
 End of 5.0 tests
+CREATE TABLE t1(a INT, KEY USING BTREE (a)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES(1),(1);
+DELETE a1 FROM t1 AS a1, t1 AS a2 WHERE a1.a=a2.a;
+DROP TABLE t1;

=== modified file 'mysql-test/t/heap_btree.test'
--- a/mysql-test/t/heap_btree.test	2007-10-19 20:12:14 +0000
+++ b/mysql-test/t/heap_btree.test	2009-02-04 08:36:11 +0000
@@ -255,3 +255,11 @@ drop table t1;
 
 --echo End of 5.0 tests
 
+#
+# bug#39918 - memory (heap) engine crashing while executing self join with delete
+#
+
+CREATE TABLE t1(a INT, KEY USING BTREE (a)) ENGINE=MEMORY;
+INSERT INTO t1 VALUES(1),(1);
+DELETE a1 FROM t1 AS a1, t1 AS a2 WHERE a1.a=a2.a;
+DROP TABLE t1;

=== modified file 'storage/heap/hp_open.c'
--- a/storage/heap/hp_open.c	2007-07-24 22:58:12 +0000
+++ b/storage/heap/hp_open.c	2009-02-04 08:36:11 +0000
@@ -106,6 +106,10 @@ HP_INFO *heap_open(const char *name, int
     info->open_list.data= (void*) info;
     heap_open_list= list_add(heap_open_list,&info->open_list);
   }
+  /*
+    Remember the number of deleted records while opening the file.
+  */
+  info->deleted= share->deleted;
   pthread_mutex_unlock(&THR_LOCK_heap);
   DBUG_RETURN(info);
 }

=== modified file 'storage/heap/hp_rnext.c'
--- a/storage/heap/hp_rnext.c	2007-09-13 12:12:46 +0000
+++ b/storage/heap/hp_rnext.c	2009-02-04 08:36:11 +0000
@@ -32,6 +32,17 @@ int heap_rnext(HP_INFO *info, uchar *rec
   {
     heap_rb_param custom_arg;
 
+    /*
+      Check if any key is deleted in tree since last_pos was
+      updated. If it has then last_pos might be holding a invalid
+      address. Invalidate last_pos and remember the current count
+      of deleted keys.
+     */
+    if (info->deleted != share->deleted) 
+    {
+        info->last_pos= 0;
+        info->deleted= share->deleted;
+    }
     if (info->last_pos)
     {
       /*