List:Commits« Previous MessageNext Message »
From:Pekka Nousiainen Date:September 24 2010 6:19pm
Subject:bzr commit into mysql-5.1-telco-7.0 branch (pekka:3792)
View as plain text  
#At file:///export/space/pekka/ms/ms-bug56829-70/ based on revid:magnus.blaudd@stripped

 3792 Pekka Nousiainen	2010-09-24 [merge]
      merge

    modified:
      storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp
      storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp
      storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp
      storage/ndb/src/kernel/blocks/dbtux/DbtuxBuild.cpp
      storage/ndb/src/kernel/blocks/dbtux/DbtuxMaint.cpp
      storage/ndb/src/kernel/blocks/dbtux/DbtuxNode.cpp
      storage/ndb/src/kernel/blocks/dbtux/DbtuxTree.cpp
      storage/ndb/test/ndbapi/testIndex.cpp
      storage/ndb/test/run-test/daily-basic-tests.txt
=== modified file 'storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2010-09-07 09:54:47 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2010-09-24 18:19:07 +0000
@@ -1669,7 +1669,7 @@ public:
    * index node.  TUX reads and writes the node directly via pointer.
    */
   int tuxAllocNode(EmulatedJamBuffer*, Uint32 fragPtrI, Uint32& pageId, Uint32& pageOffset, Uint32*& node);
-  void tuxFreeNode(Signal* signal, Uint32 fragPtrI, Uint32 pageId, Uint32 pageOffset, Uint32* node);
+  void tuxFreeNode(Uint32 fragPtrI, Uint32 pageId, Uint32 pageOffset, Uint32* node);
   void tuxGetNode(Uint32 fragPtrI, Uint32 pageId, Uint32 pageOffset, Uint32*& node);
 
   /*

=== modified file 'storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp	2010-09-03 05:35:51 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupIndex.cpp	2010-09-24 18:19:07 +0000
@@ -75,10 +75,8 @@ Dbtup::tuxAllocNode(EmulatedJamBuffer *
   return 0;
 }
 
-#if 0
 void
-Dbtup::tuxFreeNode(Signal* signal,
-                   Uint32 fragPtrI,
+Dbtup::tuxFreeNode(Uint32 fragPtrI,
                    Uint32 pageId,
                    Uint32 pageOffset,
                    Uint32* node)
@@ -90,15 +88,19 @@ Dbtup::tuxFreeNode(Signal* signal,
   TablerecPtr tablePtr;
   tablePtr.i= fragPtr.p->fragTableId;
   ptrCheckGuard(tablePtr, cnoOfTablerec, tablerec);
+
+  Local_key key;
+  key.m_page_no = pageId;
+  key.m_page_idx = pageOffset;
   PagePtr pagePtr;
-  pagePtr.i= pageId;
-  ptrCheckGuard(pagePtr, cnoOfPage, cpage);
+  Tuple_header* ptr = (Tuple_header*)get_ptr(&pagePtr, &key, tablePtr.p);
+
   Uint32 attrDescIndex= tablePtr.p->tabDescriptor + (0 << ZAD_LOG_SIZE);
   Uint32 attrDataOffset= AttributeOffset::getOffset(tableDescriptor[attrDescIndex + 1].tabDescr);
-  ndbrequire(node == &pagePtr.p->pageWord[pageOffset] + attrDataOffset);
-  freeTh(fragPtr.p, tablePtr.p, signal, pagePtr.p, pageOffset);
+  ndbrequire(node == (Uint32*)ptr + attrDataOffset);
+
+  free_fix_rec(fragPtr.p, tablePtr.p, &key, (Fix_page*)pagePtr.p);
 }
-#endif
 
 void
 Dbtup::tuxGetNode(Uint32 fragPtrI,

=== modified file 'storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp'
--- a/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp	2010-08-20 11:10:25 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtux/Dbtux.hpp	2010-09-24 18:19:07 +0000
@@ -492,7 +492,7 @@ private:
     Uint16 m_numAttrs;
     bool m_storeNullKey;
     TreeHead m_tree;
-    TupLoc m_freeLoc;           // list of free index nodes
+    TupLoc m_freeLoc;           // one free node for next op
     DLList<ScanOp> m_scanList;  // current scans on this fragment
     Uint32 m_tupIndexFragPtrI;
     Uint32 m_tupTableFragPtrI;
@@ -604,9 +604,11 @@ private:
    * DbtuxNode.cpp
    */
   int allocNode(TuxCtx&, NodeHandle& node);
+  void freeNode(NodeHandle& node);
   void selectNode(NodeHandle& node, TupLoc loc);
   void insertNode(NodeHandle& node);
   void deleteNode(NodeHandle& node);
+  void freePreallocatedNode(Frag& frag);
   void setNodePref(struct TuxCtx &, NodeHandle& node);
   // node operations
   void nodePushUp(TuxCtx&, NodeHandle& node, unsigned pos, const TreeEnt& ent, Uint32 scanList);

=== modified file 'storage/ndb/src/kernel/blocks/dbtux/DbtuxBuild.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtux/DbtuxBuild.cpp	2010-01-12 08:25:40 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtux/DbtuxBuild.cpp	2010-09-24 18:19:07 +0000
@@ -141,8 +141,6 @@ Dbtux::mt_buildIndexFragment(mt_BuildInd
       {
         break;
       }
-      // link to freelist
-      node.setLink(0, frag.m_freeLoc);
       frag.m_freeLoc = node.m_loc;
       ndbrequire(frag.m_freeLoc != NullTupLoc);
     }

=== modified file 'storage/ndb/src/kernel/blocks/dbtux/DbtuxMaint.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtux/DbtuxMaint.cpp	2009-12-14 10:58:03 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtux/DbtuxMaint.cpp	2010-09-24 11:06:19 +0000
@@ -146,8 +146,6 @@ Dbtux::execTUX_MAINT_REQ(Signal* signal)
         jam();
         break;
       }
-      // link to freelist
-      node.setLink(0, frag.m_freeLoc);
       frag.m_freeLoc = node.m_loc;
       ndbrequire(frag.m_freeLoc != NullTupLoc);
     }

=== modified file 'storage/ndb/src/kernel/blocks/dbtux/DbtuxNode.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtux/DbtuxNode.cpp	2009-12-14 22:14:34 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtux/DbtuxNode.cpp	2010-09-24 18:19:07 +0000
@@ -54,6 +54,24 @@ Dbtux::allocNode(TuxCtx& ctx, NodeHandle
 }
 
 /*
+ * Free index node in TUP
+ */
+void
+Dbtux::freeNode(NodeHandle& node)
+{
+  Frag& frag = node.m_frag;
+  Uint32 pageId = node.m_loc.getPageId();
+  Uint32 pageOffset = node.m_loc.getPageOffset();
+  Uint32* node32 = reinterpret_cast<Uint32*>(node.m_node);
+  c_tup->tuxFreeNode(frag.m_tupIndexFragPtrI,
+                     pageId, pageOffset, node32);
+  jamEntry();
+  // invalidate the handle
+  node.m_loc = NullTupLoc;
+  node.m_node = 0;
+}
+
+/*
  * Set handle to point to existing node.
  */
 void
@@ -77,9 +95,9 @@ void
 Dbtux::insertNode(NodeHandle& node)
 {
   Frag& frag = node.m_frag;
-  // unlink from freelist
+  // use up pre-allocated node
   selectNode(node, frag.m_freeLoc);
-  frag.m_freeLoc = node.getLink(0);
+  frag.m_freeLoc = NullTupLoc;
   new (node.m_node) TreeNode();
 #ifdef VM_TRACE
   TreeHead& tree = frag.m_tree;
@@ -90,19 +108,44 @@ Dbtux::insertNode(NodeHandle& node)
 }
 
 /*
- * Delete existing node.  Simply put it on the freelist.
+ * Delete existing node.  Make it the pre-allocated free node if there
+ * is none.  Otherwise return it to fragment's free list.
  */
 void
 Dbtux::deleteNode(NodeHandle& node)
 {
   Frag& frag = node.m_frag;
   ndbrequire(node.getOccup() == 0);
-  // link to freelist
-  node.setLink(0, frag.m_freeLoc);
-  frag.m_freeLoc = node.m_loc;
-  // invalidate the handle
-  node.m_loc = NullTupLoc;
-  node.m_node = 0;
+  if (frag.m_freeLoc == NullTupLoc)
+  {
+    jam();
+    frag.m_freeLoc = node.m_loc;
+    // invalidate the handle
+    node.m_loc = NullTupLoc;
+    node.m_node = 0;
+  }
+  else
+  {
+    jam();
+    freeNode(node);
+  }
+}
+
+/*
+ * Free the pre-allocated node, called when tree is empty.  This avoids
+ * leaving any used pages in DataMemory.
+ */
+void
+Dbtux::freePreallocatedNode(Frag& frag)
+{
+  if (frag.m_freeLoc != NullTupLoc)
+  {
+    jam();
+    NodeHandle node(frag);
+    selectNode(node, frag.m_freeLoc);
+    freeNode(node);
+    frag.m_freeLoc = NullTupLoc;
+  }
 }
 
 /*

=== modified file 'storage/ndb/src/kernel/blocks/dbtux/DbtuxTree.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtux/DbtuxTree.cpp	2009-12-14 22:14:34 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtux/DbtuxTree.cpp	2010-09-24 18:19:07 +0000
@@ -328,6 +328,8 @@ Dbtux::treeRemoveNode(Frag& frag, NodeHa
   }
   // tree is now empty
   tree.m_root = NullTupLoc;
+  // free even the pre-allocated node
+  freePreallocatedNode(frag);
 }
 
 /*

=== modified file 'storage/ndb/test/ndbapi/testIndex.cpp'
--- a/storage/ndb/test/ndbapi/testIndex.cpp	2010-03-19 14:15:57 +0000
+++ b/storage/ndb/test/ndbapi/testIndex.cpp	2010-09-24 18:19:07 +0000
@@ -24,6 +24,9 @@
 #include <NdbRestarts.hpp>
 #include <Vector.hpp>
 #include <signaldata/DumpStateOrd.hpp>
+#include <NodeBitmask.hpp>
+#include <NdbSqlUtil.hpp>
+#include <BlockNumbers.h>
 
 #define CHECK(b) if (!(b)) { \
   g_err << "ERR: "<< step->getName() \
@@ -2473,6 +2476,250 @@ runClearError(NDBT_Context* ctx, NDBT_St
   return NDBT_OK;
 }
 
+// bug#56829
+
+#undef CHECK2 // previous no good
+#define CHECK2(b, e) \
+  if (!(b)) { \
+    g_err << "ERR: " << #b << " failed at line " << __LINE__ \
+          << ": " << e << endl; \
+    result = NDBT_FAILED; \
+    break; \
+  }
+
+static int
+get_data_memory_pages(NdbMgmHandle h, NdbNodeBitmask dbmask, int* pages_out)
+{
+  int result = NDBT_OK;
+  int pages = 0;
+
+  while (1)
+  {
+    // sends dump 1000 and retrieves all replies
+    ndb_mgm_events* e = 0;
+    CHECK2((e = ndb_mgm_dump_events(h, NDB_LE_MemoryUsage, 0, 0)) != 0, ndb_mgm_get_latest_error_msg(h));
+
+    // sum up pages (also verify sanity)
+    for (int i = 0; i < e->no_of_events; i++)
+    {
+      ndb_logevent* le = &e->events[i];
+      CHECK2(le->type == NDB_LE_MemoryUsage, "bad event type " << le->type);
+      const ndb_logevent_MemoryUsage* lem = &le->MemoryUsage;
+      if (lem->block != DBTUP)
+        continue;
+      int nodeId = le->source_nodeid;
+      CHECK2(dbmask.get(nodeId), "duplicate event from node " << nodeId);
+      dbmask.clear(nodeId);
+      pages += lem->pages_used;
+      g_info << "i:" << i << " node:" << le->source_nodeid << " pages:" << lem->pages_used << endl;
+    }
+    free(e);
+    CHECK2(result == NDBT_OK, "failed");
+
+    char buf[NdbNodeBitmask::TextLength + 1];
+    CHECK2(dbmask.isclear(), "no response from nodes " << dbmask.getText(buf));
+    break;
+  }
+
+  *pages_out = pages;
+  return result;
+}
+
+int
+runBug56829(NDBT_Context* ctx, NDBT_Step* step)
+{
+  Ndb* pNdb = GETNDB(step);
+  NdbDictionary::Dictionary* pDic = pNdb->getDictionary();
+  const int loops = ctx->getNumLoops();
+  int result = NDBT_OK;
+  const NdbDictionary::Table tab(*ctx->getTab());
+  const int rows = ctx->getNumRecords();
+  const char* mgm = 0;//XXX ctx->getRemoteMgm();
+
+  char tabname[100];
+  strcpy(tabname, tab.getName());
+  char indname[100];
+  strcpy(indname, tabname);
+  strcat(indname, "X1");
+
+  (void)pDic->dropTable(tabname);
+
+  NdbMgmHandle h = 0;
+  NdbNodeBitmask dbmask;
+  // entry n marks if row with PK n exists
+  char* rowmask = new char [rows];
+  memset(rowmask, 0, rows);
+  int loop = 0;
+  while (loop < loops)
+  {
+    CHECK2(rows > 0, "rows must be != 0");
+    g_info << "loop " << loop << "<" << loops << endl;
+
+    // at first loop connect to mgm
+    if (loop == 0)
+    {
+      CHECK2((h = ndb_mgm_create_handle()) != 0, "mgm: failed to create handle");
+      CHECK2(ndb_mgm_set_connectstring(h, mgm) == 0, ndb_mgm_get_latest_error_msg(h));
+      CHECK2(ndb_mgm_connect(h, 0, 0, 0) == 0, ndb_mgm_get_latest_error_msg(h));
+      g_info << "mgm: connected to " << (mgm ? mgm : "default") << endl;
+
+      // make bitmask of DB nodes
+      dbmask.clear();
+      ndb_mgm_cluster_state* cs = 0;
+      CHECK2((cs = ndb_mgm_get_status(h)) != 0, ndb_mgm_get_latest_error_msg(h));
+      for (int j = 0; j < cs->no_of_nodes; j++)
+      {
+        ndb_mgm_node_state* ns = &cs->node_states[j];
+        if (ns->node_type == NDB_MGM_NODE_TYPE_NDB)
+        {
+          CHECK2(ns->node_status == NDB_MGM_NODE_STATUS_STARTED, "node " << ns->node_id << " not started status " << ns->node_status);
+          CHECK2(!dbmask.get(ns->node_id), "duplicate node id " << ns->node_id);
+          dbmask.set(ns->node_id);
+          g_info << "added DB node " << ns->node_id << endl;
+        }
+      }
+      free(cs);
+      CHECK2(result == NDBT_OK, "some DB nodes are not started");
+      CHECK2(!dbmask.isclear(), "found no DB nodes");
+    }
+
+    // data memory pages after following events
+    // 0-initial 1,2-create table,index 3-load 4-delete 5,6-drop index,table
+    int pages[7];
+
+    // initial
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[0]) == NDBT_OK, "failed");
+    g_info << "initial pages " << pages[0] << endl;
+
+    // create table
+    g_info << "create table " << tabname << endl;
+    const NdbDictionary::Table* pTab = 0;
+    CHECK2(pDic->createTable(tab) == 0, pDic->getNdbError());
+    CHECK2((pTab = pDic->getTable(tabname)) != 0, pDic->getNdbError());
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[1]) == NDBT_OK, "failed");
+    g_info << "create table pages " << pages[1] << endl;
+
+    // choice of index attributes is not relevant to this bug
+    // choose one non-PK updateable column
+    NdbDictionary::Index ind;
+    ind.setName(indname);
+    ind.setTable(tabname);
+    ind.setType(NdbDictionary::Index::OrderedIndex);
+    ind.setLogging(false);
+    {
+      HugoCalculator calc(*pTab);
+      for (int j = 0; j < pTab->getNoOfColumns(); j++)
+      {
+        const NdbDictionary::Column* col = pTab->getColumn(j);
+        if (col->getPrimaryKey() || calc.isUpdateCol(j))
+          continue;
+        CHARSET_INFO* cs = col->getCharset();
+        if (NdbSqlUtil::check_column_for_ordered_index(col->getType(), col->getCharset()) == 0)
+        {
+          ind.addColumn(*col);
+          break;
+        }
+      }
+    }
+    CHECK2(ind.getNoOfColumns() == 1, "cannot use table " << tabname);
+
+    // create index
+    g_info << "create index " << indname << " on " << ind.getColumn(0)->getName() << endl;
+    const NdbDictionary::Index* pInd = 0;
+    CHECK2(pDic->createIndex(ind, *pTab) == 0, pDic->getNdbError());
+    CHECK2((pInd = pDic->getIndex(indname, tabname)) != 0, pDic->getNdbError());
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[2]) == NDBT_OK, "failed");
+    g_info << "create index pages " << pages[2] << endl;
+
+    HugoTransactions trans(*pTab);
+
+    // load all records
+    g_info << "load records" << endl;
+    CHECK2(trans.loadTable(pNdb, rows) == 0, trans.getNdbError());
+    memset(rowmask, 1, rows);
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[3]) == NDBT_OK, "failed");
+    g_info << "load records pages " << pages[3] << endl;
+
+    // test index with random ops
+    g_info << "test index ops" << endl;
+    {
+      HugoOperations ops(*pTab);
+      for (int i = 0; i < rows; i++)
+      {
+        CHECK2(ops.startTransaction(pNdb) == 0, ops.getNdbError());
+        for (int j = 0; j < 32; j++)
+        {
+          int n = rand() % rows;
+          if (!rowmask[n])
+          {
+            CHECK2(ops.pkInsertRecord(pNdb, n) == 0, ops.getNdbError());
+            rowmask[n] = 1;
+          }
+          else if (rand() % 2 == 0)
+          {
+            CHECK2(ops.pkDeleteRecord(pNdb, n) == 0, ops.getNdbError());
+            rowmask[n] = 0;
+          }
+          else
+          {
+            CHECK2(ops.pkUpdateRecord(pNdb, n) == 0, ops.getNdbError());
+          }
+        }
+        CHECK2(result == NDBT_OK, "index ops batch failed");
+        CHECK2(ops.execute_Commit(pNdb) == 0, ops.getNdbError());
+        ops.closeTransaction(pNdb);
+      }
+      CHECK2(result == NDBT_OK, "index ops failed");
+    }
+
+    // delete all records
+    g_info << "delete records" << endl;
+    CHECK2(trans.clearTable(pNdb) == 0, trans.getNdbError());
+    memset(rowmask, 0, rows);
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[4]) == NDBT_OK, "failed");
+    g_info << "delete records pages " << pages[4] << endl;
+
+    // drop index
+    g_info << "drop index" <<  endl;
+    CHECK2(pDic->dropIndex(indname, tabname) == 0, pDic->getNdbError());
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[5]) == NDBT_OK, "failed");
+    g_info << "drop index pages " << pages[5] << endl;
+
+    // drop table
+    g_info << "drop table" << endl;
+    CHECK2(pDic->dropTable(tabname) == 0, pDic->getNdbError());
+    CHECK2(get_data_memory_pages(h, dbmask, &pages[6]) == NDBT_OK, "failed");
+    g_info << "drop table pages " << pages[6] << endl;
+
+    // verify
+    CHECK2(pages[1] == pages[0], "pages after create table " << pages[1]
+                                  << " not == initial pages " << pages[0]);
+    CHECK2(pages[2] == pages[0], "pages after create index " << pages[2]
+                                  << " not == initial pages " << pages[0]);
+    CHECK2(pages[3] >  pages[0], "pages after load " << pages[3]
+                                  << " not >  initial pages " << pages[0]);
+    CHECK2(pages[4] == pages[0], "pages after delete " << pages[4]
+                                  << " not == initial pages " << pages[0]);
+    CHECK2(pages[5] == pages[0], "pages after drop index " << pages[5]
+                                  << " not == initial pages " << pages[0]);
+    CHECK2(pages[6] == pages[0], "pages after drop table " << pages[6]
+                                  << " not == initial pages " << pages[0]);
+
+    loop++;
+
+    // at last loop disconnect from mgm
+    if (loop == loops)
+    {
+      CHECK2(ndb_mgm_disconnect(h) == 0, ndb_mgm_get_latest_error_msg(h));
+      ndb_mgm_destroy_handle(&h);
+      g_info << "mgm: disconnected" << endl;
+    }
+  }
+  delete [] rowmask;
+
+  return result;
+}
+
 
 NDBT_TESTSUITE(testIndex);
 TESTCASE("CreateAll", 
@@ -2874,6 +3121,11 @@ TESTCASE("FireTrigOverload", ""){
   FINALIZER(runClearError);
   FINALIZER(createRandomIndex_Drop);
 }
+TESTCASE("Bug56829",
+         "Return empty ordered index nodes to index fragment "
+         "so that empty fragment pages can be freed"){
+  STEP(runBug56829);
+}
   
 NDBT_TESTSUITE_END(testIndex);
 

=== modified file 'storage/ndb/test/run-test/daily-basic-tests.txt'
--- a/storage/ndb/test/run-test/daily-basic-tests.txt	2010-09-23 06:22:44 +0000
+++ b/storage/ndb/test/run-test/daily-basic-tests.txt	2010-09-24 18:19:07 +0000
@@ -1593,3 +1593,7 @@ max-time: 500
 cmd: testDict
 args: -n Bug53944 T1
 
+max-time: 300
+cmd: testIndex
+args: -n Bug56829 T1
+


Attachment: [text/bzr-bundle] bzr/pekka@mysql.com-20100924181907-9bmtnkqdff9b727q.bundle
Thread
bzr commit into mysql-5.1-telco-7.0 branch (pekka:3792) Pekka Nousiainen24 Sep