List:Commits« Previous MessageNext Message »
From:jonas Date:January 25 2006 8:48am
Subject:bk commit into 5.1 tree (jonas:1.2074) BUG#16771
View as plain text  
Below is the list of changes that have just been committed into a local
5.1 repository of jonas. When jonas does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2074 06/01/25 09:47:56 jonas@stripped +3 -0
  bug#16771 - ndb dd - fix leak of extents

  storage/ndb/src/kernel/blocks/pgman.cpp
    1.4 06/01/25 09:47:53 jonas@stripped +20 -0
    Add debug printouts

  storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp
    1.5 06/01/25 09:47:53 jonas@stripped +155 -50
    Add debug printouts
    Fix leak of extents

  storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp
    1.35 06/01/25 09:47:53 jonas@stripped +2 -1
    Add debug printouts

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	jonas
# Host:	perch.ndb.mysql.com
# Root:	/home/jonas/src/51-new

--- 1.4/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp	2006-01-24 10:47:01 +01:00
+++ 1.5/storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp	2006-01-25 09:47:53 +01:00
@@ -17,6 +17,112 @@
 #define DBTUP_C
 #include "Dbtup.hpp"
 
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Page> & ptr)
+{
+  out << "[ Page: ptr.i: " << ptr.i 
+      << " [ m_file_no: " << ptr.p->m_file_no
+      << " m_page_no: " << ptr.p->m_page_no << "]"
+      << " list_index: " << ptr.p->list_index 
+      << " free_space: " << ptr.p->free_space
+      << " uncommitted_used_space: " << ptr.p->uncommitted_used_space
+      << " ]";
+  return out;
+}
+
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Page_request> & ptr)
+{
+  out << "[ Page_request: ptr.i: " << ptr.i
+      << " " << ptr.p->m_key
+      << " m_estimated_free_space: " << ptr.p->m_estimated_free_space
+      << " m_list_index: " << ptr.p->m_list_index
+      << " m_frag_ptr_i: " << ptr.p->m_frag_ptr_i
+      << " m_extent_info_ptr: " << ptr.p->m_extent_info_ptr
+      << " m_ref_count: " << ptr.p->m_ref_count
+      << " m_uncommitted_used_space: " << ptr.p->m_uncommitted_used_space
+      << " ]";
+  
+  return out;
+}
+
+static
+NdbOut&
+operator<<(NdbOut& out, const Ptr<Dbtup::Extent_info> & ptr)
+{
+  out << "[ Extent_info: ptr.i " << ptr.i
+      << " " << ptr.p->m_key
+      << " m_first_page_no: " << ptr.p->m_first_page_no
+      << " m_free_space: " << ptr.p->m_free_space
+      << " m_free_matrix_pos: " << ptr.p->m_free_matrix_pos
+      << " m_free_page_count: [";
+
+  for(Uint32 i = 0; i<Dbtup::EXTENT_SEARCH_MATRIX_COLS; i++)
+    out << " " << ptr.p->m_free_page_count[i];
+  out << " ] ]";
+
+  return out;
+}
+
+void 
+Dbtup::dump_disk_alloc(Dbtup::Disk_alloc_info & alloc)
+{
+  ndbout_c("dirty pages");
+  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Page> ptr;
+    ArrayPool<Page> *pool= (ArrayPool<Page>*)&m_global_page_pool;
+    LocalDLList<Page> list(*pool, alloc.m_dirty_pages[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+  ndbout_c("page requests");
+  for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Page_request> ptr;
+    LocalDLList<Page_request> list(c_page_request_pool, 
+				   alloc.m_page_requests[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+
+  ndbout_c("Extent matrix");
+  for(Uint32 i = 0; i<alloc.SZ; i++)
+  {
+    printf("  %d : ", i);
+    Ptr<Extent_info> ptr;
+    LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[i]);
+    for(list.first(ptr); !ptr.isNull(); list.next(ptr))
+    {
+      ndbout << ptr << " ";
+    }
+    ndbout_c("");
+  }
+
+  if (alloc.m_curr_extent_info_ptr_i != RNIL)
+  {
+    Ptr<Extent_info> ptr;
+    c_extent_pool.getPtr(ptr, alloc.m_curr_extent_info_ptr_i);
+    ndbout << "current extent: " << ptr << endl;
+  }
+}
+
+#if defined VM_TRACE || true
+#define ddassert(x) do { if(unlikely(!(x))) { dump_disk_alloc(alloc); ndbrequire(false); } } while(0)
+#else
+#define ddassert(x)
+#endif
+
 Dbtup::Disk_alloc_info::Disk_alloc_info(const Tablerec* tabPtrP, 
 					Uint32 extent_size)
 {
@@ -60,19 +166,19 @@
    * Find the biggest available (with most free space)
    * Return position in matrix
    */
+  Uint32 col = calc_page_free_bits(sz);
   Uint32 mask= EXTENT_SEARCH_MATRIX_COLS - 1;
   for(Uint32 i= 0; i<EXTENT_SEARCH_MATRIX_SIZE; i++)
   {
     // Check that it can cater for request
-    if (m_extent_search_matrix[i] < sz)
+    if (!m_free_extents[i].isEmpty())
     {
-      i = (i + mask) & ~mask;
-      continue;
+      return i;
     }
     
-    if (!m_free_extents[i].isEmpty())
+    if ((i & mask) >= col)
     {
-      return i;
+      i = (i & ~mask) + mask;
     }
   }
   
@@ -92,13 +198,7 @@
    *   if zero (or very small free space) put 
    *     absolutly last
    */
-  {
-    
-    printf("free space %d free_page_thresholds ", free);
-    for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_ROWS; i++)
-      printf("%d ", m_total_extent_free_space_thresholds[i]);
-    ndbout_c("");
-    
+  {    
     const Uint32 *arr= m_total_extent_free_space_thresholds;
     for(; free < * arr++; row++)
       assert(row < EXTENT_SEARCH_MATRIX_ROWS);
@@ -123,11 +223,6 @@
    */
   Uint32 pos= (row * (mask + 1)) + (col & mask);
   
-  printf("free space %d free_page_count ", free);
-  for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_COLS; i++)
-    printf("%d ", extP->m_free_page_count[i]);
-  ndbout_c(" -> row: %d col: %d -> pos= %d", row, col, pos);
-
   assert(pos < EXTENT_SEARCH_MATRIX_SIZE);
   return pos;
 }
@@ -237,7 +332,9 @@
        *   and since it couldn't accomadate the request
        *   we put it on the free list
        */
+      alloc.m_curr_extent_info_ptr_i = RNIL;
       Uint32 pos= alloc.calc_extent_pos(ext.p);
+      ext.p->m_free_matrix_pos = pos;
       LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[pos]);
       list.add(ext);
     }
@@ -290,7 +387,14 @@
     alloc.m_curr_extent_info_ptr_i= ext.i;
     ext.p->m_free_matrix_pos= RNIL;
     pageBits= tsman.alloc_page_from_extent(&ext.p->m_key, bits);
-    ndbassert(pageBits >= 0);
+#ifdef VM_TRACE
+    ddassert(pageBits >= 0);
+#else
+    if (unlikely(pageBits < 0))
+    {
+      return -AllocExtentReq::NoExtentAvailable;
+    }
+#endif
   }
   
   /**
@@ -305,18 +409,18 @@
    */
   Uint32 size= alloc.calc_page_free_space((Uint32)pageBits);
   
-  ndbassert(size >= sz);
+  ddassert(size >= sz);
   Uint32 new_size = size - sz;   // Subtract alloc rec
   req.p->m_estimated_free_space= new_size; // Store on page request
 
   Uint32 newPageBits= alloc.calc_page_free_bits(new_size);
   if (newPageBits != (Uint32)pageBits)
   {
-    ndbassert(ext.p->m_free_page_count[pageBits] > 0);
+    ddassert(ext.p->m_free_page_count[pageBits] > 0);
     ext.p->m_free_page_count[pageBits]--;
     ext.p->m_free_page_count[newPageBits]++;
   }
-  ndbassert(ext.p->m_free_space >= sz);
+  ddassert(ext.p->m_free_space >= sz);
   ext.p->m_free_space -= sz;
   
   // And put page request in correct free list
@@ -365,13 +469,13 @@
 				     Ptr<Page> pagePtr, 
 				     Uint32 old_idx, Uint32 sz)
 {
-  ndbassert(pagePtr.p->list_index == old_idx);
+  ddassert(pagePtr.p->list_index == old_idx);
 
   Uint32 free= pagePtr.p->free_space;
   Uint32 used= pagePtr.p->uncommitted_used_space + sz;
   Uint32 ext= pagePtr.p->m_extent_info_ptr;
   
-  ndbassert(free >= used);
+  ddassert(free >= used);
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
@@ -385,14 +489,14 @@
     old_list.remove(pagePtr);
     new_list.add(pagePtr);
 
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
     pagePtr.p->list_index= new_idx;  
   }
 
   pagePtr.p->uncommitted_used_space = used;
-  ndbassert(extentPtr.p->m_free_space >= sz);
+  ddassert(extentPtr.p->m_free_space >= sz);
   extentPtr.p->m_free_space -= sz;
   Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
   if (old_pos != RNIL) // Current extent
@@ -417,7 +521,7 @@
 				       Ptr<Page_request> req, 
 				       Uint32 old_idx, Uint32 sz)
 {
-  ndbassert(req.p->m_list_index == old_idx);
+  ddassert(req.p->m_list_index == old_idx);
 
   Uint32 free= req.p->m_estimated_free_space;
   Uint32 used= req.p->m_uncommitted_used_space + sz;
@@ -426,7 +530,7 @@
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
-  ndbassert(free >= sz);
+  ddassert(free >= sz);
   Uint32 new_idx= alloc.calc_page_free_bits(free - sz);
   
   if (old_idx != new_idx)
@@ -437,7 +541,7 @@
     old_list.remove(req);
     new_list.add(req);
 
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
     req.p->m_list_index= new_idx;  
@@ -445,7 +549,7 @@
 
   req.p->m_uncommitted_used_space = used;
   req.p->m_estimated_free_space = free - sz;
-  ndbassert(extentPtr.p->m_free_space >= sz);
+  ddassert(extentPtr.p->m_free_space >= sz);
   extentPtr.p->m_free_space -= sz;
   Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
   if (old_pos != RNIL) // Current extent
@@ -551,11 +655,11 @@
    * 3) register callback in pgman (unmap callback)
    * 4) inform pgman about current users
    */
-  ndbassert((page->list_index & 0x8000) == 0x8000);
-  ndbassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
-  ndbassert(page->m_page_no == req.p->m_key.m_page_no);
-  ndbassert(page->m_file_no == req.p->m_key.m_file_no);
   Disk_alloc_info& alloc= fragPtr.p->m_disk_alloc_info;
+  ddassert((page->list_index & 0x8000) == 0x8000);
+  ddassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
+  ddassert(page->m_page_no == req.p->m_key.m_page_no);
+  ddassert(page->m_file_no == req.p->m_key.m_file_no);
   
   Uint32 old_idx = req.p->m_list_index;
   Uint32 free= req.p->m_estimated_free_space;
@@ -564,9 +668,9 @@
   Uint32 real_free = page->free_space;
   Uint32 real_used = used + page->uncommitted_used_space;
  
-  ndbassert(real_free >= free);
-  ndbassert(real_free >= real_used);
-  ndbassert(alloc.calc_page_free_bits(free) == old_idx);
+  ddassert(real_free >= free);
+  ddassert(real_free >= real_used);
+  ddassert(alloc.calc_page_free_bits(free) == old_idx);
   Uint32 new_idx= alloc.calc_page_free_bits(real_free - real_used);
   
   /**
@@ -587,7 +691,7 @@
     
     if (old_idx != new_idx)
     {
-      ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+      ddassert(extentPtr.p->m_free_page_count[old_idx]);
       extentPtr.p->m_free_page_count[old_idx]--;
       extentPtr.p->m_free_page_count[new_idx]++;
     }
@@ -721,13 +825,14 @@
 		       Local_key* key, PagePtr pagePtr, Uint32 gci)
 {
   Uint32 logfile_group_id= fragPtrP->m_logfile_group_id;
+  Disk_alloc_info& alloc= fragPtrP->m_disk_alloc_info;
 
   Uint64 lsn;
   Uint32 old_free = pagePtr.p->free_space;
-  Uint32 old_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(old_free);
+  Uint32 old_bits= alloc.calc_page_free_bits(old_free);
   if (tabPtrP->m_attributes[DD].m_no_of_varsize == 0)
   {
-    ndbassert(pagePtr.p->uncommitted_used_space > 0);
+    ddassert(pagePtr.p->uncommitted_used_space > 0);
     pagePtr.p->uncommitted_used_space--;
     key->m_page_idx= ((Fix_page*)pagePtr.p)->alloc_record();
     lsn= disk_page_undo_alloc(pagePtr.p, key, 1, gci, logfile_group_id);
@@ -735,7 +840,7 @@
   else
   {
     Uint32 sz= key->m_page_idx;
-    ndbassert(pagePtr.p->uncommitted_used_space >= sz);
+    ddassert(pagePtr.p->uncommitted_used_space >= sz);
     pagePtr.p->uncommitted_used_space -= sz;
     key->m_page_idx= ((Var_page*)pagePtr.p)->
       alloc_record(sz, (Var_page*)ctemp_page, 0);
@@ -744,7 +849,7 @@
   }
 
   Uint32 new_free = pagePtr.p->free_space;
-  Uint32 new_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(new_free);
+  Uint32 new_bits= alloc.calc_page_free_bits(new_free);
   
   if (old_bits != new_bits)
   {
@@ -806,20 +911,20 @@
 
   Uint32 ext = pagePtr.p->m_extent_info_ptr;
   Uint32 used = pagePtr.p->uncommitted_used_space;
-  ndbassert(old_free >= used);
-  ndbassert(new_free >= used);
-  ndbassert(new_free >= old_free);
+  ddassert(old_free >= used);
+  ddassert(new_free >= used);
+  ddassert(new_free >= old_free);
   page_idx = pagePtr.p->list_index;
   Uint32 old_idx = page_idx & 0x7FFF;
   Uint32 new_idx = alloc.calc_page_free_bits(new_free - used);
-  ndbassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
+  ddassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
 
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
 
   if (old_idx != new_idx)
   {
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
 
@@ -915,16 +1020,16 @@
   Uint32 ext = pagePtr.p->m_extent_info_ptr;
 
   Uint32 old_idx = page_idx & 0x7FFF;
-  ndbassert(free >= used);
-  ndbassert(used >= sz);
-  ndbassert(alloc.calc_page_free_bits(free - used) == old_idx);
+  ddassert(free >= used);
+  ddassert(used >= sz);
+  ddassert(alloc.calc_page_free_bits(free - used) == old_idx);
   Uint32 new_idx = alloc.calc_page_free_bits(free - used + sz);
 
   Ptr<Extent_info> extentPtr;
   c_extent_pool.getPtr(extentPtr, ext);
   if (old_idx != new_idx)
   {
-    ndbassert(extentPtr.p->m_free_page_count[old_idx]);
+    ddassert(extentPtr.p->m_free_page_count[old_idx]);
     extentPtr.p->m_free_page_count[old_idx]--;
     extentPtr.p->m_free_page_count[new_idx]++;
 

--- 1.3/storage/ndb/src/kernel/blocks/pgman.cpp	2006-01-15 20:45:04 +01:00
+++ 1.4/storage/ndb/src/kernel/blocks/pgman.cpp	2006-01-25 09:47:53 +01:00
@@ -40,6 +40,12 @@
 #define dbg(x)
 #endif
 
+#if 1
+#define DBG_LCP(x)
+#else
+#define DBG_LCP(x) ndbout << x
+#endif
+
 Pgman::Pgman(const Configuration & conf) :
   SimulatedBlock(PGMAN, conf),
   m_file_map(m_data_buffer_pool),
@@ -1083,6 +1089,7 @@
   LcpFragOrd* ord = (LcpFragOrd*)signal->getDataPtr();
   ndbrequire(ord->lcpId >= m_last_lcp_complete + 1 || m_last_lcp_complete == 0);
   m_last_lcp = ord->lcpId;
+  DBG_LCP("execLCP_FRAG_ORD" << endl);
 
   ndbrequire(!m_lcp_outstanding);
   ndbrequire(m_lcp_copy_page_free);
@@ -1104,6 +1111,8 @@
   EndLcpReq* req = (EndLcpReq*)signal->getDataPtr();
   m_end_lcp_req = *req;
 
+  DBG_LCP("execEND_LCP_REQ" << endl);
+
 #ifdef VM_TRACE
   debugOut
     << "PGMAN: execEND_LCP_REQ"
@@ -1117,6 +1126,7 @@
     ndbrequire(! m_lcp_loop_on);
     signal->theData[0] = m_end_lcp_req.senderData;
     sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
+    DBG_LCP("GSN_END_LCP_CONF" << endl);
   }
 
   m_last_lcp_complete = m_last_lcp;
@@ -1149,6 +1159,8 @@
       Ptr<Page_entry>& ptr = iter.curr;
       Uint16 state = ptr.p->m_state;
 
+      DBG_LCP("PROCESS LCP: " << ptr);
+      
       if (ptr.p->m_last_lcp < m_last_lcp &&
           (state & Page_entry::DIRTY))
       {
@@ -1159,6 +1171,7 @@
         }
         if (state & Page_entry::BUSY)
         {
+	  DBG_LCP(" BUSY" << endl);
           break;  // wait for it
         }
         if (state & Page_entry::LOCKED)
@@ -1169,6 +1182,7 @@
            */
           if (!m_lcp_copy_page_free)
           {
+	    DBG_LCP(" !m_lcp_copy_page_free" << endl);
             break;
           }
           m_lcp_copy_page_free = false;
@@ -1183,10 +1197,12 @@
         }
         else if (state & Page_entry::PAGEOUT)
         {
+	  DBG_LCP(" PAGEOUT -> state |= LCP" << endl);
           set_page_state(ptr, state | Page_entry::LCP);
         }
         else
         {
+	  DBG_LCP(" pageout()" << endl);
           ptr.p->m_state |= Page_entry::LCP;
           pageout(signal, ptr);
         }
@@ -1205,11 +1221,15 @@
     {
       signal->theData[0] = m_end_lcp_req.senderData;
       sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
+      DBG_LCP("GSN_END_LCP_CONF" << endl);
     }
+    DBG_LCP(" -- RETURN FALSE" << endl);
     m_last_lcp_complete = m_last_lcp;
     m_lcp_curr_bucket = ~(Uint32)0;
     return false;
   }
+
+  DBG_LCP(" -- RETURN TRUE" << endl);
   return true;
 }
 

--- 1.34/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2006-01-20 05:41:56 +01:00
+++ 1.35/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2006-01-25 09:47:53 +01:00
@@ -565,7 +565,6 @@
      * 
      */
     STATIC_CONST( SZ = EXTENT_SEARCH_MATRIX_SIZE );
-    Uint32 m_extent_search_matrix[SZ]; // 4x4
     DLList<Extent_info>::Head m_free_extents[SZ];
     Uint32 m_total_extent_free_space_thresholds[EXTENT_SEARCH_MATRIX_ROWS];
     Uint32 m_page_free_bits_map[EXTENT_SEARCH_MATRIX_COLS];
@@ -593,6 +592,8 @@
     SLList<Extent_info, Extent_list_t>::Head m_extent_list;
   };
   
+  void dump_disk_alloc(Disk_alloc_info&);
+
 struct Fragrecord {
   Uint32 nextStartRange;
   Uint32 currentPageRange;
Thread
bk commit into 5.1 tree (jonas:1.2074) BUG#16771jonas25 Jan