bk commit into 5.1 tree (tomas:1.1896)

[tomas]

Below is the list of changes that have just been committed into a local
5.1 repository of tomas. When tomas does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.1896 05/05/30 17:11:41 tomas@stripped +3 -0
  added buffer usage reporting
  added abstraction for lists of events

  storage/ndb/src/ndbapi/NdbEventOperationImpl.hpp
    1.29 05/05/30 17:11:35 tomas@stripped +94 -9
    added buffer usage reporting
    added abstraction for lists of events

  storage/ndb/src/ndbapi/NdbEventOperationImpl.cpp
    1.59 05/05/30 17:11:35 tomas@stripped +80 -128
    added buffer usage reporting
    added abstraction for lists of events

  sql/ha_ndbcluster.cc
    1.260 05/05/30 17:11:35 tomas@stripped +7 -3
    better timer printout

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	tomas
# Host:	poseidon.ndb.mysql.com
# Root:	/home/tomas/mysql-5.1-wl2325

--- 1.58/storage/ndb/src/ndbapi/NdbEventOperationImpl.cpp	2005-05-27 14:29:50 +02:00
+++ 1.59/storage/ndb/src/ndbapi/NdbEventOperationImpl.cpp	2005-05-30 17:11:35 +02:00
@@ -576,17 +576,11 @@
   m_sz= 0;
   m_free_data_count= 0;
   m_free_data= 0;
+  m_free_data_sz= 0;
 
   // initialize lists
   bzero(&m_complete_data, sizeof(m_complete_data));
   m_latest_complete_GCI = m_latestGCI = m_waitGCI = 0;
-  m_available_data= 0;
-  m_last_available_data= 0;
-  m_available_data_count= 0;
-
-  m_used_data= 0;
-  m_last_used_data= 0;
-  m_used_data_count= 0;
 
   bzero(&g_empty_gci_container, sizeof(Gci_container));
   m_active_gci.fill(2 * ACTIVE_GCI_DIRECTORY_SIZE - 1, g_empty_gci_container);
@@ -669,7 +663,7 @@
     ev_op->m_data_done_count= 0;
 #endif
 
-    ret= m_available_data_count;
+    ret= m_available_data.m_count;
   }
   NdbMutex_Unlock(m_mutex); // we have moved the data
 
@@ -684,27 +678,20 @@
   DBUG_ENTER("NdbEventBuffer::next");
   const char *m_latest_command_save= m_latest_command;
 
-  if (m_used_data_count > 1024)
+  if (m_used_data.m_count > 1024)
   {
     m_latest_command= "NdbEventBuffer::next (lock)";
     NdbMutex_Lock(m_mutex);
     // return m_used_data to m_free_data
-    m_last_used_data->m_next= m_free_data;
-    m_free_data= m_used_data;
-    m_free_data_count+= m_used_data_count;
-
-    // all m_used_data returned to m_free_data
-    m_used_data= 0;
-    m_last_used_data= 0;
-    m_used_data_count= 0;
+    free_list(m_used_data);
 
     NdbMutex_Unlock(m_mutex);
   }
   m_latest_command= "NdbEventBuffer::next";
 
-  while (m_available_data)
+  while (m_available_data.m_head)
   {
-    NdbEventOperationImpl *ev_op= m_available_data->m_event_op;
+    NdbEventOperationImpl *ev_op= m_available_data.m_head->m_event_op;
     int r= ev_op->next(maxGCI);
     if (r > 0)
     {
@@ -758,9 +745,9 @@
 {
   out << "[ GCI: " << gci.m_gci
       << "  state: " << hex << gci.m_state 
-      << "  head: " << hex << gci.m_head
-      << "  tail: " << hex << gci.m_tail
-      << "  cnt: " << dec << gci.m_event_count
+      << "  head: " << hex << gci.m_data.m_head
+      << "  tail: " << hex << gci.m_data.m_tail
+      << "  cnt: " << dec << gci.m_data.m_count
       << "]";
   return out;
 }
@@ -839,22 +826,13 @@
   {
     if(likely(gci == m_latestGCI + 1 || m_latestGCI == 0))
     {
-      if(bucket->m_head)
+      if(!bucket->m_data.is_empty())
       {
 #ifdef VM_TRACE
-	assert(bucket->m_event_count);
+	assert(bucket->m_data.m_count);
 #endif
-	if(m_complete_data.m_tail)
-	{
-	  m_complete_data.m_tail->m_next = bucket->m_head;
-	}
-	else
-	{
-	  m_complete_data.m_head = bucket->m_head;
-	}
-	
-	m_complete_data.m_tail = bucket->m_tail;
-	m_complete_data.m_event_count += bucket->m_event_count;
+	m_complete_data.m_data.append(bucket->m_data);
+	reportUsage();
       }
       bzero(bucket, sizeof(Gci_container));
       bucket->m_gci = gci + ACTIVE_GCI_DIRECTORY_SIZE;
@@ -913,25 +891,15 @@
     }
 
     printf("complete_outof_order_gcis - completing %lld", start_gci);
-    if(bucket->m_head)
+    if(!bucket->m_data.is_empty())
     {
 #ifdef VM_TRACE
-      assert(bucket->m_event_count);
+      assert(bucket->m_data.m_count);
 #endif
-      if(m_complete_data.m_tail)
-      {
-	m_complete_data.m_tail->m_next = bucket->m_head;
-      }
-      else
-      {
-	m_complete_data.m_head = bucket->m_head;
-      }
-      
-      m_complete_data.m_tail = bucket->m_tail;
-      m_complete_data.m_event_count += bucket->m_event_count;
+      m_complete_data.m_data.append(bucket->m_data);
 #ifdef VM_TRACE
-      ndbout_c("moved %lld rows -> %lld", bucket->m_event_count,
-	       m_complete_data.m_event_count);
+      ndbout_c("moved %lld rows -> %lld", bucket->m_data.m_count,
+	       m_complete_data.m_data.m_count);
 #else
       ndbout_c("");
 #endif
@@ -1003,12 +971,9 @@
   for(Uint32 i = 0; i<sz; i++)
   {
     Gci_container* tmp = array + i;
-    if(tmp->m_head)
+    if(!tmp->m_data.is_empty())
     {
-      tmp->m_tail->m_next = m_free_data;
-      m_free_data= tmp->m_head;
-
-      m_free_data_count += tmp->m_event_count;
+      free_list(tmp->m_data);
 #if 0
       m_free_data_count++;
       EventBufData* loop= tmp->m_head;
@@ -1095,17 +1060,13 @@
 
     if (unlikely(data == 0))
     {
-#if 0 // not decided where this should go
-      {
-	Uint32 data[4];
-	data[0]= NDB_LE_EventBufferUsage;
-	data[1]= 0;
-	data[2]= m_total_alloc;
-	data[3]= 0;
-	m_ndb->theImpl->send_event_report(data,4);
-      }
+#ifdef VM_TRACE
+      assert(m_free_data_count == 0);
+      assert(m_free_data_sz == 0);
 #endif
       expand(4000);
+      reportUsage();
+
       data= m_free_data;
       if (unlikely(data == 0))
       {
@@ -1114,10 +1075,10 @@
 	       m_latestGCI, m_waitGCI);
 	printf("m_free_data_count %d\n", m_free_data_count);
 	printf("m_available_data_count %d first gci %d last gci %d\n",
-	       m_available_data_count,
-	       m_available_data ? m_available_data->sdata->gci : 0,
-	       m_available_data ? m_last_available_data->sdata->gci : 0);
-	printf("m_used_data_count %d\n", m_used_data_count);
+	       m_available_data.m_count,
+	       m_available_data.m_head ? m_available_data.m_head->sdata->gci : 0,
+	       m_available_data.m_tail ? m_available_data.m_tail->sdata->gci : 0);
+	printf("m_used_data_count %d\n", m_used_data.m_count);
 	op->m_has_error= 2;
 	DBUG_RETURN(-1); // TODO handle this, overrun, or, skip?
       }
@@ -1126,28 +1087,17 @@
     // remove data from free list
     m_free_data= data->m_next;
     m_free_data_count--;
-    // add it to received data
-    data->m_next= 0;
+    m_free_data_sz-= data->sz;
 
-    if(likely(bucket->m_tail != 0))
-    {
-      bucket->m_tail->m_next = data;
-      bucket->m_tail = data;
-    }
-    else
-    {
-#ifdef VM_TRACE
-      assert(bucket->m_event_count == 0);
-#endif
-      bucket->m_head = bucket->m_tail = data;
-    }
-    bucket->m_event_count++;
-
-    if (copy_data_alloc(sdata, ptr, data))
+    if (unlikely(copy_data_alloc(sdata, ptr, data)))
     {
       op->m_has_error= 3;
       DBUG_RETURN(-1);
     }
+
+    // add it to received data
+    bucket->m_data.append(data);
+
     data->m_event_op= op;
 #ifdef VM_TRACE
     op->m_data_count++;
@@ -1177,7 +1127,7 @@
 
   // ToDo should start at bufferid dependant pos for optimization
   EventBufData *prev= 0;
-  for(EventBufData *data= m_available_data;
+  for(EventBufData *data= m_available_data.m_head;
       data && (data->sdata->gci <= maxGCI);
       data= data->m_next)
   {
@@ -1189,26 +1139,10 @@
       data->m_event_op->m_data_item= data;
 
       // remove item from m_available_data
-      if (likely(prev == 0))
-      {
-	m_available_data= data->m_next;
-	if (m_available_data == 0)
-	  m_last_available_data= 0;
-      }
-      else
-      {
-	prev->m_next= data->m_next;
-	if (prev->m_next == 0) // the last was removed
-	  m_last_available_data= prev;
-      }
-      m_available_data_count--;
+      m_available_data.remove(data,prev);
 
       // add it to used list
-      if (m_last_used_data == 0)
-	m_last_used_data= data;
-      data->m_next= m_used_data;
-      m_used_data= data;
-      m_used_data_count++;
+      m_used_data.append(data);
 
       if (!data->sdata->isGCIConsistent())
 	op->m_has_error= 5;
@@ -1217,7 +1151,7 @@
       op->m_data_done_count++;
 #endif
 
-      r= m_available_data_count+1;
+      r= m_available_data.m_count+1;
       break;
     }
     // keep old data item to be able to "link out"
@@ -1250,6 +1184,10 @@
     if (sdata)
     {
       NdbMem_Free((char*)sdata);
+#ifdef VM_TRACE
+      assert(m_free_data_sz >= ev_buf->sz);
+      assert(m_total_alloc >= ev_buf->sz);
+#endif
       m_total_alloc-= ev_buf->sz;
     }
     ptr= (Uint32*)NdbMem_Allocate(alloc_size);
@@ -1306,45 +1244,42 @@
 NdbEventBuffer::move_data()
 {
   // handle received data
-  if (m_complete_data.m_head)
+  if (!m_complete_data.m_data.is_empty())
   {
     // move this list to last in m_available_data
-    if (m_last_available_data)
-    {
-      m_last_available_data->m_next= m_complete_data.m_head;
-    }
-    else
-    {
-      m_available_data= m_complete_data.m_head;
-    }
-    m_last_available_data= m_complete_data.m_tail;
-    m_available_data_count+= m_complete_data.m_event_count;
+    m_available_data.append(m_complete_data.m_data);
 
     bzero(&m_complete_data, sizeof(m_complete_data));
   }
 
   // handle used data
-  if (m_used_data)
+  if (!m_used_data.is_empty())
   {
     // return m_used_data to m_free_data
-    m_last_used_data->m_next= m_free_data;
-    m_free_data= m_used_data;
-    m_free_data_count+= m_used_data_count;
-
-    // all m_used_data returned to m_free_data
-    m_used_data= 0;
-    m_last_used_data= 0;
-    m_used_data_count= 0;
+    free_list(m_used_data);
   }
-  if (m_available_data)
+  if (!m_available_data.is_empty())
   {
     DBUG_ENTER("NdbEventBuffer::move_data");
-    DBUG_PRINT("exit",("m_available_data_count %u", m_available_data_count));
-    DBUG_RETURN(m_available_data->m_event_op);
+    DBUG_PRINT("exit",("m_available_data_count %u", m_available_data.m_count));
+    DBUG_RETURN(m_available_data.m_head->m_event_op);
   }
   return 0;
 }
 
+void
+NdbEventBuffer::free_list(EventBufData_list &list)
+{
+  // return list to m_free_data
+  list.m_tail->m_next= m_free_data;
+  m_free_data= m_used_data.m_head;
+  m_free_data_count+= list.m_count;
+  m_free_data_sz+= list.m_sz;
+
+  // list returned to m_free_data
+  new (&list) EventBufData_list;
+}
+
 NdbEventOperation*
 NdbEventBuffer::createEventOperation(const char* eventName,
 				     NdbError &theError)
@@ -1382,6 +1317,23 @@
   // are _no_ more events coming
 
   //  delete tOp;
+}
+
+void
+NdbEventBuffer::reportUsage()
+{
+  if ((m_free_data_sz*100)/m_total_alloc < 10)
+  {
+    Uint32 data[4];
+    data[0]= NDB_LE_EventBufferUsage;
+    data[1]= m_total_alloc-m_free_data_sz;
+    data[2]= m_total_alloc;
+    data[3]= 0;
+    m_ndb->theImpl->send_event_report(data,4);
+#ifdef VM_TRACE
+    assert(m_total_alloc >= m_free_data_sz);
+#endif
+  }
 }
 
 template class Vector<Gci_container>;

--- 1.28/storage/ndb/src/ndbapi/NdbEventOperationImpl.hpp	2005-05-27 14:01:34 +02:00
+++ 1.29/storage/ndb/src/ndbapi/NdbEventOperationImpl.hpp	2005-05-30 17:11:35 +02:00
@@ -37,6 +37,92 @@
   EventBufData *m_next; // Next wrt to global order
 };
 
+class EventBufData_list
+{
+public:
+  EventBufData_list();
+  ~EventBufData_list();
+
+  void remove(EventBufData *data, EventBufData *prev);
+  void append(EventBufData *data);
+  void append(const EventBufData_list &list);
+
+  int is_empty();
+
+  EventBufData *m_head, *m_tail;
+  unsigned m_count;
+  unsigned m_sz;
+};
+
+inline
+EventBufData_list::EventBufData_list()
+  : m_head(0), m_tail(0), m_count(0), m_sz(0)
+{
+}
+
+inline
+EventBufData_list::~EventBufData_list()
+{
+}
+
+
+inline
+int EventBufData_list::is_empty()
+{
+  return m_head == 0;
+}
+
+inline
+void EventBufData_list::remove(EventBufData *data, EventBufData *prev)
+{
+  if (likely(prev == 0))
+  {
+    m_head= data->m_next;
+    if (m_head == 0)
+      m_tail= 0;
+  }
+  else
+  {
+    prev->m_next= data->m_next;
+    if (prev->m_next == 0) // the last was removed
+      m_tail= prev;
+  }
+  m_count--;
+  m_sz-= data->sz;
+}
+
+inline
+void EventBufData_list::append(EventBufData *data)
+{
+  data->m_next= 0;
+  if (m_tail)
+    m_tail->m_next= data;
+  else
+  {
+#ifdef VM_TRACE
+    assert(m_count == 0);
+    assert(m_sz == 0);
+#endif
+    m_head= data;
+  }
+  m_tail= data;
+
+  m_count++;
+  m_sz+= data->sz;
+}
+
+inline
+void EventBufData_list::append(const EventBufData_list &list)
+{
+  if (m_tail)
+    m_tail->m_next= list.m_head;
+  else
+    m_head= list.m_head;
+  m_tail= list.m_tail;
+  m_count+= list.m_count;
+  m_sz+= list.m_sz;
+}
+
 struct Gci_container
 {
   enum State 
@@ -47,8 +133,7 @@
   Uint32 m_state;
   Uint32 m_gcp_complete_rep_count; // Remaining SUB_GCP_COMPLETE_REP until done
   Uint64 m_gci;                    // GCI
-  EventBufData *m_head, *m_tail;
-  Uint64 m_event_count;
+  EventBufData_list m_data;
 };
 
 class NdbEventOperationImpl : public NdbEventOperation {
@@ -160,6 +245,10 @@
 		      LinearSectionPtr f_ptr[3],
 		      EventBufData *ev_buf);
 
+  void free_list(EventBufData_list &list);
+
+  void reportUsage();
+
   // Global Mutex used for some things
   static NdbMutex *p_add_drop_mutex;
 
@@ -177,15 +266,11 @@
   Gci_container m_complete_data;
   EventBufData *m_free_data;
   unsigned m_free_data_count;
+  unsigned m_free_data_sz;
 
   // user thread
-  EventBufData *m_available_data;
-  EventBufData *m_last_available_data;
-  unsigned m_available_data_count;
-
-  EventBufData *m_used_data;
-  EventBufData *m_last_used_data;
-  unsigned m_used_data_count;
+  EventBufData_list m_available_data;
+  EventBufData_list m_used_data;
 
   unsigned m_total_alloc; // total allocated memory
   unsigned m_total_used;  // used allocated memory

--- 1.259/sql/ha_ndbcluster.cc	2005-05-27 16:40:15 +02:00
+++ 1.260/sql/ha_ndbcluster.cc	2005-05-30 17:11:35 +02:00
@@ -9436,9 +9436,13 @@
 	ndb_latest_handled_binlog_epoch= gci;
 #ifdef RUN_NDB_BINLOG_TIMER
 	gci_timer.stop();
-	sql_print_information("gci %ld event_count %d write time %ld, total time %ld",
-			      (ulong)gci, event_count, write_timer.elapsed_ms(),
-			      gci_timer.elapsed_ms());
+	sql_print_information("gci %ld event_count %d write time %ld(%d e/s), "
+			      "total time %ld(%d e/s)",
+			      (ulong)gci, event_count,
+			      write_timer.elapsed_ms(),
+			      event_count/write_timer.elapsed_ms(),
+			      gci_timer.elapsed_ms(),
+			      event_count/gci_timer.elapsed_ms());
 #endif
       } // while
     } // if