bzr push into mysql-5.1-telco-6.4 branch (jonas:3254 to 3256) Bug#42215

[Jonas Oreland]

 3256 Jonas Oreland	2009-02-11
      ndb - bug#42215 - fix memory reporting for mt-lqh (tup memory)
modified:
  storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp
  storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.hpp
  storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp
  storage/ndb/src/kernel/blocks/dbtup/DbtupDebug.cpp
  storage/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp
  storage/ndb/src/kernel/blocks/dbtup/DbtupPagMan.cpp

 3255 Jonas Oreland	2009-02-11
      ndb - bug#42765 - divide IndexMemory by no of LQH instances
modified:
  storage/ndb/src/kernel/main.cpp
  storage/ndb/src/kernel/vm/Configuration.cpp
  storage/ndb/src/kernel/vm/Configuration.hpp

 3254 Jonas Oreland	2009-02-11
      ndb - add static ndbd/ndbmtd config for mtr
modified:
  mysql-test/mysql-test-run.pl

=== modified file 'storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp'
--- a/storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp	2008-11-08 21:43:03 +0000
+++ b/storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp	2009-02-11 15:07:28 +0000
@@ -43,6 +43,8 @@
 #include <NdbSleep.h>
 #include <SafeCounter.hpp>
 
+#define ZREPORT_MEMORY_USAGE 1000
+
 // Used here only to print event reports on stdout/console.
 extern EventLogger * g_eventLogger;
 extern int simulate_error_during_shutdown;
@@ -99,6 +101,8 @@ Cmvmi::Cmvmi(Block_context& ctx) :
 
   addRecSignal(GSN_TESTSIG, &Cmvmi::execTESTSIG);
   addRecSignal(GSN_NODE_START_REP, &Cmvmi::execNODE_START_REP, true);
+
+  addRecSignal(GSN_CONTINUEB, &Cmvmi::execCONTINUEB);
   
   subscriberPool.setSize(5);
   
@@ -136,6 +140,8 @@ Cmvmi::Cmvmi(Block_context& ctx) :
   setNodeInfo(getOwnNodeId()).m_connected = true;
   setNodeInfo(getOwnNodeId()).m_version = ndbGetOwnVersion();
   setNodeInfo(getOwnNodeId()).m_mysql_version = NDB_MYSQL_VERSION_D;
+
+  c_memusage_report_frequency = 0;
 }
 
 Cmvmi::~Cmvmi()
@@ -377,12 +383,21 @@ Cmvmi::execREAD_CONFIG_REQ(Signal* signa
     void* ptr = m_ctx.m_mm.get_memroot();
     m_shared_page_pool.set((GlobalPage*)ptr, ~0);
   }
-  
+
   ReadConfigConf * conf = (ReadConfigConf*)signal->getDataPtrSend();
   conf->senderRef = reference();
   conf->senderData = senderData;
   sendSignal(ref, GSN_READ_CONFIG_CONF, signal, 
 	     ReadConfigConf::SignalLength, JBB);
+
+  c_memusage_report_frequency = 0;
+  ndb_mgm_get_int_parameter(p, CFG_DB_MEMREPORT_FREQUENCY, 
+                            &c_memusage_report_frequency);
+  
+  signal->theData[0] = ZREPORT_MEMORY_USAGE;
+  signal->theData[1] = 0;
+  signal->theData[2] = 3;
+  execCONTINUEB(signal);
 }
 
 void Cmvmi::execSTTOR(Signal* signal)
@@ -1046,6 +1061,8 @@ Cmvmi::execDUMP_STATE_ORD(Signal* signal
       signal->theData[1] = 0;
       signal->theData[2] = ~0;
       sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 3, JBB);
+
+      reportDMUsage(signal, 0);
       return;
     }
     Uint32 id = signal->theData[1];
@@ -1616,3 +1633,70 @@ Cmvmi::sendFragmentedComplete(Signal* si
     }
   }
 }
+
+void
+Cmvmi::execCONTINUEB(Signal* signal)
+{
+  switch(signal->theData[0]){
+  case ZREPORT_MEMORY_USAGE:
+  {
+    jam();
+    Uint32 cnt = signal->theData[1];
+    Uint32 usedLast = signal->theData[2];
+
+    Resource_limit rl;
+    m_ctx.m_mm.get_resource_limit(RG_DATAMEM, rl);
+
+    Uint32 tmp = rl.m_min;
+    Uint32 now = tmp ? (rl.m_curr * 100)/tmp : 0;
+    static const Uint32 thresholds[] = { 100, 90, 80, 0 };
+    
+    Uint32 i = 0;
+    const Uint32 sz = sizeof(thresholds)/sizeof(thresholds[0]);
+    for(i = 0; i<sz; i++)
+    {
+      if(now >= thresholds[i])
+      {
+	now = thresholds[i];
+	break;
+      }
+    }
+    
+    if(now != usedLast || 
+       (c_memusage_report_frequency && cnt + 1 == c_memusage_report_frequency))
+    {
+      jam();
+      reportDMUsage(signal, 
+                    now > usedLast ? 1 : 
+                    now < usedLast ? -1 : 0);
+      cnt = 0;
+      usedLast = 0;
+    } 
+    else
+    {
+      jam();
+      cnt++;
+    }
+    signal->theData[0] = ZREPORT_MEMORY_USAGE;
+    signal->theData[1] = cnt; // seconds since last report
+    signal->theData[2] = usedLast; // last threshold
+    sendSignalWithDelay(reference(), GSN_CONTINUEB, signal, 1000, 3);
+    return;
+  }
+  }
+}
+
+void
+Cmvmi::reportDMUsage(Signal* signal, int incDec)
+{
+  Resource_limit rl;
+  m_ctx.m_mm.get_resource_limit(RG_DATAMEM, rl);
+  
+  signal->theData[0] = NDB_LE_MemoryUsage;
+  signal->theData[1] = incDec;
+  signal->theData[2] = sizeof(GlobalPage);
+  signal->theData[3] = rl.m_curr;
+  signal->theData[4] = rl.m_min;
+  signal->theData[5] = DBTUP;
+  sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 6, JBB);
+}

=== modified file 'storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.hpp'
--- a/storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.hpp	2008-05-23 09:26:56 +0000
+++ b/storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.hpp	2009-02-11 15:07:28 +0000
@@ -66,6 +66,8 @@ private:
 
   void execTESTSIG(Signal* signal);
   void execNODE_START_REP(Signal* signal);
+
+  void execCONTINUEB(Signal* signal);
   
   char theErrorMessage[256];
   void sendSTTORRY(Signal* signal);
@@ -112,6 +114,9 @@ private:
   void operator = (const Cmvmi &);
 
   void sendFragmentedComplete(Signal* signal, Uint32 data, Uint32 returnCode);
+
+  Uint32 c_memusage_report_frequency;
+  void reportDMUsage(Signal* signal, int incDec);
 };
 
 #endif

=== modified file 'storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2008-11-16 15:32:38 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp	2009-02-11 15:07:28 +0000
@@ -3001,8 +3001,6 @@ private:
   ArrayPool<Operationrec> c_operation_pool;
 
   ArrayPool<Page> c_page_pool;
-  Uint32 cnoOfAllocatedPages;
-  Uint32 c_no_of_pages;
 
   /* read ahead in pages during disk order scan */
   Uint32 m_max_page_read_ahead;
@@ -3045,7 +3043,6 @@ private:
 
   // Trigger variables
   Uint32 c_maxTriggersPerTable;
-  Uint32 c_memusage_report_frequency;
 
   Uint32 c_errorInsert4000TableId;
   Uint32 c_min_list_size[MAX_FREE_LIST + 1];

=== modified file 'storage/ndb/src/kernel/blocks/dbtup/DbtupDebug.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/DbtupDebug.cpp	2008-10-08 06:23:23 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupDebug.cpp	2009-02-11 15:07:28 +0000
@@ -69,50 +69,19 @@ struct Chunk {
   Uint32 pageCount;
 };
 
-void
-Dbtup::reportMemoryUsage(Signal* signal, int incDec){
-  signal->theData[0] = NDB_LE_MemoryUsage;
-  signal->theData[1] = incDec;
-  signal->theData[2] = sizeof(Page);
-  signal->theData[3] = cnoOfAllocatedPages;
-  signal->theData[4] = c_no_of_pages;
-  signal->theData[5] = DBTUP;
-  sendSignal(CMVMI_REF, GSN_EVENT_REP, signal, 6, JBB);
-}
-
 void Dbtup::execDBINFO_SCANREQ(Signal* signal)
 {
   jamEntry();
   DbinfoScanReq req= *(DbinfoScanReq*)signal->theData;
   const Uint32 reqlength= signal->getLength();
 
-  const Uint32 tableId= req.tableId;
-
-  const Uint32 senderRef= req.senderRef;
-  const Uint32 apiTxnId= req.apiTxnId;
-  const Uint32 colBitmapLo= req.colBitmapLo;
-  const Uint32 colBitmapHi= req.colBitmapHi;
-
   char buf[1024];
   struct dbinfo_row r;
   struct dbinfo_ratelimit rl;
 
   dbinfo_ratelimit_init(&rl, &req);
 
-  if(tableId == NDBINFO_MEMUSAGE_TABLEID)
-  {
-    jam();
-    dbinfo_write_row_init(&r, buf, sizeof(buf));
-    dbinfo_write_row_column(&r, "DataMemory", 10);
-    dbinfo_write_row_column_uint32(&r, getOwnNodeId());
-    Uint32 page_size_kb= sizeof(Page);;
-    dbinfo_write_row_column(&r, (char*)&page_size_kb, 4); // 8kb
-    dbinfo_write_row_column_uint32(&r, cnoOfAllocatedPages); // alloced pages
-    dbinfo_write_row_column_uint32(&r, c_no_of_pages); // number of pages
-    dbinfo_write_row_column(&r, "DBTUP", 5);
-    dbinfo_send_row(signal,r,rl,apiTxnId,senderRef);
-  }
-  else if(req.tableId == NDBINFO_POOLS_TABLEID)
+  if(req.tableId == NDBINFO_POOLS_TABLEID)
   {
     struct {
       const char* poolname;
@@ -173,10 +142,6 @@ void
 Dbtup::execDUMP_STATE_ORD(Signal* signal)
 {
   Uint32 type = signal->theData[0];
-  if(type == DumpStateOrd::DumpPageMemory && signal->getLength() == 1){
-    reportMemoryUsage(signal, 0);
-    return;
-  }
   DumpStateOrd * const dumpState = (DumpStateOrd *)&signal->theData[0];
 
 #if 0
@@ -268,7 +233,9 @@ Dbtup::execDUMP_STATE_ORD(Signal* signal
 
       // Case
       Uint32 c = (rand() % 3);
-      const Uint32 free = c_no_of_pages - cnoOfAllocatedPages;
+      Resource_limit rl;
+      m_ctx.m_mm.get_resource_limit(RG_DATAMEM, rl);
+      const Uint32 free = rl.m_max - rl.m_curr;
       
       Uint32 alloc = 0;
       if(free <= 1){

=== modified file 'storage/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp	2008-11-16 15:29:16 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupGen.cpp	2009-02-11 15:07:28 +0000
@@ -121,8 +121,6 @@ Dbtup::Dbtup(Block_context& ctx, Uint32 
   tablerec = 0;
   tableDescriptor = 0;
   totNoOfPagesAllocated = 0;
-  cnoOfAllocatedPages = 0;
-  c_no_of_pages = 0;
   
   initData();
   CLEAR_ERROR_INSERT_VALUE;
@@ -243,41 +241,6 @@ void Dbtup::execCONTINUEB(Signal* signal
     jam();
     releaseFragment(signal, dataPtr, signal->theData[2]);
     break;
-  case ZREPORT_MEMORY_USAGE:{
-    jam();
-    static int c_currentMemUsed = 0;
-    Uint32 cnt = signal->theData[1];
-    Uint32 tmp = c_no_of_pages;
-    int now = tmp ? (cnoOfAllocatedPages * 100)/tmp : 0;
-    const int thresholds[] = { 100, 90, 80, 0 };
-    
-    Uint32 i = 0;
-    const Uint32 sz = sizeof(thresholds)/sizeof(thresholds[0]);
-    for(i = 0; i<sz; i++){
-      if(now >= thresholds[i]){
-	now = thresholds[i];
-	break;
-      }
-    }
-
-    if(now != c_currentMemUsed || 
-       (c_memusage_report_frequency && cnt + 1 == c_memusage_report_frequency))
-    {
-      reportMemoryUsage(signal, 
-			now > c_currentMemUsed ? 1 : 
-			now < c_currentMemUsed ? -1 : 0);
-      cnt = 0;
-      c_currentMemUsed = now;
-    } 
-    else
-    {
-      cnt++;
-    }
-    signal->theData[0] = ZREPORT_MEMORY_USAGE;
-    signal->theData[1] = cnt;
-    sendSignalWithDelay(reference(), GSN_CONTINUEB, signal, 1000, 2);    
-    return;
-  }
   case ZBUILD_INDEX:
     jam();
     buildIndex(signal, dataPtr);
@@ -476,10 +439,6 @@ void Dbtup::execREAD_CONFIG_REQ(Signal* 
   clastBitMask = 1;
   clastBitMask = clastBitMask << 31;
 
-  c_memusage_report_frequency = 0;
-  ndb_mgm_get_int_parameter(p, CFG_DB_MEMREPORT_FREQUENCY, 
-			    &c_memusage_report_frequency);
-  
   initialiseRecordsLab(signal, 0, ref, senderData);
 }//Dbtup::execSIZEALT_REP()
 
@@ -492,12 +451,9 @@ void Dbtup::initRecords() 
     m_ctx.m_config.getOwnConfigIterator();
   ndbrequire(p != 0);
 
-  ndbrequire(!ndb_mgm_get_int_parameter(p, CFG_TUP_PAGE, &tmp));
-
   // Records with dynamic sizes
   void* ptr = m_ctx.m_mm.get_memroot();
   c_page_pool.set((Page*)ptr, (Uint32)~0);
-  c_no_of_pages = tmp;
 
   fragoperrec = (Fragoperrec*)allocRecord("Fragoperrec",
 					  sizeof(Fragoperrec),
@@ -635,14 +591,6 @@ void Dbtup::execNDB_STTOR(Signal* signal
     break;
   case ZSTARTPHASE6:
     jam();
-/*****************************************/
-/*       NOW SET THE DISK WRITE SPEED TO */
-/*       PAGES PER TICK AFTER SYSTEM     */
-/*       RESTART.                        */
-/*****************************************/
-    signal->theData[0] = ZREPORT_MEMORY_USAGE;
-    signal->theData[1] = 0;
-    sendSignalWithDelay(reference(), GSN_CONTINUEB, signal, 1000, 2);
     break;
   default:
     jam();

=== modified file 'storage/ndb/src/kernel/blocks/dbtup/DbtupPagMan.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtup/DbtupPagMan.cpp	2007-06-12 07:19:39 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtup/DbtupPagMan.cpp	2009-02-11 15:07:28 +0000
@@ -114,7 +114,6 @@ Uint32 Dbtup::nextHigherTwoLog(Uint32 in
 
 void Dbtup::initializePage() 
 {
-  cnoOfAllocatedPages = 0;
 }//Dbtup::initializePage()
 
 void Dbtup::allocConsPages(Uint32 noOfPagesToAllocate,
@@ -129,7 +128,6 @@ void Dbtup::allocConsPages(Uint32 noOfPa
 
   m_ctx.m_mm.alloc_pages(RT_DBTUP_PAGE, &allocPageRef, 
 			 &noOfPagesToAllocate, 1);
-  cnoOfAllocatedPages += noOfPagesToAllocate;
   noOfPagesAllocated = noOfPagesToAllocate;
   return;
 }//allocConsPages()
@@ -137,6 +135,5 @@ void Dbtup::allocConsPages(Uint32 noOfPa
 void Dbtup::returnCommonArea(Uint32 retPageRef, Uint32 retNo) 
 {
   m_ctx.m_mm.release_pages(RT_DBTUP_PAGE, retPageRef, retNo);
-  cnoOfAllocatedPages -= retNo;
 }//Dbtup::returnCommonArea()
 

=== modified file 'storage/ndb/src/kernel/main.cpp'
--- a/storage/ndb/src/kernel/main.cpp	2009-01-30 13:42:56 +0000
+++ b/storage/ndb/src/kernel/main.cpp	2009-02-11 14:26:06 +0000
@@ -334,22 +334,24 @@ get_multithreaded_config(EmulatorData& e
     return 0;
   }
 
-  const ndb_mgm_configuration_iterator * p =
-    ed.theConfiguration->getOwnConfigIterator();
-  if (p == 0)
+  ndb_mgm_configuration * conf = ed.theConfiguration->getClusterConfig();
+  if (conf == 0)
   {
     abort();
   }
-
+  
+  ndb_mgm_configuration_iterator * p = ndb_mgm_create_configuration_iterator(conf,
CFG_SECTION_NODE);
+  if (ndb_mgm_find(p, CFG_NODE_ID, globalData.ownId))
+  {
+    abort();
+  }
+  
   Uint32 mtthreads = 0;
   ndb_mgm_get_int_parameter(p, CFG_DB_MT_THREADS, &mtthreads);
   ndbout << "NDBMT: MaxNoOfExecutionThreads=" << mtthreads << endl;
 
   globalData.isNdbMtLqh = true;
 
-  /**
-   * TODO add config for mt-classic
-   */
   {
     Uint32 classic = 0;
     ndb_mgm_get_int_parameter(p, CFG_NDBMT_CLASSIC, &classic);
@@ -594,14 +596,15 @@ int main(int argc, char** argv)
 #else
   g_eventLogger->info("Ndb started");
 #endif
+  
+  if (get_multithreaded_config(globalEmulatorData))
+    return -1;
+  
   theConfig->setupConfiguration();
   systemInfo(* theConfig, * theConfig->m_logLevel); 
   
   NdbThread* pWatchdog = globalEmulatorData.theWatchDog->doStart();
 
-  if (get_multithreaded_config(globalEmulatorData))
-    return -1;
-
   {
     /*
      * Memory allocation can take a long time for large memory.

=== modified file 'storage/ndb/src/kernel/vm/Configuration.cpp'
--- a/storage/ndb/src/kernel/vm/Configuration.cpp	2008-12-18 08:51:37 +0000
+++ b/storage/ndb/src/kernel/vm/Configuration.cpp	2009-02-11 14:26:06 +0000
@@ -736,6 +736,12 @@ Configuration::calcSizeAlt(ConfigValues 
     }
   }
 
+  Uint32 lqhInstances = 1;
+  if (globalData.isNdbMtLqh)
+  {
+    lqhInstances = globalData.ndbMtLqhWorkers;
+  }
+
   Uint64 indexMem = 0, dataMem = 0;
   ndb_mgm_get_int64_parameter(&db, CFG_DB_DATA_MEM, &dataMem);
   ndb_mgm_get_int64_parameter(&db, CFG_DB_INDEX_MEM, &indexMem);
@@ -749,8 +755,11 @@ Configuration::calcSizeAlt(ConfigValues 
     ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, msg, buf);
   }
 
+#define DO_DIV(x,y) (((x) + (y - 1)) / (y))
+
   noOfDataPages = (dataMem / 32768);
   noOfIndexPages = (indexMem / 8192);
+  noOfIndexPages = DO_DIV(noOfIndexPages, lqhInstances);
 
   for(unsigned j = 0; j<LogLevel::LOGLEVEL_CATEGORIES; j++){
     Uint32 tmp;
@@ -861,7 +870,12 @@ Configuration::calcSizeAlt(ConfigValues 
   if (noOfLocalOperations == 0) {
     noOfLocalOperations= (11 * noOfOperations) / 10;
   }
+
   Uint32 noOfTCScanRecords = noOfScanRecords;
+  Uint32 noOfTCLocalScanRecords = noOfLocalScanRecords;
+
+  noOfLocalOperations = DO_DIV(noOfLocalOperations, lqhInstances);
+  noOfLocalScanRecords = DO_DIV(noOfLocalScanRecords, lqhInstances);
 
   {
     Uint32 noOfAccTables= noOfMetaTables/*noOfTables+noOfUniqueHashIndexes*/;
@@ -969,7 +983,7 @@ Configuration::calcSizeAlt(ConfigValues 
 	    noOfMetaTables);
     
     cfg.put(CFG_TC_LOCAL_SCAN, 
-	    noOfLocalScanRecords);
+	    noOfTCLocalScanRecords);
     
     cfg.put(CFG_TC_SCAN, 
 	    noOfTCScanRecords);

=== modified file 'storage/ndb/src/kernel/vm/Configuration.hpp'
--- a/storage/ndb/src/kernel/vm/Configuration.hpp	2008-11-07 13:23:15 +0000
+++ b/storage/ndb/src/kernel/vm/Configuration.hpp	2009-02-11 14:26:06 +0000
@@ -132,6 +132,8 @@ public:
   class LogLevel * m_logLevel;
   ndb_mgm_configuration_iterator * getClusterConfigIterator() const;
 
+  ndb_mgm_configuration* getClusterConfig() const { return m_clusterConfig; }
+
 private:
   friend class Cmvmi;
   friend class Qmgr;