List:Commits« Previous MessageNext Message »
From:Frazer Clement Date:April 24 2012 2:49pm
Subject:bzr push into mysql-5.1-telco-6.3 branch (frazer.clement:3471 to 3472)
Bug#13966723
View as plain text  
 3472 Frazer Clement	2012-04-24
      Bug#13966723 MYSQL CLUSTER - CONCURRENT OPERATIONS 'LEAK' RESULTS IN "TOO MANY ACTIVE SCANS"
      
      TC had a bug where a shortage of TC scan fragment records resulted in
      a leak of scan table records and key operations.
      
      This is fixed, and a testcase added.

    modified:
      storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp
      storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp
      storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp
      storage/ndb/test/ndbapi/testScan.cpp
      storage/ndb/test/run-test/daily-basic-tests.txt
 3471 memontgo	2012-03-21
      bug#13364905 / bug#62635 - Add support for --hostname=host:port format to ndb_size.pl 

    modified:
      storage/ndb/tools/ndb_size.pl
=== modified file 'storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp'
--- a/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp	2011-06-30 15:55:35 +0000
+++ b/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp	2012-04-24 10:55:03 +0000
@@ -171,7 +171,10 @@ public:
     DumpBackupSetCompressedLCP = 13002,
 
     SchemaResourceSnapshot = 4000, // Save resource consumption
-    SchemaResourceCheckLeak = 4001 // check same as snapshot
+    SchemaResourceCheckLeak = 4001, // check same as snapshot
+
+    TcResourceSnapshot = 2553,
+    TcResourceCheckLeak = 2554
   };
 public:
   

=== modified file 'storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp'
--- a/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp	2011-06-30 15:55:35 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp	2012-04-24 10:55:03 +0000
@@ -357,6 +357,7 @@ public:
    * Pool of trigger data record
    */
   ArrayPool<TcDefinedTriggerData> c_theDefinedTriggerPool;
+  RSS_AP_SNAPSHOT(c_theDefinedTriggerPool);
 
   /**
    * The list of active triggers
@@ -455,6 +456,7 @@ public:
   ArrayPool<TcFiredTriggerData> c_theFiredTriggerPool;
   DLHashTable<TcFiredTriggerData> c_firedTriggerHash;
   AttributeBuffer::DataBufferPool c_theTriggerAttrInfoPool;
+  RSS_AP_SNAPSHOT(c_theFiredTriggerPool);
 
   Uint32 c_maxNumberOfDefinedTriggers;
   Uint32 c_maxNumberOfFiredTriggers;
@@ -534,6 +536,7 @@ public:
    * Pool of index data record
    */
   ArrayPool<TcIndexData> c_theIndexPool;
+  RSS_AP_SNAPSHOT(c_theIndexPool);
   
   /**
    * The list of defined indexes
@@ -590,6 +593,7 @@ public:
    * Pool of index data record
    */
   ArrayPool<TcIndexOperation> c_theIndexOperationPool;
+  RSS_AP_SNAPSHOT(c_theIndexOperationPool);
 
   UintR c_maxNumberOfIndexOperations;   
 
@@ -1734,7 +1738,8 @@ private:
       return 11;
     }
   } c_counters;
-  
+  RSS_OP_SNAPSHOT(cconcurrentOp);
+
   Uint16 cownNodeid;
   Uint16 terrorCode;
 
@@ -1747,6 +1752,8 @@ private:
   UintR clastgcp;
   UintR cfirstfreeGcp;
   UintR cfirstfreeScanrec;
+  UintR cConcScanCount;
+  RSS_OP_SNAPSHOT(cConcScanCount);
 
   TableRecordPtr tabptr;
   UintR cfirstfreeApiConnectFail;
@@ -1762,6 +1769,7 @@ private:
   UintR cscanrecFileSize;
 
   UnsafeArrayPool<ScanFragRec> c_scan_frag_pool;
+  RSS_AP_SNAPSHOT(c_scan_frag_pool);
   ScanFragRecPtr scanFragptr;
 
   UintR cscanFragrecFileSize;
@@ -1864,6 +1872,7 @@ private:
   
   ArrayPool<CommitAckMarker>   m_commitAckMarkerPool;
   DLHashTable<CommitAckMarker> m_commitAckMarkerHash;
+  RSS_AP_SNAPSHOT(m_commitAckMarkerPool);
   
   void execTC_COMMIT_ACK(Signal* signal);
   void sendRemoveMarkers(Signal*, const CommitAckMarker *);

=== modified file 'storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp	2011-05-31 11:37:45 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp	2012-04-24 10:55:03 +0000
@@ -9444,6 +9444,8 @@ void Dbtc::execSCAN_TABREQ(Signal* signa
   if (unlikely(errCode))
   {
     jam();
+    transP->apiScanRec = scanptr.i;
+    releaseScanResources(scanptr, true /* NotStarted */);
     goto SCAN_TAB_error;
   }
 
@@ -9508,6 +9510,7 @@ void Dbtc::execSCAN_TABREQ(Signal* signa
     goto SCAN_TAB_error;
   }//if
   ndbrequire(cfirstfreeScanrec == RNIL);
+  ndbrequire(cConcScanCount == cscanrecFileSize);
   jam();
   errCode = ZNO_SCANREC_ERROR;
   goto SCAN_TAB_error;
@@ -9574,7 +9577,8 @@ Dbtc::initScanrec(ScanRecordPtr scanptr,
   for (Uint32 i = 0; i < scanParallel; i++) {
     jam();
     ScanFragRecPtr ptr;
-    if (unlikely(list.seize(ptr) == false))
+    if (unlikely((list.seize(ptr) == false) ||
+                 ERROR_INSERTED(8093)))
     {
       jam();
       goto errout;
@@ -9887,6 +9891,7 @@ void Dbtc::releaseScanResources(ScanReco
   scanPtr.p->scanTcrec = RNIL;
   scanPtr.p->scanApiRec = RNIL;
   cfirstfreeScanrec = scanPtr.i;
+  cConcScanCount--;
   
   apiConnectptr.p->apiScanRec = RNIL;
   apiConnectptr.p->apiConnectstate = CS_CONNECTED;
@@ -10625,6 +10630,7 @@ Dbtc::seizeScanrec(Signal* signal) {
   ptrCheckGuard(scanptr, cscanrecFileSize, scanRecord);
   cfirstfreeScanrec = scanptr.p->nextScan;
   scanptr.p->nextScan = RNIL;
+  cConcScanCount++;
   ndbrequire(scanptr.p->scanState == ScanRecord::IDLE);
   return scanptr;
 }//Dbtc::seizeScanrec()
@@ -11025,6 +11031,7 @@ void Dbtc::initialiseScanrec(Signal* sig
   ptrAss(scanptr, scanRecord);
   scanptr.p->nextScan = RNIL;
   cfirstfreeScanrec = 0;
+  cConcScanCount = 0;
 }//Dbtc::initialiseScanrec()
 
 void Dbtc::initialiseScanFragrec(Signal* signal) 
@@ -11917,6 +11924,33 @@ Dbtc::execDUMP_STATE_ORD(Signal* signal)
       warningEvent(" DBTC: dump-7019 to unknown node: %u", nodeId);
     }
   }
+
+  if (arg == DumpStateOrd::TcResourceSnapshot)
+  {
+    RSS_OP_SNAPSHOT_SAVE(cConcScanCount);
+    RSS_AP_SNAPSHOT_SAVE(c_scan_frag_pool);
+    RSS_AP_SNAPSHOT_SAVE(c_theDefinedTriggerPool);
+    RSS_AP_SNAPSHOT_SAVE(c_theFiredTriggerPool);
+    RSS_AP_SNAPSHOT_SAVE(c_theIndexPool);
+    RSS_AP_SNAPSHOT_SAVE(m_commitAckMarkerPool);
+    RSS_AP_SNAPSHOT_SAVE(c_theIndexOperationPool);
+#ifdef ERROR_INSERT
+    rss_cconcurrentOp = c_counters.cconcurrentOp;
+#endif;
+  }
+  if (arg == DumpStateOrd::TcResourceCheckLeak)
+  {
+    RSS_OP_SNAPSHOT_CHECK(cConcScanCount);
+    RSS_AP_SNAPSHOT_CHECK(c_scan_frag_pool);
+    RSS_AP_SNAPSHOT_CHECK(c_theDefinedTriggerPool);
+    RSS_AP_SNAPSHOT_CHECK(c_theFiredTriggerPool);
+    RSS_AP_SNAPSHOT_CHECK(c_theIndexPool);
+    RSS_AP_SNAPSHOT_CHECK(m_commitAckMarkerPool);
+    RSS_AP_SNAPSHOT_CHECK(c_theIndexOperationPool);
+#ifdef ERROR_INSERT
+    ndbrequire(rss_cconcurrentOp == c_counters.cconcurrentOp);
+#endif;
+  }
 }//Dbtc::execDUMP_STATE_ORD()
 
 bool

=== modified file 'storage/ndb/test/ndbapi/testScan.cpp'
--- a/storage/ndb/test/ndbapi/testScan.cpp	2011-07-04 12:36:04 +0000
+++ b/storage/ndb/test/ndbapi/testScan.cpp	2012-04-24 10:55:03 +0000
@@ -440,6 +440,47 @@ int runScanReadError(NDBT_Context* ctx,
   return result;
 }
 
+int runScanReadExhaust(NDBT_Context* ctx, NDBT_Step* step)
+{
+  int result = NDBT_OK;
+  int loops = ctx->getNumLoops();
+  int records = ctx->getNumRecords();
+  int parallelism = 240; // Max parallelism
+  int error = 8093;
+  NdbRestarter restarter;
+  
+  /* First take a TC resource snapshot */
+  int savesnapshot= DumpStateOrd::TcResourceSnapshot;
+  int checksnapshot= DumpStateOrd::TcResourceCheckLeak;
+  
+  restarter.dumpStateAllNodes(&savesnapshot, 1);
+  int i = 0;
+  HugoTransactions hugoTrans(*ctx->getTab());
+  hugoTrans.setRetryMax(1);
+  while (i<loops && !ctx->isTestStopped()) {
+    g_info << i << ": ";
+    
+    ndbout << "insertErrorInAllNodes("<<error<<")"<<endl;
+    if (restarter.insertErrorInAllNodes(error) != 0){
+      ndbout << "Could not insert error in all nodes "<<endl;
+      return NDBT_FAILED;
+    }
+    
+    if (hugoTrans.scanReadRecords(GETNDB(step), records, 0, parallelism) == 0)
+    {
+      /* Expect error 291 */
+      result = NDBT_FAILED;
+      break;
+    }
+    i++;
+  }
+  
+  restarter.insertErrorInAllNodes(0);
+
+  restarter.dumpStateAllNodes(&checksnapshot, 1);
+  return result;
+}
+
 int
 runInsertError(NDBT_Context* ctx, NDBT_Step* step){
   int error = ctx->getProperty("ErrorCode");
@@ -2024,6 +2065,13 @@ TESTCASE("Bug54945", "")
 {
   INITIALIZER(runBug54945);
 }
+TESTCASE("ScanFragRecExhaust", 
+         "Test behaviour when TC scan frag recs exhausted")
+{
+  INITIALIZER(runLoadTable);
+  INITIALIZER(runScanReadExhaust);
+  FINALIZER(runClearTable);
+}
 NDBT_TESTSUITE_END(testScan);
 
 int main(int argc, const char** argv){

=== modified file 'storage/ndb/test/run-test/daily-basic-tests.txt'
--- a/storage/ndb/test/run-test/daily-basic-tests.txt	2011-10-28 13:49:12 +0000
+++ b/storage/ndb/test/run-test/daily-basic-tests.txt	2012-04-24 10:55:03 +0000
@@ -1533,4 +1533,7 @@ max-time: 300
 cmd: testNodeRestart
 args: -nMasterFailSlowLCP T1
 
+max-time:300
+cmd: testScan
+args: -nScanFragRecExhaust T1
 

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.1-telco-6.3 branch (frazer.clement:3471 to 3472)Bug#13966723Frazer Clement25 Apr