List:Commits« Previous MessageNext Message »
From:Frazer Clement Date:September 12 2012 3:22pm
Subject:bzr push into mysql-5.1-telco-7.0 branch (frazer.clement:4968 to 4971)
View as plain text  
 4971 Frazer Clement	2012-09-12 [merge]
      Merge 6.3->7.0

    modified:
      storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp
      storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp
      storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp
      storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp
      storage/ndb/test/ndbapi/testScan.cpp
      storage/ndb/test/run-test/daily-basic-tests.txt
 4970 Frazer Clement	2012-09-12 [merge]
      Merge 6.3->7.0

 4969 Frazer Clement	2012-09-12 [merge]
      Merge 6.3->7.0

 4968 magnus.blaudd@stripped	2012-09-12 [merge]
      Merge

    modified:
      sql/ha_ndb_index_stat.cc
      sql/ha_ndb_index_stat.h
=== modified file 'storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp'
--- a/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp	2012-08-28 11:22:02 +0000
+++ b/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp	2012-09-12 14:18:33 +0000
@@ -101,6 +101,7 @@ public:
     LqhDumpAllActiveScanRec = 2302,
     LqhDumpLcpState = 2303,
     LqhErrorInsert5042 = 2315,
+    LqhDumpPoolLevels = 2353,
 
     AccDumpOneScanRec = 2400,
     AccDumpAllScanRec = 2401,
@@ -121,6 +122,7 @@ public:
     TcSetApplTransactionTimeout = 2508,
     TcStartDumpIndexOpCount = 2512,
     TcDumpIndexOpCount = 2513,
+    TcDumpPoolLevels = 2555,
     CmvmiDumpConnections = 2600,
     CmvmiDumpLongSignalMemory = 2601,
     CmvmiSetRestartOnErrorInsert = 2602,

=== modified file 'storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp'
--- a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp	2012-08-22 10:49:42 +0000
+++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp	2012-09-12 14:18:33 +0000
@@ -23687,6 +23687,74 @@ Dblqh::execDUMP_STATE_ORD(Signal* signal
     SET_ERROR_INSERT_VALUE2(5050, c_master_node_id);
 #endif
   }
+  
+  if (arg == DumpStateOrd::LqhDumpPoolLevels)
+  {
+    /* Dump some state info for internal buffers */
+    if (signal->getLength() == 1)
+    {
+      signal->theData[1] = 1;
+      signal->theData[2] = 0;
+      signal->theData[3] = 0;
+      sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB);
+      return;
+    }
+    if (signal->getLength() != 4)
+    {
+      ndbout_c("DUMP LqhDumpPoolLevels : Bad signal length : %u", signal->getLength());
+      return;
+    }
+
+    Uint32 resource = signal->theData[1];
+    Uint32 position = signal->theData[2];
+    Uint32 sum = signal->theData[3];
+    /*const Uint32 MAX_ITER = 200; */
+    
+    switch(resource)
+    {
+    case 1:
+    {
+      /* Must get all in one loop, as we're traversing a dynamic list */
+      sum = 0;
+      TcConnectionrecPtr tcp;    
+      tcp.i = cfirstfreeTcConrec;
+      while (tcp.i != RNIL)
+      {
+        sum++;
+        ptrCheckGuard(tcp, ctcConnectrecFileSize, tcConnectionrec);
+        tcp.i = tcp.p->nextTcConnectrec;
+      }
+      infoEvent("LQH : TcConnection (operation) records in use/total %u/%u (%u bytes each)",
+                ctcConnectrecFileSize - sum, ctcConnectrecFileSize, (Uint32) sizeof(TcConnectionrec));
+      resource++;
+      position = 0;
+      sum = 0;
+      break;
+    }
+    case 2:
+    {
+      infoEvent("LQH : ScanRecord (Fragment) pool in use/total %u/%u (%u bytes each)",
+                c_scanRecordPool.getSize()-
+                c_scanRecordPool.getNoOfFree(),
+                c_scanRecordPool.getSize(),
+                (Uint32) sizeof(ScanRecord));
+      resource++;
+      position = 0;
+      sum = 0;
+      break;
+    }
+    default:
+      return;
+    }
+
+    signal->theData[0] = DumpStateOrd::LqhDumpPoolLevels;
+    signal->theData[1] = resource;
+    signal->theData[2] = position;
+    signal->theData[3] = sum;
+    sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB);
+    return; 
+  }
+
 }//Dblqh::execDUMP_STATE_ORD()
 
 

=== modified file 'storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp'
--- a/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp	2012-08-24 12:00:05 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp	2012-09-12 14:18:33 +0000
@@ -1789,6 +1789,7 @@ private:
 
     // Resource usage counter(not monotonic)
     Uint32 cconcurrentOp;
+    Uint32 cconcurrentScans;
 
     MonotonicCounters() :
      cattrinfoCount(0),

=== modified file 'storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp'
--- a/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp	2012-08-24 12:00:05 +0000
+++ b/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp	2012-09-12 14:18:33 +0000
@@ -10794,12 +10794,29 @@ void Dbtc::scanKeyinfoLab(Signal* signal
   ndbassert( signal->getLength() == 
              (KeyInfo::HeaderLength + wordsInSignal) );
 
-  if (unlikely (! appendToSection(regCachePtr->keyInfoSectionI,
-                                  &signal->theData[KeyInfo::HeaderLength],
-                                  wordsInSignal)))
-  {
-    jam();
-    seizeDatabuferrorLab(signal);
+  if (unlikely ((! appendToSection(regCachePtr->keyInfoSectionI,
+                                   &signal->theData[KeyInfo::HeaderLength],
+                                   wordsInSignal)) ||
+                 ERROR_INSERTED(8094)))
+  {
+    jam();
+    Uint32 transid0 = apiConnectptr.p->transid[0];
+    Uint32 transid1 = apiConnectptr.p->transid[1];
+    ndbrequire(apiConnectptr.p->apiScanRec != RNIL);
+    ScanRecordPtr scanPtr;
+    scanPtr.i = apiConnectptr.p->apiScanRec;
+    ptrCheckGuard(scanPtr, cscanrecFileSize, scanRecord);
+    abortScanLab(signal, 
+                 scanPtr,
+                 ZGET_DATAREC_ERROR, 
+                 true /* Not started */);
+    
+    /* Prepare for up coming ATTRINFO/KEYINFO */
+    apiConnectptr.p->apiConnectstate = CS_ABORTING;
+    apiConnectptr.p->abortState = AS_IDLE;
+    apiConnectptr.p->transid[0] = transid0;
+    apiConnectptr.p->transid[1] = transid1;
+
     return;
   }
 
@@ -11223,6 +11240,7 @@ void Dbtc::releaseScanResources(Signal*
   scanPtr.p->scanTcrec = RNIL;
   scanPtr.p->scanApiRec = RNIL;
   cfirstfreeScanrec = scanPtr.i;
+  ndbassert(cConcScanCount > 0);
   cConcScanCount--;
   
   apiConnectptr.p->apiScanRec = RNIL;
@@ -13492,6 +13510,69 @@ Dbtc::execDUMP_STATE_ORD(Signal* signal)
     ndbrequire(rss_cconcurrentOp == c_counters.cconcurrentOp);
 #endif
   }
+
+  if (arg == DumpStateOrd::TcDumpPoolLevels)
+  {
+    if (signal->getLength() == 1)
+    {
+      signal->theData[1] = 1;
+      signal->theData[2] = 0;
+      signal->theData[3] = 0;
+      sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB);
+      return;
+    }
+    if (signal->getLength() != 4)
+    {
+      ndbout_c("DUMP TcDumpPoolLevels : Bad signal length : %u", signal->getLength());
+      return;
+    }
+    
+    Uint32 resource = signal->theData[1];
+    Uint32 position = signal->theData[2];
+    Uint32 sum = signal->theData[3];
+    /* const Uint32 MAX_ITER = 200; */
+
+    switch(resource)
+    {
+    case 1: 
+      infoEvent("TC : Concurrent operations in use/total : %u/%u (%u bytes each)", 
+                c_counters.cconcurrentOp, 
+                ctcConnectFilesize,
+                (Uint32) sizeof(TcConnectRecord));
+      resource++;
+      position = 0;
+      sum = 0;
+      break;
+    case 2:
+      infoEvent("TC : Concurrent scans in use/total : %u/%u (%u bytes each)", 
+                cConcScanCount, 
+                cscanrecFileSize,
+                (Uint32) sizeof(ScanRecord));
+      resource++;
+      position = 0;
+      sum = 0;
+      break;
+    case 3:
+      infoEvent("TC : Scan Frag records in use/total : %u/%u (%u bytes each)",
+                c_scan_frag_pool.getSize() - 
+                c_scan_frag_pool.getNoOfFree(), 
+                c_scan_frag_pool.getSize(),
+                (Uint32) sizeof(ScanFragRec));
+      resource++;
+      position = 0;
+      sum = 0;
+      break;
+    default:
+      return;
+    }
+
+    signal->theData[0] = DumpStateOrd::TcDumpPoolLevels;
+    signal->theData[1] = resource;
+    signal->theData[2] = position;
+    signal->theData[3] = sum;
+    sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB);
+    return;
+  }
 }//Dbtc::execDUMP_STATE_ORD()
 
 void Dbtc::execDBINFO_SCANREQ(Signal *signal)

=== modified file 'storage/ndb/test/ndbapi/testScan.cpp'
--- a/storage/ndb/test/ndbapi/testScan.cpp	2012-08-24 12:00:05 +0000
+++ b/storage/ndb/test/ndbapi/testScan.cpp	2012-09-12 14:18:33 +0000
@@ -1390,6 +1390,220 @@ finalizeBug42559(NDBT_Context* ctx, NDBT
   return NDBT_OK;
 }
 
+int takeResourceSnapshot(NDBT_Context* ctx, NDBT_Step* step)
+{
+  NdbRestarter restarter;
+  
+  int checksnapshot = DumpStateOrd::TcResourceSnapshot;
+  restarter.dumpStateAllNodes(&checksnapshot, 1);
+
+  /* TODO : Check other block's resources? */
+  return NDBT_OK;
+}
+
+int runScanReadIndexWithBounds(NDBT_Context* ctx, NDBT_Step* step){
+  int loops = ctx->getNumLoops();
+  int records = ctx->getNumRecords();
+  int numRanges = ctx->getProperty("NumRanges", 1);
+  int maxRunSecs = ctx->getProperty("MaxRunSecs", 60);
+  int maxRetries = ctx->getProperty("MaxRetries", 1000000);
+  
+  const NdbDictionary::Index * pIdx = 
+    GETNDB(step)->getDictionary()->getIndex(orderedPkIdxName, 
+					    ctx->getTab()->getName());
+
+  int i = 0;
+  HugoCalculator calc(*ctx->getTab());
+  NDBT_ResultRow row (*ctx->getTab());
+  Ndb* ndb = GETNDB(step);
+
+  Uint64 start = NdbTick_CurrentMillisecond();
+  Uint64 end = start + (1000*maxRunSecs);
+  int retries = 0;
+
+  /* Here we run an ordered index scan, with a bound.
+   * There are numRanges sub-scans with the same bound
+   * This is done to use up some KeyInfo, and expose bugs in that area
+   * If we run many of these in parallel we may exhaust the available KeyInfo storage,
+   * which may expose some bugs.
+   */
+  while (pIdx &&
+         i<loops && 
+         !ctx->isTestStopped() &&
+         NdbTick_CurrentMillisecond() < end) {
+    g_info << "Step " << step->getStepNo() 
+           << "Loop : " << i << ": ";
+    
+    /* Use specific-partition variant of startTransaction to ensure a single
+     * TC node is used
+     */
+    NdbTransaction* trans = ndb->startTransaction(ctx->getTab(), Uint32(0));
+    if (trans == NULL)
+    {
+      g_err << "Transaction start failed " << ndb->getNdbError() << endl;
+      return NDBT_FAILED;
+    }
+
+    NdbIndexScanOperation* iso = trans->getNdbIndexScanOperation(pIdx->getName(), 
+                                                                 ctx->getTab()->getName());
+    if (iso == NULL)
+    {
+      g_err << "Error obtaining IndexScanOperation : " << trans->getNdbError() << endl;
+      trans->close();
+      return NDBT_FAILED;
+    }
+
+    if (iso->readTuples(NdbOperation::LM_CommittedRead, 
+                        (NdbScanOperation::SF_OrderBy |
+                         NdbScanOperation::SF_ReadRangeNo |
+                         NdbScanOperation::SF_MultiRange), 
+                        0) != 0)
+    {
+      g_err << "Error calling readTuples : " << iso->getNdbError() << endl;
+      trans->close();
+      return NDBT_FAILED;
+    }
+
+    for (int range = 0; range < numRanges; range++)
+    {
+      /* Now define a bound... */
+      for (Uint32 k=0; k<pIdx->getNoOfColumns(); k++)
+      {
+        const NdbDictionary::Column* idxCol = pIdx->getColumn(k);
+        const char* colName = idxCol->getName();
+        /* Lower bound of <= NULL should return all rows */
+        if (iso->setBound(colName, NdbIndexScanOperation::BoundLE, NULL) != 0)
+        {
+          g_err << "Error setting bound for column %s. " 
+                << iso->getNdbError() << endl;
+          trans->close();
+          return NDBT_FAILED;
+        }
+      }
+      
+      if (iso->end_of_bound(range) != 0)
+      {
+        g_err << "Error closing range " << range << endl;
+        g_err << iso->getNdbError() << endl;
+        return NDBT_FAILED;
+      }
+    }
+    
+    const NdbDictionary::Table& tab= *ctx->getTab();
+
+    /* Now request all columns in result projection */
+    for (int a=0; a<tab.getNoOfColumns(); a++){
+      if((row.attributeStore(a) = 
+	  iso->getValue(tab.getColumn(a)->getName())) == 0) {
+	g_err << "Error defining read value " << trans->getNdbError() << endl;
+        trans->close();
+	return NDBT_FAILED;
+      }
+    }
+          
+    /* Ready to go... */
+    trans->execute(NoCommit, AbortOnError);
+
+    if (trans->getNdbError().code != 0)
+    {
+      if (trans->getNdbError().code == 218)
+      {
+        /* Out of KeyInfo buffers in TC - that's ok, let's try again */
+        trans->close();
+        if (retries++ < maxRetries)
+        {
+          g_err << "Step " << step->getStepNo()
+                << " TC out of Keyinfo buffers (218) - retrying" << endl;
+          continue;
+        }
+      }
+
+      g_err << "Error on execution : " << trans->getNdbError() << endl;
+      trans->close();
+      return NDBT_FAILED;
+    }
+    
+    int eof;
+    int rows = 0;
+    while ((eof = iso->nextResult(true)) == 0)
+    {
+      rows++;
+      if (calc.verifyRowValues(&row) != 0)
+      {
+        g_err << "Verification failed." << endl;
+        trans->close();
+        return NDBT_FAILED;
+      }
+
+#ifdef BUG_14388257_FIXED
+      int rangeNum = (rows -1) / records;
+      if (iso->get_range_no() != rangeNum)
+      {
+        g_err << "Expected row " << rows 
+              << " to be in range " << rangeNum
+              << " but it reports range num " << iso->get_range_no()
+              << " : " << row
+              << endl;
+        return NDBT_FAILED;
+      }
+#endif
+
+      //g_err << row << endl;
+    }
+
+    if (eof != 1)
+    {
+      g_err << "nextResult() returned " << eof << endl;
+      g_err << "Scan error : " << iso->getNdbError() << endl;
+
+      if (iso->getNdbError().status == NdbError::TemporaryError)
+      {
+        if (retries++ < maxRetries)
+        {
+          g_err << "Step "
+                << step->getStepNo()
+                << "  Temporary, retrying on iteration " 
+                << i << " rows so far : " << rows << endl;
+          trans->close();
+          NdbSleep_MilliSleep(2500);
+          continue;
+        }
+      }
+
+      trans->close();
+      return NDBT_FAILED;
+    }
+    
+    g_err << "Read " << rows << " rows." << endl;
+    
+    if (records != 0 && rows != (numRanges * records))
+    {
+      g_err << "Expected " << records << " rows"
+            << ", read " << rows << endl;
+#ifdef BUG_14388257_FIXED
+      trans->close();
+      assert(false);
+      return NDBT_FAILED;
+#endif
+    }
+
+    trans->close();
+    i++;
+  }
+  return NDBT_OK;
+}
+
+int checkResourceSnapshot(NDBT_Context* ctx, NDBT_Step* step)
+{
+  NdbRestarter restarter;
+  
+  int checksnapshot = DumpStateOrd::TcResourceCheckLeak;
+  restarter.dumpStateAllNodes(&checksnapshot, 1);
+
+  /* TODO : Check other block's resources? */
+  return NDBT_OK;
+}
+
 
 int
 runBug54945(NDBT_Context* ctx, NDBT_Step* step)
@@ -2364,6 +2578,47 @@ TESTCASE("extraNextResultBug11748194",
 {
   INITIALIZER(runExtraNextResult);
 }
+TESTCASE("ScanRealKeyInfoExhaust",
+         "Test behaviour when TC keyinfo buffers exhausted 4real")
+{
+  /* 55 threads, each setting 200 ranges in their keyinfo
+   * For the lightest single column PK case, each range should
+   * use 2 words, 200 ranges = 400 words per scan thread = 
+   * 400/4 = 100 Databuffers used.
+   * 55 threads = 55*100 = 5500 Databuffers which is >
+   * the 4000 statically allocated in 6.3
+   */
+  TC_PROPERTY("NumRanges", 200);
+  TC_PROPERTY("MaxRunSecs", 120);
+  INITIALIZER(createOrderedPkIndex);
+  INITIALIZER(runLoadTable);
+  INITIALIZER(takeResourceSnapshot);
+  STEPS(runScanReadIndexWithBounds,55);
+  FINALIZER(checkResourceSnapshot);
+  FINALIZER(createOrderedPkIndex_Drop);
+  FINALIZER(runClearTable);
+}
+TESTCASE("ScanKeyInfoExhaust",
+         "Test behaviour when TC keyinfo buffers exhausted with error insert")
+{
+  /* Use error insert 8094 to cause keyinfo exhaustion, then run a single scan
+   * with keyinfo to hit the error path
+   */
+  TC_PROPERTY("MaxRunSecs", 10);
+  INITIALIZER(createOrderedPkIndex);
+  INITIALIZER(runLoadTable);
+  INITIALIZER(takeResourceSnapshot);
+  TC_PROPERTY("ErrorCode", 8094);
+  INITIALIZER(runInsertError);
+  STEP(runScanReadIndexWithBounds);
+  FINALIZER(checkResourceSnapshot);
+  FINALIZER(runInsertError);
+  FINALIZER(createOrderedPkIndex_Drop);
+  FINALIZER(runClearTable);
+}
+  
+
+
 NDBT_TESTSUITE_END(testScan);
 
 int main(int argc, const char** argv){

=== modified file 'storage/ndb/test/run-test/daily-basic-tests.txt'
--- a/storage/ndb/test/run-test/daily-basic-tests.txt	2012-08-22 10:49:42 +0000
+++ b/storage/ndb/test/run-test/daily-basic-tests.txt	2012-09-12 14:18:33 +0000
@@ -1883,3 +1883,7 @@ max-time: 600
 cmd: testRedo
 args: -n RestartFD -l 2 T1
 
+max-time : 300
+cmd: testScan
+args: -n ScanKeyInfoExhaust T1
+

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.1-telco-7.0 branch (frazer.clement:4968 to 4971) Frazer Clement13 Sep