From: Frazer Clement Date: September 12 2012 3:22pm Subject: bzr push into mysql-5.1-telco-7.0 branch (frazer.clement:4968 to 4971) List-Archive: http://lists.mysql.com/commits/144746 Message-Id: <201209121522.q8CFMeOf006660@acsmt356.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 4971 Frazer Clement 2012-09-12 [merge] Merge 6.3->7.0 modified: storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp storage/ndb/test/ndbapi/testScan.cpp storage/ndb/test/run-test/daily-basic-tests.txt 4970 Frazer Clement 2012-09-12 [merge] Merge 6.3->7.0 4969 Frazer Clement 2012-09-12 [merge] Merge 6.3->7.0 4968 magnus.blaudd@stripped 2012-09-12 [merge] Merge modified: sql/ha_ndb_index_stat.cc sql/ha_ndb_index_stat.h === modified file 'storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp' --- a/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp 2012-08-28 11:22:02 +0000 +++ b/storage/ndb/include/kernel/signaldata/DumpStateOrd.hpp 2012-09-12 14:18:33 +0000 @@ -101,6 +101,7 @@ public: LqhDumpAllActiveScanRec = 2302, LqhDumpLcpState = 2303, LqhErrorInsert5042 = 2315, + LqhDumpPoolLevels = 2353, AccDumpOneScanRec = 2400, AccDumpAllScanRec = 2401, @@ -121,6 +122,7 @@ public: TcSetApplTransactionTimeout = 2508, TcStartDumpIndexOpCount = 2512, TcDumpIndexOpCount = 2513, + TcDumpPoolLevels = 2555, CmvmiDumpConnections = 2600, CmvmiDumpLongSignalMemory = 2601, CmvmiSetRestartOnErrorInsert = 2602, === modified file 'storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp' --- a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp 2012-08-22 10:49:42 +0000 +++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp 2012-09-12 14:18:33 +0000 @@ -23687,6 +23687,74 @@ Dblqh::execDUMP_STATE_ORD(Signal* signal SET_ERROR_INSERT_VALUE2(5050, c_master_node_id); #endif } + + if (arg == DumpStateOrd::LqhDumpPoolLevels) + { + /* Dump some state info for internal buffers */ + if (signal->getLength() == 1) + { + signal->theData[1] = 1; + signal->theData[2] = 0; + signal->theData[3] = 0; + sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB); + return; + } + if (signal->getLength() != 4) + { + ndbout_c("DUMP LqhDumpPoolLevels : Bad signal length : %u", signal->getLength()); + return; + } + + Uint32 resource = signal->theData[1]; + Uint32 position = signal->theData[2]; + Uint32 sum = signal->theData[3]; + /*const Uint32 MAX_ITER = 200; */ + + switch(resource) + { + case 1: + { + /* Must get all in one loop, as we're traversing a dynamic list */ + sum = 0; + TcConnectionrecPtr tcp; + tcp.i = cfirstfreeTcConrec; + while (tcp.i != RNIL) + { + sum++; + ptrCheckGuard(tcp, ctcConnectrecFileSize, tcConnectionrec); + tcp.i = tcp.p->nextTcConnectrec; + } + infoEvent("LQH : TcConnection (operation) records in use/total %u/%u (%u bytes each)", + ctcConnectrecFileSize - sum, ctcConnectrecFileSize, (Uint32) sizeof(TcConnectionrec)); + resource++; + position = 0; + sum = 0; + break; + } + case 2: + { + infoEvent("LQH : ScanRecord (Fragment) pool in use/total %u/%u (%u bytes each)", + c_scanRecordPool.getSize()- + c_scanRecordPool.getNoOfFree(), + c_scanRecordPool.getSize(), + (Uint32) sizeof(ScanRecord)); + resource++; + position = 0; + sum = 0; + break; + } + default: + return; + } + + signal->theData[0] = DumpStateOrd::LqhDumpPoolLevels; + signal->theData[1] = resource; + signal->theData[2] = position; + signal->theData[3] = sum; + sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB); + return; + } + }//Dblqh::execDUMP_STATE_ORD() === modified file 'storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp' --- a/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp 2012-08-24 12:00:05 +0000 +++ b/storage/ndb/src/kernel/blocks/dbtc/Dbtc.hpp 2012-09-12 14:18:33 +0000 @@ -1789,6 +1789,7 @@ private: // Resource usage counter(not monotonic) Uint32 cconcurrentOp; + Uint32 cconcurrentScans; MonotonicCounters() : cattrinfoCount(0), === modified file 'storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp' --- a/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp 2012-08-24 12:00:05 +0000 +++ b/storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp 2012-09-12 14:18:33 +0000 @@ -10794,12 +10794,29 @@ void Dbtc::scanKeyinfoLab(Signal* signal ndbassert( signal->getLength() == (KeyInfo::HeaderLength + wordsInSignal) ); - if (unlikely (! appendToSection(regCachePtr->keyInfoSectionI, - &signal->theData[KeyInfo::HeaderLength], - wordsInSignal))) - { - jam(); - seizeDatabuferrorLab(signal); + if (unlikely ((! appendToSection(regCachePtr->keyInfoSectionI, + &signal->theData[KeyInfo::HeaderLength], + wordsInSignal)) || + ERROR_INSERTED(8094))) + { + jam(); + Uint32 transid0 = apiConnectptr.p->transid[0]; + Uint32 transid1 = apiConnectptr.p->transid[1]; + ndbrequire(apiConnectptr.p->apiScanRec != RNIL); + ScanRecordPtr scanPtr; + scanPtr.i = apiConnectptr.p->apiScanRec; + ptrCheckGuard(scanPtr, cscanrecFileSize, scanRecord); + abortScanLab(signal, + scanPtr, + ZGET_DATAREC_ERROR, + true /* Not started */); + + /* Prepare for up coming ATTRINFO/KEYINFO */ + apiConnectptr.p->apiConnectstate = CS_ABORTING; + apiConnectptr.p->abortState = AS_IDLE; + apiConnectptr.p->transid[0] = transid0; + apiConnectptr.p->transid[1] = transid1; + return; } @@ -11223,6 +11240,7 @@ void Dbtc::releaseScanResources(Signal* scanPtr.p->scanTcrec = RNIL; scanPtr.p->scanApiRec = RNIL; cfirstfreeScanrec = scanPtr.i; + ndbassert(cConcScanCount > 0); cConcScanCount--; apiConnectptr.p->apiScanRec = RNIL; @@ -13492,6 +13510,69 @@ Dbtc::execDUMP_STATE_ORD(Signal* signal) ndbrequire(rss_cconcurrentOp == c_counters.cconcurrentOp); #endif } + + if (arg == DumpStateOrd::TcDumpPoolLevels) + { + if (signal->getLength() == 1) + { + signal->theData[1] = 1; + signal->theData[2] = 0; + signal->theData[3] = 0; + sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB); + return; + } + if (signal->getLength() != 4) + { + ndbout_c("DUMP TcDumpPoolLevels : Bad signal length : %u", signal->getLength()); + return; + } + + Uint32 resource = signal->theData[1]; + Uint32 position = signal->theData[2]; + Uint32 sum = signal->theData[3]; + /* const Uint32 MAX_ITER = 200; */ + + switch(resource) + { + case 1: + infoEvent("TC : Concurrent operations in use/total : %u/%u (%u bytes each)", + c_counters.cconcurrentOp, + ctcConnectFilesize, + (Uint32) sizeof(TcConnectRecord)); + resource++; + position = 0; + sum = 0; + break; + case 2: + infoEvent("TC : Concurrent scans in use/total : %u/%u (%u bytes each)", + cConcScanCount, + cscanrecFileSize, + (Uint32) sizeof(ScanRecord)); + resource++; + position = 0; + sum = 0; + break; + case 3: + infoEvent("TC : Scan Frag records in use/total : %u/%u (%u bytes each)", + c_scan_frag_pool.getSize() - + c_scan_frag_pool.getNoOfFree(), + c_scan_frag_pool.getSize(), + (Uint32) sizeof(ScanFragRec)); + resource++; + position = 0; + sum = 0; + break; + default: + return; + } + + signal->theData[0] = DumpStateOrd::TcDumpPoolLevels; + signal->theData[1] = resource; + signal->theData[2] = position; + signal->theData[3] = sum; + sendSignal(reference(), GSN_DUMP_STATE_ORD, signal, 4, JBB); + return; + } }//Dbtc::execDUMP_STATE_ORD() void Dbtc::execDBINFO_SCANREQ(Signal *signal) === modified file 'storage/ndb/test/ndbapi/testScan.cpp' --- a/storage/ndb/test/ndbapi/testScan.cpp 2012-08-24 12:00:05 +0000 +++ b/storage/ndb/test/ndbapi/testScan.cpp 2012-09-12 14:18:33 +0000 @@ -1390,6 +1390,220 @@ finalizeBug42559(NDBT_Context* ctx, NDBT return NDBT_OK; } +int takeResourceSnapshot(NDBT_Context* ctx, NDBT_Step* step) +{ + NdbRestarter restarter; + + int checksnapshot = DumpStateOrd::TcResourceSnapshot; + restarter.dumpStateAllNodes(&checksnapshot, 1); + + /* TODO : Check other block's resources? */ + return NDBT_OK; +} + +int runScanReadIndexWithBounds(NDBT_Context* ctx, NDBT_Step* step){ + int loops = ctx->getNumLoops(); + int records = ctx->getNumRecords(); + int numRanges = ctx->getProperty("NumRanges", 1); + int maxRunSecs = ctx->getProperty("MaxRunSecs", 60); + int maxRetries = ctx->getProperty("MaxRetries", 1000000); + + const NdbDictionary::Index * pIdx = + GETNDB(step)->getDictionary()->getIndex(orderedPkIdxName, + ctx->getTab()->getName()); + + int i = 0; + HugoCalculator calc(*ctx->getTab()); + NDBT_ResultRow row (*ctx->getTab()); + Ndb* ndb = GETNDB(step); + + Uint64 start = NdbTick_CurrentMillisecond(); + Uint64 end = start + (1000*maxRunSecs); + int retries = 0; + + /* Here we run an ordered index scan, with a bound. + * There are numRanges sub-scans with the same bound + * This is done to use up some KeyInfo, and expose bugs in that area + * If we run many of these in parallel we may exhaust the available KeyInfo storage, + * which may expose some bugs. + */ + while (pIdx && + iisTestStopped() && + NdbTick_CurrentMillisecond() < end) { + g_info << "Step " << step->getStepNo() + << "Loop : " << i << ": "; + + /* Use specific-partition variant of startTransaction to ensure a single + * TC node is used + */ + NdbTransaction* trans = ndb->startTransaction(ctx->getTab(), Uint32(0)); + if (trans == NULL) + { + g_err << "Transaction start failed " << ndb->getNdbError() << endl; + return NDBT_FAILED; + } + + NdbIndexScanOperation* iso = trans->getNdbIndexScanOperation(pIdx->getName(), + ctx->getTab()->getName()); + if (iso == NULL) + { + g_err << "Error obtaining IndexScanOperation : " << trans->getNdbError() << endl; + trans->close(); + return NDBT_FAILED; + } + + if (iso->readTuples(NdbOperation::LM_CommittedRead, + (NdbScanOperation::SF_OrderBy | + NdbScanOperation::SF_ReadRangeNo | + NdbScanOperation::SF_MultiRange), + 0) != 0) + { + g_err << "Error calling readTuples : " << iso->getNdbError() << endl; + trans->close(); + return NDBT_FAILED; + } + + for (int range = 0; range < numRanges; range++) + { + /* Now define a bound... */ + for (Uint32 k=0; kgetNoOfColumns(); k++) + { + const NdbDictionary::Column* idxCol = pIdx->getColumn(k); + const char* colName = idxCol->getName(); + /* Lower bound of <= NULL should return all rows */ + if (iso->setBound(colName, NdbIndexScanOperation::BoundLE, NULL) != 0) + { + g_err << "Error setting bound for column %s. " + << iso->getNdbError() << endl; + trans->close(); + return NDBT_FAILED; + } + } + + if (iso->end_of_bound(range) != 0) + { + g_err << "Error closing range " << range << endl; + g_err << iso->getNdbError() << endl; + return NDBT_FAILED; + } + } + + const NdbDictionary::Table& tab= *ctx->getTab(); + + /* Now request all columns in result projection */ + for (int a=0; agetValue(tab.getColumn(a)->getName())) == 0) { + g_err << "Error defining read value " << trans->getNdbError() << endl; + trans->close(); + return NDBT_FAILED; + } + } + + /* Ready to go... */ + trans->execute(NoCommit, AbortOnError); + + if (trans->getNdbError().code != 0) + { + if (trans->getNdbError().code == 218) + { + /* Out of KeyInfo buffers in TC - that's ok, let's try again */ + trans->close(); + if (retries++ < maxRetries) + { + g_err << "Step " << step->getStepNo() + << " TC out of Keyinfo buffers (218) - retrying" << endl; + continue; + } + } + + g_err << "Error on execution : " << trans->getNdbError() << endl; + trans->close(); + return NDBT_FAILED; + } + + int eof; + int rows = 0; + while ((eof = iso->nextResult(true)) == 0) + { + rows++; + if (calc.verifyRowValues(&row) != 0) + { + g_err << "Verification failed." << endl; + trans->close(); + return NDBT_FAILED; + } + +#ifdef BUG_14388257_FIXED + int rangeNum = (rows -1) / records; + if (iso->get_range_no() != rangeNum) + { + g_err << "Expected row " << rows + << " to be in range " << rangeNum + << " but it reports range num " << iso->get_range_no() + << " : " << row + << endl; + return NDBT_FAILED; + } +#endif + + //g_err << row << endl; + } + + if (eof != 1) + { + g_err << "nextResult() returned " << eof << endl; + g_err << "Scan error : " << iso->getNdbError() << endl; + + if (iso->getNdbError().status == NdbError::TemporaryError) + { + if (retries++ < maxRetries) + { + g_err << "Step " + << step->getStepNo() + << " Temporary, retrying on iteration " + << i << " rows so far : " << rows << endl; + trans->close(); + NdbSleep_MilliSleep(2500); + continue; + } + } + + trans->close(); + return NDBT_FAILED; + } + + g_err << "Read " << rows << " rows." << endl; + + if (records != 0 && rows != (numRanges * records)) + { + g_err << "Expected " << records << " rows" + << ", read " << rows << endl; +#ifdef BUG_14388257_FIXED + trans->close(); + assert(false); + return NDBT_FAILED; +#endif + } + + trans->close(); + i++; + } + return NDBT_OK; +} + +int checkResourceSnapshot(NDBT_Context* ctx, NDBT_Step* step) +{ + NdbRestarter restarter; + + int checksnapshot = DumpStateOrd::TcResourceCheckLeak; + restarter.dumpStateAllNodes(&checksnapshot, 1); + + /* TODO : Check other block's resources? */ + return NDBT_OK; +} + int runBug54945(NDBT_Context* ctx, NDBT_Step* step) @@ -2364,6 +2578,47 @@ TESTCASE("extraNextResultBug11748194", { INITIALIZER(runExtraNextResult); } +TESTCASE("ScanRealKeyInfoExhaust", + "Test behaviour when TC keyinfo buffers exhausted 4real") +{ + /* 55 threads, each setting 200 ranges in their keyinfo + * For the lightest single column PK case, each range should + * use 2 words, 200 ranges = 400 words per scan thread = + * 400/4 = 100 Databuffers used. + * 55 threads = 55*100 = 5500 Databuffers which is > + * the 4000 statically allocated in 6.3 + */ + TC_PROPERTY("NumRanges", 200); + TC_PROPERTY("MaxRunSecs", 120); + INITIALIZER(createOrderedPkIndex); + INITIALIZER(runLoadTable); + INITIALIZER(takeResourceSnapshot); + STEPS(runScanReadIndexWithBounds,55); + FINALIZER(checkResourceSnapshot); + FINALIZER(createOrderedPkIndex_Drop); + FINALIZER(runClearTable); +} +TESTCASE("ScanKeyInfoExhaust", + "Test behaviour when TC keyinfo buffers exhausted with error insert") +{ + /* Use error insert 8094 to cause keyinfo exhaustion, then run a single scan + * with keyinfo to hit the error path + */ + TC_PROPERTY("MaxRunSecs", 10); + INITIALIZER(createOrderedPkIndex); + INITIALIZER(runLoadTable); + INITIALIZER(takeResourceSnapshot); + TC_PROPERTY("ErrorCode", 8094); + INITIALIZER(runInsertError); + STEP(runScanReadIndexWithBounds); + FINALIZER(checkResourceSnapshot); + FINALIZER(runInsertError); + FINALIZER(createOrderedPkIndex_Drop); + FINALIZER(runClearTable); +} + + + NDBT_TESTSUITE_END(testScan); int main(int argc, const char** argv){ === modified file 'storage/ndb/test/run-test/daily-basic-tests.txt' --- a/storage/ndb/test/run-test/daily-basic-tests.txt 2012-08-22 10:49:42 +0000 +++ b/storage/ndb/test/run-test/daily-basic-tests.txt 2012-09-12 14:18:33 +0000 @@ -1883,3 +1883,7 @@ max-time: 600 cmd: testRedo args: -n RestartFD -l 2 T1 +max-time : 300 +cmd: testScan +args: -n ScanKeyInfoExhaust T1 + No bundle (reason: useless for push emails).