List:Commits« Previous MessageNext Message »
From:Jonas Oreland Date:March 28 2011 11:59am
Subject:bzr commit into mysql-5.1-telco-7.0 branch (jonas:4257) Bug#11769048
View as plain text  
#At file:///home/jonas/src/telco-7.0/ based on revid:ole.john.aske@stripped

 4257 Jonas Oreland	2011-03-28
      ndb - bug#11769048 - Remove hard-coded limit on 32 tux/tup scan per fragment, and instead add config variable MaxParallelScansPerFragment

    modified:
      storage/ndb/include/kernel/ndb_limits.h
      storage/ndb/include/mgmapi/mgmapi_config_parameters.h
      storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp
      storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp
      storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp
      storage/ndb/src/mgmsrv/ConfigInfo.cpp
      storage/ndb/test/ndbapi/testScan.cpp
      storage/ndb/test/run-test/daily-basic-tests.txt
=== modified file 'storage/ndb/include/kernel/ndb_limits.h'
--- a/storage/ndb/include/kernel/ndb_limits.h	2011-02-08 13:55:54 +0000
+++ b/storage/ndb/include/kernel/ndb_limits.h	2011-03-28 11:59:09 +0000
@@ -118,11 +118,6 @@
  * Maximum number of Parallel Scan queries on one hash index fragment
  */
 #define MAX_PARALLEL_SCANS_PER_FRAG 12
-/*
- * Maximum parallel ordered index scans per primary table fragment.
- * Implementation limit is (256 minus 12).
- */
-#define MAX_PARALLEL_INDEX_SCANS_PER_FRAG 32
 
 /**
  * Computed defines

=== modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h'
--- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	2011-02-15 11:41:27 +0000
+++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	2011-03-28 11:59:09 +0000
@@ -180,6 +180,7 @@
 #define CFG_DB_LATE_ALLOC             615
 
 #define CFG_DB_2PASS_INR              616
+#define CFG_DB_PARALLEL_SCANS_PER_FRAG 617
 
 #define CFG_NODE_ARBIT_RANK           200
 #define CFG_NODE_ARBIT_DELAY          201

=== modified file 'storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp'
--- a/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp	2011-02-23 19:28:26 +0000
+++ b/storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp	2011-03-28 11:59:09 +0000
@@ -3240,6 +3240,7 @@ public:
   Uint32 c_max_redo_lag;
   Uint32 c_max_redo_lag_counter;
   Uint64 cTotalLqhKeyReqCount;
+  Uint32 c_max_parallel_scans_per_frag;
 
   inline bool getAllowRead() const {
     return getNodeState().startLevel < NodeState::SL_STOPPING_3;

=== modified file 'storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp'
--- a/storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp	2011-02-15 11:41:27 +0000
+++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp	2011-03-28 11:59:09 +0000
@@ -88,6 +88,8 @@ void Dblqh::initData() 
   cTotalLqhKeyReqCount = 0;
   c_max_redo_lag = 30; // seconds
   c_max_redo_lag_counter = 3; // 3 strikes and you're out
+
+  c_max_parallel_scans_per_frag = 32;
 }//Dblqh::initData()
 
 void Dblqh::initRecords() 

=== modified file 'storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp'
--- a/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp	2011-02-23 19:28:26 +0000
+++ b/storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp	2011-03-28 11:59:09 +0000
@@ -1351,6 +1351,16 @@ void Dblqh::execREAD_CONFIG_REQ(Signal* 
   c_max_redo_lag_counter = 3;
   ndb_mgm_get_int_parameter(p, CFG_DB_REDO_OVERCOMMIT_COUNTER,
                             &c_max_redo_lag_counter);
+
+  c_max_parallel_scans_per_frag = 32;
+  ndb_mgm_get_int_parameter(p, CFG_DB_PARALLEL_SCANS_PER_FRAG,
+                            &c_max_parallel_scans_per_frag);
+
+  if (c_max_parallel_scans_per_frag > (256 - MAX_PARALLEL_SCANS_PER_FRAG) / 2)
+  {
+    jam();
+    c_max_parallel_scans_per_frag = (256 - MAX_PARALLEL_SCANS_PER_FRAG) / 2;
+  }
   return;
 }//Dblqh::execSIZEALT_REP()
 
@@ -11331,7 +11341,19 @@ Uint32 Dblqh::initScanrec(const ScanFrag
    * !idx uses 1 - (MAX_PARALLEL_SCANS_PER_FRAG - 1)  =  1-11
    *  idx uses from MAX_PARALLEL_SCANS_PER_FRAG - MAX = 12-42)
    */
+
+  /**
+   * ACC only supports 12 parallel scans per fragment (hard limit)
+   * TUP/TUX does not have any such limit...but when scanning with keyinfo
+   *         (for take-over) no more than 255 such scans can be active
+   *         at a fragment (dur to 8 bit number in scan-keyinfo protocol)
+   *
+   * TODO: Make TUP/TUX limits depend on scanKeyinfoFlag (possibly with
+   *       other config limit too)
+   */
+
   Uint32 start, stop;
+  Uint32 max_parallel_scans_per_frag = c_max_parallel_scans_per_frag;
   if (accScan)
   {
     start = 1;
@@ -11340,13 +11362,13 @@ Uint32 Dblqh::initScanrec(const ScanFrag
   else if (rangeScan)
   {
     start = MAX_PARALLEL_SCANS_PER_FRAG;
-    stop = start + MAX_PARALLEL_INDEX_SCANS_PER_FRAG - 1;
+    stop = start + max_parallel_scans_per_frag - 1;
   }
   else
   {
     ndbassert(tupScan);
-    start = MAX_PARALLEL_SCANS_PER_FRAG + MAX_PARALLEL_INDEX_SCANS_PER_FRAG;
-    stop = start + MAX_PARALLEL_INDEX_SCANS_PER_FRAG - 1;
+    start = MAX_PARALLEL_SCANS_PER_FRAG + max_parallel_scans_per_frag;
+    stop = start + max_parallel_scans_per_frag - 1;
   }
   ndbrequire((start < 32 * tFragPtr.p->m_scanNumberMask.Size) &&
              (stop < 32 * tFragPtr.p->m_scanNumberMask.Size));
@@ -11359,7 +11381,7 @@ Uint32 Dblqh::initScanrec(const ScanFrag
       jam();
       return ScanFragRef::ZTOO_MANY_ACTIVE_SCAN_ERROR;
     }
-    
+
     /**
      * Put on queue
      */

=== modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp'
--- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp	2011-02-15 11:41:27 +0000
+++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp	2011-03-28 11:59:09 +0000
@@ -1926,6 +1926,20 @@ const ConfigInfo::ParamInfo ConfigInfo::
     "true"                       /* Max */
   },
 
+  {
+    CFG_DB_PARALLEL_SCANS_PER_FRAG,
+    "MaxParallelScansPerFragment",
+    DB_TOKEN,
+    "Max parallel scans per fragment (tup or tux). If this limit is reached "
+    " scans will be serialized using a queue.",
+    ConfigInfo::CI_USED,
+    false,
+    ConfigInfo::CI_INT,
+    "32",
+    "1",                         /* Min */
+    STR_VALUE(MAX_INT_RNIL)      /* Max */
+  },
+
   /***************************************************************************
    * API
    ***************************************************************************/

=== modified file 'storage/ndb/test/ndbapi/testScan.cpp'
--- a/storage/ndb/test/ndbapi/testScan.cpp	2011-02-02 00:40:07 +0000
+++ b/storage/ndb/test/ndbapi/testScan.cpp	2011-03-28 11:59:09 +0000
@@ -296,12 +296,25 @@ int runScanRead(NDBT_Context* ctx, NDBT_
   int records = ctx->getProperty("Rows", ctx->getNumRecords());
   int parallelism = ctx->getProperty("Parallelism", 240);
   int abort = ctx->getProperty("AbortProb", 5);
+  int tupscan = ctx->getProperty("TupScan", (Uint32)0);
 
   int i = 0;
   HugoTransactions hugoTrans(*ctx->getTab());
   while (i<loops && !ctx->isTestStopped()) {
     g_info << i << ": ";
-    if (hugoTrans.scanReadRecords(GETNDB(step), records, abort, parallelism) != 0){
+
+    int scan_flags = 0;
+    if (tupscan == 1)
+    {
+      scan_flags |= NdbScanOperation::SF_TupScan;
+      if (hugoTrans.scanReadRecords(GETNDB(step), records, abort, parallelism,
+                                    NdbOperation::LM_CommittedRead,
+                                    scan_flags) != 0)
+        return NDBT_FAILED;
+    }
+    else if (hugoTrans.scanReadRecords(GETNDB(step), records, abort, parallelism)
+             != 0)
+    {
       return NDBT_FAILED;
     }
     i++;
@@ -1568,6 +1581,13 @@ TESTCASE("ScanRead100", 
   STEPS(runScanRead, 100);
   FINALIZER(runClearTable);
 }
+TESTCASE("TupScanRead100",
+	 "Verify scan requirement: Scan with 100 simultaneous threads"){
+  TC_PROPERTY("TupScan", 1);
+  INITIALIZER(runLoadTable);
+  STEPS(runScanRead, 100);
+  FINALIZER(runClearTable);
+}
 TESTCASE("Scan-bug8262", 
 	 ""){
   TC_PROPERTY("Rows", 1);

=== modified file 'storage/ndb/test/run-test/daily-basic-tests.txt'
--- a/storage/ndb/test/run-test/daily-basic-tests.txt	2011-02-01 08:36:25 +0000
+++ b/storage/ndb/test/run-test/daily-basic-tests.txt	2011-03-28 11:59:09 +0000
@@ -395,6 +395,10 @@ max-time: 1800
 cmd: testScan
 args: -n ScanRead100 -l 100 T1 D1 D2 
 
+max-time: 1800
+cmd: testScan
+args: -n TupScanRead100 -l 100 T1 D1 D2 
+
 max-time: 600
 cmd: testScan
 args: -n ScanRead40 -l 100 T1 D1 D2 


Attachment: [text/bzr-bundle] bzr/jonas@mysql.com-20110328115909-wai8mfmdk76lfxpv.bundle
Thread
bzr commit into mysql-5.1-telco-7.0 branch (jonas:4257) Bug#11769048Jonas Oreland28 Mar