List:Commits« Previous MessageNext Message »
From:Jonas Oreland Date:November 9 2010 1:03pm
Subject:bzr commit into mysql-5.1-telco-7.0-spj-scan-vs-scan branch (jonas:3354)
View as plain text  
#At file:///home/jonas/src/70-spj-svs/ based on revid:jonas@stripped

 3354 Jonas Oreland	2010-11-09
      ndb spj-svs - Add support for scan-scan queries to Hugo

    modified:
      storage/ndb/test/include/HugoQueries.hpp
      storage/ndb/test/include/HugoQueryBuilder.hpp
      storage/ndb/test/src/HugoQueries.cpp
      storage/ndb/test/src/HugoQueryBuilder.cpp
      storage/ndb/test/tools/hugoJoin.cpp
=== modified file 'storage/ndb/test/include/HugoQueries.hpp'
--- a/storage/ndb/test/include/HugoQueries.hpp	2010-05-20 11:18:08 +0000
+++ b/storage/ndb/test/include/HugoQueries.hpp	2010-11-09 13:03:09 +0000
@@ -27,6 +27,13 @@
 
 class HugoQueries
 {
+  struct Op
+  {
+    const NdbQueryOperationDef* m_query_op;
+    Vector<NDBT_ResultRow*> m_rows;
+    HugoCalculator * m_calc;
+  };
+
 public:
   HugoQueries(const NdbQueryDef & query);
   virtual ~HugoQueries();
@@ -41,7 +48,7 @@ public:
                    int scan_flags = 0);
 
   static int equalForParameters(char * buf,
-                                HugoCalculator&,
+                                Op&,
                                 NdbQueryParamValue params[],
                                 int rowNo);
   static int getValueForQueryOp(NdbQueryOperation* pOp, NDBT_ResultRow* pRow);
@@ -51,13 +58,6 @@ public:
 protected:
 
   const NdbQueryDef* m_query_def;
-  struct Op
-  {
-    const NdbQueryOperationDef* m_query_op;
-    Vector<NDBT_ResultRow*> m_rows;
-    HugoCalculator * m_calc;
-  };
-
   Vector<Op> m_ops;
   int m_retryMax;
 };

=== modified file 'storage/ndb/test/include/HugoQueryBuilder.hpp'
--- a/storage/ndb/test/include/HugoQueryBuilder.hpp	2010-07-02 09:06:49 +0000
+++ b/storage/ndb/test/include/HugoQueryBuilder.hpp	2010-11-09 13:03:09 +0000
@@ -146,14 +146,24 @@ private:
    *   ops
    */
   static bool checkBindable(Vector<const NdbDictionary::Column*> cols,
-                            Vector<Op> ops);
+                            Vector<Op> ops,
+                            bool allow_bind_nullable);
 
   Vector<Op> getParents(OpIdx); //
   NdbQueryOperand * createLink(NdbQueryBuilder&, const NdbDictionary::Column*,
-                               Vector<Op> & parents);
+                               Vector<Op> & parents,
+                               bool allow_bind_nullable);
   const NdbQueryOperationDef* createOp(NdbQueryBuilder&);
 
   void fixOptions();
+
+  /**
+   * We currently don't support busy-scan joins
+   */
+  bool checkBusyScan(Op) const;
+  bool isAncestor(const Op& parent, const Op& child) const;
+
+  friend NdbOut& operator<<(NdbOut& out, const HugoQueryBuilder::Op& op);
 };
 
 #endif

=== modified file 'storage/ndb/test/src/HugoQueries.cpp'
--- a/storage/ndb/test/src/HugoQueries.cpp	2010-05-20 11:18:08 +0000
+++ b/storage/ndb/test/src/HugoQueries.cpp	2010-11-09 13:03:09 +0000
@@ -74,21 +74,46 @@ HugoQueries::allocRows(int batch)
 
 int
 HugoQueries::equalForParameters(char * buf,
-                                HugoCalculator& calc,
+                                Op & op,
                                 NdbQueryParamValue params[],
                                 int rowNo)
 {
   Uint32 no = 0;
+  HugoCalculator & calc = * op.m_calc;
   const NdbDictionary::Table & tab = calc.getTable();
-  for (int i = 0; i<tab.getNoOfColumns(); i++)
+  if (op.m_query_op->getType() == NdbQueryOperationDef::TableScan)
   {
-    const NdbDictionary::Column* attr = tab.getColumn(i);
-    if (attr->getPrimaryKey())
+
+  }
+  else if (op.m_query_op->getType() == NdbQueryOperationDef::PrimaryKeyAccess)
+  {
+    for (int i = 0; i<tab.getNoOfColumns(); i++)
     {
+      const NdbDictionary::Column* attr = tab.getColumn(i);
+      if (attr->getPrimaryKey())
+      {
+        Uint32 len = attr->getSizeInBytes();
+        Uint32 real_len;
+        bzero(buf, len);
+        calc.calcValue((Uint32)rowNo, i, 0, buf, len, &real_len);
+        params[no++]= NdbQueryParamValue((void*)buf);
+        buf += len;
+      }
+    }
+  }
+  else if (op.m_query_op->getType() == NdbQueryOperationDef::UniqueIndexAccess||
+           op.m_query_op->getType() == NdbQueryOperationDef::OrderedIndexScan)
+  {
+    const NdbDictionary::Index* idx = op.m_query_op->getIndex();
+    for (unsigned i = 0; i < idx->getNoOfColumns(); i++)
+    {
+      const NdbDictionary::Column* attr = 
+        tab.getColumn(idx->getColumn(i)->getName());
       Uint32 len = attr->getSizeInBytes();
       Uint32 real_len;
       bzero(buf, len);
-      calc.calcValue((Uint32)rowNo, i, 0, buf, len, &real_len);
+      calc.calcValue((Uint32)rowNo, attr->getColumnNo(), 
+                     0, buf, len, &real_len);
       params[no++]= NdbQueryParamValue((void*)buf);
       buf += len;
     }
@@ -163,7 +188,7 @@ HugoQueries::runLookupQuery(Ndb* pNdb,
     {
       char buf[NDB_MAX_TUPLE_SIZE];
       NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
-      equalForParameters(buf, * m_ops[0].m_calc, params, b + r);
+      equalForParameters(buf, m_ops[0], params, b + r);
 
       NdbQuery * query = pTrans->createQuery(m_query_def, params);
       if (query == 0)
@@ -256,9 +281,10 @@ HugoQueries::runScanQuery(Ndb * pNdb,
     }
 
     NdbQuery * query = 0;
-    //char buf[NDB_MAX_TUPLE_SIZE];
-    NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
 
+    char buf[NDB_MAX_TUPLE_SIZE];
+    NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
+    equalForParameters(buf, m_ops[0], params, /* rowNo */ 0);
     query = pTrans->createQuery(m_query_def, params);
     if (query == 0)
     {

=== modified file 'storage/ndb/test/src/HugoQueryBuilder.cpp'
--- a/storage/ndb/test/src/HugoQueryBuilder.cpp	2010-07-02 09:06:49 +0000
+++ b/storage/ndb/test/src/HugoQueryBuilder.cpp	2010-11-09 13:03:09 +0000
@@ -18,6 +18,34 @@
 
 #include <HugoQueryBuilder.hpp>
 
+static
+bool
+isScan(const NdbQueryOperationDef* def)
+{
+  return
+    def->getType() == NdbQueryOperationDef::TableScan ||
+    def->getType() == NdbQueryOperationDef::OrderedIndexScan;
+}
+
+NdbOut&
+operator<<(NdbOut& out, const HugoQueryBuilder::Op& op)
+{
+  out << "[" << op.m_idx << " : " << op.m_op->getTable()->getName() << ": ";
+  switch(op.m_op->getType()){
+  case NdbQueryOperationDef::TableScan:
+    out << "table-scan";
+    break;
+  case NdbQueryOperationDef::OrderedIndexScan:
+    out << "index-scan";
+    break;
+  default:
+    out << "lookup";
+  }
+
+  out << " : parent: " << op.m_parent << " ]";
+  return out;
+}
+
 void
 HugoQueryBuilder::init()
 {
@@ -58,9 +86,28 @@ HugoQueryBuilder::addTable(Ndb* ndb, con
   TableDef def;
   def.m_table = tab;
 
-  /**
-   * TODO discover indexes...
-   */
+  NdbDictionary::Dictionary* pDict = ndb->getDictionary();
+  NdbDictionary::Dictionary::List l;
+  int res = pDict->listIndexes(l, tab->getName());
+  if (res == 0)
+  {
+    for (unsigned i = 0; i<l.count; i++)
+    {
+      const NdbDictionary::Index * idx = pDict->getIndex(l.elements[i].name,
+                                                         tab->getName());
+      if (idx)
+      {
+        if (idx->getType() == NdbDictionary::Index::UniqueHashIndex)
+        {
+          def.m_unique_indexes.push_back(idx);
+        }
+        else if (idx->getType() == NdbDictionary::Index::OrderedIndex)
+        {
+          def.m_ordered_indexes.push_back(idx);
+        }
+      }
+    }
+  }
 
   m_tables.push_back(def);
 }
@@ -166,7 +213,8 @@ found:
 
 bool
 HugoQueryBuilder::checkBindable(Vector<const NdbDictionary::Column*> cols,
-                                Vector<Op> ops)
+                                Vector<Op> ops,
+                                bool allow_bind_nullable)
 {
   for (size_t c = 0; c < cols.size(); c++)
   {
@@ -179,7 +227,9 @@ HugoQueryBuilder::checkBindable(Vector<c
       {
         for (int i = 0; i<tab->getNoOfColumns(); i++)
         {
-          if (col->isBindable(* tab->getColumn(i)) == 0)
+          if (!allow_bind_nullable && tab->getColumn(i)->getNullable())
+            continue;
+          else if (col->isBindable(* tab->getColumn(i)) == 0)
           {
             found = true;
             break;
@@ -193,6 +243,44 @@ HugoQueryBuilder::checkBindable(Vector<c
   return true;
 }
 
+bool
+HugoQueryBuilder::isAncestor(const Op& parent, const Op& child) const
+{
+  int pi = parent.m_idx;
+  int ci = child.m_idx;
+  assert(ci != pi);
+
+  while (ci != 0)
+  {
+    if (m_query[ci].m_parent == pi)
+      return true;
+    assert(m_query[ci].m_parent != -1);
+    ci = m_query[m_query[ci].m_parent].m_idx;
+  }
+  return false;
+}
+
+bool
+HugoQueryBuilder::checkBusyScan(Op op) const
+{
+  /**
+   * Iterate upwards until we find first scan...
+   */
+  while (op.m_parent != -1)
+  {
+    if (isScan(op.m_op))
+    {
+      break;
+    }
+    op = m_query[op.m_parent];
+  }
+
+  for (size_t i = op.m_idx + 1; i < m_query.size(); i++)
+    if (isAncestor(op, m_query[i]) && isScan(m_query[i].m_op))
+      return true;
+
+  return false;
+}
 
 Vector<HugoQueryBuilder::Op>
 HugoQueryBuilder::getParents(OpIdx oi)
@@ -200,6 +288,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
   /**
    * We need to be able to bind all columns in table/index
    */
+  bool allow_bind_nullable = false;
+  bool check_bushy_scan = false;
   Vector<const NdbDictionary::Column*> cols;
   if (oi.m_index == 0)
   {
@@ -207,9 +297,24 @@ HugoQueryBuilder::getParents(OpIdx oi)
       if (oi.m_table->getColumn(i)->getPrimaryKey())
         cols.push_back(oi.m_table->getColumn(i));
   }
-  else
+  else if (oi.m_index->getType() == NdbDictionary::Index::UniqueHashIndex)
+  {
+    for (unsigned i = 0; i < oi.m_index->getNoOfColumns(); i++)
+      cols.push_back(oi.m_table->getColumn
+                     (oi.m_index->getColumn(i)->getName()));
+  }
+  else if (oi.m_index->getType() == NdbDictionary::Index::OrderedIndex)
   {
-    abort(); // TODO
+    /**
+     * Binding a prefix is ok...but skip this for now...
+     */
+    allow_bind_nullable = true;
+    check_bushy_scan = true;
+    unsigned cnt = oi.m_index->getNoOfColumns();
+    unsigned val = cnt; // cnt > 1 ? (1 + (rand() % (cnt - 1))) : cnt;
+    for (unsigned i = 0; i < val; i++)
+      cols.push_back(oi.m_table->getColumn
+                     (oi.m_index->getColumn(i)->getName()));
   }
 
   int r = rand() % m_query.size();
@@ -218,6 +323,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
   {
     Vector<Op> set;
     Op op = m_query[(i + r) % cnt];
+    if (check_bushy_scan && checkBusyScan(op))
+      continue;
     set.push_back(op);
 
 #if 0
@@ -234,7 +341,7 @@ HugoQueryBuilder::getParents(OpIdx oi)
     }
 #endif
 
-    if (checkBindable(cols, set))
+    if (checkBindable(cols, set, allow_bind_nullable))
       return set;
   }
 
@@ -245,7 +352,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
 NdbQueryOperand *
 HugoQueryBuilder::createLink(NdbQueryBuilder& builder,
                              const NdbDictionary::Column* pCol,
-                             Vector<Op> & parents)
+                             Vector<Op> & parents,
+                             bool allow_bind_nullable)
 {
   int cnt = (int)parents.size();
   int r = rand();
@@ -292,6 +400,9 @@ HugoQueryBuilder::createLink(NdbQueryBui
         // already checked
         continue;
       }
+      if (!allow_bind_nullable && col->getNullable())
+        continue;
+
       if (pCol->isBindable(* col) == 0)
       {
         goto found;
@@ -338,12 +449,28 @@ HugoQueryBuilder::createOp(NdbQueryBuild
     case NdbQueryOperationDef::TableScan:
       op.m_op = builder.scanTable(oi.m_table);
       break;
-    case NdbQueryOperationDef::OrderedIndexScan:
-      abort();
+    case NdbQueryOperationDef::OrderedIndexScan:{
+      int opNo = 0;
+      NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+      for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+      {
+        operands[opNo++] = builder.paramValue();
+      }
+      operands[opNo] = 0;
+      NdbQueryIndexBound bounds(operands);
+      op.m_op = builder.scanIndex(oi.m_index, oi.m_table, &bounds);
       break;
+    }
     case NdbQueryOperationDef::UniqueIndexAccess:
-      // TODO
-      abort();
+      int opNo = 0;
+      NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+      for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+      {
+        operands[opNo++] = builder.paramValue();
+      }
+      operands[opNo] = 0;
+      op.m_op = builder.readTuple(oi.m_index, oi.m_table, operands);
+      break;
     }
   }
   else
@@ -365,7 +492,7 @@ loop:
         if (oi.m_table->getColumn(a)->getPrimaryKey())
         {
           operands[opNo++] = createLink(builder, oi.m_table->getColumn(a),
-                                        parents);
+                                        parents, false);
         }
       }
       operands[opNo] = 0;
@@ -374,15 +501,54 @@ loop:
       op.m_op = builder.readTuple(oi.m_table, operands);
       break;
     }
+    case NdbQueryOperationDef::UniqueIndexAccess: {
+      int opNo = 0;
+      NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+      for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+      {
+        operands[opNo++] = 
+          createLink(builder, 
+                     oi.m_table->getColumn(oi.m_index->getColumn(a)->getName()),
+                     parents, false);
+      }
+      operands[opNo] = 0;
+
+      op.m_parent = parents[0].m_idx;
+      op.m_op = builder.readTuple(oi.m_index, oi.m_table, operands);
+      break;
+    }
     case NdbQueryOperationDef::TableScan:
       // not supported
       abort();
-    case NdbQueryOperationDef::OrderedIndexScan:
-    case NdbQueryOperationDef::UniqueIndexAccess:
-      // TODO
-      abort();
-    }
+    case NdbQueryOperationDef::OrderedIndexScan:{
+      int opNo = 0;
+      NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+      for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+      {
+        operands[opNo++] = 
+          createLink(builder, 
+                     oi.m_table->getColumn(oi.m_index->getColumn(a)->getName()),
+                     parents, true);
+      }
+      operands[opNo] = 0;
 
+      op.m_parent = parents[0].m_idx;
+      NdbQueryIndexBound bounds(operands); // Only EQ for now
+      op.m_op = builder.scanIndex(oi.m_index, oi.m_table, &bounds);
+      if (op.m_op == 0)
+      {
+        ndbout << "Failed to add to " << endl;
+        for (size_t i = 0; i<m_query.size(); i++)
+          ndbout << m_query[i] << endl;
+
+        ndbout << "Parents: " << endl;
+        for (size_t i = 0; i<parents.size(); i++)
+          ndbout << parents[i].m_idx << " ";
+        ndbout << endl;
+      }
+      break;
+    }
+    }
   }
 
   if (op.m_op == 0)
@@ -414,11 +580,19 @@ HugoQueryBuilder::createQuery(Ndb* pNdb,
   }
 
   /**
-   * We only support lookups as child operations...
+   * We only don't support table scans as child operations...
    */
   OptionMask save = m_options;
-  clearOption(O_ORDERED_INDEX);
   clearOption(O_TABLE_SCAN);
+  
+  /**
+   * Iff root is lookup...ordered index scans are not allowed as
+   *   children
+   */
+  if (!isScan(m_query[0].m_op))
+  {
+    clearOption(O_ORDERED_INDEX);
+  }
 
   int levels = getJoinLevel();
   while (levels --)

=== modified file 'storage/ndb/test/tools/hugoJoin.cpp'
--- a/storage/ndb/test/tools/hugoJoin.cpp	2010-09-24 08:13:18 +0000
+++ b/storage/ndb/test/tools/hugoJoin.cpp	2010-11-09 13:03:09 +0000
@@ -41,7 +41,7 @@ int _loops = 100;
 int _loops_per_query = 100;
 int _depth = 4;
 unsigned int _seed = 0;
-static const char * _options = "lookup";
+static const char * _options = "";
 static const char * _db = "TEST_DB";
 
 extern const char *load_default_groups[];
@@ -188,11 +188,17 @@ int main(int argc, char** argv){
   {
     if (_verbose >= 1)
     {
-      ndbout << "\tbuilding new query" << endl;
+      ndbout << "******\tbuilding new query (mask: 0x" << hex 
+             << (Uint64)mask << ")" << endl;
     }
     HugoQueryBuilder builder(&MyNdb, tables.getBase(), mask);
     builder.setJoinLevel(_depth);
     const NdbQueryDef * q = builder.createQuery(&MyNdb);
+    if (_verbose >= 2)
+    {
+      q->print(); ndbout << endl;
+    }
+
     for (int j = 0; j < _loops_per_query && ((_loops == 0) || (i < _loops));
          i++, j++)
     {


Attachment: [text/bzr-bundle] bzr/jonas@mysql.com-20101109130309-c3spruu6bipzjy7q.bundle
Thread
bzr commit into mysql-5.1-telco-7.0-spj-scan-vs-scan branch (jonas:3354) Jonas Oreland9 Nov