#At file:///home/jonas/src/70-spj-svs/ based on revid:jonas@stripped
3354 Jonas Oreland 2010-11-09
ndb spj-svs - Add support for scan-scan queries to Hugo
modified:
storage/ndb/test/include/HugoQueries.hpp
storage/ndb/test/include/HugoQueryBuilder.hpp
storage/ndb/test/src/HugoQueries.cpp
storage/ndb/test/src/HugoQueryBuilder.cpp
storage/ndb/test/tools/hugoJoin.cpp
=== modified file 'storage/ndb/test/include/HugoQueries.hpp'
--- a/storage/ndb/test/include/HugoQueries.hpp 2010-05-20 11:18:08 +0000
+++ b/storage/ndb/test/include/HugoQueries.hpp 2010-11-09 13:03:09 +0000
@@ -27,6 +27,13 @@
class HugoQueries
{
+ struct Op
+ {
+ const NdbQueryOperationDef* m_query_op;
+ Vector<NDBT_ResultRow*> m_rows;
+ HugoCalculator * m_calc;
+ };
+
public:
HugoQueries(const NdbQueryDef & query);
virtual ~HugoQueries();
@@ -41,7 +48,7 @@ public:
int scan_flags = 0);
static int equalForParameters(char * buf,
- HugoCalculator&,
+ Op&,
NdbQueryParamValue params[],
int rowNo);
static int getValueForQueryOp(NdbQueryOperation* pOp, NDBT_ResultRow* pRow);
@@ -51,13 +58,6 @@ public:
protected:
const NdbQueryDef* m_query_def;
- struct Op
- {
- const NdbQueryOperationDef* m_query_op;
- Vector<NDBT_ResultRow*> m_rows;
- HugoCalculator * m_calc;
- };
-
Vector<Op> m_ops;
int m_retryMax;
};
=== modified file 'storage/ndb/test/include/HugoQueryBuilder.hpp'
--- a/storage/ndb/test/include/HugoQueryBuilder.hpp 2010-07-02 09:06:49 +0000
+++ b/storage/ndb/test/include/HugoQueryBuilder.hpp 2010-11-09 13:03:09 +0000
@@ -146,14 +146,24 @@ private:
* ops
*/
static bool checkBindable(Vector<const NdbDictionary::Column*> cols,
- Vector<Op> ops);
+ Vector<Op> ops,
+ bool allow_bind_nullable);
Vector<Op> getParents(OpIdx); //
NdbQueryOperand * createLink(NdbQueryBuilder&, const NdbDictionary::Column*,
- Vector<Op> & parents);
+ Vector<Op> & parents,
+ bool allow_bind_nullable);
const NdbQueryOperationDef* createOp(NdbQueryBuilder&);
void fixOptions();
+
+ /**
+ * We currently don't support busy-scan joins
+ */
+ bool checkBusyScan(Op) const;
+ bool isAncestor(const Op& parent, const Op& child) const;
+
+ friend NdbOut& operator<<(NdbOut& out, const HugoQueryBuilder::Op& op);
};
#endif
=== modified file 'storage/ndb/test/src/HugoQueries.cpp'
--- a/storage/ndb/test/src/HugoQueries.cpp 2010-05-20 11:18:08 +0000
+++ b/storage/ndb/test/src/HugoQueries.cpp 2010-11-09 13:03:09 +0000
@@ -74,21 +74,46 @@ HugoQueries::allocRows(int batch)
int
HugoQueries::equalForParameters(char * buf,
- HugoCalculator& calc,
+ Op & op,
NdbQueryParamValue params[],
int rowNo)
{
Uint32 no = 0;
+ HugoCalculator & calc = * op.m_calc;
const NdbDictionary::Table & tab = calc.getTable();
- for (int i = 0; i<tab.getNoOfColumns(); i++)
+ if (op.m_query_op->getType() == NdbQueryOperationDef::TableScan)
{
- const NdbDictionary::Column* attr = tab.getColumn(i);
- if (attr->getPrimaryKey())
+
+ }
+ else if (op.m_query_op->getType() == NdbQueryOperationDef::PrimaryKeyAccess)
+ {
+ for (int i = 0; i<tab.getNoOfColumns(); i++)
{
+ const NdbDictionary::Column* attr = tab.getColumn(i);
+ if (attr->getPrimaryKey())
+ {
+ Uint32 len = attr->getSizeInBytes();
+ Uint32 real_len;
+ bzero(buf, len);
+ calc.calcValue((Uint32)rowNo, i, 0, buf, len, &real_len);
+ params[no++]= NdbQueryParamValue((void*)buf);
+ buf += len;
+ }
+ }
+ }
+ else if (op.m_query_op->getType() == NdbQueryOperationDef::UniqueIndexAccess||
+ op.m_query_op->getType() == NdbQueryOperationDef::OrderedIndexScan)
+ {
+ const NdbDictionary::Index* idx = op.m_query_op->getIndex();
+ for (unsigned i = 0; i < idx->getNoOfColumns(); i++)
+ {
+ const NdbDictionary::Column* attr =
+ tab.getColumn(idx->getColumn(i)->getName());
Uint32 len = attr->getSizeInBytes();
Uint32 real_len;
bzero(buf, len);
- calc.calcValue((Uint32)rowNo, i, 0, buf, len, &real_len);
+ calc.calcValue((Uint32)rowNo, attr->getColumnNo(),
+ 0, buf, len, &real_len);
params[no++]= NdbQueryParamValue((void*)buf);
buf += len;
}
@@ -163,7 +188,7 @@ HugoQueries::runLookupQuery(Ndb* pNdb,
{
char buf[NDB_MAX_TUPLE_SIZE];
NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
- equalForParameters(buf, * m_ops[0].m_calc, params, b + r);
+ equalForParameters(buf, m_ops[0], params, b + r);
NdbQuery * query = pTrans->createQuery(m_query_def, params);
if (query == 0)
@@ -256,9 +281,10 @@ HugoQueries::runScanQuery(Ndb * pNdb,
}
NdbQuery * query = 0;
- //char buf[NDB_MAX_TUPLE_SIZE];
- NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
+ char buf[NDB_MAX_TUPLE_SIZE];
+ NdbQueryParamValue params[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY];
+ equalForParameters(buf, m_ops[0], params, /* rowNo */ 0);
query = pTrans->createQuery(m_query_def, params);
if (query == 0)
{
=== modified file 'storage/ndb/test/src/HugoQueryBuilder.cpp'
--- a/storage/ndb/test/src/HugoQueryBuilder.cpp 2010-07-02 09:06:49 +0000
+++ b/storage/ndb/test/src/HugoQueryBuilder.cpp 2010-11-09 13:03:09 +0000
@@ -18,6 +18,34 @@
#include <HugoQueryBuilder.hpp>
+static
+bool
+isScan(const NdbQueryOperationDef* def)
+{
+ return
+ def->getType() == NdbQueryOperationDef::TableScan ||
+ def->getType() == NdbQueryOperationDef::OrderedIndexScan;
+}
+
+NdbOut&
+operator<<(NdbOut& out, const HugoQueryBuilder::Op& op)
+{
+ out << "[" << op.m_idx << " : " << op.m_op->getTable()->getName() << ": ";
+ switch(op.m_op->getType()){
+ case NdbQueryOperationDef::TableScan:
+ out << "table-scan";
+ break;
+ case NdbQueryOperationDef::OrderedIndexScan:
+ out << "index-scan";
+ break;
+ default:
+ out << "lookup";
+ }
+
+ out << " : parent: " << op.m_parent << " ]";
+ return out;
+}
+
void
HugoQueryBuilder::init()
{
@@ -58,9 +86,28 @@ HugoQueryBuilder::addTable(Ndb* ndb, con
TableDef def;
def.m_table = tab;
- /**
- * TODO discover indexes...
- */
+ NdbDictionary::Dictionary* pDict = ndb->getDictionary();
+ NdbDictionary::Dictionary::List l;
+ int res = pDict->listIndexes(l, tab->getName());
+ if (res == 0)
+ {
+ for (unsigned i = 0; i<l.count; i++)
+ {
+ const NdbDictionary::Index * idx = pDict->getIndex(l.elements[i].name,
+ tab->getName());
+ if (idx)
+ {
+ if (idx->getType() == NdbDictionary::Index::UniqueHashIndex)
+ {
+ def.m_unique_indexes.push_back(idx);
+ }
+ else if (idx->getType() == NdbDictionary::Index::OrderedIndex)
+ {
+ def.m_ordered_indexes.push_back(idx);
+ }
+ }
+ }
+ }
m_tables.push_back(def);
}
@@ -166,7 +213,8 @@ found:
bool
HugoQueryBuilder::checkBindable(Vector<const NdbDictionary::Column*> cols,
- Vector<Op> ops)
+ Vector<Op> ops,
+ bool allow_bind_nullable)
{
for (size_t c = 0; c < cols.size(); c++)
{
@@ -179,7 +227,9 @@ HugoQueryBuilder::checkBindable(Vector<c
{
for (int i = 0; i<tab->getNoOfColumns(); i++)
{
- if (col->isBindable(* tab->getColumn(i)) == 0)
+ if (!allow_bind_nullable && tab->getColumn(i)->getNullable())
+ continue;
+ else if (col->isBindable(* tab->getColumn(i)) == 0)
{
found = true;
break;
@@ -193,6 +243,44 @@ HugoQueryBuilder::checkBindable(Vector<c
return true;
}
+bool
+HugoQueryBuilder::isAncestor(const Op& parent, const Op& child) const
+{
+ int pi = parent.m_idx;
+ int ci = child.m_idx;
+ assert(ci != pi);
+
+ while (ci != 0)
+ {
+ if (m_query[ci].m_parent == pi)
+ return true;
+ assert(m_query[ci].m_parent != -1);
+ ci = m_query[m_query[ci].m_parent].m_idx;
+ }
+ return false;
+}
+
+bool
+HugoQueryBuilder::checkBusyScan(Op op) const
+{
+ /**
+ * Iterate upwards until we find first scan...
+ */
+ while (op.m_parent != -1)
+ {
+ if (isScan(op.m_op))
+ {
+ break;
+ }
+ op = m_query[op.m_parent];
+ }
+
+ for (size_t i = op.m_idx + 1; i < m_query.size(); i++)
+ if (isAncestor(op, m_query[i]) && isScan(m_query[i].m_op))
+ return true;
+
+ return false;
+}
Vector<HugoQueryBuilder::Op>
HugoQueryBuilder::getParents(OpIdx oi)
@@ -200,6 +288,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
/**
* We need to be able to bind all columns in table/index
*/
+ bool allow_bind_nullable = false;
+ bool check_bushy_scan = false;
Vector<const NdbDictionary::Column*> cols;
if (oi.m_index == 0)
{
@@ -207,9 +297,24 @@ HugoQueryBuilder::getParents(OpIdx oi)
if (oi.m_table->getColumn(i)->getPrimaryKey())
cols.push_back(oi.m_table->getColumn(i));
}
- else
+ else if (oi.m_index->getType() == NdbDictionary::Index::UniqueHashIndex)
+ {
+ for (unsigned i = 0; i < oi.m_index->getNoOfColumns(); i++)
+ cols.push_back(oi.m_table->getColumn
+ (oi.m_index->getColumn(i)->getName()));
+ }
+ else if (oi.m_index->getType() == NdbDictionary::Index::OrderedIndex)
{
- abort(); // TODO
+ /**
+ * Binding a prefix is ok...but skip this for now...
+ */
+ allow_bind_nullable = true;
+ check_bushy_scan = true;
+ unsigned cnt = oi.m_index->getNoOfColumns();
+ unsigned val = cnt; // cnt > 1 ? (1 + (rand() % (cnt - 1))) : cnt;
+ for (unsigned i = 0; i < val; i++)
+ cols.push_back(oi.m_table->getColumn
+ (oi.m_index->getColumn(i)->getName()));
}
int r = rand() % m_query.size();
@@ -218,6 +323,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
{
Vector<Op> set;
Op op = m_query[(i + r) % cnt];
+ if (check_bushy_scan && checkBusyScan(op))
+ continue;
set.push_back(op);
#if 0
@@ -234,7 +341,7 @@ HugoQueryBuilder::getParents(OpIdx oi)
}
#endif
- if (checkBindable(cols, set))
+ if (checkBindable(cols, set, allow_bind_nullable))
return set;
}
@@ -245,7 +352,8 @@ HugoQueryBuilder::getParents(OpIdx oi)
NdbQueryOperand *
HugoQueryBuilder::createLink(NdbQueryBuilder& builder,
const NdbDictionary::Column* pCol,
- Vector<Op> & parents)
+ Vector<Op> & parents,
+ bool allow_bind_nullable)
{
int cnt = (int)parents.size();
int r = rand();
@@ -292,6 +400,9 @@ HugoQueryBuilder::createLink(NdbQueryBui
// already checked
continue;
}
+ if (!allow_bind_nullable && col->getNullable())
+ continue;
+
if (pCol->isBindable(* col) == 0)
{
goto found;
@@ -338,12 +449,28 @@ HugoQueryBuilder::createOp(NdbQueryBuild
case NdbQueryOperationDef::TableScan:
op.m_op = builder.scanTable(oi.m_table);
break;
- case NdbQueryOperationDef::OrderedIndexScan:
- abort();
+ case NdbQueryOperationDef::OrderedIndexScan:{
+ int opNo = 0;
+ NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+ for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+ {
+ operands[opNo++] = builder.paramValue();
+ }
+ operands[opNo] = 0;
+ NdbQueryIndexBound bounds(operands);
+ op.m_op = builder.scanIndex(oi.m_index, oi.m_table, &bounds);
break;
+ }
case NdbQueryOperationDef::UniqueIndexAccess:
- // TODO
- abort();
+ int opNo = 0;
+ NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+ for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+ {
+ operands[opNo++] = builder.paramValue();
+ }
+ operands[opNo] = 0;
+ op.m_op = builder.readTuple(oi.m_index, oi.m_table, operands);
+ break;
}
}
else
@@ -365,7 +492,7 @@ loop:
if (oi.m_table->getColumn(a)->getPrimaryKey())
{
operands[opNo++] = createLink(builder, oi.m_table->getColumn(a),
- parents);
+ parents, false);
}
}
operands[opNo] = 0;
@@ -374,15 +501,54 @@ loop:
op.m_op = builder.readTuple(oi.m_table, operands);
break;
}
+ case NdbQueryOperationDef::UniqueIndexAccess: {
+ int opNo = 0;
+ NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+ for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+ {
+ operands[opNo++] =
+ createLink(builder,
+ oi.m_table->getColumn(oi.m_index->getColumn(a)->getName()),
+ parents, false);
+ }
+ operands[opNo] = 0;
+
+ op.m_parent = parents[0].m_idx;
+ op.m_op = builder.readTuple(oi.m_index, oi.m_table, operands);
+ break;
+ }
case NdbQueryOperationDef::TableScan:
// not supported
abort();
- case NdbQueryOperationDef::OrderedIndexScan:
- case NdbQueryOperationDef::UniqueIndexAccess:
- // TODO
- abort();
- }
+ case NdbQueryOperationDef::OrderedIndexScan:{
+ int opNo = 0;
+ NdbQueryOperand * operands[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY + 1];
+ for (unsigned a = 0; a<oi.m_index->getNoOfColumns(); a++)
+ {
+ operands[opNo++] =
+ createLink(builder,
+ oi.m_table->getColumn(oi.m_index->getColumn(a)->getName()),
+ parents, true);
+ }
+ operands[opNo] = 0;
+ op.m_parent = parents[0].m_idx;
+ NdbQueryIndexBound bounds(operands); // Only EQ for now
+ op.m_op = builder.scanIndex(oi.m_index, oi.m_table, &bounds);
+ if (op.m_op == 0)
+ {
+ ndbout << "Failed to add to " << endl;
+ for (size_t i = 0; i<m_query.size(); i++)
+ ndbout << m_query[i] << endl;
+
+ ndbout << "Parents: " << endl;
+ for (size_t i = 0; i<parents.size(); i++)
+ ndbout << parents[i].m_idx << " ";
+ ndbout << endl;
+ }
+ break;
+ }
+ }
}
if (op.m_op == 0)
@@ -414,11 +580,19 @@ HugoQueryBuilder::createQuery(Ndb* pNdb,
}
/**
- * We only support lookups as child operations...
+ * We only don't support table scans as child operations...
*/
OptionMask save = m_options;
- clearOption(O_ORDERED_INDEX);
clearOption(O_TABLE_SCAN);
+
+ /**
+ * Iff root is lookup...ordered index scans are not allowed as
+ * children
+ */
+ if (!isScan(m_query[0].m_op))
+ {
+ clearOption(O_ORDERED_INDEX);
+ }
int levels = getJoinLevel();
while (levels --)
=== modified file 'storage/ndb/test/tools/hugoJoin.cpp'
--- a/storage/ndb/test/tools/hugoJoin.cpp 2010-09-24 08:13:18 +0000
+++ b/storage/ndb/test/tools/hugoJoin.cpp 2010-11-09 13:03:09 +0000
@@ -41,7 +41,7 @@ int _loops = 100;
int _loops_per_query = 100;
int _depth = 4;
unsigned int _seed = 0;
-static const char * _options = "lookup";
+static const char * _options = "";
static const char * _db = "TEST_DB";
extern const char *load_default_groups[];
@@ -188,11 +188,17 @@ int main(int argc, char** argv){
{
if (_verbose >= 1)
{
- ndbout << "\tbuilding new query" << endl;
+ ndbout << "******\tbuilding new query (mask: 0x" << hex
+ << (Uint64)mask << ")" << endl;
}
HugoQueryBuilder builder(&MyNdb, tables.getBase(), mask);
builder.setJoinLevel(_depth);
const NdbQueryDef * q = builder.createQuery(&MyNdb);
+ if (_verbose >= 2)
+ {
+ q->print(); ndbout << endl;
+ }
+
for (int j = 0; j < _loops_per_query && ((_loops == 0) || (i < _loops));
i++, j++)
{
Attachment: [text/bzr-bundle] bzr/jonas@mysql.com-20101109130309-c3spruu6bipzjy7q.bundle
| Thread |
|---|
| • bzr commit into mysql-5.1-telco-7.0-spj-scan-vs-scan branch (jonas:3354) | Jonas Oreland | 9 Nov |
