ok
Jonathan Miller skrev:
> Hi,
>
> I have made the changes you have suggested, plus extracted some of the code
> in to DbUtil class.
>
> http://lists.mysql.com/commits/40775
> http://lists.mysql.com/commits/40776
>
> I have pushed into new-ndb so that you can add your helper code to DbUtil
> class.
>
> I hope we will have time to review and plan expansion next week in Florida.
>
> Thanks for your time and feedback. Both are very much appreciated.
>
> Best wishes,
> /Jeb
>
> Jonathan Miller
> Austin, Texas USA
> Senior Lead Quality Assurance Developer
> MySQL AB www.mysql.com
> __ ___ ___ ____ __
> / |/ /_ __/ __/ __ \/ /
> / /|_/ / // /\ \/ /_/ / /__
> /_/ /_/\_, /___/\___\_\___/
> <___/ www.mysql.com
>
> Jumpstart your cluster!
> http://www.mysql.com/consulting/packaged/cluster.html
>
> Get training on clusters
> http://www.mysql.com/training/courses/mysql_cluster.html
>
> All-in-one Enterprise-grade Database, Support and Services
> http://www.mysql.com/network/
>
>> -----Original Message-----
>> From: Magnus Svensson [mailto:msvensson@stripped]
>> Sent: Wednesday, January 09, 2008 7:19 AM
>> To: jmiller@stripped
>> Cc: commits@stripped
>> Subject: Re: bk commit into 5.1 tree (jmiller:1.2652)
>>
>> Hi Jeb,
>>
>> approved after fixing comments marked with "magnus:" below.
>>
>> / Magnus
>>
>>
>> jmiller@stripped skrev:
>>> Below is the list of changes that have just been committed into a local
>>> 5.1 repository of jmiller. When jmiller does a push these changes
>>> will be propagated to the main repository and, within 24 hours after the
>>> push, to the public repository.
>>> For information on how to access the public repository
>>> see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html
>>>
>>> ChangeSet@stripped, 2008-01-08 23:34:30+01:00, jmiller@stripped +3 -0
>>> NdbRepStress.cpp:
>>> Updated with suggestions from Magnus and other fixes and adjustments
>> I found along the way
>>> DbUtil.hpp, DbUtil.cpp:
>>> fixes and adjustments I found along the way
>>>
>>> storage/ndb/test/include/DbUtil.hpp@stripped, 2008-01-08 23:34:16+01:00,
>> jmiller@stripped +2 -1
>>> fixes and adjustments I found along the way
>>>
>>> storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp@stripped, 2008-01-08
>> 23:33:29+01:00, jmiller@stripped +487 -0
>>> Updated with suggestions from Magnus and other fixes and adjustments
>> I found along the way
>>> storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp@stripped, 2008-01-08
>> 23:33:29+01:00, jmiller@stripped +0 -0
>>> storage/ndb/test/src/DbUtil.cpp@stripped, 2008-01-08 23:34:08+01:00,
>> jmiller@stripped +38 -4
>>> fixes and adjustments I found along the way
>>>
>>> diff -Nrup a/storage/ndb/test/include/DbUtil.hpp
>> b/storage/ndb/test/include/DbUtil.hpp
>>> --- a/storage/ndb/test/include/DbUtil.hpp 2007-12-20 21:06:41 +01:00
>>> +++ b/storage/ndb/test/include/DbUtil.hpp 2008-01-08 23:34:16 +01:00
>>> @@ -89,7 +89,8 @@ public:
>>> void mysqlCloseStmHandle(MYSQL_STMT *my_stmt);
>>>
>>> int connect();
>>> - int select_DB();
>>> + int selectDb();
>>> + int selectDb(const char * m_db);
>>> int doQuery(char * stm);
>> magnus: ^^ There should be no need for one function that takes
>> "const char*" and one "char*", remove the "char*" one. Or change it to
>> "doQuery(BaseString& str)" and then do str->c_str() in the function...
>>
>>> int doQuery(const char * stm);
>>> int getErrorNumber();
>>> diff -Nrup a/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp
>> b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp
>>> --- /dev/null Wed Dec 31 16:00:00 196900
>>> +++ b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp 2008-01-08 23:33:29
>> +01:00
>>> @@ -0,0 +1,487 @@
>>> +/* Copyright (C) 2003 MySQL AB
>>> +
>>> + This program is free software; you can redistribute it and/or modify
>>> + it under the terms of the GNU General Public License as published by
>>> + the Free Software Foundation; version 2 of the License.
>>> +
>>> + This program is distributed in the hope that it will be useful,
>>> + but WITHOUT ANY WARRANTY; without even the implied warranty of
>>> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>>> + GNU General Public License for more details.
>>> +
>>> + You should have received a copy of the GNU General Public License
>>> + along with this program; if not, write to the Free Software
>>> + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
>> USA */
>>> +
>>> +#include <NDBT_Test.hpp>
>>> +#include <NDBT_ReturnCodes.h>
>>> +#include <HugoTransactions.hpp>
>>> +#include <UtilTransactions.hpp>
>>> +#include <TestNdbEventOperation.hpp>
>>> +#include <DbUtil.hpp>
>>> +#include <NdbAutoPtr.hpp>
>>> +#include <NdbRestarter.hpp>
>>> +#include <NdbRestarts.hpp>
>>> +#include <mysql.h>
>>> +#include <signaldata/DumpStateOrd.hpp>
>> magnus: Remove the ones that are not used. Maybe fixes the "template
>> vector< >" stuff?
>>
>>
>>> +
>>> +static BaseString sqlStm;
>>> +static BaseString db;
>> magnus: You can't have static variables that are updated beween
>> different threads, move them to the functions where they are used.
>> I explained why in the prev mail.
>>
>>> +static int t1_records = 50000;
>> magnus: Suggest you take records from ctx->getNmRecords() to it can be
>> changed from commandline with "testNdbRepStress --records=10"
>>
>>> +
>>> +/**** TOOL SECTION ****/
>>> +
>>> +static uint
>>> +urandom()
>>> +{
>>> + uint r = (uint)random();
>>> + return r;
>>> +}
>>> +
>>> +static uint
>>> +urandom(uint m)
>>> +{
>>> + if (m == 0)
>>> + return 0;
>>> + uint r = urandom();
>>> + r = r % m;
>>> + return r;
>>> +}
>>> +
>>> +#define GETNDB(ps) ((NDBT_NdbApiStep*)ps)->getNdb()
>>> +/*
>>> +*/
>>> +
>>> +int
>>> +syncSlaveWithMaster()
>>> +{
>>> + /*
>>> + We need to look at the MAX epoch of the
>>> + mysql.ndb_binlog_index table so we will
>>> + know when the slave has caught up
>>> + */
>>> +
>>> + MYSQL_RES * result;
>>> + MYSQL_ROW row;
>>> + unsigned int masterEpoch = 0;
>>> + unsigned int slaveEpoch = 0;
>>> + unsigned int slaveEpochOld = 0;
>>> + int maxLoops = 100;
>>> + int loopCnt = 0;
>>> +
>>> + //Create a DbUtil object for the master
>>> + DbUtil master("mysql","");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Set the database we are wanting
>>> + if (master.selectDb())
>>> + {
>> magnus: I think you actually already have selected the db when you
>> created "DbUtil master("mysql","");", what else does "mysql" mean? ;)
>>
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Get max epoch from master
>>> + if(master.doQuery("SELECT MAX(epoch) FROM mysql.ndb_binlog_index"))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + result = mysql_use_result(master.getMysql());
>>> + row = mysql_fetch_row(result);
>>> + masterEpoch = atoi(row[0]);
>> magnus: ^ This stuff(working with result of a SQL query) can be written
>> much easier with new result set in Properties. Will show you in Orlando...
>>
>>> + mysql_free_result(result);
>>> + master.databaseLogout();
>>> +
>>> + /*
>>> + Now we will pull current epoch from slave. If not the
>>> + same as master, we will continue to retrieve the epoch
>>> + and compare until it matches or we reach the max loops
>>> + allowed.
>>> + */
>>> +
>>> + //Create a dbutil object for the slave
>>> + DbUtil slave("mysql",".slave");
>>> +
>>> + //Login to slave
>>> + if (!slave.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Set the database we are wanting
>>> + if (slave.selectDb())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + while(slaveEpoch != masterEpoch && loopCnt < maxLoops)
>>> + {
>>> + if(slave.doQuery("SELECT epoch FROM mysql.ndb_apply_status"))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + result = mysql_use_result(slave.getMysql());
>>> + row = mysql_fetch_row(result);
>>> + slaveEpoch = atoi(row[0]);
>>> + mysql_free_result(result);
>>> +
>>> + if(slaveEpoch != slaveEpochOld)
>>> + {
>>> + slaveEpochOld = slaveEpoch;
>>> + if(loopCnt > 0)
>>> + loopCnt--;
>>> + sleep(3);
>>> + }
>>> + else
>>> + {
>>> + sleep(1);
>>> + loopCnt++;
>>> + }
>>> + }
>>> +
>>> + if(slaveEpoch != masterEpoch)
>>> + {
>>> + g_err << "Slave not in sync with master!" << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +verifySlaveLoad(BaseString *table)
>>> +{
>>> + //First thing to do is sync slave
>>> + if(syncSlaveWithMaster())
>>> + {
>>> + g_err << "Verify Load -> Syncing with slave failed" <<
> endl;
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Now that slave is sync we can verify load
>>> + sqlStm.assfmt("SELECT COUNT(*) FROM %s", table);
>>> + MYSQL_RES * result;
>>> + MYSQL_ROW row;
>>> + unsigned int masterCount = 0;
>>> + unsigned int slaveCount = 0;
>>> +
>>> + //Create a DB Object for the master
>>> + DbUtil master(db.c_str()," ");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if (master.selectDb())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.doQuery(sqlStm.c_str()))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + result = mysql_use_result(master.getMysql());
>>> + row = mysql_fetch_row(result);
>>> + masterCount = atoi(row[0]);
>>> + mysql_free_result(result);
>>> + master.databaseLogout();
>>> +
>>> + //Create a DB Object for slave
>>> + DbUtil slave(db.c_str(),".slave");
>>> +
>>> + //Login to slave
>>> + if (!slave.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if (slave.selectDb())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(slave.doQuery(sqlStm.c_str()))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + result = mysql_use_result(slave.getMysql());
>>> + row = mysql_fetch_row(result);
>>> + slaveCount = atoi(row[0]);
>>> + mysql_free_result(result);
>>> +
>>> + if(slaveCount != masterCount)
>>> + {
>>> + g_err << "Verify Load -> Slave Count != Master Count "
>>> + << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +/**** Test Section ****/
>>> +
>>> +int
>>> +createDB(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + //Setup the BaseString db to use throughout
>>> + db.assign("TEST_DB");
>>> +
>>> + //Create a dbutil object
>>> + DbUtil master("mysql","");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Check to see if db already there
>>> + if (master.selectDb(db.c_str()))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Create TEST_DB
>>> + if (master.doQuery("CREATE DATABASE TEST_DB") != 0)
>> magnus: good! Still lot's of code just to create a db. We should think
>> about a way to minimize this.
>>
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +createTable_rep1(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + BaseString table;
>>> + table.assign("rep1");
>>> +
>>> + //Ensure slave is up and ready
>>> + if(syncSlaveWithMaster() != 0)
>>> + {
>>> + g_err << "Create Table -> Syncing with slave failed"
>>> + << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + //Create an SQL Object
>>> + DbUtil master(db.c_str(),"");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + // Set the database we want
>>> + if(master.selectDb())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + sqlStm.assign("CREATE TABLE rep1 (c1 MEDIUMINT NOT NULL
>> AUTO_INCREMENT,"
>>> + " c2 FLOAT, c3 CHAR(5), c4 bit(8), c5 FLOAT, c6 INT,"
>>> + " c7 INT, PRIMARY KEY (c1))ENGINE=NDB");
>>> +
>>> + if (master.doQuery(sqlStm.c_str()))
>> magnus: Many places where you use sqlStm it's just a constant string. I
>> think you shuould pass that string directly to "doQuery"
>>
>>
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + ctx->setProperty("TABLES",table.c_str());
>>> + HugoTransactions hugoTrans(*ctx->getTab());
>>> +
>>> + if (hugoTrans.loadTable(GETNDB(step), t1_records, 1, true, 0) != 0)
>>> + {
>>> + g_err << "Create Table -> Load failed!" << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(verifySlaveLoad(&table)!= 0)
>>> + {
>>> + g_err << "Create Table -> Failed on verify slave load!"
>>> + << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + //else everything is okay
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +stressNDB_rep1(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + const NdbDictionary::Table * table= ctx->getTab();
>>> + HugoTransactions hugoTrans(* table);
>>> + while(!ctx->isTestStopped())
>>> + {
>>> + if (hugoTrans.pkUpdateRecords(GETNDB(step), t1_records, 1, 30) !=
>> 0)
>>> + {
>>> + g_err << "pkUpdate Failed!" << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + if (hugoTrans.scanUpdateRecords(GETNDB(step), t1_records, 1, 30) !=
>> 0)
>>> + {
>>> + g_err << "scanUpdate Failed!" << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +stressSQL_rep1(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + //Create an SQL Object
>>> + DbUtil master(db.c_str(),"");
>>> + int loops = ctx->getNumLoops();
>>> + uint record = 0;
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.selectDb())
>>> + {
>>> + ctx->stopTest();
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + for (int j= 0; loops == 0 || j < loops; j++)
>>> + {
>>> + record = urandom(t1_records);
>>> + sqlStm.assfmt("UPDATE TEST_DB.rep1 SET c2 = 33.3221 where c1 =
>> %u", record);
>>> + if(master.doQuery(sqlStm.c_str()))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + }
>>> + ctx->stopTest();
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +verifySlave_rep1(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + if(syncSlaveWithMaster() != 0)
>>> + {
>>> + g_err << "Verify Slave rep1 -> Syncing with slave failed"
>>> + << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + //Create SQL Objects
>>> + DbUtil master(db.c_str(),"");
>>> + DbUtil slave(db.c_str(),".slave");
>>> + MYSQL_RES *resource;
>>> + MYSQL_ROW row;
>>> + float masterSum;
>>> + float slaveSum;
>>> +
>>> + sqlStm.assign("SELECT SUM(c3) FROM rep1");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.selectDb())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.doQuery(sqlStm.c_str()))
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + resource = mysql_use_result(master.getMysql());
>>> + row = mysql_fetch_row(resource);
>>> + masterSum = atoi(row[0]);
>>> + mysql_free_result(resource);
>>> + master.databaseLogout();
>>> +
>>> + //Login to slave
>>> + if (!slave.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(slave.selectDb() != 0)
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if((slave.doQuery(sqlStm.c_str())) != 0)
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> + resource = mysql_use_result(slave.getMysql());
>>> + row = mysql_fetch_row(resource);
>>> + slaveSum = atoi(row[0]);
>>> + mysql_free_result(resource);
>>> +
>>> + if(masterSum != slaveSum)
>>> + {
>>> + g_err << "VerifySlave -> masterSum != slaveSum..." <<
> endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +int
>>> +dropTEST_DB(NDBT_Context* ctx, NDBT_Step* step)
>>> +{
>>> + //Create an SQL Object
>>> + DbUtil master(db.c_str(),"");
>>> +
>>> + //Login to Master
>>> + if (!master.connect())
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.selectDb() != 0)
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(master.doQuery("DROP DATABASE TEST_DB") != 0)
>>> + {
>>> + return NDBT_FAILED;
>>> + }
>>> +
>>> + if(syncSlaveWithMaster() != 0)
>>> + {
>>> + g_err << "Drop DB -> Syncing with slave failed"
>>> + << endl;
>>> + return NDBT_FAILED;
>>> + }
>>> + return NDBT_OK;
>>> +}
>>> +
>>> +NDBT_TESTSUITE(NdbRepStress);
>>> +TESTCASE("PHASE_I_Stress","Basic Replication Stressing")
>>> +{
>>> + INITIALIZER(createDB);
>>> + INITIALIZER(createTable_rep1);
>>> + STEP(stressNDB_rep1);
>>> + STEP(stressSQL_rep1);
>>> + FINALIZER(verifySlave_rep1);
>>> + FINALIZER(dropTEST_DB);
>>> +}
>> magnus: I think it's important that we can "ramp" up. Start with some
>> simple stuff and if that works go to the next level, and so on...
>>
>>> +NDBT_TESTSUITE_END(NdbRepStress);
>>> +
>>> +int main(int argc, const char** argv){
>>> + ndb_init();
>>> + NdbRepStress.setCreateAllTables(true);
>>> + return NdbRepStress.execute(argc, argv);
>>> +}
>>> +
>>> +template class Vector<HugoOperations *>;
>>> +template class Vector<NdbEventOperation *>;
>>> +template class Vector<NdbRecAttr*>;
>>> +template class Vector<Vector<NdbRecAttr*> >;
>> magnus: Huh? Must be something wrong if you need this.
>>
>>> diff -Nrup a/storage/ndb/test/src/DbUtil.cpp
>> b/storage/ndb/test/src/DbUtil.cpp
>>> --- a/storage/ndb/test/src/DbUtil.cpp 2007-12-20 21:06:26 +01:00
>>> +++ b/storage/ndb/test/src/DbUtil.cpp 2008-01-08 23:34:08 +01:00
>>> @@ -215,23 +215,57 @@ DbUtil::printStError(MYSQL_STMT *stmt, c
>>> /* Select which database to use */
>>>
>>> int
>>> -DbUtil::select_DB()
>>> +DbUtil::selectDb()
>>> {
>>> - return mysql_select_db(this->getMysql(), this->getDbName());
>>> + if ((this->getDbName()) != NULL)
>>> + {
>>> + if(mysql_select_db(this->getMysql(), this->getDbName()))
>>> + {
>>> + this->printError("mysql_select_db failed");
>> magnus: Very good. When using the class it should automatically print
>> the error messages to avoid clutter in the testcases.
>>
>>> + return DBU_FAILED;
>>> + }
>>> + return DBU_OK;
>>> + }
>>> + this->printError("getDbName() == NULL");
>>> + return DBU_FAILED;
>>> }
>>>
>>> +int
>>> +DbUtil::selectDb(const char * m_db)
>>> +{
>>> + {
>>> + if(mysql_select_db(this->getMysql(), m_db))
>>> + {
>>> + this->printError("mysql_select_db failed");
>>> + return DBU_FAILED;
>>> + }
>>> + return DBU_OK;
>>> + }
>>> +}
>>> +
>>> +
>>> /* Run Simple Queries */
>>>
>>> int
>>> DbUtil::doQuery(char * stm)
>>> {
>>> - return mysql_query(this->getMysql(), stm);
>>> + if(mysql_query(this->getMysql(), stm))
>>> + {
>>> + this->printError(stm);
>>> + return DBU_FAILED;
>>> + }
>>> + return DBU_OK;
>>> }
>>>
>>> int
>>> DbUtil::doQuery(const char * stm)
>>> {
>>> - return mysql_query(this->getMysql(), stm);
>>> + if(mysql_query(this->getMysql(), stm))
>>> + {
>>> + this->printError(stm);
>>> + return DBU_FAILED;
>>> + }
>>> + return DBU_OK;
>>> }
>>>
>>> /* Return MySQL Error String */
>>>
>