Re: bk commit into 5.1 tree (jmiller:1.2652)

[Magnus Svensson]

Hi Jeb,

approved after fixing comments marked with "magnus:" below.

/ Magnus


jmiller@stripped skrev:
> Below is the list of changes that have just been committed into a local
> 5.1 repository of jmiller.  When jmiller does a push these changes
> will be propagated to the main repository and, within 24 hours after the
> push, to the public repository.
> For information on how to access the public repository
> see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html
> 
> ChangeSet@stripped, 2008-01-08 23:34:30+01:00, jmiller@stripped +3 -0
>   NdbRepStress.cpp:
>     Updated with suggestions from Magnus and other fixes and adjustments I found
> along the way
>   DbUtil.hpp, DbUtil.cpp:
>     fixes and adjustments I found along the way
> 
>   storage/ndb/test/include/DbUtil.hpp@stripped, 2008-01-08 23:34:16+01:00,
> jmiller@stripped +2 -1
>     fixes and adjustments I found along the way
> 
>   storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp@stripped, 2008-01-08 23:33:29+01:00,
> jmiller@stripped +487 -0
>     Updated with suggestions from Magnus and other fixes and adjustments I found
> along the way
> 
>   storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp@stripped, 2008-01-08 23:33:29+01:00,
> jmiller@stripped +0 -0
> 
>   storage/ndb/test/src/DbUtil.cpp@stripped, 2008-01-08 23:34:08+01:00, jmiller@stripped
> +38 -4
>     fixes and adjustments I found along the way
> 
> diff -Nrup a/storage/ndb/test/include/DbUtil.hpp
> b/storage/ndb/test/include/DbUtil.hpp
> --- a/storage/ndb/test/include/DbUtil.hpp	2007-12-20 21:06:41 +01:00
> +++ b/storage/ndb/test/include/DbUtil.hpp	2008-01-08 23:34:16 +01:00
> @@ -89,7 +89,8 @@ public:
>    void mysqlCloseStmHandle(MYSQL_STMT *my_stmt);
>  
>    int connect();
> -  int select_DB();
> +  int selectDb();
> +  int selectDb(const char * m_db);
>    int doQuery(char * stm);
magnus:          ^^ There should be no need for one function that takes 
"const char*" and one "char*", remove the "char*"  one. Or change it to 
"doQuery(BaseString& str)" and then do str->c_str() in the function...

>    int doQuery(const char * stm);

>    int getErrorNumber();
> diff -Nrup a/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp
> b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp
> --- /dev/null	Wed Dec 31 16:00:00 196900
> +++ b/storage/ndb/test/ndbapi/acrt/NdbRepStress.cpp	2008-01-08 23:33:29 +01:00
> @@ -0,0 +1,487 @@
> +/* Copyright (C) 2003 MySQL AB
> +
> +   This program is free software; you can redistribute it and/or modify
> +   it under the terms of the GNU General Public License as published by
> +   the Free Software Foundation; version 2 of the License.
> +
> +   This program is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +   GNU General Public License for more details.
> +
> +   You should have received a copy of the GNU General Public License
> +   along with this program; if not, write to the Free Software
> +   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
> +
> +#include <NDBT_Test.hpp>
> +#include <NDBT_ReturnCodes.h>
> +#include <HugoTransactions.hpp>
> +#include <UtilTransactions.hpp>
> +#include <TestNdbEventOperation.hpp>
> +#include <DbUtil.hpp>
> +#include <NdbAutoPtr.hpp>
> +#include <NdbRestarter.hpp>
> +#include <NdbRestarts.hpp>
> +#include <mysql.h>
> +#include <signaldata/DumpStateOrd.hpp>
magnus: Remove the ones that are not used. Maybe fixes the "template 
vector< >" stuff?


> +
> +static  BaseString sqlStm;
> +static  BaseString db;
magnus: You can't have static variables that are updated beween 
different threads, move them to the functions where they are used.
I explained why in the prev mail.

> +static  int t1_records = 50000;
magnus: Suggest you take records from ctx->getNmRecords() to it can be 
changed from commandline with "testNdbRepStress --records=10"

> +
> +/**** TOOL SECTION ****/
> +
> +static uint
> +urandom()
> +{
> +  uint r = (uint)random();
> +  return r;
> +}
> +
> +static uint
> +urandom(uint m)
> +{
> +  if (m == 0)
> +    return 0;
> +  uint r = urandom();
> +  r = r % m;
> +  return r;
> +}
> +
> +#define GETNDB(ps) ((NDBT_NdbApiStep*)ps)->getNdb()
> +/*
> +*/
> +
> +int 
> +syncSlaveWithMaster()
> +{
> +  /* 
> +     We need to look at the MAX epoch of the
> +     mysql.ndb_binlog_index table so we will
> +     know when the slave has caught up
> +  */
> +
> +  MYSQL_RES * result;
> +  MYSQL_ROW   row;
> +  unsigned int masterEpoch = 0;
> +  unsigned int slaveEpoch = 0;
> +  unsigned int slaveEpochOld = 0;
> +  int maxLoops = 100;
> +  int loopCnt = 0;
> +
> +  //Create a DbUtil object for the master
> +  DbUtil master("mysql","");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  } 
> +
> +  //Set the database we are wanting 
> +  if (master.selectDb())
> +  {

magnus: I think you actually already have selected the db when you 
created "DbUtil master("mysql","");", what else does "mysql" mean? ;)

> +    return NDBT_FAILED;
> +  }
> +
> +  //Get max epoch from master
> +  if(master.doQuery("SELECT MAX(epoch) FROM mysql.ndb_binlog_index"))
> +  {
> +    return NDBT_FAILED;
> +  }
> +  result = mysql_use_result(master.getMysql());
> +  row    = mysql_fetch_row(result);
> +  masterEpoch = atoi(row[0]);

magnus: ^ This stuff(working with result of a SQL query) can be written 
much easier with new result set in Properties. Will show you in Orlando...

> +  mysql_free_result(result);
> +  master.databaseLogout();
> +
> +  /*
> +     Now we will pull current epoch from slave. If not the
> +     same as master, we will continue to retrieve the epoch
> +     and compare until it matches or we reach the max loops
> +     allowed.
> +  */
> +
> +  //Create a dbutil object for the slave
> +  DbUtil slave("mysql",".slave");
> +
> +  //Login to slave
> +  if (!slave.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  //Set the database we are wanting
> +  if (slave.selectDb())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  while(slaveEpoch != masterEpoch && loopCnt < maxLoops)
> +  {
> +    if(slave.doQuery("SELECT epoch FROM mysql.ndb_apply_status"))
> +    {
> +      return NDBT_FAILED;
> +    }
> +    result = mysql_use_result(slave.getMysql());
> +    row    = mysql_fetch_row(result);
> +    slaveEpoch = atoi(row[0]);
> +    mysql_free_result(result);
> +
> +    if(slaveEpoch != slaveEpochOld)
> +    {
> +      slaveEpochOld = slaveEpoch;
> +      if(loopCnt > 0)
> +        loopCnt--;
> +      sleep(3);
> +    }
> +    else
> +    {
> +      sleep(1);
> +      loopCnt++;
> +    }
> +  }
> +
> +  if(slaveEpoch != masterEpoch)
> +  {
> +    g_err << "Slave not in sync with master!" << endl;
> +    return NDBT_FAILED;
> +  }
> +  return NDBT_OK;
> +}
> +
> +int
> +verifySlaveLoad(BaseString *table)
> +{
> +  //First thing to do is sync slave
> +  if(syncSlaveWithMaster())
> +  {
> +    g_err << "Verify Load -> Syncing with slave failed" << endl;
> +    return NDBT_FAILED;
> +  }
> +
> +  //Now that slave is sync we can verify load
> +  sqlStm.assfmt("SELECT COUNT(*) FROM %s", table);
> +  MYSQL_RES * result;
> +  MYSQL_ROW   row;
> +  unsigned int masterCount = 0;
> +  unsigned int slaveCount = 0;
> +
> +  //Create a DB Object for the master
> +  DbUtil master(db.c_str()," ");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if (master.selectDb())
> +  {
> +    return NDBT_FAILED;
> +  }
> + 
> +  if(master.doQuery(sqlStm.c_str()))
> +  {
> +    return NDBT_FAILED;
> +  }
> +  result = mysql_use_result(master.getMysql());
> +  row    = mysql_fetch_row(result);
> +  masterCount = atoi(row[0]);
> +  mysql_free_result(result);
> +  master.databaseLogout();
> +
> +  //Create a DB Object for slave
> +  DbUtil slave(db.c_str(),".slave");
> +
> +  //Login to slave
> +  if (!slave.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if (slave.selectDb())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if(slave.doQuery(sqlStm.c_str()))
> +  {
> +    return NDBT_FAILED;
> +  }
> +  result = mysql_use_result(slave.getMysql());
> +  row    = mysql_fetch_row(result);
> +  slaveCount = atoi(row[0]);
> +  mysql_free_result(result);
> +
> +  if(slaveCount != masterCount)
> +  {
> +    g_err << "Verify Load -> Slave Count != Master Count "
> +          << endl;
> +    return NDBT_FAILED;
> +  }
> +  return NDBT_OK;
> +}
> +
> +/**** Test Section ****/
> +
> +int 
> +createDB(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  //Setup the BaseString db to use throughout
> +  db.assign("TEST_DB");
> +
> +  //Create a dbutil object
> +  DbUtil master("mysql","");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  //Check to see if db already there
> +  if (master.selectDb(db.c_str()))
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  //Create TEST_DB
> +  if (master.doQuery("CREATE DATABASE TEST_DB") != 0)

magnus: good! Still lot's of code just to create a db. We should think 
about a way to minimize this.

> +  {
> +    return NDBT_FAILED;
> +  }
> +  return NDBT_OK;
> +}
> +
> +int
> +createTable_rep1(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  BaseString table;
> +  table.assign("rep1");
> +
> +  //Ensure slave is up and ready
> +  if(syncSlaveWithMaster() != 0)
> +  {
> +    g_err << "Create Table -> Syncing with slave failed"
> +          << endl;
> +    return NDBT_FAILED;
> +  }
> +
> +  //Create an SQL Object
> +  DbUtil master(db.c_str(),"");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  // Set the database we want
> +  if(master.selectDb())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  sqlStm.assign("CREATE TABLE rep1 (c1 MEDIUMINT NOT NULL AUTO_INCREMENT,"
> +                " c2 FLOAT, c3 CHAR(5), c4 bit(8), c5 FLOAT, c6 INT,"
> +                " c7 INT, PRIMARY KEY (c1))ENGINE=NDB");
> + 
> +  if (master.doQuery(sqlStm.c_str()))

magnus: Many places where you use sqlStm it's just a constant string. I 
think you shuould pass that string directly to "doQuery"


> +  {
> +    return NDBT_FAILED;
> +  }
> +  ctx->setProperty("TABLES",table.c_str());
> +  HugoTransactions hugoTrans(*ctx->getTab());
> +
> +  if (hugoTrans.loadTable(GETNDB(step), t1_records, 1, true, 0) != 0)
> +  {
> +    g_err << "Create Table -> Load failed!" << endl;
> +    return NDBT_FAILED;
> +  }
> +
> +  if(verifySlaveLoad(&table)!= 0)
> +  {
> +    g_err << "Create Table -> Failed on verify slave load!" 
> +          << endl;
> +    return NDBT_FAILED;
> +  }
> +  //else everything is okay  
> +  return NDBT_OK;
> +}
> +
> +int
> +stressNDB_rep1(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  const NdbDictionary::Table * table= ctx->getTab();
> +  HugoTransactions hugoTrans(* table);
> +  while(!ctx->isTestStopped())
> +  {
> +    if (hugoTrans.pkUpdateRecords(GETNDB(step), t1_records, 1, 30) != 0)
> +    {
> +      g_err << "pkUpdate Failed!" << endl;
> +      return NDBT_FAILED;
> +    }
> +    if (hugoTrans.scanUpdateRecords(GETNDB(step), t1_records, 1, 30) != 0)
> +    {
> +      g_err << "scanUpdate Failed!" << endl;
> +      return NDBT_FAILED;
> +    }
> +  }
> +  return NDBT_OK;
> +}
> +
> +int
> +stressSQL_rep1(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  //Create an SQL Object
> +  DbUtil master(db.c_str(),"");
> +  int loops = ctx->getNumLoops();
> +  uint record = 0;
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if(master.selectDb())
> +  {
> +    ctx->stopTest();
> +    return NDBT_FAILED;
> +  }
> +
> +  for (int j= 0; loops == 0 || j < loops; j++)
> +  {
> +    record = urandom(t1_records);
> +    sqlStm.assfmt("UPDATE TEST_DB.rep1 SET c2 = 33.3221 where c1 =  %u", record);
> +    if(master.doQuery(sqlStm.c_str()))
> +    {
> +      return NDBT_FAILED;
> +    }
> +  }
> +  ctx->stopTest();
> +  return NDBT_OK;
> +}
> +
> +int
> +verifySlave_rep1(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  if(syncSlaveWithMaster() != 0)
> +  {
> +    g_err << "Verify Slave rep1 -> Syncing with slave failed" 
> +          << endl;
> +    return NDBT_FAILED;
> +  }
> +  //Create SQL Objects
> +  DbUtil     master(db.c_str(),"");
> +  DbUtil     slave(db.c_str(),".slave");
> +  MYSQL_RES  *resource;
> +  MYSQL_ROW   row;
> +  float       masterSum;
> +  float       slaveSum;
> +
> +  sqlStm.assign("SELECT SUM(c3) FROM rep1");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +    
> +  if(master.selectDb())
> +  {
> +    return NDBT_FAILED;
> +  } 
> +
> +  if(master.doQuery(sqlStm.c_str()))
> +  { 
> +    return NDBT_FAILED;
> +  }
> +  resource = mysql_use_result(master.getMysql());
> +  row = mysql_fetch_row(resource);
> +  masterSum = atoi(row[0]);  
> +  mysql_free_result(resource);
> +  master.databaseLogout();
> +
> +  //Login to slave
> +  if (!slave.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> + 
> +  if(slave.selectDb() != 0)
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if((slave.doQuery(sqlStm.c_str())) != 0)
> +  {
> +     return NDBT_FAILED;
> +  }
> +  resource = mysql_use_result(slave.getMysql());
> +  row = mysql_fetch_row(resource);
> +  slaveSum = atoi(row[0]);
> +  mysql_free_result(resource);
> +  
> +  if(masterSum != slaveSum)
> +  {
> +    g_err << "VerifySlave -> masterSum != slaveSum..." << endl;
> +    return NDBT_FAILED;
> +  }
> +  return NDBT_OK;
> +}
> +
> +int
> +dropTEST_DB(NDBT_Context* ctx, NDBT_Step* step)
> +{
> +  //Create an SQL Object
> +  DbUtil master(db.c_str(),"");
> +
> +  //Login to Master
> +  if (!master.connect())
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if(master.selectDb() != 0)
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if(master.doQuery("DROP DATABASE TEST_DB") != 0)
> +  {
> +    return NDBT_FAILED;
> +  }
> +
> +  if(syncSlaveWithMaster() != 0)
> +  {
> +    g_err << "Drop DB -> Syncing with slave failed"
> +          << endl;
> +    return NDBT_FAILED;
> +  }
> +  return NDBT_OK;
> +}
> +
> +NDBT_TESTSUITE(NdbRepStress);
> +TESTCASE("PHASE_I_Stress","Basic Replication Stressing") 
> +{
> +  INITIALIZER(createDB);
> +  INITIALIZER(createTable_rep1);
> +  STEP(stressNDB_rep1);
> +  STEP(stressSQL_rep1);
> +  FINALIZER(verifySlave_rep1);
> +  FINALIZER(dropTEST_DB);
> +}

magnus: I think it's important that we can "ramp" up. Start with some 
simple stuff and if that works go to the next level, and so on...

> +NDBT_TESTSUITE_END(NdbRepStress);
> +
> +int main(int argc, const char** argv){
> +  ndb_init();
> +  NdbRepStress.setCreateAllTables(true);
> +  return NdbRepStress.execute(argc, argv);
> +}
> +
> +template class Vector<HugoOperations *>;
> +template class Vector<NdbEventOperation *>;
> +template class Vector<NdbRecAttr*>;
> +template class Vector<Vector<NdbRecAttr*> >;

magnus: Huh? Must be something wrong if you need this.

> diff -Nrup a/storage/ndb/test/src/DbUtil.cpp b/storage/ndb/test/src/DbUtil.cpp
> --- a/storage/ndb/test/src/DbUtil.cpp	2007-12-20 21:06:26 +01:00
> +++ b/storage/ndb/test/src/DbUtil.cpp	2008-01-08 23:34:08 +01:00
> @@ -215,23 +215,57 @@ DbUtil::printStError(MYSQL_STMT *stmt, c
>  /* Select which database to use */
>  
>  int 
> -DbUtil::select_DB()
> +DbUtil::selectDb()
>  {
> -  return mysql_select_db(this->getMysql(), this->getDbName());
> +  if ((this->getDbName()) != NULL)
> +  {
> +    if(mysql_select_db(this->getMysql(), this->getDbName()))
> +    {
> +      this->printError("mysql_select_db failed");

magnus: Very good. When using the class it should automatically print 
the error messages to avoid clutter in the testcases.

> +      return DBU_FAILED;
> +    }
> +    return DBU_OK;   
> +  }
> +  this->printError("getDbName() == NULL");
> +  return DBU_FAILED;
>  }
>  
> +int
> +DbUtil::selectDb(const char * m_db)
> +{
> +  {
> +    if(mysql_select_db(this->getMysql(), m_db))
> +    {
> +      this->printError("mysql_select_db failed");
> +      return DBU_FAILED;
> +    }
> +    return DBU_OK;
> +  }
> +}
> +
> +
>  /* Run Simple Queries */
>  
>  int 
>  DbUtil::doQuery(char * stm)
>  {
> -  return mysql_query(this->getMysql(), stm);
> +  if(mysql_query(this->getMysql(), stm))
> +  {
> +    this->printError(stm); 
> +    return DBU_FAILED;
> +  }
> +  return DBU_OK;
>  }
>  
>  int 
>  DbUtil::doQuery(const char * stm)
>  {
> -  return mysql_query(this->getMysql(), stm);
> +  if(mysql_query(this->getMysql(), stm))
> +  {
> +    this->printError(stm);
> +    return DBU_FAILED;
> +  }
> +  return DBU_OK;
>  }
>  
>  /* Return MySQL Error String */
>