bk commit into 5.1 tree (tomas:1.2504)

[tomas]

Below is the list of changes that have just been committed into a local
5.1 repository of tomas. When tomas does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2007-05-02 22:37:19+02:00, tomas@stripped +13 -0
  Merge whalegate.ndb.mysql.com:/home/tomas/mysql-5.1-telco-6.1-batch-slave
  into  whalegate.ndb.mysql.com:/home/tomas/mysql-5.1-telco-batch-slave
  MERGE: 1.2403.15.91

  cluster_change_hist.txt@stripped, 2007-05-02 22:32:05+02:00, tomas@stripped
+0 -4
    null merge
    MERGE: 1.6.1.12

  mysql-test/mysql-test-run.pl@stripped, 2007-05-02 22:37:16+02:00,
tomas@stripped +0 -0
    manual merge
    MERGE: 1.261.2.2

  mysql-test/r/rpl_ndb_basic.result@stripped, 2007-05-02 22:37:16+02:00,
tomas@stripped +0 -2
    manual merge
    MERGE: 1.4.5.2

  mysql-test/t/disabled.def@stripped, 2007-05-02 22:37:16+02:00,
tomas@stripped +1 -1
    manual merge
    MERGE: 1.227.1.2

  sql/ha_ndbcluster.cc@stripped, 2007-05-02 22:37:16+02:00, tomas@stripped +3
-3
    manual merge
    MERGE: 1.397.7.13

  sql/ha_ndbcluster.h@stripped, 2007-05-02 22:37:16+02:00, tomas@stripped +0
-0
    manual merge
    MERGE: 1.165.3.2

  sql/log_event.cc@stripped, 2007-05-02 22:37:16+02:00, tomas@stripped +1 -2
    manual merge
    MERGE: 1.263.3.2

  sql/mysql_priv.h@stripped, 2007-05-02 22:37:16+02:00, tomas@stripped +0 -1
    manual merge
    MERGE: 1.470.17.1

  sql/mysqld.cc@stripped, 2007-05-02 22:37:16+02:00, tomas@stripped +1 -1
    manual merge
    MERGE: 1.601.20.3

  storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp@stripped, 2007-05-02 22:31:36+02:00,
tomas@stripped +0 -0
    Auto merged
    MERGE: 1.134.2.11

  storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp@stripped, 2007-05-02 22:31:36+02:00,
tomas@stripped +0 -0
    Auto merged
    MERGE: 1.57.1.6

  storage/ndb/src/kernel/blocks/dbtup/DbtupCommit.cpp@stripped, 2007-05-02 22:31:36+02:00,
tomas@stripped +0 -0
    Auto merged
    MERGE: 1.21.1.7

  storage/ndb/src/kernel/blocks/dbtup/DbtupExecQuery.cpp@stripped, 2007-05-02 22:31:36+02:00,
tomas@stripped +0 -0
    Auto merged
    MERGE: 1.51.1.7

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	tomas
# Host:	whalegate.ndb.mysql.com
# Root:	/home/tomas/mysql-5.1-telco-batch-slave/RESYNC

--- 1.276/sql/log_event.cc	2007-04-24 19:26:48 +02:00
+++ 1.277/sql/log_event.cc	2007-05-02 22:37:16 +02:00
@@ -35,6 +35,7 @@
 #define log_cs	&my_charset_latin1
 
 #define FLAGSTR(V,F) ((V)&(F)?#F" ":"")
+extern my_bool opt_slave_allow_batching;
 
 /*
   Cache that will automatically be written to a dedicated file on
@@ -6011,6 +6012,10 @@
         thd->options|= OPTION_RELAXED_UNIQUE_CHECKS;
     else
         thd->options&= ~OPTION_RELAXED_UNIQUE_CHECKS;
+    
+    if (opt_slave_allow_batching)
+      thd->options |= OPTION_ALLOW_BATCH;
+    
     /* A small test to verify that objects have consistent types */
     DBUG_ASSERT(sizeof(thd->options) == sizeof(OPTION_RELAXED_UNIQUE_CHECKS));
 
@@ -7244,6 +7249,22 @@
                                  table->s->reclength) == 0);
 
     */
+
+    /*
+      Ndb does not need read before delete/update (and no updates are sent)
+      if primary key specified
+      
+      (Actually uniquekey will also do, but pk will be in each
+      row if table has pk)
+
+      Also set ignore no key, as we don't really know if row exists...
+    */
+    if (table->file->ht->db_type == DB_TYPE_NDBCLUSTER)
+    {
+      table->file->extra(HA_EXTRA_IGNORE_NO_KEY);
+      DBUG_RETURN(0);
+    }
+    
     table->file->position(table->record[0]);
     int error= table->file->rnd_pos(table->record[0], table->file->ref);
     /*

--- 1.503/sql/mysql_priv.h	2007-04-28 21:20:14 +02:00
+++ 1.504/sql/mysql_priv.h	2007-05-02 22:37:16 +02:00
@@ -356,7 +356,11 @@
   fulltext functions when reading from it.
 */
 #define TMP_TABLE_FORCE_MYISAM          (ULL(1) << 32)
-
+/*
+  Dont report errors for individual rows,
+  But just report error on commit (or read ofcourse)
+*/
+#define OPTION_ALLOW_BATCH              (ULL(1) << 33) // THD, intern (slave)
 
 /*
   Maximum length of time zone name that we support

--- 1.639/sql/mysqld.cc	2007-04-27 14:52:49 +02:00
+++ 1.640/sql/mysqld.cc	2007-05-02 22:37:16 +02:00
@@ -368,6 +368,7 @@
 my_bool opt_log_slave_updates= 0;
 my_bool	opt_innodb;
 bool slave_warning_issued = false; 
+my_bool opt_slave_allow_batching= 0;
 
 /*
   Legacy global handlerton. These will be removed (please do not add more).
@@ -4940,7 +4941,7 @@
   OPT_SLAVE_LOAD_TMPDIR, OPT_NO_MIX_TYPE,
   OPT_RPL_RECOVERY_RANK,OPT_INIT_RPL_ROLE,
   OPT_RELAY_LOG, OPT_RELAY_LOG_INDEX, OPT_RELAY_LOG_INFO_FILE,
-  OPT_SLAVE_SKIP_ERRORS, OPT_DES_KEY_FILE, OPT_LOCAL_INFILE,
+  OPT_SLAVE_SKIP_ERRORS, OPT_SLAVE_ALLOW_BATCHING, OPT_DES_KEY_FILE, OPT_LOCAL_INFILE,
   OPT_SSL_SSL, OPT_SSL_KEY, OPT_SSL_CERT, OPT_SSL_CA,
   OPT_SSL_CAPATH, OPT_SSL_CIPHER,
   OPT_BACK_LOG, OPT_BINLOG_CACHE_SIZE,
@@ -6420,6 +6421,11 @@
    "before giving up and stopping.",
    (gptr*) &slave_trans_retries, (gptr*) &slave_trans_retries, 0,
    GET_ULONG, REQUIRED_ARG, 10L, 0L, (longlong) ULONG_MAX, 0, 1, 0},
+  {"slave_allow_batching", OPT_SLAVE_ALLOW_BATCHING,
+   "Allow slave to batch requests.",
+   (gptr*) &opt_slave_allow_batching,
+   (gptr*) &opt_slave_allow_batching,
+   0, GET_BOOL, NO_ARG, 0, 0, 1, 0, 1, 0},
 #endif /* HAVE_REPLICATION */
   {"slow_launch_time", OPT_SLOW_LAUNCH_TIME,
    "If creating the thread takes longer than this value (in seconds), the
Slow_launch_threads counter will be incremented.",

--- 1.282/mysql-test/mysql-test-run.pl	2007-05-02 07:01:13 +02:00
+++ 1.283/mysql-test/mysql-test-run.pl	2007-05-02 22:37:16 +02:00
@@ -3811,6 +3811,7 @@
 		  $cluster->{'connect_string'});
       mtr_add_arg($args, "%s--ndb-wait-connected=20", $prefix);
       mtr_add_arg($args, "%s--ndb-cluster-connection-pool=3", $prefix);
+      mtr_add_arg($args, "%s--slave_allow_batching", $prefix);
       if ( $mysql_version_id >= 50100 )
       {
 	mtr_add_arg($args, "%s--ndb-extra-logging", $prefix);
@@ -3887,6 +3888,7 @@
 		  $clusters->[$mysqld->{'cluster'}]->{'connect_string'});
       mtr_add_arg($args, "%s--ndb-wait-connected=20", $prefix);
       mtr_add_arg($args, "%s--ndb-cluster-connection-pool=3", $prefix);
+      mtr_add_arg($args, "%s--slave_allow_batching", $prefix);
       if ( $mysql_version_id >= 50100 )
       {
 	mtr_add_arg($args, "%s--ndb-extra-logging", $prefix);

--- 1.252/mysql-test/t/disabled.def	2007-05-02 07:01:13 +02:00
+++ 1.253/mysql-test/t/disabled.def	2007-05-02 22:37:16 +02:00
@@ -40,3 +40,4 @@
 #rpl_ndb_dd_advance	 : Bug#25913 rpl_ndb_dd_advance fails randomly
 
 ndb_partition_error2	 : HF is not sure if the test can work as internded on all the
platforms
+rpl_ndb_innodb_trans     : slave batch patch breaks this

--- 1.456/sql/ha_ndbcluster.cc	2007-04-27 14:52:48 +02:00
+++ 1.457/sql/ha_ndbcluster.cc	2007-05-02 22:37:16 +02:00
@@ -116,6 +116,7 @@
 }
 
 #define NDB_AUTO_INCREMENT_RETRIES 10
+#define BATCH_FLUSH_SIZE (32768)
 
 #define ERR_PRINT(err) \
   DBUG_PRINT("error", ("%d  message: %s", err.code, err.message))
@@ -373,6 +374,7 @@
   m_error= FALSE;
   query_state&= NDB_QUERY_NORMAL;
   options= 0;
+  m_unsent_bytes= 0;
   (void) hash_init(&open_tables, &my_charset_bin, 5, 0, 0,
                    (hash_get_key)thd_ndb_share_get_key, 0, 0);
 }
@@ -1093,6 +1095,10 @@
 
   DBUG_PRINT("info", ("fetched table %s", tab->getName()));
   m_table= tab;
+  /*
+    Approx. write size in bytes over transporter
+  */
+  m_bytes_per_write= 12 + tab->getRowSizeInBytes() + 4 * tab->getNoOfColumns();
   if ((error= open_indexes(ndb, table, FALSE)) == 0)
   {
     ndbtab_g.release();
@@ -2647,6 +2653,8 @@
   NdbOperation *op;
   int res;
   THD *thd= table->in_use;
+  Thd_ndb *thd_ndb= get_thd_ndb(thd);
+  uint unsent= thd_ndb->m_unsent_bytes;
   longlong func_value= 0;
   DBUG_ENTER("ha_ndbcluster::write_row");
 
@@ -2788,20 +2796,22 @@
     to NoCommit the transaction between each row.
     Find out how this is detected!
   */
+  unsent+= m_bytes_per_write;
+  thd_ndb->m_unsent_bytes= unsent;
+  
   m_rows_inserted++;
   no_uncommitted_rows_update(1);
-  m_bulk_insert_not_flushed= TRUE;
-  if ((m_rows_to_insert == (ha_rows) 1) || 
-      ((m_rows_inserted % m_bulk_insert_rows) == 0) ||
+  if ((thd->options & OPTION_ALLOW_BATCH) == 0 ||
+      (unsent > BATCH_FLUSH_SIZE) ||
       m_primary_key_update ||
       set_blob_value)
   {
     // Send rows to NDB
     DBUG_PRINT("info", ("Sending inserts to NDB, "\
-                        "rows_inserted: %d  bulk_insert_rows: %d", 
-                        (int)m_rows_inserted, (int)m_bulk_insert_rows));
-
-    m_bulk_insert_not_flushed= FALSE;
+                        "rows_inserted:%d, unsent bytes: %d", 
+                        (int)m_rows_inserted, (int)unsent));
+    
+    thd_ndb->m_unsent_bytes= 0;
     if (m_transaction_on)
     {
       if (execute_no_commit(this,trans,FALSE) != 0)
@@ -3089,10 +3099,12 @@
 int ha_ndbcluster::delete_row(const byte *record)
 {
   THD *thd= table->in_use;
+  Thd_ndb *thd_ndb= get_thd_ndb(thd);
   NdbTransaction *trans= m_active_trans;
   NdbScanOperation* cursor= m_active_cursor;
   NdbOperation *op;
   uint32 part_id;
+  uint unsent= thd_ndb->m_unsent_bytes;
   int error;
   DBUG_ENTER("delete_row");
   m_write_op= TRUE;
@@ -3175,11 +3187,22 @@
     }
   }
 
-  // Execute delete operation
-  if (execute_no_commit(this,trans,FALSE) != 0) {
-    no_uncommitted_rows_execute_failure();
-    DBUG_RETURN(ndb_err(trans));
+  /*
+    Poor approx. let delete ~ tabsize / 4
+  */
+  unsent+= 12 + m_bytes_per_write >> 2;
+  if ((thd->options & OPTION_ALLOW_BATCH) == 0 ||
+      (unsent > BATCH_FLUSH_SIZE))
+  { 
+    thd_ndb->m_unsent_bytes = 0;
+    // Execute delete operation
+    if (execute_no_commit(this,trans,FALSE) != 0) {
+      no_uncommitted_rows_execute_failure();
+      DBUG_RETURN(ndb_err(trans));
+    }
+    DBUG_RETURN(0);
   }
+  thd_ndb->m_unsent_bytes= unsent;
   DBUG_RETURN(0);
 }
   
@@ -4068,7 +4091,8 @@
     DBUG_PRINT("info", ("Batching turned off as duplicate key is "
                         "ignored by using peek_row"));
     m_rows_to_insert= 1;
-    m_bulk_insert_rows= 1;
+    Thd_ndb *thd_ndb= get_thd_ndb(current_thd);
+    thd_ndb->m_unsent_bytes= BATCH_FLUSH_SIZE; // ensure no batching
     DBUG_VOID_RETURN;
   }
   if (rows == (ha_rows) 0)
@@ -4079,20 +4103,6 @@
   else
     m_rows_to_insert= rows; 
 
-  /* 
-    Calculate how many rows that should be inserted
-    per roundtrip to NDB. This is done in order to minimize the 
-    number of roundtrips as much as possible. However performance will 
-    degrade if too many bytes are inserted, thus it's limited by this 
-    calculation.   
-  */
-  const int bytesperbatch= 8192;
-  bytes= 12 + tab->getRowSizeInBytes() + 4 * tab->getNoOfColumns();
-  batch= bytesperbatch/bytes;
-  batch= batch == 0 ? 1 : batch;
-  DBUG_PRINT("info", ("batch: %d, bytes: %d", batch, bytes));
-  m_bulk_insert_rows= batch;
-
   DBUG_VOID_RETURN;
 }
 
@@ -4105,14 +4115,18 @@
 
   DBUG_ENTER("end_bulk_insert");
   // Check if last inserts need to be flushed
-  if (m_bulk_insert_not_flushed)
+
+  THD *thd= current_thd;
+  Thd_ndb *thd_ndb= get_thd_ndb(thd);
+  
+  if ((thd->options & OPTION_ALLOW_BATCH) == 0 &&
thd_ndb->m_unsent_bytes)
   {
     NdbTransaction *trans= m_active_trans;
     // Send rows to NDB
     DBUG_PRINT("info", ("Sending inserts to NDB, "\
-                        "rows_inserted: %d  bulk_insert_rows: %d", 
-                        (int) m_rows_inserted, (int) m_bulk_insert_rows)); 
-    m_bulk_insert_not_flushed= FALSE;
+                        "rows_inserted:%d, unsent bytes: %d", 
+                        (int) m_rows_inserted, (int)thd_ndb->m_unsent_bytes));
+    thd_ndb->m_unsent_bytes= 0;
     if (m_transaction_on)
     {
       if (execute_no_commit(this, trans,FALSE) != 0)
@@ -4576,7 +4590,27 @@
   }
 #endif /* HAVE_NDB_BINLOG */
 
-  if (execute_commit(thd,trans) != 0)
+  if (thd->options & OPTION_ALLOW_BATCH)
+  {
+    if ((res= trans->execute(Commit, AO_IgnoreError, 1)) != 0)
+    {
+      const NdbError err= trans->getNdbError();
+      if (err.classification == NdbError::ConstraintViolation ||
+	  err.classification == NdbError::NoDataFound)
+      {
+	/*
+	  This is ok...
+	*/
+	res= 0;
+      }
+    }
+  }
+  else
+  {
+    res= execute_commit(thd,trans);
+  }
+
+  if (res != 0)
   {
     const NdbError err= trans->getNdbError();
     const NdbOperation *error_op= trans->getNdbErrorOperation();
@@ -6130,9 +6164,7 @@
   m_ignore_no_key(FALSE),
   m_rows_to_insert((ha_rows) 1),
   m_rows_inserted((ha_rows) 0),
-  m_bulk_insert_rows((ha_rows) 1024),
   m_rows_changed((ha_rows) 0),
-  m_bulk_insert_not_flushed(FALSE),
   m_delete_cannot_batch(FALSE),
   m_update_cannot_batch(FALSE),
   m_ops_pending(0),

--- 1.177/sql/ha_ndbcluster.h	2007-04-24 16:48:27 +02:00
+++ 1.178/sql/ha_ndbcluster.h	2007-05-02 22:37:16 +02:00
@@ -213,6 +213,7 @@
   List<NDB_SHARE> changed_tables;
   uint query_state;
   HASH open_tables;
+  uint m_unsent_bytes;
 };
 
 class ha_ndbcluster: public handler
@@ -524,12 +525,11 @@
   bool m_ignore_no_key;
   ha_rows m_rows_to_insert; // TODO: merge it with handler::estimation_rows_to_insert?
   ha_rows m_rows_inserted;
-  ha_rows m_bulk_insert_rows;
   ha_rows m_rows_changed;
-  bool m_bulk_insert_not_flushed;
   bool m_delete_cannot_batch;
   bool m_update_cannot_batch;
   ha_rows m_ops_pending;
+  uint m_bytes_per_write;
   bool m_skip_auto_increment;
   bool m_blobs_pending;
   bool m_slow_path;

--- 1.12/mysql-test/r/rpl_ndb_basic.result	2007-04-23 20:45:22 +02:00
+++ 1.13/mysql-test/r/rpl_ndb_basic.result	2007-05-02 22:37:16 +02:00
@@ -159,8 +159,8 @@
 Replicate_Ignore_Table	<Replicate_Ignore_Table>
 Replicate_Wild_Do_Table	
 Replicate_Wild_Ignore_Table	
-Last_Errno	1105
-Last_Error	Unknown error
+Last_Errno	1205
+Last_Error	Error 'Lock wait timeout exceeded; try restarting transaction' on query.
Default database: ''. Query: 'COMMIT'
 Skip_Counter	0
 Exec_Master_Log_Pos	<Exec_Master_Log_Pos>
 Relay_Log_Space	<Relay_Log_Space>