bzr push into mysql-5.5 branch (jon.hauglid:3436 to 3437) Bug#12348348

[Jon Olav Hauglid]

 3437 Jon Olav Hauglid	2011-04-12
      Bug#12348348 - ASSERTION IN ROW0SEL.C LINE 3698:
                     TRX->CONC_STATE == 0 || TRX->CONC_STATE == 1
      
      This bug was a different manifestation of Bug#11766752,
      which was previously only fixed on mysql-trunk.
      
      This patch backports the fix for Bug#11766752 to mysql-5.5,
      which fixes the problem. The patch also adds some extra test
      coverage.

    modified:
      mysql-test/r/xa.result
      mysql-test/t/xa.test
      sql/sql_base.cc
      sql/transaction.cc
 3436 Sergey Glukhov	2011-04-12 [merge]
      automerge

    modified:
      mysql-test/collections/default.experimental
=== modified file 'mysql-test/r/xa.result'
--- a/mysql-test/r/xa.result	2011-02-14 13:16:31 +0000
+++ b/mysql-test/r/xa.result	2011-04-12 10:57:02 +0000
@@ -166,3 +166,37 @@ ERROR XA102: XA_RBDEADLOCK: Transaction
 XA END 'b';
 XA ROLLBACK 'b';
 DROP TABLE t1;
+#
+# Bug#11766752 59936: multiple xa assertions - transactional
+#              statement fuzzer
+#
+CREATE TABLE t1 (a INT) engine=InnoDB;
+XA START 'a';
+INSERT INTO t1 VALUES (1);
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ACTIVE state
+XA END 'a';
+SELECT * FROM t1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+INSERT INTO t1 VALUES (2);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+SET @a=(SELECT * FROM t1);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  IDLE state
+XA PREPARE 'a';
+SELECT * FROM t1;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+INSERT INTO t1 VALUES (2);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+SAVEPOINT savep;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+SET @a=(SELECT * FROM t1);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+UPDATE t1 SET a=1 WHERE a=2;
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  PREPARED state
+XA COMMIT 'a';
+SELECT * FROM t1;
+a
+1
+DROP TABLE t1;

=== modified file 'mysql-test/t/xa.test'
--- a/mysql-test/t/xa.test	2011-02-14 13:16:31 +0000
+++ b/mysql-test/t/xa.test	2011-04-12 10:57:02 +0000
@@ -287,6 +287,45 @@ DROP TABLE t1;
 disconnect con1;
 
 
+--echo #
+--echo # Bug#11766752 59936: multiple xa assertions - transactional
+--echo #              statement fuzzer
+--echo #
+
+CREATE TABLE t1 (a INT) engine=InnoDB;
+XA START 'a';
+INSERT INTO t1 VALUES (1);
+
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+
+XA END 'a';
+--error ER_XAER_RMFAIL
+SELECT * FROM t1;
+--error ER_XAER_RMFAIL
+INSERT INTO t1 VALUES (2);
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+--error ER_XAER_RMFAIL
+SET @a=(SELECT * FROM t1);
+
+XA PREPARE 'a';
+--error ER_XAER_RMFAIL
+SELECT * FROM t1;          # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+INSERT INTO t1 VALUES (2); # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+SAVEPOINT savep;
+--error ER_XAER_RMFAIL
+SET @a=(SELECT * FROM t1); # used to cause InnoDB assert
+--error ER_XAER_RMFAIL
+UPDATE t1 SET a=1 WHERE a=2;
+
+XA COMMIT 'a';
+SELECT * FROM t1;
+DROP TABLE t1;
+
+
 # Wait till all disconnects are completed
 --source include/wait_until_count_sessions.inc
 

=== modified file 'sql/sql_base.cc'
--- a/sql/sql_base.cc	2011-03-29 08:38:26 +0000
+++ b/sql/sql_base.cc	2011-04-12 10:57:02 +0000
@@ -4724,6 +4724,14 @@ bool open_tables(THD *thd, TABLE_LIST **
   bool has_prelocking_list;
   DBUG_ENTER("open_tables");
 
+  /* Accessing data in XA_IDLE or XA_PREPARED is not allowed. */
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (*start && (xa_state == XA_IDLE || xa_state == XA_PREPARED))
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(true);
+  }
+
   /*
     temporary mem_root for new .frm parsing.
     TODO: variables for size

=== modified file 'sql/transaction.cc'
--- a/sql/transaction.cc	2011-02-14 13:16:31 +0000
+++ b/sql/transaction.cc	2011-04-12 10:57:02 +0000
@@ -362,6 +362,13 @@ bool trans_savepoint(THD *thd, LEX_STRIN
       !opt_using_transactions)
     DBUG_RETURN(FALSE);
 
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (xa_state != XA_NOTR)
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(TRUE);
+  }
+
   sv= find_savepoint(thd, name);
 
   if (*sv) /* old savepoint of the same name exists */
@@ -435,6 +442,13 @@ bool trans_rollback_to_savepoint(THD *th
     DBUG_RETURN(TRUE);
   }
 
+  enum xa_states xa_state= thd->transaction.xid_state.xa_state;
+  if (xa_state != XA_NOTR)
+  {
+    my_error(ER_XAER_RMFAIL, MYF(0), xa_state_names[xa_state]);
+    DBUG_RETURN(TRUE);
+  }
+
   if (ha_rollback_to_savepoint(thd, sv))
     res= TRUE;
   else if (((thd->variables.option_bits & OPTION_KEEP_LOG) ||

No bundle (reason: useless for push emails).