List:Commits« Previous MessageNext Message »
From:Jonas Oreland Date:December 19 2011 2:27pm
Subject:bzr push into mysql-5.5-cluster-7.2 branch (jonas.oreland:3737 to 3738)
View as plain text  
 3738 Jonas Oreland	2011-12-19 [merge]
      ndb - merge 71 to 72

    renamed:
      storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java.not_yet => storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java
    modified:
      sql/ha_ndb_index_stat.cc
      sql/ha_ndb_index_stat.h
      sql/ha_ndbcluster.cc
      storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java
      storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionServiceImpl.java
      storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbugImpl.java
      storage/ndb/include/ndbapi/NdbIndexStat.hpp
      storage/ndb/src/kernel/blocks/LocalProxy.cpp
      storage/ndb/src/kernel/blocks/LocalProxy.hpp
      storage/ndb/src/ndbapi/NdbIndexStat.cpp
      storage/ndb/src/ndbapi/NdbIndexStatImpl.cpp
      storage/ndb/src/ndbapi/NdbIndexStatImpl.hpp
      storage/ndb/src/ndbapi/ndberror.c
      storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java
 3737 John David Duncan	2011-12-18
      Fix compiler warnings about integer types.

    modified:
      storage/ndb/memcache/src/ExternalValue.cc
      storage/ndb/memcache/src/ndb_error_logger.cc
      storage/ndb/memcache/src/ndb_flush.cc
      storage/ndb/memcache/src/ndb_pipeline.cc
=== modified file 'sql/ha_ndb_index_stat.cc'
--- a/sql/ha_ndb_index_stat.cc	2011-12-08 11:53:27 +0000
+++ b/sql/ha_ndb_index_stat.cc	2011-12-19 14:23:00 +0000
@@ -50,7 +50,6 @@ Ndb_index_stat_thread::Ndb_index_stat_th
   pthread_mutex_init(&LOCK, MY_MUTEX_INIT_FAST);
   pthread_cond_init(&COND, NULL);
   pthread_cond_init(&COND_ready, NULL);
-  pthread_mutex_init(&list_mutex, MY_MUTEX_INIT_FAST);
   pthread_mutex_init(&stat_mutex, MY_MUTEX_INIT_FAST);
   pthread_cond_init(&stat_cond, NULL);
 }
@@ -60,7 +59,6 @@ Ndb_index_stat_thread::~Ndb_index_stat_t
   pthread_mutex_destroy(&LOCK);
   pthread_cond_destroy(&COND);
   pthread_cond_destroy(&COND_ready);
-  pthread_mutex_destroy(&list_mutex);
   pthread_mutex_destroy(&stat_mutex);
   pthread_cond_destroy(&stat_cond);
 }
@@ -90,9 +88,12 @@ struct Ndb_index_stat {
   time_t check_time;    /* when checked for updated stats (>= read_time) */
   uint query_bytes;     /* cache query bytes in use */
   uint clean_bytes;     /* cache clean bytes waiting to be deleted */
+  uint drop_bytes;      /* cache bytes waiting for drop */
+  uint evict_bytes;     /* cache bytes waiting for evict */
   bool force_update;    /* one-time force update from analyze table */
   bool no_stats;        /* have detected that no stats exist */
   NdbIndexStat::Error error;
+  NdbIndexStat::Error client_error;
   time_t error_time;
   uint error_count;
   struct Ndb_index_stat *share_next; /* per-share list */
@@ -101,7 +102,9 @@ struct Ndb_index_stat {
   struct Ndb_index_stat *list_next;
   struct Ndb_index_stat *list_prev;
   struct NDB_SHARE *share;
+  uint ref_count;       /* from client requests */
   bool to_delete;       /* detached from share and marked for delete */
+  bool abort_request;   /* abort all requests and allow no more */
   Ndb_index_stat();
 };
 
@@ -556,12 +559,16 @@ struct Ndb_index_stat_glob {
   uint event_ok;          /* Events received for known index */
   uint event_miss;        /* Events received for unknown index */
   uint refresh_count;     /* Successful cache refreshes */
+  uint clean_count;       /* Times old caches (1 or more) cleaned */
+  uint pinned_count;      /* Times not cleaned due to old cache ref count */
   uint drop_count;        /* From index drop */
   uint evict_count;       /* From LRU cleanup */
   /* Cache */
   uint cache_query_bytes; /* In use */
   uint cache_clean_bytes; /* Obsolete versions not yet removed */
   uint cache_high_bytes;  /* Max ever of above */
+  uint cache_drop_bytes;  /* Part of above waiting to be evicted */
+  uint cache_evict_bytes; /* Part of above waiting to be evicted */
   char status[2][512];
   uint status_i;
 
@@ -588,11 +595,15 @@ Ndb_index_stat_glob::Ndb_index_stat_glob
   event_ok= 0;
   event_miss= 0;
   refresh_count= 0;
+  clean_count= 0;
+  pinned_count= 0;
   drop_count= 0;
   evict_count= 0;
   cache_query_bytes= 0;
   cache_clean_bytes= 0;
   cache_high_bytes= 0;
+  cache_drop_bytes= 0;
+  cache_evict_bytes= 0;
   memset(status, 0, sizeof(status));
   status_i= 0;
 }
@@ -601,6 +612,8 @@ Ndb_index_stat_glob::Ndb_index_stat_glob
 void
 Ndb_index_stat_glob::set_status()
 {
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
   const Ndb_index_stat_opt &opt= ndb_index_stat_opt;
   char* p= status[status_i];
 
@@ -635,11 +648,13 @@ Ndb_index_stat_glob::set_status()
   p+= strlen(p);
   sprintf(p, "analyze:(all:%u,error:%u)", analyze_count, analyze_error);
   p+= strlen(p);
-  sprintf(p, ",query:(all:%u,nostats:%u,error:%u)", query_count, query_no_stats, query_error);
+  sprintf(p, ",query:(all:%u,nostats:%u,error:%u)",
+             query_count, query_no_stats, query_error);
   p+= strlen(p);
   sprintf(p, ",event:(ok:%u,miss:%u)", event_ok, event_miss);
   p+= strlen(p);
-  sprintf(p, ",cache:(refresh:%u,drop:%u,evict:%u)", refresh_count, drop_count, evict_count);
+  sprintf(p, ",cache:(refresh:%u,clean:%u,pinned:%u,drop:%u,evict:%u)",
+             refresh_count, clean_count, pinned_count, drop_count, evict_count);
   p+= strlen(p);
   sprintf(p, ")");
   p+= strlen(p);
@@ -654,8 +669,12 @@ Ndb_index_stat_glob::set_status()
     cache_pct= (double)100.0 * (double)cache_total / (double)cache_limit;
     cache_high_pct= (double)100.0 * (double)cache_high_bytes / (double)cache_limit;
   }
-  sprintf(p, ",cache:(query:%u,clean:%u,usedpct:%.2f,highpct:%.2f)",
-             cache_query_bytes, cache_clean_bytes, cache_pct, cache_high_pct);
+  sprintf(p, ",cache:(query:%u,clean:%u"
+             ",drop:%u,evict:%u"
+             ",usedpct:%.2f,highpct:%.2f)",
+             cache_query_bytes, cache_clean_bytes,
+             cache_drop_bytes, cache_evict_bytes,
+             cache_pct, cache_high_pct);
   p+= strlen(p);
 
   // alternating status buffers to keep this lock short
@@ -679,6 +698,8 @@ Ndb_index_stat_glob::zero_total()
   event_ok= 0;
   event_miss= 0;
   refresh_count= 0;
+  clean_count= 0;
+  pinned_count= 0;
   drop_count= 0;
   evict_count= 0;
   /* Reset highest use seen to current */
@@ -704,6 +725,8 @@ Ndb_index_stat::Ndb_index_stat()
   check_time= 0;
   query_bytes= 0;
   clean_bytes= 0;
+  drop_bytes= 0;
+  evict_bytes= 0;
   force_update= false;
   no_stats= false;
   error_time= 0;
@@ -714,24 +737,40 @@ Ndb_index_stat::Ndb_index_stat()
   list_next= 0;
   list_prev= 0;
   share= 0;
+  ref_count= 0;
   to_delete= false;
+  abort_request= false;
 }
 
+/*
+  Called by stats thread and (rarely) by client.  Caller must hold
+  stat_mutex.  Client errors currently have no effect on execution
+  since they are probably local e.g. bad range (internal error).
+  Argument "from" is 0=stats thread 1=client.
+*/
 void
-ndb_index_stat_error(Ndb_index_stat *st, const char* place, int line)
+ndb_index_stat_error(Ndb_index_stat *st,
+                     int from, const char* place, int line)
 {
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
   time_t now= ndb_index_stat_time();
   NdbIndexStat::Error error= st->is->getNdbError();
   if (error.code == 0)
   {
-    // XXX why this if
+    /* Make sure code is not 0 */
     NdbIndexStat::Error error2;
     error= error2;
     error.code= NdbIndexStat::InternalError;
     error.status= NdbError::TemporaryError;
   }
-  st->error= error;
-  st->error_time= now;
+  if (from == 0)
+  {
+    st->error= error;
+    st->error_time= now; /* Controls proc_error */
+  }
+  else
+    st->client_error= error;
   st->error_count++;
 
   DBUG_PRINT("index_stat", ("%s line %d: error %d line %d extra %d",
@@ -839,6 +878,8 @@ ndb_index_stat_list_move(Ndb_index_stat
 void
 ndb_index_stat_force_update(Ndb_index_stat *st, bool onoff)
 {
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   if (onoff)
   {
@@ -864,6 +905,8 @@ ndb_index_stat_force_update(Ndb_index_st
 void
 ndb_index_stat_no_stats(Ndb_index_stat *st, bool flag)
 {
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   if (st->no_stats != flag)
   {
@@ -882,8 +925,36 @@ ndb_index_stat_no_stats(Ndb_index_stat *
   }
 }
 
+void
+ndb_index_stat_ref_count(Ndb_index_stat *st, bool flag)
+{
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
+  uint old_count= st->ref_count;
+  (void)old_count; // USED
+  if (flag)
+  {
+    st->ref_count++;
+  }
+  else
+  {
+    assert(st->ref_count != 0);
+    st->ref_count--;
+  }
+  DBUG_PRINT("index_stat", ("st %s ref_count:%u->%u",
+                            st->id, old_count, st->ref_count));
+}
+
 /* Find or add entry under the share */
 
+/* Saved in get_share() under stat_mutex */
+struct Ndb_index_stat_snap {
+  time_t load_time;
+  uint sample_version;
+  Ndb_index_stat_snap() { load_time= 0; sample_version= 0; }
+};
+
+/* Subroutine, have lock */
 Ndb_index_stat*
 ndb_index_stat_alloc(const NDBINDEX *index,
                      const NDBTAB *table,
@@ -902,8 +973,8 @@ ndb_index_stat_alloc(const NDBINDEX *ind
 #endif
     if (is->set_index(*index, *table) == 0)
       return st;
-    ndb_index_stat_error(st, "set_index", __LINE__);
-    err_out= st->error.code;
+    ndb_index_stat_error(st, 1, "set_index", __LINE__);
+    err_out= st->client_error.code;
   }
   else
   {
@@ -956,6 +1027,7 @@ Ndb_index_stat*
 ndb_index_stat_get_share(NDB_SHARE *share,
                          const NDBINDEX *index,
                          const NDBTAB *table,
+                         Ndb_index_stat_snap &snap,
                          int &err_out,
                          bool allow_add,
                          bool force_update)
@@ -963,7 +1035,6 @@ ndb_index_stat_get_share(NDB_SHARE *shar
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
 
   pthread_mutex_lock(&share->mutex);
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
   pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   time_t now= ndb_index_stat_time();
   err_out= 0;
@@ -974,7 +1045,7 @@ ndb_index_stat_get_share(NDB_SHARE *shar
   {
     if (unlikely(!ndb_index_stat_allow()))
     {
-      err_out= Ndb_index_stat_error_NOT_ALLOW;
+      err_out= NdbIndexStat::MyNotAllow;
       break;
     }
     st= ndb_index_stat_find_share(share, index, st_last);
@@ -982,7 +1053,7 @@ ndb_index_stat_get_share(NDB_SHARE *shar
     {
       if (!allow_add)
       {
-        err_out= Ndb_index_stat_error_NOT_FOUND;
+        err_out= NdbIndexStat::MyNotFound;
         break;
       }
       st= ndb_index_stat_alloc(index, table, err_out);
@@ -995,14 +1066,28 @@ ndb_index_stat_get_share(NDB_SHARE *shar
       ndb_index_stat_list_add(st, Ndb_index_stat::LT_New);
       glob.set_status();
     }
+    else if (unlikely(st->abort_request))
+    {
+      err_out= NdbIndexStat::MyAbortReq;
+      break;
+    }
     if (force_update)
       ndb_index_stat_force_update(st, true);
+    snap.load_time= st->load_time;
+    snap.sample_version= st->sample_version;
     st->access_time= now;
   }
   while (0);
+ 
+  if (err_out == 0)
+  {
+    assert(st != 0);
+    ndb_index_stat_ref_count(st, true);
+  }
+  else
+    st= 0;
 
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
   pthread_mutex_unlock(&share->mutex);
   return st;
 }
@@ -1012,10 +1097,12 @@ ndb_index_stat_get_share(NDB_SHARE *shar
   list and set "to_delete" flag.  Stats thread does real delete.
 */
 
-/* caller must hold list_mutex */
+/* caller must hold stat_mutex */
 void
 ndb_index_stat_free(Ndb_index_stat *st)
 {
+  safe_mutex_assert_owner(&ndb_index_stat_thread.stat_mutex);
+
   DBUG_ENTER("ndb_index_stat_free");
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   NDB_SHARE *share= st->share;
@@ -1037,6 +1124,7 @@ ndb_index_stat_free(Ndb_index_stat *st)
       assert(st->lt != Ndb_index_stat::LT_Delete);
       assert(!st->to_delete);
       st->to_delete= true;
+      st->abort_request= true;
       found++;
     }
     else
@@ -1053,9 +1141,7 @@ ndb_index_stat_free(Ndb_index_stat *st)
   assert(found == 1);
   share->index_stat_list= st_head;
 
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   glob.set_status();
-  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
   DBUG_VOID_RETURN;
 }
 
@@ -1067,7 +1153,7 @@ ndb_index_stat_free(NDB_SHARE *share, in
   DBUG_PRINT("index_stat", ("(index_id:%d index_version:%d",
                             index_id, index_version));
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
 
   uint found= 0;
   Ndb_index_stat *st= share->index_stat_list;
@@ -1078,16 +1164,17 @@ ndb_index_stat_free(NDB_SHARE *share, in
     {
       ndb_index_stat_free(st);
       found++;
+      glob.drop_count++;
+      assert(st->drop_bytes == 0);
+      st->drop_bytes= st->query_bytes + st->clean_bytes;
+      glob.cache_drop_bytes+= st->drop_bytes;
       break;
     }
     st= st->share_next;
   }
 
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
-  glob.drop_count+= found;
   glob.set_status();
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
   DBUG_VOID_RETURN;
 }
 
@@ -1096,7 +1183,8 @@ ndb_index_stat_free(NDB_SHARE *share)
 {
   DBUG_ENTER("ndb_index_stat_free");
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+
   uint found= 0;
   Ndb_index_stat *st;
   while ((st= share->index_stat_list) != 0)
@@ -1109,13 +1197,16 @@ ndb_index_stat_free(NDB_SHARE *share)
     assert(st->lt != Ndb_index_stat::LT_Delete);
     assert(!st->to_delete);
     st->to_delete= true;
+    st->abort_request= true;
     found++;
+    glob.drop_count++;
+    assert(st->drop_bytes == 0);
+    st->drop_bytes+= st->query_bytes + st->clean_bytes;
+    glob.cache_drop_bytes+= st->drop_bytes;
   }
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
-  glob.drop_count+= found;
+
   glob.set_status();
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
   DBUG_VOID_RETURN;
 }
 
@@ -1126,7 +1217,7 @@ ndb_index_stat_find_entry(int index_id,
 {
   DBUG_ENTER("ndb_index_stat_find_entry");
   pthread_mutex_lock(&ndbcluster_mutex);
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   DBUG_PRINT("index_stat", ("find index:%d version:%d table:%d",
                             index_id, index_version, table_id));
 
@@ -1139,7 +1230,7 @@ ndb_index_stat_find_entry(int index_id,
       if (st->index_id == index_id &&
           st->index_version == index_version)
       {
-        pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
+        pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
         pthread_mutex_unlock(&ndbcluster_mutex);
         DBUG_RETURN(st);
       }
@@ -1147,7 +1238,7 @@ ndb_index_stat_find_entry(int index_id,
     }
   }
 
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
   pthread_mutex_unlock(&ndbcluster_mutex);
   DBUG_RETURN(0);
 }
@@ -1179,7 +1270,7 @@ ndb_index_stat_cache_move(Ndb_index_stat
     glob.cache_high_bytes= cache_total;
 }
 
-void
+bool
 ndb_index_stat_cache_clean(Ndb_index_stat *st)
 {
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
@@ -1187,12 +1278,46 @@ ndb_index_stat_cache_clean(Ndb_index_sta
 
   st->is->get_cache_info(infoClean, NdbIndexStat::CacheClean);
   const uint old_clean_bytes= infoClean.m_totalBytes;
-  DBUG_PRINT("index_stat", ("st %s cache clean: clean:%u",
-                            st->id, old_clean_bytes));
+  const uint ref_count= infoClean.m_ref_count;
+  DBUG_PRINT("index_stat", ("st %s cache clean: clean:%u ref_count:%u",
+                            st->id, old_clean_bytes, ref_count));
+  if (ref_count != 0)
+    return false;
   st->is->clean_cache();
   st->clean_bytes= 0;
   assert(glob.cache_clean_bytes >= old_clean_bytes);
   glob.cache_clean_bytes-= old_clean_bytes;
+  return true;
+}
+
+void
+ndb_index_stat_cache_evict(Ndb_index_stat *st)
+{
+  NdbIndexStat::Head head;
+  NdbIndexStat::CacheInfo infoBuild;
+  NdbIndexStat::CacheInfo infoQuery;
+  NdbIndexStat::CacheInfo infoClean;
+  st->is->get_head(head);
+  st->is->get_cache_info(infoBuild, NdbIndexStat::CacheBuild);
+  st->is->get_cache_info(infoQuery, NdbIndexStat::CacheQuery);
+  st->is->get_cache_info(infoClean, NdbIndexStat::CacheClean);
+
+  DBUG_PRINT("index_stat",
+             ("evict table: %u index: %u version: %u"
+              " sample version: %u"
+              " cache bytes build:%u query:%u clean:%u",
+              head.m_tableId, head.m_indexId, head.m_indexVersion,
+              head.m_sampleVersion,
+              infoBuild.m_totalBytes, infoQuery.m_totalBytes, infoClean.m_totalBytes));
+
+  /* Twice to move all caches to clean */
+  ndb_index_stat_cache_move(st);
+  ndb_index_stat_cache_move(st);
+  /* Unused variable release vs debug nonsense */
+  bool ok= false;
+  (void)ok; // USED
+  ok= ndb_index_stat_cache_clean(st);
+  assert(ok);
 }
 
 /* Misc in/out parameters for process steps */
@@ -1231,7 +1356,7 @@ void
 ndb_index_stat_proc_new(Ndb_index_stat_proc &pr)
 {
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   const int lt= Ndb_index_stat::LT_New;
   Ndb_index_stat_list &list= ndb_index_stat_list[lt];
 
@@ -1245,10 +1370,8 @@ ndb_index_stat_proc_new(Ndb_index_stat_p
     assert(pr.lt != lt);
     ndb_index_stat_list_move(st, pr.lt);
   }
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   glob.set_status();
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
 }
 
 void
@@ -1256,8 +1379,8 @@ ndb_index_stat_proc_update(Ndb_index_sta
 {
   if (st->is->update_stat(pr.ndb) == -1)
   {
-    ndb_index_stat_error(st, "update_stat", __LINE__);
     pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+    ndb_index_stat_error(st, 0, "update_stat", __LINE__);
 
     /*
       Turn off force update or else proc_error() thinks
@@ -1311,7 +1434,7 @@ ndb_index_stat_proc_read(Ndb_index_stat_
   if (st->is->read_stat(pr.ndb) == -1)
   {
     pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
-    ndb_index_stat_error(st, "read_stat", __LINE__);
+    ndb_index_stat_error(st, 0, "read_stat", __LINE__);
     const bool force_update= st->force_update;
     ndb_index_stat_force_update(st, false);
 
@@ -1384,6 +1507,7 @@ ndb_index_stat_proc_read(Ndb_index_stat_
 void
 ndb_index_stat_proc_idle(Ndb_index_stat_proc &pr, Ndb_index_stat *st)
 {
+  Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   const Ndb_index_stat_opt &opt= ndb_index_stat_opt;
   const longlong clean_delay= opt.get(Ndb_index_stat_opt::Iclean_delay);
   const longlong check_delay= opt.get(Ndb_index_stat_opt::Icheck_delay);
@@ -1408,7 +1532,10 @@ ndb_index_stat_proc_idle(Ndb_index_stat_
 
   if (st->clean_bytes != 0 && clean_wait <= 0)
   {
-    ndb_index_stat_cache_clean(st);
+    if (ndb_index_stat_cache_clean(st))
+      glob.clean_count++;
+    else
+      glob.pinned_count++;
   }
   if (st->force_update)
   {
@@ -1482,7 +1609,8 @@ ndb_index_stat_proc_check(Ndb_index_stat
   NdbIndexStat::Head head;
   if (st->is->read_head(pr.ndb) == -1)
   {
-    ndb_index_stat_error(st, "read_head", __LINE__);
+    pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+    ndb_index_stat_error(st, 0, "read_head", __LINE__);
     /* no stats is not unexpected error */
     if (st->is->getNdbError().code == NdbIndexStat::NoIndexStats)
     {
@@ -1493,6 +1621,8 @@ ndb_index_stat_proc_check(Ndb_index_stat
     {
       pr.lt= Ndb_index_stat::LT_Error;
     }
+    pthread_cond_broadcast(&ndb_index_stat_thread.stat_cond);
+    pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
     return;
   }
   st->is->get_head(head);
@@ -1537,39 +1667,6 @@ ndb_index_stat_proc_check(Ndb_index_stat
     pr.busy= true;
 }
 
-/* Only evict the caches */
-void
-ndb_index_stat_proc_evict(Ndb_index_stat_proc &pr, Ndb_index_stat *st)
-{
-  Ndb_index_stat_glob &glob= ndb_index_stat_glob;
-
-  NdbIndexStat::Head head;
-  NdbIndexStat::CacheInfo infoBuild;
-  NdbIndexStat::CacheInfo infoQuery;
-  NdbIndexStat::CacheInfo infoClean;
-  st->is->get_head(head);
-  st->is->get_cache_info(infoBuild, NdbIndexStat::CacheBuild);
-  st->is->get_cache_info(infoQuery, NdbIndexStat::CacheQuery);
-  st->is->get_cache_info(infoClean, NdbIndexStat::CacheClean);
-
-  DBUG_PRINT("index_stat",
-             ("evict table: %u index: %u version: %u"
-              " sample version: %u"
-              " cache bytes build:%u query:%u clean:%u",
-              head.m_tableId, head.m_indexId, head.m_indexVersion,
-              head.m_sampleVersion,
-              infoBuild.m_totalBytes, infoQuery.m_totalBytes, infoClean.m_totalBytes));
-
-  /* Twice to move all caches to clean */
-  ndb_index_stat_cache_move(st);
-  ndb_index_stat_cache_move(st);
-  ndb_index_stat_cache_clean(st);
-
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
-  glob.set_status();
-  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-}
-
 /* Check if need to evict more */
 bool
 ndb_index_stat_proc_evict()
@@ -1577,6 +1674,11 @@ ndb_index_stat_proc_evict()
   const Ndb_index_stat_opt &opt= ndb_index_stat_opt;
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   uint curr_size= glob.cache_query_bytes + glob.cache_clean_bytes;
+
+  /* Subtract bytes already scheduled for evict */
+  assert(curr_size >= glob.cache_evict_bytes);
+  curr_size-= glob.cache_evict_bytes;
+
   const uint cache_lowpct= opt.get(Ndb_index_stat_opt::Icache_lowpct);
   const uint cache_limit= opt.get(Ndb_index_stat_opt::Icache_limit);
   if (100 * curr_size <= cache_lowpct * cache_limit)
@@ -1612,6 +1714,9 @@ ndb_index_stat_proc_evict(Ndb_index_stat
   if (!ndb_index_stat_proc_evict())
     return;
 
+  /* Mutex entire routine (protect access_time) */
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+
   /* Create a LRU batch */
   Ndb_index_stat* st_lru_arr[ndb_index_stat_max_evict_batch + 1];
   uint st_lru_cnt= 0;
@@ -1690,19 +1795,19 @@ ndb_index_stat_proc_evict(Ndb_index_stat
 
     Ndb_index_stat *st= st_lru_arr[cnt];
     DBUG_PRINT("index_stat", ("st %s proc evict %s", st->id, list.name));
-    ndb_index_stat_proc_evict(pr, st);
-    pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+
+    /* Entry may have requests.  Cache is evicted at delete. */
     ndb_index_stat_free(st);
-    pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
+    assert(st->evict_bytes == 0);
+    st->evict_bytes= st->query_bytes + st->clean_bytes;
+    glob.cache_evict_bytes+= st->evict_bytes;
     cnt++;
   }
+  if (cnt == batch)
+    pr.busy= true;
 
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   glob.evict_count+= cnt;
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
-
-  if (cnt == batch)
-    pr.busy= true;
 }
 
 void
@@ -1722,6 +1827,9 @@ ndb_index_stat_proc_delete(Ndb_index_sta
   const uint delete_batch= opt.get(Ndb_index_stat_opt::Idelete_batch);
   const uint batch= !pr.end ? delete_batch : ~(uint)0;
 
+  /* Mutex entire routine */
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+
   Ndb_index_stat *st_loop= list.head;
   uint cnt= 0;
   while (st_loop != 0 && cnt < batch)
@@ -1731,12 +1839,26 @@ ndb_index_stat_proc_delete(Ndb_index_sta
     DBUG_PRINT("index_stat", ("st %s proc %s", st->id, list.name));
 
     // adjust global counters at drop
-    pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
     ndb_index_stat_force_update(st, false);
     ndb_index_stat_no_stats(st, false);
-    pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
 
-    ndb_index_stat_proc_evict(pr, st);
+    /*
+      Do not wait for requests to terminate since this could
+      risk stats thread hanging.  Instead try again next time.
+      Presumably clients will eventually notice abort_request.
+    */
+    if (st->ref_count != 0)
+    {
+      DBUG_PRINT("index_stat", ("st %s proc %s: ref_count:%u",
+                 st->id, list.name, st->ref_count));
+      continue;
+    }
+
+    ndb_index_stat_cache_evict(st);
+    assert(glob.cache_drop_bytes >= st->drop_bytes);
+    glob.cache_drop_bytes-= st->drop_bytes;
+    assert(glob.cache_evict_bytes >= st->evict_bytes);
+    glob.cache_evict_bytes-= st->evict_bytes;
     ndb_index_stat_list_remove(st);
     delete st->is;
     delete st;
@@ -1745,7 +1867,6 @@ ndb_index_stat_proc_delete(Ndb_index_sta
   if (cnt == batch)
     pr.busy= true;
 
-  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   glob.set_status();
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
 }
@@ -2004,7 +2125,7 @@ void
 ndb_index_stat_list_verify(Ndb_index_stat_proc &pr)
 {
   const Ndb_index_stat_glob &glob= ndb_index_stat_glob;
-  pthread_mutex_lock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
   pr.cache_query_bytes= 0;
   pr.cache_clean_bytes= 0;
 
@@ -2013,7 +2134,7 @@ ndb_index_stat_list_verify(Ndb_index_sta
 
   assert(glob.cache_query_bytes == pr.cache_query_bytes);
   assert(glob.cache_clean_bytes == pr.cache_clean_bytes);
-  pthread_mutex_unlock(&ndb_index_stat_thread.list_mutex);
+  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
 }
 
 void
@@ -2498,12 +2619,16 @@ ndb_index_stat_round(double x)
   return n;
 }
 
+/*
+  Client waits for query or analyze.  The routines are
+  similar but separated for clarity.
+*/
+
 int
-ndb_index_stat_wait(Ndb_index_stat *st,
-                    uint sample_version,
-                    bool from_analyze)
+ndb_index_stat_wait_query(Ndb_index_stat *st,
+                          const Ndb_index_stat_snap &snap)
 {
-  DBUG_ENTER("ndb_index_stat_wait");
+  DBUG_ENTER("ndb_index_stat_wait_query");
 
   Ndb_index_stat_glob &glob= ndb_index_stat_glob;
   pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
@@ -2515,24 +2640,16 @@ ndb_index_stat_wait(Ndb_index_stat *st,
     int ret= 0;
     if (count == 0)
     {
-      if (!from_analyze)
-      {
-        glob.wait_stats++;
-        glob.query_count++;
-      }
-      else
+      glob.wait_stats++;
+      glob.query_count++;
+      if (st->lt == Ndb_index_stat::LT_Error)
       {
-        glob.wait_update++;
-        glob.analyze_count++;
-      }
-      if (st->lt == Ndb_index_stat::LT_Error && !from_analyze)
-      {
-        err= Ndb_index_stat_error_HAS_ERROR;
+        err= NdbIndexStat::MyHasError;
         break;
       }
       ndb_index_stat_clear_error(st);
     }
-    if (st->no_stats && !from_analyze)
+    if (st->no_stats)
     {
       /* Have detected no stats now or before */
       err= NdbIndexStat::NoIndexStats;
@@ -2543,16 +2660,29 @@ ndb_index_stat_wait(Ndb_index_stat *st,
     {
       /* A new error has occured */
       err= st->error.code;
-      if (!from_analyze)
-        glob.query_error++;
-      else
-        glob.analyze_error++;
+      glob.query_error++;
+      break;
+    }
+    /* Query waits for any samples */
+    if (st->sample_version > 0)
+      break;
+    /*
+      Try to detect changes behind our backs.  Should really not
+      happen but make sure.
+    */
+    if (st->load_time != snap.load_time ||
+        st->sample_version != snap.sample_version)
+    {
+      err= NdbIndexStat::NoIndexStats;
       break;
     }
-    if (st->sample_version > sample_version)
+    if (st->abort_request)
+    {
+      err= NdbIndexStat::MyAbortReq;
       break;
+    }
     count++;
-    DBUG_PRINT("index_stat", ("st %s wait count:%u",
+    DBUG_PRINT("index_stat", ("st %s wait_query count:%u",
                               st->id, count));
     pthread_mutex_lock(&ndb_index_stat_thread.LOCK);
     ndb_index_stat_waiter= true;
@@ -2568,25 +2698,93 @@ ndb_index_stat_wait(Ndb_index_stat *st,
       break;
     }
   }
-  if (!from_analyze)
+  assert(glob.wait_stats != 0);
+  glob.wait_stats--;
+  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
+  if (err != 0)
   {
-    assert(glob.wait_stats != 0);
-    glob.wait_stats--;
+    DBUG_PRINT("index_stat", ("st %s wait_query error: %d",
+                               st->id, err));
+    DBUG_RETURN(err);
   }
-  else
+  DBUG_PRINT("index_stat", ("st %s wait_query ok: sample_version %u -> %u",
+                            st->id, snap.sample_version, st->sample_version));
+  DBUG_RETURN(0);
+}
+
+int
+ndb_index_stat_wait_analyze(Ndb_index_stat *st,
+                            const Ndb_index_stat_snap &snap)
+{
+  DBUG_ENTER("ndb_index_stat_wait_analyze");
+
+  Ndb_index_stat_glob &glob= ndb_index_stat_glob;
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+  int err= 0;
+  uint count= 0;
+  struct timespec abstime;
+  while (true)
   {
-    assert(glob.wait_update != 0);
-    glob.wait_update--;
+    int ret= 0;
+    if (count == 0)
+    {
+      glob.wait_update++;
+      glob.analyze_count++;
+      ndb_index_stat_clear_error(st);
+    }
+    if (st->error.code != 0)
+    {
+      /* A new error has occured */
+      err= st->error.code;
+      glob.analyze_error++;
+      break;
+    }
+    /* Analyze waits for newer samples */
+    if (st->sample_version > snap.sample_version)
+      break;
+    /*
+      Try to detect changes behind our backs.  If another process
+      deleted stats, an analyze here could wait forever.
+    */
+    if (st->load_time != snap.load_time ||
+        st->sample_version != snap.sample_version)
+    {
+      err= NdbIndexStat::AlienUpdate;
+      break;
+    }
+    if (st->abort_request)
+    {
+      err= NdbIndexStat::MyAbortReq;
+      break;
+    }
+    count++;
+    DBUG_PRINT("index_stat", ("st %s wait_analyze count:%u",
+                              st->id, count));
+    pthread_mutex_lock(&ndb_index_stat_thread.LOCK);
+    ndb_index_stat_waiter= true;
+    pthread_cond_signal(&ndb_index_stat_thread.COND);
+    pthread_mutex_unlock(&ndb_index_stat_thread.LOCK);
+    set_timespec(abstime, 1);
+    ret= pthread_cond_timedwait(&ndb_index_stat_thread.stat_cond,
+                                &ndb_index_stat_thread.stat_mutex,
+                                &abstime);
+    if (ret != 0 && ret != ETIMEDOUT)
+    {
+      err= ret;
+      break;
+    }
   }
+  assert(glob.wait_update != 0);
+  glob.wait_update--;
   pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
   if (err != 0)
   {
-    DBUG_PRINT("index_stat", ("st %s wait error: %d",
+    DBUG_PRINT("index_stat", ("st %s wait_analyze error: %d",
                                st->id, err));
     DBUG_RETURN(err);
   }
-  DBUG_PRINT("index_stat", ("st %s wait ok: sample_version %u -> %u",
-                            st->id, sample_version, st->sample_version));
+  DBUG_PRINT("index_stat", ("st %s wait_analyze ok: sample_version %u -> %u",
+                            st->id, snap.sample_version, st->sample_version));
   DBUG_RETURN(0);
 }
 
@@ -2611,37 +2809,51 @@ ha_ndbcluster::ndb_index_stat_query(uint
   compute_index_bounds(ib, key_info, min_key, max_key, from);
   ib.range_no= 0;
 
+  Ndb_index_stat_snap snap;
   Ndb_index_stat *st=
-    ndb_index_stat_get_share(m_share, index, m_table, err, true, false);
+    ndb_index_stat_get_share(m_share, index, m_table, snap, err, true, false);
   if (st == 0)
     DBUG_RETURN(err);
+  /* Now holding reference to st */
 
-  /* Pass old version 0 so existing stats terminates wait at once */
-  err= ndb_index_stat_wait(st, 0, false);
-  if (err != 0)
-    DBUG_RETURN(err);
-  assert(st->sample_version != 0);
-
-  uint8 bound_lo_buffer[NdbIndexStat::BoundBufferBytes];
-  uint8 bound_hi_buffer[NdbIndexStat::BoundBufferBytes];
-  NdbIndexStat::Bound bound_lo(st->is, bound_lo_buffer);
-  NdbIndexStat::Bound bound_hi(st->is, bound_hi_buffer);
-  NdbIndexStat::Range range(bound_lo, bound_hi);
-
-  const NdbRecord* key_record= data.ndb_record_key;
-  if (st->is->convert_range(range, key_record, &ib) == -1)
-  {
-    ndb_index_stat_error(st, "convert_range", __LINE__);
-    DBUG_RETURN(st->error.code);
-  }
-  if (st->is->query_stat(range, stat) == -1)
+  do
   {
-    /* Invalid cache - should remove the entry */
-    ndb_index_stat_error(st, "query_stat", __LINE__);
-    DBUG_RETURN(st->error.code);
+    err= ndb_index_stat_wait_query(st, snap);
+    if (err != 0)
+      break;
+    assert(st->sample_version != 0);
+    uint8 bound_lo_buffer[NdbIndexStat::BoundBufferBytes];
+    uint8 bound_hi_buffer[NdbIndexStat::BoundBufferBytes];
+    NdbIndexStat::Bound bound_lo(st->is, bound_lo_buffer);
+    NdbIndexStat::Bound bound_hi(st->is, bound_hi_buffer);
+    NdbIndexStat::Range range(bound_lo, bound_hi);
+
+    const NdbRecord* key_record= data.ndb_record_key;
+    if (st->is->convert_range(range, key_record, &ib) == -1)
+    {
+      pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+      ndb_index_stat_error(st, 1, "convert_range", __LINE__);
+      err= st->client_error.code;
+      pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
+      break;
+    }
+    if (st->is->query_stat(range, stat) == -1)
+    {
+      /* Invalid cache - should remove the entry */
+      pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+      ndb_index_stat_error(st, 1, "query_stat", __LINE__);
+      err= st->client_error.code;
+      pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
+      break;
+    }
   }
+  while (0);
 
-  DBUG_RETURN(0);
+  /* Release reference to st */
+  pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+  ndb_index_stat_ref_count(st, false);
+  pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
+  DBUG_RETURN(err);
 }
 
 int
@@ -2713,50 +2925,62 @@ ha_ndbcluster::ndb_index_stat_analyze(Nd
 {
   DBUG_ENTER("ha_ndbcluster::ndb_index_stat_analyze");
 
-  struct {
-    uint sample_version;
-    uint error_count;
-  } old[MAX_INDEXES];
-
-  int err= 0;
-  uint i;
+  struct Req {
+    Ndb_index_stat *st;
+    Ndb_index_stat_snap snap;
+    int err;
+    Req() { st= 0; err= 0; }
+  };
+  Req req[MAX_INDEXES];
 
   /* Force stats update on each index */
-  for (i= 0; i < inx_count; i++)
+  for (uint i= 0; i < inx_count; i++)
   {
+    Req &r= req[i];
     uint inx= inx_list[i];
     const NDB_INDEX_DATA &data= m_index[inx];
     const NDBINDEX *index= data.index;
     DBUG_PRINT("index_stat", ("force update: %s", index->getName()));
 
-    Ndb_index_stat *st=
-      ndb_index_stat_get_share(m_share, index, m_table, err, true, true);
-    if (st == 0)
-      DBUG_RETURN(err);
-
-    old[i].sample_version= st->sample_version;
-    old[i].error_count= st->error_count;
+    r.st=
+      ndb_index_stat_get_share(m_share, index, m_table, r.snap, r.err, true, true);
+    assert((r.st != 0) == (r.err == 0));
+    /* Now holding reference to r.st if r.err == 0 */
   }
 
-  /* Wait for each update (or error) */
-  for (i = 0; i < inx_count; i++)
+  /* Wait for each update */
+  for (uint i = 0; i < inx_count; i++)
   {
+    Req &r= req[i];
     uint inx= inx_list[i];
     const NDB_INDEX_DATA &data= m_index[inx];
     const NDBINDEX *index= data.index;
-    DBUG_PRINT("index_stat", ("wait for update: %s", index->getName()));
+    (void)index; // USED
 
-    Ndb_index_stat *st=
-      ndb_index_stat_get_share(m_share, index, m_table, err, false, false);
-    if (st == 0)
-      DBUG_RETURN(err);
+    if (r.err == 0)
+    {
+      DBUG_PRINT("index_stat", ("wait for update: %s", index->getName()));
+      r.err=ndb_index_stat_wait_analyze(r.st, r.snap);
+      /* Release reference to r.st */
+      pthread_mutex_lock(&ndb_index_stat_thread.stat_mutex);
+      ndb_index_stat_ref_count(r.st, false);
+      pthread_mutex_unlock(&ndb_index_stat_thread.stat_mutex);
+    }
+  }
 
-    err= ndb_index_stat_wait(st, old[i].sample_version, true);
-    if (err != 0)
-      DBUG_RETURN(err);
+  /* Return first error if any */
+  int err= 0;
+  for (uint i= 0; i < inx_count; i++)
+  {
+    Req &r= req[i];
+    if (r.err != 0)
+    {
+      err= r.err;
+      break;
+    }
   }
 
-  DBUG_RETURN(0);
+  DBUG_RETURN(err);
 }
 
 #endif

=== modified file 'sql/ha_ndb_index_stat.h'
--- a/sql/ha_ndb_index_stat.h	2011-11-22 08:56:11 +0000
+++ b/sql/ha_ndb_index_stat.h	2011-12-18 12:20:44 +0000
@@ -40,10 +40,10 @@ public:
   pthread_cond_t COND;
   pthread_cond_t COND_ready;
 
-  /* protect entry lists where needed */
-  pthread_mutex_t list_mutex;
-
-  /* protect and signal changes in stats entries */
+  /*
+    protect stats entry lists where needed
+    protect and signal changes in stats entries
+  */
   pthread_mutex_t stat_mutex;
   pthread_cond_t stat_cond;
 
@@ -82,5 +82,8 @@ compute_index_bounds(NdbIndexScanOperati
 
 /* request on stats entry with recent error was ignored */
 #define Ndb_index_stat_error_HAS_ERROR          9003
+ 
+/* stats thread aborted request on stats entry */
+#define Ndb_index_stat_error_ABORT_REQUEST      9004
 
 #endif

=== modified file 'sql/ha_ndbcluster.cc'
--- a/sql/ha_ndbcluster.cc	2011-12-14 02:31:38 +0000
+++ b/sql/ha_ndbcluster.cc	2011-12-19 14:23:00 +0000
@@ -1286,7 +1286,9 @@ void ha_ndbcluster::set_rec_per_key()
             /* no stats is not unexpected error */
             err != NdbIndexStat::NoIndexStats &&
             /* warning was printed at first error */
-            err != Ndb_index_stat_error_HAS_ERROR)
+            err != NdbIndexStat::MyHasError &&
+            /* stats thread aborted request */
+            err != NdbIndexStat::MyAbortReq)
         {
           push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                               ER_CANT_GET_STAT, /* pun? */
@@ -12893,7 +12895,9 @@ ha_ndbcluster::records_in_range(uint inx
           /* no stats is not unexpected error */
           err != NdbIndexStat::NoIndexStats &&
           /* warning was printed at first error */
-          err != Ndb_index_stat_error_HAS_ERROR)
+          err != NdbIndexStat::MyHasError &&
+          /* stats thread aborted request */
+          err != NdbIndexStat::MyAbortReq)
       {
         push_warning_printf(thd, MYSQL_ERROR::WARN_LEVEL_WARN,
                             ER_CANT_GET_STAT, /* pun? */

=== renamed file 'storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java.not_yet' => 'storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java'
--- a/storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java.not_yet	2011-12-15 12:54:52 +0000
+++ b/storage/ndb/clusterj/clusterj-test/src/main/java/testsuite/clusterj/DbugTest.java	2011-12-18 18:37:39 +0000
@@ -25,15 +25,28 @@ import com.mysql.clusterj.Dbug;
  */
 public class DbugTest extends AbstractClusterJTest{
 
+    static String tmpFileName = System.getProperty("MYSQL_TMP_DIR", "/tmp") + "/clusterj-test-dbug";
+
     public boolean getDebug() {
         return false;
     }
 
     public void test() {
-        String originalState = "t";
-        String newState = "d,jointx:o,/tmp/jointx";
         Dbug dbug = ClusterJHelper.newDbug();
-        errorIfEqual("Failed to get new Dbug", null, dbug);
+        if (dbug == null) {
+            // nothing else can be tested
+            fail("Failed to get new Dbug");
+        }
+        if (dbug.get() == null) {
+            // ndbclient is compiled without DBUG; just make sure nothing blows up
+            dbug.set("nothing");
+            dbug.push("nada");
+            dbug.pop();
+            dbug.print("keyword", "message");
+            return;
+        }
+        String originalState = "t";
+        String newState = "d,jointx:o," + tmpFileName;
         dbug.set(originalState);
         String actualState = dbug.get();
         errorIfNotEqual("Failed to set original state", originalState, actualState);
@@ -45,15 +58,17 @@ public class DbugTest extends AbstractCl
         errorIfNotEqual("Failed to pop original state", originalState, actualState);
 
         dbug = ClusterJHelper.newDbug();
-        dbug.output("/fully/qualified").flush().debug(new String[] {"how", "now"}).push();
+        dbug.output(tmpFileName).flush().debug(new String[] {"a", "b", "c", "d", "e", "f"}).push();
         actualState = dbug.get();
-        errorIfNotEqual("Wrong state created", "d,how,now:O,/fully/qualified", actualState);
+        // keywords are stored LIFO
+        errorIfNotEqual("Wrong state created", "d,f,e,d,c,b,a:O," + tmpFileName, actualState);
         dbug.pop();
 
         dbug = ClusterJHelper.newDbug();
-        dbug.append("partly/qualified").trace().debug(new String[] {"brown", "cow"}).set();
+        dbug.append(tmpFileName).trace().debug("a,b,c,d,e,f").set();
         actualState = dbug.get();
-        errorIfNotEqual("Wrong state created", "t:d,brown,cow:a,partly/qualified", actualState);
+        // keywords are stored LIFO
+        errorIfNotEqual("Wrong state created", "d,f,e,d,c,b,a:a," + tmpFileName + ":t", actualState);
         dbug.pop();
 
         failOnError();

=== modified file 'storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java'
--- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java	2011-03-08 00:44:56 +0000
+++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionImpl.java	2011-12-18 11:47:59 +0000
@@ -45,13 +45,6 @@ public class ClusterConnectionImpl
     static final Logger logger = LoggerFactoryService.getFactory()
             .getInstance(com.mysql.clusterj.core.store.ClusterConnection.class);
 
-    /** Load the ndbjtie system library */
-    static {
-        loadSystemLibrary("ndbclient");
-        // initialize the charset map
-        Utility.getCharsetMap();
-    }
-
     /** Ndb_cluster_connection is wrapped by ClusterConnection */
     protected Ndb_cluster_connection clusterConnection;
 
@@ -77,40 +70,6 @@ public class ClusterConnectionImpl
         logger.info(local.message("INFO_Create_Cluster_Connection", connectString, nodeId));
     }
 
-    static protected void loadSystemLibrary(String name) {
-        String message;
-        String path;
-        try {
-            System.loadLibrary(name);
-        } catch (UnsatisfiedLinkError e) {
-            path = getLoadLibraryPath();
-            message = local.message("ERR_Failed_Loading_Library",
-                    name, path, "UnsatisfiedLinkError", e.getLocalizedMessage());
-            logger.fatal(message);
-            throw e;
-        } catch (SecurityException e) {
-            path = getLoadLibraryPath();
-            message = local.message("ERR_Failed_Loading_Library",
-                    name, path, "SecurityException", e.getLocalizedMessage());
-            logger.fatal(message);
-            throw e;
-        }
-    }
-
-    /**
-     * @return the load library path or the Exception string
-     */
-    private static String getLoadLibraryPath() {
-        String path;
-        try {
-            path = System.getProperty("java.library.path");
-        } catch (Exception ex) {
-            path = "<Exception: " + ex.getMessage() + ">";
-        }
-        return path;
-    }
-
-
     public void connect(int connectRetries, int connectDelay, boolean verbose) {
         checkConnection();
         int returnCode = clusterConnection.connect(connectRetries, connectDelay, verbose?1:0);

=== modified file 'storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionServiceImpl.java'
--- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionServiceImpl.java	2011-03-08 00:44:56 +0000
+++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/ClusterConnectionServiceImpl.java	2011-12-18 11:47:59 +0000
@@ -40,7 +40,30 @@ public class ClusterConnectionServiceImp
         LoggerFactoryService.getFactory().registerLogger("com.mysql.clusterj.tie");
     }
 
+    /** Load the ndbclient system library only once */
+    static boolean ndbclientLoaded = false;
+
+    static protected void loadSystemLibrary(String name) {
+        // this is not thread-protected so it might be executed multiple times but no matter
+        if (ndbclientLoaded) {
+            return;
+        }
+        try {
+            System.loadLibrary(name);
+            // initialize the charset map
+            Utility.getCharsetMap();
+            ndbclientLoaded = true;
+        } catch (Throwable e) {
+            String path = getLoadLibraryPath();
+            String message = local.message("ERR_Failed_Loading_Library",
+                    name, path, e.getClass(), e.getLocalizedMessage());
+            logger.fatal(message);
+            throw new ClusterJFatalUserException(message, e);
+        }
+    }
+
     public ClusterConnection create(String connectString, int nodeId) {
+        loadSystemLibrary("ndbclient");
         try {
             return new ClusterConnectionImpl(connectString, nodeId);
         } catch (ClusterJFatalUserException cjex) {
@@ -52,4 +75,17 @@ public class ClusterConnectionServiceImp
         }
     }
 
+    /**
+     * @return the load library path or the Exception string
+     */
+    private static String getLoadLibraryPath() {
+        String path;
+        try {
+            path = System.getProperty("java.library.path");
+        } catch (Exception ex) {
+            path = "<Exception: " + ex.getMessage() + ">";
+        }
+        return path;
+    }
+
 }

=== modified file 'storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbugImpl.java'
--- a/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbugImpl.java	2011-12-15 02:18:32 +0000
+++ b/storage/ndb/clusterj/clusterj-tie/src/main/java/com/mysql/clusterj/tie/DbugImpl.java	2011-12-18 11:47:59 +0000
@@ -47,6 +47,11 @@ public class DbugImpl implements Dbug {
     Character fileStrategy = 'o';
     String debugList;
     
+    public DbugImpl() {
+        // Load the native library so we can set up debugging before anything else
+        ClusterConnectionServiceImpl.loadSystemLibrary("ndbclient");
+    }
+
     public String get() {
         ByteBuffer buffer = ByteBuffer.allocateDirect(DBUG_SIZE);
         String result = Utils.dbugExplain(buffer, DBUG_SIZE);

=== modified file 'storage/ndb/include/ndbapi/NdbIndexStat.hpp'
--- a/storage/ndb/include/ndbapi/NdbIndexStat.hpp	2011-09-29 09:23:04 +0000
+++ b/storage/ndb/include/ndbapi/NdbIndexStat.hpp	2011-12-19 14:23:00 +0000
@@ -104,7 +104,16 @@ public:
     HaveSysTables = 4244, // create error if all sys tables exist
     NoSysEvents = 4710,
     BadSysEvents = BadSysTables,
-    HaveSysEvents = 746
+    HaveSysEvents = 746,
+    /*
+     * Following are for mysqld.  Most are consumed by mysqld itself
+     * and should therefore not be seen by clients.
+     */
+    MyNotAllow = 4721,    // stats thread not open for requests
+    MyNotFound = 4722,    // stats entry unexpectedly not found
+    MyHasError = 4723,    // request ignored due to recent error
+    MyAbortReq = 4724,    // request aborted by stats thread
+    AlienUpdate = 4725    // somebody else messed with stats
   };
 
   /*
@@ -180,6 +189,7 @@ public:
     Uint32 m_totalBytes;  // total bytes memory used
     Uint64 m_save_time;   // microseconds to read stats into cache
     Uint64 m_sort_time;   // microseconds to sort the cache
+    Uint32 m_ref_count;   // in use by query_stat
     // end v4 fields
   };
 

=== modified file 'storage/ndb/src/kernel/blocks/LocalProxy.cpp'
--- a/storage/ndb/src/kernel/blocks/LocalProxy.cpp	2011-10-07 18:15:59 +0000
+++ b/storage/ndb/src/kernel/blocks/LocalProxy.cpp	2011-12-19 14:23:00 +0000
@@ -27,8 +27,6 @@ LocalProxy::LocalProxy(BlockNumber block
   for (i = 0; i < MaxWorkers; i++)
     c_worker[i] = 0;
 
-  c_ssIdSeq = 0;
-
   c_typeOfStart = NodeState::ST_ILLEGAL_TYPE;
   c_masterNodeId = ZNIL;
 

=== modified file 'storage/ndb/src/kernel/blocks/LocalProxy.hpp'
--- a/storage/ndb/src/kernel/blocks/LocalProxy.hpp	2011-10-07 08:07:21 +0000
+++ b/storage/ndb/src/kernel/blocks/LocalProxy.hpp	2011-12-19 13:58:28 +0000
@@ -192,11 +192,17 @@ protected:
 
   template <class Ss>
   Ss& ssSeize() {
-    const Uint32 base = SsIdBase;
-    const Uint32 mask = ~base;
-    const Uint32 ssId = base | c_ssIdSeq;
-    c_ssIdSeq = (c_ssIdSeq + 1) & mask;
-    return ssSeize<Ss>(ssId);
+    SsPool<Ss>& sp = Ss::pool(this);
+    Ss* ssptr = ssSearch<Ss>(0);
+    ndbrequire(ssptr != 0);
+    // Use position in array as ssId
+    UintPtr pos = ssptr - sp.m_pool;
+    Uint32 ssId = Uint32(pos) + 1;
+    new (ssptr) Ss;
+    ssptr->m_ssId = ssId;
+    sp.m_usage++;
+    D("ssSeize()" << V(sp.m_usage) << hex << V(ssId) << " " << Ss::name());
+    return *ssptr;
   }
 
   template <class Ss>

=== modified file 'storage/ndb/src/ndbapi/NdbIndexStat.cpp'
--- a/storage/ndb/src/ndbapi/NdbIndexStat.cpp	2011-09-02 09:01:29 +0000
+++ b/storage/ndb/src/ndbapi/NdbIndexStat.cpp	2011-12-19 14:23:00 +0000
@@ -369,6 +369,7 @@ NdbIndexStat::clean_cache()
 void
 NdbIndexStat::get_cache_info(CacheInfo& info, CacheType type) const
 {
+  NdbMutex_Lock(m_impl.m_query_mutex);
   const NdbIndexStatImpl::Cache* c = 0;
   switch (type) {
   case CacheBuild:
@@ -387,6 +388,7 @@ NdbIndexStat::get_cache_info(CacheInfo&
   info.m_totalBytes = 0;
   info.m_save_time = 0;
   info.m_sort_time = 0;
+  info.m_ref_count = 0;
   while (c != 0)
   {
     info.m_count += 1;
@@ -395,10 +397,12 @@ NdbIndexStat::get_cache_info(CacheInfo&
     info.m_totalBytes += c->m_keyBytes + c->m_valueBytes + c->m_addrBytes;
     info.m_save_time += c->m_save_time;
     info.m_sort_time += c->m_sort_time;
+    info.m_ref_count += c->m_ref_count;
     c = c->m_nextClean;
   }
   // build and query cache have at most one instance
   require(type == CacheClean || info.m_count <= 1);
+  NdbMutex_Unlock(m_impl.m_query_mutex);
 }
 
 // read

=== modified file 'storage/ndb/src/ndbapi/NdbIndexStatImpl.cpp'
--- a/storage/ndb/src/ndbapi/NdbIndexStatImpl.cpp	2011-12-10 19:02:03 +0000
+++ b/storage/ndb/src/ndbapi/NdbIndexStatImpl.cpp	2011-12-19 14:23:00 +0000
@@ -23,6 +23,7 @@
 #include <NdbSqlUtil.hpp>
 #include <NdbRecord.hpp>
 #include <NdbEventOperation.hpp>
+#include <NdbSleep.h>
 #include "NdbIndexStatImpl.hpp"
 
 #undef min
@@ -1460,6 +1461,8 @@ NdbIndexStatImpl::Cache::Cache()
   // performance
   m_save_time = 0;
   m_sort_time = 0;
+  // in use by query_stat
+  m_ref_count = 0;
 }
 
 int
@@ -2002,6 +2005,21 @@ NdbIndexStatImpl::convert_range(Range& r
       return -1;
     }
   }
+
+#ifdef VM_TRACE
+  {
+    const char* p = NdbEnv_GetEnv("NDB_INDEX_STAT_RANGE_ERROR", (char*)0, 0);
+    if (p != 0 && strchr("1Y", p[0]) != 0)
+    {
+      if (rand() % 10 == 0)
+      {
+        setError(InternalError, __LINE__, NdbIndexStat::InternalError);
+        return -1;
+      }
+    }
+  }
+#endif
+
   return 0;
 }
 
@@ -2033,23 +2051,41 @@ int
 NdbIndexStatImpl::query_stat(const Range& range, Stat& stat)
 {
   NdbMutex_Lock(m_query_mutex);
-  const Cache* cacheTmp = m_cacheQuery;
-  NdbMutex_Unlock(m_query_mutex);
-
-  if (unlikely(cacheTmp == 0))
+  if (unlikely(m_cacheQuery == 0))
   {
+    NdbMutex_Unlock(m_query_mutex);
     setError(UsageError, __LINE__);
     return -1;
   }
-  const Cache& c = *cacheTmp;
+  const Cache& c = *m_cacheQuery;
   if (unlikely(!c.m_valid))
   {
+    NdbMutex_Unlock(m_query_mutex);
     setError(InvalidCache, __LINE__);
     return -1;
   }
+  c.m_ref_count++;
+  NdbMutex_Unlock(m_query_mutex);
+
+#ifdef VM_TRACE
+  {
+    const char* p = NdbEnv_GetEnv("NDB_INDEX_STAT_SLOW_QUERY", (char*)0, 0);
+    if (p != 0 && strchr("1Y", p[0]) != 0)
+    {
+      int ms = 1 + rand() % 20;
+      NdbSleep_MilliSleep(ms);
+    }
+  }
+#endif
 
+  // clients run these in parallel
   query_interpolate(c, range, stat);
   query_normalize(c, stat.m_value);
+
+  NdbMutex_Lock(m_query_mutex);
+  assert(c.m_ref_count != 0);
+  c.m_ref_count--;
+  NdbMutex_Unlock(m_query_mutex);
   return 0;
 }
 

=== modified file 'storage/ndb/src/ndbapi/NdbIndexStatImpl.hpp'
--- a/storage/ndb/src/ndbapi/NdbIndexStatImpl.hpp	2011-09-19 10:13:44 +0000
+++ b/storage/ndb/src/ndbapi/NdbIndexStatImpl.hpp	2011-12-19 14:23:00 +0000
@@ -70,7 +70,7 @@ public:
   Cache* m_cacheBuild;
   Cache* m_cacheQuery;
   Cache* m_cacheClean;
-  // mutex for query cache switch, memory barrier would do
+  // mutex for query cache switch and reference count
   NdbMutex* m_query_mutex;
   NdbEventOperation* m_eventOp;
   Mem* m_mem_handler;
@@ -185,6 +185,8 @@ public:
     // performance
     mutable Uint64 m_save_time;
     mutable Uint64 m_sort_time;
+    // in use by query_stat
+    mutable uint m_ref_count;
     Cache();
     // pos is index < sampleCount, addr is offset in keyArray
     uint get_keyaddr(uint pos) const;

=== modified file 'storage/ndb/src/ndbapi/ndberror.c'
--- a/storage/ndb/src/ndbapi/ndberror.c	2011-11-29 17:32:59 +0000
+++ b/storage/ndb/src/ndbapi/ndberror.c	2011-12-19 14:23:00 +0000
@@ -552,6 +552,11 @@ ErrorBundle ErrorCodes[] = {
   { 4718, DMEC, IE, "Index stats samples data or memory cache is invalid" },
   { 4719, DMEC, IE, "Index stats internal error" },
   { 4720, DMEC, AE, "Index stats sys tables " NDB_INDEX_STAT_PREFIX " partly missing or invalid" },
+  { 4721, DMEC, IE, "Mysqld: index stats thread not open for requests" },
+  { 4722, DMEC, IE, "Mysqld: index stats entry unexpectedly not found" },
+  { 4723, DMEC, AE, "Mysqld: index stats request ignored due to recent error" },
+  { 4724, DMEC, AE, "Mysqld: index stats request aborted by stats thread" },
+  { 4725, DMEC, AE, "Index stats were deleted by another process" },
   
   /**
    * Still uncategorized

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.5-cluster-7.2 branch (jonas.oreland:3737 to 3738) Jonas Oreland19 Dec