List:Commits« Previous MessageNext Message »
From:Mikael Ronstrom Date:January 27 2012 4:19pm
Subject:bzr push into mysql-5.5-cluster-7.2 branch (mikael.ronstrom:3739 to 3741)
View as plain text  
 3741 Mikael Ronstrom	2012-01-27
      add static to local method

    modified:
      storage/ndb/src/kernel/vm/mt.cpp
 3740 Mikael Ronstrom	2012-01-27
      Enhanced handling of send buffer memory

    modified:
      storage/ndb/include/mgmapi/mgmapi_config_parameters.h
      storage/ndb/include/transporter/TransporterRegistry.hpp
      storage/ndb/src/common/transporter/TCP_Transporter.cpp
      storage/ndb/src/common/transporter/TransporterRegistry.cpp
      storage/ndb/src/kernel/ndbd.cpp
      storage/ndb/src/kernel/vm/Configuration.cpp
      storage/ndb/src/kernel/vm/Emulator.hpp
      storage/ndb/src/kernel/vm/dummy_nonmt.cpp
      storage/ndb/src/kernel/vm/mt.cpp
      storage/ndb/src/mgmsrv/ConfigInfo.cpp
      storage/ndb/src/ndbapi/TransporterFacade.cpp
 3739 Mikael Ronstrom	2012-01-26
      Remove pack of non-existing nodes, remove extra printout

    modified:
      storage/ndb/src/common/transporter/Transporter.hpp
      storage/ndb/src/kernel/vm/mt.cpp
=== modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h'
--- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	revid:mikael.ronstrom@stripped
@@ -206,6 +206,7 @@
 #define CFG_NODE_ARBIT_RANK           200
 #define CFG_NODE_ARBIT_DELAY          201
 #define CFG_RESERVED_SEND_BUFFER_MEMORY 202
+#define CFG_EXTRA_SEND_BUFFER_MEMORY  203
 
 #define CFG_MIN_LOGLEVEL          250
 #define CFG_LOGLEVEL_STARTUP      250

=== modified file 'storage/ndb/include/transporter/TransporterRegistry.hpp'
--- a/storage/ndb/include/transporter/TransporterRegistry.hpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/include/transporter/TransporterRegistry.hpp	revid:mikael.ronstrom@stripped
@@ -270,8 +270,10 @@ public:
    *
    * Argument is the value of config parameter TotalSendBufferMemory. If 0,
    * a default will be used of sum(max send buffer) over all transporters.
+   * The second is the config parameter ExtraSendBufferMemory
    */
-  void allocate_send_buffers(Uint64 total_send_buffer);
+  void allocate_send_buffers(Uint64 total_send_buffer,
+                             Uint64 extra_send_buffer);
 
   /**
    * Get sum of max send buffer over all transporters, to be used as a default

=== modified file 'storage/ndb/src/common/transporter/TCP_Transporter.cpp'
--- a/storage/ndb/src/common/transporter/TCP_Transporter.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/common/transporter/TCP_Transporter.cpp	revid:mikael.ronstrom@stripped
@@ -112,7 +112,10 @@ TCP_Transporter::TCP_Transporter(Transpo
   setIf(sockOptTcpMaxSeg, conf->tcp.tcpMaxsegSize, 0);
 
   m_overload_limit = overload_limit(conf);
-  m_slowdown_limit = conf->tcp.sendBufferSize/2;
+  /**
+   * Always set slowdown limit to 60% of overload limit
+   */
+  m_slowdown_limit = m_overload_limit * 6 / 10; 
 }
 
 

=== modified file 'storage/ndb/src/common/transporter/TransporterRegistry.cpp'
--- a/storage/ndb/src/common/transporter/TransporterRegistry.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/common/transporter/TransporterRegistry.cpp	revid:mikael.ronstrom@stripped
@@ -250,8 +250,10 @@ TransporterRegistry::TransporterRegistry
   DBUG_VOID_RETURN;
 }
 
+#define MIN_SEND_BUFFER_SIZE 4 * 1024 * 1024
 void
-TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer)
+TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer,
+                                           Uint64 extra_send_buffer)
 {
   if (!m_use_default_send_buffer)
     return;
@@ -259,6 +261,21 @@ TransporterRegistry::allocate_send_buffe
   if (total_send_buffer == 0)
     total_send_buffer = get_total_max_send_buffer();
 
+  total_send_buffer += extra_send_buffer;
+
+  if (!extra_send_buffer)
+  {
+    /**
+      * If extra send buffer memory is 0 it means we can decide on an
+      * appropriate value for it. We select to always ensure that the
+      * minimum send buffer memory is 4M, otherwise we simply don't
+      * add any extra send buffer memory at all.
+      */
+    if (total_send_buffer < MIN_SEND_BUFFER_SIZE)
+    {
+      total_send_buffer = (Uint64)MIN_SEND_BUFFER_SIZE;
+    }
+  }
   if (m_send_buffers)
   {
     /* Send buffers already allocated -> resize the buffer pages */

=== modified file 'storage/ndb/src/kernel/ndbd.cpp'
--- a/storage/ndb/src/kernel/ndbd.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/ndbd.cpp	revid:mikael.ronstrom@stripped
@@ -214,15 +214,28 @@ init_global_memory_manager(EmulatorData 
   Uint32 sbpages = 0;
   if (globalTransporterRegistry.get_using_default_send_buffer() == false)
   {
-    Uint64 mem = globalTransporterRegistry.get_total_max_send_buffer();
+    Uint64 mem;
+    Uint32 tot_mem = 0;
+
+    ndb_mgm_get_int_parameter(p, CFG_TOTAL_SEND_BUFFER_MEMORY, &tot_mem);
+    if (tot_mem)
+    {
+      mem = (Uint64)tot_mem;
+    }
+    else
+    {
+      mem = globalTransporterRegistry.get_total_max_send_buffer();
+    }
     sbpages = Uint32((mem + GLOBAL_PAGE_SIZE - 1) / GLOBAL_PAGE_SIZE);
     Resource_limit rl;
-    /*
-      Add one allocation of 32 pages per thread since each thread is
-      likely to seize at least one such in addition to the memory
-      needed to buffer in front of transporters
-    */
-    sbpages += 32 * get_total_number_of_block_threads();
+    /**
+     * Add extra send buffer pages for NDB multithreaded case
+     */
+    Uint64 extra_mem;
+    ndb_mgm_get_int64_parameter(p, CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_mem);
+    Uint32 extra_mem_pages = Uint32((extra_mem + GLOBAL_PAGE_SIZE - 1) /
+                                    GLOBAL_PAGE_SIZE);
+    sbpages += mt_get_extra_send_buffer_pages(sbpages, extra_mem_pages);
     rl.m_min = sbpages;
     rl.m_max = sbpages;
     rl.m_resource_id = RG_TRANSPORTER_BUFFERS;

=== modified file 'storage/ndb/src/kernel/vm/Configuration.cpp'
--- a/storage/ndb/src/kernel/vm/Configuration.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/Configuration.cpp	revid:mikael.ronstrom@stripped
@@ -316,7 +316,10 @@ Configuration::setupConfiguration(){
 
   Uint32 total_send_buffer = 0;
   iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
-  globalTransporterRegistry.allocate_send_buffers(total_send_buffer);
+  Uint64 extra_send_buffer = 0;
+  iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+  globalTransporterRegistry.allocate_send_buffers(total_send_buffer,
+                                                  extra_send_buffer);
   
   if(iter.get(CFG_DB_NO_SAVE_MSGS, &_maxErrorLogs)){
     ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, "Invalid configuration fetched", 

=== modified file 'storage/ndb/src/kernel/vm/Emulator.hpp'
--- a/storage/ndb/src/kernel/vm/Emulator.hpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/Emulator.hpp	revid:mikael.ronstrom@stripped
@@ -78,9 +78,10 @@ struct EmulatorData {
 extern struct EmulatorData globalEmulatorData;
 
 /**
- * Compute total number of block threads in data node
+ * Get number of extra send buffer pages to use
  */
-Uint32 get_total_number_of_block_threads(void);
+Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                                      Uint32 extra_mem_pages);
 
 /**
  * Compute no of pages to be used as job-buffer

=== modified file 'storage/ndb/src/kernel/vm/dummy_nonmt.cpp'
--- a/storage/ndb/src/kernel/vm/dummy_nonmt.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/dummy_nonmt.cpp	revid:mikael.ronstrom@stripped
@@ -44,9 +44,12 @@ mt_get_instance_count(Uint32 block)
   return 0;
 }
 
-Uint32 get_total_number_of_block_threads(void)
+Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                                      Uint32 extra_mem_pages)
 {
-  return 1;
+  (void)curr_num_pages;
+  (void)extra_mem_pages;
+  return 0;
 }
 
 Uint32
@@ -55,7 +58,6 @@ compute_jb_pages(struct EmulatorData*)
   return 0;
 }
 
-
 bool
 NdbIsMultiThreaded()
 {

=== modified file 'storage/ndb/src/kernel/vm/mt.cpp'
--- a/storage/ndb/src/kernel/vm/mt.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/mt.cpp	revid:mikael.ronstrom@stripped
@@ -554,7 +554,7 @@ public:
     {
       T *save_last = find_last(m_freelist);
       T *new_list = m_global_pool->seize_list(mm, rg,
-                                              fill_level - m_free,
+                                              m_alloc_size,
                                               &alloced);
       m_free += alloced;
       if (save_last)
@@ -845,6 +845,20 @@ struct thr_tq
 };
 
 /*
+ * THR_SEND_BUFFER_ALLOC_SIZE is the amount of 32k pages allocated
+ * when we allocate pages from the global pool of send buffers to
+ * the thread_local_pool (which is local to a thread).
+ */
+#define THR_SEND_BUFFER_ALLOC_SIZE 32
+/*
+ * THR_MINIMUM_SEND_BUFFERS is the minimum amount of send buffer pages
+ * which is kept in thread_local_pool before starting execution of a
+ * new loop in a block thread to execute signals.
+ */
+#define THR_MINIMUM_SEND_BUFFERS 32
+#define THR_SEND_BUFFER_MAX_FREE \
+  (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS)
+/*
  * Max number of thread-local job buffers to keep before releasing to
  * global pool.
  */
@@ -914,7 +928,9 @@ struct thr_send_queue
 struct thr_data
 {
   thr_data() : m_jba_write_lock("jbalock"),
-               m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX) {}
+               m_send_buffer_pool(0,
+                                  THR_SEND_BUFFER_MAX_FREE,
+                                  THR_SEND_BUFFER_ALLOC_SIZE) {}
 
   thr_wait m_waiter;
   unsigned m_thr_no;
@@ -1170,7 +1186,9 @@ struct thr_send_thread_instance
                m_awake(FALSE),
                m_thread(NULL),
                m_waiter_struct(),
-               m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX)
+               m_send_buffer_pool(0,
+                                  THR_SEND_BUFFER_MAX_FREE,
+                                  THR_SEND_BUFFER_ALLOC_SIZE)
   {}
   Uint32 m_instance_no;
   Uint32 m_watchdog_counter;
@@ -3781,7 +3799,7 @@ mt_job_thread_main(void *thr_arg)
     watchDogCounter = 11;
     if (!selfptr->m_send_buffer_pool.fill(g_thr_repository.m_mm,
                                           RG_TRANSPORTER_BUFFERS,
-                                          THR_FREE_BUF_MAX))
+                                          THR_MINIMUM_SEND_BUFFERS))
     {
       pack_send_buffers(selfptr);
     }
@@ -4213,7 +4231,7 @@ rep_init(struct thr_repository* rep, uns
 #include "ThreadConfig.hpp"
 #include <signaldata/StartOrd.hpp>
 
-Uint32
+static Uint32
 get_total_number_of_block_threads(void)
 {
   return (NUM_MAIN_THREADS +
@@ -4222,6 +4240,73 @@ get_total_number_of_block_threads(void)
           globalData.ndbMtReceiveThreads);
 }
 
+static Uint32
+get_num_nodes()
+{
+  Uint32 count = 0;
+  for (Uint32 nodeId = 1; nodeId < MAX_NODES; nodeId++)
+  {
+    if (globalTransporterRegistry.get_transporter(nodeId))
+    {
+      count++;
+    }
+  }
+  return count;
+}
+
+/**
+ * This function returns the amount of extra send buffer pages
+ * that we should allocate in addition to the amount allocated
+ * for each node send buffer.
+ */
+#define MIN_SEND_BUFFER_GENERAL (512) //16M
+#define MIN_SEND_BUFFER_PER_NODE (8) //256k
+#define MIN_SEND_BUFFER_PER_THREAD (64) //2M
+Uint32
+mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                               Uint32 extra_mem_pages)
+{
+  Uint32 num_threads = get_total_number_of_block_threads();
+  Uint32 num_nodes = get_num_nodes();
+
+  /**
+   * First allocate 2M for each thread since we allocate 1M every
+   * time we allocate and also we ensure there is also a minimum
+   * of 1M of send buffer in each thread. Thus we can easily have
+   * 2M of send buffer just to keep the contention around the
+   * send buffer page spinlock small. This memory we add independent
+   * of the configuration settings since the user cannot be
+   * expected to handle this and also since we could change this
+   * behaviour at any time.
+   */
+  Uint32 extra_pages = 
+    (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS) *
+      num_threads;
+
+  extra_pages += extra_mem_pages;
+  if (!extra_mem_pages)
+  {
+    /**
+     * The user have set extra send buffer memory to 0 and left for us
+     * to decide on our own how much extra memory is needed.
+     * 
+     * We'll make sure that we have at least a minimum of 16M +
+     * 2M per thread + 256k per node. If we have this based on
+     * curr_num_pages and our local additions we don't add
+     * anything more, if we don't come up to this level we add to
+     * reach this minimum level.
+     */
+    Uint32 min_pages = MIN_SEND_BUFFER_GENERAL +
+      (MIN_SEND_BUFFER_PER_NODE * num_nodes) +
+      (MIN_SEND_BUFFER_PER_THREAD * num_threads);
+    if ((curr_num_pages + extra_pages) < min_pages)
+    {
+      extra_pages = min_pages - curr_num_pages;
+    }
+  }
+  return extra_pages;
+}
+
 Uint32
 compute_jb_pages(struct EmulatorData * ed)
 {

=== modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp'
--- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp	revid:mikael.ronstrom@stripped
@@ -1688,11 +1688,24 @@ const ConfigInfo::ParamInfo ConfigInfo::
     "true"},
 
   {
+    CFG_EXTRA_SEND_BUFFER_MEMORY,
+    "ExtraSendBufferMemory",
+    DB_TOKEN,
+    "Extra send buffer memory to use for send buffers in all transporters",
+    ConfigInfo::CI_USED,
+    false,
+    ConfigInfo::CI_INT64,
+    "0",
+    "0",
+    "32G"
+  },
+
+  {
     CFG_TOTAL_SEND_BUFFER_MEMORY,
     "TotalSendBufferMemory",
     DB_TOKEN,
     "Total memory to use for send buffers in all transporters",
-    ConfigInfo::CI_USED,
+    ConfigInfo::CI_DEPRECATED,
     false,
     ConfigInfo::CI_INT,
     "0",
@@ -1707,7 +1720,7 @@ const ConfigInfo::ParamInfo ConfigInfo::
     "Amount of bytes (out of TotalSendBufferMemory) to reserve for connection\n"
     "between data nodes. This memory will not be available for connections to\n"
     "management server or API nodes.",
-    ConfigInfo::CI_USED,
+    ConfigInfo::CI_DEPRECATED,
     false,
     ConfigInfo::CI_INT,
     "0",

=== modified file 'storage/ndb/src/ndbapi/TransporterFacade.cpp'
--- a/storage/ndb/src/ndbapi/TransporterFacade.cpp	revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/ndbapi/TransporterFacade.cpp	revid:mikael.ronstrom@stripped
@@ -643,7 +643,10 @@ TransporterFacade::configure(NodeId node
   // Configure send buffers
   Uint32 total_send_buffer = 0;
   iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
-  theTransporterRegistry->allocate_send_buffers(total_send_buffer);
+  Uint64 extra_send_buffer = 0;
+  iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+  theTransporterRegistry->allocate_send_buffers(total_send_buffer,
+                                                extra_send_buffer);
 
   Uint32 auto_reconnect=1;
   iter.get(CFG_AUTO_RECONNECT, &auto_reconnect);

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.5-cluster-7.2 branch (mikael.ronstrom:3739 to 3741) Mikael Ronstrom30 Jan