List:Commits« Previous MessageNext Message »
From:Jonas Oreland Date:January 30 2012 2:41pm
Subject:bzr push into mysql-5.1-telco-7.0 branch (jonas.oreland:4833 to 4834)
Bug#13633845
View as plain text  
 4834 Jonas Oreland	2012-01-30
      ndb - bug#13633845 - try to improve inconsistent transporter send buffer handling

    modified:
      storage/ndb/include/mgmapi/mgmapi_config_parameters.h
      storage/ndb/include/transporter/TransporterRegistry.hpp
      storage/ndb/src/common/transporter/TransporterRegistry.cpp
      storage/ndb/src/kernel/ndbd.cpp
      storage/ndb/src/kernel/vm/Configuration.cpp
      storage/ndb/src/kernel/vm/Emulator.hpp
      storage/ndb/src/kernel/vm/dummy_nonmt.cpp
      storage/ndb/src/kernel/vm/mt.cpp
      storage/ndb/src/mgmsrv/ConfigInfo.cpp
      storage/ndb/src/ndbapi/TransporterFacade.cpp
 4833 Jonas Oreland	2012-01-30
      ndb - bug#13618181 - patch 2 - make sure to release pages before (maybe) looping on sb->m_force_send

    modified:
      storage/ndb/src/kernel/vm/mt.cpp
=== modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h'
--- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	2011-12-01 10:45:07 +0000
+++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h	2012-01-30 14:28:55 +0000
@@ -206,6 +206,7 @@
 #define CFG_NODE_ARBIT_RANK           200
 #define CFG_NODE_ARBIT_DELAY          201
 #define CFG_RESERVED_SEND_BUFFER_MEMORY 202
+#define CFG_EXTRA_SEND_BUFFER_MEMORY  203
 
 #define CFG_MIN_LOGLEVEL          250
 #define CFG_LOGLEVEL_STARTUP      250

=== modified file 'storage/ndb/include/transporter/TransporterRegistry.hpp'
--- a/storage/ndb/include/transporter/TransporterRegistry.hpp	2012-01-19 11:48:54 +0000
+++ b/storage/ndb/include/transporter/TransporterRegistry.hpp	2012-01-30 14:28:55 +0000
@@ -270,8 +270,10 @@ public:
    *
    * Argument is the value of config parameter TotalSendBufferMemory. If 0,
    * a default will be used of sum(max send buffer) over all transporters.
+   * The second is the config parameter ExtraSendBufferMemory
    */
-  void allocate_send_buffers(Uint64 total_send_buffer);
+  void allocate_send_buffers(Uint64 total_send_buffer,
+                             Uint64 extra_send_buffer);
 
   /**
    * Get sum of max send buffer over all transporters, to be used as a default

=== modified file 'storage/ndb/src/common/transporter/TransporterRegistry.cpp'
--- a/storage/ndb/src/common/transporter/TransporterRegistry.cpp	2012-01-23 19:53:55 +0000
+++ b/storage/ndb/src/common/transporter/TransporterRegistry.cpp	2012-01-30 14:28:55 +0000
@@ -250,8 +250,11 @@ TransporterRegistry::TransporterRegistry
   DBUG_VOID_RETURN;
 }
 
+#define MIN_SEND_BUFFER_SIZE (4 * 1024 * 1024)
+
 void
-TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer)
+TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer,
+                                           Uint64 extra_send_buffer)
 {
   if (!m_use_default_send_buffer)
     return;
@@ -259,6 +262,22 @@ TransporterRegistry::allocate_send_buffe
   if (total_send_buffer == 0)
     total_send_buffer = get_total_max_send_buffer();
 
+  total_send_buffer += extra_send_buffer;
+
+  if (!extra_send_buffer)
+  {
+    /**
+     * If extra send buffer memory is 0 it means we can decide on an
+     * appropriate value for it. We select to always ensure that the
+     * minimum send buffer memory is 4M, otherwise we simply don't
+     * add any extra send buffer memory at all.
+     */
+    if (total_send_buffer < MIN_SEND_BUFFER_SIZE)
+    {
+      total_send_buffer = (Uint64)MIN_SEND_BUFFER_SIZE;
+    }
+  }
+
   if (m_send_buffers)
   {
     /* Send buffers already allocated -> resize the buffer pages */

=== modified file 'storage/ndb/src/kernel/ndbd.cpp'
--- a/storage/ndb/src/kernel/ndbd.cpp	2012-01-16 07:14:30 +0000
+++ b/storage/ndb/src/kernel/ndbd.cpp	2012-01-30 14:28:55 +0000
@@ -214,11 +214,40 @@ init_global_memory_manager(EmulatorData
   Uint32 sbpages = 0;
   if (globalTransporterRegistry.get_using_default_send_buffer() == false)
   {
-    Uint64 mem = globalTransporterRegistry.get_total_max_send_buffer();
+    Uint64 mem;
+    {
+      Uint32 tot_mem = 0;
+      ndb_mgm_get_int_parameter(p, CFG_TOTAL_SEND_BUFFER_MEMORY, &tot_mem);
+      if (tot_mem)
+      {
+        mem = (Uint64)tot_mem;
+      }
+      else
+      {
+        mem = globalTransporterRegistry.get_total_max_send_buffer();
+      }
+    }
+
     sbpages = Uint32((mem + GLOBAL_PAGE_SIZE - 1) / GLOBAL_PAGE_SIZE);
+
+    /**
+     * Add extra send buffer pages for NDB multithreaded case
+     */
+    {
+      Uint64 extra_mem;
+      ndb_mgm_get_int64_parameter(p, CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_mem);
+      Uint32 extra_mem_pages = Uint32((extra_mem + GLOBAL_PAGE_SIZE - 1) /
+                                      GLOBAL_PAGE_SIZE);
+      sbpages += mt_get_extra_send_buffer_pages(sbpages, extra_mem_pages);
+    }
+
     Resource_limit rl;
     rl.m_min = sbpages;
-    rl.m_max = sbpages;
+    /**
+     * allow over allocation (from SharedGlobalMemory) of up to 25% of
+     *   totally allocated SendBuffer
+     */
+    rl.m_max = sbpages + (sbpages * 25) / 100;
     rl.m_resource_id = RG_TRANSPORTER_BUFFERS;
     ed.m_mem_manager->set_resource_limit(rl);
   }

=== modified file 'storage/ndb/src/kernel/vm/Configuration.cpp'
--- a/storage/ndb/src/kernel/vm/Configuration.cpp	2012-01-16 07:14:30 +0000
+++ b/storage/ndb/src/kernel/vm/Configuration.cpp	2012-01-30 14:28:55 +0000
@@ -316,7 +316,10 @@ Configuration::setupConfiguration(){
 
   Uint32 total_send_buffer = 0;
   iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
-  globalTransporterRegistry.allocate_send_buffers(total_send_buffer);
+  Uint64 extra_send_buffer = 0;
+  iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+  globalTransporterRegistry.allocate_send_buffers(total_send_buffer,
+                                                  extra_send_buffer);
   
   if(iter.get(CFG_DB_NO_SAVE_MSGS, &_maxErrorLogs)){
     ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, "Invalid configuration fetched", 

=== modified file 'storage/ndb/src/kernel/vm/Emulator.hpp'
--- a/storage/ndb/src/kernel/vm/Emulator.hpp	2012-01-19 06:21:05 +0000
+++ b/storage/ndb/src/kernel/vm/Emulator.hpp	2012-01-30 14:28:55 +0000
@@ -78,6 +78,12 @@ struct EmulatorData {
 extern struct EmulatorData globalEmulatorData;
 
 /**
+ * Get number of extra send buffer pages to use
+ */
+Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                                      Uint32 extra_mem_pages);
+
+/**
  * Compute no of pages to be used as job-buffer
  */
 Uint32 compute_jb_pages(struct EmulatorData* ed);

=== modified file 'storage/ndb/src/kernel/vm/dummy_nonmt.cpp'
--- a/storage/ndb/src/kernel/vm/dummy_nonmt.cpp	2011-10-07 08:07:21 +0000
+++ b/storage/ndb/src/kernel/vm/dummy_nonmt.cpp	2012-01-30 14:28:55 +0000
@@ -45,6 +45,15 @@ mt_get_instance_count(Uint32 block)
 }
 
 Uint32
+mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                               Uint32 extra_mem_pages)
+{
+  (void)curr_num_pages;
+  (void)extra_mem_pages;
+  return 0;
+}
+
+Uint32
 compute_jb_pages(struct EmulatorData*)
 {
   return 0;

=== modified file 'storage/ndb/src/kernel/vm/mt.cpp'
--- a/storage/ndb/src/kernel/vm/mt.cpp	2012-01-30 13:13:37 +0000
+++ b/storage/ndb/src/kernel/vm/mt.cpp	2012-01-30 14:28:55 +0000
@@ -4109,6 +4109,83 @@ rep_init(struct thr_repository* rep, uns
 #include "ThreadConfig.hpp"
 #include <signaldata/StartOrd.hpp>
 
+static Uint32
+get_total_number_of_block_threads(void)
+{
+  return (NUM_MAIN_THREADS +
+          globalData.ndbMtLqhThreads + 
+          globalData.ndbMtTcThreads +
+          globalData.ndbMtReceiveThreads);
+}
+
+static Uint32
+get_num_nodes()
+{
+  Uint32 count = 0;
+  for (Uint32 nodeId = 1; nodeId < MAX_NODES; nodeId++)
+  {
+    if (globalTransporterRegistry.get_transporter(nodeId))
+    {
+      count++;
+    }
+  }
+  return count;
+}
+
+/**
+ * This function returns the amount of extra send buffer pages
+ * that we should allocate in addition to the amount allocated
+ * for each node send buffer.
+ */
+#define MIN_SEND_BUFFER_GENERAL (512) //16M
+#define MIN_SEND_BUFFER_PER_NODE (8) //256k
+#define MIN_SEND_BUFFER_PER_THREAD (64) //2M
+
+Uint32
+mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+                               Uint32 extra_mem_pages)
+{
+  Uint32 num_threads = get_total_number_of_block_threads();
+  Uint32 num_nodes = get_num_nodes();
+
+  Uint32 extra_pages = extra_mem_pages;
+
+  /**
+   * Add 2M for each thread since we allocate 1M every
+   * time we allocate and also we ensure there is also a minimum
+   * of 1M of send buffer in each thread. Thus we can easily have
+   * 2M of send buffer just to keep the contention around the
+   * send buffer page spinlock small. This memory we add independent
+   * of the configuration settings since the user cannot be
+   * expected to handle this and also since we could change this
+   * behaviour at any time.
+   */
+  extra_pages += num_threads * 0;
+
+  if (extra_mem_pages == 0)
+  {
+    /**
+     * The user have set extra send buffer memory to 0 and left for us
+     * to decide on our own how much extra memory is needed.
+     *
+     * We'll make sure that we have at least a minimum of 16M +
+     * 2M per thread + 256k per node. If we have this based on
+     * curr_num_pages and our local additions we don't add
+     * anything more, if we don't come up to this level we add to
+     * reach this minimum level.
+     */
+    Uint32 min_pages = MIN_SEND_BUFFER_GENERAL +
+      (MIN_SEND_BUFFER_PER_NODE * num_nodes) +
+      (MIN_SEND_BUFFER_PER_THREAD * num_threads);
+
+    if ((curr_num_pages + extra_pages) < min_pages)
+    {
+      extra_pages = min_pages - curr_num_pages;
+    }
+  }
+  return extra_pages;
+}
+
 Uint32
 compute_jb_pages(struct EmulatorData * ed)
 {

=== modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp'
--- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp	2012-01-25 09:18:33 +0000
+++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp	2012-01-30 14:28:55 +0000
@@ -1688,11 +1688,24 @@ const ConfigInfo::ParamInfo ConfigInfo::
     "true"},
 
   {
+    CFG_EXTRA_SEND_BUFFER_MEMORY,
+    "ExtraSendBufferMemory",
+    DB_TOKEN,
+    "Extra send buffer memory to use for send buffers in all transporters",
+    ConfigInfo::CI_USED,
+    false,
+    ConfigInfo::CI_INT64,
+    "0",
+    "0",
+    "32G"
+  },
+
+  {
     CFG_TOTAL_SEND_BUFFER_MEMORY,
     "TotalSendBufferMemory",
     DB_TOKEN,
     "Total memory to use for send buffers in all transporters",
-    ConfigInfo::CI_USED,
+    ConfigInfo::CI_DEPRECATED,
     false,
     ConfigInfo::CI_INT,
     "0",
@@ -1707,7 +1720,7 @@ const ConfigInfo::ParamInfo ConfigInfo::
     "Amount of bytes (out of TotalSendBufferMemory) to reserve for connection\n"
     "between data nodes. This memory will not be available for connections to\n"
     "management server or API nodes.",
-    ConfigInfo::CI_USED,
+    ConfigInfo::CI_DEPRECATED,
     false,
     ConfigInfo::CI_INT,
     "0",

=== modified file 'storage/ndb/src/ndbapi/TransporterFacade.cpp'
--- a/storage/ndb/src/ndbapi/TransporterFacade.cpp	2012-01-16 08:25:04 +0000
+++ b/storage/ndb/src/ndbapi/TransporterFacade.cpp	2012-01-30 14:28:55 +0000
@@ -643,7 +643,10 @@ TransporterFacade::configure(NodeId node
   // Configure send buffers
   Uint32 total_send_buffer = 0;
   iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
-  theTransporterRegistry->allocate_send_buffers(total_send_buffer);
+  Uint64 extra_send_buffer = 0;
+  iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+  theTransporterRegistry->allocate_send_buffers(total_send_buffer,
+                                                extra_send_buffer);
 
   Uint32 auto_reconnect=1;
   iter.get(CFG_AUTO_RECONNECT, &auto_reconnect);

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-5.1-telco-7.0 branch (jonas.oreland:4833 to 4834)Bug#13633845Jonas Oreland31 Jan