3741 Mikael Ronstrom 2012-01-27
add static to local method
modified:
storage/ndb/src/kernel/vm/mt.cpp
3740 Mikael Ronstrom 2012-01-27
Enhanced handling of send buffer memory
modified:
storage/ndb/include/mgmapi/mgmapi_config_parameters.h
storage/ndb/include/transporter/TransporterRegistry.hpp
storage/ndb/src/common/transporter/TCP_Transporter.cpp
storage/ndb/src/common/transporter/TransporterRegistry.cpp
storage/ndb/src/kernel/ndbd.cpp
storage/ndb/src/kernel/vm/Configuration.cpp
storage/ndb/src/kernel/vm/Emulator.hpp
storage/ndb/src/kernel/vm/dummy_nonmt.cpp
storage/ndb/src/kernel/vm/mt.cpp
storage/ndb/src/mgmsrv/ConfigInfo.cpp
storage/ndb/src/ndbapi/TransporterFacade.cpp
3739 Mikael Ronstrom 2012-01-26
Remove pack of non-existing nodes, remove extra printout
modified:
storage/ndb/src/common/transporter/Transporter.hpp
storage/ndb/src/kernel/vm/mt.cpp
=== modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h'
--- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h revid:mikael.ronstrom@stripped
+++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h revid:mikael.ronstrom@stripped
@@ -206,6 +206,7 @@
#define CFG_NODE_ARBIT_RANK 200
#define CFG_NODE_ARBIT_DELAY 201
#define CFG_RESERVED_SEND_BUFFER_MEMORY 202
+#define CFG_EXTRA_SEND_BUFFER_MEMORY 203
#define CFG_MIN_LOGLEVEL 250
#define CFG_LOGLEVEL_STARTUP 250
=== modified file 'storage/ndb/include/transporter/TransporterRegistry.hpp'
--- a/storage/ndb/include/transporter/TransporterRegistry.hpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/include/transporter/TransporterRegistry.hpp revid:mikael.ronstrom@stripped
@@ -270,8 +270,10 @@ public:
*
* Argument is the value of config parameter TotalSendBufferMemory. If 0,
* a default will be used of sum(max send buffer) over all transporters.
+ * The second is the config parameter ExtraSendBufferMemory
*/
- void allocate_send_buffers(Uint64 total_send_buffer);
+ void allocate_send_buffers(Uint64 total_send_buffer,
+ Uint64 extra_send_buffer);
/**
* Get sum of max send buffer over all transporters, to be used as a default
=== modified file 'storage/ndb/src/common/transporter/TCP_Transporter.cpp'
--- a/storage/ndb/src/common/transporter/TCP_Transporter.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/common/transporter/TCP_Transporter.cpp revid:mikael.ronstrom@stripped
@@ -112,7 +112,10 @@ TCP_Transporter::TCP_Transporter(Transpo
setIf(sockOptTcpMaxSeg, conf->tcp.tcpMaxsegSize, 0);
m_overload_limit = overload_limit(conf);
- m_slowdown_limit = conf->tcp.sendBufferSize/2;
+ /**
+ * Always set slowdown limit to 60% of overload limit
+ */
+ m_slowdown_limit = m_overload_limit * 6 / 10;
}
=== modified file 'storage/ndb/src/common/transporter/TransporterRegistry.cpp'
--- a/storage/ndb/src/common/transporter/TransporterRegistry.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/common/transporter/TransporterRegistry.cpp revid:mikael.ronstrom@stripped
@@ -250,8 +250,10 @@ TransporterRegistry::TransporterRegistry
DBUG_VOID_RETURN;
}
+#define MIN_SEND_BUFFER_SIZE 4 * 1024 * 1024
void
-TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer)
+TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer,
+ Uint64 extra_send_buffer)
{
if (!m_use_default_send_buffer)
return;
@@ -259,6 +261,21 @@ TransporterRegistry::allocate_send_buffe
if (total_send_buffer == 0)
total_send_buffer = get_total_max_send_buffer();
+ total_send_buffer += extra_send_buffer;
+
+ if (!extra_send_buffer)
+ {
+ /**
+ * If extra send buffer memory is 0 it means we can decide on an
+ * appropriate value for it. We select to always ensure that the
+ * minimum send buffer memory is 4M, otherwise we simply don't
+ * add any extra send buffer memory at all.
+ */
+ if (total_send_buffer < MIN_SEND_BUFFER_SIZE)
+ {
+ total_send_buffer = (Uint64)MIN_SEND_BUFFER_SIZE;
+ }
+ }
if (m_send_buffers)
{
/* Send buffers already allocated -> resize the buffer pages */
=== modified file 'storage/ndb/src/kernel/ndbd.cpp'
--- a/storage/ndb/src/kernel/ndbd.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/ndbd.cpp revid:mikael.ronstrom@stripped
@@ -214,15 +214,28 @@ init_global_memory_manager(EmulatorData
Uint32 sbpages = 0;
if (globalTransporterRegistry.get_using_default_send_buffer() == false)
{
- Uint64 mem = globalTransporterRegistry.get_total_max_send_buffer();
+ Uint64 mem;
+ Uint32 tot_mem = 0;
+
+ ndb_mgm_get_int_parameter(p, CFG_TOTAL_SEND_BUFFER_MEMORY, &tot_mem);
+ if (tot_mem)
+ {
+ mem = (Uint64)tot_mem;
+ }
+ else
+ {
+ mem = globalTransporterRegistry.get_total_max_send_buffer();
+ }
sbpages = Uint32((mem + GLOBAL_PAGE_SIZE - 1) / GLOBAL_PAGE_SIZE);
Resource_limit rl;
- /*
- Add one allocation of 32 pages per thread since each thread is
- likely to seize at least one such in addition to the memory
- needed to buffer in front of transporters
- */
- sbpages += 32 * get_total_number_of_block_threads();
+ /**
+ * Add extra send buffer pages for NDB multithreaded case
+ */
+ Uint64 extra_mem;
+ ndb_mgm_get_int64_parameter(p, CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_mem);
+ Uint32 extra_mem_pages = Uint32((extra_mem + GLOBAL_PAGE_SIZE - 1) /
+ GLOBAL_PAGE_SIZE);
+ sbpages += mt_get_extra_send_buffer_pages(sbpages, extra_mem_pages);
rl.m_min = sbpages;
rl.m_max = sbpages;
rl.m_resource_id = RG_TRANSPORTER_BUFFERS;
=== modified file 'storage/ndb/src/kernel/vm/Configuration.cpp'
--- a/storage/ndb/src/kernel/vm/Configuration.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/Configuration.cpp revid:mikael.ronstrom@stripped
@@ -316,7 +316,10 @@ Configuration::setupConfiguration(){
Uint32 total_send_buffer = 0;
iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
- globalTransporterRegistry.allocate_send_buffers(total_send_buffer);
+ Uint64 extra_send_buffer = 0;
+ iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+ globalTransporterRegistry.allocate_send_buffers(total_send_buffer,
+ extra_send_buffer);
if(iter.get(CFG_DB_NO_SAVE_MSGS, &_maxErrorLogs)){
ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, "Invalid configuration fetched",
=== modified file 'storage/ndb/src/kernel/vm/Emulator.hpp'
--- a/storage/ndb/src/kernel/vm/Emulator.hpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/Emulator.hpp revid:mikael.ronstrom@stripped
@@ -78,9 +78,10 @@ struct EmulatorData {
extern struct EmulatorData globalEmulatorData;
/**
- * Compute total number of block threads in data node
+ * Get number of extra send buffer pages to use
*/
-Uint32 get_total_number_of_block_threads(void);
+Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+ Uint32 extra_mem_pages);
/**
* Compute no of pages to be used as job-buffer
=== modified file 'storage/ndb/src/kernel/vm/dummy_nonmt.cpp'
--- a/storage/ndb/src/kernel/vm/dummy_nonmt.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/dummy_nonmt.cpp revid:mikael.ronstrom@stripped
@@ -44,9 +44,12 @@ mt_get_instance_count(Uint32 block)
return 0;
}
-Uint32 get_total_number_of_block_threads(void)
+Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+ Uint32 extra_mem_pages)
{
- return 1;
+ (void)curr_num_pages;
+ (void)extra_mem_pages;
+ return 0;
}
Uint32
@@ -55,7 +58,6 @@ compute_jb_pages(struct EmulatorData*)
return 0;
}
-
bool
NdbIsMultiThreaded()
{
=== modified file 'storage/ndb/src/kernel/vm/mt.cpp'
--- a/storage/ndb/src/kernel/vm/mt.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/kernel/vm/mt.cpp revid:mikael.ronstrom@stripped
@@ -554,7 +554,7 @@ public:
{
T *save_last = find_last(m_freelist);
T *new_list = m_global_pool->seize_list(mm, rg,
- fill_level - m_free,
+ m_alloc_size,
&alloced);
m_free += alloced;
if (save_last)
@@ -845,6 +845,20 @@ struct thr_tq
};
/*
+ * THR_SEND_BUFFER_ALLOC_SIZE is the amount of 32k pages allocated
+ * when we allocate pages from the global pool of send buffers to
+ * the thread_local_pool (which is local to a thread).
+ */
+#define THR_SEND_BUFFER_ALLOC_SIZE 32
+/*
+ * THR_MINIMUM_SEND_BUFFERS is the minimum amount of send buffer pages
+ * which is kept in thread_local_pool before starting execution of a
+ * new loop in a block thread to execute signals.
+ */
+#define THR_MINIMUM_SEND_BUFFERS 32
+#define THR_SEND_BUFFER_MAX_FREE \
+ (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS)
+/*
* Max number of thread-local job buffers to keep before releasing to
* global pool.
*/
@@ -914,7 +928,9 @@ struct thr_send_queue
struct thr_data
{
thr_data() : m_jba_write_lock("jbalock"),
- m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX) {}
+ m_send_buffer_pool(0,
+ THR_SEND_BUFFER_MAX_FREE,
+ THR_SEND_BUFFER_ALLOC_SIZE) {}
thr_wait m_waiter;
unsigned m_thr_no;
@@ -1170,7 +1186,9 @@ struct thr_send_thread_instance
m_awake(FALSE),
m_thread(NULL),
m_waiter_struct(),
- m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX)
+ m_send_buffer_pool(0,
+ THR_SEND_BUFFER_MAX_FREE,
+ THR_SEND_BUFFER_ALLOC_SIZE)
{}
Uint32 m_instance_no;
Uint32 m_watchdog_counter;
@@ -3781,7 +3799,7 @@ mt_job_thread_main(void *thr_arg)
watchDogCounter = 11;
if (!selfptr->m_send_buffer_pool.fill(g_thr_repository.m_mm,
RG_TRANSPORTER_BUFFERS,
- THR_FREE_BUF_MAX))
+ THR_MINIMUM_SEND_BUFFERS))
{
pack_send_buffers(selfptr);
}
@@ -4213,7 +4231,7 @@ rep_init(struct thr_repository* rep, uns
#include "ThreadConfig.hpp"
#include <signaldata/StartOrd.hpp>
-Uint32
+static Uint32
get_total_number_of_block_threads(void)
{
return (NUM_MAIN_THREADS +
@@ -4222,6 +4240,73 @@ get_total_number_of_block_threads(void)
globalData.ndbMtReceiveThreads);
}
+static Uint32
+get_num_nodes()
+{
+ Uint32 count = 0;
+ for (Uint32 nodeId = 1; nodeId < MAX_NODES; nodeId++)
+ {
+ if (globalTransporterRegistry.get_transporter(nodeId))
+ {
+ count++;
+ }
+ }
+ return count;
+}
+
+/**
+ * This function returns the amount of extra send buffer pages
+ * that we should allocate in addition to the amount allocated
+ * for each node send buffer.
+ */
+#define MIN_SEND_BUFFER_GENERAL (512) //16M
+#define MIN_SEND_BUFFER_PER_NODE (8) //256k
+#define MIN_SEND_BUFFER_PER_THREAD (64) //2M
+Uint32
+mt_get_extra_send_buffer_pages(Uint32 curr_num_pages,
+ Uint32 extra_mem_pages)
+{
+ Uint32 num_threads = get_total_number_of_block_threads();
+ Uint32 num_nodes = get_num_nodes();
+
+ /**
+ * First allocate 2M for each thread since we allocate 1M every
+ * time we allocate and also we ensure there is also a minimum
+ * of 1M of send buffer in each thread. Thus we can easily have
+ * 2M of send buffer just to keep the contention around the
+ * send buffer page spinlock small. This memory we add independent
+ * of the configuration settings since the user cannot be
+ * expected to handle this and also since we could change this
+ * behaviour at any time.
+ */
+ Uint32 extra_pages =
+ (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS) *
+ num_threads;
+
+ extra_pages += extra_mem_pages;
+ if (!extra_mem_pages)
+ {
+ /**
+ * The user have set extra send buffer memory to 0 and left for us
+ * to decide on our own how much extra memory is needed.
+ *
+ * We'll make sure that we have at least a minimum of 16M +
+ * 2M per thread + 256k per node. If we have this based on
+ * curr_num_pages and our local additions we don't add
+ * anything more, if we don't come up to this level we add to
+ * reach this minimum level.
+ */
+ Uint32 min_pages = MIN_SEND_BUFFER_GENERAL +
+ (MIN_SEND_BUFFER_PER_NODE * num_nodes) +
+ (MIN_SEND_BUFFER_PER_THREAD * num_threads);
+ if ((curr_num_pages + extra_pages) < min_pages)
+ {
+ extra_pages = min_pages - curr_num_pages;
+ }
+ }
+ return extra_pages;
+}
+
Uint32
compute_jb_pages(struct EmulatorData * ed)
{
=== modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp'
--- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp revid:mikael.ronstrom@stripped
@@ -1688,11 +1688,24 @@ const ConfigInfo::ParamInfo ConfigInfo::
"true"},
{
+ CFG_EXTRA_SEND_BUFFER_MEMORY,
+ "ExtraSendBufferMemory",
+ DB_TOKEN,
+ "Extra send buffer memory to use for send buffers in all transporters",
+ ConfigInfo::CI_USED,
+ false,
+ ConfigInfo::CI_INT64,
+ "0",
+ "0",
+ "32G"
+ },
+
+ {
CFG_TOTAL_SEND_BUFFER_MEMORY,
"TotalSendBufferMemory",
DB_TOKEN,
"Total memory to use for send buffers in all transporters",
- ConfigInfo::CI_USED,
+ ConfigInfo::CI_DEPRECATED,
false,
ConfigInfo::CI_INT,
"0",
@@ -1707,7 +1720,7 @@ const ConfigInfo::ParamInfo ConfigInfo::
"Amount of bytes (out of TotalSendBufferMemory) to reserve for connection\n"
"between data nodes. This memory will not be available for connections to\n"
"management server or API nodes.",
- ConfigInfo::CI_USED,
+ ConfigInfo::CI_DEPRECATED,
false,
ConfigInfo::CI_INT,
"0",
=== modified file 'storage/ndb/src/ndbapi/TransporterFacade.cpp'
--- a/storage/ndb/src/ndbapi/TransporterFacade.cpp revid:mikael.ronstrom@stripped
+++ b/storage/ndb/src/ndbapi/TransporterFacade.cpp revid:mikael.ronstrom@stripped
@@ -643,7 +643,10 @@ TransporterFacade::configure(NodeId node
// Configure send buffers
Uint32 total_send_buffer = 0;
iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer);
- theTransporterRegistry->allocate_send_buffers(total_send_buffer);
+ Uint64 extra_send_buffer = 0;
+ iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer);
+ theTransporterRegistry->allocate_send_buffers(total_send_buffer,
+ extra_send_buffer);
Uint32 auto_reconnect=1;
iter.get(CFG_AUTO_RECONNECT, &auto_reconnect);
No bundle (reason: useless for push emails).
| Thread |
|---|
| • bzr push into mysql-5.5-cluster-7.2 branch (mikael.ronstrom:3739 to 3741) | Mikael Ronstrom | 30 Jan |
