From: Mikael Ronstrom Date: January 27 2012 4:19pm Subject: bzr push into mysql-5.5-cluster-7.2 branch (mikael.ronstrom:3739 to 3741) List-Archive: http://lists.mysql.com/commits/142605 Message-Id: <201201271620.q0RGKQYK024127@dator6> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 3741 Mikael Ronstrom 2012-01-27 add static to local method modified: storage/ndb/src/kernel/vm/mt.cpp 3740 Mikael Ronstrom 2012-01-27 Enhanced handling of send buffer memory modified: storage/ndb/include/mgmapi/mgmapi_config_parameters.h storage/ndb/include/transporter/TransporterRegistry.hpp storage/ndb/src/common/transporter/TCP_Transporter.cpp storage/ndb/src/common/transporter/TransporterRegistry.cpp storage/ndb/src/kernel/ndbd.cpp storage/ndb/src/kernel/vm/Configuration.cpp storage/ndb/src/kernel/vm/Emulator.hpp storage/ndb/src/kernel/vm/dummy_nonmt.cpp storage/ndb/src/kernel/vm/mt.cpp storage/ndb/src/mgmsrv/ConfigInfo.cpp storage/ndb/src/ndbapi/TransporterFacade.cpp 3739 Mikael Ronstrom 2012-01-26 Remove pack of non-existing nodes, remove extra printout modified: storage/ndb/src/common/transporter/Transporter.hpp storage/ndb/src/kernel/vm/mt.cpp === modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h' --- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h revid:mikael.ronstrom@stripped +++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h revid:mikael.ronstrom@stripped @@ -206,6 +206,7 @@ #define CFG_NODE_ARBIT_RANK 200 #define CFG_NODE_ARBIT_DELAY 201 #define CFG_RESERVED_SEND_BUFFER_MEMORY 202 +#define CFG_EXTRA_SEND_BUFFER_MEMORY 203 #define CFG_MIN_LOGLEVEL 250 #define CFG_LOGLEVEL_STARTUP 250 === modified file 'storage/ndb/include/transporter/TransporterRegistry.hpp' --- a/storage/ndb/include/transporter/TransporterRegistry.hpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/include/transporter/TransporterRegistry.hpp revid:mikael.ronstrom@stripped @@ -270,8 +270,10 @@ public: * * Argument is the value of config parameter TotalSendBufferMemory. If 0, * a default will be used of sum(max send buffer) over all transporters. + * The second is the config parameter ExtraSendBufferMemory */ - void allocate_send_buffers(Uint64 total_send_buffer); + void allocate_send_buffers(Uint64 total_send_buffer, + Uint64 extra_send_buffer); /** * Get sum of max send buffer over all transporters, to be used as a default === modified file 'storage/ndb/src/common/transporter/TCP_Transporter.cpp' --- a/storage/ndb/src/common/transporter/TCP_Transporter.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/common/transporter/TCP_Transporter.cpp revid:mikael.ronstrom@stripped @@ -112,7 +112,10 @@ TCP_Transporter::TCP_Transporter(Transpo setIf(sockOptTcpMaxSeg, conf->tcp.tcpMaxsegSize, 0); m_overload_limit = overload_limit(conf); - m_slowdown_limit = conf->tcp.sendBufferSize/2; + /** + * Always set slowdown limit to 60% of overload limit + */ + m_slowdown_limit = m_overload_limit * 6 / 10; } === modified file 'storage/ndb/src/common/transporter/TransporterRegistry.cpp' --- a/storage/ndb/src/common/transporter/TransporterRegistry.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/common/transporter/TransporterRegistry.cpp revid:mikael.ronstrom@stripped @@ -250,8 +250,10 @@ TransporterRegistry::TransporterRegistry DBUG_VOID_RETURN; } +#define MIN_SEND_BUFFER_SIZE 4 * 1024 * 1024 void -TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer) +TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer, + Uint64 extra_send_buffer) { if (!m_use_default_send_buffer) return; @@ -259,6 +261,21 @@ TransporterRegistry::allocate_send_buffe if (total_send_buffer == 0) total_send_buffer = get_total_max_send_buffer(); + total_send_buffer += extra_send_buffer; + + if (!extra_send_buffer) + { + /** + * If extra send buffer memory is 0 it means we can decide on an + * appropriate value for it. We select to always ensure that the + * minimum send buffer memory is 4M, otherwise we simply don't + * add any extra send buffer memory at all. + */ + if (total_send_buffer < MIN_SEND_BUFFER_SIZE) + { + total_send_buffer = (Uint64)MIN_SEND_BUFFER_SIZE; + } + } if (m_send_buffers) { /* Send buffers already allocated -> resize the buffer pages */ === modified file 'storage/ndb/src/kernel/ndbd.cpp' --- a/storage/ndb/src/kernel/ndbd.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/kernel/ndbd.cpp revid:mikael.ronstrom@stripped @@ -214,15 +214,28 @@ init_global_memory_manager(EmulatorData Uint32 sbpages = 0; if (globalTransporterRegistry.get_using_default_send_buffer() == false) { - Uint64 mem = globalTransporterRegistry.get_total_max_send_buffer(); + Uint64 mem; + Uint32 tot_mem = 0; + + ndb_mgm_get_int_parameter(p, CFG_TOTAL_SEND_BUFFER_MEMORY, &tot_mem); + if (tot_mem) + { + mem = (Uint64)tot_mem; + } + else + { + mem = globalTransporterRegistry.get_total_max_send_buffer(); + } sbpages = Uint32((mem + GLOBAL_PAGE_SIZE - 1) / GLOBAL_PAGE_SIZE); Resource_limit rl; - /* - Add one allocation of 32 pages per thread since each thread is - likely to seize at least one such in addition to the memory - needed to buffer in front of transporters - */ - sbpages += 32 * get_total_number_of_block_threads(); + /** + * Add extra send buffer pages for NDB multithreaded case + */ + Uint64 extra_mem; + ndb_mgm_get_int64_parameter(p, CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_mem); + Uint32 extra_mem_pages = Uint32((extra_mem + GLOBAL_PAGE_SIZE - 1) / + GLOBAL_PAGE_SIZE); + sbpages += mt_get_extra_send_buffer_pages(sbpages, extra_mem_pages); rl.m_min = sbpages; rl.m_max = sbpages; rl.m_resource_id = RG_TRANSPORTER_BUFFERS; === modified file 'storage/ndb/src/kernel/vm/Configuration.cpp' --- a/storage/ndb/src/kernel/vm/Configuration.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/kernel/vm/Configuration.cpp revid:mikael.ronstrom@stripped @@ -316,7 +316,10 @@ Configuration::setupConfiguration(){ Uint32 total_send_buffer = 0; iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer); - globalTransporterRegistry.allocate_send_buffers(total_send_buffer); + Uint64 extra_send_buffer = 0; + iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer); + globalTransporterRegistry.allocate_send_buffers(total_send_buffer, + extra_send_buffer); if(iter.get(CFG_DB_NO_SAVE_MSGS, &_maxErrorLogs)){ ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, "Invalid configuration fetched", === modified file 'storage/ndb/src/kernel/vm/Emulator.hpp' --- a/storage/ndb/src/kernel/vm/Emulator.hpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/kernel/vm/Emulator.hpp revid:mikael.ronstrom@stripped @@ -78,9 +78,10 @@ struct EmulatorData { extern struct EmulatorData globalEmulatorData; /** - * Compute total number of block threads in data node + * Get number of extra send buffer pages to use */ -Uint32 get_total_number_of_block_threads(void); +Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages); /** * Compute no of pages to be used as job-buffer === modified file 'storage/ndb/src/kernel/vm/dummy_nonmt.cpp' --- a/storage/ndb/src/kernel/vm/dummy_nonmt.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/kernel/vm/dummy_nonmt.cpp revid:mikael.ronstrom@stripped @@ -44,9 +44,12 @@ mt_get_instance_count(Uint32 block) return 0; } -Uint32 get_total_number_of_block_threads(void) +Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages) { - return 1; + (void)curr_num_pages; + (void)extra_mem_pages; + return 0; } Uint32 @@ -55,7 +58,6 @@ compute_jb_pages(struct EmulatorData*) return 0; } - bool NdbIsMultiThreaded() { === modified file 'storage/ndb/src/kernel/vm/mt.cpp' --- a/storage/ndb/src/kernel/vm/mt.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/kernel/vm/mt.cpp revid:mikael.ronstrom@stripped @@ -554,7 +554,7 @@ public: { T *save_last = find_last(m_freelist); T *new_list = m_global_pool->seize_list(mm, rg, - fill_level - m_free, + m_alloc_size, &alloced); m_free += alloced; if (save_last) @@ -845,6 +845,20 @@ struct thr_tq }; /* + * THR_SEND_BUFFER_ALLOC_SIZE is the amount of 32k pages allocated + * when we allocate pages from the global pool of send buffers to + * the thread_local_pool (which is local to a thread). + */ +#define THR_SEND_BUFFER_ALLOC_SIZE 32 +/* + * THR_MINIMUM_SEND_BUFFERS is the minimum amount of send buffer pages + * which is kept in thread_local_pool before starting execution of a + * new loop in a block thread to execute signals. + */ +#define THR_MINIMUM_SEND_BUFFERS 32 +#define THR_SEND_BUFFER_MAX_FREE \ + (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS) +/* * Max number of thread-local job buffers to keep before releasing to * global pool. */ @@ -914,7 +928,9 @@ struct thr_send_queue struct thr_data { thr_data() : m_jba_write_lock("jbalock"), - m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX) {} + m_send_buffer_pool(0, + THR_SEND_BUFFER_MAX_FREE, + THR_SEND_BUFFER_ALLOC_SIZE) {} thr_wait m_waiter; unsigned m_thr_no; @@ -1170,7 +1186,9 @@ struct thr_send_thread_instance m_awake(FALSE), m_thread(NULL), m_waiter_struct(), - m_send_buffer_pool(0, THR_FREE_BUF_MAX, THR_FREE_BUF_MAX) + m_send_buffer_pool(0, + THR_SEND_BUFFER_MAX_FREE, + THR_SEND_BUFFER_ALLOC_SIZE) {} Uint32 m_instance_no; Uint32 m_watchdog_counter; @@ -3781,7 +3799,7 @@ mt_job_thread_main(void *thr_arg) watchDogCounter = 11; if (!selfptr->m_send_buffer_pool.fill(g_thr_repository.m_mm, RG_TRANSPORTER_BUFFERS, - THR_FREE_BUF_MAX)) + THR_MINIMUM_SEND_BUFFERS)) { pack_send_buffers(selfptr); } @@ -4213,7 +4231,7 @@ rep_init(struct thr_repository* rep, uns #include "ThreadConfig.hpp" #include -Uint32 +static Uint32 get_total_number_of_block_threads(void) { return (NUM_MAIN_THREADS + @@ -4222,6 +4240,73 @@ get_total_number_of_block_threads(void) globalData.ndbMtReceiveThreads); } +static Uint32 +get_num_nodes() +{ + Uint32 count = 0; + for (Uint32 nodeId = 1; nodeId < MAX_NODES; nodeId++) + { + if (globalTransporterRegistry.get_transporter(nodeId)) + { + count++; + } + } + return count; +} + +/** + * This function returns the amount of extra send buffer pages + * that we should allocate in addition to the amount allocated + * for each node send buffer. + */ +#define MIN_SEND_BUFFER_GENERAL (512) //16M +#define MIN_SEND_BUFFER_PER_NODE (8) //256k +#define MIN_SEND_BUFFER_PER_THREAD (64) //2M +Uint32 +mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages) +{ + Uint32 num_threads = get_total_number_of_block_threads(); + Uint32 num_nodes = get_num_nodes(); + + /** + * First allocate 2M for each thread since we allocate 1M every + * time we allocate and also we ensure there is also a minimum + * of 1M of send buffer in each thread. Thus we can easily have + * 2M of send buffer just to keep the contention around the + * send buffer page spinlock small. This memory we add independent + * of the configuration settings since the user cannot be + * expected to handle this and also since we could change this + * behaviour at any time. + */ + Uint32 extra_pages = + (THR_SEND_BUFFER_ALLOC_SIZE + THR_MINIMUM_SEND_BUFFERS) * + num_threads; + + extra_pages += extra_mem_pages; + if (!extra_mem_pages) + { + /** + * The user have set extra send buffer memory to 0 and left for us + * to decide on our own how much extra memory is needed. + * + * We'll make sure that we have at least a minimum of 16M + + * 2M per thread + 256k per node. If we have this based on + * curr_num_pages and our local additions we don't add + * anything more, if we don't come up to this level we add to + * reach this minimum level. + */ + Uint32 min_pages = MIN_SEND_BUFFER_GENERAL + + (MIN_SEND_BUFFER_PER_NODE * num_nodes) + + (MIN_SEND_BUFFER_PER_THREAD * num_threads); + if ((curr_num_pages + extra_pages) < min_pages) + { + extra_pages = min_pages - curr_num_pages; + } + } + return extra_pages; +} + Uint32 compute_jb_pages(struct EmulatorData * ed) { === modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp' --- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp revid:mikael.ronstrom@stripped @@ -1688,11 +1688,24 @@ const ConfigInfo::ParamInfo ConfigInfo:: "true"}, { + CFG_EXTRA_SEND_BUFFER_MEMORY, + "ExtraSendBufferMemory", + DB_TOKEN, + "Extra send buffer memory to use for send buffers in all transporters", + ConfigInfo::CI_USED, + false, + ConfigInfo::CI_INT64, + "0", + "0", + "32G" + }, + + { CFG_TOTAL_SEND_BUFFER_MEMORY, "TotalSendBufferMemory", DB_TOKEN, "Total memory to use for send buffers in all transporters", - ConfigInfo::CI_USED, + ConfigInfo::CI_DEPRECATED, false, ConfigInfo::CI_INT, "0", @@ -1707,7 +1720,7 @@ const ConfigInfo::ParamInfo ConfigInfo:: "Amount of bytes (out of TotalSendBufferMemory) to reserve for connection\n" "between data nodes. This memory will not be available for connections to\n" "management server or API nodes.", - ConfigInfo::CI_USED, + ConfigInfo::CI_DEPRECATED, false, ConfigInfo::CI_INT, "0", === modified file 'storage/ndb/src/ndbapi/TransporterFacade.cpp' --- a/storage/ndb/src/ndbapi/TransporterFacade.cpp revid:mikael.ronstrom@stripped +++ b/storage/ndb/src/ndbapi/TransporterFacade.cpp revid:mikael.ronstrom@stripped @@ -643,7 +643,10 @@ TransporterFacade::configure(NodeId node // Configure send buffers Uint32 total_send_buffer = 0; iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer); - theTransporterRegistry->allocate_send_buffers(total_send_buffer); + Uint64 extra_send_buffer = 0; + iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer); + theTransporterRegistry->allocate_send_buffers(total_send_buffer, + extra_send_buffer); Uint32 auto_reconnect=1; iter.get(CFG_AUTO_RECONNECT, &auto_reconnect); No bundle (reason: useless for push emails).