From: Jonas Oreland Date: January 30 2012 2:41pm Subject: bzr push into mysql-5.1-telco-7.0 branch (jonas.oreland:4833 to 4834) Bug#13633845 List-Archive: http://lists.mysql.com/commits/142639 X-Bug: 13633845 Message-Id: <20120130144119.E992755C2D4@perch.localdomain> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 4834 Jonas Oreland 2012-01-30 ndb - bug#13633845 - try to improve inconsistent transporter send buffer handling modified: storage/ndb/include/mgmapi/mgmapi_config_parameters.h storage/ndb/include/transporter/TransporterRegistry.hpp storage/ndb/src/common/transporter/TransporterRegistry.cpp storage/ndb/src/kernel/ndbd.cpp storage/ndb/src/kernel/vm/Configuration.cpp storage/ndb/src/kernel/vm/Emulator.hpp storage/ndb/src/kernel/vm/dummy_nonmt.cpp storage/ndb/src/kernel/vm/mt.cpp storage/ndb/src/mgmsrv/ConfigInfo.cpp storage/ndb/src/ndbapi/TransporterFacade.cpp 4833 Jonas Oreland 2012-01-30 ndb - bug#13618181 - patch 2 - make sure to release pages before (maybe) looping on sb->m_force_send modified: storage/ndb/src/kernel/vm/mt.cpp === modified file 'storage/ndb/include/mgmapi/mgmapi_config_parameters.h' --- a/storage/ndb/include/mgmapi/mgmapi_config_parameters.h 2011-12-01 10:45:07 +0000 +++ b/storage/ndb/include/mgmapi/mgmapi_config_parameters.h 2012-01-30 14:28:55 +0000 @@ -206,6 +206,7 @@ #define CFG_NODE_ARBIT_RANK 200 #define CFG_NODE_ARBIT_DELAY 201 #define CFG_RESERVED_SEND_BUFFER_MEMORY 202 +#define CFG_EXTRA_SEND_BUFFER_MEMORY 203 #define CFG_MIN_LOGLEVEL 250 #define CFG_LOGLEVEL_STARTUP 250 === modified file 'storage/ndb/include/transporter/TransporterRegistry.hpp' --- a/storage/ndb/include/transporter/TransporterRegistry.hpp 2012-01-19 11:48:54 +0000 +++ b/storage/ndb/include/transporter/TransporterRegistry.hpp 2012-01-30 14:28:55 +0000 @@ -270,8 +270,10 @@ public: * * Argument is the value of config parameter TotalSendBufferMemory. If 0, * a default will be used of sum(max send buffer) over all transporters. + * The second is the config parameter ExtraSendBufferMemory */ - void allocate_send_buffers(Uint64 total_send_buffer); + void allocate_send_buffers(Uint64 total_send_buffer, + Uint64 extra_send_buffer); /** * Get sum of max send buffer over all transporters, to be used as a default === modified file 'storage/ndb/src/common/transporter/TransporterRegistry.cpp' --- a/storage/ndb/src/common/transporter/TransporterRegistry.cpp 2012-01-23 19:53:55 +0000 +++ b/storage/ndb/src/common/transporter/TransporterRegistry.cpp 2012-01-30 14:28:55 +0000 @@ -250,8 +250,11 @@ TransporterRegistry::TransporterRegistry DBUG_VOID_RETURN; } +#define MIN_SEND_BUFFER_SIZE (4 * 1024 * 1024) + void -TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer) +TransporterRegistry::allocate_send_buffers(Uint64 total_send_buffer, + Uint64 extra_send_buffer) { if (!m_use_default_send_buffer) return; @@ -259,6 +262,22 @@ TransporterRegistry::allocate_send_buffe if (total_send_buffer == 0) total_send_buffer = get_total_max_send_buffer(); + total_send_buffer += extra_send_buffer; + + if (!extra_send_buffer) + { + /** + * If extra send buffer memory is 0 it means we can decide on an + * appropriate value for it. We select to always ensure that the + * minimum send buffer memory is 4M, otherwise we simply don't + * add any extra send buffer memory at all. + */ + if (total_send_buffer < MIN_SEND_BUFFER_SIZE) + { + total_send_buffer = (Uint64)MIN_SEND_BUFFER_SIZE; + } + } + if (m_send_buffers) { /* Send buffers already allocated -> resize the buffer pages */ === modified file 'storage/ndb/src/kernel/ndbd.cpp' --- a/storage/ndb/src/kernel/ndbd.cpp 2012-01-16 07:14:30 +0000 +++ b/storage/ndb/src/kernel/ndbd.cpp 2012-01-30 14:28:55 +0000 @@ -214,11 +214,40 @@ init_global_memory_manager(EmulatorData Uint32 sbpages = 0; if (globalTransporterRegistry.get_using_default_send_buffer() == false) { - Uint64 mem = globalTransporterRegistry.get_total_max_send_buffer(); + Uint64 mem; + { + Uint32 tot_mem = 0; + ndb_mgm_get_int_parameter(p, CFG_TOTAL_SEND_BUFFER_MEMORY, &tot_mem); + if (tot_mem) + { + mem = (Uint64)tot_mem; + } + else + { + mem = globalTransporterRegistry.get_total_max_send_buffer(); + } + } + sbpages = Uint32((mem + GLOBAL_PAGE_SIZE - 1) / GLOBAL_PAGE_SIZE); + + /** + * Add extra send buffer pages for NDB multithreaded case + */ + { + Uint64 extra_mem; + ndb_mgm_get_int64_parameter(p, CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_mem); + Uint32 extra_mem_pages = Uint32((extra_mem + GLOBAL_PAGE_SIZE - 1) / + GLOBAL_PAGE_SIZE); + sbpages += mt_get_extra_send_buffer_pages(sbpages, extra_mem_pages); + } + Resource_limit rl; rl.m_min = sbpages; - rl.m_max = sbpages; + /** + * allow over allocation (from SharedGlobalMemory) of up to 25% of + * totally allocated SendBuffer + */ + rl.m_max = sbpages + (sbpages * 25) / 100; rl.m_resource_id = RG_TRANSPORTER_BUFFERS; ed.m_mem_manager->set_resource_limit(rl); } === modified file 'storage/ndb/src/kernel/vm/Configuration.cpp' --- a/storage/ndb/src/kernel/vm/Configuration.cpp 2012-01-16 07:14:30 +0000 +++ b/storage/ndb/src/kernel/vm/Configuration.cpp 2012-01-30 14:28:55 +0000 @@ -316,7 +316,10 @@ Configuration::setupConfiguration(){ Uint32 total_send_buffer = 0; iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer); - globalTransporterRegistry.allocate_send_buffers(total_send_buffer); + Uint64 extra_send_buffer = 0; + iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer); + globalTransporterRegistry.allocate_send_buffers(total_send_buffer, + extra_send_buffer); if(iter.get(CFG_DB_NO_SAVE_MSGS, &_maxErrorLogs)){ ERROR_SET(fatal, NDBD_EXIT_INVALID_CONFIG, "Invalid configuration fetched", === modified file 'storage/ndb/src/kernel/vm/Emulator.hpp' --- a/storage/ndb/src/kernel/vm/Emulator.hpp 2012-01-19 06:21:05 +0000 +++ b/storage/ndb/src/kernel/vm/Emulator.hpp 2012-01-30 14:28:55 +0000 @@ -78,6 +78,12 @@ struct EmulatorData { extern struct EmulatorData globalEmulatorData; /** + * Get number of extra send buffer pages to use + */ +Uint32 mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages); + +/** * Compute no of pages to be used as job-buffer */ Uint32 compute_jb_pages(struct EmulatorData* ed); === modified file 'storage/ndb/src/kernel/vm/dummy_nonmt.cpp' --- a/storage/ndb/src/kernel/vm/dummy_nonmt.cpp 2011-10-07 08:07:21 +0000 +++ b/storage/ndb/src/kernel/vm/dummy_nonmt.cpp 2012-01-30 14:28:55 +0000 @@ -45,6 +45,15 @@ mt_get_instance_count(Uint32 block) } Uint32 +mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages) +{ + (void)curr_num_pages; + (void)extra_mem_pages; + return 0; +} + +Uint32 compute_jb_pages(struct EmulatorData*) { return 0; === modified file 'storage/ndb/src/kernel/vm/mt.cpp' --- a/storage/ndb/src/kernel/vm/mt.cpp 2012-01-30 13:13:37 +0000 +++ b/storage/ndb/src/kernel/vm/mt.cpp 2012-01-30 14:28:55 +0000 @@ -4109,6 +4109,83 @@ rep_init(struct thr_repository* rep, uns #include "ThreadConfig.hpp" #include +static Uint32 +get_total_number_of_block_threads(void) +{ + return (NUM_MAIN_THREADS + + globalData.ndbMtLqhThreads + + globalData.ndbMtTcThreads + + globalData.ndbMtReceiveThreads); +} + +static Uint32 +get_num_nodes() +{ + Uint32 count = 0; + for (Uint32 nodeId = 1; nodeId < MAX_NODES; nodeId++) + { + if (globalTransporterRegistry.get_transporter(nodeId)) + { + count++; + } + } + return count; +} + +/** + * This function returns the amount of extra send buffer pages + * that we should allocate in addition to the amount allocated + * for each node send buffer. + */ +#define MIN_SEND_BUFFER_GENERAL (512) //16M +#define MIN_SEND_BUFFER_PER_NODE (8) //256k +#define MIN_SEND_BUFFER_PER_THREAD (64) //2M + +Uint32 +mt_get_extra_send_buffer_pages(Uint32 curr_num_pages, + Uint32 extra_mem_pages) +{ + Uint32 num_threads = get_total_number_of_block_threads(); + Uint32 num_nodes = get_num_nodes(); + + Uint32 extra_pages = extra_mem_pages; + + /** + * Add 2M for each thread since we allocate 1M every + * time we allocate and also we ensure there is also a minimum + * of 1M of send buffer in each thread. Thus we can easily have + * 2M of send buffer just to keep the contention around the + * send buffer page spinlock small. This memory we add independent + * of the configuration settings since the user cannot be + * expected to handle this and also since we could change this + * behaviour at any time. + */ + extra_pages += num_threads * 0; + + if (extra_mem_pages == 0) + { + /** + * The user have set extra send buffer memory to 0 and left for us + * to decide on our own how much extra memory is needed. + * + * We'll make sure that we have at least a minimum of 16M + + * 2M per thread + 256k per node. If we have this based on + * curr_num_pages and our local additions we don't add + * anything more, if we don't come up to this level we add to + * reach this minimum level. + */ + Uint32 min_pages = MIN_SEND_BUFFER_GENERAL + + (MIN_SEND_BUFFER_PER_NODE * num_nodes) + + (MIN_SEND_BUFFER_PER_THREAD * num_threads); + + if ((curr_num_pages + extra_pages) < min_pages) + { + extra_pages = min_pages - curr_num_pages; + } + } + return extra_pages; +} + Uint32 compute_jb_pages(struct EmulatorData * ed) { === modified file 'storage/ndb/src/mgmsrv/ConfigInfo.cpp' --- a/storage/ndb/src/mgmsrv/ConfigInfo.cpp 2012-01-25 09:18:33 +0000 +++ b/storage/ndb/src/mgmsrv/ConfigInfo.cpp 2012-01-30 14:28:55 +0000 @@ -1688,11 +1688,24 @@ const ConfigInfo::ParamInfo ConfigInfo:: "true"}, { + CFG_EXTRA_SEND_BUFFER_MEMORY, + "ExtraSendBufferMemory", + DB_TOKEN, + "Extra send buffer memory to use for send buffers in all transporters", + ConfigInfo::CI_USED, + false, + ConfigInfo::CI_INT64, + "0", + "0", + "32G" + }, + + { CFG_TOTAL_SEND_BUFFER_MEMORY, "TotalSendBufferMemory", DB_TOKEN, "Total memory to use for send buffers in all transporters", - ConfigInfo::CI_USED, + ConfigInfo::CI_DEPRECATED, false, ConfigInfo::CI_INT, "0", @@ -1707,7 +1720,7 @@ const ConfigInfo::ParamInfo ConfigInfo:: "Amount of bytes (out of TotalSendBufferMemory) to reserve for connection\n" "between data nodes. This memory will not be available for connections to\n" "management server or API nodes.", - ConfigInfo::CI_USED, + ConfigInfo::CI_DEPRECATED, false, ConfigInfo::CI_INT, "0", === modified file 'storage/ndb/src/ndbapi/TransporterFacade.cpp' --- a/storage/ndb/src/ndbapi/TransporterFacade.cpp 2012-01-16 08:25:04 +0000 +++ b/storage/ndb/src/ndbapi/TransporterFacade.cpp 2012-01-30 14:28:55 +0000 @@ -643,7 +643,10 @@ TransporterFacade::configure(NodeId node // Configure send buffers Uint32 total_send_buffer = 0; iter.get(CFG_TOTAL_SEND_BUFFER_MEMORY, &total_send_buffer); - theTransporterRegistry->allocate_send_buffers(total_send_buffer); + Uint64 extra_send_buffer = 0; + iter.get(CFG_EXTRA_SEND_BUFFER_MEMORY, &extra_send_buffer); + theTransporterRegistry->allocate_send_buffers(total_send_buffer, + extra_send_buffer); Uint32 auto_reconnect=1; iter.get(CFG_AUTO_RECONNECT, &auto_reconnect); No bundle (reason: useless for push emails).