From: Mauritz Sundell Date: November 30 2012 8:59am Subject: bzr push into mysql-5.1-telco-7.0 branch (mauritz.sundell:5054 to 5055) List-Archive: http://lists.mysql.com/commits/145415 Message-Id: <20121130085954.18145.55261.5055@msundell-laptop> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 5055 Mauritz Sundell 2012-11-30 [merge] ndb - ConfigValues warnings, refactor, and bug modified: storage/ndb/include/util/ConfigValues.hpp storage/ndb/src/common/util/ConfigValues.cpp 5054 Frazer Clement 2012-11-29 Bug #15935206 NDB : IMPROVE VISIBILITY OF TRANSPORTER OVERLOAD OCCURRENCES NdbInfo.transporters extended with 5 new columns : connect_count (Number of times connection established on this transporter) overloaded (Whether this transporter is currently overloaded. 0 or 1) overload_count (How many times this transporter has entered overloaded state since connecting) slowdown (Whether this transporter is currently in scan slowdown state. 0 or 1) slowdown_count (How many times this transporter has entered scan slowdown state since connecting) Note that the counters are reset on *CONNECT*, so will retain their values after the remote node disconnects. Note also that the existing bytes_send and bytes_received counters are *CHANGED* to be reset on *CONNECT* as well, so will retain their values after the remote node disconnects. This is a change to the previous behaviour, where they were reset on DISCONNECT. Additionally 6 new per-LQH-instance counters are added to NDBINFO.COUNTERS : LQHKEY_OVERLOAD Number of primary key requests rejected at LQH block instance due to transporter overload LQHKEY_OVERLOAD_TC Count of instances of LQHKEY_OVERLOAD where the TC node transporter was overloaded LQHKEY_OVERLOAD_READER Count of instances of LQHKEY_OVERLOAD where the Api reader (only reads) node was overloaded. LQHKEY_OVERLOAD_NODE_PEER Count of instances of LQHKEY_OVERLOAD where the next backup data node (only writes) was overloaded LQHKEY_OVERLOAD_SUBSCRIBER Count of instances of LQHKEY_OVERLOAD where a event subscriber (only writes) was overloaded. LQHSCAN_SLOWDOWNS Count of instances where a fragment scan batchsize was reduced due to scanning Api transporter overload. These new values and counters make transporter overload and SendBuffer sizing problems more visible and debuggable. modified: mysql-test/suite/ndb/r/ndbinfo.result scripts/mysql_system_tables.sql storage/ndb/include/transporter/TransporterRegistry.hpp storage/ndb/src/common/transporter/Transporter.cpp storage/ndb/src/common/transporter/Transporter.hpp storage/ndb/src/common/transporter/TransporterRegistry.cpp storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp storage/ndb/src/kernel/blocks/trpman.cpp storage/ndb/src/kernel/vm/Ndbinfo.hpp storage/ndb/src/kernel/vm/NdbinfoTables.cpp storage/ndb/tools/ndbinfo_sql.cpp === modified file 'storage/ndb/include/util/ConfigValues.hpp' --- a/storage/ndb/include/util/ConfigValues.hpp 2011-06-30 15:59:25 +0000 +++ b/storage/ndb/include/util/ConfigValues.hpp 2012-11-30 08:46:59 +0000 @@ -27,6 +27,8 @@ class ConfigValues { ConfigValues(Uint32 sz, Uint32 data); public: + static ConfigValues* constructInPlace(Uint32 keys, Uint32 data, void* place, size_t size); + static size_t sizeInBytes(Uint32 keys, Uint32 data); ~ConfigValues(); enum ValueType { @@ -100,8 +102,18 @@ private: Uint32 m_stringCount; Uint32 m_int64Count; + /** + * ConfigValues are constructed with different sizes. + * If we used dynamicly sized arrays the end of the + * class would look like: + * struct alignas(8) { Uint32 key; Uint32 value_or_index; } m_values[m_size]; + * Uint64 alignas(8) m_data[m_int64Count]; + * char m_free[m_dataSize - m_int64Count*8 - m_stringCount*sizeof(char*)]; + * char * alignas(sizeof(char *)) m_dataString[m_stringCount]; + * Where m_dataString grows from top and down with index 0 on top. + * But now we only have a place holder for the first entry. + */ Uint32 m_values[1]; - void * m_data[1]; }; class ConfigValuesFactory { @@ -112,7 +124,7 @@ public: Uint32 m_freeData; public: - ConfigValuesFactory(Uint32 keys = 50, Uint32 data = 10); // Initial + ConfigValuesFactory(Uint32 keys = 50, Uint32 data = 16); // Initial ConfigValuesFactory(ConfigValues * m_cfg); // ~ConfigValuesFactory(); === modified file 'storage/ndb/src/common/util/ConfigValues.cpp' --- a/storage/ndb/src/common/util/ConfigValues.cpp 2011-10-21 08:59:23 +0000 +++ b/storage/ndb/src/common/util/ConfigValues.cpp 2012-11-30 08:46:59 +0000 @@ -60,16 +60,35 @@ getTypeOf(Uint32 k) { return (ConfigValues::ValueType)((k >> KP_TYPE_SHIFT) & KP_TYPE_MASK); } -ConfigValues::ConfigValues(Uint32 sz, Uint32 dsz){ - m_size = sz; - m_dataSize = dsz; +ConfigValues::ConfigValues(Uint32 sz, Uint32 dsz) +: m_size(sz), + m_dataSize(dsz) +{ + assert(dsz % sizeof(char const*) == 0); + assert(dsz % sizeof(Uint64) == 0); + assert(my_offsetof(ConfigValues, m_values) % 8 == 0); m_stringCount = 0; m_int64Count = 0; for(Uint32 i = 0; i= sizeInBytes(keys, data)); + return new (place) ConfigValues(keys, data); +} + +size_t +ConfigValues::sizeInBytes(Uint32 keys, Uint32 data) +{ + return my_offsetof(ConfigValues, m_values) + + keys * 2 * sizeof(Uint32) + + data * sizeof(char); +} + ConfigValues::~ConfigValues(){ for(Uint32 i = 0; i 0); switch(entry.m_type){ case ConfigValues::IntType: case ConfigValues::SectionType: @@ -494,6 +512,7 @@ ConfigValuesFactory::put(const ConfigVal char ** ref = m_cfg->getString(index); * ref = strdup(entry.m_string ? entry.m_string : ""); m_freeKeys--; + assert(m_freeData >= sizeof(char*)); m_freeData -= sizeof(char *); DEBUG printf("Putting at: %d(%d) (loop = %d) key: %d value(%d): %s\n", pos, sz, 0, @@ -507,6 +526,7 @@ ConfigValuesFactory::put(const ConfigVal m_cfg->m_values[pos+1] = index; * m_cfg->get64(index) = entry.m_int64; m_freeKeys--; + assert(m_freeData >= 8); m_freeData -= 8; DEBUG printf("Putting at: %d(%d) (loop = %d) key: %d value64(%d): %lld\n", pos, sz, 0, @@ -566,7 +586,7 @@ ConfigValuesFactory::extractCurrentSecti ConfigValues * ConfigValuesFactory::getConfigValues(){ ConfigValues * ret = m_cfg; - m_cfg = create(10, 10); + m_cfg = create(10, 16); return ret; } @@ -580,8 +600,9 @@ Uint32 ConfigValues::getPackedSize() const { Uint32 size = 0; + Uint32 const* values = m_values; for(Uint32 i = 0; i < 2 * m_size; i += 2){ - Uint32 key = m_values[i]; + Uint32 key = values[i]; if(key != CFV_KEY_FREE){ switch(::getTypeOf(key)){ case IntType: @@ -593,7 +614,7 @@ ConfigValues::getPackedSize() const { break; case StringType: size += 8; // key + len - size += mod4((unsigned)strlen(* getString(m_values[i+1])) + 1); + size += mod4((unsigned)strlen(* getString(values[i+1])) + 1); break; case InvalidType: default: @@ -611,9 +632,10 @@ ConfigValues::pack(void * _dst, Uint32 _ char * dst = (char*)_dst; memcpy(dst, Magic, sizeof(Magic)); dst += sizeof(Magic); + Uint32 const* values = m_values; for(i = 0; i < 2 * m_size; i += 2){ - Uint32 key = m_values[i]; - Uint32 val = m_values[i+1]; + Uint32 key = values[i]; + Uint32 val = values[i+1]; if(key != CFV_KEY_FREE){ switch(::getTypeOf(key)){ case IntType: No bundle (reason: useless for push emails).