From: magnus.blaudd Date: May 23 2012 12:57pm Subject: bzr push into mysql-5.5-cluster-7.3 branch (magnus.blaudd:3890 to 3891) List-Archive: http://lists.mysql.com/commits/143958 Message-Id: <201205231257.q4NCv6bG018831@acsmt356.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 3891 magnus.blaudd@stripped 2012-05-23 [merge] Merge 7.2 -> 7.3 modified: mysql-test/suite/ndb/r/ndb_condition_pushdown.result mysql-test/suite/ndb/t/ndb_condition_pushdown.test sql/ha_ndbcluster_cond.cc storage/ndb/src/common/util/SocketClient.cpp storage/ndb/src/kernel/vm/mt.cpp support-files/mysql.spec.sh 3890 Frazer Clement 2012-05-22 [merge] Merge 7.2->7.3 modified: storage/ndb/include/kernel/GlobalSignalNumbers.h storage/ndb/include/kernel/signaldata/LCP.hpp storage/ndb/include/kernel/signaldata/SignalData.hpp storage/ndb/src/common/debugger/signaldata/LCP.cpp storage/ndb/src/common/debugger/signaldata/SignalDataPrint.cpp storage/ndb/src/common/debugger/signaldata/SignalNames.cpp storage/ndb/src/kernel/blocks/backup/Backup.cpp storage/ndb/src/kernel/blocks/backup/Backup.hpp storage/ndb/src/kernel/blocks/backup/BackupInit.cpp storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp storage/ndb/src/kernel/blocks/dblqh/DblqhInit.cpp storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp storage/ndb/test/include/NdbMgmd.hpp storage/ndb/test/ndbapi/testNodeRestart.cpp storage/ndb/test/run-test/daily-devel-tests.txt === modified file 'mysql-test/suite/ndb/r/ndb_condition_pushdown.result' --- a/mysql-test/suite/ndb/r/ndb_condition_pushdown.result 2012-02-23 15:41:31 +0000 +++ b/mysql-test/suite/ndb/r/ndb_condition_pushdown.result 2012-05-23 12:43:48 +0000 @@ -2422,3 +2422,11 @@ test_data@stripped 1 drop table escapetest; set @@session.optimizer_switch = @old_ecpd; DROP TABLE t1,t2,t3,t4,t5; +# +# Bug#14106592 DEBUG BUILD CRASH IN NDB_SERIALIZE_COND() +# +create table t1(a time) engine=ndbcluster; +insert into t1 values ('00:00:00'),('01:01:01'); +select 1 from t1 where a >= geomfromtext('1'); +1 +drop table t1; === modified file 'mysql-test/suite/ndb/t/ndb_condition_pushdown.test' --- a/mysql-test/suite/ndb/t/ndb_condition_pushdown.test 2012-02-23 15:41:31 +0000 +++ b/mysql-test/suite/ndb/t/ndb_condition_pushdown.test 2012-05-23 12:43:48 +0000 @@ -2437,3 +2437,11 @@ drop table escapetest; set @@session.optimizer_switch = @old_ecpd; DROP TABLE t1,t2,t3,t4,t5; +--echo # +--echo # Bug#14106592 DEBUG BUILD CRASH IN NDB_SERIALIZE_COND() +--echo # +create table t1(a time) engine=ndbcluster; +insert into t1 values ('00:00:00'),('01:01:01'); +select 1 from t1 where a >= geomfromtext('1'); +drop table t1; + === modified file 'sql/ha_ndbcluster_cond.cc' --- a/sql/ha_ndbcluster_cond.cc 2012-02-23 15:41:31 +0000 +++ b/sql/ha_ndbcluster_cond.cc 2012-05-23 12:43:48 +0000 @@ -766,11 +766,9 @@ ndb_serialize_cond(const Item *item, voi { #ifndef DBUG_OFF char buff[256]; - String str(buff,0, system_charset_info); - //str.length(0);// Magnus - Item_string *string_item= (Item_string *) item; - DBUG_PRINT("info", ("value \"%s\"", - string_item->val_str(&str)->ptr())); + String str(buff, 0, system_charset_info); + const_cast(item)->print(&str, QT_ORDINARY); + DBUG_PRINT("info", ("value: '%s'", str.c_ptr_safe())); #endif NDB_ITEM_QUALIFICATION q; q.value_type= Item::STRING_ITEM; @@ -1029,11 +1027,9 @@ ndb_serialize_cond(const Item *item, voi { #ifndef DBUG_OFF char buff[256]; - String str(buff,0, system_charset_info); - //str.length(0);// Magnus - Item_string *string_item= (Item_string *) item; - DBUG_PRINT("info", ("value \"%s\"", - string_item->val_str(&str)->ptr())); + String str(buff, 0, system_charset_info); + const_cast(item)->print(&str, QT_ORDINARY); + DBUG_PRINT("info", ("value: '%s'", str.c_ptr_safe())); #endif NDB_ITEM_QUALIFICATION q; q.value_type= Item::STRING_ITEM; === modified file 'storage/ndb/src/common/util/SocketClient.cpp' --- a/storage/ndb/src/common/util/SocketClient.cpp 2011-02-01 23:27:25 +0000 +++ b/storage/ndb/src/common/util/SocketClient.cpp 2012-05-23 11:45:13 +0000 @@ -189,7 +189,7 @@ SocketClient::connect(const char *toaddr } done: - if (my_socket_nonblock(m_sockfd, true) < 0) + if (my_socket_nonblock(m_sockfd, false) < 0) { my_socket_close(m_sockfd); my_socket_invalidate(&m_sockfd); === modified file 'storage/ndb/src/kernel/vm/mt.cpp' --- a/storage/ndb/src/kernel/vm/mt.cpp 2012-03-21 12:31:02 +0000 +++ b/storage/ndb/src/kernel/vm/mt.cpp 2012-05-23 06:40:37 +0000 @@ -637,6 +637,17 @@ struct thr_jb_write_state /* Number of signals inserted since last wakeup */ Uint32 m_pending_signals_wakeup; + + /* + * Is this job buffer open for communication at all? + * Several threads are not expected to communicate, and thus does + * not allocate thr_job_buffer for exchange of signals. + * Don't access any job_buffers without ensuring 'is_open()==true'. + */ + bool is_open() const + { + return (m_write_buffer != NULL); + } }; /* @@ -669,6 +680,17 @@ struct thr_jb_read_state Uint32 m_write_index; // Last available thr_job_buffer. + /* + * Is this job buffer open for communication at all? + * Several threads are not expected to communicate, and thus does + * not allocate thr_job_buffer for exchange of signals. + * Don't access any job_buffers without ensuring 'is_open()==true'. + */ + bool is_open() const + { + return (m_read_buffer != NULL); + } + bool is_empty() const { assert(m_read_index != m_write_index || m_read_pos <= m_read_end); @@ -2949,6 +2971,7 @@ insert_signal(thr_job_queue *q, thr_job_ { Uint32 write_pos = w->m_write_pos; Uint32 datalen = sh->theLength; + assert(w->is_open()); assert(w->m_write_buffer == q->m_buffers[w->m_write_index]); memcpy(w->m_write_buffer->m_data + write_pos, sh, sizeof(*sh)); write_pos += (sizeof(*sh) >> 2); @@ -3021,19 +3044,22 @@ read_jbb_state(thr_data *selfptr, Uint32 const thr_job_queue_head *h = selfptr->m_in_queue_head; for (Uint32 i = 0; i < count; i++,r++) { - Uint32 read_index = r->m_read_index; - - /** - * Optimization: Only reload when possibly empty. - * Avoid cache reload of shared thr_job_queue_head - * Read head directly to avoid unnecessary cache - * load of first cache line of m_in_queue entry. - */ - if (r->m_write_index == read_index) + if (r->is_open()) { - r->m_write_index = h[i].m_write_index; - read_barrier_depends(); - r->m_read_end = q[i].m_buffers[read_index]->m_len; + Uint32 read_index = r->m_read_index; + + /** + * Optimization: Only reload when possibly empty. + * Avoid cache reload of shared thr_job_queue_head + * Read head directly to avoid unnecessary cache + * load of first cache line of m_in_queue entry. + */ + if (r->m_write_index == read_index) + { + r->m_write_index = h[i].m_write_index; + read_barrier_depends(); + r->m_read_end = q[i].m_buffers[read_index]->m_len; + } } } } @@ -3739,7 +3765,7 @@ mt_job_thread_main(void *thr_arg) /** * prefill our thread local send buffers - * up to THR_MINIMUM_SEND_BUFFERS (1Mb) + * up to THR_SEND_BUFFER_PRE_ALLOC (1Mb) * * and if this doesnt work pack buffers before start to execute signals */ @@ -4025,6 +4051,75 @@ sendprioa_STOP_FOR_CRASH(const struct th } /** + * Identify type of thread. + * Based on assumption that threads are allocated in the order: + * main, ldm, tc, recv, send + */ +static bool +is_main_thread(unsigned thr_no) +{ + return thr_no < NUM_MAIN_THREADS; +} + +static bool +is_ldm_thread(unsigned thr_no) +{ + return thr_no >= NUM_MAIN_THREADS && + thr_no < NUM_MAIN_THREADS+globalData.ndbMtLqhThreads; +} + +static bool +is_tc_thread(unsigned thr_no) +{ + unsigned tc_base = NUM_MAIN_THREADS+globalData.ndbMtLqhThreads; + return thr_no >= tc_base && + thr_no < tc_base+globalData.ndbMtTcThreads; +} + +static bool +is_recv_thread(unsigned thr_no) +{ + unsigned recv_base = NUM_MAIN_THREADS+globalData.ndbMtLqhThreads+globalData.ndbMtTcThreads; + return thr_no >= recv_base && + thr_no < recv_base+globalData.ndbMtReceiveThreads; +} + +/** + * Implements the rules for which threads are allowed to have + * communication with each other. + * Also see compute_jb_pages() which has similar logic. + */ +static bool +may_communicate(unsigned from, unsigned to) +{ + if (is_main_thread(from)) + { + // Main threads communicates with all other threads + return true; + } + else if (is_ldm_thread(from)) + { + // LQH threads can communicates with TC-, main- and itself + return is_main_thread(to) || + is_tc_thread(to) || + (to == from); + } + else if (is_tc_thread(from)) + { + // TC threads can communicate with LQH-, main- and itself + return is_main_thread(to) || + is_ldm_thread(to) || + (to == from); + } + else + { + assert(is_recv_thread(from)); + // Receive treads communicate with all, except other receivers + return !is_recv_thread(to); + } +} + +/** * init functions */ static @@ -4072,7 +4167,8 @@ thr_init(struct thr_repository* rep, str { selfptr->m_in_queue_head[i].m_read_index = 0; selfptr->m_in_queue_head[i].m_write_index = 0; - buffer = seize_buffer(rep, thr_no, false); + buffer = may_communicate(i,thr_no) + ? seize_buffer(rep, thr_no, false) : NULL; selfptr->m_in_queue[i].m_buffers[0] = buffer; selfptr->m_read_states[i].m_read_index = 0; selfptr->m_read_states[i].m_read_buffer = buffer; @@ -4258,37 +4354,93 @@ mt_get_extra_send_buffer_pages(Uint32 cu Uint32 compute_jb_pages(struct EmulatorData * ed) { - Uint32 cnt = NUM_MAIN_THREADS + - globalData.ndbMtReceiveThreads + globalData.ndbMtTcThreads + globalData.ndbMtLqhThreads + 1; + Uint32 cnt = get_total_number_of_block_threads(); + Uint32 num_receive_threads = globalData.ndbMtReceiveThreads; + Uint32 num_lqh_threads = globalData.ndbMtLqhThreads; + Uint32 num_tc_threads = globalData.ndbMtTcThreads; + Uint32 num_main_threads = NUM_MAIN_THREADS; + + /** + * Number of pages each thread needs to communicate with another + * thread. + */ + Uint32 job_queue_pages_per_thread = thr_job_queue::SIZE; + /** + * In 'perthread' we calculate number of pages required by + * all 'block threads' (excludes 'send-threads'). 'perthread' + * usage is independent of whether this thread will communicate + * with other 'block threads' or not. + */ Uint32 perthread = 0; /** - * Each thread can have thr_job_queue::SIZE pages in out-queues - * to each other thread + * Each threads has its own job_queue for 'prio A' signals */ - perthread += cnt * (1 + thr_job_queue::SIZE); + perthread += job_queue_pages_per_thread; /** - * And thr_job_queue::SIZE prio A signals + * Each thread keeps a available free page in 'm_next_buffer' + * in case it is required by insert_signal() into JBA or JBB. */ - perthread += (1 + thr_job_queue::SIZE); + perthread += 1; /** - * And XXX time-queue signals + * Each thread keeps time-queued signals in 'struct thr_tq' + * thr_tq::PAGES are used to store these. */ - perthread += 32; // Say 1M for now + perthread += thr_tq::PAGES; /** - * Each thread also keeps an own cache with max THR_FREE_BUF_MAX + * Each thread has its own 'm_free_fifo[THR_FREE_BUF_MAX]' cache. + * As it is filled to MAX *before* a page is allocated, which consumes a page, + * it will never cache more than MAX-1 pages. Pages are also returned to global + * allocator as soon as MAX is reached. */ - perthread += THR_FREE_BUF_MAX; + perthread += THR_FREE_BUF_MAX-1; /** - * Multiply by no of threads + * Start by calculating the basic number of pages required for + * our 'cnt' block threads. + * (no inter-thread communication assumed so far) */ Uint32 tot = cnt * perthread; + /** + * We then start adding pages required for inter-thread communications: + * + * Receiver threads will be able to communicate with all other + * threads except other receive threads. + */ + tot += num_receive_threads * + (cnt - num_receive_threads) * + job_queue_pages_per_thread; + + /** + * LQH threads can communicate with TC threads and main threads. + * Cannot communicate with receive threads and other LQH threads, + * but it can communicate with itself. + */ + tot += num_lqh_threads * + (num_tc_threads + num_main_threads + 1) * + job_queue_pages_per_thread; + + /** + * TC threads can communicate with LQH threads and main threads. + * Cannot communicate with receive threads and other TC threads, + * but it can communicate with itself. + */ + tot += num_tc_threads * + (num_lqh_threads + num_main_threads + 1) * + job_queue_pages_per_thread; + + /** + * Main threads can communicate with all other threads + */ + tot += num_main_threads * + cnt * + job_queue_pages_per_thread; + return tot; } @@ -4710,7 +4862,7 @@ FastScheduler::dumpSignalMemory(Uint32 t const thr_job_queue *q = thr_ptr->m_in_queue + thr_no; const thr_jb_read_state *r = thr_ptr->m_read_states + thr_no; Uint32 read_pos = r->m_read_pos; - if (read_pos > 0) + if (r->is_open() && read_pos > 0) { jbs[num_jbs].m_jb = q->m_buffers[r->m_read_index]; jbs[num_jbs].m_pos = 0; === modified file 'support-files/mysql.spec.sh' --- a/support-files/mysql.spec.sh 2012-03-16 08:07:11 +0000 +++ b/support-files/mysql.spec.sh 2012-05-23 09:04:42 +0000 @@ -283,11 +283,13 @@ documentation and the manual for more in Summary: MySQL: a very fast and reliable SQL database server Group: Applications/Databases Requires: %{distro_requires} -Provides: MySQL-Cluster-server +Provides: MySQL-Cluster-server msqlormysql mysql MySQL mysql-server MySQL-server Obsoletes: MySQL-Cluster-server Obsoletes: MySQL-Cluster-management MySQL-Cluster-storage Obsoletes: MySQL-Cluster-extra MySQL-Cluster-tools -Conflicts: MySQL mysql mysql-server MySQL-server +Obsoletes: mysql MySQL mysql-server MySQL-server +Obsoletes: MySQL-server-classic MySQL-server-community MySQL-server-enterprise +Obsoletes: MySQL-server-advanced MySQL-server-advanced-gpl MySQL-server-enterprise-gpl %description -n MySQL-Cluster-server%{product_suffix} The MySQL(TM) software delivers a very fast, multi-threaded, multi-user, @@ -1009,6 +1011,7 @@ echo "=====" %doc %attr(644, root, man) %{_mandir}/man1/ndb_drop_index.1* %doc %attr(644, root, man) %{_mandir}/man1/ndb_drop_table.1* %doc %attr(644, root, man) %{_mandir}/man1/ndb_error_reporter.1* +%doc %attr(644, root, man) %{_mandir}/man1/ndb_index_stat.1* %doc %attr(644, root, man) %{_mandir}/man1/ndb_mgm.1* %doc %attr(644, root, man) %{_mandir}/man1/ndb_print_backup_file.1* %doc %attr(644, root, man) %{_mandir}/man1/ndb_print_schema_file.1* @@ -1023,6 +1026,7 @@ echo "=====" %doc %attr(644, root, man) %{_mandir}/man8/ndb_mgmd.8* %doc %attr(644, root, man) %{_mandir}/man8/ndbd.8* %doc %attr(644, root, man) %{_mandir}/man8/ndbmtd.8* +%doc %attr(644, root, man) %{_mandir}/man1/ndbinfo_select_all.1* %doc %attr(644, root, man) %{_mandir}/man1/perror.1* %doc %attr(644, root, man) %{_mandir}/man1/replace.1* %doc %attr(644, root, man) %{_mandir}/man1/resolve_stack_dump.1* No bundle (reason: useless for push emails).