#At file:///mnt/raid/alik/MySQL/bzr/00/bug27863/mysql-trunk-bf-bug27863/ based on revid:alik@stripped
3044 Alexander Nozdrin 2010-05-14
Patch for Bug#27863 (excessive memory usage for many small queries in a
multiquery packet).
Background:
- a query can contain multiple SQL statements;
- the server frees resources allocated to process a query when the
whole query is handled. In other words, resources allocated to process
one SQL statement from a multi-statement query are freed when all SQL
statements are handled.
The problem was that the parser allocated a buffer of size of the whole
query for each SQL statement in a multi-statement query. Thus, if a query
had many SQL-statements (so, the query was long), but each SQL statement
was short, ther parser tried to allocate huge amount of memory (number of
small SQL statements * length of the whole query).
The memory was allocated for a so-called "cpp buffer", which is intended to
store pre-processed SQL statement -- SQL text without version specific
comments.
The fix is to allocate memory for the "cpp buffer" once for all SQL
statements (once for a query).
modified:
mysql-test/t/parser_stack.test
sql/ha_ndbcluster_binlog.cc
sql/log_event.cc
sql/sql_lex.cc
sql/sql_lex.h
sql/sql_parse.cc
sql/sql_parse.h
sql/sql_signal.cc
=== modified file 'mysql-test/t/parser_stack.test'
--- a/mysql-test/t/parser_stack.test 2008-07-14 21:41:30 +0000
+++ b/mysql-test/t/parser_stack.test 2010-05-14 18:11:25 +0000
@@ -399,4 +399,12 @@ delimiter ;$$
drop procedure p_37228;
+#
+# Bug#27863 (excessive memory usage for many small queries in a multiquery
+# packet).
+#
+let $i=`select repeat("set @a=1;", 65535)`;
+--disable_query_log
+eval $i;
+--enable_query_log
=== modified file 'sql/ha_ndbcluster_binlog.cc'
--- a/sql/ha_ndbcluster_binlog.cc 2010-03-31 14:05:33 +0000
+++ b/sql/ha_ndbcluster_binlog.cc 2010-05-14 18:11:25 +0000
@@ -263,7 +263,7 @@ static void run_query(THD *thd, char *bu
ulonglong save_thd_options= thd->variables.option_bits;
DBUG_ASSERT(sizeof(save_thd_options) == sizeof(thd->variables.option_bits));
NET save_thd_net= thd->net;
- const char* found_semicolon= NULL;
+ Parser_state parser_state(thd, thd->query(), thd->query_length());
bzero((char*) &thd->net, sizeof(NET));
thd->set_query(buf, (uint) (end - buf));
@@ -277,7 +277,7 @@ static void run_query(THD *thd, char *bu
DBUG_ASSERT(!thd->in_sub_stmt);
DBUG_ASSERT(!thd->locked_tables_mode);
- mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
+ mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
if (no_print_error && thd->is_slave_error)
{
=== modified file 'sql/log_event.cc'
--- a/sql/log_event.cc 2010-04-28 12:47:49 +0000
+++ b/sql/log_event.cc 2010-05-14 18:11:25 +0000
@@ -3290,8 +3290,8 @@ int Query_log_event::do_apply_event(Rela
thd->table_map_for_update= (table_map)table_map_for_update;
/* Execute the query (note that we bypass dispatch_command()) */
- const char* found_semicolon= NULL;
- mysql_parse(thd, thd->query(), thd->query_length(), &found_semicolon);
+ Parser_state parser_state(thd, thd->query(), thd->query_length());
+ mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
log_slow_statement(thd);
/*
=== modified file 'sql/sql_lex.cc'
--- a/sql/sql_lex.cc 2010-04-19 11:05:07 +0000
+++ b/sql/sql_lex.cc 2010-05-14 18:11:25 +0000
@@ -142,37 +142,64 @@ st_parsing_options::reset()
allows_derived= TRUE;
}
+
+/**
+ Perform initialization of Lex_input_stream instance.
+
+ Basically, a buffer for pre-processed query. This buffer should be large
+ enough to keep multi-statement query. The allocation is done once in the
+ Lex_input_stream constructor in order to prevent memory pollution when
+ the server is processing large multi-statement queries.
+
+ @todo Check return value of THD::alloc().
+*/
+
Lex_input_stream::Lex_input_stream(THD *thd,
const char* buffer,
unsigned int length)
-: m_thd(thd),
- yylineno(1),
- yytoklen(0),
- yylval(NULL),
- lookahead_token(-1),
- lookahead_yylval(NULL),
- m_ptr(buffer),
- m_tok_start(NULL),
- m_tok_end(NULL),
- m_end_of_query(buffer + length),
- m_tok_start_prev(NULL),
- m_buf(buffer),
- m_buf_length(length),
- m_echo(TRUE),
- m_cpp_tok_start(NULL),
- m_cpp_tok_start_prev(NULL),
- m_cpp_tok_end(NULL),
- m_body_utf8(NULL),
- m_cpp_utf8_processed_ptr(NULL),
- next_state(MY_LEX_START),
- found_semicolon(NULL),
- ignore_space(test(thd->variables.sql_mode & MODE_IGNORE_SPACE)),
- stmt_prepare_mode(FALSE),
- multi_statements(TRUE),
- in_comment(NO_COMMENT),
- m_underscore_cs(NULL)
+ :m_thd(thd)
{
m_cpp_buf= (char*) thd->alloc(length + 1);
+ reset(buffer, length);
+}
+
+
+/**
+ Prepare Lex_input_stream instance state for use for handling next SQL statement.
+
+ It should be called between two statements in a multi-statement query.
+ The operation resets the input stream to the beginning-of-parse state,
+ but does not reallocate m_cpp_buf.
+*/
+
+void
+Lex_input_stream::reset(const char *buffer, unsigned int length)
+{
+ yylineno= 1;
+ yytoklen= 0;
+ yylval= NULL;
+ lookahead_token= -1;
+ lookahead_yylval= NULL;
+ m_ptr= buffer;
+ m_tok_start= NULL;
+ m_tok_end= NULL;
+ m_end_of_query= buffer + length;
+ m_tok_start_prev= NULL;
+ m_buf= buffer;
+ m_buf_length= length;
+ m_echo= TRUE;
+ m_cpp_tok_start= NULL;
+ m_cpp_tok_start_prev= NULL;
+ m_cpp_tok_end= NULL;
+ m_body_utf8= NULL;
+ m_cpp_utf8_processed_ptr= NULL;
+ next_state= MY_LEX_START;
+ found_semicolon= NULL;
+ ignore_space= test(m_thd->variables.sql_mode & MODE_IGNORE_SPACE);
+ stmt_prepare_mode= FALSE;
+ multi_statements= TRUE;
+ in_comment=NO_COMMENT;
+ m_underscore_cs= NULL;
m_cpp_ptr= m_cpp_buf;
}
=== modified file 'sql/sql_lex.h'
--- a/sql/sql_lex.h 2010-04-19 11:05:07 +0000
+++ b/sql/sql_lex.h 2010-05-14 18:11:25 +0000
@@ -1377,6 +1377,8 @@ public:
Lex_input_stream(THD *thd, const char* buff, unsigned int length);
~Lex_input_stream();
+ void reset(const char *buff, unsigned int length);
+
/**
Set the echo mode.
@@ -2206,8 +2208,8 @@ struct LEX: public Query_tables_list
class Set_signal_information
{
public:
- /** Constructor. */
- Set_signal_information();
+ /** Empty default constructor, use clear() */
+ Set_signal_information() {}
/** Copy constructor. */
Set_signal_information(const Set_signal_information& set);
@@ -2220,7 +2222,7 @@ public:
void clear();
/**
- For each contition item assignment, m_item[] contains the parsed tree
+ For each condition item assignment, m_item[] contains the parsed tree
that represents the expression assigned, if any.
m_item[] is an array indexed by Diag_condition_item_name.
*/
@@ -2237,8 +2239,16 @@ class Yacc_state
{
public:
Yacc_state()
- : yacc_yyss(NULL), yacc_yyvs(NULL)
- {}
+ {
+ reset();
+ }
+
+ void reset()
+ {
+ yacc_yyss= NULL;
+ yacc_yyvs= NULL;
+ m_set_signal_info.clear();
+ }
~Yacc_state();
@@ -2283,6 +2293,12 @@ public:
Lex_input_stream m_lip;
Yacc_state m_yacc;
+
+ void reset(const char *found_semicolon, unsigned int length)
+ {
+ m_lip.reset(found_semicolon, length);
+ m_yacc.reset();
+ }
};
=== modified file 'sql/sql_parse.cc'
--- a/sql/sql_parse.cc 2010-05-14 05:28:51 +0000
+++ b/sql/sql_parse.cc 2010-05-14 18:11:25 +0000
@@ -458,7 +458,6 @@ static void handle_bootstrap_impl(THD *t
{
MYSQL_FILE *file= bootstrap_file;
char *buff;
- const char* found_semicolon= NULL;
DBUG_ENTER("handle_bootstrap");
@@ -531,7 +530,8 @@ static void handle_bootstrap_impl(THD *t
mode we have only one thread.
*/
thd->set_time();
- mysql_parse(thd, thd->query(), length, & found_semicolon);
+ Parser_state parser_state(thd, thd->query(), length);
+ mysql_parse(thd, thd->query(), length, &parser_state);
close_thread_tables(thd); // Free tables
bootstrap_error= thd->is_error();
@@ -1074,19 +1074,21 @@ bool dispatch_command(enum enum_server_c
(char *) thd->security_ctx->host_or_ip);
char *packet_end= thd->query() + thd->query_length();
/* 'b' stands for 'buffer' parameter', special for 'my_snprintf' */
- const char* end_of_stmt= NULL;
general_log_write(thd, command, thd->query(), thd->query_length());
DBUG_PRINT("query",("%-.4096s",thd->query()));
#if defined(ENABLED_PROFILING)
thd->profiling.set_query_source(thd->query(), thd->query_length());
#endif
+ Parser_state parser_state(thd, thd->query(), thd->query_length());
- mysql_parse(thd, thd->query(), thd->query_length(), &end_of_stmt);
+ mysql_parse(thd, thd->query(), thd->query_length(), &parser_state);
- while (!thd->killed && (end_of_stmt != NULL) && ! thd->is_error())
+ while (!thd->killed && (parser_state.m_lip.found_semicolon != NULL) &&
+ ! thd->is_error())
{
- char *beginning_of_next_stmt= (char*) end_of_stmt;
+ char *beginning_of_next_stmt= (char*)
+ parser_state.m_lip.found_semicolon;
thd->protocol->end_statement();
query_cache_end_of_result(thd);
@@ -1127,8 +1129,9 @@ bool dispatch_command(enum enum_server_c
*/
statistic_increment(thd->status_var.questions, &LOCK_status);
thd->set_time(); /* Reset the query start time. */
+ parser_state.reset(beginning_of_next_stmt, length);
/* TODO: set thd->lex->sql_command to SQLCOM_END here */
- mysql_parse(thd, beginning_of_next_stmt, length, &end_of_stmt);
+ mysql_parse(thd, beginning_of_next_stmt, length, &parser_state);
}
DBUG_PRINT("info",("query ready"));
@@ -5721,7 +5724,7 @@ void mysql_init_multi_delete(LEX *lex)
*/
void mysql_parse(THD *thd, const char *inBuf, uint length,
- const char ** found_semicolon)
+ Parser_state *parser_state)
{
int error;
DBUG_ENTER("mysql_parse");
@@ -5751,10 +5754,7 @@ void mysql_parse(THD *thd, const char *i
{
LEX *lex= thd->lex;
- Parser_state parser_state(thd, inBuf, length);
-
- bool err= parse_sql(thd, & parser_state, NULL);
- *found_semicolon= parser_state.m_lip.found_semicolon;
+ bool err= parse_sql(thd, parser_state, NULL);
if (!err)
{
@@ -5769,6 +5769,7 @@ void mysql_parse(THD *thd, const char *i
{
if (! thd->is_error())
{
+ const char *found_semicolon= parser_state->m_lip.found_semicolon;
/*
Binlog logs a string starting from thd->query and having length
thd->query_length; so we set thd->query_length correctly (to not
@@ -5779,12 +5780,12 @@ void mysql_parse(THD *thd, const char *i
PROCESSLIST.
Note that we don't need LOCK_thread_count to modify query_length.
*/
- if (*found_semicolon && (ulong) (*found_semicolon - thd->query()))
+ if (found_semicolon && (ulong) (found_semicolon - thd->query()))
thd->set_query_inner(thd->query(),
- (uint32) (*found_semicolon -
+ (uint32) (found_semicolon -
thd->query() - 1));
/* Actually execute the query */
- if (*found_semicolon)
+ if (found_semicolon)
{
lex->safe_to_cache_query= 0;
thd->server_status|= SERVER_MORE_RESULTS_EXISTS;
@@ -5821,11 +5822,6 @@ void mysql_parse(THD *thd, const char *i
thd->cleanup_after_query();
DBUG_ASSERT(thd->change_list.is_empty());
}
- else
- {
- /* There are no multi queries in the cache. */
- *found_semicolon= NULL;
- }
DBUG_VOID_RETURN;
}
=== modified file 'sql/sql_parse.h'
--- a/sql/sql_parse.h 2010-04-12 13:17:37 +0000
+++ b/sql/sql_parse.h 2010-05-14 18:11:25 +0000
@@ -84,7 +84,7 @@ bool is_log_table_write_query(enum enum_
bool alloc_query(THD *thd, const char *packet, uint packet_length);
void mysql_init_select(LEX *lex);
void mysql_parse(THD *thd, const char *inBuf, uint length,
- const char ** semicolon);
+ Parser_state *parser_state);
void mysql_reset_thd_for_next_command(THD *thd);
bool mysql_new_select(LEX *lex, bool move_down);
void create_select_for_variable(const char *var_name);
=== modified file 'sql/sql_signal.cc'
--- a/sql/sql_signal.cc 2010-05-14 05:28:51 +0000
+++ b/sql/sql_signal.cc 2010-05-14 18:11:25 +0000
@@ -75,10 +75,6 @@ const LEX_STRING Diag_statement_item_nam
{ C_STRING_WITH_LEN("TRANSACTION_ACTIVE") }
};
-Set_signal_information::Set_signal_information()
-{
- clear();
-}
Set_signal_information::Set_signal_information(
const Set_signal_information& set)
Attachment: [text/bzr-bundle] bzr/alik@sun.com-20100514181125-g96dzokifgz1n8a3.bundle
