3569 Tor Didriksen 2011-11-07
Bug#11748783 37359: FILESORT CAN BE MORE EFFICIENT
filesort() initializes space for
MIN(Estimate rows in table, max rows that can fit in sort buffer) rows
before doing the sort. For queries that have a "where" clause, the
number of rows estimated by estimate_rows_upper_bound can be much
higher than rows that match the where clause.
The initialization is done via make_char_array() in filesort.cc and is
very expensive when the estimate is off.
Solution: lazy-init of the pointers in the sort buffer.
@ mysql-test/r/filesort_debug.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/filesort_pq.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/general2_no_prot.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/general2_ps_prot.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/general_no_prot_none.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/general_ps_prot_none.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/subquery_no_prot.result
Add sort_buffer_size to opt_trace.
@ mysql-test/suite/opt_trace/r/subquery_ps_prot.result
Add sort_buffer_size to opt_trace.
@ mysql-test/t/filesort_debug.test
Add sort_buffer_size to opt_trace.
@ sql/filesort.cc
Use Filesort_buffer* rather than uchar **
@ sql/filesort_utils.cc
New class Filesort_buffer, which wraps the uchar** buffer.
@ sql/filesort_utils.h
New class Filesort_buffer, which wraps the uchar** buffer.
@ sql/table.h
New class Filesort_buffer, which wraps the uchar** buffer.
Rename FILESORT_INFO => Filesort_info.
@ unittest/gunit/CMakeLists.txt
New unit test.
@ unittest/gunit/filesort_buffer-t.cc
New unit test.
added:
unittest/gunit/filesort_buffer-t.cc
modified:
mysql-test/r/filesort_debug.result
mysql-test/suite/opt_trace/r/filesort_pq.result
mysql-test/suite/opt_trace/r/general2_no_prot.result
mysql-test/suite/opt_trace/r/general2_ps_prot.result
mysql-test/suite/opt_trace/r/general_no_prot_none.result
mysql-test/suite/opt_trace/r/general_ps_prot_none.result
mysql-test/suite/opt_trace/r/subquery_no_prot.result
mysql-test/suite/opt_trace/r/subquery_ps_prot.result
mysql-test/t/filesort_debug.test
sql/filesort.cc
sql/filesort_utils.cc
sql/filesort_utils.h
sql/table.h
unittest/gunit/CMakeLists.txt
3568 Tor Didriksen 2011-11-07
WL#5825 Using C++ Standard Library with MySQL code
Post-push fix: different library layout for stlport on intel/sparc.
modified:
configure.cmake
=== modified file 'mysql-test/r/filesort_debug.result'
--- a/mysql-test/r/filesort_debug.result 2011-02-24 07:18:03 +0000
+++ b/mysql-test/r/filesort_debug.result 2011-11-07 15:32:36 +0000
@@ -16,7 +16,7 @@ DROP TABLE t1;
#
CREATE TABLE t1(f0 int auto_increment primary key, f1 int);
INSERT INTO t1(f1) VALUES (0),(1),(2),(3),(4),(5);
-SET session debug= '+d,make_char_array_fail';
+SET session debug= '+d,alloc_sort_buffer_fail';
CALL mtr.add_suppression("Out of sort memory");
SELECT * FROM t1 ORDER BY f1 ASC, f0;
ERROR HY001: Out of sort memory, consider increasing server sort buffer size
=== modified file 'mysql-test/suite/opt_trace/r/filesort_pq.result'
--- a/mysql-test/suite/opt_trace/r/filesort_pq.result 2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/filesort_pq.result 2011-11-07 15:32:36 +0000
@@ -269,6 +269,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
"rows": 100,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 25080,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -446,6 +447,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 7068,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -658,6 +660,7 @@ SELECT * FROM t1 ORDER BY f2 DESC, f0 LI
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 13144,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -884,6 +887,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -1120,6 +1124,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -1291,6 +1296,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 11,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 4664,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -1429,6 +1435,7 @@ SELECT CONCAT("hello ", f2) AS foo FROM
"rows": 3,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 1254,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -1566,6 +1573,7 @@ SELECT * from t1 ORDER BY rand(2) LIMIT
"rows": 3,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 72,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -1744,6 +1752,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 7068,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -1956,6 +1965,7 @@ SELECT * FROM t1 ORDER BY f2 DESC, f0 LI
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 13144,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -2182,6 +2192,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -2418,6 +2429,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -2589,6 +2601,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 11,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 4664,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -2774,6 +2787,7 @@ ORDER BY f1, f0 LIMIT 30 {
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 7068,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -2926,6 +2940,7 @@ ORDER BY f1, f0 LIMIT 0 {
"rows": 1,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 228,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -3122,6 +3137,7 @@ ORDER BY f2, f0 LIMIT 20 {
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -3298,6 +3314,7 @@ ORDER BY f2, f0 LIMIT 0 {
"rows": 1,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 424,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -3484,6 +3501,7 @@ ORDER BY f2, f0 LIMIT 10 OFFSET 10 {
"rows": 21,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 8904,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -3660,6 +3678,7 @@ ORDER BY f2, f0 LIMIT 0 OFFSET 10 {
"rows": 11,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 4664,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -3942,6 +3961,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 {
"rows": 31,
"examined_rows": 1500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 775,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -4216,6 +4236,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30 {
"rows": 61,
"examined_rows": 1500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 1525,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -4495,6 +4516,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30 {
"rows": 61,
"examined_rows": 1500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 1525,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -4802,6 +4824,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30 {
"rows": 61,
"examined_rows": 1185,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 1525,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -5109,6 +5132,7 @@ SELECT * FROM v1 {
"rows": 31,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 7068,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -5405,6 +5429,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
"rows": 101,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 23028,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -5439,6 +5464,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 6851,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -6039,6 +6065,7 @@ LIMIT 30 {
"rows": 101,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 23028,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -6077,6 +6104,7 @@ LIMIT 30 {
"rows": 101,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 42824,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -6117,6 +6145,7 @@ LIMIT 30 {
"rows": 31,
"examined_rows": 325,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 6975,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -6284,6 +6313,7 @@ GROUP BY 1 ORDER BY 2,1 LIMIT 5 {
"rows": 6,
"examined_rows": 10,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 198,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -6534,6 +6564,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 31,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 13144,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -6718,6 +6749,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
"rows": 31,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 13144,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -6903,6 +6935,7 @@ ORDER BY f2, f0 LIMIT 15 OFFSET 15 {
"rows": 31,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 13144,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -7194,6 +7227,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
"rows": 101,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 23028,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -7228,6 +7262,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
"rows": 31,
"examined_rows": 100,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 6851,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -7591,6 +7626,7 @@ ORDER BY d1.f2 DESC LIMIT 30 {
"rows": 31,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 6944,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -7619,6 +7655,7 @@ ORDER BY d1.f2 DESC LIMIT 30 {
"rows": 31,
"examined_rows": 620,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 6727,
"sort_mode": "<sort_key, rowid>"
}
}
@@ -7881,6 +7918,7 @@ SELECT * FROM t1 WHERE f1 = (SELECT f1 F
"rows": 2,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 36,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -11637,6 +11675,7 @@ SELECT * FROM t1 WHERE f1 = (SELECT f1 F
"rows": 3,
"examined_rows": 500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 54,
"sort_mode": "<sort_key, additional_fields>"
}
}
@@ -11903,6 +11942,7 @@ SELECT * FROM t1 ORDER BY f2 LIMIT 100 {
"rows": 101,
"examined_rows": 438500,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 21715,
"sort_mode": "<sort_key, rowid>"
}
}
=== modified file 'mysql-test/suite/opt_trace/r/general2_no_prot.result'
--- a/mysql-test/suite/opt_trace/r/general2_no_prot.result 2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general2_no_prot.result 2011-11-07 15:32:36 +0000
@@ -1119,6 +1119,7 @@ TRACE
"rows": 4,
"examined_rows": 4,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 252,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
@@ -1354,6 +1355,7 @@ TRACE
"rows": 3,
"examined_rows": 3,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 273,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
@@ -4860,6 +4862,7 @@ GROUP BY field2 ORDER BY alias1.col_int_
"rows": 8,
"examined_rows": 8,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 378,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/suite/opt_trace/r/general2_ps_prot.result'
--- a/mysql-test/suite/opt_trace/r/general2_ps_prot.result 2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general2_ps_prot.result 2011-11-07 15:32:36 +0000
@@ -1137,6 +1137,7 @@ TRACE
"rows": 4,
"examined_rows": 4,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 252,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
@@ -1372,6 +1373,7 @@ TRACE
"rows": 3,
"examined_rows": 3,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 273,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
@@ -4913,6 +4915,7 @@ GROUP BY field2 ORDER BY alias1.col_int_
"rows": 8,
"examined_rows": 8,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 378,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/suite/opt_trace/r/general_no_prot_none.result'
--- a/mysql-test/suite/opt_trace/r/general_no_prot_none.result 2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general_no_prot_none.result 2011-11-07 15:32:36 +0000
@@ -5835,6 +5835,7 @@ trace
"rows": 2,
"examined_rows": 2,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 252,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/suite/opt_trace/r/general_ps_prot_none.result'
--- a/mysql-test/suite/opt_trace/r/general_ps_prot_none.result 2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general_ps_prot_none.result 2011-11-07 15:32:36 +0000
@@ -5793,6 +5793,7 @@ trace
"rows": 2,
"examined_rows": 2,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 252,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/suite/opt_trace/r/subquery_no_prot.result'
--- a/mysql-test/suite/opt_trace/r/subquery_no_prot.result 2011-10-27 08:52:27 +0000
+++ b/mysql-test/suite/opt_trace/r/subquery_no_prot.result 2011-11-07 15:32:36 +0000
@@ -1762,6 +1762,7 @@ field4,field5,field6 {
"rows": 0,
"examined_rows": 0,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 260,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/suite/opt_trace/r/subquery_ps_prot.result'
--- a/mysql-test/suite/opt_trace/r/subquery_ps_prot.result 2011-10-27 08:52:27 +0000
+++ b/mysql-test/suite/opt_trace/r/subquery_ps_prot.result 2011-11-07 15:32:36 +0000
@@ -1740,6 +1740,7 @@ field4,field5,field6 {
"rows": 0,
"examined_rows": 0,
"number_of_tmp_files": 0,
+ "sort_buffer_size": 260,
"sort_mode": "<sort_key, rowid>"
} /* filesort_summary */
}
=== modified file 'mysql-test/t/filesort_debug.test'
--- a/mysql-test/t/filesort_debug.test 2011-02-24 07:18:03 +0000
+++ b/mysql-test/t/filesort_debug.test 2011-11-07 15:32:36 +0000
@@ -28,7 +28,7 @@ DROP TABLE t1;
CREATE TABLE t1(f0 int auto_increment primary key, f1 int);
INSERT INTO t1(f1) VALUES (0),(1),(2),(3),(4),(5);
-SET session debug= '+d,make_char_array_fail';
+SET session debug= '+d,alloc_sort_buffer_fail';
CALL mtr.add_suppression("Out of sort memory");
--error ER_OUT_OF_SORTMEMORY
SELECT * FROM t1 ORDER BY f1 ASC, f0;
=== modified file 'sql/filesort.cc'
--- a/sql/filesort.cc 2011-10-28 12:45:35 +0000
+++ b/sql/filesort.cc 2011-11-07 15:32:36 +0000
@@ -43,11 +43,11 @@ using std::min;
/* functions defined in this file */
-static void make_char_array(FILESORT_INFO *info, uint fields, uint length);
static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count,
uchar *buf);
static ha_rows find_all_keys(Sort_param *param,SQL_SELECT *select,
- uchar **sort_keys, IO_CACHE *buffer_file,
+ Filesort_info *fs_info,
+ IO_CACHE *buffer_file,
IO_CACHE *tempfile,
Bounded_queue<uchar, uchar> *pq,
ha_rows *found_rows);
@@ -60,7 +60,7 @@ static int merge_index(Sort_param *param
uint maxbuffer,IO_CACHE *tempfile,
IO_CACHE *outfile);
static bool save_index(Sort_param *param,uchar **sort_keys, uint count,
- FILESORT_INFO *table_sort);
+ Filesort_info *table_sort);
static uint suffix_length(ulong string_length);
static uint sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length,
bool *multi_byte_charset);
@@ -70,7 +70,7 @@ static SORT_ADDON_FIELD *get_addon_field
static void unpack_addon_fields(struct st_sort_addon_field *addon_field,
uchar *buff);
static bool check_if_pq_applicable(Opt_trace_context *trace,
- Sort_param *param, FILESORT_INFO *info,
+ Sort_param *param, Filesort_info *info,
TABLE *table,
ha_rows records, ulong memory_available);
@@ -177,7 +177,6 @@ ha_rows filesort(THD *thd, TABLE *table,
uint maxbuffer;
BUFFPEK *buffpek;
ha_rows num_rows= HA_POS_ERROR;
- uchar **sort_keys= NULL;
IO_CACHE tempfile, buffpek_pointers, *outfile;
Sort_param param;
bool multi_byte_charset;
@@ -212,7 +211,7 @@ ha_rows filesort(THD *thd, TABLE *table,
QUICK_INDEX_MERGE_SELECT. Work with a copy and put it back at the end
when index_merge select has finished with it.
*/
- FILESORT_INFO table_sort= table->sort;
+ Filesort_info table_sort= table->sort;
table->sort.io_cache= NULL;
DBUG_ASSERT(table_sort.record_pointers == NULL);
@@ -258,18 +257,19 @@ ha_rows filesort(THD *thd, TABLE *table,
true, // max_at_top
NULL, // compare_function
compare_length,
- &make_sortkey, ¶m, table_sort.sort_keys))
+ &make_sortkey, ¶m, table_sort.get_sort_keys()))
{
/*
If we fail to init pq, we have to give up:
out of memory means my_malloc() will call my_error().
*/
DBUG_PRINT("info", ("failed to allocate PQ"));
- my_free(table_sort.sort_keys);
- table_sort.sort_keys= NULL;
+ table_sort.free_sort_buffer();
DBUG_ASSERT(thd->is_error());
goto err;
}
+ // For PQ queries (with limit) we initialize all pointers.
+ table_sort.init_record_pointers();
}
else
{
@@ -280,9 +280,9 @@ ha_rows filesort(THD *thd, TABLE *table,
while (memory_available >= min_sort_memory)
{
ha_rows keys= memory_available / (param.rec_length + sizeof(char*));
- param.max_keys_per_buffer= static_cast<uint>(min(num_rows, keys));
- make_char_array(&table_sort, param.max_keys_per_buffer, param.rec_length);
- if (table_sort.sort_keys)
+ param.max_keys_per_buffer= (uint) min(num_rows, keys);
+ table_sort.alloc_sort_buffer(param.max_keys_per_buffer, param.rec_length);
+ if (table_sort.get_sort_keys())
break;
ulong old_memory_available= memory_available;
memory_available= memory_available/4*3;
@@ -297,7 +297,6 @@ ha_rows filesort(THD *thd, TABLE *table,
}
}
- sort_keys= table_sort.sort_keys;
if (open_cached_file(&buffpek_pointers,mysql_tmpdir,TEMP_PREFIX,
DISK_BUFFER_SIZE, MYF(MY_WME)))
goto err;
@@ -307,7 +306,9 @@ ha_rows filesort(THD *thd, TABLE *table,
// New scope, because subquery execution must be traced within an array.
{
Opt_trace_array ota(trace, "filesort_execution");
- num_rows= find_all_keys(¶m, select, sort_keys, &buffpek_pointers,
+ num_rows= find_all_keys(¶m, select,
+ &table_sort,
+ &buffpek_pointers,
&tempfile,
pq.is_initialized() ? &pq : NULL,
found_rows);
@@ -321,13 +322,15 @@ ha_rows filesort(THD *thd, TABLE *table,
.add("rows", num_rows)
.add("examined_rows", param.examined_rows)
.add("number_of_tmp_files", maxbuffer)
+ .add("sort_buffer_size", table_sort.sort_buffer_size())
.add_alnum("sort_mode",
param.addon_field ?
"<sort_key, additional_fields>" : "<sort_key, rowid>");
if (maxbuffer == 0) // The whole set is in memory
{
- if (save_index(¶m, sort_keys, (uint) num_rows, &table_sort))
+ if (save_index(¶m, table_sort.get_sort_keys(),
+ (uint) num_rows, &table_sort))
goto err;
}
else
@@ -363,13 +366,19 @@ ha_rows filesort(THD *thd, TABLE *table,
(param.rec_length + sizeof(char*))) /
param.rec_length - 1);
maxbuffer--; // Offset from 0
- if (merge_many_buff(¶m,(uchar*) sort_keys,buffpek,&maxbuffer,
+ if (merge_many_buff(¶m,
+ (uchar*) table_sort.get_sort_keys(),
+ buffpek,&maxbuffer,
&tempfile))
goto err;
if (flush_io_cache(&tempfile) ||
reinit_io_cache(&tempfile,READ_CACHE,0L,0,0))
goto err;
- if (merge_index(¶m,(uchar*) sort_keys,buffpek,maxbuffer,&tempfile,
+ if (merge_index(¶m,
+ (uchar*) table_sort.get_sort_keys(),
+ buffpek,
+ maxbuffer,
+ &tempfile,
outfile))
goto err;
}
@@ -385,8 +394,7 @@ ha_rows filesort(THD *thd, TABLE *table,
my_free(param.tmp_buffer);
if (!subselect || !subselect->is_uncacheable())
{
- my_free(sort_keys);
- table_sort.sort_keys= 0;
+ table_sort.free_sort_buffer();
my_free(buffpek);
table_sort.buffpek= 0;
table_sort.buffpek_len= 0;
@@ -450,8 +458,7 @@ void filesort_free_buffers(TABLE *table,
if (full)
{
- my_free(table->sort.sort_keys);
- table->sort.sort_keys= NULL;
+ table->sort.free_sort_buffer();
my_free(table->sort.buffpek);
table->sort.buffpek= NULL;
table->sort.buffpek_len= 0;
@@ -467,36 +474,11 @@ void filesort_free_buffers(TABLE *table,
/**
Makes an array of string pointers for info->sort_keys.
- @param info FILESORT_INFO struct owning the allocated array.
+ @param info Filesort_info struct owning the allocated array.
@param num_records Number of records.
@param length Length of each record.
*/
-static void make_char_array(FILESORT_INFO *info, uint num_records, uint length)
-{
- DBUG_ENTER("make_char_array");
-
- DBUG_PRINT("info", ("num_records %u length %u", num_records, length));
-
- DBUG_EXECUTE_IF("make_char_array_fail",
- DBUG_SET("+d,simulate_out_of_memory"););
-
- if (!info->sort_keys)
- info->sort_keys=
- (uchar**) my_malloc(num_records * (length + sizeof(uchar*)), MYF(0));
-
- if (info->sort_keys)
- {
- uchar **pos= info->sort_keys;
- uchar *char_pos= ((uchar*) (pos+num_records)) - length;
- while (num_records--)
- *(pos++)= (char_pos+= length);
- }
-
- DBUG_VOID_RETURN;
-}
-
-
/** Read 'count' number of buffer pointers into memory. */
static uchar *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count,
@@ -619,7 +601,7 @@ static void dbug_print_record(TABLE *tab
*/
static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
- uchar **sort_keys,
+ Filesort_info *fs_info,
IO_CACHE *buffpek_pointers,
IO_CACHE *tempfile,
Bounded_queue<uchar, uchar> *pq,
@@ -740,12 +722,13 @@ static ha_rows find_all_keys(Sort_param
{
if (idx == param->max_keys_per_buffer)
{
- if (write_keys(param,sort_keys, idx, buffpek_pointers, tempfile))
+ if (write_keys(param, fs_info->get_sort_keys(),
+ idx, buffpek_pointers, tempfile))
DBUG_RETURN(HA_POS_ERROR);
idx= 0;
indexpos++;
}
- make_sortkey(param, sort_keys[idx++], ref_pos);
+ make_sortkey(param, fs_info->get_record_buffer(idx++), ref_pos);
}
}
else
@@ -770,11 +753,12 @@ static ha_rows find_all_keys(Sort_param
DBUG_PRINT("test",("error: %d indexpos: %d",error,indexpos));
if (error != HA_ERR_END_OF_FILE)
{
- file->print_error(error,MYF(ME_ERROR | ME_WAITTANG)); /* purecov: inspected */
+ file->print_error(error,MYF(ME_ERROR | ME_WAITTANG)); // purecov: inspected
DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */
}
if (indexpos && idx &&
- write_keys(param,sort_keys,idx,buffpek_pointers,tempfile))
+ write_keys(param, fs_info->get_sort_keys(),
+ idx, buffpek_pointers, tempfile))
DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */
const ha_rows retval=
my_b_inited(tempfile) ?
@@ -1151,7 +1135,7 @@ static void register_used_fields(Sort_pa
static bool save_index(Sort_param *param, uchar **sort_keys, uint count,
- FILESORT_INFO *table_sort)
+ Filesort_info *table_sort)
{
uint offset,res_length;
uchar *to;
@@ -1204,7 +1188,7 @@ static bool save_index(Sort_param *param
bool check_if_pq_applicable(Opt_trace_context *trace,
Sort_param *param,
- FILESORT_INFO *filesort_info,
+ Filesort_info *filesort_info,
TABLE *table, ha_rows num_rows,
ulong memory_available)
{
@@ -1248,10 +1232,10 @@ bool check_if_pq_applicable(Opt_trace_co
// The whole source set fits into memory.
if (param->max_rows < num_rows/PQ_slowness )
{
- make_char_array(filesort_info,
- param->max_keys_per_buffer, param->rec_length);
+ filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+ param->rec_length);
trace_filesort.add("chosen", true);
- DBUG_RETURN(filesort_info->sort_keys != NULL);
+ DBUG_RETURN(filesort_info->get_sort_keys() != NULL);
}
else
{
@@ -1265,10 +1249,10 @@ bool check_if_pq_applicable(Opt_trace_co
// Do we have space for LIMIT rows in memory?
if (param->max_keys_per_buffer < num_available_keys)
{
- make_char_array(filesort_info,
- param->max_keys_per_buffer, param->rec_length);
+ filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+ param->rec_length);
trace_filesort.add("chosen", true);
- DBUG_RETURN(filesort_info->sort_keys != NULL);
+ DBUG_RETURN(filesort_info->get_sort_keys() != NULL);
}
// Try to strip off addon fields.
@@ -1317,10 +1301,9 @@ bool check_if_pq_applicable(Opt_trace_co
}
trace_addon.add("chosen", true);
- make_char_array(filesort_info,
- param->max_keys_per_buffer,
- param->sort_length + param->ref_length);
- if (filesort_info->sort_keys)
+ filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+ param->sort_length + param->ref_length);
+ if (filesort_info->get_sort_keys())
{
// Make attached data to be references instead of fields.
my_free(filesort_info->addon_buf);
=== modified file 'sql/filesort_utils.cc'
--- a/sql/filesort_utils.cc 2011-06-30 15:50:45 +0000
+++ b/sql/filesort_utils.cc 2011-11-07 15:32:36 +0000
@@ -16,6 +16,7 @@
#include "filesort_utils.h"
#include "sql_const.h"
#include "sql_sort.h"
+#include "table.h"
namespace {
@@ -82,3 +83,36 @@ double get_merge_many_buffs_cost_fast(ha
}
+uchar **Filesort_buffer::alloc_sort_buffer(uint num_records, uint record_length)
+{
+ DBUG_ENTER("alloc_sort_buffer");
+
+ DBUG_EXECUTE_IF("alloc_sort_buffer_fail",
+ DBUG_SET("+d,simulate_out_of_memory"););
+
+ if (m_idx_array.is_null())
+ {
+ uchar **sort_keys=
+ (uchar**) my_malloc(num_records * (record_length + sizeof(uchar*)),
+ MYF(0));
+ m_idx_array= Idx_array(sort_keys, num_records);
+ m_record_length= record_length;
+ uchar **start_of_data= m_idx_array.array() + m_idx_array.size();
+ m_start_of_data= reinterpret_cast<uchar*>(start_of_data);
+ }
+ else
+ {
+ DBUG_ASSERT(num_records == m_idx_array.size());
+ DBUG_ASSERT(record_length == m_record_length);
+ }
+ DBUG_RETURN(m_idx_array.array());
+}
+
+
+void Filesort_buffer::free_sort_buffer()
+{
+ my_free(m_idx_array.array());
+ m_idx_array= Idx_array();
+ m_record_length= 0;
+ m_start_of_data= NULL;
+}
=== modified file 'sql/filesort_utils.h'
--- a/sql/filesort_utils.h 2011-06-30 15:50:45 +0000
+++ b/sql/filesort_utils.h 2011-11-07 15:32:36 +0000
@@ -14,8 +14,11 @@
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
#ifndef FILESORT_UTILS_INCLUDED
+#define FILESORT_UTILS_INCLUDED
+#include "my_global.h"
#include "my_base.h"
+#include "sql_array.h"
/*
Calculate cost of merge sort
@@ -36,9 +39,79 @@
See also comments get_merge_many_buffs_cost().
*/
-#define FILESORT_UTILS_INCLUDED
double get_merge_many_buffs_cost_fast(ha_rows num_rows,
ha_rows num_keys_per_buffer,
uint elem_size);
+
+/**
+ A wrapper class around the buffer used by filesort().
+ The buffer is a contiguous chunk of memory,
+ where the first part is <num_records> pointers to the actual data.
+
+ We wrap the buffer in order to be able to do lazy initialization of the
+ pointers: the buffer is often much larger than what we actually need.
+
+ The buffer must be kept available for multiple executions of the
+ same sort operation, so we have explicit allocate and free functions,
+ rather than doing alloc/free in CTOR/DTOR.
+ */
+class Filesort_buffer
+{
+public:
+ Filesort_buffer() :
+ m_idx_array(), m_record_length(0), m_start_of_data(NULL)
+ {}
+
+ /// Initializes a record pointer.
+ uchar *get_record_buffer(uint idx)
+ {
+ m_idx_array[idx]= m_start_of_data + (idx * m_record_length);
+ return m_idx_array[idx];
+ }
+
+ /// Initializes all the record pointers.
+ void init_record_pointers()
+ {
+ for (uint ix= 0; ix < m_idx_array.size(); ++ix)
+ (void) get_record_buffer(ix);
+ }
+
+ /// Returns total size: pointer array + record buffers.
+ size_t sort_buffer_size() const
+ {
+ return m_idx_array.size() * (m_record_length + sizeof(uchar*));
+ }
+
+ /// Allocates the buffer, but does *not* initialize pointers.
+ uchar **alloc_sort_buffer(uint num_records, uint record_length);
+
+ /// Frees the buffer.
+ void free_sort_buffer();
+
+ /// Getter, for calling routines which still use the uchar** interface.
+ uchar **get_sort_keys() { return m_idx_array.array(); }
+
+ /**
+ We need an assignment operator, see filesort().
+ This happens to have the same semantics as the one that would be
+ generated by the compiler. We still implement it here, to show shallow
+ assignment explicitly: we have two objects sharing the same array.
+ */
+ Filesort_buffer &operator=(const Filesort_buffer &rhs)
+ {
+ m_idx_array= rhs.m_idx_array;
+ m_record_length= rhs.m_record_length;
+ m_start_of_data= rhs.m_start_of_data;
+ return *this;
+ }
+
+private:
+ typedef Bounds_checked_array<uchar*> Idx_array;
+
+ Idx_array m_idx_array;
+ uint m_record_length;
+ uchar *m_start_of_data;
+};
+
#endif // FILESORT_UTILS_INCLUDED
=== modified file 'sql/table.h'
--- a/sql/table.h 2011-10-18 14:25:42 +0000
+++ b/sql/table.h 2011-11-07 15:32:36 +0000
@@ -28,6 +28,7 @@
#include "handler.h" /* row_type, ha_choice, handler */
#include "mysql_com.h" /* enum_field_types */
#include "thr_lock.h" /* thr_lock_type */
+#include "filesort_utils.h"
/* Structs that defines the TABLE */
@@ -300,10 +301,14 @@ enum tmp_table_type
};
enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP };
-typedef struct st_filesort_info
+
+class Filesort_info
{
+ /// Buffer for sorting keys.
+ Filesort_buffer filesort_buffer;
+
+public:
IO_CACHE *io_cache; /* If sorted through filesort */
- uchar **sort_keys; /* Buffer for sorting keys */
uchar *buffpek; /* Buffer for buffpek structures */
uint buffpek_len; /* Max number of buffpeks in the buffer */
uchar *addon_buf; /* Pointer to a buffer if sorted with fields */
@@ -312,7 +317,28 @@ typedef struct st_filesort_info
void (*unpack)(struct st_sort_addon_field *, uchar *); /* To unpack back */
uchar *record_pointers; /* If sorted in memory */
ha_rows found_records; /* How many records in sort */
-} FILESORT_INFO;
+
+ /**
+ Accessors for Filesort_buffer (which @c).
+ */
+ uchar *get_record_buffer(uint idx)
+ { return filesort_buffer.get_record_buffer(idx); }
+
+ uchar **get_sort_keys()
+ { return filesort_buffer.get_sort_keys(); }
+
+ uchar **alloc_sort_buffer(uint num_records, uint record_length)
+ { return filesort_buffer.alloc_sort_buffer(num_records, record_length); }
+
+ void free_sort_buffer()
+ { filesort_buffer.free_sort_buffer(); }
+
+ void init_record_pointers()
+ { filesort_buffer.init_record_pointers(); }
+
+ size_t sort_buffer_size() const
+ { return filesort_buffer.sort_buffer_size(); }
+};
/*
@@ -1134,7 +1160,7 @@ public:
REGINFO reginfo; /* field connections */
MEM_ROOT mem_root;
GRANT_INFO grant;
- FILESORT_INFO sort;
+ Filesort_info sort;
#ifdef WITH_PARTITION_STORAGE_ENGINE
partition_info *part_info; /* Partition related information */
bool no_partitions_used; /* If true, all partitions have been pruned away */
=== modified file 'unittest/gunit/CMakeLists.txt'
--- a/unittest/gunit/CMakeLists.txt 2011-10-28 12:45:35 +0000
+++ b/unittest/gunit/CMakeLists.txt 2011-11-07 15:32:36 +0000
@@ -215,6 +215,7 @@ SET(TESTS
dbug
decimal
dynarray
+ filesort_buffer
mdl
mdl_mytap
my_bitmap
=== added file 'unittest/gunit/filesort_buffer-t.cc'
--- a/unittest/gunit/filesort_buffer-t.cc 1970-01-01 00:00:00 +0000
+++ b/unittest/gunit/filesort_buffer-t.cc 2011-11-07 15:32:36 +0000
@@ -0,0 +1,87 @@
+/* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02111-1307 USA */
+
+// First include (the generated) my_config.h, to get correct platform defines,
+// then gtest.h (before any other MySQL headers), to avoid min() macros etc ...
+#include "my_config.h"
+#include <gtest/gtest.h>
+
+#include "filesort_utils.h"
+#include "table.h"
+
+
+namespace {
+
+class FileSortBufferTest : public ::testing::Test
+{
+protected:
+ virtual void TearDown()
+ {
+ fs_info.free_sort_buffer();
+ }
+
+ Filesort_info fs_info;
+};
+
+
+TEST_F(FileSortBufferTest, FileSortBuffer)
+{
+ const char letters[10]= "abcdefghi";
+ uchar **sort_keys= fs_info.alloc_sort_buffer(10, sizeof(char));
+ uchar **null_sort_keys= NULL;
+ EXPECT_NE(null_sort_keys, sort_keys);
+ for (uint ix= 0; ix < 10; ++ix)
+ {
+ uchar *ptr= fs_info.get_record_buffer(ix);
+ *ptr= letters[ix];
+ }
+ uchar *data= *fs_info.get_sort_keys();
+ const char *str= reinterpret_cast<const char*>(data);
+ EXPECT_STREQ(letters, str);
+
+ const size_t expected_size= 10 * (sizeof(char*) + sizeof(char));
+ EXPECT_EQ(expected_size, fs_info.sort_buffer_size());
+}
+
+
+TEST_F(FileSortBufferTest, InitRecordPointers)
+{
+ fs_info.alloc_sort_buffer(10, sizeof(char));
+ fs_info.init_record_pointers();
+ uchar **ptr= fs_info.get_sort_keys();
+ for (uint ix= 0; ix < 10 - 1; ++ix)
+ {
+ uchar **nxt= ptr + 1;
+ EXPECT_EQ(1, *nxt - *ptr);
+ ++ptr;
+ }
+}
+
+
+TEST_F(FileSortBufferTest, AssignmentOperator)
+{
+ fs_info.alloc_sort_buffer(10, sizeof(char));
+ Filesort_info fs_copy;
+ fs_copy= fs_info;
+ for (uint ix= 0; ix < 10 - 1; ++ix)
+ {
+ EXPECT_EQ(fs_copy.get_record_buffer(ix), fs_info.get_record_buffer(ix));
+ }
+ EXPECT_EQ(fs_copy.get_sort_keys(), fs_info.get_sort_keys());
+ EXPECT_EQ(fs_copy.sort_buffer_size(), fs_info.sort_buffer_size());
+}
+
+
+} // namespace
No bundle (reason: useless for push emails).
| Thread |
|---|
| • bzr push into mysql-trunk branch (tor.didriksen:3568 to 3569) Bug#11748783 | Tor Didriksen | 7 Nov |
