List:Commits« Previous MessageNext Message »
From:Tor Didriksen Date:November 7 2011 3:33pm
Subject:bzr push into mysql-trunk branch (tor.didriksen:3568 to 3569) Bug#11748783
View as plain text  
 3569 Tor Didriksen	2011-11-07
      Bug#11748783 37359: FILESORT CAN BE MORE EFFICIENT
      
      filesort() initializes space for
      MIN(Estimate rows in table, max rows that can fit in sort buffer) rows
      before doing the sort. For queries that have a "where" clause, the
      number of rows estimated by estimate_rows_upper_bound can be much
      higher than rows that match the where clause.
      
      The initialization is done via make_char_array() in filesort.cc and is
      very expensive when the estimate is off.
      
      Solution: lazy-init of the pointers in the sort buffer.
     @ mysql-test/r/filesort_debug.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/filesort_pq.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/general2_no_prot.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/general2_ps_prot.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/general_no_prot_none.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/general_ps_prot_none.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/subquery_no_prot.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/suite/opt_trace/r/subquery_ps_prot.result
        Add sort_buffer_size to opt_trace.
     @ mysql-test/t/filesort_debug.test
        Add sort_buffer_size to opt_trace.
     @ sql/filesort.cc
        Use Filesort_buffer* rather than uchar **
     @ sql/filesort_utils.cc
        New class Filesort_buffer, which wraps the uchar** buffer.
     @ sql/filesort_utils.h
        New class Filesort_buffer, which wraps the uchar** buffer.
     @ sql/table.h
        New class Filesort_buffer, which wraps the uchar** buffer.
        Rename FILESORT_INFO => Filesort_info.
     @ unittest/gunit/CMakeLists.txt
        New unit test.
     @ unittest/gunit/filesort_buffer-t.cc
        New unit test.

    added:
      unittest/gunit/filesort_buffer-t.cc
    modified:
      mysql-test/r/filesort_debug.result
      mysql-test/suite/opt_trace/r/filesort_pq.result
      mysql-test/suite/opt_trace/r/general2_no_prot.result
      mysql-test/suite/opt_trace/r/general2_ps_prot.result
      mysql-test/suite/opt_trace/r/general_no_prot_none.result
      mysql-test/suite/opt_trace/r/general_ps_prot_none.result
      mysql-test/suite/opt_trace/r/subquery_no_prot.result
      mysql-test/suite/opt_trace/r/subquery_ps_prot.result
      mysql-test/t/filesort_debug.test
      sql/filesort.cc
      sql/filesort_utils.cc
      sql/filesort_utils.h
      sql/table.h
      unittest/gunit/CMakeLists.txt
 3568 Tor Didriksen	2011-11-07
      WL#5825 Using C++ Standard Library with MySQL code
      
      Post-push fix: different library layout for stlport on intel/sparc.

    modified:
      configure.cmake
=== modified file 'mysql-test/r/filesort_debug.result'
--- a/mysql-test/r/filesort_debug.result	2011-02-24 07:18:03 +0000
+++ b/mysql-test/r/filesort_debug.result	2011-11-07 15:32:36 +0000
@@ -16,7 +16,7 @@ DROP TABLE t1;
 #
 CREATE TABLE t1(f0 int auto_increment primary key, f1 int);
 INSERT INTO t1(f1) VALUES (0),(1),(2),(3),(4),(5);
-SET session debug= '+d,make_char_array_fail';
+SET session debug= '+d,alloc_sort_buffer_fail';
 CALL mtr.add_suppression("Out of sort memory");
 SELECT * FROM t1 ORDER BY f1 ASC, f0;
 ERROR HY001: Out of sort memory, consider increasing server sort buffer size

=== modified file 'mysql-test/suite/opt_trace/r/filesort_pq.result'
--- a/mysql-test/suite/opt_trace/r/filesort_pq.result	2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/filesort_pq.result	2011-11-07 15:32:36 +0000
@@ -269,6 +269,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
               "rows": 100,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 25080,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -446,6 +447,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 7068,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -658,6 +660,7 @@ SELECT * FROM t1 ORDER BY f2 DESC, f0 LI
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 13144,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -884,6 +887,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -1120,6 +1124,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -1291,6 +1296,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 11,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 4664,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -1429,6 +1435,7 @@ SELECT CONCAT("hello ", f2) AS foo FROM 
               "rows": 3,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 1254,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -1566,6 +1573,7 @@ SELECT * from t1 ORDER BY rand(2) LIMIT 
               "rows": 3,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 72,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -1744,6 +1752,7 @@ SELECT * FROM t1 ORDER BY f1 ASC, f0 LIM
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 7068,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -1956,6 +1965,7 @@ SELECT * FROM t1 ORDER BY f2 DESC, f0 LI
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 13144,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -2182,6 +2192,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -2418,6 +2429,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -2589,6 +2601,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 11,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 4664,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -2774,6 +2787,7 @@ ORDER BY f1, f0 LIMIT 30	{
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 7068,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -2926,6 +2940,7 @@ ORDER BY f1, f0 LIMIT 0	{
               "rows": 1,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 228,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -3122,6 +3137,7 @@ ORDER BY f2, f0 LIMIT 20	{
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -3298,6 +3314,7 @@ ORDER BY f2, f0 LIMIT 0	{
               "rows": 1,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 424,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -3484,6 +3501,7 @@ ORDER BY f2, f0 LIMIT 10 OFFSET 10	{
               "rows": 21,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 8904,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -3660,6 +3678,7 @@ ORDER BY f2, f0 LIMIT 0 OFFSET 10	{
               "rows": 11,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 4664,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -3942,6 +3961,7 @@ ORDER BY tmp.f1, f0 LIMIT 30	{
               "rows": 31,
               "examined_rows": 1500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 775,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -4216,6 +4236,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30	{
               "rows": 61,
               "examined_rows": 1500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 1525,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -4495,6 +4516,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30	{
               "rows": 61,
               "examined_rows": 1500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 1525,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -4802,6 +4824,7 @@ ORDER BY tmp.f1, f0 LIMIT 30 OFFSET 30	{
               "rows": 61,
               "examined_rows": 1185,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 1525,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -5109,6 +5132,7 @@ SELECT * FROM v1	{
                     "rows": 31,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 7068,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -5405,6 +5429,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
                     "rows": 101,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 23028,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -5439,6 +5464,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 6851,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -6039,6 +6065,7 @@ LIMIT 30	{
                     "rows": 101,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 23028,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -6077,6 +6104,7 @@ LIMIT 30	{
                     "rows": 101,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 42824,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -6117,6 +6145,7 @@ LIMIT 30	{
               "rows": 31,
               "examined_rows": 325,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 6975,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -6284,6 +6313,7 @@ GROUP BY 1 ORDER BY 2,1 LIMIT 5	{
               "rows": 6,
               "examined_rows": 10,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 198,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -6534,6 +6564,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 31,
               "examined_rows": 500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 13144,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -6718,6 +6749,7 @@ SELECT * FROM t1 WHERE f1>10 ORDER BY f2
               "rows": 31,
               "examined_rows": 500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 13144,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -6903,6 +6935,7 @@ ORDER BY f2, f0 LIMIT 15 OFFSET 15	{
               "rows": 31,
               "examined_rows": 500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 13144,
               "sort_mode": "<sort_key, additional_fields>"
             }
           }
@@ -7194,6 +7227,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
                     "rows": 101,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 23028,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -7228,6 +7262,7 @@ SELECT * FROM v1 ORDER BY f2, f0 LIMIT 3
               "rows": 31,
               "examined_rows": 100,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 6851,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -7591,6 +7626,7 @@ ORDER BY d1.f2 DESC LIMIT 30	{
                     "rows": 31,
                     "examined_rows": 500,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 6944,
                     "sort_mode": "<sort_key, additional_fields>"
                   }
                 }
@@ -7619,6 +7655,7 @@ ORDER BY d1.f2 DESC LIMIT 30	{
               "rows": 31,
               "examined_rows": 620,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 6727,
               "sort_mode": "<sort_key, rowid>"
             }
           }
@@ -7881,6 +7918,7 @@ SELECT * FROM t1 WHERE f1 = (SELECT f1 F
                           "rows": 2,
                           "examined_rows": 500,
                           "number_of_tmp_files": 0,
+                          "sort_buffer_size": 36,
                           "sort_mode": "<sort_key, additional_fields>"
                         }
                       }
@@ -11637,6 +11675,7 @@ SELECT * FROM t1 WHERE f1 = (SELECT f1 F
                           "rows": 3,
                           "examined_rows": 500,
                           "number_of_tmp_files": 0,
+                          "sort_buffer_size": 54,
                           "sort_mode": "<sort_key, additional_fields>"
                         }
                       }
@@ -11903,6 +11942,7 @@ SELECT * FROM t1 ORDER BY f2 LIMIT 100	{
               "rows": 101,
               "examined_rows": 438500,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 21715,
               "sort_mode": "<sort_key, rowid>"
             }
           }

=== modified file 'mysql-test/suite/opt_trace/r/general2_no_prot.result'
--- a/mysql-test/suite/opt_trace/r/general2_no_prot.result	2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general2_no_prot.result	2011-11-07 15:32:36 +0000
@@ -1119,6 +1119,7 @@ TRACE
               "rows": 4,
               "examined_rows": 4,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 252,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }
@@ -1354,6 +1355,7 @@ TRACE
                     "rows": 3,
                     "examined_rows": 3,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 273,
                     "sort_mode": "<sort_key, rowid>"
                   } /* filesort_summary */
                 }
@@ -4860,6 +4862,7 @@ GROUP BY field2 ORDER BY alias1.col_int_
               "rows": 8,
               "examined_rows": 8,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 378,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/suite/opt_trace/r/general2_ps_prot.result'
--- a/mysql-test/suite/opt_trace/r/general2_ps_prot.result	2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general2_ps_prot.result	2011-11-07 15:32:36 +0000
@@ -1137,6 +1137,7 @@ TRACE
               "rows": 4,
               "examined_rows": 4,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 252,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }
@@ -1372,6 +1373,7 @@ TRACE
                     "rows": 3,
                     "examined_rows": 3,
                     "number_of_tmp_files": 0,
+                    "sort_buffer_size": 273,
                     "sort_mode": "<sort_key, rowid>"
                   } /* filesort_summary */
                 }
@@ -4913,6 +4915,7 @@ GROUP BY field2 ORDER BY alias1.col_int_
               "rows": 8,
               "examined_rows": 8,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 378,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/suite/opt_trace/r/general_no_prot_none.result'
--- a/mysql-test/suite/opt_trace/r/general_no_prot_none.result	2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general_no_prot_none.result	2011-11-07 15:32:36 +0000
@@ -5835,6 +5835,7 @@ trace
               "rows": 2,
               "examined_rows": 2,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 252,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/suite/opt_trace/r/general_ps_prot_none.result'
--- a/mysql-test/suite/opt_trace/r/general_ps_prot_none.result	2011-10-13 13:22:45 +0000
+++ b/mysql-test/suite/opt_trace/r/general_ps_prot_none.result	2011-11-07 15:32:36 +0000
@@ -5793,6 +5793,7 @@ trace
               "rows": 2,
               "examined_rows": 2,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 252,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/suite/opt_trace/r/subquery_no_prot.result'
--- a/mysql-test/suite/opt_trace/r/subquery_no_prot.result	2011-10-27 08:52:27 +0000
+++ b/mysql-test/suite/opt_trace/r/subquery_no_prot.result	2011-11-07 15:32:36 +0000
@@ -1762,6 +1762,7 @@ field4,field5,field6	{
               "rows": 0,
               "examined_rows": 0,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 260,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/suite/opt_trace/r/subquery_ps_prot.result'
--- a/mysql-test/suite/opt_trace/r/subquery_ps_prot.result	2011-10-27 08:52:27 +0000
+++ b/mysql-test/suite/opt_trace/r/subquery_ps_prot.result	2011-11-07 15:32:36 +0000
@@ -1740,6 +1740,7 @@ field4,field5,field6	{
               "rows": 0,
               "examined_rows": 0,
               "number_of_tmp_files": 0,
+              "sort_buffer_size": 260,
               "sort_mode": "<sort_key, rowid>"
             } /* filesort_summary */
           }

=== modified file 'mysql-test/t/filesort_debug.test'
--- a/mysql-test/t/filesort_debug.test	2011-02-24 07:18:03 +0000
+++ b/mysql-test/t/filesort_debug.test	2011-11-07 15:32:36 +0000
@@ -28,7 +28,7 @@ DROP TABLE t1;
 CREATE TABLE t1(f0 int auto_increment primary key, f1 int);
 INSERT INTO t1(f1) VALUES (0),(1),(2),(3),(4),(5);
 
-SET session debug= '+d,make_char_array_fail';
+SET session debug= '+d,alloc_sort_buffer_fail';
 CALL mtr.add_suppression("Out of sort memory");
 --error ER_OUT_OF_SORTMEMORY
 SELECT * FROM t1 ORDER BY f1 ASC, f0;

=== modified file 'sql/filesort.cc'
--- a/sql/filesort.cc	2011-10-28 12:45:35 +0000
+++ b/sql/filesort.cc	2011-11-07 15:32:36 +0000
@@ -43,11 +43,11 @@ using std::min;
 
 	/* functions defined in this file */
 
-static void make_char_array(FILESORT_INFO *info, uint fields, uint length);
 static uchar *read_buffpek_from_file(IO_CACHE *buffer_file, uint count,
                                      uchar *buf);
 static ha_rows find_all_keys(Sort_param *param,SQL_SELECT *select,
-                             uchar **sort_keys, IO_CACHE *buffer_file,
+                             Filesort_info *fs_info,
+                             IO_CACHE *buffer_file,
                              IO_CACHE *tempfile,
                              Bounded_queue<uchar, uchar> *pq,
                              ha_rows *found_rows);
@@ -60,7 +60,7 @@ static int merge_index(Sort_param *param
                        uint maxbuffer,IO_CACHE *tempfile,
                        IO_CACHE *outfile);
 static bool save_index(Sort_param *param,uchar **sort_keys, uint count, 
-                       FILESORT_INFO *table_sort);
+                       Filesort_info *table_sort);
 static uint suffix_length(ulong string_length);
 static uint sortlength(THD *thd, SORT_FIELD *sortorder, uint s_length,
 		       bool *multi_byte_charset);
@@ -70,7 +70,7 @@ static SORT_ADDON_FIELD *get_addon_field
 static void unpack_addon_fields(struct st_sort_addon_field *addon_field,
                                 uchar *buff);
 static bool check_if_pq_applicable(Opt_trace_context *trace,
-                                   Sort_param *param, FILESORT_INFO *info,
+                                   Sort_param *param, Filesort_info *info,
                                    TABLE *table,
                                    ha_rows records, ulong memory_available);
 
@@ -177,7 +177,6 @@ ha_rows filesort(THD *thd, TABLE *table,
   uint maxbuffer;
   BUFFPEK *buffpek;
   ha_rows num_rows= HA_POS_ERROR;
-  uchar **sort_keys= NULL;
   IO_CACHE tempfile, buffpek_pointers, *outfile; 
   Sort_param param;
   bool multi_byte_charset;
@@ -212,7 +211,7 @@ ha_rows filesort(THD *thd, TABLE *table,
     QUICK_INDEX_MERGE_SELECT. Work with a copy and put it back at the end 
     when index_merge select has finished with it.
   */
-  FILESORT_INFO table_sort= table->sort;
+  Filesort_info table_sort= table->sort;
   table->sort.io_cache= NULL;
   DBUG_ASSERT(table_sort.record_pointers == NULL);
   
@@ -258,18 +257,19 @@ ha_rows filesort(THD *thd, TABLE *table,
                 true,                           // max_at_top
                 NULL,                           // compare_function
                 compare_length,
-                &make_sortkey, &param, table_sort.sort_keys))
+                &make_sortkey, &param, table_sort.get_sort_keys()))
     {
       /*
        If we fail to init pq, we have to give up:
        out of memory means my_malloc() will call my_error().
       */
       DBUG_PRINT("info", ("failed to allocate PQ"));
-      my_free(table_sort.sort_keys);
-      table_sort.sort_keys= NULL;
+      table_sort.free_sort_buffer();
       DBUG_ASSERT(thd->is_error());
       goto err;
     }
+    // For PQ queries (with limit) we initialize all pointers.
+    table_sort.init_record_pointers();
   }
   else
   {
@@ -280,9 +280,9 @@ ha_rows filesort(THD *thd, TABLE *table,
     while (memory_available >= min_sort_memory)
     {
       ha_rows keys= memory_available / (param.rec_length + sizeof(char*));
-      param.max_keys_per_buffer= static_cast<uint>(min(num_rows, keys));
-      make_char_array(&table_sort, param.max_keys_per_buffer, param.rec_length);
-      if (table_sort.sort_keys)
+      param.max_keys_per_buffer= (uint) min(num_rows, keys);
+      table_sort.alloc_sort_buffer(param.max_keys_per_buffer, param.rec_length);
+      if (table_sort.get_sort_keys())
         break;
       ulong old_memory_available= memory_available;
       memory_available= memory_available/4*3;
@@ -297,7 +297,6 @@ ha_rows filesort(THD *thd, TABLE *table,
     }
   }
 
-  sort_keys= table_sort.sort_keys;
   if (open_cached_file(&buffpek_pointers,mysql_tmpdir,TEMP_PREFIX,
 		       DISK_BUFFER_SIZE, MYF(MY_WME)))
     goto err;
@@ -307,7 +306,9 @@ ha_rows filesort(THD *thd, TABLE *table,
   // New scope, because subquery execution must be traced within an array.
   {
     Opt_trace_array ota(trace, "filesort_execution");
-    num_rows= find_all_keys(&param, select, sort_keys, &buffpek_pointers,
+    num_rows= find_all_keys(&param, select,
+                            &table_sort,
+                            &buffpek_pointers,
                             &tempfile, 
                             pq.is_initialized() ? &pq : NULL,
                             found_rows);
@@ -321,13 +322,15 @@ ha_rows filesort(THD *thd, TABLE *table,
     .add("rows", num_rows)
     .add("examined_rows", param.examined_rows)
     .add("number_of_tmp_files", maxbuffer)
+    .add("sort_buffer_size", table_sort.sort_buffer_size())
     .add_alnum("sort_mode",
                param.addon_field ?
                "<sort_key, additional_fields>" : "<sort_key, rowid>");
 
   if (maxbuffer == 0)			// The whole set is in memory
   {
-    if (save_index(&param, sort_keys, (uint) num_rows, &table_sort))
+    if (save_index(&param, table_sort.get_sort_keys(),
+                   (uint) num_rows, &table_sort))
       goto err;
   }
   else
@@ -363,13 +366,19 @@ ha_rows filesort(THD *thd, TABLE *table,
                                 (param.rec_length + sizeof(char*))) /
                                param.rec_length - 1);
     maxbuffer--;				// Offset from 0
-    if (merge_many_buff(&param,(uchar*) sort_keys,buffpek,&maxbuffer,
+    if (merge_many_buff(&param,
+                        (uchar*) table_sort.get_sort_keys(),
+                        buffpek,&maxbuffer,
 			&tempfile))
       goto err;
     if (flush_io_cache(&tempfile) ||
 	reinit_io_cache(&tempfile,READ_CACHE,0L,0,0))
       goto err;
-    if (merge_index(&param,(uchar*) sort_keys,buffpek,maxbuffer,&tempfile,
+    if (merge_index(&param,
+                    (uchar*) table_sort.get_sort_keys(),
+                    buffpek,
+                    maxbuffer,
+                    &tempfile,
 		    outfile))
       goto err;
   }
@@ -385,8 +394,7 @@ ha_rows filesort(THD *thd, TABLE *table,
   my_free(param.tmp_buffer);
   if (!subselect || !subselect->is_uncacheable())
   {
-    my_free(sort_keys);
-    table_sort.sort_keys= 0;
+    table_sort.free_sort_buffer();
     my_free(buffpek);
     table_sort.buffpek= 0;
     table_sort.buffpek_len= 0;
@@ -450,8 +458,7 @@ void filesort_free_buffers(TABLE *table,
 
   if (full)
   {
-    my_free(table->sort.sort_keys);
-    table->sort.sort_keys= NULL;
+    table->sort.free_sort_buffer();
     my_free(table->sort.buffpek);
     table->sort.buffpek= NULL;
     table->sort.buffpek_len= 0;
@@ -467,36 +474,11 @@ void filesort_free_buffers(TABLE *table,
 /**
   Makes an array of string pointers for info->sort_keys.
 
-  @param info         FILESORT_INFO struct owning the allocated array.
+  @param info         Filesort_info struct owning the allocated array.
   @param num_records  Number of records.
   @param length       Length of each record.
 */
 
-static void make_char_array(FILESORT_INFO *info, uint num_records, uint length)
-{
-  DBUG_ENTER("make_char_array");
-
-  DBUG_PRINT("info", ("num_records %u length %u", num_records, length));
-
-  DBUG_EXECUTE_IF("make_char_array_fail",
-                  DBUG_SET("+d,simulate_out_of_memory"););
-
-  if (!info->sort_keys)
-    info->sort_keys= 
-      (uchar**) my_malloc(num_records * (length + sizeof(uchar*)), MYF(0));
-
-  if (info->sort_keys)
-  {
-    uchar **pos= info->sort_keys;
-    uchar *char_pos= ((uchar*) (pos+num_records)) - length;
-    while (num_records--)
-      *(pos++)= (char_pos+= length);
-  }
-
-  DBUG_VOID_RETURN;
-}
-
-
 /** Read 'count' number of buffer pointers into memory. */
 
 static uchar *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count,
@@ -619,7 +601,7 @@ static void dbug_print_record(TABLE *tab
 */
 
 static ha_rows find_all_keys(Sort_param *param, SQL_SELECT *select,
-                             uchar **sort_keys,
+                             Filesort_info *fs_info,
                              IO_CACHE *buffpek_pointers,
                              IO_CACHE *tempfile,
                              Bounded_queue<uchar, uchar> *pq,
@@ -740,12 +722,13 @@ static ha_rows find_all_keys(Sort_param 
       {
         if (idx == param->max_keys_per_buffer)
         {
-          if (write_keys(param,sort_keys, idx, buffpek_pointers, tempfile))
+          if (write_keys(param, fs_info->get_sort_keys(),
+                         idx, buffpek_pointers, tempfile))
              DBUG_RETURN(HA_POS_ERROR);
           idx= 0;
           indexpos++;
         }
-        make_sortkey(param, sort_keys[idx++], ref_pos);
+        make_sortkey(param, fs_info->get_record_buffer(idx++), ref_pos);
       }
     }
     else
@@ -770,11 +753,12 @@ static ha_rows find_all_keys(Sort_param 
   DBUG_PRINT("test",("error: %d  indexpos: %d",error,indexpos));
   if (error != HA_ERR_END_OF_FILE)
   {
-    file->print_error(error,MYF(ME_ERROR | ME_WAITTANG)); /* purecov: inspected */
+    file->print_error(error,MYF(ME_ERROR | ME_WAITTANG)); // purecov: inspected
     DBUG_RETURN(HA_POS_ERROR);			/* purecov: inspected */
   }
   if (indexpos && idx &&
-      write_keys(param,sort_keys,idx,buffpek_pointers,tempfile))
+      write_keys(param, fs_info->get_sort_keys(),
+                 idx, buffpek_pointers, tempfile))
     DBUG_RETURN(HA_POS_ERROR);			/* purecov: inspected */
   const ha_rows retval= 
     my_b_inited(tempfile) ?
@@ -1151,7 +1135,7 @@ static void register_used_fields(Sort_pa
 
 
 static bool save_index(Sort_param *param, uchar **sort_keys, uint count, 
-                       FILESORT_INFO *table_sort)
+                       Filesort_info *table_sort)
 {
   uint offset,res_length;
   uchar *to;
@@ -1204,7 +1188,7 @@ static bool save_index(Sort_param *param
 
 bool check_if_pq_applicable(Opt_trace_context *trace,
                             Sort_param *param,
-                            FILESORT_INFO *filesort_info,
+                            Filesort_info *filesort_info,
                             TABLE *table, ha_rows num_rows,
                             ulong memory_available)
 {
@@ -1248,10 +1232,10 @@ bool check_if_pq_applicable(Opt_trace_co
     // The whole source set fits into memory.
     if (param->max_rows < num_rows/PQ_slowness )
     {
-      make_char_array(filesort_info,
-                      param->max_keys_per_buffer, param->rec_length);
+      filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+                                       param->rec_length);
       trace_filesort.add("chosen", true);
-      DBUG_RETURN(filesort_info->sort_keys != NULL);
+      DBUG_RETURN(filesort_info->get_sort_keys() != NULL);
     }
     else
     {
@@ -1265,10 +1249,10 @@ bool check_if_pq_applicable(Opt_trace_co
   // Do we have space for LIMIT rows in memory?
   if (param->max_keys_per_buffer < num_available_keys)
   {
-    make_char_array(filesort_info,
-                    param->max_keys_per_buffer, param->rec_length);
+    filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+                                     param->rec_length);
     trace_filesort.add("chosen", true);
-    DBUG_RETURN(filesort_info->sort_keys != NULL);
+    DBUG_RETURN(filesort_info->get_sort_keys() != NULL);
   }
 
   // Try to strip off addon fields.
@@ -1317,10 +1301,9 @@ bool check_if_pq_applicable(Opt_trace_co
       }
 
       trace_addon.add("chosen", true);
-      make_char_array(filesort_info,
-                      param->max_keys_per_buffer,
-                      param->sort_length + param->ref_length);
-      if (filesort_info->sort_keys)
+      filesort_info->alloc_sort_buffer(param->max_keys_per_buffer,
+                                       param->sort_length + param->ref_length);
+      if (filesort_info->get_sort_keys())
       {
         // Make attached data to be references instead of fields.
         my_free(filesort_info->addon_buf);

=== modified file 'sql/filesort_utils.cc'
--- a/sql/filesort_utils.cc	2011-06-30 15:50:45 +0000
+++ b/sql/filesort_utils.cc	2011-11-07 15:32:36 +0000
@@ -16,6 +16,7 @@
 #include "filesort_utils.h"
 #include "sql_const.h"
 #include "sql_sort.h"
+#include "table.h"
 
 
 namespace {
@@ -82,3 +83,36 @@ double get_merge_many_buffs_cost_fast(ha
 }
 
 
+uchar **Filesort_buffer::alloc_sort_buffer(uint num_records, uint record_length)
+{
+  DBUG_ENTER("alloc_sort_buffer");
+
+  DBUG_EXECUTE_IF("alloc_sort_buffer_fail",
+                  DBUG_SET("+d,simulate_out_of_memory"););
+
+  if (m_idx_array.is_null())
+  {
+    uchar **sort_keys=
+      (uchar**) my_malloc(num_records * (record_length + sizeof(uchar*)),
+                          MYF(0));
+    m_idx_array= Idx_array(sort_keys, num_records);
+    m_record_length= record_length;
+    uchar **start_of_data= m_idx_array.array() + m_idx_array.size();
+    m_start_of_data= reinterpret_cast<uchar*>(start_of_data);
+  }
+  else
+  {
+    DBUG_ASSERT(num_records == m_idx_array.size());
+    DBUG_ASSERT(record_length == m_record_length);
+  }
+  DBUG_RETURN(m_idx_array.array());
+}
+
+
+void Filesort_buffer::free_sort_buffer()
+{
+  my_free(m_idx_array.array());
+  m_idx_array= Idx_array();
+  m_record_length= 0;
+  m_start_of_data= NULL;
+}

=== modified file 'sql/filesort_utils.h'
--- a/sql/filesort_utils.h	2011-06-30 15:50:45 +0000
+++ b/sql/filesort_utils.h	2011-11-07 15:32:36 +0000
@@ -14,8 +14,11 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
 
 #ifndef FILESORT_UTILS_INCLUDED
+#define FILESORT_UTILS_INCLUDED
 
+#include "my_global.h"
 #include "my_base.h"
+#include "sql_array.h"
 
 /*
   Calculate cost of merge sort
@@ -36,9 +39,79 @@
     See also comments get_merge_many_buffs_cost().
 */
 
-#define FILESORT_UTILS_INCLUDED
 double get_merge_many_buffs_cost_fast(ha_rows num_rows,
                                       ha_rows num_keys_per_buffer,
                                       uint    elem_size);
 
+
+/**
+  A wrapper class around the buffer used by filesort().
+  The buffer is a contiguous chunk of memory,
+  where the first part is <num_records> pointers to the actual data.
+
+  We wrap the buffer in order to be able to do lazy initialization of the
+  pointers: the buffer is often much larger than what we actually need.
+
+  The buffer must be kept available for multiple executions of the
+  same sort operation, so we have explicit allocate and free functions,
+  rather than doing alloc/free in CTOR/DTOR.
+ */
+class Filesort_buffer
+{
+public:
+  Filesort_buffer() :
+    m_idx_array(), m_record_length(0), m_start_of_data(NULL)
+  {}
+
+  /// Initializes a record pointer.
+  uchar *get_record_buffer(uint idx)
+  {
+    m_idx_array[idx]= m_start_of_data + (idx * m_record_length);
+    return m_idx_array[idx];
+  }
+
+  /// Initializes all the record pointers.
+  void init_record_pointers()
+  {
+    for (uint ix= 0; ix < m_idx_array.size(); ++ix)
+      (void) get_record_buffer(ix);
+  }
+
+  /// Returns total size: pointer array + record buffers.
+  size_t sort_buffer_size() const
+  {
+    return m_idx_array.size() * (m_record_length + sizeof(uchar*));
+  }
+
+  /// Allocates the buffer, but does *not* initialize pointers.
+  uchar **alloc_sort_buffer(uint num_records, uint record_length);
+
+  /// Frees the buffer.
+  void free_sort_buffer();
+
+  /// Getter, for calling routines which still use the uchar** interface.
+  uchar **get_sort_keys() { return m_idx_array.array(); }
+
+  /**
+    We need an assignment operator, see filesort().
+    This happens to have the same semantics as the one that would be
+    generated by the compiler. We still implement it here, to show shallow
+    assignment explicitly: we have two objects sharing the same array.
+  */
+  Filesort_buffer &operator=(const Filesort_buffer &rhs)
+  {
+    m_idx_array= rhs.m_idx_array;
+    m_record_length= rhs.m_record_length;
+    m_start_of_data= rhs.m_start_of_data;
+    return *this;
+  }
+
+private:
+  typedef Bounds_checked_array<uchar*> Idx_array;
+
+  Idx_array  m_idx_array;
+  uint       m_record_length;
+  uchar     *m_start_of_data;
+};
+
 #endif  // FILESORT_UTILS_INCLUDED

=== modified file 'sql/table.h'
--- a/sql/table.h	2011-10-18 14:25:42 +0000
+++ b/sql/table.h	2011-11-07 15:32:36 +0000
@@ -28,6 +28,7 @@
 #include "handler.h"                /* row_type, ha_choice, handler */
 #include "mysql_com.h"              /* enum_field_types */
 #include "thr_lock.h"                  /* thr_lock_type */
+#include "filesort_utils.h"
 
 /* Structs that defines the TABLE */
 
@@ -300,10 +301,14 @@ enum tmp_table_type
 };
 enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP };
 
-typedef struct st_filesort_info
+
+class Filesort_info
 {
+  /// Buffer for sorting keys.
+  Filesort_buffer filesort_buffer;
+
+public:
   IO_CACHE *io_cache;           /* If sorted through filesort */
-  uchar     **sort_keys;        /* Buffer for sorting keys */
   uchar     *buffpek;           /* Buffer for buffpek structures */
   uint      buffpek_len;        /* Max number of buffpeks in the buffer */
   uchar     *addon_buf;         /* Pointer to a buffer if sorted with fields */
@@ -312,7 +317,28 @@ typedef struct st_filesort_info
   void    (*unpack)(struct st_sort_addon_field *, uchar *); /* To unpack back */
   uchar     *record_pointers;    /* If sorted in memory */
   ha_rows   found_records;      /* How many records in sort */
-} FILESORT_INFO;
+
+  /**
+     Accessors for Filesort_buffer (which @c).
+  */
+  uchar *get_record_buffer(uint idx)
+  { return filesort_buffer.get_record_buffer(idx); }
+
+  uchar **get_sort_keys()
+  { return filesort_buffer.get_sort_keys(); }
+
+  uchar **alloc_sort_buffer(uint num_records, uint record_length)
+  { return filesort_buffer.alloc_sort_buffer(num_records, record_length); }
+
+  void free_sort_buffer()
+  { filesort_buffer.free_sort_buffer(); }
+
+  void init_record_pointers()
+  { filesort_buffer.init_record_pointers(); }
+
+  size_t sort_buffer_size() const
+  { return filesort_buffer.sort_buffer_size(); }
+};
 
 
 /*
@@ -1134,7 +1160,7 @@ public:
   REGINFO reginfo;			/* field connections */
   MEM_ROOT mem_root;
   GRANT_INFO grant;
-  FILESORT_INFO sort;
+  Filesort_info sort;
 #ifdef WITH_PARTITION_STORAGE_ENGINE
   partition_info *part_info;            /* Partition related information */
   bool no_partitions_used; /* If true, all partitions have been pruned away */

=== modified file 'unittest/gunit/CMakeLists.txt'
--- a/unittest/gunit/CMakeLists.txt	2011-10-28 12:45:35 +0000
+++ b/unittest/gunit/CMakeLists.txt	2011-11-07 15:32:36 +0000
@@ -215,6 +215,7 @@ SET(TESTS
   dbug
   decimal
   dynarray
+  filesort_buffer
   mdl
   mdl_mytap
   my_bitmap

=== added file 'unittest/gunit/filesort_buffer-t.cc'
--- a/unittest/gunit/filesort_buffer-t.cc	1970-01-01 00:00:00 +0000
+++ b/unittest/gunit/filesort_buffer-t.cc	2011-11-07 15:32:36 +0000
@@ -0,0 +1,87 @@
+/* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA */
+
+// First include (the generated) my_config.h, to get correct platform defines,
+// then gtest.h (before any other MySQL headers), to avoid min() macros etc ...
+#include "my_config.h"
+#include <gtest/gtest.h>
+
+#include "filesort_utils.h"
+#include "table.h"
+
+
+namespace {
+
+class FileSortBufferTest : public ::testing::Test
+{
+protected:
+  virtual void TearDown()
+  {
+    fs_info.free_sort_buffer();
+  }
+
+  Filesort_info fs_info;
+};
+
+
+TEST_F(FileSortBufferTest, FileSortBuffer)
+{
+  const char letters[10]= "abcdefghi";
+  uchar **sort_keys= fs_info.alloc_sort_buffer(10, sizeof(char));
+  uchar **null_sort_keys= NULL;
+  EXPECT_NE(null_sort_keys, sort_keys);
+  for (uint ix= 0; ix < 10; ++ix)
+  {
+    uchar *ptr= fs_info.get_record_buffer(ix);
+    *ptr= letters[ix];
+  }
+  uchar *data= *fs_info.get_sort_keys();
+  const char *str= reinterpret_cast<const char*>(data);
+  EXPECT_STREQ(letters, str);
+
+  const size_t expected_size= 10 * (sizeof(char*) + sizeof(char));
+  EXPECT_EQ(expected_size, fs_info.sort_buffer_size());
+}
+
+
+TEST_F(FileSortBufferTest, InitRecordPointers)
+{
+  fs_info.alloc_sort_buffer(10, sizeof(char));
+  fs_info.init_record_pointers();
+  uchar **ptr= fs_info.get_sort_keys();
+  for (uint ix= 0; ix < 10 - 1; ++ix)
+  {
+    uchar **nxt= ptr + 1;
+    EXPECT_EQ(1, *nxt - *ptr);
+    ++ptr;
+  }
+}
+
+
+TEST_F(FileSortBufferTest, AssignmentOperator)
+{
+  fs_info.alloc_sort_buffer(10, sizeof(char));
+  Filesort_info fs_copy;
+  fs_copy= fs_info;
+  for (uint ix= 0; ix < 10 - 1; ++ix)
+  {
+    EXPECT_EQ(fs_copy.get_record_buffer(ix), fs_info.get_record_buffer(ix));
+  }
+  EXPECT_EQ(fs_copy.get_sort_keys(), fs_info.get_sort_keys());
+  EXPECT_EQ(fs_copy.sort_buffer_size(), fs_info.sort_buffer_size());
+}
+
+
+}  // namespace

No bundle (reason: useless for push emails).
Thread
bzr push into mysql-trunk branch (tor.didriksen:3568 to 3569) Bug#11748783Tor Didriksen7 Nov