MySQL Lists are EOL. Please join:

List:Commits« Previous MessageNext Message »
From:Satya B Date:November 12 2009 11:31am
Subject:bzr commit into mysql-5.0-bugteam branch (satya.bn:2845) Bug#37408
View as plain text  
#At file:///home/satya/WORK/37408/mysql-5.0-bugteam-37408/ based on revid:joro@stripped

 2845 Satya B	2009-11-12
      Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
                        
      When compressed myisam files are opened, they are always memory mapped
      sometimes causing memory swapping problems.
      
      When we mmap the myisam compressed tables of size greater than the memory 
      available, the kswapd0 process utilization is very high consuming 30-40% of 
      the cpu. This happens only with linux kernels olders than 2.6.9
      
      With newer linux kernels, we don't have this problem of high cpu consumption
      and this option may not be required.
       
      The option 'myisam_mmap_size' is added to limit the amount of memory used for
      memory mapping of myisam files. This option is not dynamic.
      
      The default value on 32 bit system is 4294967295 bytes and on 64 bit system it
      is 18446744073709547520 bytes.
                      
      Also for mmaping, we use system's mmap call directly instead of using mysql's 
      wrapper function my_mmap() which made the implementation of the option 
      'mmap_memory_size' difficult and so myisam level option is introduced.
      
      Note: Testcase is not added and has to be tested manually
     @ include/my_global.h
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        define SIZE_T_MAX
     @ include/myisam.h
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        declare 'myisam_mmap_size' and 'myisam_mmap_used' variables
     @ myisam/mi_packrec.c
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        add 'myisam_mmap_size' option which limits the memory available to mmap of 
        myisam files
     @ myisam/mi_static.c
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        declare 'myisam_mmap_size' and 'myisam_mmap_used' variables
     @ sql/mysqld.cc
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        add the 'myisam_mmap_size' option
     @ sql/set_var.cc
        Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
        
        add the 'myisam_mmap_size' to the SHOW VARIABLES list

    modified:
      include/my_global.h
      include/myisam.h
      myisam/mi_packrec.c
      myisam/mi_static.c
      sql/mysqld.cc
      sql/set_var.cc
=== modified file 'include/my_global.h'
--- a/include/my_global.h	2009-09-17 15:10:30 +0000
+++ b/include/my_global.h	2009-11-12 11:31:53 +0000
@@ -793,6 +793,9 @@ typedef SOCKET_SIZE_TYPE size_socket;
 #define DBL_MAX		1.79769313486231470e+308
 #define FLT_MAX		((float)3.40282346638528860e+38)
 #endif
+#ifndef SIZE_T_MAX
+#define SIZE_T_MAX      ~((size_t) 0)
+#endif
 
 #ifndef HAVE_FINITE
 #define finite(x) (1.0 / fabs(x) > 0.0)

=== modified file 'include/myisam.h'
--- a/include/myisam.h	2009-10-27 13:11:06 +0000
+++ b/include/myisam.h	2009-11-12 11:31:53 +0000
@@ -270,6 +270,7 @@ extern ulong myisam_bulk_insert_tree_siz
 /* usually used to check if a symlink points into the mysql data home */
 /* which is normally forbidden                                        */
 extern int (*myisam_test_invalid_symlink)(const char *filename);
+extern ulonglong myisam_mmap_size, myisam_mmap_used;
 
 	/* Prototypes for myisam-functions */
 

=== modified file 'myisam/mi_packrec.c'
--- a/myisam/mi_packrec.c	2009-08-28 15:51:31 +0000
+++ b/myisam/mi_packrec.c	2009-11-12 11:31:53 +0000
@@ -1504,7 +1504,8 @@ my_bool _mi_memmap_file(MI_INFO *info)
   if (!share->file_map)
   {
     my_off_t data_file_length= share->state.state.data_file_length;
-    if (data_file_length > (my_off_t) (~((size_t) 0)) - MEMMAP_EXTRA_MARGIN)
+    if ((data_file_length + myisam_mmap_used + MEMMAP_EXTRA_MARGIN) > 
+         myisam_mmap_size)
     {
       DBUG_PRINT("warning", ("File is too large for mmap"));
       DBUG_RETURN(0);
@@ -1525,6 +1526,7 @@ my_bool _mi_memmap_file(MI_INFO *info)
       DBUG_RETURN(0);
     }
     share->file_map= file_map;
+    myisam_mmap_used+= (data_file_length + MEMMAP_EXTRA_MARGIN);
   }
   info->opt_flag|= MEMMAP_USED;
   info->read_record=share->read_record=_mi_read_mempack_record;
@@ -1538,6 +1540,8 @@ void _mi_unmap_file(MI_INFO *info)
   VOID(my_munmap(info->s->file_map,
 	      (size_t) info->s->state.state.data_file_length+
 	      MEMMAP_EXTRA_MARGIN));
+  myisam_mmap_used-= info->s->state.state.data_file_length +
+                          MEMMAP_EXTRA_MARGIN;
 }
 
 

=== modified file 'myisam/mi_static.c'
--- a/myisam/mi_static.c	2009-10-27 13:11:06 +0000
+++ b/myisam/mi_static.c	2009-11-12 11:31:53 +0000
@@ -40,6 +40,7 @@ ulong myisam_concurrent_insert= 0;
 my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
 ulong    myisam_bulk_insert_tree_size=8192*1024;
 ulong    myisam_data_pointer_size=4;
+ulonglong    myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
 
 
 static int always_valid(const char *filename __attribute__((unused)))

=== modified file 'sql/mysqld.cc'
--- a/sql/mysqld.cc	2009-10-27 13:11:06 +0000
+++ b/sql/mysqld.cc	2009-11-12 11:31:53 +0000
@@ -4975,7 +4975,8 @@ enum options_mysqld
   OPT_MAX_WRITE_LOCK_COUNT, OPT_BULK_INSERT_BUFFER_SIZE,
   OPT_MAX_ERROR_COUNT, OPT_MULTI_RANGE_COUNT, OPT_MYISAM_DATA_POINTER_SIZE,
   OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
-  OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
+  OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_MMAP_SIZE,
+  OPT_MYISAM_SORT_BUFFER_SIZE,
   OPT_MYISAM_STATS_METHOD,
   OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
   OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
@@ -6254,6 +6255,10 @@ The minimum value for this variable is 4
    (gptr*) &max_system_variables.myisam_max_sort_file_size, 0,
    GET_ULL, REQUIRED_ARG, (longlong) LONG_MAX, 0, (ulonglong) MAX_FILE_SIZE,
    0, 1024*1024, 0},
+  {"myisam_mmap_size", OPT_MYISAM_MMAP_SIZE,
+   "Can be used to restrict the total memory used for memory mmaping of myisam files",
+   (gptr*) &myisam_mmap_size, (gptr*) &myisam_mmap_size, 0,
+   GET_ULL, REQUIRED_ARG, SIZE_T_MAX, 0, SIZE_T_MAX, 0, 1, 0},
   {"myisam_repair_threads", OPT_MYISAM_REPAIR_THREADS,
    "Number of threads to use when repairing MyISAM tables. The value of 1 disables parallel repair.",
    (gptr*) &global_system_variables.myisam_repair_threads,

=== modified file 'sql/set_var.cc'
--- a/sql/set_var.cc	2009-07-16 12:37:38 +0000
+++ b/sql/set_var.cc	2009-11-12 11:31:53 +0000
@@ -288,6 +288,8 @@ sys_var_thd_ulong       sys_multi_range_
 sys_var_long_ptr	sys_myisam_data_pointer_size("myisam_data_pointer_size",
                                                     &myisam_data_pointer_size);
 sys_var_thd_ulonglong	sys_myisam_max_sort_file_size("myisam_max_sort_file_size", &SV::myisam_max_sort_file_size, fix_myisam_max_sort_file_size, 1);
+sys_var_ulonglong_ptr   sys_myisam_mmap_size("myisam_mmap_size",
+                                             &myisam_mmap_size);
 sys_var_thd_ulong       sys_myisam_repair_threads("myisam_repair_threads", &SV::myisam_repair_threads);
 sys_var_thd_ulong	sys_myisam_sort_buffer_size("myisam_sort_buffer_size", &SV::myisam_sort_buff_size);
 
@@ -1026,6 +1028,7 @@ struct show_var_st init_vars[]= {
   {sys_myisam_data_pointer_size.name, (char*) &sys_myisam_data_pointer_size, SHOW_SYS},
   {sys_myisam_max_sort_file_size.name, (char*) &sys_myisam_max_sort_file_size,
    SHOW_SYS},
+  {sys_myisam_mmap_size.name, (char*) &sys_myisam_mmap_size, SHOW_SYS},
   {"myisam_recover_options",  (char*) &myisam_recover_options_str,  SHOW_CHAR_PTR},
   {sys_myisam_repair_threads.name, (char*) &sys_myisam_repair_threads,
    SHOW_SYS},


Attachment: [text/bzr-bundle] bzr/satya.bn@sun.com-20091112113153-f9vaxv2gk97hybi6.bundle
Thread
bzr commit into mysql-5.0-bugteam branch (satya.bn:2845) Bug#37408Satya B12 Nov
  • Re: bzr commit into mysql-5.0-bugteam branch (satya.bn:2845)Bug#37408Sergei Golubchik19 Nov