#At file:///home/satya/WORK/37408/mysql-5.0-bugteam-37408/ based on revid:epotemkin@stripped
2859 Satya B 2009-12-09
Fix for Bug#37408 - Compressed MyISAM files should not require/use mmap()
When compressed myisam files are opened, they are always memory mapped
sometimes causing memory swapping problems.
When we mmap the myisam compressed tables of size greater than the memory
available, the kswapd0 process utilization is very high consuming 30-40% of
the cpu. This happens only with linux kernels older than 2.6.9
With newer linux kernels, we don't have this problem of high cpu consumption
and this option may not be required.
The option 'myisam_mmap_size' is added to limit the amount of memory used for
memory mapping of myisam files. This option is not dynamic.
The default value on 32 bit system is 4294967295 bytes and on 64 bit system it
is 18446744073709547520 bytes.
Note: Testcase only tests the option variable. The actual bug has be to
tested manually.
@ include/my_global.h
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
define SIZE_T_MAX
@ include/myisam.h
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex
THR_LOCK_myisam_mmap
@ myisam/mi_packrec.c
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
add 'myisam_mmap_size' option which limits the memory available to mmap of
myisam files
@ myisam/mi_static.c
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
declare 'myisam_mmap_size' and 'myisam_mmap_used' variables and the mutex
THR_LOCK_myisam_mmap
@ mysql-test/r/variables.result
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
Testcase for BUG#37408 to test the myisam_mmap_size option
@ mysql-test/t/variables.test
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
Testcase for BUG#37408 to test the myisam_mmap_size option
@ mysys/my_thr_init.c
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
intialize the lock THR_LOCK_myisam_mmap
@ sql/mysqld.cc
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
add the 'myisam_mmap_size' option
@ sql/set_var.cc
Fix for Bug #37408 - Compressed MyISAM files should not require/use mmap()
add the 'myisam_mmap_size' to the SHOW VARIABLES list
modified:
include/my_global.h
include/myisam.h
myisam/mi_packrec.c
myisam/mi_static.c
mysql-test/r/variables.result
mysql-test/t/variables.test
mysys/my_thr_init.c
sql/mysqld.cc
sql/set_var.cc
=== modified file 'include/my_global.h'
--- a/include/my_global.h 2009-09-17 15:10:30 +0000
+++ b/include/my_global.h 2009-12-09 12:35:26 +0000
@@ -793,6 +793,9 @@ typedef SOCKET_SIZE_TYPE size_socket;
#define DBL_MAX 1.79769313486231470e+308
#define FLT_MAX ((float)3.40282346638528860e+38)
#endif
+#ifndef SIZE_T_MAX
+#define SIZE_T_MAX (~((size_t) 0))
+#endif
#ifndef HAVE_FINITE
#define finite(x) (1.0 / fabs(x) > 0.0)
=== modified file 'include/myisam.h'
--- a/include/myisam.h 2009-10-27 13:11:06 +0000
+++ b/include/myisam.h 2009-12-09 12:35:26 +0000
@@ -270,6 +270,8 @@ extern ulong myisam_bulk_insert_tree_siz
/* usually used to check if a symlink points into the mysql data home */
/* which is normally forbidden */
extern int (*myisam_test_invalid_symlink)(const char *filename);
+extern ulonglong myisam_mmap_size, myisam_mmap_used;
+extern pthread_mutex_t THR_LOCK_myisam_mmap;
/* Prototypes for myisam-functions */
=== modified file 'myisam/mi_packrec.c'
--- a/myisam/mi_packrec.c 2009-08-28 15:51:31 +0000
+++ b/myisam/mi_packrec.c 2009-12-09 12:35:26 +0000
@@ -1500,36 +1500,45 @@ my_bool _mi_memmap_file(MI_INFO *info)
byte *file_map;
MYISAM_SHARE *share=info->s;
DBUG_ENTER("mi_memmap_file");
+ bool mmap_status= FALSE;
if (!share->file_map)
{
my_off_t data_file_length= share->state.state.data_file_length;
- if (data_file_length > (my_off_t) (~((size_t) 0)) - MEMMAP_EXTRA_MARGIN)
- {
- DBUG_PRINT("warning", ("File is too large for mmap"));
- DBUG_RETURN(0);
- }
+
if (my_seek(info->dfile,0L,MY_SEEK_END,MYF(0)) <
data_file_length + MEMMAP_EXTRA_MARGIN)
{
DBUG_PRINT("warning",("File isn't extended for memmap"));
DBUG_RETURN(0);
}
- file_map=(byte*)
- my_mmap(0, (size_t) (data_file_length + MEMMAP_EXTRA_MARGIN), PROT_READ,
- MAP_SHARED | MAP_NORESERVE, info->dfile, 0L);
- if (file_map == (byte*) MAP_FAILED)
+
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ if ((data_file_length + myisam_mmap_used + MEMMAP_EXTRA_MARGIN) >
+ myisam_mmap_size)
{
- DBUG_PRINT("warning",("mmap failed: errno: %d",errno));
- my_errno=errno;
- DBUG_RETURN(0);
+ DBUG_PRINT("warning", ("File is too large for mmap"));
}
- share->file_map= file_map;
+ else
+ {
+ file_map=(byte*)
+ my_mmap(0, (size_t) (data_file_length + MEMMAP_EXTRA_MARGIN), PROT_READ,
+ MAP_SHARED | MAP_NORESERVE, info->dfile, 0L);
+ if (file_map == (byte*) MAP_FAILED)
+ {
+ DBUG_PRINT("warning",("mmap failed: errno: %d",errno));
+ my_errno= errno;
+ }
+ share->file_map= file_map;
+ myisam_mmap_used+= (data_file_length + MEMMAP_EXTRA_MARGIN);
+ info->opt_flag|= MEMMAP_USED;
+ info->read_record= share->read_record= _mi_read_mempack_record;
+ share->read_rnd= _mi_read_rnd_mempack_record;
+ mmap_status= TRUE;
+ }
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
}
- info->opt_flag|= MEMMAP_USED;
- info->read_record=share->read_record=_mi_read_mempack_record;
- share->read_rnd=_mi_read_rnd_mempack_record;
- DBUG_RETURN(1);
+ DBUG_RETURN(mmap_status);
}
@@ -1538,6 +1547,11 @@ void _mi_unmap_file(MI_INFO *info)
VOID(my_munmap(info->s->file_map,
(size_t) info->s->state.state.data_file_length+
MEMMAP_EXTRA_MARGIN));
+
+ pthread_mutex_lock(&THR_LOCK_myisam_mmap);
+ myisam_mmap_used-= info->s->state.state.data_file_length +
+ MEMMAP_EXTRA_MARGIN;
+ pthread_mutex_unlock(&THR_LOCK_myisam_mmap);
}
=== modified file 'myisam/mi_static.c'
--- a/myisam/mi_static.c 2009-10-27 13:11:06 +0000
+++ b/myisam/mi_static.c 2009-12-09 12:35:26 +0000
@@ -40,7 +40,8 @@ ulong myisam_concurrent_insert= 0;
my_off_t myisam_max_temp_length= MAX_FILE_SIZE;
ulong myisam_bulk_insert_tree_size=8192*1024;
ulong myisam_data_pointer_size=4;
-
+ulonglong myisam_mmap_size= SIZE_T_MAX, myisam_mmap_used= 0;
+pthread_mutex_t THR_LOCK_myisam_mmap;
static int always_valid(const char *filename __attribute__((unused)))
{
=== modified file 'mysql-test/r/variables.result'
--- a/mysql-test/r/variables.result 2009-06-19 09:27:19 +0000
+++ b/mysql-test/r/variables.result 2009-12-09 12:35:26 +0000
@@ -904,3 +904,9 @@ set global server_id =@m
set global slow_launch_time =@my_slow_launch_time;
set global storage_engine =@my_storage_engine;
set global thread_cache_size =@my_thread_cache_size;
+#
+# BUG#37408 - Compressed MyISAM files should not require/use mmap()
+#
+# Test 'myisam_mmap_size' option is not dynamic
+SET @@myisam_mmap_size= 500M;
+ERROR HY000: Variable 'myisam_mmap_size' is a read only variable
=== modified file 'mysql-test/t/variables.test'
--- a/mysql-test/t/variables.test 2009-06-19 09:27:19 +0000
+++ b/mysql-test/t/variables.test 2009-12-09 12:35:26 +0000
@@ -765,3 +765,9 @@ set global slow_launch_time =@m
set global storage_engine =@my_storage_engine;
set global thread_cache_size =@my_thread_cache_size;
+--echo #
+--echo # BUG#37408 - Compressed MyISAM files should not require/use mmap()
+--echo #
+--echo # Test 'myisam_mmap_size' option is not dynamic
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SET @@myisam_mmap_size= 500M;
=== modified file 'mysys/my_thr_init.c'
--- a/mysys/my_thr_init.c 2009-09-24 06:30:31 +0000
+++ b/mysys/my_thr_init.c 2009-12-09 12:35:26 +0000
@@ -30,7 +30,9 @@ pthread_key(struct st_my_thread_var, THR
#endif /* USE_TLS */
pthread_mutex_t THR_LOCK_malloc,THR_LOCK_open,
THR_LOCK_lock,THR_LOCK_isam,THR_LOCK_myisam,THR_LOCK_heap,
- THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads;
+ THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads,
+ THR_LOCK_myisam_mmap;
+
pthread_cond_t THR_COND_threads;
uint THR_thread_count= 0;
uint my_thread_end_wait_time= 5;
@@ -143,6 +145,7 @@ my_bool my_thread_global_init(void)
pthread_mutex_init(&THR_LOCK_lock,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_isam,MY_MUTEX_INIT_SLOW);
pthread_mutex_init(&THR_LOCK_myisam,MY_MUTEX_INIT_SLOW);
+ pthread_mutex_init(&THR_LOCK_myisam_mmap,MY_MUTEX_INIT_SLOW);
pthread_mutex_init(&THR_LOCK_heap,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_net,MY_MUTEX_INIT_FAST);
pthread_mutex_init(&THR_LOCK_charset,MY_MUTEX_INIT_FAST);
@@ -208,6 +211,7 @@ void my_thread_global_end(void)
pthread_mutex_destroy(&THR_LOCK_lock);
pthread_mutex_destroy(&THR_LOCK_isam);
pthread_mutex_destroy(&THR_LOCK_myisam);
+ pthread_mutex_destroy(&THR_LOCK_myisam_mmap);
pthread_mutex_destroy(&THR_LOCK_heap);
pthread_mutex_destroy(&THR_LOCK_net);
pthread_mutex_destroy(&THR_LOCK_charset);
=== modified file 'sql/mysqld.cc'
--- a/sql/mysqld.cc 2009-10-27 13:11:06 +0000
+++ b/sql/mysqld.cc 2009-12-09 12:35:26 +0000
@@ -4975,7 +4975,8 @@ enum options_mysqld
OPT_MAX_WRITE_LOCK_COUNT, OPT_BULK_INSERT_BUFFER_SIZE,
OPT_MAX_ERROR_COUNT, OPT_MULTI_RANGE_COUNT, OPT_MYISAM_DATA_POINTER_SIZE,
OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
- OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
+ OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_MMAP_SIZE,
+ OPT_MYISAM_SORT_BUFFER_SIZE,
OPT_MYISAM_STATS_METHOD,
OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
@@ -6254,6 +6255,10 @@ The minimum value for this variable is 4
(gptr*) &max_system_variables.myisam_max_sort_file_size, 0,
GET_ULL, REQUIRED_ARG, (longlong) LONG_MAX, 0, (ulonglong) MAX_FILE_SIZE,
0, 1024*1024, 0},
+ {"myisam_mmap_size", OPT_MYISAM_MMAP_SIZE,
+ "Can be used to restrict the total memory used for memory mmaping of myisam files",
+ (gptr*) &myisam_mmap_size, (gptr*) &myisam_mmap_size, 0,
+ GET_ULL, REQUIRED_ARG, SIZE_T_MAX, 0, SIZE_T_MAX, 0, 1, 0},
{"myisam_repair_threads", OPT_MYISAM_REPAIR_THREADS,
"Number of threads to use when repairing MyISAM tables. The value of 1 disables parallel repair.",
(gptr*) &global_system_variables.myisam_repair_threads,
=== modified file 'sql/set_var.cc'
--- a/sql/set_var.cc 2009-07-16 12:37:38 +0000
+++ b/sql/set_var.cc 2009-12-09 12:35:26 +0000
@@ -122,6 +122,7 @@ static byte *get_error_count(THD *thd);
static byte *get_warning_count(THD *thd);
static byte *get_have_innodb(THD *thd);
static byte *get_tmpdir(THD *thd);
+static byte *get_myisam_mmap_size(THD *thd);
/*
Variable definition list
@@ -623,6 +624,10 @@ sys_var_thd_bool sys_keep_files_on_crea
&SV::keep_files_on_create);
+static sys_var_readonly sys_myisam_mmap_size("myisam_mmap_size",
+ OPT_GLOBAL,
+ SHOW_LONGLONG,
+ get_myisam_mmap_size);
/*
@@ -723,6 +728,7 @@ sys_var *sys_variables[]=
&sys_multi_range_count,
&sys_myisam_data_pointer_size,
&sys_myisam_max_sort_file_size,
+ &sys_myisam_mmap_size,
&sys_myisam_repair_threads,
&sys_myisam_sort_buffer_size,
&sys_myisam_stats_method,
@@ -1026,6 +1032,7 @@ struct show_var_st init_vars[]= {
{sys_myisam_data_pointer_size.name, (char*) &sys_myisam_data_pointer_size, SHOW_SYS},
{sys_myisam_max_sort_file_size.name, (char*) &sys_myisam_max_sort_file_size,
SHOW_SYS},
+ {sys_myisam_mmap_size.name, (char*) &sys_myisam_mmap_size, SHOW_LONGLONG},
{"myisam_recover_options", (char*) &myisam_recover_options_str, SHOW_CHAR_PTR},
{sys_myisam_repair_threads.name, (char*) &sys_myisam_repair_threads,
SHOW_SYS},
@@ -3181,6 +3188,11 @@ static byte *get_tmpdir(THD *thd)
return (byte*)mysql_tmpdir;
}
+static byte *get_myisam_mmap_size(THD *thd)
+{
+ return (byte *)myisam_mmap_size;
+}
+
/****************************************************************************
Main handling of variables:
- Initialisation
Attachment: [text/bzr-bundle] bzr/satya.bn@sun.com-20091209123526-wpsfa4lvfm2zy2go.bundle
| Thread |
|---|
| • bzr commit into mysql-5.0-bugteam branch (satya.bn:2859) Bug#37408 | Satya B | 9 Dec |
