bk commit into 5.0 tree (vtkachenko:1.1874)

[Vadim Tkachenko]

Below is the list of changes that have just been committed into a local
5.0 repository of vtkachenko. When vtkachenko does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.1874 05/05/27 10:14:20 vtkachenko@stripped +13 -0
  Many files:
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  sql/set_var.cc
    1.113 05/05/27 10:10:53 vtkachenko@stripped +3 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  sql/mysqld.cc
    1.458 05/05/27 10:10:53 vtkachenko@stripped +7 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  sql/mysql_priv.h
    1.298 05/05/27 10:10:52 vtkachenko@stripped +1 -1
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  sql/ha_myisam.cc
    1.149 05/05/27 10:10:52 vtkachenko@stripped +4 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/myisamdef.h
    1.77 05/05/27 10:10:52 vtkachenko@stripped +1 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/mi_statrec.c
    1.12 05/05/27 10:10:52 vtkachenko@stripped +9 -9
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/mi_packrec.c
    1.31 05/05/27 10:10:51 vtkachenko@stripped +2 -2
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/mi_extra.c
    1.45 05/05/27 10:10:51 vtkachenko@stripped +22 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/mi_dynrec.c
    1.40 05/05/27 10:10:51 vtkachenko@stripped +40 -7
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  myisam/mi_close.c
    1.19 05/05/27 10:10:51 vtkachenko@stripped +7 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  include/my_sys.h
    1.156 05/05/27 10:10:51 vtkachenko@stripped +5 -0
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  include/my_base.h
    1.70 05/05/27 10:10:51 vtkachenko@stripped +2 -1
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

  configure.in
    1.312 05/05/27 10:10:50 vtkachenko@stripped +1 -1
    WL#2596 MMAP for read/write MyISAM tables, with Monty's commnets

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	vtkachenko
# Host:	quadxeon.mysql.com
# Root:	/users/vtkachenko/bk/HP/mysql-5.0-hp

--- 1.311/configure.in	2005-05-20 08:33:57 +02:00
+++ 1.312/configure.in	2005-05-27 10:10:50 +02:00
@@ -1920,7 +1920,7 @@
   getcwd gethostbyaddr_r gethostbyname_r getpass getpassphrase getpwnam \
   getpwuid getrlimit getrusage getwd gmtime_r index initgroups isnan \
   localtime_r locking longjmp lrand48 madvise mallinfo memcpy memmove \
-  mkstemp mlockall perror poll pread pthread_attr_create mmap getpagesize \
+  mkstemp mlockall perror poll pread pthread_attr_create mmap mmap64 getpagesize \
   pthread_attr_getstacksize pthread_attr_setprio pthread_attr_setschedparam \
   pthread_attr_setstacksize pthread_condattr_create pthread_getsequence_np \
   pthread_key_delete pthread_rwlock_rdlock pthread_setprio \

--- 1.69/include/my_base.h	2005-04-07 20:17:33 +02:00
+++ 1.70/include/my_base.h	2005-05-27 10:10:51 +02:00
@@ -152,7 +152,8 @@
     other fields intact. When this is off (by default) InnoDB will use memcpy
     to overwrite entire row.
   */
-  HA_EXTRA_KEYREAD_PRESERVE_FIELDS
+  HA_EXTRA_KEYREAD_PRESERVE_FIELDS,
+  HA_EXTRA_MMAP
 };
 
 	/* The following is parameter to ha_panic() */

--- 1.155/include/my_sys.h	2005-05-20 08:33:58 +02:00
+++ 1.156/include/my_sys.h	2005-05-27 10:10:51 +02:00
@@ -378,6 +378,7 @@
   /* the non-inclusive boundary in the buffer for the currently valid read */
   byte  *read_end;
   byte  *buffer;				/* The read buffer */
+  void  *mmaparea;   /* mmap address */
   /* Used in ASYNC_IO */
   byte  *request_pos;
 
@@ -815,6 +816,10 @@
 #endif
 
 #define my_mmap(a,b,c,d,e,f)    mmap(a,b,c,d,e,f)
+#ifdef HAVE_MMAP64
+#undef my_mmap
+#define my_mmap(a,b,c,d,e,f)    mmap64(a,b,c,d,e,f)
+#endif
 #ifdef HAVE_GETPAGESIZE
 #define my_getpagesize()        getpagesize()
 #else

--- 1.18/myisam/mi_close.c	2004-10-06 14:09:38 +02:00
+++ 1.19/myisam/mi_close.c	2005-05-27 10:10:51 +02:00
@@ -81,6 +81,13 @@
       if (my_close(share->kfile,MYF(0)))
         error = my_errno;
     }
+
+    if (share->file_map)
+    {
+      my_munmap(share->file_map, share->mmaped_length);
+      share->file_map= NULL;
+    }
+
 #ifdef HAVE_MMAP
     if (share->file_map)
       _mi_unmap_file(info);

--- 1.39/myisam/mi_dynrec.c	2005-05-13 11:08:02 +02:00
+++ 1.40/myisam/mi_dynrec.c	2005-05-27 10:10:51 +02:00
@@ -50,6 +50,39 @@
 
 	/* Interface function from MI_INFO */
 
+uint _mi_read(MI_INFO *info, byte *Buffer, uint Count, my_off_t offset, myf MyFlags)
+{
+  if ((info->s->file_map) && 
+      (min(info->s->mmaped_length, info->state->data_file_length) > offset
+ Count))
+  {
+    DBUG_PRINT("info", ("_mi_read with mmap %d\n", info->dfile));
+    memcpy(Buffer, info->s->file_map + offset, Count); 
+    return 0;
+  }
+  else
+  {
+    return my_pread(info->dfile, Buffer, Count, offset, MyFlags);
+  }
+
+}
+
+uint _mi_write(MI_INFO *info, byte *Buffer, uint Count, my_off_t offset, myf MyFlags)
+{
+  if ((info->s->file_map) && 
+      (min(info->s->mmaped_length, info->state->data_file_length) > offset
+ Count))
+  {
+    DBUG_PRINT("info", ("_mi_write with mmap %d\n", info->dfile));
+    memcpy(info->s->file_map + offset, Buffer, Count); 
+    return 0;
+  }
+  else
+  {
+    return my_pwrite(info->dfile, Buffer, Count, offset, MyFlags);
+  }
+
+}
+
+
 int _mi_write_dynamic_record(MI_INFO *info, const byte *record)
 {
   ulong reclength=_mi_rec_pack(info,info->rec_buff,record);
@@ -243,7 +276,7 @@
 	  & BLOCK_DELETED))
       DBUG_RETURN(1);				/* Something is wrong */
     mi_sizestore(tmp.header+4,block_info->next_filepos);
-    if (my_pwrite(info->dfile,(char*) tmp.header+4,8,
+    if (_mi_write(info,(char*) tmp.header+4,8,
 		  block_info->prev_filepos+4, MYF(MY_NABP)))
       DBUG_RETURN(1);
     /* Unlink block from next block */
@@ -253,7 +286,7 @@
 	    & BLOCK_DELETED))
 	DBUG_RETURN(1);				/* Something is wrong */
       mi_sizestore(tmp.header+12,block_info->prev_filepos);
-      if (my_pwrite(info->dfile,(char*) tmp.header+12,8,
+      if (_mi_write(info,(char*) tmp.header+12,8,
 		    block_info->next_filepos+12,
 		    MYF(MY_NABP)))
 	DBUG_RETURN(1);
@@ -304,7 +337,7 @@
     {
       char buff[8];
       mi_sizestore(buff,filepos);
-      if (my_pwrite(info->dfile,buff, 8, delete_block+12, MYF(MY_NABP)))
+      if (_mi_write(info,buff, 8, delete_block+12, MYF(MY_NABP)))
 	DBUG_RETURN(1);				/* Error on write */
     }
     else
@@ -362,7 +395,7 @@
       bfill(block_info.header+12,8,255);
     else
       mi_sizestore(block_info.header+12,block_info.next_filepos);
-    if (my_pwrite(info->dfile,(byte*) block_info.header,20,filepos,
+    if (_mi_write(info,(byte*) block_info.header,20,filepos,
 		  MYF(MY_NABP)))
       DBUG_RETURN(1);
     info->s->state.dellink = filepos;
@@ -545,7 +578,7 @@
   else
   {
     info->rec_cache.seek_not_done=1;
-    if (my_pwrite(info->dfile,(byte*) *record-head_length,length+extra_length+
+    if (_mi_write(info,(byte*) *record-head_length,length+extra_length+
 		  del_length,filepos,info->s->write_flag))
       goto err;
   }
@@ -655,7 +688,7 @@
 	      mi_int3store(del_block.header+1, rest_length);
 	      mi_sizestore(del_block.header+4,info->s->state.dellink);
 	      bfill(del_block.header+12,8,255);
-	      if (my_pwrite(info->dfile,(byte*) del_block.header,20, next_pos,
+	      if (_mi_write(info,(byte*) del_block.header,20, next_pos,
 			    MYF(MY_NABP)))
 		DBUG_RETURN(1);
 	      info->s->state.dellink= next_pos;
@@ -1182,7 +1215,7 @@
       }
       if (left_length < block_info.data_len || ! block_info.data_len)
 	goto panic;			/* Wrong linked record */
-      if (my_pread(file,(byte*) to,block_info.data_len,block_info.filepos,
+      if (_mi_read(info,(byte*) to,block_info.data_len,block_info.filepos,
 		   MYF(MY_NABP)))
 	goto panic;
       left_length-=block_info.data_len;

--- 1.44/myisam/mi_extra.c	2005-02-23 23:25:45 +01:00
+++ 1.45/myisam/mi_extra.c	2005-05-27 10:10:51 +02:00
@@ -367,6 +367,28 @@
   case HA_EXTRA_CHANGE_KEY_TO_DUP:
     mi_extra_keyflag(info, function);
     break;
+  case HA_EXTRA_MMAP:
+#if defined(HAVE_MMAP) && defined(HAVE_MADVICE)
+    pthread_mutex_lock(&share->intern_lock);
+    if ((!share->file_map) && (share->state.state.data_file_length > 0))
+    {
+      share->file_map=(byte*)
+                      my_mmap(0,
+                              (size_t)(share->state.state.data_file_length +
MEMMAP_EXTRA_MARGIN),
+                              PROT_READ|PROT_WRITE, MAP_SHARED | MAP_NORESERVE,
info->dfile, 0L);
+      madvise(share->file_map, share->state.state.data_file_length +
MEMMAP_EXTRA_MARGIN, MADV_RANDOM);
+
+      if (share->file_map == (byte*) MAP_FAILED)
+      {
+        DBUG_PRINT("warning",("mmap failed: errno: %d",errno));
+        error= my_errno= errno;
+        break;
+      }
+      share->mmaped_length= share->state.state.data_file_length +
MEMMAP_EXTRA_MARGIN;
+    }
+    pthread_mutex_unlock(&share->intern_lock);
+#endif
+    break;
   case HA_EXTRA_KEY_CACHE:
   case HA_EXTRA_NO_KEY_CACHE:
   default:

--- 1.30/myisam/mi_packrec.c	2005-03-19 01:12:20 +01:00
+++ 1.31/myisam/mi_packrec.c	2005-05-27 10:10:51 +02:00
@@ -1221,6 +1221,7 @@
       DBUG_RETURN(0);
     }
     info->s->file_map=file_map;
+    info->s->mmaped_length=
share->state.state.data_file_length+MEMMAP_EXTRA_MARGIN;
   }
   info->opt_flag|= MEMMAP_USED;
   info->read_record=share->read_record=_mi_read_mempack_record;
@@ -1232,8 +1233,7 @@
 void _mi_unmap_file(MI_INFO *info)
 {
   VOID(my_munmap(info->s->file_map,
-	      (size_t) info->s->state.state.data_file_length+
-	      MEMMAP_EXTRA_MARGIN));
+	      (size_t) info->s->mmaped_length));
 }
 
 

--- 1.11/myisam/mi_statrec.c	2005-05-13 11:08:02 +02:00
+++ 1.12/myisam/mi_statrec.c	2005-05-27 10:10:52 +02:00
@@ -28,14 +28,14 @@
   {
     my_off_t filepos=info->s->state.dellink;
     info->rec_cache.seek_not_done=1;		/* We have done a seek */
-    if (my_pread(info->dfile,(char*) &temp[0],info->s->base.rec_reflength,
+    if (_mi_read(info,(char*) &temp[0],info->s->base.rec_reflength,
 		info->s->state.dellink+1,
 		 MYF(MY_NABP)))
       goto err;
     info->s->state.dellink= _mi_rec_pos(info->s,temp);
     info->state->del--;
     info->state->empty-=info->s->base.pack_reclength;
-    if (my_pwrite(info->dfile, (char*) record, info->s->base.reclength,
+    if (_mi_write(info, (char*) record, info->s->base.reclength,
 		  filepos,
 		  MYF(MY_NABP)))
       goto err;
@@ -64,7 +64,7 @@
     else
     {
       info->rec_cache.seek_not_done=1;		/* We have done a seek */
-      if (my_pwrite(info->dfile,(char*) record,info->s->base.reclength,
+      if (_mi_write(info,(char*) record,info->s->base.reclength,
 		    info->state->data_file_length,
 		    info->s->write_flag))
 	goto err;
@@ -72,7 +72,7 @@
       {
 	uint length=info->s->base.pack_reclength - info->s->base.reclength;
 	bzero((char*) temp,length);
-	if (my_pwrite(info->dfile, (byte*) temp,length,
+	if (_mi_write(info, (byte*) temp,length,
 		      info->state->data_file_length+
 		      info->s->base.reclength,
 		      info->s->write_flag))
@@ -90,7 +90,7 @@
 int _mi_update_static_record(MI_INFO *info, my_off_t pos, const byte *record)
 {
   info->rec_cache.seek_not_done=1;		/* We have done a seek */
-  return (my_pwrite(info->dfile,
+  return (_mi_write(info,
 		    (char*) record,info->s->base.reclength,
 		    pos,
 		    MYF(MY_NABP)) != 0);
@@ -107,7 +107,7 @@
   _mi_dpointer(info,temp+1,info->s->state.dellink);
   info->s->state.dellink = info->lastpos;
   info->rec_cache.seek_not_done=1;
-  return (my_pwrite(info->dfile,(byte*) temp, 1+info->s->rec_reflength,
+  return (_mi_write(info,(byte*) temp, 1+info->s->rec_reflength,
 		    info->lastpos, MYF(MY_NABP)) != 0);
 }
 
@@ -131,7 +131,7 @@
   if ((info->opt_flag & READ_CHECK_USED))
   {						/* If check isn't disabled  */
     info->rec_cache.seek_not_done=1;		/* We have done a seek */
-    if (my_pread(info->dfile, (char*) info->rec_buff,
info->s->base.reclength,
+    if (_mi_read(info, (char*) info->rec_buff, info->s->base.reclength,
 		 info->lastpos,
 		 MYF(MY_NABP)))
       DBUG_RETURN(-1);
@@ -154,7 +154,7 @@
   DBUG_ENTER("_mi_cmp_static_unique");
 
   info->rec_cache.seek_not_done=1;		/* We have done a seek */
-  if (my_pread(info->dfile, (char*) info->rec_buff, info->s->base.reclength,
+  if (_mi_read(info, (char*) info->rec_buff, info->s->base.reclength,
 	       pos, MYF(MY_NABP)))
     DBUG_RETURN(-1);
   DBUG_RETURN(mi_unique_comp(def, record, info->rec_buff,
@@ -180,7 +180,7 @@
       return(-1);
     info->rec_cache.seek_not_done=1;		/* We have done a seek */
 
-    error=my_pread(info->dfile,(char*) record,info->s->base.reclength,
+    error=_mi_read(info,(char*) record,info->s->base.reclength,
 		   pos,MYF(MY_NABP)) != 0;
     fast_mi_writeinfo(info);
     if (! error)

--- 1.76/myisam/myisamdef.h	2005-05-13 11:08:02 +02:00
+++ 1.77/myisam/myisamdef.h	2005-05-27 10:10:52 +02:00
@@ -206,6 +206,7 @@
   pthread_mutex_t intern_lock;		/* Locking for use with _locking */
   rw_lock_t *key_root_lock;
 #endif
+  my_off_t mmaped_length;
 } MYISAM_SHARE;
 
 

--- 1.148/sql/ha_myisam.cc	2005-04-27 11:25:04 +02:00
+++ 1.149/sql/ha_myisam.cc	2005-05-27 10:10:52 +02:00
@@ -238,6 +238,10 @@
   
   if (test_if_locked & (HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_TMP_TABLE))
     VOID(mi_extra(file, HA_EXTRA_NO_WAIT_LOCK, 0));
+
+  if (opt_myisam_use_mmap)
+    VOID(mi_extra(file, HA_EXTRA_MMAP, 0));
+
   info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST);
   if (!(test_if_locked & HA_OPEN_WAIT_IF_LOCKED))
     VOID(mi_extra(file, HA_EXTRA_WAIT_LOCK, 0));

--- 1.297/sql/mysql_priv.h	2005-05-20 08:33:59 +02:00
+++ 1.298/sql/mysql_priv.h	2005-05-27 10:10:52 +02:00
@@ -1092,7 +1092,7 @@
 extern bool mysql_proc_table_exists;
 extern uint volatile thread_count, thread_running, global_read_lock;
 extern my_bool opt_sql_bin_update, opt_safe_user_create, opt_no_mix_types;
-extern my_bool opt_safe_show_db, opt_local_infile;
+extern my_bool opt_safe_show_db, opt_local_infile, opt_myisam_use_mmap;
 extern my_bool opt_slave_compressed_protocol, use_temp_pool;
 extern my_bool opt_readonly, lower_case_file_system;
 extern my_bool opt_enable_named_pipe, opt_sync_frm, opt_allow_suspicious_udfs;

--- 1.457/sql/mysqld.cc	2005-05-18 20:15:08 +02:00
+++ 1.458/sql/mysqld.cc	2005-05-27 10:10:53 +02:00
@@ -328,6 +328,7 @@
 my_bool opt_secure_auth= 0;
 my_bool lower_case_file_system= 0;
 my_bool opt_large_pages= 0;
+my_bool opt_myisam_use_mmap= 0;
 uint    opt_large_page_size= 0;
 my_bool opt_old_style_user_limits= 0, trust_routine_creators= 0;
 /*
@@ -4232,6 +4233,7 @@
   OPT_MAX_ERROR_COUNT, OPT_MULTI_RANGE_COUNT, OPT_MYISAM_DATA_POINTER_SIZE,
   OPT_MYISAM_BLOCK_SIZE, OPT_MYISAM_MAX_EXTRA_SORT_FILE_SIZE,
   OPT_MYISAM_MAX_SORT_FILE_SIZE, OPT_MYISAM_SORT_BUFFER_SIZE,
+  OPT_MYISAM_USE_MMAP,
   OPT_NET_BUFFER_LENGTH, OPT_NET_RETRY_COUNT,
   OPT_NET_READ_TIMEOUT, OPT_NET_WRITE_TIMEOUT,
   OPT_OPEN_FILES_LIMIT,
@@ -5368,6 +5370,11 @@
    (gptr*) &global_system_variables.myisam_sort_buff_size,
    (gptr*) &max_system_variables.myisam_sort_buff_size, 0,
    GET_ULONG, REQUIRED_ARG, 8192*1024, 4, ~0L, 0, 1, 0},
+  {"myisam_use_mmap", OPT_MYISAM_USE_MMAP,
+   "Use mmap/mmap64 for read/write MyISAM tables.",
+   (gptr*) &opt_myisam_use_mmap,
+   (gptr*) &opt_myisam_use_mmap, 0, GET_BOOL, NO_ARG, 0, 
+    0, 0, 0, 0, 0},
   {"net_buffer_length", OPT_NET_BUFFER_LENGTH,
    "Buffer length for TCP/IP and socket communication.",
    (gptr*) &global_system_variables.net_buffer_length,

--- 1.112/sql/set_var.cc	2005-05-18 11:46:02 +02:00
+++ 1.113/sql/set_var.cc	2005-05-27 10:10:53 +02:00
@@ -272,6 +272,7 @@
 sys_var_thd_ulonglong	sys_myisam_max_sort_file_size("myisam_max_sort_file_size",
&SV::myisam_max_sort_file_size, fix_myisam_max_sort_file_size, 1);
 sys_var_thd_ulong       sys_myisam_repair_threads("myisam_repair_threads",
&SV::myisam_repair_threads);
 sys_var_thd_ulong	sys_myisam_sort_buffer_size("myisam_sort_buffer_size",
&SV::myisam_sort_buff_size);
+sys_var_bool_ptr	sys_myisam_use_mmap("myisam_use_mmap", &opt_myisam_use_mmap);
 sys_var_thd_ulong	sys_net_buffer_length("net_buffer_length",
 					      &SV::net_buffer_length);
 sys_var_thd_ulong	sys_net_read_timeout("net_read_timeout",
@@ -623,6 +624,7 @@
   &sys_myisam_data_pointer_size,
   &sys_myisam_max_sort_file_size,
   &sys_myisam_repair_threads,
+  &sys_myisam_use_mmap,
   &sys_myisam_sort_buffer_size,
   &sys_net_buffer_length,
   &sys_net_read_timeout,
@@ -885,6 +887,7 @@
   {sys_myisam_repair_threads.name, (char*) &sys_myisam_repair_threads,
    SHOW_SYS},
   {sys_myisam_sort_buffer_size.name, (char*) &sys_myisam_sort_buffer_size, SHOW_SYS},
+  {sys_myisam_use_mmap.name, (char*) &sys_myisam_use_mmap, SHOW_SYS},
 #ifdef __NT__
   {"named_pipe",	      (char*) &opt_enable_named_pipe,       SHOW_MY_BOOL},
 #endif