List:Internals« Previous MessageNext Message »
From:Patrick Galbraith Date:November 16 2005 12:33am
Subject:bk commit into 5.0 tree (patg:1.1958) BUG#13406
View as plain text  
Below is the list of changes that have just been committed into a local
5.0 repository of patg. When patg does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.1958 05/11/15 16:32:58 patg@stripped +2 -0
  BUG #13406, Worklog #2949 
  
  Added meta-file support to ha_tina, CSV storage engine. This fixes the 
  problem where records is never the correct value, particularly on delete 
  operations (delete_all_rows and delete_row), also beefed up info.

  sql/examples/ha_tina.h
    1.6 05/11/15 16:32:41 patg@stripped +18 -1
    BUG #13406, Worklog #2949
    
    Added meta file to ha_tina, CSV storage engine. Added share members, class
    methods read/write_meta_file, made get_share and free_share class methods.

  sql/examples/ha_tina.cc
    1.22 05/11/15 16:32:41 patg@stripped +195 -19
    BUG #13406, WL# 2949 Add meta file to CSV storage engine
    
    Added the methods read_meta_file, write_meta_file, added share members 
    required, open meta file in get_share. Set share->dirty and share->crashed 
    where appropriate. Added new CSV and CSM (meta file) defines.

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	patg
# Host:	krsna.patg.net
# Root:	/home/patg/mysql-build/mysql-5.0.test4

--- 1.21/sql/examples/ha_tina.cc	2005-11-07 14:12:32 -08:00
+++ 1.22/sql/examples/ha_tina.cc	2005-11-15 16:32:41 -08:00
@@ -54,6 +54,15 @@
 static HASH tina_open_tables;
 static int tina_init= 0;
 
+/* file extension */
+#define CSM ".CSM"               // Meta file
+#define CSV ".CSV"               // The data file
+/*
+  uchar + uchar + ulonglong + ulonglong + uchar
+*/
+#define META_BUFFER_SIZE 19      // Size of the data used in the meta file
+#define TINA_CHECK_HEADER 254 // The number we use to determine corruption
+
 handlerton tina_hton= {
   "CSV",
   SHOW_OPTION_YES,
@@ -78,6 +87,73 @@
   HTON_CAN_RECREATE
 };
 
+/*
+  This method reads the header of a meta file and returns whether or not it was successful.
+  *rows will contain the current number of rows in the data file upon success.
+*/
+int ha_tina::read_meta_file(File meta_file, ha_rows *rows)
+{
+  uchar meta_buffer[META_BUFFER_SIZE];
+  ulonglong check_point;
+
+  DBUG_ENTER("ha_tina::read_meta_file");
+
+  VOID(my_seek(meta_file, 0, MY_SEEK_SET, MYF(0)));
+  if (my_read(meta_file, (byte*)meta_buffer, META_BUFFER_SIZE, 0) != META_BUFFER_SIZE)
+    DBUG_RETURN(-1);
+
+  /*
+    Parse out the meta data, we ignore version at the moment
+  */
+  *rows= (ha_rows)uint8korr(meta_buffer + 2);
+  check_point= uint8korr(meta_buffer + 10);
+
+  DBUG_PRINT("info", ("Check %d", (uint)meta_buffer[0]));
+  DBUG_PRINT("info", ("Version %d", (uint)meta_buffer[1]));
+  DBUG_PRINT("info", ("Rows %lld", *rows));
+  DBUG_PRINT("info", ("Checkpoint %lld", check_point));
+  DBUG_PRINT("info", ("Dirty %d", (int)meta_buffer[18]));
+
+  if ((meta_buffer[0] != (uchar)TINA_CHECK_HEADER) || 
+      ((bool)meta_buffer[18] == TRUE))
+    DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+
+  my_sync(meta_file, MYF(MY_WME));
+
+  DBUG_RETURN(0);
+}
+
+/*
+  This method writes out the header of a meta file and returns whether or not it was successful.
+  By setting dirty you say whether or not the file represents the actual state of the data file.
+  Upon ::open() we set to dirty, and upon ::close() we set to clean.
+*/
+int ha_tina::write_meta_file(File meta_file, ha_rows rows, bool dirty)
+{
+  uchar meta_buffer[META_BUFFER_SIZE];
+  ulonglong check_point= 0; //Reserved for the future
+
+  DBUG_ENTER("ha_tina::write_meta_file");
+
+  meta_buffer[0]= (uchar)TINA_CHECK_HEADER;
+  meta_buffer[1]= (uchar)TINA_VERSION;
+  int8store(meta_buffer + 2, (ulonglong)rows); 
+  int8store(meta_buffer + 10, check_point); 
+  *(meta_buffer + 18)= (uchar)dirty;
+  DBUG_PRINT("info", ("Check %d", (uint)TINA_CHECK_HEADER));
+  DBUG_PRINT("info", ("Version %d", (uint)TINA_VERSION));
+  DBUG_PRINT("info", ("Rows %llu", (ulonglong)rows));
+  DBUG_PRINT("info", ("Checkpoint %llu", check_point));
+  DBUG_PRINT("info", ("Dirty %d", (uint)dirty));
+
+  VOID(my_seek(meta_file, 0, MY_SEEK_SET, MYF(0)));
+  if (my_write(meta_file, (byte *)meta_buffer, META_BUFFER_SIZE, 0) != META_BUFFER_SIZE)
+    DBUG_RETURN(-1);
+
+  my_sync(meta_file, MYF(MY_WME));
+
+  DBUG_RETURN(0);
+}
 /*****************************************************************************
  ** TINA tables
  *****************************************************************************/
@@ -136,16 +212,17 @@
       DBUG_RETURN(1);
     }
   }
-  else 
+  else
     share->mapped_file= NULL;
 
   DBUG_RETURN(0);
 }
 
+
 /*
   Simple lock controls.
 */
-static TINA_SHARE *get_share(const char *table_name, TABLE *table)
+TINA_SHARE *ha_tina::get_share(const char *table_name, TABLE *table)
 {
   TINA_SHARE *share;
   char *tmp_name;
@@ -170,7 +247,7 @@
                                         (byte*) table_name,
                                         length)))
   {
-    char data_file_name[FN_REFLEN];
+    char meta_file_name[FN_REFLEN];
     if (!my_multi_malloc(MYF(MY_WME | MY_ZEROFILL),
                          &share, sizeof(*share),
                          &tmp_name, length+1,
@@ -183,14 +260,27 @@
     share->use_count=0;
     share->table_name_length=length;
     share->table_name=tmp_name;
+    share->crashed= FALSE;
+    fn_format(share->data_file_name, table_name, "", CSV,MY_REPLACE_EXT|MY_UNPACK_FILENAME);
+    fn_format(meta_file_name,table_name,"",CSM,MY_REPLACE_EXT|MY_UNPACK_FILENAME);
     strmov(share->table_name,table_name);
-    fn_format(data_file_name, table_name, "", ".CSV",MY_REPLACE_EXT|MY_UNPACK_FILENAME);
-    if (my_hash_insert(&tina_open_tables, (byte*) share))
-      goto error;
-    thr_lock_init(&share->lock);
+
     pthread_mutex_init(&share->mutex,MY_MUTEX_INIT_FAST);
 
-    if ((share->data_file= my_open(data_file_name, O_RDWR|O_APPEND,
+    if ((share->meta_file= my_open(meta_file_name, O_RDWR, MYF(0))) == -1)
+      goto error2;
+    /*
+      After we read, we set the file to dirty. When we close, we will do the 
+      opposite. If the meta file will not open we assume it is crashed and
+      leave it up to the user to fix.
+    */
+    if (read_meta_file(share->meta_file, &share->rows_recorded))
+      share->crashed= TRUE;
+    else
+      (void)write_meta_file(share->meta_file, share->rows_recorded, TRUE);
+
+
+    if ((share->data_file= my_open(share->data_file_name, O_RDWR|O_APPEND,
                                    MYF(0))) == -1)
       goto error2;
 
@@ -201,15 +291,22 @@
     share->mapped_file= NULL; // We don't know the state since we just allocated it
     if (get_mmap(share, 0) > 0)
       goto error3;
+
+    if (my_hash_insert(&tina_open_tables, (byte*) share))
+      goto error;
+    thr_lock_init(&share->lock);
   }
   share->use_count++;
   pthread_mutex_unlock(&tina_mutex);
 
   return share;
 
+error4:
+  my_close(share->meta_file,MYF(0));
 error3:
   my_close(share->data_file,MYF(0));
 error2:
+  share->crashed= TRUE;
   thr_lock_delete(&share->lock);
   pthread_mutex_destroy(&share->mutex);
 error:
@@ -223,7 +320,7 @@
 /* 
   Free lock controls.
 */
-static int free_share(TINA_SHARE *share)
+int ha_tina::free_share(TINA_SHARE *share)
 {
   DBUG_ENTER("ha_tina::free_share");
   pthread_mutex_lock(&tina_mutex);
@@ -236,6 +333,9 @@
     hash_delete(&tina_open_tables, (byte*) share);
     thr_lock_delete(&share->lock);
     pthread_mutex_destroy(&share->mutex);
+    (void)write_meta_file(share->meta_file, share->rows_recorded, FALSE);
+    if (my_close(share->meta_file, MYF(0)))
+      result_code= 1;
     my_free((gptr) share, MYF(0));
   }
   pthread_mutex_unlock(&tina_mutex);
@@ -437,7 +537,8 @@
   extensions exist for this handler.
 */
 static const char *ha_tina_exts[] = {
-  ".CSV",
+  CSV,
+  CSM,
   NullS
 };
 
@@ -485,6 +586,9 @@
   int size;
   DBUG_ENTER("ha_tina::write_row");
 
+  if (share->crashed)
+      DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+
   statistic_increment(table->in_use->status_var.ha_write_count, &LOCK_status);
 
   if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_INSERT)
@@ -492,9 +596,12 @@
 
   size= encode_quote(buf);
 
+  share->rows_recorded++;
   if (my_write(share->data_file, buffer.ptr(), size, MYF(MY_WME | MY_NABP)))
     DBUG_RETURN(-1);
 
+  share->dirty= TRUE;
+
   /* 
     Ok, this is means that we will be doing potentially bad things 
     during a bulk insert on some OS'es. What we need is a cleanup
@@ -534,6 +641,7 @@
 
   if (my_write(share->data_file, buffer.ptr(), size, MYF(MY_WME | MY_NABP)))
     DBUG_RETURN(-1);
+  share->dirty= TRUE;
   DBUG_RETURN(0);
 }
 
@@ -554,7 +662,9 @@
   if (chain_append())
     DBUG_RETURN(-1);
 
-  --records;
+  share->rows_recorded--;
+  records= share->rows_recorded;
+  share->dirty= TRUE;
 
   DBUG_RETURN(0);
 }
@@ -666,13 +776,28 @@
 {
   DBUG_ENTER("ha_tina::rnd_init");
 
+  if (share->crashed)
+      DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+
   current_position= next_position= 0;
   records= 0;
   chain_ptr= chain;
-#ifdef HAVE_MADVISE
   if (scan)
+  {
+    scan_rows= share->rows_recorded;
+#ifdef HAVE_MADVISE
     (void)madvise(share->mapped_file,share->file_stat.st_size,MADV_SEQUENTIAL);
 #endif
+    if (share->dirty == TRUE)
+    {
+      /*
+        in archive, we do a gzflush(share->archive_write, Z_SYNC_FLUSH);
+        at this point. I wonder if we need to do something similar here
+        for a CSV file?
+      */
+        share->dirty= FALSE;
+    }
+  }
 
   DBUG_RETURN(0);
 }
@@ -692,13 +817,19 @@
 {
   DBUG_ENTER("ha_tina::rnd_next");
 
+  if (share->crashed)
+      DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE);
+
   statistic_increment(table->in_use->status_var.ha_read_rnd_next_count,
 		      &LOCK_status);
 
   current_position= next_position;
-  if (!share->mapped_file) 
+  if (!scan_rows)
     DBUG_RETURN(HA_ERR_END_OF_FILE);
-  if (HA_ERR_END_OF_FILE == find_current_row(buf) ) 
+  scan_rows--;
+  if (!share->mapped_file)
+    DBUG_RETURN(HA_ERR_END_OF_FILE);
+  if (HA_ERR_END_OF_FILE == find_current_row(buf) )
     DBUG_RETURN(HA_ERR_END_OF_FILE);
 
   records++;
@@ -745,8 +876,25 @@
 {
   DBUG_ENTER("ha_tina::info");
   /* This is a lie, but you don't want the optimizer to see zero or 1 */
-  if (records < 2) 
-    records= 2;
+  records= share->rows_recorded;
+  deleted= 0;
+
+  /* Costs quite a bit more to get all information */
+  if (flag & HA_STATUS_TIME)
+  {
+    MY_STAT file_stat;  // Stat information for the data file
+
+    VOID(my_stat(share->data_file_name, &file_stat, MYF(MY_WME)));
+
+    mean_rec_length= table->s->reclength + buffer.alloced_length();
+    data_file_length= file_stat.st_size;
+    create_time= file_stat.st_ctime;
+    update_time= file_stat.st_mtime;
+    max_data_file_length= share->rows_recorded * mean_rec_length;
+  }
+  delete_length= 0;
+  index_file_length=0;
+
   DBUG_VOID_RETURN;
 }
 
@@ -823,7 +971,7 @@
   DBUG_RETURN(0);
 }
 
-/* 
+/*
   Truncate table and others of its ilk call this. 
 */
 int ha_tina::delete_all_rows()
@@ -832,13 +980,27 @@
 
   int rc= my_chsize(share->data_file, 0, 0, MYF(MY_WME));
 
-  if (get_mmap(share, 0) > 0) 
+  if (get_mmap(share, 0) > 0)
     DBUG_RETURN(-1);
 
+  deleted= share->rows_recorded;
+  records= share->rows_recorded = 0;
+
   DBUG_RETURN(rc);
 }
 
 /*
+  We just return state if asked.
+*/
+/*
+bool ha_archive::is_crashed() const
+{
+  return share->crashed;
+}
+*/
+
+
+/*
   Always called by the start of a transaction (or by "lock tables");
 */
 int ha_tina::external_lock(THD *thd, int lock_type)
@@ -872,10 +1034,24 @@
   File create_file;
   DBUG_ENTER("ha_tina::create");
 
-  if ((create_file= my_create(fn_format(name_buff,name,"",".CSV",MY_REPLACE_EXT|MY_UNPACK_FILENAME),0,
+  /*
+    create the CSV data file
+  */
+  if ((create_file= my_create(fn_format(name_buff, name,"", CSV,
+                                        MY_REPLACE_EXT|MY_UNPACK_FILENAME), 0,
                               O_RDWR | O_TRUNC,MYF(MY_WME))) < 0)
     DBUG_RETURN(-1);
 
+  my_close(create_file,MYF(0));
+
+  /*
+    create the meta file
+  */
+  if ((create_file= my_create(fn_format(name_buff, name,"", CSM,
+                                        MY_REPLACE_EXT|MY_UNPACK_FILENAME), 0,
+                              O_RDWR | O_TRUNC,MYF(MY_WME))) < 0)
+    DBUG_RETURN(-1);
+  write_meta_file(create_file, 0, FALSE);
   my_close(create_file,MYF(0));
 
   DBUG_RETURN(0);

--- 1.5/sql/examples/ha_tina.h	2005-10-03 18:41:51 -07:00
+++ 1.6/sql/examples/ha_tina.h	2005-11-15 16:32:41 -08:00
@@ -22,14 +22,25 @@
 
 typedef struct st_tina_share {
   char *table_name;
+  char data_file_name[FN_REFLEN];
   byte *mapped_file;                /* mapped region of file */
   uint table_name_length,use_count;
   MY_STAT file_stat;                /* Stat information for the data file */
   File data_file;                   /* Current open data file */
+  File meta_file;                   /* Meta file we use */
+  bool dirty;                       /* Flag for if a flush should occur */
+  bool crashed;                     /* Meta file is crashed */
+  ha_rows rows_recorded;            /* Number of rows in tables */
   pthread_mutex_t mutex;
   THR_LOCK lock;
 } TINA_SHARE;
 
+/*
+  version for file format ...
+  1 initial version
+*/
+#define TINA_VERSION 1
+
 typedef struct tina_set {
 	off_t begin;
 	off_t end;
@@ -43,6 +54,7 @@
   off_t next_position;     /* Next position in the file scan */
   byte byte_buffer[IO_SIZE];
   String buffer;
+  ha_rows scan_rows;         /* Number of rows left in scan */
   tina_set chain_buffer[DEFAULT_CHAIN_LENGTH];
   tina_set *chain;
   tina_set *chain_ptr;
@@ -61,7 +73,7 @@
   const char **bas_ext() const;
   ulong table_flags() const
   {
-    return (HA_REC_NOT_IN_SEQ | HA_NOT_EXACT_COUNT | 
+    return (HA_REC_NOT_IN_SEQ | HA_NOT_EXACT_COUNT |
       HA_NO_AUTO_INCREMENT );
   }
   ulong index_flags(uint idx, uint part, bool all_parts) const
@@ -102,6 +114,10 @@
   int rnd_init(bool scan=1);
   int rnd_next(byte *buf);
   int rnd_pos(byte * buf, byte *pos);
+  int read_meta_file(File meta_file, ha_rows *rows);
+  int write_meta_file(File meta_file, ha_rows rows, bool dirty);
+  TINA_SHARE *get_share(const char *table_name, TABLE *table);
+  int free_share(TINA_SHARE *share);
   int rnd_end();
   void position(const byte *record);
   void info(uint);
@@ -118,6 +134,7 @@
   int encode_quote(byte *buf);
   int find_current_row(byte *buf);
   int chain_append();
+  //bool is_crashed() const;
 };
 
 bool tina_end();
Thread
bk commit into 5.0 tree (patg:1.1958) BUG#13406Patrick Galbraith16 Nov