MySQL Lists are EOL. Please join:

List:Commits« Previous MessageNext Message »
From:Rafal Somla Date:April 15 2010 1:51pm
Subject:bzr push into mysql-6.0-backup branch (Rafal.Somla:2942 to 2943)
View as plain text  
 2943 Rafal Somla	2010-04-15
      Backup driver: Tablespace data copy + skeleton for copy of log data.
      
      Note change in INB_CHUNK_DATA chunk format - each data chunk stores the number
      of empty pages preceeding the given one (can be 0). INB_CHUNK_DATA_EMPTY is no
      longer needed.
     @ inb/backup.cc
        Added methods and code for sending the backup info chunk and tablespace data
        chunks.
     @ inb/backup.h
        Added new members used by driver methods.
     @ inb/inb_impl.h
        The header of INB_CHUNK_DATA is now longer, because it
        stores empty_count.
     @ inb/restore.cc
        Rename: get_dno -> get_dnum.

    modified:
      inb/backup.cc
      inb/backup.h
      inb/inb_impl.h
      inb/restore.cc
 2942 Rafal Somla	2010-04-14
      Backup driver skeleton.

    added:
      inb/backup.cc
      inb/backup.h
    modified:
      inb/Makefile.am
      inb/engine.cc
=== modified file 'inb/backup.cc'
--- a/inb/backup.cc	2010-04-14 11:28:39 +0000
+++ b/inb/backup.cc	2010-04-15 13:50:48 +0000
@@ -1,5 +1,11 @@
 #include <mysql_priv.h>
-#include <srv0srv.h>
+
+extern "C" {
+#include <os0file.h>
+#include <log0recv.h>
+#include <mach0data.h>
+#include <page0page.h>
+}
 
 #include "backup.h"
 #include "inb_impl.h"
@@ -21,8 +27,16 @@ void msg(const char *str, ...)
 
 
 inb::Backup_driver::Backup_driver(const Table_list &tables)
-  : ::Backup_driver(tables)
+  : ::Backup_driver(tables), stage(INIT), log_stage(INIT), m_table(0),
+  m_dnum(0), m_pnum(0), m_lnum(0), m_bnum(0)
 {
+  m_dcount= srv_n_data_files;
+  m_data_dir= srv_data_home;
+  m_dname= (const char**)srv_data_file_names;
+  m_dsize= srv_data_file_sizes;
+  m_lcount= srv_n_log_files;
+  m_lsize= (srv_log_file_size * UNIV_PAGE_SIZE) / OS_FILE_LOG_BLOCK_SIZE;
+
   msg("Driver created");
   msg("                   data_home= %s", srv_data_home);
   msg("              file_per_table= %s", srv_file_per_table ? "yes" : "no");
@@ -33,14 +47,15 @@ inb::Backup_driver::Backup_driver(const 
   msg("                n_log_groups= %lu", srv_n_log_groups);
   msg("                 n_log_files= %lu", srv_n_log_files);
   msg("log_file_size (in 16k pages)= %lu", srv_log_file_size);
+  msg("   log_file_size (in blocks)= %lu", m_lsize);
+  
   msg("");
   msg(" data file list:");
-  
   for (unsigned int i=0; i < srv_n_data_files; ++i)
   {
-    msg("   %s (%lu%s)",  srv_data_file_names[i],
-                        srv_data_file_sizes[i],
-                        srv_data_file_is_raw_partition[i] ? ", raw":"");
+    msg("   %s (%lu%s)", srv_data_file_names[i],
+                         srv_data_file_sizes[i],
+                         srv_data_file_is_raw_partition[i] ? ", raw":"");
   }
 
   msg("");
@@ -51,13 +66,70 @@ inb::Backup_driver::Backup_driver(const 
     msg("   %s",  srv_log_group_home_dirs[i]);
   }
 
+  m_log_dir= srv_log_group_home_dirs[0];
+
   msg("");  
 }
 
 
+static char name_buf[5000];
+
+
 result_t inb::Backup_driver::begin(size_t)
 {
-  msg("Driver started");
+  // read first log block
+
+  os_file_t  log_file;
+  ibool      success;
+  byte       buf[LOG_CHECKPOINT_2 + OS_FILE_LOG_BLOCK_SIZE];
+
+  sprintf(name_buf, "%s%s%lu", m_log_dir, "ib_logfile", (ulint)0);
+
+  log_file = os_file_create_simple(name_buf, OS_FILE_OPEN, OS_FILE_READ_ONLY,
+                                   &success);
+
+  if (!success)
+  {
+    perror("Error when reading first log file");
+    return ERROR;
+  }
+
+  os_file_read(log_file, buf, 0, 0,
+               LOG_CHECKPOINT_2 + OS_FILE_LOG_BLOCK_SIZE);
+
+  os_file_close(log_file);
+
+  // Scan log beginning to find starting position
+
+  dulint    lsn;            /* out: checkpoint lsn */
+  ulint     offset;         /* out: checkpoint offset in
+                                        the log group */
+  ulint     fsp_limit;      /* out: fsp limit */
+  dulint    cp_no;          /* out: checkpoint number */
+  dulint    first_header_lsn;       /* out: lsn of the start of */
+  
+  success = recv_read_cp_info_for_backup(buf, 
+                                         &lsn, 
+                                         &offset,
+                                         &fsp_limit,
+                                         &cp_no, 
+                                         &first_header_lsn);
+
+  if (!success)
+  {
+    msg("Error when getting log info");
+    return ERROR;
+  }
+
+  m_start_cp=       lsn;
+  m_start_lsn.high= lsn.high;
+  m_start_lsn.low=  ut_2pow_round(lsn.low, OS_FILE_LOG_BLOCK_SIZE);
+
+  msg("Driver started:");
+
+  msg("  start_cp= %lu", m_start_cp.low);
+  msg(" start_lsn= %lu", m_start_lsn.low);
+  msg("");
   return OK;
 }
 
@@ -92,6 +164,249 @@ result_t inb::Backup_driver::unlock()
 
 result_t inb::Backup_driver::get_data(Buffer &buf)
 {
-  buf.size= 0;
+  result_t ret;
+
+  buf.table_num= 0;
+  buf.last= FALSE;
+
+  switch (stage) {
+   
+  case INIT:
+    buf.table_num= m_table++;
+
+    if (!buf.table_num)
+    {
+      msg("Sending info chunk.");
+      return send_info(buf);
+    }
+    else if (buf.table_num <= m_tables.count())
+    {
+      msg("Closing table %u.", buf.table_num);
+      buf.size= 0;
+      buf.last= TRUE;
+      return OK;
+    }
+
+    stage= DATA;
+    buf.table_num= 0;
+    msg("Starting data transfer.");
+
+  case DATA:
+    ret= send_data(buf);
+    
+    if (ret != backup::DONE)
+      return ret;
+
+    stage= LOG;
+    msg("Data transfer finished, starting log transfer.");
+    return READY;
+
+  case LOG:
+    ret= send_log(buf);
+    
+    if (ret != backup::DONE)
+      return ret;
+
+    stage= FINI;
+    msg("Log transfer finished.");
+
+  case FINI:
+    buf.last= TRUE;
+    msg("Closed common stream.");
+    stage= DONE;
+
+  case DONE:
+    msg("Backup done.");
+    return backup::DONE;
+  }
+
   return ERROR;
 }
+
+
+result_t inb::Backup_driver::send_info(Buffer &buf)
+{
+  buf.table_num= 0;
+  buf.last= FALSE;
+
+  if (buf.size < INB_CHUNK_INFO_LEN)
+    return ERROR;
+
+  buf.size= INB_CHUNK_INFO_LEN;
+
+  set_type(buf.data,       INB_CHUNK_INFO);
+  set_block_size(buf.data, OS_FILE_LOG_BLOCK_SIZE);
+  set_page_size(buf.data,  UNIV_PAGE_SIZE);
+  set_data_count(buf.data, srv_n_data_files);
+  // set_logg_count(buf, srv_n_log_groups); TODO: handle multiple log groups.
+  set_log_count(buf.data,  srv_n_log_files);
+  //set_log_size(buf.data,   (srv_log_file_size * UNIV_PAGE_SIZE) / OS_FILE_LOG_BLOCK_SIZE);
+
+  // TODO: store list of data files (name,size)
+  // TODO: store initial log position info
+  set_data_size(buf.data,  srv_data_file_sizes[0]); // in pages
+//  set_start_lsn(buf, m_start_lsn.low, m_start_lsn.high);
+//  set_start_cp(buf,  m_start_cp.low, m_start_cp.high);
+
+  return OK;
+}
+
+result_t inb::Backup_driver::send_data(Buffer &buf)
+{
+  ibool         success;
+  
+  buf.table_num= 0;
+  buf.last= FALSE;
+
+  if (buf.size < UNIV_PAGE_SIZE + 100)
+    return ERROR;
+
+  set_type(buf.data, INB_CHUNK_DATA);
+  set_dnum(buf.data, m_dnum);
+
+  if (m_dnum > m_dcount)
+    return backup::DONE;
+
+  if (!m_dnum || m_pnum >= m_dsize[m_dnum-1]) // switch to next data file
+  {
+    if (m_dnum)
+    {
+      msg("Closing data file %s%s", m_data_dir, m_dname[m_dnum-1]);
+      os_file_close(m_file);
+    }
+    else  // first call - do some initialization
+    {
+      m_max_lsn.high= m_max_lsn.low= 0;
+    }
+
+    ++m_dnum;
+    m_pnum= 0;
+
+    if (m_dnum > m_dcount)
+    {
+      // send_empty_pages(buf);
+      buf.size= 0;
+      if (m_empty)
+      {
+        msg("Sending %lu empty pages.", m_empty);
+        set_empty_count(buf.data, m_empty);
+        buf.size = INB_CHUNK_DATA_LEN; // no payload
+      }
+      return backup::DONE;
+    }
+
+    m_empty= 0;
+
+    snprintf(name_buf, sizeof(name_buf), "%s%s", m_data_dir, m_dname[m_dnum-1]);
+    msg("Opening data file %s", name_buf);
+  
+    m_file = os_file_create_simple_no_error_handling(
+                name_buf, OS_FILE_OPEN, OS_FILE_READ_ONLY, &success
+              );
+    if (!success)
+    {
+      msg("Failed to open data file %s.", name_buf);
+      perror("Opening data file");
+      buf.size= 0;
+      return ERROR;
+    }  
+  }
+
+  byte  *page=       buf.data + INB_CHUNK_DATA_LEN;
+  ulint offset_high= 0;
+  ulint offset_low=  (m_pnum * UNIV_PAGE_SIZE);
+
+  success= os_file_read(m_file, page, offset_low, offset_high, UNIV_PAGE_SIZE);
+
+  if (!success)
+  {
+    msg("Error reading page %lu from data file %s", m_pnum, m_dname[m_dnum-1]);
+    os_file_close(m_file);
+    return ERROR;
+  }
+
+  ++m_pnum;
+
+  bool all_zero= TRUE;
+
+  for(ulint i=0; i < UNIV_PAGE_SIZE/sizeof(ulint); ++i)
+   if (*((ulint*)(page)+i))
+     all_zero= FALSE;
+
+  if (all_zero)
+  {
+    msg("Page %lu is empty.", m_pnum-1);
+    ++m_empty;
+    return BUSY;  // kernel will re-try but we don't grab the buffer.
+  }
+  else
+  {
+    set_empty_count(buf.data, m_empty);
+    m_empty= 0;
+
+    dulint page_lsn;
+    page_lsn= mach_read_from_8(page + FIL_PAGE_LSN);
+    //msg(" page lsn %lu %lu", page_lsn.high, page_lsn.low);
+
+    if (page_lsn.high > m_max_lsn.high ||
+        (page_lsn.high == m_max_lsn.high && page_lsn.low > m_max_lsn.low))
+    {
+      m_max_lsn= page_lsn;
+      msg(" found new page lsn %lu %lu", m_max_lsn.high, m_max_lsn.low);
+    }
+  }
+
+  buf.size= INB_CHUNK_DATA_LEN + UNIV_PAGE_SIZE;
+
+  msg("Sending page %lu with %lu preceding empty pages", 
+      m_pnum-1, get_empty_count(buf.data));
+  
+  return OK;
+}
+
+
+result_t inb::Backup_driver::send_log(Buffer &buf)
+{
+  buf.table_num= 0;
+  buf.last= FALSE;
+
+  if (buf.size < OS_FILE_LOG_BLOCK_SIZE + INB_CHUNK_LOG_LEN)
+    return ERROR;
+
+  set_type(buf.data, INB_CHUNK_LOG);
+
+  switch (log_stage) {
+  
+  case INIT:
+    msg("Opening log file %lu", m_lnum);
+    log_stage= LOG;
+    break;
+
+  case LOG:
+    break;
+
+  case DONE:
+    buf.size= 0;
+    return backup::DONE;
+
+  default: DBUG_ASSERT(FALSE);
+  }
+
+  if (m_bnum >= m_lsize)
+  {
+    msg("Closing log file %lu", m_lnum);
+    m_lnum= (m_lnum+1) % m_lcount;
+    msg("Opening log file %lu", m_lnum);
+    m_bnum= 0;
+  }
+
+  msg("Sending log block %lu.", m_bnum);
+  
+  buf.size= INB_CHUNK_LOG_LEN;
+  ++m_bnum;
+  
+  if (m_lnum == m_lcount-1) 
+    log_stage= DONE; 
+
+  return backup::OK;
+}

=== modified file 'inb/backup.h'
--- a/inb/backup.h	2010-04-14 11:28:39 +0000
+++ b/inb/backup.h	2010-04-15 13:50:48 +0000
@@ -3,10 +3,25 @@
 
 #include <backup/backup_engine.h>
 
+extern "C" {
+#include <srv0srv.h>    // for dulint
+#include <os0file.h>
+}
+
 class InnoDB_backup_engine;
 
+
 namespace inb {
-        
+
+#if 0
+struct dulint { 
+  unsigned long high, low; 
+
+  dulint(const typename ::dulint&);
+  dulint(): high(0), low(0) {}
+};
+#endif
+
 using namespace backup;
 
 class Backup_driver: public ::Backup_driver
@@ -28,6 +43,38 @@ class Backup_driver: public ::Backup_dri
    result_t lock();
    result_t unlock();
 
+ private:
+
+   enum { INIT, DATA, LOG, FINI, DONE } stage, log_stage;
+
+   uint m_table;
+   
+   const char *m_log_dir;
+   ulint      m_lcount;    // number of files in log group
+   ulint      m_lsize;     // size of log file (in blocks)
+   
+   const char *m_data_dir;
+   ulint       m_dcount;   // number of data files in the tablespace
+   const char* *m_dname;   // names of data files
+   ulint       *m_dsize;   // sizes of data files
+   
+   dulint     m_start_cp;
+   dulint     m_start_lsn;
+
+   result_t send_info(Buffer&);
+   result_t send_data(Buffer&);
+   result_t send_log(Buffer&);
+
+   ulint      m_dnum;   // current data file being processed (1 based, 0- not started)
+   ulint      m_pnum;   // page number within the current data file
+   os_file_t  m_file;   // handle to opened data file
+   ulint      m_empty;  // number of consecutive empty pages found since last non-empty one.
+   dulint     m_max_lsn; // maximum lsn found on a page.
+
+   ulint      m_lnum;   // current log file number
+   ulint      m_bnum;   // current block within log file
+   os_file_t  m_lfile;  // handle to opened log file
+ 
  friend class ::InnoDB_backup_engine;
 };
 

=== modified file 'inb/inb_impl.h'
--- a/inb/inb_impl.h	2010-04-14 08:26:08 +0000
+++ b/inb/inb_impl.h	2010-04-15 13:50:48 +0000
@@ -11,20 +11,20 @@ extern "C" {
 #define INB_CHUNK_DATA_EMPTY  0x03
 
 #define INB_CHUNK_INFO_LEN  (1+5*4+2*8)
-#define INB_CHUNK_DATA_LEN  (1+2)
+#define INB_CHUNK_DATA_LEN  (1+2+2)
 #define INB_CHUNK_LOG_LEN   1
 #define INB_CHUNK_EMPTY_LEN (1+2+2)
 
 
 inline
-void set_dno(unsigned char chunk[], unsigned int val)
+void set_dnum(unsigned char chunk[], unsigned int val)
 {
   chunk[1] = val & 0xFF;
   chunk[2] = (val >> 8) & 0xFF;
 }
 
 inline
-unsigned int get_dno(const unsigned char chunk[])
+unsigned int get_dnum(const unsigned char chunk[])
 {
   return chunk[1] + (chunk[2] << 8);
 }

=== modified file 'inb/restore.cc'
--- a/inb/restore.cc	2010-04-14 08:20:30 +0000
+++ b/inb/restore.cc	2010-04-15 13:50:48 +0000
@@ -79,7 +79,7 @@ result_t inb::Restore_driver::send_data(
     
   case INB_CHUNK_DATA:
   {
-    unsigned int dno= get_dno(buf.data);
+    unsigned int dno= get_dnum(buf.data);
     msg("inb:restore: Got %lu bytes of table data from ts file %d", 
         buf.size - INB_CHUNK_DATA_LEN , dno);
     ++(m_page_count[dno-1]);
@@ -88,7 +88,7 @@ result_t inb::Restore_driver::send_data(
   
   case INB_CHUNK_DATA_EMPTY:
   {
-    unsigned int dno =   get_dno(buf.data);
+    unsigned int dno =   get_dnum(buf.data);
     unsigned int count = get_empty_count(buf.data);
     msg("inb:restore: Got %d empty data pages from file %d", count, dno);
     m_page_count[dno-1] += count;


Attachment: [text/bzr-bundle] bzr/rafal.somla@sun.com-20100415135048-he1vwfpfw3apkojy.bundle
Thread
bzr push into mysql-6.0-backup branch (Rafal.Somla:2942 to 2943)Rafal Somla15 Apr