MySQL Lists are EOL. Please join:

List:Commits« Previous MessageNext Message »
From:tomas Date:May 19 2007 8:35pm
Subject:bk commit into 5.1 tree (tomas:1.2544)
View as plain text  
Below is the list of changes that have just been committed into a local
5.1 repository of tomas. When tomas does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2007-05-19 22:35:31+02:00, tomas@stripped +4 -0
  progress reporting in ndb_restore

  storage/ndb/tools/restore/Restore.cpp@stripped, 2007-05-19 22:35:26+02:00, tomas@stripped +25 -0
    progress reporting in ndb_restore

  storage/ndb/tools/restore/Restore.hpp@stripped, 2007-05-19 22:35:26+02:00, tomas@stripped +7 -0
    progress reporting in ndb_restore

  storage/ndb/tools/restore/consumer_restore.cpp@stripped, 2007-05-19 22:35:26+02:00, tomas@stripped +3 -0
    progress reporting in ndb_restore

  storage/ndb/tools/restore/restore_main.cpp@stripped, 2007-05-19 22:35:27+02:00, tomas@stripped +60 -0
    progress reporting in ndb_restore

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	tomas
# Host:	whalegate.ndb.mysql.com
# Root:	/home/tomas/mysql-5.1-telco-merge

--- 1.54/storage/ndb/tools/restore/Restore.cpp	2007-04-23 13:07:20 +02:00
+++ 1.55/storage/ndb/tools/restore/Restore.cpp	2007-05-19 22:35:26 +02:00
@@ -741,6 +741,9 @@
   m_buffer = malloc(m_buffer_sz);
   m_buffer_ptr = m_buffer;
   m_buffer_data_left = 0;
+
+  m_file_size = 0;
+  m_file_pos = 0;
 }
 
 BackupFile::~BackupFile(){
@@ -755,9 +758,30 @@
   if(m_file != NULL){
     fclose(m_file);
     m_file = 0;
+    m_file_size = 0;
+    m_file_pos = 0;
   }
   
+  info.setLevel(254);
+  info << "Opening file '" << m_fileName << "'\n";
   m_file = fopen(m_fileName, "r");
+
+  if (m_file)
+  {
+    struct stat buf;
+    if (fstat(fileno(m_file), &buf) == 0)
+    {
+      m_file_size = (Uint64)buf.st_size;
+      info << "File size " << m_file_size << " bytes\n";
+    }
+    else
+    {
+      info << "Progress reporting degraded output since fstat failed,"
+           << "errno: " << errno << endl;
+      m_file_size = 0;
+    }
+  }
+
   return m_file != 0;
 }
 
@@ -772,6 +796,7 @@
     memcpy(m_buffer, m_buffer_ptr, m_buffer_data_left);
 
     size_t r = fread(((char *)m_buffer) + m_buffer_data_left, 1, m_buffer_sz - m_buffer_data_left, m_file);
+    m_file_pos += r;
     m_buffer_data_left += r;
     m_buffer_ptr = m_buffer;
 

--- 1.32/storage/ndb/tools/restore/Restore.hpp	2007-04-23 13:07:20 +02:00
+++ 1.33/storage/ndb/tools/restore/Restore.hpp	2007-05-19 22:35:26 +02:00
@@ -270,6 +270,10 @@
   void * m_buffer_ptr;
   Uint32 m_buffer_sz;
   Uint32 m_buffer_data_left;
+
+  Uint64 m_file_size;
+  Uint64 m_file_pos;
+
   void (* free_data_callback)();
 
   bool openFile();
@@ -295,6 +299,9 @@
   Uint32 getNodeId() const { return m_nodeId;}
   const BackupFormat::FileHeader & getFileHeader() const { return m_fileHeader;}
   bool Twiddle(const AttributeDesc *  attr_desc, AttributeData * attr_data, Uint32 arraySize = 0);
+
+  Uint64 get_file_size() const { return m_file_size; }
+  Uint64 get_file_pos() const { return m_file_pos; }
 };
 
 struct DictObject {

--- 1.59/storage/ndb/tools/restore/restore_main.cpp	2007-05-08 18:45:38 +02:00
+++ 1.60/storage/ndb/tools/restore/restore_main.cpp	2007-05-19 22:35:27 +02:00
@@ -50,6 +50,8 @@
 const char *opt_ndb_table= NULL;
 unsigned int opt_verbose;
 unsigned int opt_hex_format;
+unsigned int opt_progress_frequency;
+unsigned int g_report_next;
 Vector<BaseString> g_databases;
 Vector<BaseString> g_tables;
 NdbRecordPrintFormat g_ndbrecord_print_format;
@@ -86,6 +88,7 @@
   OPT_FIELDS_OPTIONALLY_ENCLOSED_BY,
   OPT_LINES_TERMINATED_BY,
   OPT_APPEND,
+  OPT_PROGRESS_FREQUENCY,
   OPT_VERBOSE
 };
 static const char *opt_fields_enclosed_by= NULL;
@@ -190,6 +193,10 @@
   { "lines-terminated-by", OPT_LINES_TERMINATED_BY, "",
     (gptr*) &opt_lines_terminated_by, (gptr*) &opt_lines_terminated_by, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+  { "progress-frequency", OPT_PROGRESS_FREQUENCY,
+    "Print status uf restore periodically in given seconds", 
+    (gptr*) &opt_progress_frequency, (gptr*) &opt_progress_frequency, 0,
+    GET_INT, REQUIRED_ARG, 0, 0, 65535, 0, 0, 0 },
   { "verbose", OPT_VERBOSE,
     "verbosity", 
     (gptr*) &opt_verbose, (gptr*) &opt_verbose, 0,
@@ -639,6 +646,37 @@
     exit(code);
 }
 
+static void init_progress()
+{
+  struct timeval the_time;
+  gettimeofday(&the_time, 0);
+  g_report_next = the_time.tv_sec + opt_progress_frequency;
+}
+
+static int check_progress()
+{
+  if (!opt_progress_frequency)
+    return 0;
+  struct timeval the_time;
+  gettimeofday(&the_time, 0);
+  if (the_time.tv_sec >= g_report_next)
+  {
+    g_report_next = the_time.tv_sec + opt_progress_frequency;
+    return 1;
+  }
+  return 0;
+}
+
+static void report_progress(const char *prefix, const BackupFile &f)
+{
+  info.setLevel(255);
+  if (f.get_file_size())
+    info << prefix << (f.get_file_pos() * 100 + f.get_file_size()-1) / f.get_file_size() 
+         << "%(" << f.get_file_pos() << " bytes)\n";
+  else
+    info << prefix << f.get_file_pos() << " bytes\n";
+}
+
 int
 main(int argc, char** argv)
 {
@@ -667,6 +705,9 @@
   g_options.appfmt(" -p %d", ga_nParallelism);
 
   g_connect_string = opt_connect_str;
+
+  init_progress();
+
   /**
    * we must always load meta data, even if we will only print it to stdout
    */
@@ -752,6 +793,13 @@
         err << metaData[i]->getTableName() << " ... Exiting " << endl;
 	exitHandler(NDBT_FAILED);
       } 
+    if (check_progress())
+    {
+      info.setLevel(255);
+      info << "Object create progress: "
+           << i+1 << " objects out of "
+           << metaData.getNoOfObjects() << endl;
+    }
   }
 
   Vector<OutputStream *> table_output(metaData.getNoOfTables());
@@ -807,6 +855,13 @@
           exitHandler(NDBT_FAILED);
         }
     }
+    if (check_progress())
+    {
+      info.setLevel(255);
+      info << "Table create progress: "
+           << i+1 << " tables out of "
+           << metaData.getNoOfTables() << endl;
+    }
   }
   debug << "Close tables" << endl; 
   for(i= 0; i < g_consumers.size(); i++)
@@ -858,6 +913,8 @@
           for(Uint32 j= 0; j < g_consumers.size(); j++) 
             g_consumers[j]->tuple(* tuple, fragmentId);
           ndbout.m_out =  tmp;
+          if (check_progress())
+            report_progress("Data file progress: ", dataIter);
 	} // while (tuple != NULL);
 	
 	if (res < 0)
@@ -905,6 +962,9 @@
           continue;
         for(Uint32 j= 0; j < g_consumers.size(); j++)
           g_consumers[j]->logEntry(* logEntry);
+
+        if (check_progress())
+          report_progress("Log file progress: ", logIter);
       }
       if (res < 0)
       {

--- 1.49/storage/ndb/tools/restore/consumer_restore.cpp	2007-05-08 18:45:38 +02:00
+++ 1.50/storage/ndb/tools/restore/consumer_restore.cpp	2007-05-19 22:35:26 +02:00
@@ -929,6 +929,7 @@
       }
       return false;
     }
+    info.setLevel(254);
     info << "Successfully restored table `"
          << table.getTableName() << "`" << endl;
   }  
@@ -979,6 +980,7 @@
 	dict->dropTable(split[2].c_str());
 	return false;
       }
+      info.setLevel(254);
       info << "Successfully restored table event " << event_name << endl ;
     }
   }
@@ -1489,6 +1491,7 @@
   if (!m_restore)
     return;
 
+  info.setLevel(254);
   info << "Restored " << m_dataCount << " tuples and "
        << m_logCount << " log entries" << endl;
 }
Thread
bk commit into 5.1 tree (tomas:1.2544)tomas19 May