List:Internals« Previous MessageNext Message »
From:sasha Date:March 27 2002 5:19am
Subject:bk commit into 4.0 tree
View as plain text  
Below is the list of changes that have just been committed into a
4.0 repository of sasha. When sasha does a push, they will be propogated to 
the main repository and within 24 hours after the push to the public repository. 
For information on how to access the public repository
see http://www.mysql.com/doc/I/n/Installing_source_tree.html

ChangeSet@stripped, 2002-03-26 22:19:23-07:00, sasha@stripped
  coverted my_thread_init() /end to use my_malloc()/my_free() to help track
  down replication corruption

  dbug/dbug.c
    1.11 02/03/26 22:19:22 sasha@stripped +22 -6
    fixed DBUG_ macros to not crash if my_thread_init() had not been called

  include/my_sys.h
    1.69 02/03/26 22:19:22 sasha@stripped +4 -0
    keep track of shutdown, main, and signal threads for safemalloc

  mysys/my_static.h
    1.8 02/03/26 22:19:22 sasha@stripped +6 -0
    keep track of thread in safemalloc blocks

  mysys/my_thr_init.c
    1.13 02/03/26 22:19:22 sasha@stripped +34 -5
    use my_malloc()/my_free() in my_thread_init()

  mysys/safemalloc.c
    1.9 02/03/26 22:19:22 sasha@stripped +69 -1
    keep track of threads
    do not fuss if main,signal, or shutdown thread still has allocated blocks 
     when we call TERMINATE() as some blocks are freed later and there is no
     easy way to make TERMINATE() wait for those threads

  sql/mysqld.cc
    1.286 02/03/26 22:19:22 sasha@stripped +21 -2
    keep track of signal, main, and shutdown threads

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	sasha
# Host:	mysql.sashanet.com
# Root:	/reiser-data/mysql-4.0

--- 1.10/dbug/dbug.c	Tue Jan 22 17:52:25 2002
+++ 1.11/dbug/dbug.c	Tue Mar 26 22:19:22 2002
@@ -21,7 +21,8 @@
  *	all copies and derivative works.  Thank you.			      *
  *									      *
  *	The author makes no warranty of any kind  with	respect  to  this     *
- *	product  and  explicitly disclaims any implied warranties of mer-     *
+ *	product  and  explicitly disclaims any implied warranties of mer-     *ct_lex.table_list.first=0;
+  thd->lex.selec
  *	chantability or fitness for any particular purpose.		      *
  *									      *
  ******************************************************************************
@@ -58,7 +59,7 @@
  *	seismo!bpa!sjuvax!bbanerje
  *
  *	Michael Widenius:
- *	DBUG_DUMP	- To dump a pice of memory.
+ *	DBUG_DUMP	- To dump a block of memory.
  *	PUSH_FLAG "O"	- To be used insted of "o" if we don't
  *			  want flushing (for slow systems)
  *	PUSH_FLAG "A"	- as 'O', but we will append to the out file instead
@@ -707,7 +708,13 @@
     int save_errno=errno;
     if (!init_done)
       _db_push_ (_DBUG_START_CONDITION_);
-    state=code_state();
+    /* Sasha: the test below is so we could call functions with DBUG_ENTER
+       before my_thread_init(). I needed this because I suspected corruption
+       of a block allocated by my_thread_init() itself, so I wanted to use
+       my_malloc()/my_free() in my_thread_init()/my_thread_end()
+    */
+    if (!(state=code_state()))
+      return;
 
     *_sfunc_ = state->func;
     *_sfile_ = state->file;
@@ -855,6 +862,9 @@
 const char *keyword)
 {
   CODE_STATE *state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!state)
+    return;
   state->u_line = _line_;
   state->u_keyword = (char*) keyword;
 }
@@ -890,7 +900,9 @@
 {
   va_list args;
   CODE_STATE *state;
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return;
 
   va_start(args,format);
 
@@ -942,7 +954,9 @@
   int pos;
   char dbuff[90];
   CODE_STATE *state;
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return;
 
   if (_db_keyword_ ((char*) keyword))
   {
@@ -1224,7 +1238,9 @@
 
   if (!init_done)
     _db_push_ ("");
-  state=code_state();
+  /* Sasha: pre-my_thread_init() safety */
+  if (!(state=code_state()))
+    return FALSE;
   result = FALSE;
   if (DEBUGGING &&
       state->level <= stack -> maxdepth &&

--- 1.68/include/my_sys.h	Fri Mar  8 09:33:14 2002
+++ 1.69/include/my_sys.h	Tue Mar 26 22:19:22 2002
@@ -137,6 +137,10 @@
 #define NORMAL_SAFEMALLOC sf_malloc_quick=0
 extern uint sf_malloc_prehunc,sf_malloc_endhunc,sf_malloc_quick;
 extern ulonglong safemalloc_mem_limit;
+/* keep track of shutdown,signal, and main threads so that my_end() will not
+   report errors with them
+*/
+extern pthread_t shutdown_th, main_th,signal_th;
 #define CALLER_INFO_PROTO   , const char *sFile, uint uLine
 #define CALLER_INFO         , __FILE__, __LINE__
 #define ORIG_CALLER_INFO    , sFile, uLine

--- 1.7/mysys/my_static.h	Thu Dec  6 05:10:43 2001
+++ 1.8/mysys/my_static.h	Tue Mar 26 22:19:22 2002
@@ -38,6 +38,7 @@
     my_string _sFileName;		/* File in which memory was new'ed */
     uint _uLineNum;			/* Line number in above file	   */
     uint _uDataSize;			/* Size requested		   */
+    pthread_t thread_id;
     long _lSpecialValue;		/* Underrun marker value	   */
 };
 
@@ -55,6 +56,11 @@
 
 extern USED_MEM* my_once_root_block;
 extern uint	 my_once_extra;
+
+/* these threads are exept from safemalloc leak scrutiny unless
+   PEDANTIC_SAFEMALLOC is defined
+*/
+extern pthread_t signal_thread,kill_thread;
 
 #ifndef HAVE_TEMPNAM
 extern int	_my_tempnam_used;

--- 1.12/mysys/my_thr_init.c	Thu Dec  6 05:10:43 2001
+++ 1.13/mysys/my_thr_init.c	Tue Mar 26 22:19:22 2002
@@ -105,19 +105,33 @@
 my_bool my_thread_init(void)
 {
   struct st_my_thread_var *tmp;
+#ifdef EXTRA_DEBUG
+  fprintf(stderr,"my_thread_init(): thread_id=%ld\n",pthread_self());
+#endif  
 #if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX)
   pthread_mutex_lock(&THR_LOCK_lock);
 #endif
 #if !defined(__WIN__) || defined(USE_TLS)
   if (my_pthread_getspecific(struct st_my_thread_var *,THR_KEY_mysys))
   {
+#ifdef EXTRA_DEBUG
+    fprintf(stderr,"my_thread_init() called more than once in thread %ld\n",
+	    pthread_self());
+#endif    
     pthread_mutex_unlock(&THR_LOCK_lock);
     return 0;						/* Safequard */
   }
     /* We must have many calloc() here because these are freed on
        pthread_exit */
+    /*
+      Sasha: the above comment does not make sense. I have changed calloc() to
+      equivalent my_malloc() but it was calloc() before. It seems like the
+      comment is out of date - we always call my_thread_end() before
+      pthread_exit() to clean up. Note that I have also fixed up DBUG
+      code to be able to call it from my_thread_init()
+     */
   if (!(tmp=(struct st_my_thread_var *)
-	calloc(1,sizeof(struct st_my_thread_var))))
+	my_malloc(sizeof(struct st_my_thread_var),MYF(MY_WME|MY_ZEROFILL))))
   {
     pthread_mutex_unlock(&THR_LOCK_lock);
     return 1;
@@ -125,6 +139,9 @@
   pthread_setspecific(THR_KEY_mysys,tmp);
 
 #else
+  /* Sasha: TODO - explain what exactly we are doing on Windows
+     At first glance, I have a hard time following the code
+   */
   if (THR_KEY_mysys.id)   /* Already initialized */
   {
 #if !defined(__WIN__) || defined(USE_TLS) || ! defined(SAFE_MUTEX)
@@ -146,9 +163,18 @@
 void my_thread_end(void)
 {
   struct st_my_thread_var *tmp=my_thread_var;
+#ifdef EXTRA_DEBUG
+  fprintf(stderr,"my_thread_end(): tmp=%p,thread_id=%ld\n",
+	  tmp,pthread_self());
+#endif  
   if (tmp)
   {
 #if !defined(DBUG_OFF)
+    /* Sasha:  tmp->dbug is allocated inside DBUG library
+       so for now we will not mess with trying to use my_malloc()/
+       my_free(), but in the future it would be nice to figure out a
+       way to do it
+    */
     if (tmp->dbug)
     {
       free(tmp->dbug);
@@ -160,12 +186,15 @@
 #endif
     pthread_mutex_destroy(&tmp->mutex);
 #if (!defined(__WIN__) && !defined(OS2)) || defined(USE_TLS)
-    free(tmp);
+    /* we need to setspecific to 0 BEFORE we call my_free, as my_free
+       uses some DBUG_ macros that will use the follow the specific
+       pointer after the block it is pointing to has been freed if
+       specific does not get reset first
+    */
+    pthread_setspecific(THR_KEY_mysys,0);
+    my_free((gptr)tmp,MYF(MY_WME));
 #endif
   }
-#if (!defined(__WIN__) && !defined(OS2)) || defined(USE_TLS)
-  pthread_setspecific(THR_KEY_mysys,0);
-#endif
 }
 
 struct st_my_thread_var *_my_thread_var(void)

--- 1.8/mysys/safemalloc.c	Thu Dec  6 05:10:43 2001
+++ 1.9/mysys/safemalloc.c	Tue Mar 26 22:19:22 2002
@@ -73,14 +73,25 @@
 #include "mysys_err.h"
 
 ulonglong safemalloc_mem_limit = ~(ulonglong)0;
+pthread_t shutdown_th=0,main_th=0,signal_th=0;
 
 #define pNext		tInt._pNext
 #define pPrev		tInt._pPrev
 #define sFileName	tInt._sFileName
 #define uLineNum	tInt._uLineNum
 #define uDataSize	tInt._uDataSize
+#define thread_id       tInt.thread_id
 #define lSpecialValue	tInt._lSpecialValue
 
+#ifndef PEDANTIC_SAFEMALLOC
+static int sf_malloc_tampered = 0; /* set to 1 after TERMINATE() if we had
+				    to fiddle with cNewCount and the linked
+				    list of blocks so that _sanity() will
+				    not fuss when it is not supposed to
+				 */
+#endif
+				   
+
 	/* Static functions prototypes */
 
 static int check_ptr(const char *where, byte *ptr, const char *sFile,
@@ -174,6 +185,7 @@
     pTmp -> sFileName = (my_string) sFile;
     pTmp -> uLineNum = uLine;
     pTmp -> uDataSize = uSize;
+    pTmp->thread_id = pthread_self();
     pTmp -> pPrev = NULL;
 
     /* Add this remember structure to the linked list */
@@ -359,6 +371,12 @@
   return 0;
 }
 
+static int legal_leak(struct remember* pPtr)
+{
+ return pthread_self() == pPtr->thread_id || main_th == pPtr->thread_id
+	    || shutdown_th == pPtr->thread_id
+   || signal_th == pPtr->thread_id;
+}
 
 /*
  * TERMINATE(FILE *file)
@@ -376,6 +394,47 @@
   /* NEW and the number of calls to FREE.  >0 means more	 */
   /* NEWs than FREEs.  <0, etc.				 */
 
+#ifndef PEDANTIC_SAFEMALLOC
+  /* Avoid false alarms for blocks that we cannot free before my_end()
+     This does miss some positives, but that is ok. This will only miss
+     failures to free things allocated in the main thread which 
+     performs only one-time allocations. If you really need to
+     debug memory allocations in the main thread,
+     #define PEDANTIC_SAFEMALLOC
+  */
+  if ((pPtr=pRememberRoot))
+  {
+    while (pPtr)
+    {
+      if (legal_leak(pPtr))
+      {
+	sf_malloc_tampered=1;
+	cNewCount--;
+	lCurMemory -= pPtr->uDataSize;
+	if (pPtr->pPrev)
+	{
+	  struct remember* tmp;
+	  tmp = pPtr->pPrev->pNext = pPtr->pNext;
+	  if (tmp)
+	   tmp->pPrev = pPtr->pPrev; 
+	  pPtr->pNext = pPtr->pPrev = 0;
+	  pPtr = tmp;
+	}
+	else
+	{
+	  pRememberRoot = pPtr->pNext;
+	  pPtr->pNext = pPtr->pPrev = 0;
+	  pPtr = pRememberRoot;
+	  if (pPtr)
+	    pPtr->pPrev=0;
+	}
+      }
+      else
+	pPtr = pPtr->pNext;
+    }
+  }
+#endif  
+  
   if (cNewCount)
   {
     if (file)
@@ -402,10 +461,14 @@
       if (file)
       {
 	fprintf (file,
-		 "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'\n",
+		 "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'",
 		 pPtr -> uDataSize,
 		 (ulong) &(pPtr -> aData[sf_malloc_prehunc]),
 		 pPtr -> uLineNum, pPtr -> sFileName);
+#ifdef THREAD
+	fprintf(file, " in thread %ld", pPtr->thread_id);
+#endif
+	fprintf(file, "\n");
 	(void) fflush(file);
       }
       DBUG_PRINT("safe",
@@ -484,6 +547,10 @@
   uint count=0;
 
   pthread_mutex_lock(&THR_LOCK_malloc);
+#ifndef PEDANTIC_SAFEMALLOC  
+  if (sf_malloc_tampered && cNewCount < 0)
+    cNewCount=0;
+#endif  
   count=cNewCount;
   for (pTmp = pRememberRoot; pTmp != NULL && count-- ; pTmp = pTmp -> pNext)
     flag+=_checkchunk (pTmp, sFile, uLine);
@@ -492,6 +559,7 @@
   {
     const char *format="Safemalloc link list destroyed, discovered at '%s:%d'";
     fprintf (stderr, format, sFile, uLine); fputc('\n',stderr);
+    fprintf (stderr, "root=%p,count=%d,pTmp=%p\n", pRememberRoot,count,pTmp);
     (void) fflush(stderr);
     DBUG_PRINT("safe",(format, sFile, uLine));
     flag=1;

--- 1.285/sql/mysqld.cc	Fri Mar 15 18:44:43 2002
+++ 1.286/sql/mysqld.cc	Tue Mar 26 22:19:22 2002
@@ -38,7 +38,17 @@
 #define ONE_THREAD
 #endif
 
-/* do stack traces are only supported on linux intel */
+#ifdef SAFEMALLOC
+#define SHUTDOWN_THD shutdown_th=pthread_self();
+#define MAIN_THD main_th=pthread_self();
+#define SIGNAL_THD signal_th=pthread_self();
+#else
+#define SHUTDOWN_THD
+#define MAIN_THD
+#define SIGNAL_THD
+#endif
+
+/* stack traces are only supported on linux intel */
 #if defined(__linux__)  && defined(__i386__) && defined(USE_PSTACK)
 #define	HAVE_STACK_TRACE_ON_SEGV
 #include "../pstack/pstack.h"
@@ -694,6 +704,7 @@
     sql_print_error(ER(ER_GOT_SIGNAL),my_progname,sig); /* purecov: inspected */
 
 #if defined(USE_ONE_SIGNAL_HAND) && !defined(__WIN__) && !defined(OS2)
+  SHUTDOWN_THD;
   my_thread_init();				// If this is a new thread
 #endif
   close_connections();
@@ -709,6 +720,7 @@
 #ifdef USE_ONE_SIGNAL_HAND
 static pthread_handler_decl(kill_server_thread,arg __attribute__((unused)))
 {
+  SHUTDOWN_THD;
   my_thread_init();				// Initialize new thread
   kill_server(0);
   my_thread_end();				// Normally never reached
@@ -1252,6 +1264,7 @@
   signal(SIGALRM, SIG_IGN);
   signal(SIGBREAK,SIG_IGN);
   signal_thread = pthread_self();
+  SIGNAL_THD;
 }
 
 static void start_signal_handler(void)
@@ -1445,7 +1458,7 @@
   int sig;
   my_thread_init();				// Init new thread
   DBUG_ENTER("signal_hand");
-
+  SIGNAL_THD;
   /* Setup alarm handler */
   init_thr_alarm(max_connections+max_insert_delayed_threads);
 #if SIGINT != THR_KILL_SIGNAL
@@ -1500,7 +1513,10 @@
     else
       while ((error=my_sigwait(&set,&sig)) == EINTR) ;
     if (cleanup_done)
+    {
+      my_thread_end();
       pthread_exit(0);				// Safety
+    }
     switch (sig) {
     case SIGTERM:
     case SIGQUIT:
@@ -1594,6 +1610,7 @@
 pthread_handler_decl(handle_shutdown,arg)
 {
   MSG msg;
+  SHUTDOWN_THD;
   my_thread_init();
 
   /* this call should create the message queue for this thread */
@@ -1620,6 +1637,7 @@
 #ifdef OS2
 pthread_handler_decl(handle_shutdown,arg)
 {
+  SHUTDOWN_THD;
   my_thread_init();
 
   // wait semaphore
@@ -1691,6 +1709,7 @@
 
   my_umask=0660;		// Default umask for new files
   my_umask_dir=0700;		// Default umask for new directories
+  MAIN_THD;
   MY_INIT(argv[0]);		// init my_sys library & pthreads
   tzset();			// Set tzname
 
Thread
bk commit into 4.0 treesasha27 Mar