bzr commit into mysql-6.0-falcon-team branch (christopher.powers:2991) Bug#42424

[Christopher Powers]

#At file:///home/cpowers/work/dev/dev-04/mysql/

 2991 Christopher Powers	2009-02-01
      Bug #42424, "Serious performance degradation after new scavenger"
      
      Several fixes for the Scavenger:
      
      1. Don't run updateCardinalities() in a low memory state
      2. Distinguish between scheduled and load-based or 'forced' scavenges
      3. Progessively increase the time that record alloc waits for a scavenge
      4. Retire records during a low memory state, even if active memory is
         below the cache threshold
modified:
  storage/falcon/Database.cpp
  storage/falcon/Database.h
  storage/falcon/Record.cpp
  storage/falcon/RecordScavenge.cpp
  storage/falcon/RecordScavenge.h
  storage/falcon/Table.cpp

per-file messages:
  storage/falcon/Database.cpp
    Database::scavenge()
    - Added 'forced' parameter
    - Skip updateCardinalities() if scavenge is forced
    
    Database::scavengeRecords()
    - Added 'forced' parameter
    - Clear low memory flag if record cache level is nominal
    
    Database::updateCardinalities()
    - Abort cardinality count if forced scavenge is pending
  storage/falcon/Database.h
    Added Database::clearLowMemory()
    Added 'forced' parameter to scavenger methods
  storage/falcon/Record.cpp
    Record::allocRecordData()
    - Added 'forced = true' to signalScavenger()
  storage/falcon/RecordScavenge.cpp
    RecordScavenge::RecordScavenge()
    - Added generation and forced parameters
    
    RecordScavenge::computeThreshold()
    
    If the age group total is less than the memory actually
    in use, assign the current scavenge generation as a starting
    point rather than zero
  storage/falcon/RecordScavenge.h
    Added RecordScavenge::forced
  storage/falcon/Table.cpp
    Specify 'forced = true' for all calls to signalScavenger()
=== modified file 'storage/falcon/Database.cpp'
--- a/storage/falcon/Database.cpp	2009-01-31 23:22:42 +0000
+++ b/storage/falcon/Database.cpp	2009-02-01 08:18:51 +0000
@@ -471,6 +471,7 @@ Database::Database(const char *dbName, C
 	scavengerThread = NULL;
 	scavengerThreadSleeping = 0;
 	scavengerThreadSignaled = 0;
+	scavengeForced = 0;
 	serialLog = NULL;
 	pageWriter = NULL;
 	zombieTables = NULL;
@@ -1730,9 +1731,14 @@ void Database::validate(int optionMask)
 	Log::debug ("Database::validate: validation complete\n");
 }
 
-void Database::scavenge()
+void Database::scavenge(bool forced)
 {
-	signalCardinality();
+	// Signal the cardinality task unless a forced scavenge is pending
+
+	if (!forced)
+		signalCardinality();
+	
+	scavengeForced = 0;
 
 	// Start by scavenging compiled statements.  If they're moldy and not in use,
 	// off with their heads!
@@ -1763,7 +1769,7 @@ void Database::scavenge()
 	transactionManager->purgeTransactions();
 
 	// Scavenge the record cache
-	scavengeRecords();
+	scavengeRecords(forced);
 
 	// Scavenge expired licenses
 	
@@ -1791,15 +1797,16 @@ void Database::scavenge()
 		backLog->reportStatistics();
 }
 
-void Database::scavengeRecords(void)
+void Database::scavengeRecords(bool forced)
 {
 	Sync syncScavenger(&syncScavenge, "Database::scavengeRecords(Scavenge)");
 	syncScavenger.lock(Exclusive);
 
-	// Create an object to track this record scavenge cycle.
-
-	RecordScavenge recordScavenge(this);
+	// Create an object to track this record scavenge cycle. Scavenge up to and
+	// including the current generation.
 
+	RecordScavenge recordScavenge(this, currentGeneration, forced);
+	
 	// Take inventory of the record cache and prune invisible record versions
 
 	pruneRecords(&recordScavenge);
@@ -1814,10 +1821,12 @@ void Database::scavengeRecords(void)
 	recordScavenge.retiredActiveMemory = recordDataPool->activeMemory;
 	recordScavenge.retireStop = deltaTime;
 
-	// Check for low memory 
+	// Enable backlogging if memory is low
 
 	if (recordScavenge.retiredActiveMemory > recordScavengeFloor)
 		setLowMemory();
+	else
+		clearLowMemory();
 
 	recordScavenge.print();
 	// Log::log(analyze(analyzeRecordLeafs));
@@ -1864,9 +1873,11 @@ void Database::pruneRecords(RecordScaven
 
 void Database::retireRecords(RecordScavenge *recordScavenge)
 {
-	// If we passed the upper limit, scavenge.
+	// Scavenge if we passed the upper limit or if a forced scavenge
+	// was requested.
 
-	if (recordDataPool->activeMemory < recordScavengeThreshold)
+	if (recordDataPool->activeMemory < recordScavengeThreshold
+		&& !recordScavenge->forced)
 		return;
 
 	//LogStream stream;
@@ -1927,7 +1938,7 @@ void Database::scavengerThreadMain(void)
 
 	while (!thread->shutdownInProgress)
 		{
-		scavenge();
+		scavenge((scavengeForced > 0));
 		
 		if (recordDataPool->activeMemory < recordScavengeThreshold)
 			{
@@ -2467,7 +2478,11 @@ void Database::updateCardinalities(void)
 	
 	try
 		{
-		for (Table *table = tableList; table; table = table->next)
+		
+		// Establish the record cardinality for each table. Abandon the effort
+		// if a forced scavenge operation is pending.
+		
+		for (Table *table = tableList; (table && scavengeForced == 0); table =
table->next)
 			{
 			uint64 cardinality = table->cardinality;
 			
@@ -2611,7 +2626,7 @@ void Database::checkRecordScavenge(void)
 
 // Signal the scavenger thread
 
-void Database::signalScavenger(void)
+void Database::signalScavenger(bool force)
 {
 	Sync syncMem(&syncMemory, "Database::signalScavenger");
 	syncMem.lock(Exclusive);
@@ -2619,6 +2634,10 @@ void Database::signalScavenger(void)
 	if (scavengerThreadSleeping && !scavengerThreadSignaled)
 		{
 		INTERLOCKED_INCREMENT(scavengerThreadSignaled);
+		
+		if (force)
+			INTERLOCKED_INCREMENT(scavengeForced);
+		
 		scavengerThreadWakeup();
 		}
 }
@@ -2734,3 +2753,8 @@ void Database::setLowMemory(void)
 
 	lowMemory = true;
 }
+
+void Database::clearLowMemory(void)
+{
+	lowMemory = false;
+}

=== modified file 'storage/falcon/Database.h'
--- a/storage/falcon/Database.h	2009-01-27 17:32:40 +0000
+++ b/storage/falcon/Database.h	2009-02-01 08:18:51 +0000
@@ -40,6 +40,8 @@
 #define VERSION_CURRENT					COMBINED_VERSION(ODS_VERSION, ODS_MINOR_VERSION)					
 #define VERSION_SERIAL_LOG				COMBINED_VERSION(ODS_VERSION2, ODS_MINOR_VERSION1)
 
+#define SCAVENGE_WAIT_MS   10       // Milliseconds per iteration to wait for the
Scavenger
+
 static const int FALC0N_TRACE_TRANSACTIONS	= 1;
 static const int FALC0N_SYNC_TEST			= 2;
 static const int FALC0N_SYNC_OBJECTS		= 4;
@@ -132,7 +134,7 @@ public:
 	const char*		fetchTemplate (JString applicationName, JString templateName,
TemplateContext *context);
 	void			licenseCheck();
 	void			serverOperation (int op, Parameters *parameters);
-	void			scavengeRecords(void);
+	void			scavengeRecords(bool forced = false);
 	void			pruneRecords(RecordScavenge* recordScavenge);
 	void			retireRecords(RecordScavenge* recordScavenge);
 	int				getMemorySize (const char *string);
@@ -159,7 +161,7 @@ public:
 	static void		scavengerThreadMain(void * database);
 	void			scavengerThreadMain(void);
 	void			scavengerThreadWakeup(void);
-	void			scavenge();
+	void			scavenge(bool forced = false);
 	void			validate (int optionMask);
 	Role*			findRole(const char *schemaName, const char * roleName);
 	User*			findUser (const char *account);
@@ -233,7 +235,7 @@ public:
 	void			setRecordScavengeFloor(int value);
 	void			checkRecordScavenge(void);
 	void			signalCardinality(void);
-	void			signalScavenger(void);
+	void			signalScavenger(bool force = false);
 	void			debugTrace(void);
 	void			pageCacheFlushed(int64 flushArg);
 	JString			setLogRoot(const char *defaultPath, bool create);
@@ -241,6 +243,7 @@ public:
 	void			clearIOError(void);
 	void			flushWait(void);
 	void			setLowMemory(void);
+	void			clearLowMemory(void);
 
 
 	Dbb					*dbb;
@@ -313,6 +316,7 @@ public:
 	volatile INTERLOCK_TYPE	cardinalityThreadSignaled;
 	volatile INTERLOCK_TYPE	scavengerThreadSleeping;
 	volatile INTERLOCK_TYPE	scavengerThreadSignaled;
+	volatile INTERLOCK_TYPE	scavengeForced;
 	PageWriter			*pageWriter;
 	PreparedStatement	*updateCardinality;
 	MemMgr				*recordDataPool;

=== modified file 'storage/falcon/Record.cpp'
--- a/storage/falcon/Record.cpp	2009-01-14 22:33:44 +0000
+++ b/storage/falcon/Record.cpp	2009-02-01 08:18:51 +0000
@@ -953,7 +953,7 @@ char* Record::allocRecordData(int length
 				|| n > OUT_OF_RECORD_MEMORY_RETRIES)
 				throw;
 			
-			format->table->database->signalScavenger();
+			format->table->database->signalScavenger(true);
 
 			// Give the scavenger thread a chance to release some memory
 

=== modified file 'storage/falcon/RecordScavenge.cpp'
--- a/storage/falcon/RecordScavenge.cpp	2009-01-14 22:33:44 +0000
+++ b/storage/falcon/RecordScavenge.cpp	2009-02-01 08:18:51 +0000
@@ -25,24 +25,22 @@
 #include "Transaction.h"
 #include "TransactionManager.h"
 
-RecordScavenge::RecordScavenge(Database *db)
+RecordScavenge::RecordScavenge(Database *db, uint64 generation, bool forceScavenge)
+								: database(db), baseGeneration(generation), forced(forceScavenge)
 {
-	database = db;
 	cycle = ++database->scavengeCycle;
 
 	memset(ageGroups, 0, sizeof(ageGroups));
 	veryOldRecords = 0;
 	veryOldRecordSpace = 0;
 
-	startingActiveMemory = db->recordDataPool->activeMemory;
+	startingActiveMemory = database->recordDataPool->activeMemory;
 	prunedActiveMemory = 0;
 	retiredActiveMemory = 0;
 
-	scavengeStart = db->deltaTime;
+	scavengeStart = database->deltaTime;
 	pruneStop = 0;
 	retireStop = 0;
-
-	baseGeneration = database->currentGeneration;
 	scavengeGeneration = 0;
 
 	// Results of Scavenging
@@ -63,10 +61,10 @@ RecordScavenge::RecordScavenge(Database 
 	unScavengeableRecords = 0;
 	unScavengeableSpace = 0;
 
-	Sync syncActive(&db->transactionManager->activeTransactions.syncObject,
"RecordScavenge::RecordScavenge");
+	Sync syncActive(&database->transactionManager->activeTransactions.syncObject,
"RecordScavenge::RecordScavenge");
 	syncActive.lock(Shared);
 
-	oldestActiveTransaction = db->transactionManager->findOldestInActiveList();
+	oldestActiveTransaction = database->transactionManager->findOldestInActiveList();
 }
 
 RecordScavenge::~RecordScavenge(void)
@@ -251,7 +249,10 @@ uint64 RecordScavenge::computeThreshold(
 			scavengeGeneration = baseGeneration - n;
 		}
 
-	return scavengeGeneration;
+	// We still may want to scavenge even if the age group total is
+	// too small, so use the base generation as a starting point.
+	
+	return (scavengeGeneration ? scavengeGeneration : baseGeneration);
 }
 
 void RecordScavenge::print(void)
@@ -272,8 +273,10 @@ void RecordScavenge::print(void)
 
 	Log::log (LogScavenge,"Cycle=" I64FORMAT 
 		"  Base Generation=" I64FORMAT 
-		"  Scavenge Generation=" I64FORMAT "\n", 
-		cycle, baseGeneration, scavengeGeneration);
+		"  Scavenge Generation=" I64FORMAT
+		"  Forced=%d"
+		"  Low Memory=%d\n", 
+		cycle, baseGeneration, scavengeGeneration, (int)forced, (int)database->lowMemory);
 	Log::log (LogScavenge,"Cycle=" I64FORMAT 
 		"  Oldest Active Transaction=%d\n", 
 		cycle, oldestActiveTransaction);

=== modified file 'storage/falcon/RecordScavenge.h'
--- a/storage/falcon/RecordScavenge.h	2009-01-08 09:05:26 +0000
+++ b/storage/falcon/RecordScavenge.h	2009-02-01 08:18:51 +0000
@@ -29,7 +29,7 @@ class Record;
 class RecordScavenge
 {
 public:
-	RecordScavenge(Database *db);
+	RecordScavenge(Database *db, uint64 generation, bool forceScavenge = false);
 	~RecordScavenge(void);
 
 	bool     canBeRetired(Record* record);
@@ -71,6 +71,8 @@ public:
 	uint64		ageGroups[AGE_GROUPS];
 	uint64		veryOldRecords;
 	uint64		veryOldRecordSpace;
+	
+	bool		forced;
 };
 
 #endif

=== modified file 'storage/falcon/Table.cpp'
--- a/storage/falcon/Table.cpp	2009-01-28 19:19:44 +0000
+++ b/storage/falcon/Table.cpp	2009-02-01 08:18:51 +0000
@@ -3642,12 +3642,13 @@ RecordVersion* Table::allocRecordVersion
 				|| n > OUT_OF_RECORD_MEMORY_RETRIES)
 				throw;
 
-			database->signalScavenger();
+			database->signalScavenger(true);
 
-			// Give the scavenger thread a chance to release some memory
+			// Give the scavenger thread a chance to release memory.
+			// Increase the wait time per iteration.
 
 			Thread *thread = Thread::getThread("Database::ticker");
-			thread->sleep(10);
+			thread->sleep(n * SCAVENGE_WAIT_MS);
 			}
 		}
 
@@ -3678,7 +3679,7 @@ Record* Table::allocRecord(int recordNum
 				|| n > OUT_OF_RECORD_MEMORY_RETRIES)
 				throw;
 
-			database->signalScavenger();
+			database->signalScavenger(true);
 
 			// Give the scavenger thread a chance to release some memory