2993 Christopher Powers 2009-02-01
Bug #42510, "Falcon: Default memory parameters must be changed"
Bug #36442, "Falcon: crash during optimize table"
Increased default page cache size from 4MB to 250MB
Increased record scavenge threshold from 67 to 90
Increased record scavenge floor from 50 to 80
modified:
storage/falcon/SerialLog.cpp
storage/falcon/StorageParameters.h
storage/falcon/ha_falcon.cpp
2992 Christopher Powers 2009-02-01
Bug #42424, "Serious performance degradation after new scavenger"
Increased scavenger wait increment from 10 to 20ms.
Use progressive scavenger wait timeout for record allocation methods.
modified:
storage/falcon/Database.cpp
storage/falcon/Database.h
storage/falcon/Record.cpp
storage/falcon/Table.cpp
2991 Christopher Powers 2009-02-01
Bug #42424, "Serious performance degradation after new scavenger"
Several fixes for the Scavenger:
1. Don't run updateCardinalities() in a low memory state
2. Distinguish between scheduled and load-based or 'forced' scavenges
3. Progessively increase the time that record alloc waits for a scavenge
4. Retire records during a low memory state, even if active memory is
below the cache threshold
modified:
storage/falcon/Database.cpp
storage/falcon/Database.h
storage/falcon/Record.cpp
storage/falcon/RecordScavenge.cpp
storage/falcon/RecordScavenge.h
storage/falcon/Table.cpp
2990 Christopher Powers 2009-01-31
Bug #42505, "Falcon: Record backlogging not enabled by the Scavenger"
Fixed several issues with record backlogging.
modified:
storage/falcon/BackLog.cpp
storage/falcon/Database.cpp
storage/falcon/TableSpaceManager.cpp
storage/falcon/TableSpaceManager.h
2989 Vladislav Vaintroub 2009-01-30
remove unintentionally introduced new assert
modified:
storage/falcon/RecordLocatorPage.cpp
=== modified file 'storage/falcon/BackLog.cpp'
--- a/storage/falcon/BackLog.cpp 2008-08-18 20:17:15 +0000
+++ b/storage/falcon/BackLog.cpp 2009-01-31 23:22:42 +0000
@@ -16,6 +16,7 @@
#include "Engine.h"
#include "BackLog.h"
#include "Database.h"
+#include "TableSpaceManager.h"
#include "Dbb.h"
#include "Section.h"
#include "Index.h"
@@ -33,16 +34,15 @@
static const char THIS_FILE[]=__FILE__;
#endif
-BackLog::BackLog(Database *db, const char *fileName)
+BackLog::BackLog(Database *db, const char *fileName) : database(db)
{
- database = db;
dbb = new Dbb(database->dbb, 0);
#ifndef FALCONDB
dbb->createPath(fileName);
#endif
dbb->create(fileName, dbb->pageSize, 0, HdrTableSpace, 0, NULL);
dbb->noLog = true;
- dbb->tableSpaceId = -1;
+ dbb->tableSpaceId = database->tableSpaceManager->createTableSpaceId();
int32 sectionId = Section::createSection (dbb, NO_TRANSACTION);
section = new Section(dbb, sectionId, NO_TRANSACTION);
recordsBacklogged = 0;
=== modified file 'storage/falcon/Database.cpp'
--- a/storage/falcon/Database.cpp 2009-01-27 17:32:40 +0000
+++ b/storage/falcon/Database.cpp 2009-02-01 09:00:52 +0000
@@ -471,6 +471,7 @@ Database::Database(const char *dbName, C
scavengerThread = NULL;
scavengerThreadSleeping = 0;
scavengerThreadSignaled = 0;
+ scavengeForced = 0;
serialLog = NULL;
pageWriter = NULL;
zombieTables = NULL;
@@ -1730,9 +1731,14 @@ void Database::validate(int optionMask)
Log::debug ("Database::validate: validation complete\n");
}
-void Database::scavenge()
+void Database::scavenge(bool forced)
{
- signalCardinality();
+ // Signal the cardinality task unless a forced scavenge is pending
+
+ if (!forced)
+ signalCardinality();
+
+ scavengeForced = 0;
// Start by scavenging compiled statements. If they're moldy and not in use,
// off with their heads!
@@ -1763,7 +1769,7 @@ void Database::scavenge()
transactionManager->purgeTransactions();
// Scavenge the record cache
- scavengeRecords();
+ scavengeRecords(forced);
// Scavenge expired licenses
@@ -1791,15 +1797,16 @@ void Database::scavenge()
backLog->reportStatistics();
}
-void Database::scavengeRecords(void)
+void Database::scavengeRecords(bool forced)
{
Sync syncScavenger(&syncScavenge, "Database::scavengeRecords(Scavenge)");
syncScavenger.lock(Exclusive);
- // Create an object to track this record scavenge cycle.
-
- RecordScavenge recordScavenge(this);
+ // Create an object to track this record scavenge cycle. Scavenge up to and
+ // including the current generation.
+ RecordScavenge recordScavenge(this, currentGeneration, forced);
+
// Take inventory of the record cache and prune invisible record versions
pruneRecords(&recordScavenge);
@@ -1814,10 +1821,12 @@ void Database::scavengeRecords(void)
recordScavenge.retiredActiveMemory = recordDataPool->activeMemory;
recordScavenge.retireStop = deltaTime;
- // Check for low memory
+ // Enable backlogging if memory is low
- if (recordScavenge.spaceRemaining > recordScavengeFloor)
+ if (recordScavenge.retiredActiveMemory > recordScavengeFloor)
setLowMemory();
+ else
+ clearLowMemory();
recordScavenge.print();
// Log::log(analyze(analyzeRecordLeafs));
@@ -1864,9 +1873,11 @@ void Database::pruneRecords(RecordScaven
void Database::retireRecords(RecordScavenge *recordScavenge)
{
- // If we passed the upper limit, scavenge.
+ // Scavenge if we passed the upper limit or if a forced scavenge
+ // was requested.
- if (recordDataPool->activeMemory < recordScavengeThreshold)
+ if (recordDataPool->activeMemory < recordScavengeThreshold
+ && !recordScavenge->forced)
return;
//LogStream stream;
@@ -1927,7 +1938,7 @@ void Database::scavengerThreadMain(void)
while (!thread->shutdownInProgress)
{
- scavenge();
+ scavenge((scavengeForced > 0));
if (recordDataPool->activeMemory < recordScavengeThreshold)
{
@@ -2463,11 +2474,16 @@ void Database::updateCardinalities(void)
Sync syncTbl(&syncTables, "Database::updateCardinalities(2)");
syncTbl.lock(Shared);
+ Log::log("Update cardinalities begin...\n");
bool hit = false;
try
{
- for (Table *table = tableList; table; table = table->next)
+
+ // Establish the record cardinality for each table. Abandon the effort
+ // if a forced scavenge operation is pending.
+
+ for (Table *table = tableList; (table && scavengeForced == 0); table = table->next)
{
uint64 cardinality = table->cardinality;
@@ -2507,6 +2523,8 @@ void Database::updateCardinalities(void)
// Situations where this might happen can be due to problems with
// writing to the serial log
}
+
+ Log::log("Update cardinalities complete.\n");
}
void Database::sync()
@@ -2611,7 +2629,7 @@ void Database::checkRecordScavenge(void)
// Signal the scavenger thread
-void Database::signalScavenger(void)
+void Database::signalScavenger(bool force)
{
Sync syncMem(&syncMemory, "Database::signalScavenger");
syncMem.lock(Exclusive);
@@ -2619,6 +2637,10 @@ void Database::signalScavenger(void)
if (scavengerThreadSleeping && !scavengerThreadSignaled)
{
INTERLOCKED_INCREMENT(scavengerThreadSignaled);
+
+ if (force)
+ INTERLOCKED_INCREMENT(scavengeForced);
+
scavengerThreadWakeup();
}
}
@@ -2734,3 +2756,8 @@ void Database::setLowMemory(void)
lowMemory = true;
}
+
+void Database::clearLowMemory(void)
+{
+ lowMemory = false;
+}
=== modified file 'storage/falcon/Database.h'
--- a/storage/falcon/Database.h 2009-01-27 17:32:40 +0000
+++ b/storage/falcon/Database.h 2009-02-01 09:00:52 +0000
@@ -40,6 +40,8 @@
#define VERSION_CURRENT COMBINED_VERSION(ODS_VERSION, ODS_MINOR_VERSION)
#define VERSION_SERIAL_LOG COMBINED_VERSION(ODS_VERSION2, ODS_MINOR_VERSION1)
+#define SCAVENGE_WAIT_MS 20 // Milliseconds per iteration to wait for the Scavenger
+
static const int FALC0N_TRACE_TRANSACTIONS = 1;
static const int FALC0N_SYNC_TEST = 2;
static const int FALC0N_SYNC_OBJECTS = 4;
@@ -132,7 +134,7 @@ public:
const char* fetchTemplate (JString applicationName, JString templateName, TemplateContext *context);
void licenseCheck();
void serverOperation (int op, Parameters *parameters);
- void scavengeRecords(void);
+ void scavengeRecords(bool forced = false);
void pruneRecords(RecordScavenge* recordScavenge);
void retireRecords(RecordScavenge* recordScavenge);
int getMemorySize (const char *string);
@@ -159,7 +161,7 @@ public:
static void scavengerThreadMain(void * database);
void scavengerThreadMain(void);
void scavengerThreadWakeup(void);
- void scavenge();
+ void scavenge(bool forced = false);
void validate (int optionMask);
Role* findRole(const char *schemaName, const char * roleName);
User* findUser (const char *account);
@@ -233,7 +235,7 @@ public:
void setRecordScavengeFloor(int value);
void checkRecordScavenge(void);
void signalCardinality(void);
- void signalScavenger(void);
+ void signalScavenger(bool force = false);
void debugTrace(void);
void pageCacheFlushed(int64 flushArg);
JString setLogRoot(const char *defaultPath, bool create);
@@ -241,6 +243,7 @@ public:
void clearIOError(void);
void flushWait(void);
void setLowMemory(void);
+ void clearLowMemory(void);
Dbb *dbb;
@@ -313,6 +316,7 @@ public:
volatile INTERLOCK_TYPE cardinalityThreadSignaled;
volatile INTERLOCK_TYPE scavengerThreadSleeping;
volatile INTERLOCK_TYPE scavengerThreadSignaled;
+ volatile INTERLOCK_TYPE scavengeForced;
PageWriter *pageWriter;
PreparedStatement *updateCardinality;
MemMgr *recordDataPool;
=== modified file 'storage/falcon/Record.cpp'
--- a/storage/falcon/Record.cpp 2009-01-14 22:33:44 +0000
+++ b/storage/falcon/Record.cpp 2009-02-01 09:00:52 +0000
@@ -953,12 +953,13 @@ char* Record::allocRecordData(int length
|| n > OUT_OF_RECORD_MEMORY_RETRIES)
throw;
- format->table->database->signalScavenger();
+ format->table->database->signalScavenger(true);
- // Give the scavenger thread a chance to release some memory
+ // Give the scavenger thread a chance to release memory.
+ // Increase the wait time per iteration.
Thread *thread = Thread::getThread("Database::ticker");
- thread->sleep(10);
+ thread->sleep(n * SCAVENGE_WAIT_MS);
}
return NULL;
=== modified file 'storage/falcon/RecordScavenge.cpp'
--- a/storage/falcon/RecordScavenge.cpp 2009-01-14 22:33:44 +0000
+++ b/storage/falcon/RecordScavenge.cpp 2009-02-01 08:18:51 +0000
@@ -25,24 +25,22 @@
#include "Transaction.h"
#include "TransactionManager.h"
-RecordScavenge::RecordScavenge(Database *db)
+RecordScavenge::RecordScavenge(Database *db, uint64 generation, bool forceScavenge)
+ : database(db), baseGeneration(generation), forced(forceScavenge)
{
- database = db;
cycle = ++database->scavengeCycle;
memset(ageGroups, 0, sizeof(ageGroups));
veryOldRecords = 0;
veryOldRecordSpace = 0;
- startingActiveMemory = db->recordDataPool->activeMemory;
+ startingActiveMemory = database->recordDataPool->activeMemory;
prunedActiveMemory = 0;
retiredActiveMemory = 0;
- scavengeStart = db->deltaTime;
+ scavengeStart = database->deltaTime;
pruneStop = 0;
retireStop = 0;
-
- baseGeneration = database->currentGeneration;
scavengeGeneration = 0;
// Results of Scavenging
@@ -63,10 +61,10 @@ RecordScavenge::RecordScavenge(Database
unScavengeableRecords = 0;
unScavengeableSpace = 0;
- Sync syncActive(&db->transactionManager->activeTransactions.syncObject, "RecordScavenge::RecordScavenge");
+ Sync syncActive(&database->transactionManager->activeTransactions.syncObject, "RecordScavenge::RecordScavenge");
syncActive.lock(Shared);
- oldestActiveTransaction = db->transactionManager->findOldestInActiveList();
+ oldestActiveTransaction = database->transactionManager->findOldestInActiveList();
}
RecordScavenge::~RecordScavenge(void)
@@ -251,7 +249,10 @@ uint64 RecordScavenge::computeThreshold(
scavengeGeneration = baseGeneration - n;
}
- return scavengeGeneration;
+ // We still may want to scavenge even if the age group total is
+ // too small, so use the base generation as a starting point.
+
+ return (scavengeGeneration ? scavengeGeneration : baseGeneration);
}
void RecordScavenge::print(void)
@@ -272,8 +273,10 @@ void RecordScavenge::print(void)
Log::log (LogScavenge,"Cycle=" I64FORMAT
" Base Generation=" I64FORMAT
- " Scavenge Generation=" I64FORMAT "\n",
- cycle, baseGeneration, scavengeGeneration);
+ " Scavenge Generation=" I64FORMAT
+ " Forced=%d"
+ " Low Memory=%d\n",
+ cycle, baseGeneration, scavengeGeneration, (int)forced, (int)database->lowMemory);
Log::log (LogScavenge,"Cycle=" I64FORMAT
" Oldest Active Transaction=%d\n",
cycle, oldestActiveTransaction);
=== modified file 'storage/falcon/RecordScavenge.h'
--- a/storage/falcon/RecordScavenge.h 2009-01-08 09:05:26 +0000
+++ b/storage/falcon/RecordScavenge.h 2009-02-01 08:18:51 +0000
@@ -29,7 +29,7 @@ class Record;
class RecordScavenge
{
public:
- RecordScavenge(Database *db);
+ RecordScavenge(Database *db, uint64 generation, bool forceScavenge = false);
~RecordScavenge(void);
bool canBeRetired(Record* record);
@@ -71,6 +71,8 @@ public:
uint64 ageGroups[AGE_GROUPS];
uint64 veryOldRecords;
uint64 veryOldRecordSpace;
+
+ bool forced;
};
#endif
=== modified file 'storage/falcon/SerialLog.cpp'
--- a/storage/falcon/SerialLog.cpp 2009-01-22 18:40:02 +0000
+++ b/storage/falcon/SerialLog.cpp 2009-02-01 09:50:43 +0000
@@ -285,7 +285,7 @@ void SerialLog::recover()
SerialLogBlock *recoveryBlock = recoveryWindow->firstBlock();
recoveryBlockNumber = recoveryBlock->blockNumber;
SerialLogBlock *lastBlock = findLastBlock(recoveryWindow);
- Log::log("/nFirst recovery block is " I64FORMAT "\n",
+ Log::log("\nFirst recovery block is " I64FORMAT "\n",
(otherWindow) ? otherWindow->firstBlock()->blockNumber : recoveryBlockNumber);
Log::log("Last recovery block is " I64FORMAT "\n", lastBlock->blockNumber);
@@ -354,7 +354,7 @@ void SerialLog::recover()
// Next, make a second pass to reallocate any necessary pages
- Log::log("\nRecovery phase 2...\n");
+ Log::log("Recovery phase 2...\n");
recordCount = 0;
while ( (record = control.nextRecord()) )
@@ -383,7 +383,7 @@ void SerialLog::recover()
// Make a third pass doing things
- Log::log("\nRecovery phase 3...\n");
+ Log::log("Recovery phase 3...\n");
recordCount = 0;
while ( (record = control.nextRecord()) )
=== modified file 'storage/falcon/StorageParameters.h'
--- a/storage/falcon/StorageParameters.h 2008-10-20 21:28:11 +0000
+++ b/storage/falcon/StorageParameters.h 2009-02-01 09:50:43 +0000
@@ -26,8 +26,8 @@ PARAMETER_UINT(large_blob_threshold, "Th
PARAMETER_UINT(lock_wait_timeout, "Transaction lock time period (seconds)", 0, 50, INT_MAX, 0, NULL)
PARAMETER_UINT(page_size, "The page size used when creating a Falcon tablespace.", 2048, 4096, 32768, 0x0200, NULL)
PARAMETER_UINT(record_chill_threshold, "Bytes of pending record data that is 'frozen' to the Falcon serial log.", 1, 5*1024*1024, 1024*1024*1024, 0, &updateRecordChillThreshold)
-PARAMETER_UINT(record_scavenge_floor, "A percentage of falcon_record_memory_threshold that defines the amount of record data that will remain in the record cache after a scavenge run.", 10, 50, 90, 0x2000, &StorageInterface::updateRecordScavengeFloor)
-PARAMETER_UINT(record_scavenge_threshold, "The percentage of falcon_record_memory_max that will cause the scavenger thread to start scavenging records from the record cache.", 10, 67, 100, 0x2000, &StorageInterface::updateRecordScavengeThreshold)
+PARAMETER_UINT(record_scavenge_floor, "A percentage of falcon_record_memory_threshold that defines the amount of record data that will remain in the record cache after a scavenge run.", 10, 80, 90, 0x2000, &StorageInterface::updateRecordScavengeFloor)
+PARAMETER_UINT(record_scavenge_threshold, "The percentage of falcon_record_memory_max that will cause the scavenger thread to start scavenging records from the record cache.", 10, 90, 100, 0x2000, &StorageInterface::updateRecordScavengeThreshold)
PARAMETER_UINT(serial_log_block_size, "Minimum block size for serial log.", 0, 0, 4096, 0, NULL)
PARAMETER_UINT(serial_log_buffers, "The number of buffers allocated for Falcon serial log.", SRL_MIN_WINDOWS, 20, 1000, 0x0200, NULL)
PARAMETER_UINT(serial_log_priority, "Whether or not serial log has write priority over other writes.", 0, 1, 1, 0, NULL)
=== modified file 'storage/falcon/Table.cpp'
--- a/storage/falcon/Table.cpp 2009-01-28 19:19:44 +0000
+++ b/storage/falcon/Table.cpp 2009-02-01 09:00:52 +0000
@@ -3581,14 +3581,22 @@ void Table::optimize(Connection *connect
record = record->fetchVersion(transaction);
if (record)
+ {
+ record->release(); // no need to keep it around
++count;
+ }
}
cardinality = count;
+ // Disable index optimization until a more
+ // efficient method is implemented using
+ // the IndexWalker (Bug#36442)
+#if 0
FOR_INDEXES(index, this);
index->optimize(count, connection);
END_FOR;
+#endif
database->commitSystemTransaction();
}
@@ -3642,12 +3650,13 @@ RecordVersion* Table::allocRecordVersion
|| n > OUT_OF_RECORD_MEMORY_RETRIES)
throw;
- database->signalScavenger();
+ database->signalScavenger(true);
- // Give the scavenger thread a chance to release some memory
+ // Give the scavenger thread a chance to release memory.
+ // Increase the wait time per iteration.
Thread *thread = Thread::getThread("Database::ticker");
- thread->sleep(10);
+ thread->sleep(n * SCAVENGE_WAIT_MS);
}
}
@@ -3678,12 +3687,13 @@ Record* Table::allocRecord(int recordNum
|| n > OUT_OF_RECORD_MEMORY_RETRIES)
throw;
- database->signalScavenger();
+ database->signalScavenger(true);
- // Give the scavenger thread a chance to release some memory
+ // Give the scavenger thread a chance to release memory.
+ // Increase the wait time per iteration.
Thread *thread = Thread::getThread("Database::ticker");
- thread->sleep(10);
+ thread->sleep(n * SCAVENGE_WAIT_MS);
}
}
=== modified file 'storage/falcon/TableSpaceManager.cpp'
--- a/storage/falcon/TableSpaceManager.cpp 2009-01-28 14:26:37 +0000
+++ b/storage/falcon/TableSpaceManager.cpp 2009-01-31 23:22:42 +0000
@@ -157,9 +157,8 @@ TableSpace* TableSpaceManager::createTab
Sync syncDDL(&database->syncSysDDL, "TableSpaceManager::createTableSpace");
syncDDL.lock(Exclusive);
- Sequence *sequence = database->sequenceManager->getSequence(database->getSymbol("SYSTEM"), database->getSymbol("TABLESPACE_IDS"));
int type = (repository) ? TABLESPACE_TYPE_REPOSITORY : TABLESPACE_TYPE_TABLESPACE;
- int id = (int) sequence->update(1, database->getSystemTransaction());
+ int id = createTableSpaceId();
TableSpace *tableSpace = new TableSpace(database, name, id, fileName, type, tsInit);
@@ -565,3 +564,9 @@ void TableSpaceManager::getTableSpaceFil
}
}
+int TableSpaceManager::createTableSpaceId()
+{
+ Sequence *sequence = database->sequenceManager->getSequence(database->getSymbol("SYSTEM"), database->getSymbol("TABLESPACE_IDS"));
+ int id = (int) sequence->update(1, database->getSystemTransaction());
+ return id;
+}
=== modified file 'storage/falcon/TableSpaceManager.h'
--- a/storage/falcon/TableSpaceManager.h 2008-10-31 00:29:13 +0000
+++ b/storage/falcon/TableSpaceManager.h 2009-01-31 23:22:42 +0000
@@ -63,6 +63,7 @@ public:
void reportWrites(void);
void redoCreateTableSpace(int id, int nameLength, const char* name, int fileNameLength, const char* fileName, int type, TableSpaceInit* tsInit);
void initialize(void);
+ int createTableSpaceId();
Database *database;
TableSpace *tableSpaces;
=== modified file 'storage/falcon/ha_falcon.cpp'
--- a/storage/falcon/ha_falcon.cpp 2009-01-21 16:48:25 +0000
+++ b/storage/falcon/ha_falcon.cpp 2009-02-01 09:50:43 +0000
@@ -3930,7 +3930,7 @@ static MYSQL_SYSVAR_UINT(allocation_exte
static MYSQL_SYSVAR_ULONGLONG(page_cache_size, falcon_page_cache_size,
PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
"The amount of memory to be used for the database page cache.",
- NULL, NULL, LL(4)<<20, LL(2)<<20, (ulonglong) max_memory_address, LL(1)<<20);
+ NULL, NULL, LL(250)<<20, LL(2)<<20, (ulonglong) max_memory_address, LL(1)<<20);
static MYSQL_THDVAR_BOOL(consistent_read, PLUGIN_VAR_OPCMDARG,
"Enable Consistent Read Mode for Repeatable Reads",
| Thread |
|---|
| • bzr push into mysql-6.0-falcon-team branch (christopher.powers:2989 to2993) Bug#36442 Bug#42510 | Christopher Powers | 1 Feb |
