From: Date: March 23 2006 4:59am Subject: bk commit into 5.1 tree (pekka:1.2200) BUG#17813 List-Archive: http://lists.mysql.com/commits/4057 X-Bug: 17813 Message-Id: <200603230359.k2N3xM9t005585@localhost.localdomain> Below is the list of changes that have just been committed into a local 5.1 repository of pekka. When pekka does a push these changes will be propagated to the main repository and, within 24 hours after the push, to the public repository. For information on how to access the public repository see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html ChangeSet 1.2200 06/03/23 04:59:14 pekka@stripped +9 -0 ndb - bug#17813 schema.query => blob storage/ndb/tools/restore/restore_main.cpp 1.42 06/03/23 04:56:54 pekka@stripped +14 -13 allow 'system tables' to have blob attrs storage/ndb/tools/restore/Restore.hpp 1.24 06/03/23 04:56:54 pekka@stripped +6 -1 allow 'system tables' to have blob attrs storage/ndb/tools/restore/Restore.cpp 1.35 06/03/23 04:56:54 pekka@stripped +48 -0 allow 'system tables' to have blob attrs sql/ha_ndbcluster_binlog.cc 1.38 06/03/23 04:21:11 pekka@stripped +58 -22 schema.query => blob storage/ndb/src/ndbapi/NdbBlob.cpp 1.40 06/03/23 04:19:18 pekka@stripped +6 -1 atNextEvent: ignore non-data event storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp 1.24 06/03/23 04:17:22 pekka@stripped +12 -3 ER_ALL: omit unchanged blob inlines storage/ndb/src/kernel/blocks/dbtup/DbtupMeta.cpp 1.25 06/03/23 04:17:22 pekka@stripped +6 -0 ER_ALL: omit unchanged blob inlines storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp 1.43 06/03/23 04:17:22 pekka@stripped +1 -0 ER_ALL: omit unchanged blob inlines storage/ndb/include/ndbapi/NdbDictionary.hpp 1.72 06/03/23 04:17:22 pekka@stripped +1 -1 ER_ALL: omit unchanged blob inlines # This is a BitKeeper patch. What follows are the unified diffs for the # set of deltas contained in the patch. The rest of the patch, the part # that BitKeeper cares about, is below these diffs. # User: pekka # Host: clam.(none) # Root: /export/space/pekka/ndb/version/my51-rbr --- 1.71/storage/ndb/include/ndbapi/NdbDictionary.hpp 2006-03-06 11:13:55 +01:00 +++ 1.72/storage/ndb/include/ndbapi/NdbDictionary.hpp 2006-03-23 04:17:22 +01:00 @@ -1164,7 +1164,7 @@ */ enum EventReport { ER_UPDATED = 0, - ER_ALL = 1, + ER_ALL = 1, // except not-updated blob inlines ER_SUBSCRIBE = 2 }; --- 1.34/storage/ndb/tools/restore/Restore.cpp 2006-02-14 11:48:13 +01:00 +++ 1.35/storage/ndb/tools/restore/Restore.cpp 2006-03-23 04:56:54 +01:00 @@ -26,6 +26,8 @@ #include #include +#include "../../../../sql/ha_ndbcluster_tables.h" + Uint16 Twiddle16(Uint16 in); // Byte shift 16-bit data Uint32 Twiddle32(Uint32 in); // Byte shift 32-bit data Uint64 Twiddle64(Uint64 in); // Byte shift 64-bit data @@ -111,6 +113,8 @@ return 0; } } + if (! markSysTables()) + return 0; if(!readGCPEntry()) return 0; return 1; @@ -276,6 +280,49 @@ } bool +RestoreMetaData::markSysTables() +{ + Uint32 i; + for (i = 0; i < getNoOfTables(); i++) { + TableS* table = allTables[i]; + const char* tableName = table->getTableName(); + if ( // XXX should use type + strcmp(tableName, "SYSTAB_0") == 0 || + strcmp(tableName, "NDB$EVENTS_0") == 0 || + strcmp(tableName, "sys/def/SYSTAB_0") == 0 || + strcmp(tableName, "sys/def/NDB$EVENTS_0") == 0 || + strcmp(tableName, NDB_REP_DB "/def/" NDB_APPLY_TABLE) == 0 || + strcmp(tableName, NDB_REP_DB "/def/" NDB_SCHEMA_TABLE)== 0 ) + table->isSysTable = true; + } + for (i = 0; i < getNoOfTables(); i++) { + TableS* blobTable = allTables[i]; + const char* blobTableName = blobTable->getTableName(); + // yet another match blob + int cnt, id1, id2; + char buf[256]; + cnt = sscanf(blobTableName, "%[^/]/%[^/]/NDB$BLOB_%d_%d", + buf, buf, &id1, &id2); + if (cnt == 4) { + Uint32 j; + for (j = 0; j < getNoOfTables(); j++) { + TableS* table = allTables[j]; + if (table->getTableId() == id1) { + if (table->isSysTable) + blobTable->isSysTable = true; + break; + } + } + if (j == getNoOfTables()) { + err << "Restore: Bad primary table id in " << blobTableName << endl; + return false; + } + } + } + return true; +} + +bool RestoreMetaData::readGCPEntry() { Uint32 data[4]; @@ -312,6 +359,7 @@ m_auto_val_id= ~(Uint32)0; m_max_auto_val= 0; backupVersion = version; + isSysTable = false; for (int i = 0; i < tableImpl->getNoOfColumns(); i++) createAttr(tableImpl->getColumn(i)); --- 1.23/storage/ndb/tools/restore/Restore.hpp 2006-01-17 09:24:54 +01:00 +++ 1.24/storage/ndb/tools/restore/Restore.hpp 2006-03-23 04:56:54 +01:00 @@ -134,7 +134,7 @@ Uint32 m_auto_val_id; Uint64 m_max_auto_val; - int pos; + bool isSysTable; void createAttr(NdbDictionary::Column *column); @@ -222,6 +222,10 @@ return allAttributesDesc[attributeId]; } + bool getSysTable() const { + return isSysTable; + } + TableS& operator=(TableS& org) ; }; // TableS; @@ -279,6 +283,7 @@ Vector allTables; bool readMetaFileHeader(); bool readMetaTableDesc(); + bool markSysTables(); bool readGCPEntry(); Uint32 readMetaTableList(); --- 1.41/storage/ndb/tools/restore/restore_main.cpp 2006-02-13 17:07:06 +01:00 +++ 1.42/storage/ndb/tools/restore/restore_main.cpp 2006-03-23 04:56:54 +01:00 @@ -411,16 +411,17 @@ g_consumers.clear(); } -static bool -checkSysTable(const char *tableName) +static inline bool +checkSysTable(const TableS* table) { - return ga_dont_ignore_systab_0 || - (strcmp(tableName, "SYSTAB_0") != 0 && - strcmp(tableName, "NDB$EVENTS_0") != 0 && - strcmp(tableName, "sys/def/SYSTAB_0") != 0 && - strcmp(tableName, "sys/def/NDB$EVENTS_0") != 0 && - strcmp(tableName, NDB_REP_DB "/def/" NDB_APPLY_TABLE) != 0 && - strcmp(tableName, NDB_REP_DB "/def/" NDB_SCHEMA_TABLE) != 0); + return ga_dont_ignore_systab_0 || ! table->getSysTable(); +} + +static inline bool +checkSysTable(const RestoreMetaData& metaData, uint i) +{ + assert(i < metaData.getNoOfTables()); + return checkSysTable(metaData[i]); } static void @@ -534,7 +535,7 @@ debug << "Restoring tables" << endl; for(i = 0; igetTableName())) + if (checkSysTable(metaData, i)) { for(Uint32 j= 0; j < g_consumers.size(); j++) if (!g_consumers[j]->table(* metaData[i])) @@ -572,7 +573,7 @@ const TupleS* tuple; while ((tuple = dataIter.getNextTuple(res= 1)) != 0) { - if (checkSysTable(tuple->getTable()->getTableName())) + if (checkSysTable(tuple->getTable())) for(Uint32 i= 0; i < g_consumers.size(); i++) g_consumers[i]->tuple(* tuple, fragmentId); } // while (tuple != NULL); @@ -617,7 +618,7 @@ bool alloc_flag = false; while ((logEntry = logIter.getNextLogEntry(res= 0, &alloc_flag)) != 0) { - if (checkSysTable(logEntry->m_table->getTableName())) + if (checkSysTable(logEntry->m_table)) for(Uint32 i= 0; i < g_consumers.size(); i++) g_consumers[i]->logEntry(* logEntry); if (alloc_flag) @@ -638,7 +639,7 @@ { for(i = 0; igetTableName())) + if (checkSysTable(metaData, i)) { for(Uint32 j= 0; j < g_consumers.size(); j++) if (!g_consumers[j]->finalize_table(* metaData[i])) --- 1.42/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp 2006-03-02 16:44:09 +01:00 +++ 1.43/storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp 2006-03-23 04:17:22 +01:00 @@ -878,6 +878,7 @@ {} Bitmask notNullAttributeMask; + Bitmask blobAttributeMask; ReadFunction* readFunctionArray; UpdateFunction* updateFunctionArray; --- 1.24/storage/ndb/src/kernel/blocks/dbtup/DbtupMeta.cpp 2006-03-02 16:44:09 +01:00 +++ 1.25/storage/ndb/src/kernel/blocks/dbtup/DbtupMeta.cpp 2006-03-23 04:17:22 +01:00 @@ -201,6 +201,7 @@ regTabPtr.p->m_no_of_attributes= noOfAttributes; regTabPtr.p->notNullAttributeMask.clear(); + regTabPtr.p->blobAttributeMask.clear(); Uint32 offset[10]; Uint32 tableDescriptorRef= allocTabDescr(regTabPtr.p, offset); @@ -286,6 +287,7 @@ ptrCheckGuard(fragOperPtr, cnoOfFragoprec, fragoperrec); Uint32 attrId = signal->theData[2]; Uint32 attrDescriptor = signal->theData[3]; + Uint32 extType = AttributeDescriptor::getType(attrDescriptor); // DICT sends charset number in upper half Uint32 csNumber = (signal->theData[4] >> 16); @@ -351,6 +353,10 @@ else { regTabPtr.p->notNullAttributeMask.set(attrId); + } + + if (extType == NDB_TYPE_BLOB || extType == NDB_TYPE_TEXT) { + regTabPtr.p->blobAttributeMask.set(attrId); } switch (AttributeDescriptor::getArrayType(attrDescriptor)) { --- 1.23/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp 2006-02-13 13:12:42 +01:00 +++ 1.24/storage/ndb/src/kernel/blocks/dbtup/DbtupTrigger.cpp 2006-03-23 04:17:22 +01:00 @@ -867,10 +867,19 @@ } else { ljam(); //-------------------------------------------------------------------- -// All others send all attributes that are monitored +// All others send all attributes that are monitored, except: +// Omit unchanged blob inlines on update i.e. +// attributeMask & ~ (blobAttributeMask & ~ changeMask) //-------------------------------------------------------------------- - numAttrsToRead = setAttrIds(trigPtr->attributeMask, - regTabPtr->m_no_of_attributes, &readBuffer[0]); + Bitmask attributeMask; + attributeMask = trigPtr->attributeMask; + if (regOperPtr->op_struct.op_type == ZUPDATE) { + Bitmask tmpMask = regTabPtr->blobAttributeMask; + tmpMask.bitANDC(req_struct->changeMask); + attributeMask.bitANDC(tmpMask); + } + numAttrsToRead = setAttrIds(attributeMask, regTabPtr->m_no_of_attributes, + &readBuffer[0]); } ndbrequire(numAttrsToRead < MAX_ATTRIBUTES_IN_TABLE); //-------------------------------------------------------------------- --- 1.39/storage/ndb/src/ndbapi/NdbBlob.cpp 2006-03-09 17:07:08 +01:00 +++ 1.40/storage/ndb/src/ndbapi/NdbBlob.cpp 2006-03-23 04:19:18 +01:00 @@ -598,6 +598,8 @@ theNullFlag = theHeadInlineRecAttr->isNULL(); assert(theEventBlobVersion >= 0 || theNullFlag != -1); theLength = ! theNullFlag ? theHead->length : 0; + DBUG_PRINT("info", ("theNullFlag=%d theLength=%llu", + theNullFlag, theLength)); DBUG_VOID_RETURN; } @@ -1835,10 +1837,13 @@ NdbBlob::atNextEvent() { DBUG_ENTER("NdbBlob::atNextEvent"); - DBUG_PRINT("info", ("this=%p op=%p blob op=%p version=%d", this, theEventOp, theBlobEventOp, theEventBlobVersion)); + Uint32 optype = theEventOp->m_data_item->sdata->operation; + DBUG_PRINT("info", ("this=%p op=%p blob op=%p version=%d optype=%u", this, theEventOp, theBlobEventOp, theEventBlobVersion, optype)); if (theState == Invalid) DBUG_RETURN(-1); assert(theEventBlobVersion >= 0); + if (optype >= NdbDictionary::Event::_TE_FIRST_NON_DATA_EVENT) + DBUG_RETURN(0); getHeadFromRecAttr(); if (theNullFlag == -1) // value not defined DBUG_RETURN(0); --- 1.37/sql/ha_ndbcluster_binlog.cc 2006-03-21 16:54:45 +01:00 +++ 1.38/sql/ha_ndbcluster_binlog.cc 2006-03-23 04:21:11 +01:00 @@ -746,10 +746,10 @@ */ end= strmov(buf, "CREATE TABLE IF NOT EXISTS " NDB_REP_DB "." NDB_SCHEMA_TABLE - " ( db VARCHAR(63) NOT NULL," - " name VARCHAR(63) NOT NULL," + " ( db VARBINARY(63) NOT NULL," + " name VARBINARY(63) NOT NULL," " slock BINARY(32) NOT NULL," - " query VARCHAR(4094) NOT NULL," + " query BLOB NOT NULL," " node_id INT UNSIGNED NOT NULL," " epoch BIGINT UNSIGNED NOT NULL," " id INT UNSIGNED NOT NULL," @@ -802,7 +802,6 @@ #define SCHEMA_TYPE_I 8u #define SCHEMA_SIZE 9u #define SCHEMA_SLOCK_SIZE 32u -#define SCHEMA_QUERY_SIZE 4096u struct Cluster_schema { @@ -813,7 +812,7 @@ unsigned char slock_length; uint32 slock[SCHEMA_SLOCK_SIZE/4]; unsigned short query_length; - char query[SCHEMA_QUERY_SIZE]; + char *query; Uint64 epoch; uint32 node_id; uint32 id; @@ -824,10 +823,26 @@ /* Transfer schema table data into corresponding struct */ -static void ndbcluster_get_schema(TABLE *table, +static void ndbcluster_get_schema(NDB_SHARE *share, Cluster_schema *s) { + TABLE *table= share->table; Field **field; + /* unpack blob values */ + byte* blobs_buffer= 0; + uint blobs_buffer_size= 0; + { + ptrdiff_t ptrdiff= 0; + int ret= get_ndb_blobs_value(table, share->ndb_value[0], + blobs_buffer, blobs_buffer_size, + ptrdiff); + if (ret != 0) + { + my_free(blobs_buffer, MYF(MY_ALLOW_ZERO_PTR)); + DBUG_PRINT("info", ("blob read error")); + DBUG_ASSERT(false); + } + } /* db varchar 1 length byte */ field= table->field; s->db_length= *(uint8*)(*field)->ptr; @@ -847,13 +862,19 @@ s->slock_length= (*field)->field_length; DBUG_ASSERT((*field)->field_length == sizeof(s->slock)); memcpy(s->slock, (*field)->ptr, s->slock_length); - /* query varchar 2 length bytes */ + /* query blob */ field++; - s->query_length= uint2korr((*field)->ptr); - DBUG_ASSERT(s->query_length <= (*field)->field_length); - DBUG_ASSERT((*field)->field_length + 2 == sizeof(s->query)); - memcpy(s->query, (*field)->ptr + 2, s->query_length); - s->query[s->query_length]= 0; + { + Field_blob *field_blob= (Field_blob*)(*field); + uint blob_len= field_blob->get_length((*field)->ptr); + char *blob_ptr= 0; + field_blob->get_ptr(&blob_ptr); + assert(blob_len == 0 || blob_ptr != 0); + s->query_length= blob_len; + s->query= sql_alloc(blob_len+1); + memcpy(s->query, blob_ptr, blob_len); + s->query[blob_len]= 0; + } /* node_id */ field++; s->node_id= ((Field_long *)*field)->val_int(); @@ -869,6 +890,8 @@ /* type */ field++; s->type= ((Field_long *)*field)->val_int(); + /* free blobs buffer */ + my_free(blobs_buffer, MYF(MY_ALLOW_ZERO_PTR)); } /* @@ -1013,7 +1036,7 @@ char save_db[FN_REFLEN]; strcpy(save_db, ndb->getDatabaseName()); - char tmp_buf[SCHEMA_QUERY_SIZE]; + char tmp_buf[FN_REFLEN]; NDBDICT *dict= ndb->getDictionary(); ndb->setDatabaseName(NDB_REP_DB); const NDBTAB *ndbtab= dict->getTable(NDB_SCHEMA_TABLE); @@ -1037,8 +1060,11 @@ for (i= 0; i < SCHEMA_SIZE; i++) { col[i]= ndbtab->getColumn(i); - sz[i]= col[i]->getLength(); - DBUG_ASSERT(sz[i] <= sizeof(tmp_buf)); + if (i != SCHEMA_QUERY_I) + { + sz[i]= col[i]->getLength(); + DBUG_ASSERT(sz[i] <= sizeof(tmp_buf)); + } } } @@ -1068,9 +1094,14 @@ r|= op->setValue(SCHEMA_SLOCK_I, (char*)schema_subscribers.bitmap); DBUG_ASSERT(r == 0); /* query */ - ndb_pack_varchar(col[SCHEMA_QUERY_I], tmp_buf, query, query_length); - r|= op->setValue(SCHEMA_QUERY_I, tmp_buf); - DBUG_ASSERT(r == 0); + { + NdbBlob *ndb_blob= op->getBlobHandle(SCHEMA_QUERY_I); + DBUG_ASSERT(ndb_blob != 0); + uint blob_len= query_length; + const char* blob_ptr= query; + r|= ndb_blob->setValue(blob_ptr, blob_len); + DBUG_ASSERT(r == 0); + } /* node_id */ r|= op->setValue(SCHEMA_NODE_ID_I, node_id); DBUG_ASSERT(r == 0); @@ -1203,7 +1234,7 @@ char save_db[FN_HEADLEN]; strcpy(save_db, ndb->getDatabaseName()); - char tmp_buf[SCHEMA_QUERY_SIZE]; + char tmp_buf[FN_REFLEN]; NDBDICT *dict= ndb->getDictionary(); ndb->setDatabaseName(NDB_REP_DB); const NDBTAB *ndbtab= dict->getTable(NDB_SCHEMA_TABLE); @@ -1227,8 +1258,11 @@ for (i= 0; i < SCHEMA_SIZE; i++) { col[i]= ndbtab->getColumn(i); - sz[i]= col[i]->getLength(); - DBUG_ASSERT(sz[i] <= sizeof(tmp_buf)); + if (i != SCHEMA_QUERY_I) + { + sz[i]= col[i]->getLength(); + DBUG_ASSERT(sz[i] <= sizeof(tmp_buf)); + } } } @@ -1506,7 +1540,7 @@ MY_BITMAP slock; bitmap_init(&slock, schema->slock, 8*SCHEMA_SLOCK_SIZE, false); uint node_id= g_ndb_cluster_connection->node_id(); - ndbcluster_get_schema(share->table, schema); + ndbcluster_get_schema(share, schema); if (schema->node_id != node_id) { int log_query= 0, post_epoch_unlock= 0; @@ -2265,6 +2299,7 @@ */ DBUG_ENTER("ndbcluster_create_event_ops"); + DBUG_PRINT("enter", ("table: %s event: %s", ndbtab->getName(), event_name)); DBUG_ASSERT(! IS_NDB_BLOB_PREFIX(ndbtab->getName())); DBUG_ASSERT(share != 0); @@ -2374,6 +2409,7 @@ else { DBUG_PRINT("info", ("%s blob", col_name)); + DBUG_ASSERT(share->flags & NSF_BLOB_FLAG); attr0.blob= op->getBlobHandle(col_name); attr1.blob= op->getPreBlobHandle(col_name); if (attr0.blob == NULL || attr1.blob == NULL)