bk commit into 5.1 tree (jonas:1.2604) BUG#36950

[jonas]

Below is the list of changes that have just been committed into a local
5.1 repository of jonas.  When jonas does a push these changes
will be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet@stripped, 2008-05-26 18:41:15+02:00, jonas@stripped +1 -0
  ndb - bug#36950
    handle mysql-style index varsize-keys (2 bytes len regardless of datatype)
    reuse tmp buffer and silently fail (dont prune) when running out of buffer

  storage/ndb/src/ndbapi/NdbScanOperation.cpp@stripped, 2008-05-26 18:41:13+02:00,
jonas@stripped +28 -1
    handle mysql-style index varsize-keys (2 bytes len regardless of datatype)
    reuse tmp buffer and silently fail (dont prune) when running out of buffer

diff -Nrup a/storage/ndb/src/ndbapi/NdbScanOperation.cpp
b/storage/ndb/src/ndbapi/NdbScanOperation.cpp
--- a/storage/ndb/src/ndbapi/NdbScanOperation.cpp	2008-05-26 11:22:35 +02:00
+++ b/storage/ndb/src/ndbapi/NdbScanOperation.cpp	2008-05-26 18:41:13 +02:00
@@ -516,13 +516,40 @@ NdbIndexScanOperation::setDistKeyFromRan
                                            Uint32 distkeyMax)
 {
   Uint64 tmp[1000];
+  char* tmpshrink = (char*)tmp;
+  size_t tmplen = sizeof(tmp);
+  
   Ndb::Key_part_ptr ptrs[NDB_MAX_NO_OF_ATTRIBUTES_IN_KEY+1];
   Uint32 i;
   for (i = 0; i<key_record->distkey_index_length; i++)
   {
     const NdbRecord::Attr *col =
       &key_record->columns[key_record->distkey_indexes[i]];
-    ptrs[i].ptr = row + col->offset;
+    if (col->flags & NdbRecord::IsMysqldShrinkVarchar)
+    {
+      if (tmplen >= 256)
+      {
+        Uint32 len;
+        bool len_ok = col->shrink_varchar(row, len, tmpshrink);
+        if (!len_ok)
+        {
+          assert(false);
+          return;
+        }
+        ptrs[i].ptr = tmpshrink;
+        tmpshrink += len;
+        tmplen -= len;
+      }
+      else
+      {
+        // no buffer...
+        return;
+      }
+    }
+    else
+    {
+      ptrs[i].ptr = row + col->offset;
+    }
     ptrs[i].len = col->maxSize;
   }
   ptrs[i].ptr = 0;