bzr commit into mysql-5.1-telco-6.2 branch (jonas:2990) Bug#47170

[Jonas Oreland]

#At file:///home/jonas/src/telco-6.2/ based on revid:magnus.blaudd@stripped

 2990 Jonas Oreland	2009-09-09
      ndb - bug#47170
        Calculation of StringMemory could overflow for big values 
          of MaxNoOfTables/Attributes

    modified:
      storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp
      storage/ndb/src/kernel/vm/Rope.hpp
=== modified file 'storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp'
--- a/storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp	2009-08-21 09:39:39 +0000
+++ b/storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp	2009-09-09 08:57:13 +0000
@@ -2212,17 +2212,20 @@ void Dbdict::execREAD_CONFIG_REQ(Signal*
   if (sm == 0)
     sm = 25;
   
-  Uint32 sb = 0;
+  Uint64 sb = 0;
   if (sm <= 100)
   {
-    sb = (rps * sm) / 100;
+    sb = (Uint64(rps) * Uint64(sm)) / 100;
   }
   else
   {
     sb = sm;
   }
   
-  c_rope_pool.setSize(sb/28 + 100);
+  sb /= (Rope::getSegmentSize() * sizeof(Uint32));
+  sb += 100; // more safty
+  ndbrequire(sb < (Uint64(1) << 32));
+  c_rope_pool.setSize(Uint32(sb));
   
   // Initialize BAT for interface to file system
   NewVARIABLE* bat = allocateBat(2);

=== modified file 'storage/ndb/src/kernel/vm/Rope.hpp'
--- a/storage/ndb/src/kernel/vm/Rope.hpp	2009-05-26 18:53:34 +0000
+++ b/storage/ndb/src/kernel/vm/Rope.hpp	2009-09-09 08:57:13 +0000
@@ -86,6 +86,7 @@ public:
   
   static Uint32 hash(const char * str, Uint32 len);
 
+  static Uint32 getSegmentSize() { return RopeBase::getSegmentSize();}
 private:
   Uint32 m_hash;
   RopeHandle & src;


Attachment: [text/bzr-bundle] bzr/[e-mail]