List:Commits« Previous MessageNext Message »
From:jonas Date:January 22 2006 5:23pm
Subject:bk commit into 5.1 tree (jonas:1.2066)
View as plain text  
Below is the list of changes that have just been committed into a local
5.1 repository of jonas. When jonas does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html

ChangeSet
  1.2066 06/01/22 18:23:48 jonas@eel.(none) +8 -0
  wl1497 - ndb - dynamic mem in ndbd
    Impl. NdbdSuperPool (subclass of SuperPool) that uses Ndbd_mem_manager
    Impl. micro benchmark

  storage/ndb/src/kernel/vm/bench_pool.cpp
    1.1 06/01/22 18:23:44 jonas@eel.(none) +249 -0
    New BitKeeper file ``storage/ndb/src/kernel/vm/bench_pool.cpp''

  storage/ndb/src/kernel/vm/bench_pool.cpp
    1.0 06/01/22 18:23:44 jonas@eel.(none) +0 -0
    BitKeeper file /home/jonas/src/51-ndb/storage/ndb/src/kernel/vm/bench_pool.cpp

  storage/ndb/src/kernel/vm/NdbdSuperPool.hpp
    1.1 06/01/22 18:23:43 jonas@eel.(none) +55 -0
    New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.hpp''

  storage/ndb/src/kernel/vm/NdbdSuperPool.cpp
    1.1 06/01/22 18:23:43 jonas@eel.(none) +228 -0
    New BitKeeper file ``storage/ndb/src/kernel/vm/NdbdSuperPool.cpp''

  storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp
    1.2 06/01/22 18:23:43 jonas@eel.(none) +7 -2
    Add util methods used by NdbdSuperPool

  storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp
    1.3 06/01/22 18:23:43 jonas@eel.(none) +20 -1
    Add util methods used by NdbdSuperPool

  storage/ndb/src/kernel/vm/SuperPool.hpp
    1.6 06/01/22 18:23:43 jonas@eel.(none) +2 -1
    Remove unneccesary virtuality

  storage/ndb/src/kernel/vm/SuperPool.cpp
    1.5 06/01/22 18:23:43 jonas@eel.(none) +2 -2
    remove virtuallity

  storage/ndb/src/kernel/vm/NdbdSuperPool.hpp
    1.0 06/01/22 18:23:43 jonas@eel.(none) +0 -0
    BitKeeper file /home/jonas/src/51-ndb/storage/ndb/src/kernel/vm/NdbdSuperPool.hpp

  storage/ndb/src/kernel/vm/NdbdSuperPool.cpp
    1.0 06/01/22 18:23:43 jonas@eel.(none) +0 -0
    BitKeeper file /home/jonas/src/51-ndb/storage/ndb/src/kernel/vm/NdbdSuperPool.cpp

  storage/ndb/src/kernel/vm/Makefile.am
    1.13 06/01/22 18:23:43 jonas@eel.(none) +11 -2
    Add NdbdSuperPool , which impl. SuperPool

# This is a BitKeeper patch.  What follows are the unified diffs for the
# set of deltas contained in the patch.  The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User:	jonas
# Host:	eel.(none)
# Root:	/home/jonas/src/51-ndb
--- New file ---
+++ storage/ndb/src/kernel/vm/NdbdSuperPool.cpp	06/01/22 18:23:43
/* Copyright (C) 2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#include <ndb_global.h>
#include "SuperPool.hpp"
#include "ndbd_malloc_impl.hpp"
#include "NdbdSuperPool.hpp"

#define PSI (1 << (BMW_2LOG + 2))

struct AllocArea
{
  AllocArea(AllocArea* next);

  Uint16 m_currPage;                        // 2
  Uint16 m_numPages;  // number of pages    // 2
  SuperPool::PtrI m_firstPageI;             // 4
  void* m_memory;     // page-aligned pages // 4/8
  struct AllocArea* m_nextArea;             // 4/8
  // tot 16/24
};

AllocArea::AllocArea(AllocArea* next)
{
  m_nextArea = next;
  m_firstPageI = RNIL;
  m_currPage = m_numPages = 0;
  m_memory = 0;
}

NdbdSuperPool::NdbdSuperPool(class Ndbd_mem_manager & mm,
			     Uint32 pageSize, Uint32 pageBits) :
  SuperPool(pageSize, pageBits),
  m_mm(mm),
  m_currArea(0), m_firstArea(0)
{
  m_memRoot = m_mm.get_memroot();
  
  m_shift = Ndbd_mem_manager::log2((1 << (BMW_2LOG + 2)) / pageSize) - 1;
  m_add = (1 << m_shift) - 1;
}

NdbdSuperPool::~NdbdSuperPool()
{
  Uint32 cnt = PSI / sizeof(AllocArea);
  AllocArea* ap = m_firstArea;
  while(ap != 0)
  {
    AllocArea * first = ap;
    for(Uint32 i = 0; i<cnt; i++)
    {
      if (ap->m_numPages)
      {
	m_mm.release(ap->m_memory, ap->m_numPages >> m_shift);
      }
      ap = ap->m_nextArea;
    }
    m_mm.release((void*)first, 1);
  }
}

bool
NdbdSuperPool::init_1()
{
  Uint32 pageCount = (1 << m_pageBits);
  if (m_pageEnt == 0) {
    // allocate page entry array
    Uint32 bytes = pageCount * sizeof(PageEnt);
    m_pageEnt = static_cast<PageEnt*>(malloc(bytes));
    if (m_pageEnt == 0)
      return false;
    for (Uint32 i = 0; i < pageCount; i++)
      new (&m_pageEnt[i]) PageEnt();
  }
  if (m_pageType == 0) {
    // allocate type check array
    Uint32 bytes = pageCount;
    m_pageType = static_cast<Uint8*>(malloc(bytes));
    if (m_pageType == 0)
      return false;
    memset(m_pageType, 0, bytes);
  }
  
  return true;
}

static
void
initAllocAreaPage(AllocArea * p1)
{
  AllocArea * ap = p1;
  Uint32 cnt = PSI / sizeof(AllocArea);
  for(Uint32 i = 0; i<cnt; i++, ap++)
  {
    new (ap) AllocArea(ap + 1);
  }

  (p1 + cnt - 1)->m_nextArea = 0;
}

bool
NdbdSuperPool::init_2()
{
  m_memRoot = m_mm.get_memroot();

  Uint32 cnt = 1;
  AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1);
  if (p1 == 0)
    return false;

  initAllocAreaPage(p1);
  m_currArea = p1;
  m_firstArea = p1;
  return true;
}

SuperPool::PtrI
NdbdSuperPool::getNewPage()
{
  AllocArea* ap = m_currArea;
  Uint32 curr = ap->m_currPage;
  Uint32 cnt = ap->m_numPages;
  if (curr == cnt)
  {
    // area is used up
    if (! (ap = allocMem()))
    {
      abort();
      return RNIL;
    }
    curr = ap->m_currPage;
    cnt = ap->m_numPages;
  }

  assert(curr < cnt);
  PtrI pageI = ap->m_firstPageI;
  Uint32 recBits = m_recBits;
  Int32 ip = ((Int32)pageI >> recBits) + curr;
  pageI = ip << recBits;
  ap->m_currPage = curr + 1;
  return pageI;
}

Uint32
NdbdSuperPool::allocAreaMemory(AllocArea* ap, Uint32 tryPages)
{
  Uint32 cnt = (tryPages + m_add) >> m_shift;
  void* p1 = m_mm.alloc(&cnt, 1);
  if (p1 == 0)
  {
    abort();
    return 0;
  }
  Uint32 pageI = getPageI(p1);
  ap->m_firstPageI = pageI;
  ap->m_currPage = 0;
  ap->m_memory = p1;
  ap->m_numPages = cnt << m_shift;
  return cnt;
}

AllocArea*
NdbdSuperPool::allocArea()
{
  AllocArea * curr = m_currArea;
  AllocArea * next = curr->m_nextArea;
  if (next == 0)
  {
    Uint32 cnt = 1;
    AllocArea* p1 = (AllocArea*)m_mm.alloc(&cnt, 1);
    if (p1 == 0)
      return 0;
    
    initAllocAreaPage(p1);

    m_currArea->m_nextArea = p1;
    return m_currArea = p1;
  }
  else
  {
    m_currArea = m_currArea->m_nextArea;
    return m_currArea;
  }
}

AllocArea*
NdbdSuperPool::allocMem()
{
  // compute number of additional pages needed
  if (m_totPages >= m_maxPages)
  {
    abort();
    return 0;
  }
  Uint32 needPages = (m_totPages == 0 ? m_initPages : m_incrPages);
  
  // add new area
  AllocArea* ap = allocArea();
  if (ap == 0)
  {
    abort();
    return 0;
  }
  
  Uint32 numPages;
  if (!(numPages = allocAreaMemory(ap, needPages)))
  {
    abort();
    return 0;
  }
  
  // update counts
  m_totPages += numPages;
  return ap;
}

--- New file ---
+++ storage/ndb/src/kernel/vm/NdbdSuperPool.hpp	06/01/22 18:23:43
/* Copyright (C) 2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#ifndef NDBD_SUPER_POOL_HPP
#define NDBD_SUPER_POOL_HPP

#include "SuperPool.hpp"

struct AllocArea;

class NdbdSuperPool : public SuperPool
{
public:
  NdbdSuperPool(class Ndbd_mem_manager&, Uint32 pageSize, Uint32 pageBits);
  
  // Destructor.
  virtual ~NdbdSuperPool();
  
  // Get new page from current area.
  virtual PtrI getNewPage();

  // Call first...on all superpools (uses malloc)
  bool init_1(); 
  
  // Call second...uses mm
  bool init_2();
  
  virtual bool allocMemory() { return allocMem() != 0; }
private:
  Uint32 allocAreaMemory(AllocArea*, Uint32 pages);
  AllocArea* allocArea();
  AllocArea* allocMem();
  
  // List of malloc areas.
  Uint32 m_shift, m_add;
  class Ndbd_mem_manager & m_mm;

  AllocArea* m_currArea;
  AllocArea* m_firstArea;
};

#endif

--- New file ---
+++ storage/ndb/src/kernel/vm/bench_pool.cpp	06/01/22 18:23:44
/* Copyright (C) 2003 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */


#include "NdbdSuperPool.hpp"
#include "ArrayPool.hpp"
#include <NdbTick.h>
#include "ndbd_malloc_impl.hpp"

template <typename T>
inline
void
init(ArrayPool<T> & pool, Uint32 cnt)
{
  pool.setSize(cnt + 1);
}

template <typename T>
inline
void
init(RecordPool<T> & pool, Uint32 cnt)
{
}


template<typename T, typename R>
inline
void 
test_pool(R& pool, Uint32 cnt, Uint32 loops)
{
  init(pool, cnt);
  Ptr<T> ptr;
  Uint32 *arr = (Uint32*)alloca(cnt * sizeof(Uint32));
  {
    printf(" ; seize "); fflush(stdout);
    Uint64 sum = 0;
    for(Uint32 i = 0; i<loops; i++)
    {
      Uint64 start = NdbTick_CurrentMillisecond();
      for(Uint32 j = 0; j<cnt; j++)
      {
	bool b = pool.seize(ptr);
	arr[j] = ptr.i;
	assert(b);
      }
      Uint64 stop = NdbTick_CurrentMillisecond();      

      for(Uint32 j = 0; j<cnt; j++)
      {
	ptr.i = arr[j];
	pool.getPtr(ptr);
	pool.release(ptr);
	arr[j] = RNIL;
      }
      
      sum += (stop - start);
      if (i == 0)
	printf("; first ; %lld", (stop - start));
    }
    printf(" ; avg ; %lld ; tot ; %lld", sum/loops, sum);fflush(stdout);
  }

  {
    printf(" ; release "); fflush(stdout);
    Uint64 sum = 0;
    for(Uint32 i = 0; i<loops; i++)
    {
      for(Uint32 j = 0; j<cnt; j++)
      {
	bool b = pool.seize(ptr);
	arr[j] = ptr.i;
	assert(b);
      }

      Uint64 start = NdbTick_CurrentMillisecond();
      for(Uint32 j = 0; j<cnt; j++)
      {
	ptr.i = arr[j];
	pool.release(ptr);
	arr[j] = RNIL;
      }
      Uint64 stop = NdbTick_CurrentMillisecond();      
      
      sum += (stop - start);
    }
    printf("; avg ; %lld ; tot ; %lld", sum/loops, sum); fflush(stdout);
  }

  {
    printf(" ; mix"); fflush(stdout);
    
    Uint64 sum = 0;
    Uint64 start = NdbTick_CurrentMillisecond();
    for(Uint32 i = 0; i<loops * cnt; i++)
    {
      int pos = rand() % cnt;
      ptr.i = arr[pos];
      if (ptr.i == RNIL)
      {
	pool.seize(ptr);
	arr[pos] = ptr.i;
	assert(ptr.i != RNIL);
      }
      else
      {
	pool.release(ptr);
	arr[pos] = RNIL;
      }
    }
    Uint64 stop = NdbTick_CurrentMillisecond();    
    
    for(Uint32 j = 0; j<cnt; j++)
    {
      ptr.i = arr[j];
      if (ptr.i != RNIL)
      {
	pool.getPtr(ptr);
	pool.release(ptr);
      }
      arr[j] = RNIL;
    }
    
    sum += (stop - start);
    printf(" ; %lld", sum); fflush(stdout);
  }
  
  {
    printf(" ; getPtr"); fflush(stdout);

    for(Uint32 j = 0; j<cnt; j++)
    {
      bool b = pool.seize(ptr);
      arr[j] = ptr.i;
      assert(b);
    }

    Uint64 sum = 0;
    Uint64 start = NdbTick_CurrentMillisecond();
    for(Uint32 i = 0; i<loops * cnt; i++)
    {
      int pos = rand() % cnt;
      ptr.i = arr[pos];
      pool.getPtr(ptr);
    }
    Uint64 stop = NdbTick_CurrentMillisecond();    

    for(Uint32 j = 0; j<cnt; j++)
    {
      ptr.i = arr[j];
      pool.getPtr(ptr);
      pool.release(ptr);
      arr[j] = RNIL;
    }
    
    sum += (stop - start);
    printf(" ; %lld", sum); fflush(stdout);
  }
  ndbout_c("");
}

template <Uint32 sz> struct Rec { char data[sz-4]; Uint32 nextPool; };
typedef Rec<32> Rec32;
typedef Rec<36> Rec36;
typedef Rec<256> Rec256;
typedef Rec<260> Rec260;

Ndbd_mem_manager mem;

template <typename T>
inline
void test_rp(Uint32 cnt, Uint32 loop, Uint32 pgsz)
{
  printf("RP ; %d ; ws ; %d ; page ; %d", 
	 sizeof(T), (sizeof(T)*cnt) >> 10, pgsz >> 10);
  NdbdSuperPool sp(mem, pgsz, 19);
  GroupPool gp(sp);
  sp.init_1();
  sp.init_2();
  
  sp.setInitPages(4);
  sp.setIncrPages(4);
  sp.setMaxPages(~0);
  sp.allocMemory();
  
  RecordPool<T> pool(gp);
  test_pool<T, RecordPool<T> >(pool, cnt, loop);
}

template <typename T>
inline
void test_ap(Uint32 cnt, Uint32 loop)
{
  printf("AP ; %d ; ws ; %d ; page ; n/a", sizeof(T), (cnt * sizeof(T))>>10);
  ArrayPool<T> pool;
  test_pool<T, ArrayPool<T> >(pool, cnt, loop);
}

int
main(int argc, char **argv)
{
  mem.init(10000);

  Uint32 cnt = 100;
  Uint32 loop = 300000;

  while(cnt <= 1000000)
  {
    test_rp<Rec32>(cnt, loop, 8192);
    test_rp<Rec32>(cnt, loop, 32768);
    test_ap<Rec32>(cnt, loop);
    
    test_rp<Rec36>(cnt, loop, 8192);
    test_rp<Rec36>(cnt, loop, 32768);
    test_ap<Rec36>(cnt, loop);

    test_rp<Rec256>(cnt, loop, 8192);
    test_rp<Rec256>(cnt, loop, 32768);
    test_ap<Rec256>(cnt, loop);

    test_rp<Rec260>(cnt, loop, 8192);
    test_rp<Rec260>(cnt, loop, 32768);
    test_ap<Rec260>(cnt, loop);

    cnt *= 100;
    loop /= 100;
  }
}

void
ErrorReporter::handleAssert(const char * msg, const char * file, 
			    int line, int)
{
  ndbout << "ErrorReporter::handleAssert activated - " 
	 << " line= " << line << endl;
  abort();
}


--- 1.2/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp	2006-01-12 15:39:45 +01:00
+++ 1.3/storage/ndb/src/kernel/vm/ndbd_malloc_impl.cpp	2006-01-22 18:23:43 +01:00
@@ -84,7 +84,7 @@
     release(start+1, end - 1 - start);    
   }
 
-  return 0;
+  return true;
 }
 
 void
@@ -198,6 +198,25 @@
     }
   }
   * pages = 0;
+}
+
+void*
+Ndbd_mem_manager::alloc(Uint32 *pages, Uint32 min)
+{
+  Uint32 ret;
+  alloc(&ret, pages, min);
+  if (pages)
+  {
+    return m_base_page + ret;
+  }
+  return 0;
+}
+
+void
+Ndbd_mem_manager::release(void* ptr, Uint32 cnt)
+{
+  Uint32 page = ((Alloc_page*)ptr) - m_base_page;
+  release(page, cnt);
 }
 
 void

--- 1.1/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp	2006-01-11 11:35:20 +01:00
+++ 1.2/storage/ndb/src/kernel/vm/ndbd_malloc_impl.hpp	2006-01-22 18:23:43 +01:00
@@ -64,8 +64,12 @@
   bool grow(Uint32 pages = 0);
 
   void dump() const ;
-private:
-  
+
+  void* get_memroot() const { return (void*)m_base_page;}
+
+  void* alloc(Uint32 * pages, Uint32 min_requested);
+  void release(void* ptr, Uint32 cnt);
+
   /**
    * Compute 2log of size 
    * @note size = 0     -> 0
@@ -73,6 +77,7 @@
    */
   static Uint32 log2(Uint32 size);
 
+private:
   /**
    * Return pointer to free page data on page
    */

--- 1.4/storage/ndb/src/kernel/vm/SuperPool.cpp	2005-11-08 23:47:38 +01:00
+++ 1.5/storage/ndb/src/kernel/vm/SuperPool.cpp	2006-01-22 18:23:43 +01:00
@@ -644,7 +644,7 @@
   if (ap->m_currPage == ap->m_numPages) {
     // area is used up
     if (ap->m_nextArea == 0) {
-      if (! allocMemory())
+      if (! allocMemoryImpl())
         return RNIL;
     }
     ap = m_currArea = ap->m_nextArea;
@@ -711,7 +711,7 @@
 }
 
 bool
-HeapPool::allocMemory()
+HeapPool::allocMemoryImpl()
 {
   if (! allocInit())
     return false;

--- 1.5/storage/ndb/src/kernel/vm/SuperPool.hpp	2005-11-08 23:47:38 +01:00
+++ 1.6/storage/ndb/src/kernel/vm/SuperPool.hpp	2006-01-22 18:23:43 +01:00
@@ -580,7 +580,8 @@
   bool allocArea(Area* ap, Uint32 tryPages);
 
   // Allocate memory.
-  virtual bool allocMemory();
+  virtual bool allocMemory() { return allocMemoryImpl();}
+  bool allocMemoryImpl();
 
   // List of malloc areas.
   Area m_areaHead;

--- 1.12/storage/ndb/src/kernel/vm/Makefile.am	2006-01-11 09:26:03 +01:00
+++ 1.13/storage/ndb/src/kernel/vm/Makefile.am	2006-01-22 18:23:43 +01:00
@@ -20,7 +20,8 @@
         Mutex.cpp SafeCounter.cpp \
         Rope.cpp \
 	SuperPool.cpp \
-	ndbd_malloc.cpp ndbd_malloc_impl.cpp
+	ndbd_malloc.cpp ndbd_malloc_impl.cpp \
+        NdbdSuperPool.cpp
 
 INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/mgmapi
 
@@ -44,10 +45,18 @@
 	@$(top_srcdir)/storage/ndb/config/win-sources $@ $(libkernel_a_SOURCES)
 	@$(top_srcdir)/storage/ndb/config/win-libraries $@ LIB $(LDADD)
 
-EXTRA_PROGRAMS = ndbd_malloc_impl_test
+EXTRA_PROGRAMS = ndbd_malloc_impl_test bench_pool
 ndbd_malloc_impl_test_CXXFLAGS = -DUNIT_TEST
 ndbd_malloc_impl_test_SOURCES = ndbd_malloc_impl.cpp
 ndbd_malloc_impl_test_LDFLAGS = @ndb_bin_am_ldflags@ \
+  $(top_builddir)/storage/ndb/src/libndbclient.la \
+  $(top_builddir)/mysys/libmysys.a \
+  $(top_builddir)/dbug/libdbug.a \
+  $(top_builddir)/strings/libmystrings.a
+
+bench_pool_SOURCES = bench_pool.cpp ndbd_malloc.cpp \
+                     SuperPool.cpp NdbdSuperPool.cpp ndbd_malloc_impl.cpp
+bench_pool_LDFLAGS = @ndb_bin_am_ldflags@ \
   $(top_builddir)/storage/ndb/src/libndbclient.la \
   $(top_builddir)/mysys/libmysys.a \
   $(top_builddir)/dbug/libdbug.a \
Thread
bk commit into 5.1 tree (jonas:1.2066)jonas22 Jan