#At bzr+ssh://bk-internal.mysql.com/bzrroot/mysql-falcon/ based on revid:john.embretsen@stripped
403 John H. Embretsen 2009-06-08
Falcon DBT2 automation scripts: Adjustments to be able to run on Solaris (in TRD). Part 1.
- "cp -a" => "cp -pPR" (-a option not supported in default cp on Solaris).
- "hostname -s" => "hostname" (made platform dependent, -s option means "set" not "short" on Solaris).
- host-specific locations of data directories + size of these directories (may depend on filesystem).
- fixed path to include /opt/csw/bin. Also using Perl in this location (has most modules installed).
- added solaris x86-64-specific configure script.
- changed configure --prefix to local directory and removed other paths pointing to central locations.
Other fixes:
- "mkdir" => "mkdir -p" (make parent directories as needed) for warehouse dirs.
- redirecting output from configure command to build log.
- removed -W options from configure scripts (solaris). Warnings should be detected and handled elsewhere (e.g. Pushbuild).
added:
dbt2/include/configure_solaris_x86-64_gcc.inc
modified:
dbt2/bzr_checkout_dbt2.sh
dbt2/include/configure_solaris_x86_gcc.inc
dbt2/include/host_caneland.inc
dbt2/include/host_fimafeng13.inc
dbt2/include/host_lu0009.inc
dbt2/include/host_nehalem-1.inc
dbt2/include/host_tor06.inc
dbt2/include/host_walldorf.inc
dbt2/machine_info.sh
dbt2/wait_for_dbt2_and_collect.sh
dbt2/weekly_dbt2_overview.sh
=== modified file 'dbt2/bzr_checkout_dbt2.sh'
--- a/dbt2/bzr_checkout_dbt2.sh 2008-12-18 06:59:59 +0000
+++ b/dbt2/bzr_checkout_dbt2.sh 2009-06-08 11:53:58 +0000
@@ -15,9 +15,19 @@
# $Id$
#
-# Hostname. Needed for various paths, directories, includes.
+# Environment.
+#
+PLATFORM=$(uname)
+
#
-HOSTNAME=$(hostname -s)
+# Hostname. Needed for various paths, directories, includes.
+# "hostname -s" means different things on Linux vs. Solaris, so adapt.
+# Using -s on linux to get short hostname only.
+if [ "${PLATFORM}" = "SunOS" ]; then
+ HOSTNAME=$(hostname)
+else
+ HOSTNAME=$(hostname -s)
+fi
#
# Includes.
=== added file 'dbt2/include/configure_solaris_x86-64_gcc.inc'
--- a/dbt2/include/configure_solaris_x86-64_gcc.inc 1970-01-01 00:00:00 +0000
+++ b/dbt2/include/configure_solaris_x86-64_gcc.inc 2009-06-08 11:53:58 +0000
@@ -0,0 +1,52 @@
+#
+# Configuration for compiling MySQL on Solaris/x86-64 with gcc.
+#
+# $Id$
+# Hakan Kuecuekyilmaz, <hakan at mysql dot com>, 2008-07-16.
+
+CCACHE_TEST=$(ccache -V 2> /dev/null)
+if [ "x$CCACHE_TEST" != "x" ]; then
+ CCACHE=$(which ccache)
+else
+ CCACHE=''
+fi
+
+export CC="$CCACHE gcc"
+export CXX="$CCACHE g++"
+
+export CFLAGS="-g \
+ -m64 \
+ -O3 \
+ -fno-omit-frame-pointer \
+ -DFORCE_INIT_OF_VARS"
+
+export CXXFLAGS="-g \
+ -m64 \
+ -O3 \
+ -fno-omit-frame-pointer \
+ -DFORCE_INIT_OF_VARS"
+
+export LDFLAGS="-g \
+ -m64"
+
+export ASFLAGS="-g \
+ -m64"
+
+export CONFIGURATION="\
+ --disable-shared \
+ --enable-assembler \
+ --enable-local-infile \
+ --enable-thread-safe-client \
+ --prefix=$PWD/install \
+ --with-big-tables \
+ --with-client-ldflags='-static' \
+ --with-comment='MySQL-Community-Server' \
+ --with-extra-charsets=all \
+ --with-libevent \
+ --with-mysqld-ldflags='-static' \
+ --with-plugins=max-no-ndb \
+ --with-readline \
+ --with-ssl \
+ --with-zlib-dir=bundled"
+
+# --enable-dtrace causes build error on Solaris x86_64 (unknown why).
=== modified file 'dbt2/include/configure_solaris_x86_gcc.inc'
--- a/dbt2/include/configure_solaris_x86_gcc.inc 2008-12-11 11:03:37 +0000
+++ b/dbt2/include/configure_solaris_x86_gcc.inc 2009-06-08 11:53:58 +0000
@@ -17,49 +17,11 @@ export CXX="$CCACHE g++"
export CFLAGS="-g \
-O3 \
-fno-omit-frame-pointer \
- -Wunused-function \
- -Wunused-label \
- -Wunused-value \
- -Wunused-variable \
- -Wimplicit \
- -Wreturn-type \
- -Wswitch \
- -Wtrigraphs \
- -Wcomment \
- -W \
- -Wa \
- -Wchar-subscripts \
- -Wformat \
- -Wparentheses \
- -Wsign-compare \
- -Wwrite-strings \
- -Wuninitialized \
- -Wunused \
-DFORCE_INIT_OF_VARS"
export CXXFLAGS="-g \
-O3 \
-fno-omit-frame-pointer \
- -Wunused-function \
- -Wunused-label \
- -Wunused-value \
- -Wunused-variable \
- -Wimplicit \
- -Wreturn-type \
- -Wswitch \
- -Wtrigraphs \
- -Wcomment \
- -W \
- -Wa \
- -Woverloaded-virtual \
- -Wchar-subscripts \
- -Wformat \
- -Wparentheses \
- -Wsign-compare \
- -Wwrite-strings \
- -Wctor-dtor-privacy \
- -Wnon-virtual-dtor \
- -Wuninitialized \
-DFORCE_INIT_OF_VARS"
export CONFIGURATION="--disable-shared \
@@ -67,15 +29,13 @@ export CONFIGURATION="--disable-shared \
--enable-dtrace \
--enable-local-infile \
--enable-thread-safe-client \
- --libexecdir=/usr/local/mysql/bin \
- --localstatedir=/usr/local/mysql/data \
--with-big-tables \
--with-client-ldflags='-static' \
+ --with-mysqld-ldflags='-static' \
--with-comment='MySQL-Community-Server' \
--with-extra-charsets=all \
--with-plugins=max-no-ndb \
- --with-mysqld-ldflags='-static' \
- --prefix=/usr/local/mysql \
+ --prefix=$PWD/install \
--with-readline \
--with-ssl \
--with-libevent \
=== modified file 'dbt2/include/host_caneland.inc'
--- a/dbt2/include/host_caneland.inc 2009-05-04 17:52:24 +0000
+++ b/dbt2/include/host_caneland.inc 2009-06-08 11:53:58 +0000
@@ -58,6 +58,11 @@ W100_DATA_COPY='/data0/tmp/dbt2-w100-dat
# Where to find datagen (a DBT2 tool).
DATAGEN='/usr/local/bin/datagen'
+# DBT2 data directories, per warehouse setting
+W_DIR[10]='/tmp/dbt2-w10'
+W_DIR[20]='/tmp/dbt2-w20'
+W_DIR[100]='/tmp/dbt2-w100'
+
# Expected dbt2 data directory sizes; may vary between platforms and file systems.
W_SIZE[10]='677M'
W_SIZE[20]='1.4G'
=== modified file 'dbt2/include/host_fimafeng13.inc'
--- a/dbt2/include/host_fimafeng13.inc 2008-12-18 06:38:48 +0000
+++ b/dbt2/include/host_fimafeng13.inc 2009-06-08 11:53:58 +0000
@@ -14,8 +14,6 @@
# John H. Embretsen, <john dot embretsen at sun dot com>, 2008-07-18.
# $Id$
-## Todo: Rename DBT2_DIR to DBT2_SCRIPTS?
-
# Where to send email in case of severe errors.
ADMIN='john.embretsen@stripped'
@@ -33,14 +31,14 @@ FALCONQA_BASEDIR="/export/home/tmp/falco
WORK="${FALCONQA_BASEDIR}/dbt2/work"
# Where to find patches to apply to the checked out code base before building.
-PATCHES="$HOME/work/mysql/patches/falcon/$(hostname)"
+PATCHES="$HOME/work/mysql/patches/falcon/dbt2/$(hostname)"
# Base directory of the helper scripts for DBT2 w/Falcon.
# Also the place where the results are stored.
BZR_BASE="${FALCONQA_BASEDIR}/bzr/mysql-falcon/dbt2"
# Where to find the DBT2 for MySQL installation
-DBT2_DIR="${FALCONQA_BASEDIR}/dbt2/dbt2-fixed-20042007"
+DBT2_DIR="${FALCONQA_BASEDIR}/dbt2/dbt2-fixed-20090401"
# Where to find DBT2 test scripts.
DBT2_SCRIPTS="${DBT2_DIR}/scripts"
@@ -57,10 +55,15 @@ LOG_DIR="${FALCONQA_BASEDIR}/dbt2/log"
# MySQL and reuse it for consecutive runs.
W100_DATA_COPY="${FALCONQA_BASEDIR}/dbt2/tmp/dbt2-w100-datadir"
+# DBT2 data directories, per warehouse setting
+W_DIR[10]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w10"
+W_DIR[20]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w20"
+W_DIR[100]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w100"
+
# Expected dbt2 data directory sizes; may vary between platforms.
W_SIZE[10]='675M'
-W_SIZE[20]='1.4G'
-W_SIZE[100]='6.7G'
+W_SIZE[20]='1.3G'
+W_SIZE[100]='6.6G'
# Which runs should be oprofile'd?
#
@@ -76,17 +79,17 @@ DO_OPROFILE[100]=''
# Binaries
##################
# On Sun Trondheim hosts /usr/local/bin is central installation, /usr/bin/ is local
-PATH="/opt/csw/bin:/usr/local/bin:/usr/bin:/bin:/usr/sfw/bin:/usr/sbin:/opt/SUNWspro/bin:/usr/ucb:/usr/ccs/bin:$PATH"
+PATH="/opt/csw/bin:/opt/csw/php5/bin:/usr/local/bin:/usr/bin:/bin:/usr/sfw/bin:/usr/sbin:/opt/SUNWspro/bin:/usr/ucb:/usr/ccs/bin:$PATH"
PATCH=gpatch
MAKE=gmake
-OPCONTROL='sudo /usr/bin/opcontrol'
-OPREPORT='sudo /usr/bin/opreport'
+OPCONTROL='sudo /usr/bin/opcontrol' # NOT YET AVAILABLE ON TRD SOLARIS HOSTS
+OPREPORT='sudo /usr/bin/opreport' # NOT YET AVAILABLE ON TRD SOLARIS HOSTS
# Path to mail program which accepts -s for sending mails.
MAIL=/usr/ucb/mail
# Path to Perl with Statistics::Descriptive.
-PERL=/usr/bin/perl
+PERL=/opt/csw/bin/perl
# Where to find datagen (a DBT2 tool).
DATAGEN="${DBT2_DIR}/src/datagen"
=== modified file 'dbt2/include/host_lu0009.inc'
--- a/dbt2/include/host_lu0009.inc 2008-12-18 06:38:48 +0000
+++ b/dbt2/include/host_lu0009.inc 2009-06-08 11:53:58 +0000
@@ -59,6 +59,11 @@ W100_DATA_COPY='/data0/tmp/dbt2-w100-dat
# Where to find datagen (a DBT2 tool).
DATAGEN='/usr/local/bin/datagen'
+# DBT2 data directories, per warehouse setting
+W_DIR[10]='/tmp/dbt2-w10'
+W_DIR[20]='/tmp/dbt2-w20'
+W_DIR[100]='/tmp/dbt2-w100'
+
# Expected dbt2 data directory sizes; may vary between platforms and file systems.
W_SIZE[10]='677M'
W_SIZE[20]='1.4G'
=== modified file 'dbt2/include/host_nehalem-1.inc'
--- a/dbt2/include/host_nehalem-1.inc 2009-04-01 02:24:48 +0000
+++ b/dbt2/include/host_nehalem-1.inc 2009-06-08 11:53:58 +0000
@@ -60,6 +60,11 @@ W100_DATA_COPY='/data0/tmp/dbt2-w100-dat
# Where to find datagen (a DBT2 tool).
DATAGEN='/usr/local/bin/datagen'
+# DBT2 data directories, per warehouse setting
+W_DIR[10]='/tmp/dbt2-w10'
+W_DIR[20]='/tmp/dbt2-w20'
+W_DIR[100]='/tmp/dbt2-w100'
+
# Expected dbt2 data directory sizes; may vary between platforms and file systems.
W_SIZE[10]='677M'
W_SIZE[20]='1.4G'
=== modified file 'dbt2/include/host_tor06.inc'
--- a/dbt2/include/host_tor06.inc 2008-12-18 06:38:48 +0000
+++ b/dbt2/include/host_tor06.inc 2009-06-08 11:53:58 +0000
@@ -57,10 +57,15 @@ LOG_DIR="${FALCONQA_BASEDIR}/dbt2/log"
# MySQL and reuse it for consecutive runs.
W100_DATA_COPY="${FALCONQA_BASEDIR}/dbt2/tmp/dbt2-w100-datadir"
+# DBT2 data directories, per warehouse setting
+W_DIR[10]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w10"
+W_DIR[20]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w20"
+W_DIR[100]="${FALCONQA_BASEDIR}/dbt2/warehousedata/dbt2-w100"
+
# Expected dbt2 data directory sizes; may vary between platforms.
W_SIZE[10]='675M'
-W_SIZE[20]='1.4G'
-W_SIZE[100]='6.7G'
+W_SIZE[20]='1.3G'
+W_SIZE[100]='6.6G'
# Which runs should be oprofile'd?
#
=== modified file 'dbt2/include/host_walldorf.inc'
--- a/dbt2/include/host_walldorf.inc 2009-03-31 23:50:38 +0000
+++ b/dbt2/include/host_walldorf.inc 2009-06-08 11:53:58 +0000
@@ -58,6 +58,11 @@ W100_DATA_COPY='/data0/tmp/dbt2-w100-dat
# Where to find datagen (a DBT2 tool).
DATAGEN='/usr/local/bin/datagen'
+# DBT2 data directories, per warehouse setting
+W_DIR[10]='/tmp/dbt2-w10'
+W_DIR[20]='/tmp/dbt2-w20'
+W_DIR[100]='/tmp/dbt2-w100'
+
# Expected dbt2 data directory sizes; may vary between platforms and file systems.
W_SIZE[10]='677M'
W_SIZE[20]='1.4G'
=== modified file 'dbt2/machine_info.sh'
--- a/dbt2/machine_info.sh 2008-09-12 22:05:34 +0000
+++ b/dbt2/machine_info.sh 2009-06-08 11:53:58 +0000
@@ -5,7 +5,18 @@
# Hakan Kuecuekyilmaz <hakan at mysql dot com>, 2007-08-07.
# $Id$
-HOSTNAME=$(hostname -s)
+PLATFORM=$(uname)
+
+#
+# Hostname. Needed for various paths, directories, includes.
+# "hostname -s" means different things on Linux vs. Solaris, so adapt.
+# Using -s on linux to get short hostname only.
+if [ "${PLATFORM}" = "SunOS" ]; then
+ HOSTNAME=$(hostname)
+else
+ HOSTNAME=$(hostname -s)
+fi
+
UNAME=$(uname -a)
MEM=$(cat /proc/meminfo | grep MemTotal | awk -F : '{ print $2 }')
CPUS=$(cat /proc/cpuinfo | grep processor | wc -l)
=== modified file 'dbt2/wait_for_dbt2_and_collect.sh'
--- a/dbt2/wait_for_dbt2_and_collect.sh 2008-10-24 16:53:59 +0000
+++ b/dbt2/wait_for_dbt2_and_collect.sh 2009-06-08 11:53:58 +0000
@@ -2,21 +2,44 @@
#
# Wait for DBT2 runs to finish and generate report.
#
+# First author:
# Hakan Kuecuekyilmaz, <hakan at mysql dot com>, 2007-07-19.
# $Id$
#
-# PATH.
+# Environment.
#
-export PATH="/usr/local/php/bin:/usr/local/bin:$PATH"
-HOSTNAME=$(hostname -s)
+#
+# Hostname. Needed for various paths, directories, includes.
+# "hostname -s" means different things on Linux vs. Solaris, so adapt.
+# Using -s on linux to get short hostname only.
+if [ "${PLATFORM}" = "SunOS" ]; then
+ HOSTNAME=$(hostname)
+else
+ HOSTNAME=$(hostname -s)
+fi
+
+#
+# Includes.
+#
+# There should be one include file per host, specifying
+# platform/host specific variables (paths etc.).
+#
+HOST_INCLUDE_FILE="include/host_${HOSTNAME}.inc"
+if [ ! -f "$HOST_INCLUDE_FILE" ]; then
+ echo "[ERROR]: Include file for host $HOSTNAME not found."
+ echo "We were looking for: $HOST_INCLUDE_FILE"
+ echo "Current directory: $PWD"
+ echo "Exiting."
+ exit 1
+fi
+source "$HOST_INCLUDE_FILE"
+
#
# Directories and files.
#
-WORK="/data0/work/mysql"
-BZR_BASE="${HOME}/work/mysql-falcon/dbt2"
PRODUCTION="hakank@stripped:~/public_html/${HOSTNAME}"
#FALCON="${WORK}/mysql-6.0"
=== modified file 'dbt2/weekly_dbt2_overview.sh'
--- a/dbt2/weekly_dbt2_overview.sh 2009-04-24 10:27:48 +0000
+++ b/dbt2/weekly_dbt2_overview.sh 2009-06-08 11:53:58 +0000
@@ -55,9 +55,20 @@ else
fi
#
-# Hostname. Needed for various paths, directories, includes.
+# Environment.
#
-HOSTNAME=$(hostname -s)
+PLATFORM=$(uname)
+ARCH=$(uname -m)
+
+#
+# Hostname. Needed for various paths, directories, includes.
+# "hostname -s" means different things on Linux vs. Solaris, so adapt.
+# Using -s on linux to get short hostname only.
+if [ "${PLATFORM}" = "SunOS" ]; then
+ HOSTNAME=$(hostname)
+else
+ HOSTNAME=$(hostname -s)
+fi
#
# Includes.
@@ -75,11 +86,6 @@ if [ ! -f "$HOST_INCLUDE_FILE" ]; then
fi
source "$HOST_INCLUDE_FILE"
-#
-# Environment.
-#
-PLATFORM=$(uname)
-ARCH=$(uname -m)
#
# Timestamps.
@@ -102,10 +108,6 @@ OLD_DBT2_RUNS="${BZR_BASE}/old_weekly_db
DBT2_ERROR_LOGS="${DBT2_RUNS}/w${WAREHOUSES}/err"
DBT2_IO_STATS="${DBT2_RUNS}/w${WAREHOUSES}/io_stats"
-W_DIR[10]='/tmp/dbt2-w10'
-W_DIR[20]='/tmp/dbt2-w20'
-W_DIR[100]='/tmp/dbt2-w100'
-
#
# Binaries.
#
@@ -273,8 +275,8 @@ if [ $COMPILE = 'compile' ]; then
;;
SunOS)
if [ x"$ARCH" = x'i86pc' ]; then
- # Solaris/x86 gcc
- source ${BZR_BASE}/include/configure_solaris_x86_gcc.inc >> $BUILD_LOG 2>&1
+ # Solaris/x86. Assume 64-bit for now. Not using Sun Studio due to Bug#43449.
+ source ${BZR_BASE}/include/configure_solaris_x86-64_gcc.inc >> $BUILD_LOG 2>&1
J=$(psrinfo | wc -l)
J=$(( $J + 1 ))
@@ -291,7 +293,7 @@ if [ $COMPILE = 'compile' ]; then
;;
esac
- ./configure $CONFIGURATION
+ ./configure $CONFIGURATION >> $BUILD_LOG 2>&1
$MAKE -j$J >> $BUILD_LOG 2>&1
if [ $? != 0 ]; then
@@ -309,7 +311,7 @@ fi
# Check that warehouse directory exists and has correct size.
if [ ! -d ${W_DIR[$WAREHOUSES]} ]; then
echo "[INFO]: Directory ${W_DIR[$WAREHOUSES]} does not exist. Creating and filling it." >> $LOG
- mkdir ${W_DIR[$WAREHOUSES]}
+ mkdir -p ${W_DIR[$WAREHOUSES]}
$DATAGEN -w $WAREHOUSES -d ${W_DIR[$WAREHOUSES]} --mysql
fi
@@ -343,11 +345,11 @@ if [ $ROTATE = 'rotate' ]; then
OLD_DATE=$(ls FALCON* | grep -v OPROFILE | sort -r | head -n1 | awk -F_ '{ print $2 }')
mkdir -p ${OLD_DBT2_RUNS}/${OLD_DATE}
mv ${DBT2_RUNS}/* ${OLD_DBT2_RUNS}/${OLD_DATE}
- cp -a ${BZR_BASE}/dbt2.css ${OLD_DBT2_RUNS}/${OLD_DATE}
- cp -a ${BZR_BASE}/images ${OLD_DBT2_RUNS}/${OLD_DATE}
- cp -a ${BZR_BASE}/include ${OLD_DBT2_RUNS}/${OLD_DATE}
- cp -a ${BZR_BASE}/weekly_dbt2_overview.php ${OLD_DBT2_RUNS}/${OLD_DATE}
- cp -a ${BZR_BASE}/weekly_dbt2_overview.png ${OLD_DBT2_RUNS}/${OLD_DATE}
+ cp -pPR ${BZR_BASE}/dbt2.css ${OLD_DBT2_RUNS}/${OLD_DATE}
+ cp -pPR ${BZR_BASE}/images ${OLD_DBT2_RUNS}/${OLD_DATE}
+ cp -pPR ${BZR_BASE}/include ${OLD_DBT2_RUNS}/${OLD_DATE}
+ cp -pPR ${BZR_BASE}/weekly_dbt2_overview.php ${OLD_DBT2_RUNS}/${OLD_DATE}
+ cp -pPR ${BZR_BASE}/weekly_dbt2_overview.png ${OLD_DBT2_RUNS}/${OLD_DATE}
break
fi
@@ -445,8 +447,8 @@ for ENGINE in FALCON INNODB
if [ $W100_DATA_LOADED -eq 1 -a $WAREHOUSES -eq 100 ]; then
# cp back stuff
cd $W100_DATA_COPY
- cp -a $DATA_DIR_FILES $DATA_DIR
- cp -a $LOG_DIR_FILES $LOG_DIR
+ cp -pPR $DATA_DIR_FILES $DATA_DIR
+ cp -pPR $LOG_DIR_FILES $LOG_DIR
start_mysqld
fi
@@ -468,9 +470,9 @@ for ENGINE in FALCON INNODB
mkdir $W100_DATA_COPY
cd $DATA_DIR
- cp -a $DATA_DIR_FILES $W100_DATA_COPY
+ cp -pPR $DATA_DIR_FILES $W100_DATA_COPY
cd $LOG_DIR
- cp -a $LOG_DIR_FILES $W100_DATA_COPY
+ cp -pPR $LOG_DIR_FILES $W100_DATA_COPY
start_mysqld
W100_DATA_LOADED=1
Attachment: [text/bzr-bundle] bzr/john.embretsen@sun.com-20090608115358-y0rlxz838nh0db83.bundle
| Thread |
|---|
| • bzr commit into mysql-falcon branch (john.embretsen:403) | John H. Embretsen | 8 Jun |
