From: Dyre Tjeldvoll Date: October 3 2012 7:38pm Subject: bzr push into mysql-5.5-cluster-7.2 branch (Dyre.Tjeldvoll:4082 to 4083) List-Archive: http://lists.mysql.com/commits/144964 Message-Id: <20121003193804.22424.52310.4083@khepri17.no.oracle.com> MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit 4083 Dyre Tjeldvoll 2012-10-03 Support configurable unit tests for backend modified: storage/ndb/mcc/tst/tst_main.py 4082 Dyre Tjeldvoll 2012-10-02 Use socket.gethostname() as the server name when using the WindowsDefault browser modified: storage/ndb/mcc/request_handler.py === modified file 'storage/ndb/mcc/tst/tst_main.py' --- a/storage/ndb/mcc/tst/tst_main.py 2012-09-26 09:39:16 +0000 +++ b/storage/ndb/mcc/tst/tst_main.py 2012-10-03 19:37:27 +0000 @@ -30,13 +30,16 @@ if pyminor < 7: else: import unittest utmod = unittest - + +import traceback import json import urlparse import time import socket import stat import os.path +import tempfile +import os import request_handler import config_parser @@ -46,20 +49,45 @@ import util from util import mock_msg, is_set from clusterhost import ABClusterHost, produce_ABClusterHost, LocalClusterHost -tst_datadir = os.path.join(os.path.expanduser('~'), 'MySQL_Cluster', 'data') +# Default tst configuration. Can be overridden in config.json +_cfg = { 'debuglevel': None, 'username': None, 'password': None, 'host_list': ['localhost'] } +_cfg_loaded = False + +def cfg(): + global _cfg_loaded + if _cfg_loaded: + return _cfg + + try: + cfgfile = open('config.json') + fcfg = json.load(cfgfile) + except: + traceback.print_exc() + else: + for k in fcfg.keys(): + _cfg[k] = fcfg[k] + finally: + _cfg_loaded = True + + print '_cfg='+str(_cfg) + return _cfg + +request_handler.configdir = '..' + +def_datadir = os.path.join(os.path.expanduser('~'), 'MySQL_Cluster', 'data') SUNOS_HOST = {'name' : 'fimafeng08.no.oracle.com', 'memory' : 4096, 'hostInfoRep': { 'host': { 'name' : 'fimafeng08.no.oracle.com' }, 'hostRes' : {'ram' : 4096, 'cores' : 4, 'uname' : 'SunOS', - 'datadir': tst_datadir, + 'datadir': util.get_val(cfg(), 'datadir', def_datadir), 'installdir': None }}} LINUX_HOST = {'name': 'fimafeng09.no.oracle.com', 'memory': 7991, 'hostInfoRep': { 'host': { 'name' : 'fimafeng09.no.oracle.com' }, 'hostRes' : {'ram' : 7991, 'cores' : 4, 'uname': 'Linux', - 'datadir': tst_datadir, + 'datadir': util.get_val(cfg(), 'datadir', def_datadir), 'installdir': None }}} WINDOWS_HOST = {'name' : 'tyr04.no.oracle.com', @@ -67,11 +95,11 @@ WINDOWS_HOST = {'name' : 'tyr04.no.oracl 'hostInfoRep': { 'host': { 'name' : 'tyr04.no.oracle.com' }, 'hostRes' : {'ram' : 8190, 'cores' : 8, 'uname': 'CYGWIN' }}} -HOST_A = 'siv27.no.oracle.com' -HOST_B = 'siv28.no.oracle.com' +#HOST_A = 'siv27.no.oracle.com' +#HOST_B = 'siv28.no.oracle.com' def mock_produce_ABClusterHost(hostname='localhost'): - return produce_ABClusterHost(hostname=hostname, user='', pwd='') + return produce_ABClusterHost(hostname=hostname) def host_is_unreachable(hostname, port=22): return util.try_connect(hostname, port, False, retdict={ socket.EAI_NONAME : True}) @@ -82,16 +110,7 @@ def mock_msg_as_json(cmd, body): class Test00Utils(utmod.TestCase): def test_version_tuple(self): print [ int(filter(str.isdigit, vn)) for vn in ('2', '7', '2+')] - def test_ex(self): - self.assertTrue(True) - -class Test0ConfigIni(utmod.TestCase): - def setUp(self): - self.cp = config_parser.parse_config_ini('example_config.ini') - - def tearDown(self): - pass - + def test_Param(self): print util.Param({'name':'--c', 'val':'bar', 'sep':'@'}) @@ -99,12 +118,22 @@ class Test0ConfigIni(utmod.TestCase): print json.dumps(x) print util.Param('--b','foo') print util.Param('--d','baz','#') - - + def test_host_is_unreachable(self): - self.assertFalse(host_is_unreachable(LINUX_HOST['name'])) self.assertTrue(host_is_unreachable('some_non_existent_host_name')) + @utmod.skipIf(not cfg().has_key('host_list'), 'No host list') + def test_host_is_unreachable(self): + for h in cfg()['host_list']: + self.assertFalse(host_is_unreachable(h['name'])) + +class Test0ConfigIni(utmod.TestCase): + def setUp(self): + self.cp = config_parser.parse_config_ini('example_config.ini') + + def tearDown(self): + pass + def test_get_option_values(self): self.assertEqual(config_parser.get_option_value_set(self.cp, 'HostName'), set(['siv27','siv28'])) @@ -115,18 +144,15 @@ class Test0ConfigIni(utmod.TestCase): print config_parser.get_configvalues(self.cp) print config_parser.get_processes(self.cp) - #@utmod.skipIf(host_is_unreachable('siv27'), 'Skipping because siv27 is unreachable') @utmod.skip('hangs...') def test_create_cluster_from_config_ini(self): print clumodel.create_cluster_from_config_ini('example_config.ini') class Test1Json(utmod.TestCase): def setUp(self): - self.ssh = {'user': 'dtjeldvo', 'pwd': None } - pass + self.ssh = {'user': None, 'pwd': None } def tearDown(self): - #self.ch.drop() pass def test_to_json(self): @@ -134,46 +160,45 @@ class Test1Json(utmod.TestCase): json_string = json.dumps(obj) jobj = json.loads(json_string) self.assertEqual(obj, jobj) - + def test_hostInfoReq(self): json_str = mock_msg_as_json('hostInfoReq', {'ssh': self.ssh, - 'hostName': LINUX_HOST['name']}) + 'hostName': 'localhost'}) print 'hostInfoReq: '+json_str print request_handler.handle_req(json.loads(json_str)) + + @utmod.skipIf(not cfg().has_key('local_installdir'), 'local_installdir not set') + def test_createFileReq(self): + json_str = mock_msg_as_json('createFileReq', + {'ssh': self.ssh, 'file': {'hostName': 'localhost', 'path': tempfile.gettempdir(), 'name': 'foobar'}, + 'contentString': 'a=0\nb=1\n', 'contents': {'params': {'sep': None, 'para': [{'name': None, 'sep': None, 'val': None}]}}}) + print json_str + try: + print request_handler.handle_req(json.loads(json_str)) + finally: + try: + os.remove(os.path.join(tempfile.gettempdir(),'foobar')) + except: + pass + + @utmod.skipIf(not cfg().has_key('local_installdir'), 'local_installdir not set') def test_startClusterReq(self): - json_str = mock_msg_as_json('startClusterReq', {'ssh': self.ssh, - 'procs': [ - {'file': {'autoComplete': True, 'hostName': LINUX_HOST['name'], 'path': '/usr/local/cluster-mgt/cluster-7.1.19', 'name': 'ndb_mgmd'}, + json_str = mock_msg_as_json('startClusterReq', + {'ssh': self.ssh, + 'procs': [ + {'file': {'autoComplete': True, 'hostName': 'localhost', 'path': cfg()['local_installdir'], 'name': 'ndb_mgmd'}, 'procCtrl': {'hup':True, 'getStd': False}, 'params': { 'param': [ util.Param({'name':'Dummy1', 'sep':'=', 'val':42}) ], 'sep': ' ' }}]}) print json_str print request_handler.handle_req(json.loads(json_str)) - - @utmod.skip('Get going...') - def test_createFileReq(self): - json_str = mock_msg_as_json('createFileReq', - {'ssh': self.ssh, - 'file': {'hostName': LINUX_HOST['name'], 'path': '/tmp', 'name': 'foobar'}, - 'contentString': 'a=0\nb=1\n', - 'contents': {'params': {'sep': None, 'para': [{'name': None, 'sep': None, 'val': None}]}}}) - print json_str - print request_handler.handle_req(json.loads(json_str)) + + -@stripped('We do not rely on url-parsing anymore. Initial page is displayed using GET. The client uses POST') -class Test2GetUrl(utmod.TestCase): - def setUp(self): - self.path = "/page?A=a&B=b&C=c" - - def test_parse(self): - urldict = urlparse.parse_qs(urlparse.urlparse(self.path).query) - for urlkey in urldict.keys(): - self.assertEqual(urlkey, urldict[urlkey][0].upper()) - -@stripped(host_is_unreachable(LINUX_HOST['name']), 'Skipping because '+LINUX_HOST['name']+' is unreachable') +@stripped(not cfg().has_key('linux_host') or host_is_unreachable(cfg()['linux_host']['name']), 'Skipping because no reachable linux host') class Test5SSH(utmod.TestCase): def setUp(self): - self.chost = mock_produce_ABClusterHost(LINUX_HOST['name']) + self.chost = mock_produce_ABClusterHost(cfg()['linux_host']['name']) self.flst = [] example = open('example.txt', 'w') example.write("Some text here\nSome more text on another line\n") @@ -185,23 +210,18 @@ class Test5SSH(utmod.TestCase): def test_fg_exec(self): self.assertEqual(self.chost.exec_blocking(['uname']).strip(), 'Linux') - @utmod.skip('Pid has gone away?') - def test_daemon_exec(self): - self.chost.exec_blocking(['nohup', 'sleep', '30', '&', 'echo', '$!', '>', '/tmp/sleep_pid']) - - out = self.chost.exec_blocking(['cat', '/tmp/sleep_pid']) - #print "err=" + streams[2].read() - pid = out.strip() - print 'Will kill pid=' + pid; - - self.chost.exec_blocking(['kill', str(pid)]) - def test_copy_file(self): self.chost.sftp.put('example.txt', '/tmp/example.txt') + self.flst.append('/tmp/example.txt') self.chost.sftp.get('/tmp/example.txt', 'remote_example.txt') example = open('example.txt') remote_example = open('remote_example.txt') - self.assertEqual(example.read(), remote_example.read()) + try: + self.assertEqual(example.read(), remote_example.read()) + finally: + example.close() + remote_example.close() + os.remove('remote_example.txt') def test_mkdir(self): self.chost.sftp.mkdir('/tmp/some_new_dir/') @@ -214,10 +234,12 @@ class Test5SSH(utmod.TestCase): def test_stat_dir(self): self.assertEqual(stat.S_IFMT(self.chost.sftp.stat('/tmp').st_mode), stat.S_IFDIR) + @utmod.skipIf(not cfg().has_key('windows_host') or host_is_unreachable(cfg()['windows_host']['name']), 'Skipping, no windows host') def test_stat_dir_win(self): whost = mock_produce_ABClusterHost(WINDOWS_HOST['name']) self.assertTrue(stat.S_ISDIR(whost.sftp.stat('/tmp').st_mode)) - + + @utmod.skip('Cannot create files with files with attributes') def test_list_dir_no_x(self): nox_mode = self.chost.sftp.stat('/tmp/nox').st_mode notmine_mode = self.chost.sftp.stat('/tmp/not_mine').st_mode @@ -235,26 +257,26 @@ class Test5SSH(utmod.TestCase): self.chost.mkdir_p('/tmp/some_dir/') self.flst.append('/tmp/some_dir/') -@stripped(host_is_unreachable(LINUX_HOST['name']), 'Skipping because '+LINUX_HOST['name']+' is unreachable') +@stripped(not cfg().has_key('linux_host') or host_is_unreachable(cfg()['linux_host']['name']), 'Skipping because linux host is unavailable') class Test3Ports(utmod.TestCase): def testPortNotAvailable(self): - self.assertFalse(util.is_port_available(LINUX_HOST['name'], 22)) + self.assertFalse(util.is_port_available(cfg()['linux_host']['name'], 22)) def testPortAvailable(self): - self.assertTrue(util.is_port_available(LINUX_HOST['name'], 23)) + self.assertTrue(util.is_port_available(cfg()['linux_host']['name'], 23)) def testFindPort(self): - self.assertEqual(util.first_available_port(LINUX_HOST['name'], 22, 23), 23) + self.assertEqual(util.first_available_port(cfg()['linux_host']['name'], 22, 23), 23) def testNotFindPort(self): self.assertRaises(util.NoPortAvailableException, - util.first_available_port, LINUX_HOST['name'], 22, 22) + util.first_available_port, cfg()['linux_host']['name'], 22, 22) -@stripped(host_is_unreachable(LINUX_HOST['name']), 'Skipping because '+LINUX_HOST['name']+' is unreachable') +@stripped(not cfg().has_key('host_list'), 'No host list') class Test4ABClusterHost(utmod.TestCase): def setUp(self): self.ch = None - self.chlst = map(mock_produce_ABClusterHost, ['localhost', 'loki24', 'fimafeng08', 'tyr04']) + self.chlst = map(mock_produce_ABClusterHost, cfg()['host_list']) def tearDown(self): if self.ch != None: @@ -263,7 +285,6 @@ class Test4ABClusterHost(utmod.TestCase) def test_hostInfo(self): print [ch.hostInfo for ch in self.chlst] - #print [ch.open_func for ch in self.chlst] def test_fileops(self): for ch in self.chlst: @@ -280,25 +301,28 @@ class Test4ABClusterHost(utmod.TestCase) if f: f.close() ch.rm_r('/tmp/foo') - + + @utmod.skipIf(not cfg().has_key('linux_host'), 'No linux_host') def test_hostInfoRep_lin(self): - self.ch = mock_produce_ABClusterHost(LINUX_HOST['name']) - self.assertEqual(self.ch.hostInfo.rep, LINUX_HOST['hostInfoRep']) + self.maxDiff = None + self.ch = mock_produce_ABClusterHost(cfg()['linux_host']['name']) + self.assertEqual(self.ch.hostInfo.rep, cfg()['linux_host']['hostInfoRep']) + @utmod.skipIf(not cfg().has_key('sunos_host'), 'No sunos_host') def test_hostInfoRep_sol(self): - self.ch = mock_produce_ABClusterHost(SUNOS_HOST['name']) - self.assertEqual(self.ch.hostInfo.rep, SUNOS_HOST['hostInfoRep']) - + self.ch = mock_produce_ABClusterHost(cfg()['sunos_host']['name']) + self.assertEqual(self.ch.hostInfo.rep, cfg()['sunos_host']['hostInfoRep']) + + @utmod.skipIf(not cfg().has_key('windows_host'), 'No windows_host') def test_hostInfoRep_win(self): - self.ch = mock_produce_ABClusterHost(WINDOWS_HOST['name']) - self.assertEqual(self.ch.hostInfo.rep, WINDOWS_HOST['hostInfoRep']) + self.ch = mock_produce_ABClusterHost(cfg()['windows_host']['name']) + self.assertEqual(self.ch.hostInfo.rep, cfg()['windows_host']['hostInfoRep']) def test_hostInfoRep_localhost(self): self.ch = mock_produce_ABClusterHost('localhost') self.assertTrue(isinstance(self.ch, LocalClusterHost)) print self.ch.hostInfo.rep -#@utmod.skipIf(host_is_unreachable('siv27'), 'Skipping because siv27 is unreachable') @utmod.skip('Hangs') class Test6Cluster(utmod.TestCase): def setUp(self): @@ -344,19 +368,19 @@ class Test6Cluster(utmod.TestCase): print 'ndb config as json: ', json.dumps(util.xml_to_python(self.cluster.ndb_config_xml())) + + if __name__ == '__main__': - if len(sys.argv) > 1 and sys.argv[1] == '-d': + if cfg()['debuglevel'] is not None: import logging fmt = '%(asctime)s: %(levelname)s [%(funcName)s;%(filename)s:%(lineno)d]: %(message)s ' - logging.basicConfig(level=logging.DEBUG, format=fmt) - utmod.main(argv=sys.argv[:1]+sys.argv[2:]) + logging.basicConfig(level=getattr(logging, cfg()['debuglevel']), format=fmt) + + try: + import xmlrunner2 + except: + traceback.print_exc() + assert(False) + utmod.main(argv=sys.argv) else: - try: - import xmlrunner2 - except: - import traceback - traceback.print_exc() - assert(False) - utmod.main(argv=sys.argv) - else: - utmod.main(argv=sys.argv, testRunner=xmlrunner2.XMLTestRunner) + utmod.main(argv=sys.argv, testRunner=xmlrunner2.XMLTestRunner) No bundle (reason: useless for push emails).