import subprocess, time, shutil, os, sys, glob, email.Message, smtplib, traceback, re def run(cmd, onestring=True, fatal=True): global log process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output=[] while True: line = process.stdout.readline() if not line: break print >>log, line, output.append(line) retcode = process.wait() if retcode and fatal: raise "running program failed (%s)" + " ".join(cmd) if onestring: return retcode, "".join(output) else: return retcode, output debug_on=False engine_re = re.compile("[\-\+]\) (ENGINE|TYPE)=M(yISAM|ARIA)") select_types = ["SIMPLE","PRIMARY","UNION","DEPENDENT UNION","UNION RESULT","SUBQUERY","DEPENDENT SUBQUERY","DERIVED","UNCACHEABLE SUBQUERY","UNCACHEABLE UNION"] def dprint(thing): global debug_on if debug_on: print thing def filter_hunk(hunk): global engine_re global select_types dprint("dealing with") dprint(hunk) for i in hunk: if i[0] in ("-","+"): # changed line if engine_re.match(i): # ENGINE clause in SHOW CREATE TABLE continue cols = i.split("\t") if len(cols)==10 and cols[1] in select_types: # EXPLAIN output continue dprint("Line not ignored :") dprint(i) return "".join(hunk) return "" def find_and_filter_hunks(lines): output="" in_hunk=False for i in lines: if i.startswith("@@ -"): if in_hunk: output += filter_hunk(hunk) else: in_hunk=True hunk=[] if in_hunk: hunk.append(i) if i.startswith("mysqltest: Result"): # end of hunk in_hunk=False output += filter_hunk(hunk) hunk=[] else: if i in ("Stopping All Servers\n","Restoring snapshot of databases\n","Resuming Tests\n","\n"): continue output += i return output assert(len(sys.argv) <= 5) nopull = nobuild = norotate = False use_parent=[] for i in sys.argv: if i == "nopull": nopull = True elif i == "nobuild": nobuild = True elif i == "norotate": norotate = True elif i.startswith("parent="): use_parent=[i[7:]] if not norotate: assert(not nobuild) branch = os.getcwd() branch_base = os.path.basename(branch) checkouts = "%s/../../checkouts/%s" % (branch, branch_base) log = open(("%s/loop_log" % checkouts), "w", 1) # line-buffered to be up-to-date def try_build(): global nopull global nobuild global norotate global use_parent global branch date = time.asctime() mailtext = date+"\n" os.chdir(branch) if not nopull: retcode, output = run((["bzr", "missing"] + use_parent), fatal=False) if "Branches are up to date." in output: return 0 print "Pulling" mailtext += "Pulling:\n"+output run(["bzr", "pull"] + use_parent) retcode, output = run(["bzr", "log", "-r-1", "--show-ids"]) mailtext += "Latest:\n"+output for i in range(-1,0+1): try: os.mkdir("%s/checkout%d" % (checkouts,i)) except: pass if not norotate: shutil.rmtree("%s/checkout-1" % checkouts) shutil.move(("%s/checkout0" % checkouts),("%s/checkout-1" % checkouts)) if not nobuild: try: shutil.rmtree("%s/checkout0" % checkouts) except: pass print "Making checkout" run(["bzr", "checkout", "--lightweight", ".", ("%s/checkout0" % checkouts)]) os.chdir("%s/checkout0" % checkouts) if not nobuild: print "Building" retcode, output = run("BUILD/compile-pentium-valgrind-max", fatal=False) if retcode: mailtext += "BUILD FAILURE" tell_user(mailtext, big_problem=False) return 1 keepdir = date.replace(' ','_').replace(':','_') keepdir = "%s/%s" % (checkouts,keepdir) os.mkdir(keepdir) os.chdir("./mysql-test") print "Testing" os.environ["MTR_BUILD_THREAD"]="auto" test_types={"nm":[], "pr":["--ps-protocol","--mysqld=--binlog-format=row"]} for test_type in ["nm","pr"]: print test_type mailtext+= "test type: "+test_type+"\n" retcode, output = run(["./mtr", "--mysqld=--default-storage-engine=maria", "--force", "--notimer", "--mem"] + test_types[test_type], onestring=False, fatal=False) output2 = [] for i in output: # no time in diff if i.find(".result\t200") != -1 or i.find(".reject\t200") != -1: continue for varying in ["mysql-test-run: WARNING: Forcing kill of process ","Saving core."]: if i.startswith(varying): i = varying + "\n" # eliminate core and PID numbers output2.append(i) output2 = find_and_filter_hunks(output2) mtr_out_name = test_type+"_mtr.out" mtr_out = open(mtr_out_name,"w") mtr_out.write("".join(output2)) mtr_out.close() shutil.copy(mtr_out_name, keepdir+"/"+mtr_out_name) for err_file in glob.glob("var/log/*.err"): shutil.copy(err_file, keepdir+"/"+test_type+"_"+os.path.basename(err_file)) bad_crashes = [] current_test = "unknown" for i in open(err_file).readlines(): if i.find("CURRENT_TEST") != -1: current_test = i[14:-1] if i.find("This could be because you hit a bug") != -1: bad_crashes.append(current_test) if bad_crashes: mailtext += "%d bad crashes in %s\n" % (len(bad_crashes), err_file) mailtext += ", ".join(bad_crashes)+"\n" retcode, output = run(["diff","-u",("%s/checkout-1/mysql-test/%s" % (checkouts,mtr_out_name)),mtr_out_name], fatal=False) if retcode in [0,1]: mailtext += "diff of mtr's output\n" mailtext += output else: mailtext += "some mtr.out file must be missing for diff\n" shutil.rmtree(os.path.realpath("var")) # free RAM; race cond as we just freed MTR_BUILD_THREAD # todo: attach mtr.out and *.err as (gzipped) files # not urgent as they are saved locally tell_user(mailtext, big_problem=False) print "Done" return 2 def tell_user(body, big_problem): global branch_base print "Mailing" msg = email.Message.Message() msg["from"]=some address msg["to"]=your address subject="result of mtr default-engine=maria branch %s" % branch_base if big_problem: subject+= " BIG PROBLEM" msg["subject"] = subject msg.set_payload(body) mailserver = smtplib.SMTP("localhost") msg_failed = mailserver.sendmail(msg["from"],msg["to"],str(msg)) assert(not msg_failed) while True: try: ret = try_build() if nopull: break if ret == 0: #NOTHING_TO_DO time.sleep(10*60) continue if ret == 1: #BUILD_FAILURE continue # won't do anything until next push if ret == 2: #TEST_WENT_THROUGH continue except: tell_user(traceback.format_exc(), big_problem=True) break print "Dying" sys.exit(1)