#!/usr/bin/env python## run-tests.py - Run a set of tests on Mercurial## Copyright 2006 Matt Mackall <mpm@selenic.com>## This software may be used and distributed according to the terms of the# GNU General Public License version 2, incorporated herein by reference.# Modifying this script is tricky because it has many modes:# - serial (default) vs parallel (-jN, N > 1)# - no coverage (default) vs coverage (-c, -C, -s)# - temp install (default) vs specific hg script (--with-hg, --local)# - tests are a mix of shell scripts and Python scripts## If you change this script, it is recommended that you ensure you# haven't broken it by running it in various modes with a representative# sample of test scripts. For example:# # 1) serial, no coverage, temp install:# ./run-tests.py test-s*# 2) serial, no coverage, local hg:# ./run-tests.py --local test-s*# 3) serial, coverage, temp install:# ./run-tests.py -c test-s*# 4) serial, coverage, local hg:# ./run-tests.py -c --local test-s* # unsupported# 5) parallel, no coverage, temp install:# ./run-tests.py -j2 test-s*# 6) parallel, no coverage, local hg:# ./run-tests.py -j2 --local test-s*# 7) parallel, coverage, temp install:# ./run-tests.py -j2 -c test-s* # currently broken# 8) parallel, coverage, local install# ./run-tests.py -j2 -c --local test-s* # unsupported (and broken)## (You could use any subset of the tests: test-s* happens to match# enough that it's worth doing parallel runs, few enough that it# completes fairly quickly, includes both shell and Python scripts, and# includes some scripts that run daemon processes.)importdifflibimporterrnoimportoptparseimportosimportsubprocessimportshutilimportsignalimportsysimporttempfileimporttimeclosefds=os.name=='posix'defPopen4(cmd,bufsize=-1):p=subprocess.Popen(cmd,shell=True,bufsize=bufsize,close_fds=closefds,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)p.fromchild=p.stdoutp.tochild=p.stdinp.childerr=p.stderrreturnp# reserved exit code to skip test (used by hghave)SKIPPED_STATUS=80SKIPPED_PREFIX='skipped: 'FAILED_PREFIX='hghave check failed: 'PYTHON=sys.executablerequiredtools=["python","diff","grep","unzip","gunzip","bunzip2","sed"]defaults={'jobs':('HGTEST_JOBS',1),'timeout':('HGTEST_TIMEOUT',180),'port':('HGTEST_PORT',20059),}defparseargs():parser=optparse.OptionParser("%prog [options] [tests]")parser.add_option("-C","--annotate",action="store_true",help="output files annotated with coverage")parser.add_option("--child",type="int",help="run as child process, summary to given fd")parser.add_option("-c","--cover",action="store_true",help="print a test coverage report")parser.add_option("-f","--first",action="store_true",help="exit on the first test failure")parser.add_option("-i","--interactive",action="store_true",help="prompt to accept changed output")parser.add_option("-j","--jobs",type="int",help="number of jobs to run in parallel"" (default: $%s or %d)"%defaults['jobs'])parser.add_option("--keep-tmpdir",action="store_true",help="keep temporary directory after running tests"" (best used with --tmpdir)")parser.add_option("-R","--restart",action="store_true",help="restart at last error")parser.add_option("-p","--port",type="int",help="port on which servers should listen"" (default: $%s or %d)"%defaults['port'])parser.add_option("-r","--retest",action="store_true",help="retest failed tests")parser.add_option("-s","--cover_stdlib",action="store_true",help="print a test coverage report inc. standard libraries")parser.add_option("-t","--timeout",type="int",help="kill errant tests after TIMEOUT seconds"" (default: $%s or %d)"%defaults['timeout'])parser.add_option("--tmpdir",type="string",help="run tests in the given temporary directory")parser.add_option("-v","--verbose",action="store_true",help="output verbose messages")parser.add_option("-n","--nodiff",action="store_true",help="skip showing test changes")parser.add_option("--with-hg",type="string",metavar="HG",help="test using specified hg script rather than a ""temporary installation")parser.add_option("--local",action="store_true",help="shortcut for --with-hg=<testdir>/../hg")parser.add_option("--pure",action="store_true",help="use pure Python code instead of C extensions")foroption,defaultindefaults.items():defaults[option]=int(os.environ.get(*default))parser.set_defaults(**defaults)(options,args)=parser.parse_args()ifoptions.with_hg:ifnot(os.path.isfile(options.with_hg)andos.access(options.with_hg,os.X_OK)):parser.error('--with-hg must specify an executable hg script')ifnotos.path.basename(options.with_hg)=='hg':sys.stderr.write('warning: --with-hg should specify an hg script')ifoptions.local:testdir=os.path.dirname(os.path.realpath(sys.argv[0]))hgbin=os.path.join(os.path.dirname(testdir),'hg')ifnotos.access(hgbin,os.X_OK):parser.error('--local specified, but %r not found or not executable'%hgbin)options.with_hg=hgbinoptions.anycoverage=(options.coveroroptions.cover_stdliboroptions.annotate)ifoptions.anycoverageandoptions.with_hg:# I'm not sure if this is a fundamental limitation or just a# bug. But I don't want to waste people's time and energy doing# test runs that don't give the results they want.parser.error("sorry, coverage options do not work when --with-hg ""or --local specified")globalvlogifoptions.verbose:ifoptions.jobs>1oroptions.childisnotNone:pid="[%d]"%os.getpid()else:pid=Nonedefvlog(*msg):ifpid:printpid,forminmsg:printm,printelse:vlog=lambda*msg:Noneifoptions.jobs<1:print>>sys.stderr,'ERROR: -j/--jobs must be positive'sys.exit(1)ifoptions.interactiveandoptions.jobs>1:print'(--interactive overrides --jobs)'options.jobs=1return(options,args)defrename(src,dst):"""Like os.rename(), trade atomicity and opened files friendliness for existing destination support. """shutil.copy(src,dst)os.remove(src)defsplitnewlines(text):'''like str.splitlines, but only split on newlines. keep line endings.'''i=0lines=[]whileTrue:n=text.find('\n',i)ifn==-1:last=text[i:]iflast:lines.append(last)returnlineslines.append(text[i:n+1])i=n+1defparsehghaveoutput(lines):'''Parse hghave log lines. Return tuple of lists (missing, failed): * the missing/unknown features * the features for which existence check failed'''missing=[]failed=[]forlineinlines:ifline.startswith(SKIPPED_PREFIX):line=line.splitlines()[0]missing.append(line[len(SKIPPED_PREFIX):])elifline.startswith(FAILED_PREFIX):line=line.splitlines()[0]failed.append(line[len(FAILED_PREFIX):])returnmissing,faileddefshowdiff(expected,output):forlineindifflib.unified_diff(expected,output,"Expected output","Test output"):sys.stdout.write(line)deffindprogram(program):"""Search PATH for a executable program"""forpinos.environ.get('PATH',os.defpath).split(os.pathsep):name=os.path.join(p,program)ifos.access(name,os.X_OK):returnnamereturnNonedefchecktools():# Before we go any further, check for pre-requisite tools# stuff from coreutils (cat, rm, etc) are not testedforpinrequiredtools:ifos.name=='nt':p+='.exe'found=findprogram(p)iffound:vlog("# Found prerequisite",p,"at",found)else:print"WARNING: Did not find prerequisite tool: "+pdefcleanup(options):ifnotoptions.keep_tmpdir:vlog("# Cleaning up HGTMP",HGTMP)shutil.rmtree(HGTMP,True)defusecorrectpython():# some tests run python interpreter. they must use same# interpreter we use or bad things will happen.exedir,exename=os.path.split(sys.executable)ifexename=='python':path=findprogram('python')ifos.path.dirname(path)==exedir:returnvlog('# Making python executable in test path use correct Python')mypython=os.path.join(BINDIR,'python')try:os.symlink(sys.executable,mypython)exceptAttributeError:# windows fallbackshutil.copyfile(sys.executable,mypython)shutil.copymode(sys.executable,mypython)definstallhg(options):vlog("# Performing temporary installation of HG")installerrs=os.path.join("tests","install.err")pure=options.pureand"--pure"or""# Run installer in hg rootos.chdir(os.path.join(os.path.dirname(sys.argv[0]),'..'))cmd=('%s setup.py %s clean --all'' install --force --prefix="%s" --install-lib="%s"'' --install-scripts="%s" >%s 2>&1'%(sys.executable,pure,INST,PYTHONDIR,BINDIR,installerrs))vlog("# Running",cmd)ifos.system(cmd)==0:ifnotoptions.verbose:os.remove(installerrs)else:f=open(installerrs)forlineinf:printline,f.close()sys.exit(1)os.chdir(TESTDIR)usecorrectpython()vlog("# Installing dummy diffstat")f=open(os.path.join(BINDIR,'diffstat'),'w')f.write('#!'+sys.executable+'\n''import sys\n''files = 0\n''for line in sys.stdin:\n'' if line.startswith("diff "):\n'' files += 1\n''sys.stdout.write("files patched: %d\\n" % files)\n')f.close()os.chmod(os.path.join(BINDIR,'diffstat'),0700)ifoptions.anycoverage:vlog("# Installing coverage wrapper")os.environ['COVERAGE_FILE']=COVERAGE_FILEifos.path.exists(COVERAGE_FILE):os.unlink(COVERAGE_FILE)# Create a wrapper script to invoke hg via coverage.pyos.rename(os.path.join(BINDIR,"hg"),os.path.join(BINDIR,"_hg.py"))f=open(os.path.join(BINDIR,'hg'),'w')f.write('#!'+sys.executable+'\n')f.write('import sys, os; os.execv(sys.executable, [sys.executable, ''"%s", "-x", "-p", "%s"] + sys.argv[1:])\n'%(os.path.join(TESTDIR,'coverage.py'),os.path.join(BINDIR,'_hg.py')))f.close()os.chmod(os.path.join(BINDIR,'hg'),0700)defoutputcoverage(options):vlog('# Producing coverage report')os.chdir(PYTHONDIR)defcovrun(*args):start=sys.executable,os.path.join(TESTDIR,'coverage.py')cmd='"%s" "%s" %s'%(start[0],start[1],' '.join(args))vlog('# Running: %s'%cmd)os.system(cmd)omit=[BINDIR,TESTDIR,PYTHONDIR]ifnotoptions.cover_stdlib:# Exclude as system paths (ignoring empty strings seen on win)omit+=[xforxinsys.pathifx!='']omit=','.join(omit)covrun('-c')# combine from parallel processesforfninos.listdir(TESTDIR):iffn.startswith('.coverage.'):os.unlink(os.path.join(TESTDIR,fn))covrun('-i','-r','"--omit=%s"'%omit)# reportifoptions.annotate:adir=os.path.join(TESTDIR,'annotated')ifnotos.path.isdir(adir):os.mkdir(adir)covrun('-i','-a','"--directory=%s"'%adir,'"--omit=%s"'%omit)classTimeout(Exception):passdefalarmed(signum,frame):raiseTimeoutdefrun(cmd,options):"""Run command in a sub-process, capturing the output (stdout and stderr). Return the exist code, and output."""# TODO: Use subprocess.Popen if we're running on Python 2.4ifos.name=='nt'orsys.platform.startswith('java'):tochild,fromchild=os.popen4(cmd)tochild.close()output=fromchild.read()ret=fromchild.close()ifret==None:ret=0else:proc=Popen4(cmd)try:output=''proc.tochild.close()output=proc.fromchild.read()ret=proc.wait()ifos.WIFEXITED(ret):ret=os.WEXITSTATUS(ret)exceptTimeout:vlog('# Process %d timed out - killing it'%proc.pid)os.kill(proc.pid,signal.SIGTERM)ret=proc.wait()ifret==0:ret=signal.SIGTERM<<8output+=("\n### Abort: timeout after %d seconds.\n"%options.timeout)returnret,splitnewlines(output)defrunone(options,test,skips,fails):'''tristate output: None -> skipped True -> passed False -> failed'''defskip(msg):ifnotoptions.verbose:skips.append((test,msg))else:print"\nSkipping %s: %s"%(test,msg)returnNonedeffail(msg):fails.append((test,msg))ifnotoptions.nodiff:print"\nERROR: %s%s"%(test,msg)returnNonevlog("# Test",test)# create a fresh hgrchgrc=file(HGRCPATH,'w+')hgrc.write('[ui]\n')hgrc.write('slash = True\n')hgrc.write('[defaults]\n')hgrc.write('backout = -d "0 0"\n')hgrc.write('commit = -d "0 0"\n')hgrc.write('tag = -d "0 0"\n')hgrc.close()err=os.path.join(TESTDIR,test+".err")ref=os.path.join(TESTDIR,test+".out")testpath=os.path.join(TESTDIR,test)ifos.path.exists(err):os.remove(err)# Remove any previous output files# Make a tmp subdirectory to work intmpd=os.path.join(HGTMP,test)os.mkdir(tmpd)os.chdir(tmpd)try:tf=open(testpath)firstline=tf.readline().rstrip()tf.close()except:firstline=''lctest=test.lower()iflctest.endswith('.py')orfirstline=='#!/usr/bin/env python':cmd='%s "%s"'%(PYTHON,testpath)eliflctest.endswith('.bat'):# do not run batch scripts on non-windowsifos.name!='nt':returnskip("batch script")# To reliably get the error code from batch files on WinXP,# the "cmd /c call" prefix is needed. Grrrcmd='cmd /c call "%s"'%testpathelse:# do not run shell scripts on windowsifos.name=='nt':returnskip("shell script")# do not try to run non-executable programsifnotos.path.exists(testpath):returnfail("does not exist")elifnotos.access(testpath,os.X_OK):returnskip("not executable")cmd='"%s"'%testpathifoptions.timeout>0:signal.alarm(options.timeout)vlog("# Running",cmd)ret,out=run(cmd,options)vlog("# Ret was:",ret)ifoptions.timeout>0:signal.alarm(0)mark='.'skipped=(ret==SKIPPED_STATUS)# If reference output file exists, check test output against itifos.path.exists(ref):f=open(ref,"r")refout=splitnewlines(f.read())f.close()else:refout=[]ifskipped:mark='s'missing,failed=parsehghaveoutput(out)ifnotmissing:missing=['irrelevant']iffailed:fail("hghave failed checking for %s"%failed[-1])skipped=Falseelse:skip(missing[-1])elifout!=refout:mark='!'ifret:fail("output changed and returned error code %d"%ret)else:fail("output changed")ifnotoptions.nodiff:showdiff(refout,out)ret=1elifret:mark='!'fail("returned error code %d"%ret)ifnotoptions.verbose:sys.stdout.write(mark)sys.stdout.flush()ifret!=0andnotskipped:# Save errors to a file for diagnosisf=open(err,"wb")forlineinout:f.write(line)f.close()# Kill off any leftover daemon processestry:fp=file(DAEMON_PIDS)forlineinfp:try:pid=int(line)exceptValueError:continuetry:os.kill(pid,0)vlog('# Killing daemon process %d'%pid)os.kill(pid,signal.SIGTERM)time.sleep(0.25)os.kill(pid,0)vlog('# Daemon process %d is stuck - really killing it'%pid)os.kill(pid,signal.SIGKILL)exceptOSError,err:iferr.errno!=errno.ESRCH:raisefp.close()os.unlink(DAEMON_PIDS)exceptIOError:passos.chdir(TESTDIR)ifnotoptions.keep_tmpdir:shutil.rmtree(tmpd,True)ifskipped:returnNonereturnret==0_hgpath=Nonedef_gethgpath():"""Return the path to the mercurial package that is actually found by the current Python interpreter."""global_hgpathif_hgpathisnotNone:return_hgpathcmd='%s -c "import mercurial; print mercurial.__path__[0]"'pipe=os.popen(cmd%PYTHON)try:_hgpath=pipe.read().strip()finally:pipe.close()return_hgpathdef_checkhglib(verb):"""Ensure that the 'mercurial' package imported by python is the one we expect it to be. If not, print a warning to stderr."""expecthg=os.path.join(PYTHONDIR,'mercurial')actualhg=_gethgpath()ifactualhg!=expecthg:sys.stderr.write('warning: %s with unexpected mercurial lib: %s\n'' (expected %s)\n'%(verb,actualhg,expecthg))defrunchildren(options,tests):ifINST:installhg(options)_checkhglib("Testing")optcopy=dict(options.__dict__)optcopy['jobs']=1ifoptcopy['with_hg']isNone:optcopy['with_hg']=os.path.join(BINDIR,"hg")opts=[]foropt,valueinoptcopy.iteritems():name='--'+opt.replace('_','-')ifvalueisTrue:opts.append(name)elifvalueisnotNone:opts.append(name+'='+str(value))tests.reverse()jobs=[[]forjinxrange(options.jobs)]whiletests:forjobinjobs:ifnottests:breakjob.append(tests.pop())fps={}forj,jobinenumerate(jobs):ifnotjob:continuerfd,wfd=os.pipe()childopts=['--child=%d'%wfd,'--port=%d'%(options.port+j*3)]cmdline=[PYTHON,sys.argv[0]]+opts+childopts+jobvlog(' '.join(cmdline))fps[os.spawnvp(os.P_NOWAIT,cmdline[0],cmdline)]=os.fdopen(rfd,'r')os.close(wfd)failures=0tested,skipped,failed=0,0,0skips=[]fails=[]whilefps:pid,status=os.wait()fp=fps.pop(pid)l=fp.read().splitlines()test,skip,fail=map(int,l[:3])split=-failorlen(l)forsinl[3:split]:skips.append(s.split(" ",1))forsinl[split:]:fails.append(s.split(" ",1))tested+=testskipped+=skipfailed+=failvlog('pid %d exited, status %d'%(pid,status))failures|=statusprintforsinskips:print"Skipped %s: %s"%(s[0],s[1])forsinfails:print"Failed %s: %s"%(s[0],s[1])_checkhglib("Tested")print"# Ran %d tests, %d skipped, %d failed."%(tested,skipped,failed)sys.exit(failures!=0)defruntests(options,tests):globalDAEMON_PIDS,HGRCPATHDAEMON_PIDS=os.environ["DAEMON_PIDS"]=os.path.join(HGTMP,'daemon.pids')HGRCPATH=os.environ["HGRCPATH"]=os.path.join(HGTMP,'.hgrc')try:ifINST:installhg(options)_checkhglib("Testing")ifoptions.timeout>0:try:signal.signal(signal.SIGALRM,alarmed)vlog('# Running each test with %d second timeout'%options.timeout)exceptAttributeError:print'WARNING: cannot run tests with timeouts'options.timeout=0tested=0failed=0skipped=0ifoptions.restart:orig=list(tests)whiletests:ifos.path.exists(tests[0]+".err"):breaktests.pop(0)ifnottests:print"running all tests"tests=origskips=[]fails=[]fortestintests:ifoptions.retestandnotos.path.exists(test+".err"):skipped+=1continueret=runone(options,test,skips,fails)ifretisNone:skipped+=1elifnotret:ifoptions.interactive:print"Accept this change? [n] ",answer=sys.stdin.readline().strip()ifanswer.lower()in"y yes".split():rename(test+".err",test+".out")tested+=1fails.pop()continuefailed+=1ifoptions.first:breaktested+=1ifoptions.child:fp=os.fdopen(options.child,'w')fp.write('%d\n%d\n%d\n'%(tested,skipped,failed))forsinskips:fp.write("%s%s\n"%s)forsinfails:fp.write("%s%s\n"%s)fp.close()else:printforsinskips:print"Skipped %s: %s"%sforsinfails:print"Failed %s: %s"%s_checkhglib("Tested")print"# Ran %d tests, %d skipped, %d failed."%(tested,skipped,failed)ifoptions.anycoverage:outputcoverage(options)exceptKeyboardInterrupt:failed=Trueprint"\ninterrupted!"iffailed:sys.exit(1)defmain():(options,args)=parseargs()ifnotoptions.child:os.umask(022)checktools()# Reset some environment variables to well-known values so that# the tests produce repeatable output.os.environ['LANG']=os.environ['LC_ALL']='C'os.environ['TZ']='GMT'os.environ["EMAIL"]="Foo Bar <foo.bar@example.com>"os.environ['CDPATH']=''globalTESTDIR,HGTMP,INST,BINDIR,PYTHONDIR,COVERAGE_FILETESTDIR=os.environ["TESTDIR"]=os.getcwd()HGTMP=os.environ['HGTMP']=os.path.realpath(tempfile.mkdtemp('','hgtests.',options.tmpdir))DAEMON_PIDS=NoneHGRCPATH=Noneos.environ["HGEDITOR"]=sys.executable+' -c "import sys; sys.exit(0)"'os.environ["HGMERGE"]="internal:merge"os.environ["HGUSER"]="test"os.environ["HGENCODING"]="ascii"os.environ["HGENCODINGMODE"]="strict"os.environ["HGPORT"]=str(options.port)os.environ["HGPORT1"]=str(options.port+1)os.environ["HGPORT2"]=str(options.port+2)ifoptions.with_hg:INST=NoneBINDIR=os.path.dirname(os.path.realpath(options.with_hg))# This looks redundant with how Python initializes sys.path from# the location of the script being executed. Needed because the# "hg" specified by --with-hg is not the only Python script# executed in the test suite that needs to import 'mercurial'# ... which means it's not really redundant at all.PYTHONDIR=BINDIRelse:INST=os.path.join(HGTMP,"install")BINDIR=os.environ["BINDIR"]=os.path.join(INST,"bin")PYTHONDIR=os.path.join(INST,"lib","python")os.environ["BINDIR"]=BINDIRos.environ["PYTHON"]=PYTHONifnotoptions.child:path=[BINDIR]+os.environ["PATH"].split(os.pathsep)os.environ["PATH"]=os.pathsep.join(path)# Include TESTDIR in PYTHONPATH so that out-of-tree extensions# can run .../tests/run-tests.py test-foo where test-foo# adds an extension to HGRCpypath=[PYTHONDIR,TESTDIR]# We have to augment PYTHONPATH, rather than simply replacing# it, in case external libraries are only available via current# PYTHONPATH. (In particular, the Subversion bindings on OS X# are in /opt/subversion.)oldpypath=os.environ.get('PYTHONPATH')ifoldpypath:pypath.append(oldpypath)os.environ['PYTHONPATH']=os.pathsep.join(pypath)COVERAGE_FILE=os.path.join(TESTDIR,".coverage")iflen(args)==0:args=os.listdir(".")args.sort()tests=[]fortestinargs:if(test.startswith("test-")and'~'notintestand('.'notintestortest.endswith('.py')ortest.endswith('.bat'))):tests.append(test)ifnottests:print"# Ran 0 tests, 0 skipped, 0 failed."returnvlog("# Using TESTDIR",TESTDIR)vlog("# Using HGTMP",HGTMP)vlog("# Using PATH",os.environ["PATH"])vlog("# Using PYTHONPATH",os.environ["PYTHONPATH"])try:iflen(tests)>1andoptions.jobs>1:runchildren(options,tests)else:runtests(options,tests)finally:cleanup(options)main()