test: comparison tests/run-tests.py

equal deleted inserted replaced

-:06ccaeb6cfc1
+:629956d13cbc
 #   - tests are a mix of shell scripts and Python scripts
 #
 # If you change this script, it is recommended that you ensure you
 # haven't broken it by running it in various modes with a representative
 # sample of test scripts.  For example:
 #
 #  1) serial, no coverage, temp install:
 #      ./run-tests.py test-s*
 #  2) serial, no coverage, local hg:
 #      ./run-tests.py --local test-s*
 #  3) serial, coverage, temp install:
 #      ./run-tests.py -j2 test-s*
 #  6) parallel, no coverage, local hg:
 #      ./run-tests.py -j2 --local test-s*
 #  7) parallel, coverage, temp install:
 #      ./run-tests.py -j2 -c test-s*          # currently broken
-#  8) parallel, coverage, local install
+#  8) parallel, coverage, local install:
 #      ./run-tests.py -j2 -c --local test-s*  # unsupported (and broken)
+#  9) parallel, custom tmp dir:
+#      ./run-tests.py -j2 --tmpdir /tmp/myhgtests
 #
 # (You could use any subset of the tests: test-s* happens to match
 # enough that it's worth doing parallel runs, few enough that it
 # completes fairly quickly, includes both shell and Python scripts, and
 # includes some scripts that run daemon processes.)
+from ConfigParser import ConfigParser
 import difflib
 import errno
 import optparse
 import os
 import subprocess
 parser.add_option("-i", "--interactive", action="store_true",
 help="prompt to accept changed output")
 parser.add_option("-j", "--jobs", type="int",
 help="number of jobs to run in parallel"
 " (default: $%s or %d)" % defaults['jobs'])
+parser.add_option("-k", "--keywords",
+help="run tests matching keywords")
 parser.add_option("--keep-tmpdir", action="store_true",
-help="keep temporary directory after running tests"
+help="keep temporary directory after running tests")
-" (best used with --tmpdir)")
+parser.add_option("--tmpdir", type="string",
+help="run tests in the given temporary directory"
+" (implies --keep-tmpdir)")
+parser.add_option("-d", "--debug", action="store_true",
+help="debug mode: write output of test scripts to console"
+" rather than capturing and diff'ing it (disables timeout)")
 parser.add_option("-R", "--restart", action="store_true",
 help="restart at last error")
 parser.add_option("-p", "--port", type="int",
 help="port on which servers should listen"
 " (default: $%s or %d)" % defaults['port'])
 parser.add_option("-r", "--retest", action="store_true",
 help="retest failed tests")
 parser.add_option("-s", "--cover_stdlib", action="store_true",
 help="print a test coverage report inc. standard libraries")
+parser.add_option("-S", "--noskips", action="store_true",
+help="don't report skip tests verbosely")
 parser.add_option("-t", "--timeout", type="int",
 help="kill errant tests after TIMEOUT seconds"
 " (default: $%s or %d)" % defaults['timeout'])
-parser.add_option("--tmpdir", type="string",
-help="run tests in the given temporary directory")
 parser.add_option("-v", "--verbose", action="store_true",
 help="output verbose messages")
 parser.add_option("-n", "--nodiff", action="store_true",
 help="skip showing test changes")
 parser.add_option("--with-hg", type="string",
 "temporary installation")
 parser.add_option("--local", action="store_true",
 help="shortcut for --with-hg=<testdir>/../hg")
 parser.add_option("--pure", action="store_true",
 help="use pure Python code instead of C extensions")
+parser.add_option("-3", "--py3k-warnings", action="store_true",
+help="enable Py3k warnings on Python 2.6+")
+parser.add_option("--inotify", action="store_true",
+help="enable inotify extension when running tests")
+parser.add_option("--blacklist", action="append",
+help="skip tests listed in the specified section of "
+"the blacklist file")
 for option, default in defaults.items():
 defaults[option] = int(os.environ.get(*default))
 parser.set_defaults(**defaults)
 (options, args) = parser.parse_args()
 if pid:
 print pid,
 for m in msg:
 print m,
 print
+sys.stdout.flush()
 else:
 vlog = lambda *msg: None
+if options.tmpdir:
+options.tmpdir = os.path.expanduser(options.tmpdir)
 if options.jobs < 1:
-print >> sys.stderr, 'ERROR: -j/--jobs must be positive'
+parser.error('--jobs must be positive')
-sys.exit(1)
 if options.interactive and options.jobs > 1:
 print '(--interactive overrides --jobs)'
 options.jobs = 1
+if options.interactive and options.debug:
+parser.error("-i/--interactive and -d/--debug are incompatible")
+if options.debug:
+if options.timeout != defaults['timeout']:
+sys.stderr.write(
+'warning: --timeout option ignored with --debug\n')
+options.timeout = 0
+if options.py3k_warnings:
+if sys.version_info[:2] < (2, 6) or sys.version_info[:2] >= (3, 0):
+parser.error('--py3k-warnings can only be used on Python 2.6+')
+if options.blacklist:
+configparser = ConfigParser()
+configparser.read("blacklist")
+blacklist = dict()
+for section in options.blacklist:
+for (item, value) in configparser.items(section):
+blacklist["test-" + item] = section
+options.blacklist = blacklist
 return (options, args)
 def rename(src, dst):
 """Like os.rename(), trade atomicity and opened files friendliness
 line = line.splitlines()[0]
 failed.append(line[len(FAILED_PREFIX):])
 return missing, failed
-def showdiff(expected, output):
+def showdiff(expected, output, ref, err):
-for line in difflib.unified_diff(expected, output,
+for line in difflib.unified_diff(expected, output, ref, err):
-"Expected output", "Test output"):
 sys.stdout.write(line)
 def findprogram(program):
 """Search PATH for a executable program"""
 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
 vlog("# Performing temporary installation of HG")
 installerrs = os.path.join("tests", "install.err")
 pure = options.pure and "--pure" or ""
 # Run installer in hg root
-os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+script = os.path.realpath(sys.argv[0])
+hgroot = os.path.dirname(os.path.dirname(script))
+os.chdir(hgroot)
+nohome = '--home=""'
+if os.name == 'nt':
+# The --home="" trick works only on OS where os.sep == '/'
+# because of a distutils convert_path() fast-path. Avoid it at
+# least on Windows for now, deal with .pydistutils.cfg bugs
+# when they happen.
+nohome = ''
 cmd = ('%s setup.py %s clean --all'
 ' install --force --prefix="%s" --install-lib="%s"'
-' --install-scripts="%s" >%s 2>&1'
+' --install-scripts="%s" %s >%s 2>&1'
-% (sys.executable, pure, INST, PYTHONDIR, BINDIR, installerrs))
+% (sys.executable, pure, INST, PYTHONDIR, BINDIR, nohome,
+installerrs))
 vlog("# Running", cmd)
 if os.system(cmd) == 0:
 if not options.verbose:
 os.remove(installerrs)
 else:
 '    if line.startswith("diff "):\n'
 '        files += 1\n'
 'sys.stdout.write("files patched: %d\\n" % files)\n')
 f.close()
 os.chmod(os.path.join(BINDIR, 'diffstat'), 0700)
+if options.py3k_warnings and not options.anycoverage:
+vlog("# Updating hg command to enable Py3k Warnings switch")
+f = open(os.path.join(BINDIR, 'hg'), 'r')
+lines = [line.rstrip() for line in f]
+lines[0] += ' -3'
+f.close()
+f = open(os.path.join(BINDIR, 'hg'), 'w')
+for line in lines:
+f.write(line + '\n')
+f.close()
 if options.anycoverage:
 vlog("# Installing coverage wrapper")
 os.environ['COVERAGE_FILE'] = COVERAGE_FILE
 if os.path.exists(COVERAGE_FILE):
 def alarmed(signum, frame):
 raise Timeout
 def run(cmd, options):
 """Run command in a sub-process, capturing the output (stdout and stderr).
-Return the exist code, and output."""
+Return a tuple (exitcode, output).  output is None in debug mode."""
 # TODO: Use subprocess.Popen if we're running on Python 2.4
+if options.debug:
+proc = subprocess.Popen(cmd, shell=True)
+ret = proc.wait()
+return (ret, None)
 if os.name == 'nt' or sys.platform.startswith('java'):
 tochild, fromchild = os.popen4(cmd)
 tochild.close()
 output = fromchild.read()
 ret = fromchild.close()
 def skip(msg):
 if not options.verbose:
 skips.append((test, msg))
 else:
-print "\nSkipping %s: %s" % (test, msg)
+print "\nSkipping %s: %s" % (testpath, msg)
 return None
 def fail(msg):
 fails.append((test, msg))
 if not options.nodiff:
-print "\nERROR: %s %s" % (test, msg)
+print "\nERROR: %s %s" % (testpath, msg)
 return None
 vlog("# Test", test)
 # create a fresh hgrc
-hgrc = file(HGRCPATH, 'w+')
+hgrc = open(HGRCPATH, 'w+')
 hgrc.write('[ui]\n')
 hgrc.write('slash = True\n')
 hgrc.write('[defaults]\n')
 hgrc.write('backout = -d "0 0"\n')
 hgrc.write('commit = -d "0 0"\n')
 hgrc.write('tag = -d "0 0"\n')
+if options.inotify:
+hgrc.write('[extensions]\n')
+hgrc.write('inotify=\n')
+hgrc.write('[inotify]\n')
+hgrc.write('pidfile=%s\n' % DAEMON_PIDS)
+hgrc.write('appendpid=True\n')
 hgrc.close()
 err = os.path.join(TESTDIR, test+".err")
 ref = os.path.join(TESTDIR, test+".out")
 testpath = os.path.join(TESTDIR, test)
 except:
 firstline = ''
 lctest = test.lower()
 if lctest.endswith('.py') or firstline == '#!/usr/bin/env python':
-cmd = '%s "%s"' % (PYTHON, testpath)
+py3kswitch = options.py3k_warnings and ' -3' or ''
+cmd = '%s%s "%s"' % (PYTHON, py3kswitch, testpath)
 elif lctest.endswith('.bat'):
 # do not run batch scripts on non-windows
 if os.name != 'nt':
 return skip("batch script")
 # To reliably get the error code from batch files on WinXP,
 signal.alarm(0)
 mark = '.'
 skipped = (ret == SKIPPED_STATUS)
-# If reference output file exists, check test output against it
+# If we're not in --debug mode and reference output file exists,
-if os.path.exists(ref):
+# check test output against it.
+if options.debug:
+refout = None                   # to match out == None
+elif os.path.exists(ref):
 f = open(ref, "r")
 refout = splitnewlines(f.read())
 f.close()
 else:
 refout = []
 if skipped:
 mark = 's'
-missing, failed = parsehghaveoutput(out)
+if out is None:                 # debug mode: nothing to parse
+missing = ['unknown']
+failed = None
+else:
+missing, failed = parsehghaveoutput(out)
 if not missing:
 missing = ['irrelevant']
 if failed:
 fail("hghave failed checking for %s" % failed[-1])
 skipped = False
 if ret:
 fail("output changed and returned error code %d" % ret)
 else:
 fail("output changed")
 if not options.nodiff:
-showdiff(refout, out)
+showdiff(refout, out, ref, err)
 ret = 1
 elif ret:
 mark = '!'
 fail("returned error code %d" % ret)
 if not options.verbose:
 sys.stdout.write(mark)
 sys.stdout.flush()
-if ret != 0 and not skipped:
+if ret != 0 and not skipped and not options.debug:
 # Save errors to a file for diagnosis
 f = open(err, "wb")
 for line in out:
 f.write(line)
 f.close()
 # Kill off any leftover daemon processes
 try:
-fp = file(DAEMON_PIDS)
+fp = open(DAEMON_PIDS)
 for line in fp:
 try:
 pid = int(line)
 except ValueError:
 continue
 for j, job in enumerate(jobs):
 if not job:
 continue
 rfd, wfd = os.pipe()
 childopts = ['--child=%d' % wfd, '--port=%d' % (options.port + j * 3)]
+childtmp = os.path.join(HGTMP, 'child%d' % j)
+childopts += ['--tmpdir', childtmp]
 cmdline = [PYTHON, sys.argv[0]] + opts + childopts + job
 vlog(' '.join(cmdline))
 fps[os.spawnvp(os.P_NOWAIT, cmdline[0], cmdline)] = os.fdopen(rfd, 'r')
 os.close(wfd)
 failures = 0
 skipped += skip
 failed += fail
 vlog('pid %d exited, status %d' % (pid, status))
 failures |= status
 print
-for s in skips:
+if not options.noskips:
-print "Skipped %s: %s" % (s[0], s[1])
+for s in skips:
+print "Skipped %s: %s" % (s[0], s[1])
 for s in fails:
 print "Failed %s: %s" % (s[0], s[1])
 _checkhglib("Tested")
 print "# Ran %d tests, %d skipped, %d failed." % (
 print "running all tests"
 tests = orig
 skips = []
 fails = []
 for test in tests:
+if options.blacklist:
+section = options.blacklist.get(test)
+if section is not None:
+skips.append((test, "blacklisted (%s section)" % section))
+skipped += 1
+continue
 if options.retest and not os.path.exists(test + ".err"):
 skipped += 1
 continue
+if options.keywords:
+t = open(test).read().lower() + test.lower()
+for k in options.keywords.lower().split():
+if k in t:
+break
+else:
+skipped +=1
+continue
 ret = runone(options, test, skips, fails)
 if ret is None:
 skipped += 1
 elif not ret:
 if options.interactive:
 checktools()
 # Reset some environment variables to well-known values so that
 # the tests produce repeatable output.
-os.environ['LANG'] = os.environ['LC_ALL'] = 'C'
+os.environ['LANG'] = os.environ['LC_ALL'] = os.environ['LANGUAGE'] = 'C'
 os.environ['TZ'] = 'GMT'
 os.environ["EMAIL"] = "Foo Bar <foo.bar@example.com>"
 os.environ['CDPATH'] = ''
+os.environ['COLUMNS'] = '80'
 global TESTDIR, HGTMP, INST, BINDIR, PYTHONDIR, COVERAGE_FILE
 TESTDIR = os.environ["TESTDIR"] = os.getcwd()
-HGTMP = os.environ['HGTMP'] = os.path.realpath(tempfile.mkdtemp('', 'hgtests.',
+if options.tmpdir:
-options.tmpdir))
+options.keep_tmpdir = True
+tmpdir = options.tmpdir
+if os.path.exists(tmpdir):
+# Meaning of tmpdir has changed since 1.3: we used to create
+# HGTMP inside tmpdir; now HGTMP is tmpdir.  So fail if
+# tmpdir already exists.
+sys.exit("error: temp dir %r already exists" % tmpdir)
+# Automatically removing tmpdir sounds convenient, but could
+# really annoy anyone in the habit of using "--tmpdir=/tmp"
+# or "--tmpdir=$HOME".
+#vlog("# Removing temp dir", tmpdir)
+#shutil.rmtree(tmpdir)
+os.makedirs(tmpdir)
+else:
+tmpdir = tempfile.mkdtemp('', 'hgtests.')
+HGTMP = os.environ['HGTMP'] = os.path.realpath(tmpdir)
 DAEMON_PIDS = None
 HGRCPATH = None
 os.environ["HGEDITOR"] = sys.executable + ' -c "import sys; sys.exit(0)"'
 os.environ["HGMERGE"] = "internal:merge"

branch	stable
changeset 671	629956d13cbc
parent 670	80d0ed025a02
child 674	0efa9d6ac11f