Commit 32bb8741 authored by Carlos Garcia Campos's avatar Carlos Garcia Campos

regtest: Reduce the noise of the default output when running tests

Show permanent information only about failed tests, without the details
about the failing pages. Previous verbose output is available passing
--verbose command line output.
parent ca6afce2
# Printer.py
#
# Copyright (C) 2012 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import sys
from Config import Config
class Printer:
__single = None
def __init__(self):
if Printer.__single is not None:
raise Printer.__single
self._verbose = Config().verbose
self._stream = sys.stdout
self._rewrite = self._stream.isatty() #and not self.config.verbose
self._current_line = None
Printer.__single = self
def _erase_current_line(self):
if not self._rewrite or self._current_line is None:
return
line_len = len(self._current_line)
self._stream.write('\b' * line_len + ' ' * line_len + '\b' * line_len)
self._current_line = None
def _ensure_new_line(self, msg):
if not msg.endswith('\n'):
msg += '\n'
return msg
def _print(self, msg):
self._stream.write(msg)
self._stream.flush()
def printout(self, msg):
self._erase_current_line()
self._print(msg)
self._current_line = msg[msg.rfind('\n') + 1:]
def printout_update(self, msg):
if self._rewrite and self._current_line is not None:
msg = self._current_line + msg
elif not self._rewrite:
msg = self._ensure_new_line(msg)
self.printout(msg)
def printout_ln(self, msg):
if self._current_line is not None:
self._current_line = None
msg = '\n' + msg
self._print(self._ensure_new_line(msg))
def printerr(self, msg):
self.stderr.write(self._ensure_new_line(msg))
self.stderr.flush()
def print_test_start(self, msg):
self.printout(msg)
def print_test_result(self, msg):
self.printout_update(msg)
def print_default(self, msg):
if self._verbose:
self.printout_ln(msg)
def get_printer():
try:
instance = Printer()
except Printer, i:
instance = i
return instance
......@@ -20,6 +20,7 @@ import os
import errno
from backends import get_backend, get_all_backends
from Config import Config
from Printer import get_printer
from Utils import get_document_paths_from_dir, get_skipped_tests
class TestReferences:
......@@ -29,6 +30,7 @@ class TestReferences:
self._refsdir = refsdir
self._skipped = get_skipped_tests(docsdir)
self.config = Config()
self.printer = get_printer()
try:
os.makedirs(self._refsdir)
......@@ -40,7 +42,7 @@ class TestReferences:
def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1):
if filename in self._skipped:
print("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
self.printer.print_default("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
return
refs_path = os.path.join(self._refsdir, filename)
......@@ -60,9 +62,9 @@ class TestReferences:
for backend in backends:
if not self.config.force and backend.has_results(refs_path):
print("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
self.printer.print_default("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
continue
print("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
self.printer.printout_ln("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
......
......@@ -19,6 +19,7 @@
from backends import get_backend, get_all_backends
from Config import Config
from Utils import get_document_paths_from_dir, get_skipped_tests
from Printer import get_printer
import sys
import os
import errno
......@@ -31,6 +32,7 @@ class TestRun:
self._outdir = outdir
self._skip = get_skipped_tests(docsdir)
self.config = Config()
self.printer = get_printer()
# Results
self._n_tests = 0
......@@ -56,12 +58,11 @@ class TestRun:
ref_is_failed = backend.is_failed(refs_path)
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
self._skipped.append("%s (%s)" % (doc_path, backend.get_name()))
print("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
self.printer.print_default("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
return
self._n_tests += 1
sys.stdout.write("Testing '%s' using %s backend (%d/%d): " % (doc_path, backend.get_name(), n_doc, total_docs))
sys.stdout.flush()
self.printer.print_test_start("Testing '%s' using %s backend (%d/%d): " % (doc_path, backend.get_name(), n_doc, total_docs))
test_has_md5 = backend.create_refs(doc_path, test_path)
if backend.has_stderr(test_path):
......@@ -70,7 +71,7 @@ class TestRun:
if ref_has_md5 and test_has_md5:
if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs):
# FIXME: remove dir if it's empty?
print("PASS")
self.printer.print_test_result("PASS")
self._n_passed += 1
else:
print("FAIL")
......@@ -78,32 +79,31 @@ class TestRun:
return
elif test_has_md5:
if ref_is_crashed:
print("DOES NOT CRASH")
self.printer.print_test_result("DOES NOT CRASH")
elif ref_is_failed:
print("DOES NOT FAIL")
self.printer.print_test_result("DOES NOT FAIL")
return
test_is_crashed = backend.is_crashed(test_path)
if ref_is_crashed and test_is_crashed:
print("PASS (Expected crash)")
self.printer.print_test_result("PASS (Expected crash)")
self._n_passed += 1
return
test_is_failed = backend.is_failed(test_path)
if ref_is_failed and test_is_failed:
# FIXME: compare status errors
print("PASS (Expected fail with status error %d)" % (test_is_failed))
self.printer.print_test_result("PASS (Expected fail with status error %d)" % (test_is_failed))
self._n_passed += 1
return
if test_is_crashed:
print("CRASH")
self.printer.print_test_result("CRASH")
self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
return
if test_is_failed:
print("FAIL (status error %d)" % (test_is_failed))
self.printer.print_test_result("FAIL (status error %d)" % (test_is_failed))
self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
return
......@@ -111,7 +111,7 @@ class TestRun:
if filename in self._skip:
doc_path = os.path.join(self._docsdir, filename)
self._skipped.append("%s" % (doc_path))
print("Skipping test '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
self.printer.print_default("Skipping test '%s' (%d/%d)" % (doc_path, n_doc, total_docs))
return
out_path = os.path.join(self._outdir, filename)
......@@ -127,7 +127,7 @@ class TestRun:
if not os.path.isdir(refs_path):
self._skipped.append("%s" % (doc_path))
print("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
self.printer.print_default("Reference dir not found for %s, skipping (%d/%d)" % (doc_path, n_doc, total_docs))
return
if self.config.backends:
......@@ -147,16 +147,17 @@ class TestRun:
def summary(self):
if not self._n_tests:
print("No tests run")
self.printer.printout_ln("No tests run")
return
print("Total %d tests" % (self._n_tests))
print("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
self.printer.printout_ln("Total %d tests" % (self._n_tests))
self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests))
def report_tests(test_list, test_type):
n_tests = len(test_list)
if not n_tests:
return
print("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
self.printer.printout_ln("%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list)))
report_tests(self._failed, "failed")
report_tests(self._crashed, "crashed")
report_tests(self._failed_status_error, "failed to run")
......
......@@ -21,6 +21,7 @@ import os
import shutil
import errno
from Config import Config
from Printer import get_printer
__all__ = [ 'register_backend',
'get_backend',
......@@ -38,6 +39,8 @@ class Backend:
self._diff_ext = diff_ext
self._utilsdir = Config().utils_dir
self.printer = get_printer()
def get_name(self):
return self._name
......@@ -83,7 +86,7 @@ class Backend:
if not basename in tests:
retval = False
print("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
self.printer.print_default("%s found in md5 ref file but missing in output dir %s" % (basename, out_path))
continue
result_path = os.path.join(out_path, basename)
......@@ -99,10 +102,10 @@ class Backend:
if remove_results:
os.remove(result_path)
else:
print("Differences found in %s" % (basename))
self.printer.print_default("Differences found in %s" % (basename))
if create_diffs:
if not os.path.exists(ref_path):
print("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
self.printer.print_default("Reference file %s not found, skipping diff for %s" % (ref_path, result_path))
else:
try:
self._create_diff(ref_path, result_path)
......@@ -112,14 +115,14 @@ class Backend:
if update_refs:
if os.path.exists(ref_path):
print("Updating image reference %s" % (ref_path))
self.printer.print_default("Updating image reference %s" % (ref_path))
shutil.copyfile(result_path, ref_path)
retval = False
md5_file.close()
if update_refs and not retval:
print("Updating md5 reference %s" % (md5_path))
self.printer.print_default("Updating md5 reference %s" % (md5_path))
f = open(md5_path + '.md5.tmp', 'wb')
f.writelines(result_md5)
f.close()
......
......@@ -20,6 +20,7 @@ from commands import Command, register_command
from TestReferences import TestReferences
from Timer import Timer
from Config import Config
from Printer import get_printer
import os
import tempfile
......@@ -60,6 +61,6 @@ class CreateRefs(Command):
refs.create_refs()
else:
refs.create_refs_for_file(os.path.basename(doc))
print("Refs created in %s" % (t.elapsed_str()))
get_printer().printout_ln("Refs created in %s" % (t.elapsed_str()))
register_command('create-refs', CreateRefs)
......@@ -20,6 +20,7 @@ from commands import Command, register_command
from Bisect import Bisect
from Timer import Timer
from Config import Config
from Printer import get_printer
import os
import tempfile
......@@ -66,12 +67,12 @@ class FindRegression(Command):
doc = options['test']
if not os.path.isfile(doc):
print("Invalid test %s: not a regulat file" % (doc))
get_printer().printerr("Invalid test %s: not a regulat file" % (doc))
return
t = Timer()
bisect = Bisect(options['test'], options['refs_dir'], options['out_dir'])
bisect.run()
print("Tests run in %s" % (t.elapsed_str()))
get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))
register_command('find-regression', FindRegression)
......@@ -20,6 +20,7 @@ from commands import Command, register_command
from TestRun import TestRun
from Timer import Timer
from Config import Config
from Printer import get_printer
import os
import tempfile
......@@ -68,6 +69,6 @@ class RunTests(Command):
else:
tests.run_test(os.path.basename(doc))
tests.summary()
print("Tests run in %s" % (t.elapsed_str()))
get_printer().printout_ln("Tests run in %s" % (t.elapsed_str()))
register_command('run-tests', RunTests)
......@@ -49,6 +49,9 @@ def main(args):
parser.add_argument('--help-command', metavar = 'COMMAND',
action = HelpAction,
help = 'Show help for a given command')
parser.add_argument('-v', '--verbose',
action = 'store_true', dest = 'verbose', default = False,
help = 'Run in verbose mode')
parser.add_argument('--utils-dir',
action = 'store', dest = 'utils_dir', default = os.path.abspath("../utils"),
help = 'Directory of poppler utils used for the tests')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment