Commit 5241c0cc authored by Carlos Garcia Campos's avatar Carlos Garcia Campos

regtest: Use number of tests run to show the progress

Instead of using the number of document tested. We don't really need to
set a number to a particular document, we only want to know the progress
of the whole process. This ensures that the progress is shown in the
correct order. It also simplifies the code a bit.
Also improved the output formatting to make it easier to read.
parent f20fe89d
......@@ -34,14 +34,12 @@ class Printer:
self._rewrite = self._stream.isatty() and not self._verbose
self._current_line = None
self._tests = {}
self._lock = RLock()
Printer.__single = self
def _erase_current_line(self):
if not self._rewrite or self._current_line is None:
if self._current_line is None:
return
line_len = len(self._current_line)
......@@ -58,25 +56,17 @@ class Printer:
self._stream.flush()
def printout(self, msg):
if not self._rewrite:
self.printout_ln(msg)
with self._lock:
self._erase_current_line()
self._print(msg)
self._current_line = msg[msg.rfind('\n') + 1:]
def printout_update(self, msg):
def printout_ln(self, msg=''):
with self._lock:
if self._rewrite and self._current_line is not None:
msg = self._current_line + msg
elif not self._rewrite:
msg = self._ensure_new_line(msg)
self.printout(msg)
def printout_ln(self, msg):
with self._lock:
if self._current_line is not None:
self._current_line = None
msg = '\n' + msg
self._erase_current_line()
self._print(self._ensure_new_line(msg))
def printerr(self, msg):
......@@ -84,25 +74,11 @@ class Printer:
self.stderr.write(self._ensure_new_line(msg))
self.stderr.flush()
def print_test_start(self, doc_path, backend_name, n_doc, total_docs):
with self._lock:
self._tests[(doc_path, backend_name)] = n_doc, total_docs
def print_test_result(self, doc_path, backend_name, msg):
if not self._rewrite:
self.print_test_result_ln(doc_path, backend_name, msg)
return
def print_test_result(self, doc_path, backend_name, n_test, total_tests, msg):
self.printout("[%d/%d] %s (%s): %s" % (n_test, total_tests, doc_path, backend_name, msg))
with self._lock:
n_doc, total_docs = self._tests.pop((doc_path, backend_name))
msg = "Tested '%s' using %s backend (%d/%d): %s" % (doc_path, backend_name, n_doc, total_docs, msg)
self.printout(msg)
def print_test_result_ln(self, doc_path, backend_name, msg):
with self._lock:
n_doc, total_docs = self._tests.pop((doc_path, backend_name))
msg = "Tested '%s' using %s backend (%d/%d): %s" % (doc_path, backend_name, n_doc, total_docs, msg)
self.printout_ln(msg)
def print_test_result_ln(self, doc_path, backend_name, n_test, total_tests, msg):
self.printout_ln("[%d/%d] %s (%s): %s" % (n_test, total_tests, doc_path, backend_name, msg))
def print_default(self, msg):
if self._verbose:
......
......@@ -24,7 +24,7 @@ from Printer import get_printer
from Utils import get_document_paths_from_dir, get_skipped_tests
from Queue import Queue
from threading import Thread
from threading import Thread, RLock
class TestReferences:
......@@ -34,8 +34,11 @@ class TestReferences:
self._skipped = get_skipped_tests(docsdir)
self.config = Config()
self.printer = get_printer()
self._total_tests = 1
self._n_tests = 0
self._queue = Queue()
self._lock = RLock()
try:
os.makedirs(self._refsdir)
......@@ -45,9 +48,19 @@ class TestReferences:
except:
raise
def create_refs_for_file(self, filename, n_doc = 1, total_docs = 1):
def _get_backends(self):
if self.config.backends:
return [get_backend(name) for name in self.config.backends]
return get_all_backends()
def create_refs_for_file(self, filename):
backends = self._get_backends()
if filename in self._skipped:
self.printer.print_default("Skipping test '%s' (%d/%d)" % (os.path.join(self._docsdir, filename), n_doc, total_docs))
with self._lock:
self._n_tests += len(backends)
self.printer.print_default("Skipping test '%s'" % (os.path.join(self._docsdir, filename)))
return
refs_path = os.path.join(self._refsdir, filename)
......@@ -60,38 +73,43 @@ class TestReferences:
raise
doc_path = os.path.join(self._docsdir, filename)
if self.config.backends:
backends = [get_backend(name) for name in self.config.backends]
else:
backends = get_all_backends()
for backend in backends:
if not self.config.force and backend.has_results(refs_path):
self.printer.print_default("Results found, skipping '%s' for %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
with self._lock:
self._n_tests += 1
self.printer.print_default("Results found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
continue
self.printer.printout_ln("Creating refs for '%s' using %s backend (%d/%d)" % (doc_path, backend.get_name(), n_doc, total_docs))
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
with self._lock:
self._n_tests += 1
self.printer.printout_ln("[%d/%d] %s (%s): done" % (self._n_tests, self._total_tests, doc_path, backend.get_name()))
def _worker_thread(self):
while True:
doc, n_doc, total_docs = self._queue.get()
self.create_refs_for_file(doc, n_doc, total_docs)
doc = self._queue.get()
self.create_refs_for_file(doc)
self._queue.task_done()
def create_refs(self):
docs, total_docs = get_document_paths_from_dir(self._docsdir)
backends = self._get_backends()
self._total_tests = total_docs * len(backends)
self.printer.printout_ln('Found %d documents' % (total_docs))
self.printer.printout_ln('Backends: %s' % ', '.join([backend.get_name() for backend in backends]))
self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), self.config.threads))
self.printer.printout_ln()
self.printer.printout_ln('Process %d is spawning %d worker threads...' % (os.getpid(), self.config.threads))
self.printer.printout('Spawning %d workers...' % (self.config.threads))
for n_thread in range(self.config.threads):
thread = Thread(target=self._worker_thread)
thread.daemon = True
thread.start()
n_doc = 0
for doc in docs:
n_doc += 1
self._queue.put( (doc, n_doc, total_docs) )
self._queue.put(doc)
self._queue.join()
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment