Commit 10801b6f authored by Carlos Garcia Campos's avatar Carlos Garcia Campos

Add initial poppler regressions test program

parent 245e331a
# Config.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
class Config:
shared_state = {}
def __init__(self, config = None):
if config is not None:
self.__class__.shared_state = config
self.__dict__ = self.__class__.shared_state
if __name__ == '__main__':
c = Config({'foo' : 25})
print c.foo
cc = Config()
print cc.foo
# TestReferences.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os
import errno
from backends import get_backend, get_all_backends
from Config import Config
class TestReferences:
def __init__(self, docsdir, refsdir):
self._docsdir = docsdir
self._refsdir = refsdir
self.config = Config()
try:
os.makedirs(self._refsdir)
except OSError, e:
if e.errno != errno.EEXIST:
raise
except:
raise
def create_refs_for_file(self, filename):
refs_path = os.path.join(self._refsdir, filename)
try:
os.makedirs(refs_path)
except OSError, e:
if e.errno != errno.EEXIST:
raise
except:
raise
doc_path = os.path.join(self._docsdir, filename)
if self.config.backends:
backends = [get_backend(name) for name in self.config.backends]
else:
backends = get_all_backends()
for backend in backends:
if not self.config.force and backend.has_md5(refs_path):
print "Checksum file found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
continue
print "Creating refs for '%s' using %s backend" % (doc_path, backend.get_name())
if backend.create_refs(doc_path, refs_path):
backend.create_checksums(refs_path, self.config.checksums_only)
def create_refs(self):
for root, dirs, files in os.walk(self._docsdir, False):
for entry in files:
if not entry.lower().endswith('.pdf'):
continue
test_path = os.path.join(root[len(self._docsdir):], entry)
self.create_refs_for_file(test_path.lstrip(os.path.sep))
# TestRun.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from backends import get_backend, get_all_backends
from Config import Config
import sys
import os
import errno
class TestRun:
def __init__(self, docsdir, refsdir, outdir):
self._docsdir = docsdir
self._refsdir = refsdir
self._outdir = outdir
self.config = Config()
# Results
self._n_tests = 0
self._n_passed = 0
self._failed = []
self._crashed = []
self._failed_status_error = []
self._stderr = []
try:
os.makedirs(self._outdir);
except OSError, e:
if e.errno != errno.EEXIST:
raise
except:
raise
def test(self, refs_path, doc_path, test_path, backend):
# First check whether there are test results for the backend
ref_has_md5 = backend.has_md5(refs_path)
ref_is_crashed = backend.is_crashed(refs_path)
ref_is_failed = backend.is_failed(refs_path)
if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
print "Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name())
return
self._n_tests += 1
sys.stdout.write("Testing '%s' using %s backend: " % (doc_path, backend.get_name()))
sys.stdout.flush()
test_has_md5 = backend.create_refs(doc_path, test_path)
if backend.has_stderr(test_path):
self._stderr.append("%s (%s)" % (doc_path, backend.get_name()))
if ref_has_md5 and test_has_md5:
if backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs):
# FIXME: remove dir if it's empty?
print "PASS"
self._n_passed += 1
else:
print "FAIL"
self._failed.append("%s (%s)" % (doc_path, backend.get_name()))
return
elif test_has_md5:
if ref_is_crashed:
print "DOES NOT CRASH"
elif ref_is_failed:
print "DOES NOT FAIL"
return
test_is_crashed = backend.is_crashed(test_path)
if ref_is_crashed and test_is_crashed:
print "PASS (Expected crash)"
self._n_passed += 1
return
test_is_failed = backend.is_failed(test_path)
if ref_is_failed and test_is_failed:
# FIXME: compare status errors
print "PASS (Expected fail with status error %d)" % (test_is_failed)
self._n_passed += 1
return
if test_is_crashed:
print "CRASH"
self._crashed.append("%s (%s)" % (doc_path, backend.get_name()))
return
if test_is_failed:
print "FAIL (status error %d)" % (test_is_failed)
self._failed_status_error("%s (%s)" % (doc_path, backend.get_name()))
return
def run_test(self, filename):
out_path = os.path.join(self._outdir, filename)
try:
os.makedirs(out_path)
except OSError, e:
if e.errno != errno.EEXIST:
raise
except:
raise
doc_path = os.path.join(self._docsdir, filename)
refs_path = os.path.join(self._refsdir, filename)
if not os.path.isdir(refs_path):
print "Reference dir not found for %s, skipping" % (doc_path)
return
if self.config.backends:
backends = [get_backend(name) for name in self.config.backends]
else:
backends = get_all_backends()
for backend in backends:
self.test(refs_path, doc_path, out_path, backend)
def run_tests(self):
for root, dirs, files in os.walk(self._docsdir, False):
for entry in files:
if not entry.lower().endswith('.pdf'):
continue
test_path = os.path.join(root[len(self._docsdir):], entry)
self.run_test(test_path.lstrip(os.path.sep))
def summary(self):
if not self._n_tests:
print "No tests run"
return
print "Total %d tests" % (self._n_tests)
print "%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_tests)
def report_tests(test_list, test_type):
n_tests = len(test_list)
if not n_tests:
return
print "%d tests %s (%.2f%%): %s" % (n_tests, test_type, (n_tests * 100.) / self._n_tests, ", ".join(test_list))
report_tests(self._failed, "failed")
report_tests(self._crashed, "crashed")
report_tests(self._failed_status_error, "failed to run")
report_tests(self._stderr, "have stderr output")
# Timer.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from time import time, strftime, gmtime
class Timer:
def __init__(self, start = True):
self._stop = None
if start:
self.start()
else:
self._start = None
def start(self):
self._start = time()
def stop(self):
self._stop = time()
def elapsed(self):
if self._start is None:
return 0
if self._stop is None:
return time() - self._start
return self._stop - self._start
def elapsed_str(self):
h, m, s = [int(i) for i in strftime('%H:%M:%S', gmtime(self.elapsed())).split(':')]
retval = "%d seconds" % (s)
if h == 0 and m == 0:
return retval
retval = "%d minutes and %s" % (m, retval)
if h == 0:
return retval
retval = "%d hours, %s" % (h, retval)
return retval
if __name__ == '__main__':
from time import sleep
t = Timer()
sleep(5)
print "Elapsed: %s" % (t.elapsed_str())
sleep(1)
print "Elapsed: %s" % (t.elapsed_str())
t.start()
sleep(2)
t.stop()
print "Elapsed: %s" % (t.elapsed_str())
sleep(2)
print "Elapsed: %s" % (t.elapsed_str())
# backends
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from hashlib import md5
import os
from Config import Config
__all__ = [ 'register_backend',
'get_backend',
'get_all_backends',
'UnknownBackendError',
'Backend' ]
class UnknownBackendError(Exception):
'''Unknown backend type'''
class Backend:
def __init__(self, name):
self._name = name
self._utilsdir = Config().utils_dir
def get_name(self):
return self._name
def create_checksums(self, refs_path, delete_refs = False):
path = os.path.join(refs_path, self._name)
md5_file = open(path + '.md5', 'w')
for entry in os.listdir(refs_path):
if not entry.startswith(self._name) or entry.endswith('.md5'):
continue
ref_path = os.path.join(refs_path, entry)
f = open(ref_path, 'r')
md5_file.write("%s %s\n" % (md5(f.read()).hexdigest(), ref_path))
f.close()
if delete_refs:
os.remove(ref_path)
md5_file.close()
def compare_checksums(self, refs_path, out_path, remove_results = True, create_diffs = True):
retval = True
md5_path = os.path.join(refs_path, self._name)
md5_file = open(md5_path + '.md5', 'r')
tests = os.listdir(out_path)
for line in md5_file.readlines():
md5sum, ref_path = line.strip('\n').split(' ')
basename = os.path.basename(ref_path)
if not basename in tests:
retval = False
print "%s found in md5 ref file but missing in output dir %s" % (basename, out_path)
continue
result_path = os.path.join(out_path, basename)
f = open(result_path, 'r')
matched = md5sum == md5(f.read()).hexdigest()
f.close()
if matched:
if remove_results:
os.remove(result_path)
else:
print "Differences found in %s" % (basename)
if create_diffs:
if not os.path.exists(ref_path):
print "Reference file %s not found, skipping diff for %s" % (ref_path, result_path)
else:
try:
self._create_diff(ref_path, result_path)
except NotImplementedError:
# Diff not supported by backend
pass
retval = False
md5_file.close()
return retval
def has_md5(self, test_path):
return os.path.exists(os.path.join(test_path, self._name + '.md5'))
def is_crashed(self, test_path):
return os.path.exists(os.path.join(test_path, self._name + '.crashed'))
def is_failed(self, test_path):
failed_path = os.path.join(test_path, self._name + '.failed')
if not os.path.exists(failed_path):
return 0
f = open(failed_path, 'r')
status = int(f.read())
f.close()
return status
def has_stderr(self, test_path):
return os.path.exists(os.path.join(test_path, self._name + '.stderr'))
def __create_stderr_file(self, stderr, out_path):
if not stderr:
return
stderr_file = open(out_path + '.stderr', 'w')
stderr_file.write(stderr)
stderr_file.close()
def __create_failed_file_if_needed(self, status, out_path):
if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
return False
failed_file = open(out_path + '.failed', 'w')
failed_file.write("%d" % (os.WEXITSTATUS(status)))
failed_file.close()
return True
def _check_exit_status(self, p, out_path):
status = p.wait()
stderr = p.stderr.read()
self.__create_stderr_file(stderr, out_path)
if not os.WIFEXITED(status):
open(out_path + '.crashed', 'w').close()
return False
if self.__create_failed_file_if_needed(status, out_path):
return False
return True
def _check_exit_status2(self, p1, p2, out_path):
status1 = p1.wait()
status2 = p2.wait()
p1_stderr = p1.stderr.read()
p2_stderr = p2.stderr.read()
if p1_stderr or p2_stderr:
self.__create_stderr_file(p1_stderr + p2_stderr, out_path)
if not os.WIFEXITED(status1) or not os.WIFEXITED(status2):
open(out_path + '.crashed', 'w').close()
return False
if self.__create_failed_file_if_needed(status1, out_path):
return False
if self.__create_failed_file_if_needed(status2, out_path):
return False
return True
def _diff_png(self, ref_path, result_path):
try:
import Image, ImageChops
except ImportError:
raise NotImplementedError
ref = Image.open(ref_path)
result = Image.open(result_path)
diff = ImageChops.difference(ref, result)
diff.save(result_path + '.diff', 'png')
def _create_diff(self, ref_path, result_path):
raise NotImplementedError
def create_refs(self, doc_path, refs_path):
raise NotImplementedError
_backends = {}
def register_backend(backend_name, backend_class):
_backends[backend_name] = backend_class
def _get_backend(backend_name):
if backend_name not in _backends:
try:
__import__('backends.%s' % backend_name)
except ImportError:
pass
if backend_name not in _backends:
raise UnknownBackendError('Backend %s does not exist' % backend_name)
return _backends[backend_name]
def get_backend(backend_name):
backend_class = _get_backend(backend_name)
return backend_class(backend_name)
def get_all_backends():
backends = []
thisdir = os.path.abspath(os.path.dirname(__file__))
for fname in os.listdir(os.path.join(thisdir)):
name, ext = os.path.splitext(fname)
if not ext == '.py':
continue
try:
__import__('backends.%s' % name)
except ImportError:
continue
if name in _backends:
backends.append(_backends[name](name))
return backends
# cairo.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from backends import Backend, register_backend
import subprocess
import os
class Cairo(Backend):
def __init__(self, name):
Backend.__init__(self, name)
self._pdftocairo = os.path.join(self._utilsdir, 'pdftocairo');
def create_refs(self, doc_path, refs_path):
out_path = os.path.join(refs_path, 'cairo')
p1 = subprocess.Popen([self._pdftocairo, '-cropbox', '-e', '-png', doc_path, out_path], stderr = subprocess.PIPE)
p2 = subprocess.Popen([self._pdftocairo, '-cropbox', '-o', '-png', doc_path, out_path], stderr = subprocess.PIPE)
return self._check_exit_status2(p1, p2, out_path)
def _create_diff(self, ref_path, result_path):
self._diff_png(ref_path, result_path)
register_backend('cairo', Cairo)
# postscript.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from backends import Backend, register_backend
import subprocess
import os
class PostScript(Backend):
def __init__(self, name):
Backend.__init__(self, name)
self._pdftops = os.path.join(self._utilsdir, 'pdftops');
def create_refs(self, doc_path, refs_path):
out_path = os.path.join(refs_path, 'postscript')
p = subprocess.Popen([self._pdftops, doc_path, out_path + '.ps'], stderr = subprocess.PIPE)
return self._check_exit_status(p, out_path)
register_backend('postscript', PostScript)
# splash.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from backends import Backend, register_backend
import subprocess
import os
class Splash(Backend):
def __init__(self, name):
Backend.__init__(self, name)
self._pdftoppm = os.path.join(self._utilsdir, 'pdftoppm');
def create_refs(self, doc_path, refs_path):
out_path = os.path.join(refs_path, 'splash')
p1 = subprocess.Popen([self._pdftoppm, '-cropbox', '-e', '-png', doc_path, out_path], stderr = subprocess.PIPE)
p2 = subprocess.Popen([self._pdftoppm, '-cropbox', '-o', '-png', doc_path, out_path], stderr = subprocess.PIPE)
return self._check_exit_status2(p1, p2, out_path)
def _create_diff(self, ref_path, result_path):
self._diff_png(ref_path, result_path)
register_backend('splash', Splash)
# text.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from backends import Backend, register_backend
import subprocess
import os
class Text(Backend):
def __init__(self, name):
Backend.__init__(self, name)
self._pdftotext = os.path.join(self._utilsdir, 'pdftotext');
def create_refs(self, doc_path, refs_path):
out_path = os.path.join(refs_path, 'text')
p = subprocess.Popen([self._pdftotext, doc_path, out_path + '.txt'], stderr = subprocess.PIPE)
return self._check_exit_status(p, out_path)
def _create_diff(self, ref_path, result_path):
import difflib
ref = open(ref_path, 'r')
result = open(result_path, 'r')
diff = difflib.unified_diff(ref.readlines(), result.readlines(), ref_path, result_path)
ref.close()
result.close()
diff_file = open(result_path + '.diff', 'w')
diff_file.writelines(diff)
diff_file.close()
register_backend('text', Text)