TestRun.py 13.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# TestRun.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18
from __future__ import print_function
19
20
21

from backends import get_backend, get_all_backends
from Config import Config
22
from Utils import get_document_paths_from_dir, get_skipped_tests, get_passwords
23
from Printer import get_printer
24
25
26
import sys
import os
import errno
27
import shutil
28

29
from InterruptibleQueue import InterruptibleQueue
30
31
from threading import Thread, RLock

32
33
class TestRun:

34
    def __init__(self, docsdir, refsdir, outdir, max_failures = None):
35
36
37
        self._docsdir = docsdir
        self._refsdir = refsdir
        self._outdir = outdir
38
        self._max_failures = max_failures
39
        self._skip = get_skipped_tests(docsdir)
40
        self._passwords = get_passwords(docsdir)
41
        self.config = Config()
42
        self.printer = get_printer()
43
        self._total_tests = 1
44
        self._exited_early = False
45
46
47

        # Results
        self._n_tests = 0
48
        self._n_run = 0
49
        self._n_passed = 0
50
51
52
        self._failed = {}
        self._crashed = {}
        self._failed_status_error = {}
53
54
        self._did_not_crash = {}
        self._did_not_fail_status_error = {}
55
        self._stderr = {}
56
        self._skipped = []
57
        self._new = []
58

59
        self._queue = InterruptibleQueue()
60
61
        self._lock = RLock()

62
63
        try:
            os.makedirs(self._outdir);
Albert Astals Cid's avatar
Albert Astals Cid committed
64
        except OSError as e:
65
66
67
68
69
            if e.errno != errno.EEXIST:
                raise
        except:
            raise

70
71
72
73
74
75
    def _get_backends(self):
        if self.config.backends:
            return [get_backend(name) for name in self.config.backends]

        return get_all_backends()

76
    def test(self, refs_path, doc_path, test_path, backend, password):
77
78
79
80
81
        # First check whether there are test results for the backend
        ref_has_md5 = backend.has_md5(refs_path)
        ref_is_crashed = backend.is_crashed(refs_path)
        ref_is_failed = backend.is_failed(refs_path)
        if not ref_has_md5 and not ref_is_crashed and not ref_is_failed:
82
            with self._lock:
83
                self._new.append("%s (%s)" % (doc_path, backend.get_name()))
84
                self._n_tests += 1
85
            self.printer.print_default("Reference files not found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
86
87
            return

88
        test_has_md5 = backend.create_refs(doc_path, test_path, password)
89
90
91
        test_passed = False
        if ref_has_md5 and test_has_md5:
            test_passed = backend.compare_checksums(refs_path, test_path, not self.config.keep_results, self.config.create_diffs, self.config.update_refs)
92
93
        elif self.config.update_refs:
            backend.update_results(refs_path, test_path)
94

95
96
        with self._lock:
            self._n_tests += 1
97
            self._n_run += 1
98

99
            if backend.has_stderr(test_path):
100
                self._stderr.setdefault(backend.get_name(), []).append(doc_path)
101

102
103
104
105
            if ref_has_md5 and test_has_md5:
                if test_passed:
                    # FIXME: remove dir if it's empty?
                    self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS")
106
                    self._n_passed += 1
107
108
                else:
                    self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "FAIL")
109
                    self._failed.setdefault(backend.get_name(), []).append(doc_path)
110
                return
111

112
113
114
            if test_has_md5:
                if ref_is_crashed:
                    self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "DOES NOT CRASH")
115
                    self._did_not_crash.setdefault(backend.get_name(), []).append(doc_path)
116
117
                elif ref_is_failed:
                    self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "DOES NOT FAIL")
118
                    self._did_not_fail_status_error.setdefault(backend.get_name(), []).append(doc_path)
119
120
121
122
123
                return

            test_is_crashed = backend.is_crashed(test_path)
            if ref_is_crashed and test_is_crashed:
                self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS (Expected crash)")
124
                self._n_passed += 1
125
                return
126

127
128
129
130
            test_is_failed = backend.is_failed(test_path)
            if ref_is_failed and test_is_failed:
                # FIXME: compare status errors
                self.printer.print_test_result(doc_path, backend.get_name(), self._n_tests, self._total_tests, "PASS (Expected fail with status error %d)" % (test_is_failed))
131
                self._n_passed += 1
132
                return
133

134
135
            if test_is_crashed:
                self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "CRASH")
136
                self._crashed.setdefault(backend.get_name(), []).append(doc_path)
137
                return
138

139
140
            if test_is_failed:
                self.printer.print_test_result_ln(doc_path, backend.get_name(), self._n_tests, self._total_tests, "FAIL (status error %d)" % (test_is_failed))
141
                self._failed_status_error.setdefault(backend.get_name(), []).append(doc_path)
142
143
144
145
                return

    def run_test(self, filename):
        backends = self._get_backends()
146

147
148
        if filename in self._skip:
            doc_path = os.path.join(self._docsdir, filename)
149
150
            with self._lock:
                self._skipped.append("%s" % (doc_path))
151
152
                self._n_tests += len(backends)
            self.printer.print_default("Skipping test '%s'" % (doc_path))
153
154
            return

155
156
157
        out_path = os.path.join(self._outdir, filename)
        try:
            os.makedirs(out_path)
Albert Astals Cid's avatar
Albert Astals Cid committed
158
        except OSError as e:
159
160
161
162
163
164
165
166
            if e.errno != errno.EEXIST:
                raise
        except:
            raise
        doc_path = os.path.join(self._docsdir, filename)
        refs_path = os.path.join(self._refsdir, filename)

        if not os.path.isdir(refs_path):
167
            with self._lock:
168
                self._new.append("%s" % (doc_path))
169
170
                self._n_tests += len(backends)
            self.printer.print_default("Reference dir not found for %s, skipping" % (doc_path))
171
172
            return

173
174
        password = self._passwords.get(filename)

175
        for backend in backends:
176
            self.test(refs_path, doc_path, out_path, backend, password)
177

178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
    def _should_exit_early(self):
        if self._max_failures is None:
            return False

        def _len(tests):
            retval = 0
            for backend in tests:
                retval += len(tests[backend])
            return retval

        with self._lock:
            if _len(self._failed) + _len(self._crashed) + _len(self._failed_status_error) >= self._max_failures:
                if not self._exited_early:
                    self.printer.printout_ln('Exiting early after %d failures, waiting for running jobs to finish...' % self._max_failures)
                    self.printer.block()
                self._exited_early = True
                return True
        return False

197
198
    def _worker_thread(self):
        while True:
199
            doc = self._queue.get()
200
201
            if not self._should_exit_early():
                self.run_test(doc)
202
            self._queue.task_done()
203

204
    def run_tests(self, tests = []):
205
206
207
208
209
210
211
        # Clean the output dir.
        try:
            shutil.rmtree(self._outdir)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        except:
212
            raise
213

214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
        if not tests:
            docs, total_docs = get_document_paths_from_dir(self._docsdir)
        else:
            docs = []
            total_docs = 0
            for test in tests:
                if os.path.isdir(test):
                    test_dir = test
                elif os.path.isdir(os.path.join(self._docsdir, test)):
                    test_dir = os.path.join(self._docsdir, test)
                else:
                    test_dir = None

                if test_dir is not None:
                    dir_docs, dir_n_docs = get_document_paths_from_dir(test_dir, self._docsdir)
                    docs.extend(dir_docs)
                    total_docs += dir_n_docs
                else:
                    if test.startswith(self._docsdir):
                        test = test[len(self._docsdir):].lstrip(os.path.sep)
                    docs.append(test)
                    total_docs += 1

237
238
239
        backends = self._get_backends()
        self._total_tests = total_docs * len(backends)

240
241
242
243
244
        if total_docs == 1:
            n_workers = 0
        else:
            n_workers = min(self.config.threads, total_docs)

245
246
        self.printer.printout_ln('Found %d documents' % (total_docs))
        self.printer.printout_ln('Backends: %s' % ', '.join([backend.get_name() for backend in backends]))
247
        self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), n_workers))
248
        self.printer.printout_ln()
249

250
        if n_workers > 0:
251
            self.printer.printout('Spawning %d workers...' % (n_workers))
252

253
254
255
256
            for n_thread in range(n_workers):
                thread = Thread(target=self._worker_thread)
                thread.daemon = True
                thread.start()
257

258
259
            for doc in docs:
                self._queue.put(doc)
260

261
262
263
264
            self._queue.join()
        else:
            for doc in docs:
                self.run_test(doc)
265
266
267
268
269
270
271
                if self._should_exit_early():
                    break

        if self._exited_early:
            open(os.path.join(self._outdir, '.exited_early'), 'w').close()
            self.printer.unblock()
            return -self._max_failures
272

273
274
        return int(self._n_passed != self._n_run)

275
    def summary(self):
276
        self.printer.printout_ln()
277
278

        if self._n_run:
279
280
281
            if self._exited_early:
                self.printer.printout_ln("Testing exited early")
                self.printer.printout_ln()
282
            self.printer.printout_ln("%d tests passed (%.2f%%)" % (self._n_passed, (self._n_passed * 100.) / self._n_run))
283
            self.printer.printout_ln()
284

285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
            def result_tests(test_dict):
                if not test_dict:
                    return 0, None

                n_tests = 0
                tests = ""
                for backend in test_dict:
                    backend_docs = test_dict[backend]
                    n_tests += len(backend_docs)
                    tests += "\n".join(["  %s (%s)" % (doc_path, backend) for doc_path in backend_docs])
                    tests += "\n"

                return n_tests, tests

            def backends_summary(test_dict, n_tests):
                percs = []
                for backend in test_dict:
                    n_docs = len(test_dict[backend])
                    percs.append("%d %s (%.2f%%)" % (n_docs, backend, (n_docs * 100.) / n_tests))
                return ", ".join(percs)

            test_results = [(self._failed, "unexpected failures"),
                            (self._crashed, "unexpected crashes"),
                            (self._failed_status_error, "unexpected failures (test program returned with an exit error status)"),
309
310
311
                            (self._stderr, "tests have stderr output"),
                            (self._did_not_crash, "expected to crash, but didn't crash"),
                            (self._did_not_fail_status_error, "expected to fail to run, but didn't fail")]
312
313
314
315
316
317
318
319
320

            for test_dict, test_msg in test_results:
                n_tests, tests = result_tests(test_dict)
                if n_tests == 0:
                    continue

                self.printer.printout_ln("%d %s (%.2f%%) [%s]" % (n_tests, test_msg, (n_tests * 100.) / self._n_run, backends_summary(test_dict, n_tests)))
                self.printer.printout_ln(tests)
                self.printer.printout_ln()
321
322
        else:
            self.printer.printout_ln("No tests run")
323

324
        if self._skipped:
325
326
            self.printer.printout_ln("%d tests skipped" % len(self._skipped))
            self.printer.printout_ln("\n".join(["  %s" % skipped for skipped in self._skipped]))
327
            self.printer.printout_ln()
328
329

        if self._new:
330
331
332
            self.printer.printout_ln("%d new documents" % len(self._new))
            self.printer.printout_ln("\n".join(["  %s" % new for new in self._new]))
            self.printer.printout_ln("Use create-refs command to add reference results for them")
333
            self.printer.printout_ln()