TestReferences.py 4.42 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
# TestReferences.py
#
# Copyright (C) 2011 Carlos Garcia Campos <carlosgc@gnome.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18
from __future__ import absolute_import, division, print_function
19 20 21 22 23

import os
import errno
from backends import get_backend, get_all_backends
from Config import Config
24
from Printer import get_printer
25
from Utils import get_document_paths_from_dir, get_skipped_tests, get_passwords
26

27
from InterruptibleQueue import InterruptibleQueue
28
from threading import Thread, RLock
29

30 31 32 33 34
class TestReferences:

    def __init__(self, docsdir, refsdir):
        self._docsdir = docsdir
        self._refsdir = refsdir
35
        self._skipped = get_skipped_tests(docsdir)
36
        self._passwords = get_passwords(docsdir)
37
        self.config = Config()
38
        self.printer = get_printer()
39 40
        self._total_tests = 1
        self._n_tests = 0
41

42
        self._queue = InterruptibleQueue()
43
        self._lock = RLock()
44

45 46
        try:
            os.makedirs(self._refsdir)
47
        except OSError as e:
48 49 50 51 52
            if e.errno != errno.EEXIST:
                raise
        except:
            raise

53 54 55 56 57 58 59 60 61
    def _get_backends(self):
        if self.config.backends:
            return [get_backend(name) for name in self.config.backends]

        return get_all_backends()

    def create_refs_for_file(self, filename):
        backends = self._get_backends()

62
        if filename in self._skipped:
63 64 65
            with self._lock:
                self._n_tests += len(backends)
            self.printer.print_default("Skipping test '%s'" % (os.path.join(self._docsdir, filename)))
66 67
            return

68 69 70
        refs_path = os.path.join(self._refsdir, filename)
        try:
            os.makedirs(refs_path)
71
        except OSError as e:
72 73 74 75 76 77
            if e.errno != errno.EEXIST:
                raise
        except:
            raise
        doc_path = os.path.join(self._docsdir, filename)

78 79
        password = self._passwords.get(filename)

80
        for backend in backends:
81
            if not self.config.force and backend.has_results(refs_path):
82 83 84
                with self._lock:
                    self._n_tests += 1
                self.printer.print_default("Results found, skipping '%s' for %s backend" % (doc_path, backend.get_name()))
85
                continue
86

87
            if backend.create_refs(doc_path, refs_path, password):
88
                backend.create_checksums(refs_path, self.config.checksums_only)
89 90
            with self._lock:
                self._n_tests += 1
91
            self.printer.printout_ln("[%d/%d] %s (%s): done" % (self._n_tests, self._total_tests, doc_path, backend.get_name()))
92

93 94
    def _worker_thread(self):
        while True:
95 96
            doc = self._queue.get()
            self.create_refs_for_file(doc)
97
            self._queue.task_done()
98

99
    def create_refs(self):
100
        docs, total_docs = get_document_paths_from_dir(self._docsdir)
101 102 103
        backends = self._get_backends()
        self._total_tests = total_docs * len(backends)

104 105 106 107 108
        if total_docs == 1:
            n_workers = 0
        else:
            n_workers = min(self.config.threads, total_docs)

109 110
        self.printer.printout_ln('Found %d documents' % (total_docs))
        self.printer.printout_ln('Backends: %s' % ', '.join([backend.get_name() for backend in backends]))
111
        self.printer.printout_ln('Process %d using %d worker threads' % (os.getpid(), n_workers))
112
        self.printer.printout_ln()
113

114 115
        if n_workers > 0:
            self.printer.printout('Spawning %d workers...' % (n_workers))
116

117 118 119 120
            for n_thread in range(n_workers):
                thread = Thread(target=self._worker_thread)
                thread.daemon = True
                thread.start()
121

122 123
            for doc in docs:
                self._queue.put(doc)
124

125 126 127 128
            self._queue.join()
        else:
            for doc in docs:
                self.create_refs_for_file(doc)