Commit 817cc333 authored by Carlos Garcia Campos's avatar Carlos Garcia Campos

regtest: Do not buffer stderr output

Some buggy documents can produce a huge stderr output because of parsing
errors or whatever. We could give a file directly to Popen to write the
stderr file, but we only want to create the file when there's output,
because it's what we use to know whether the command produced output or
not. So, instead of buffering the whole output and then write it to the
file, now we read from the pipe while the command is running, writing
the output in chunks to the file. This improves a lot the memory
consumption when running some tests.
parent f8f82f1c
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
import hashlib import hashlib
import os import os
import select
import shutil import shutil
import errno import errno
from Config import Config from Config import Config
...@@ -193,13 +194,6 @@ class Backend: ...@@ -193,13 +194,6 @@ class Backend:
return False return False
return os.path.exists(test_result + self._diff_ext) return os.path.exists(test_result + self._diff_ext)
def __create_stderr_file(self, stderr, out_path):
if not stderr:
return
stderr_file = open(out_path + '.stderr', 'wb')
stderr_file.write(stderr)
stderr_file.close()
def __create_failed_file_if_needed(self, status, out_path): def __create_failed_file_if_needed(self, status, out_path):
if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0: if os.WIFEXITED(status) or os.WEXITSTATUS(status) == 0:
return False return False
...@@ -210,10 +204,36 @@ class Backend: ...@@ -210,10 +204,36 @@ class Backend:
return True return True
def _check_exit_status(self, p, out_path): def __redirect_stderr_to_file(self, fd, out_path):
stderr = p.stderr.read() stderr_file = None
self.__create_stderr_file(stderr, out_path) read_set = [fd]
while read_set:
try:
rlist, wlist, xlist = select.select(read_set, [], [])
except select.error as e:
continue
if fd in rlist:
try:
chunk = os.read(fd, 1024)
except OSError as e:
if e.errno == errno.EIO:
# Child process finished.
chunk = ''
else:
raise e
if chunk:
if stderr_file is None:
stderr_file = open(out_path + '.stderr', 'wb')
stderr_file.write(chunk)
else:
read_set.remove(fd)
if stderr_file is not None:
stderr_file.close()
def _check_exit_status(self, p, out_path):
self.__redirect_stderr_to_file(p.stderr.fileno(), out_path)
status = p.wait() status = p.wait()
if not os.WIFEXITED(status): if not os.WIFEXITED(status):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment