validate.py 8.55 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# -*- Mode: Python -*- vi:si:et:sw=4:sts=4:ts=4:syntax=python
#
# Copyright (c) 2014,Thibault Saunier <thibault.saunier@collabora.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.

"""
The GstValidate default testsuite
"""

24
import os
25
from testsuiteutils import update_assets
26
from launcher.baseclasses import MediaFormatCombination
27
28


29
30
31
32
TEST_MANAGER = "validate"

BLACKLIST = [('validate.file.transcode.to_vorbis_and_vp8_in_webm.GH1_00094_1920x1280_MTS',
              'Got error: Internal data stream error. -- Debug message: mpegtsbase.c(1371):'
33
34
              'mpegts_base_loop (): ...: stream stopped, reason not-negotiated'),
             ('validate.file.*.simple.fast_forward.synchronized',
35
36
37
              'https://bugzilla.gnome.org/show_bug.cgi?id=775102'),
             ('validate.hls.playback.change_state_intensive.*',
              'https://bugzilla.gnome.org/show_bug.cgi?id=775118'),
Thibault Saunier's avatar
Thibault Saunier committed
38
            ('validate.rtsp.*playback.switch.*',
39
              'https://bugzilla.gnome.org/show_bug.cgi?id=783436'),
Thibault Saunier's avatar
Thibault Saunier committed
40
            ('validate.rtsp.*playback.*seek.*mxf$|validate.rtsp.*playback.*change_state_intensive.*mxf$',
41
              'Actions on MXF streams with rtsp-server fail in racy ways.'
42
             ' (Deactivating as it is not very important.)'),
43
44
45
            ('validate.rtsp.*pal-dv25_mxf$',
              'File has decoding issues with rtsp-server.'
             ' (Deactivating as it is not very important.)'),
46
47
             ("(?!.*.media_check.qtdemux-test-frag-basic_zero_dur_no_mehd_mp4).*.qtdemux-test-frag-basic_zero_dur_no_mehd_mp4",
              '`qtdemux-test-frag-basic_zero_dur_no_mehd_mp4` is there only for media_check tests.'),
48
            ]
49

50

51
EXPECTED_ISSUES = {
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
    'validate.dash.playback.*seek.*|validate.dash.playback.*reverse.*'
    '|validate.dash.playback.*fast.':
    [
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=775266',
            'detected-on': 'playbin',
            'summary': 'We got an ERROR message on the bus',
            'level': 'critical',
            'sometimes': True,
        },
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=775266',
            'summary': "flow return from pad push doesn't match expected value",
            'details': '.*Wrong combined flow return error.*',
            'level': 'critical',
            'sometimes': True,
        },
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=775266',
            'level': 'critical',
            'summary': 'All the actions were not executed before the program stopped',
            'sometimes': True,
        }
    ],
76
77
78
79
80
81
    'validate.http.*.vorbis_theora_1_ogg':
    [
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=775107',
            'detected-on': 'playbin',
            'summary': 'We got an ERROR message on the bus',
82
            'details': '.*No valid frames decoded before end of stream.*',
83
84
85
86
87
88
            'level': 'critical',
            'sometimes': True,
        },
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=775107',
            'level': 'critical',
89
            'summary': '.*All the actions were not executed before the program stopped.*',
90
91
92
            'sometimes': True,
        }
    ],
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    'validate.rtsp.playback.change_state_intensive.*':
    [
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=783516',
            'level': 'critical',
            'summary': 'We got an ERROR message on the bus',
            'details': '.*Got error: Could not read from resource.*',
            'sometimes': True,
        },
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=783516',
            'level': 'critical',
            'summary': 'The program stopped before some actions were executed',
            'sometimes': True,
        }
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
    ],
    'validate.file.playback.scrub_forward_seeking.op2b-mpeg2-wave_hd_mxf':
    [
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=796746',
            'level': 'critical',
            'summary': 'We got an ERROR message on the bus',
            'details': '.*Got error: No valid frames decoded before end of stream.*',
        },
        {
            'bug': 'https://bugzilla.gnome.org/show_bug.cgi?id=796746',
            'level': 'critical',
            'summary': 'The program stopped before some actions were executed',
        }
    ],
123
124
}

125

126
def setup_tests(test_manager, options):
127
    print("Setting up GstValidate default tests")
128

129
130
131
    testsuite_dir = os.path.realpath(os.path.join(os.path.dirname(__file__)))

    assets_dir = os.path.realpath(os.path.join(testsuite_dir, os.path.pardir, "medias", "defaults"))
132
    if options.sync:
133
        if not update_assets(options, assets_dir):
134
135
            return False

136
    options.add_paths(assets_dir)
137
    options.set_http_server_dir(os.path.join(testsuite_dir, os.path.pardir, "medias"))
138
    test_manager.set_default_blacklist(BLACKLIST)
139
140
141
142
143
144

    pipelines_tests = os.path.join(testsuite_dir, 'pipelines.json')
    test_manager.add_generators(
        test_manager.GstValidatePipelineTestsGenerator.from_json(test_manager, pipelines_tests)
    )

145
    test_manager.add_expected_issues(EXPECTED_ISSUES)
146
147
    test_manager.register_defaults()

148
149
    test_manager.add_encoding_formats([MediaFormatCombination("quicktime", "rawaudio", "prores")])

150
151
152
153
154
155
156
157
158
159
    valid_mixing_scenarios = ["play_15s",
                              "fast_forward",
                              "seek_forward",
                              "seek_backward",
                              "seek_with_stop",
                              "scrub_forward_seeking"]

    for compositor in ["compositor", "glvideomixer"]:
            test_manager.add_generators(
                test_manager.GstValidateMixerTestsGenerator(compositor + ".simple", test_manager,
160
161
162
163
164
165
166
167
168
169
170
171
                                                            compositor,
                                                            "video",
                                                            converter="deinterlace ! videoconvert",
                                                            mixed_srcs={
                                                                 "synchronized": {"mixer_props": "sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50",  # noqa
                                                                                  "sources":
                                                                                  ("videotestsrc pattern=snow timestamp-offset=3000000000 ! 'video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1' !  timeoverlay",  # noqa
                                                                                   "videotestsrc pattern=smpte ! 'video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1' ! timeoverlay")},  # noqa
                                                                 "bgra": ("videotestsrc ! video/x-raw, framerate=\(fraction\)10/1, width=100, height=100",  # noqa
                                                                          "videotestsrc ! video/x-raw, framerate=\(fraction\)5/1, width=320, height=240")
                                                            },
                                                            valid_scenarios=valid_mixing_scenarios))
172
173
174

    test_manager.add_generators(
        test_manager.GstValidateMixerTestsGenerator("audiomixer.simple", test_manager,
175
176
177
178
179
180
181
                                                    "audiomixer",
                                                    "audio",
                                                    converter="audioconvert ! audioresample",
                                                    mixed_srcs={"basic": {"mixer_props": "",
                                                                "sources": ("audiotestsrc wave=triangle",
                                                                            "audiotestsrc wave=ticks")}},
                                                    valid_scenarios=valid_mixing_scenarios))
182
183

    return True