Commit 5ae7be07 authored by Edward Hervey's avatar Edward Hervey 🤘 Committed by Edward Hervey
Browse files

testsuites: New DASH-IF conformity testsuite

Allows testing various DASH-IF Test Assets
parent d25630c0
# Ignore caps changing issues (aacparse and container reporting bogus streams)
change-severity, issue-id=caps::field-unexpected-value, new-severity=warning
<file duration="654000000000" frame-detection="0" uri="http://dash.akamaized.net/dash264/TestCases/1a/netflix/exMPD_BIP_TC1.mpd" seekable="true">
<streams caps="application/dash+xml">
<stream type="audio" caps="audio/mpeg, mpegversion=(int)4, framed=(boolean)true, stream-format=(string)raw, level=(string)1, base-profile=(string)lc, profile=(string)lc, codec_data=(buffer)1310, rate=(int)24000, channels=(int)2" id="f49d93aa67485a428786d40153e50898f96d960963626eb89bc361ba90b971fb/audio_00/001">
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)119813, bitrate=(uint)65319, datetime=(datetime)2012-05-30T10:53:59Z, container-format=(string)&quot;ISO\ fMP4&quot;;"/>
</tags>
</stream>
<stream type="video" caps="video/x-h264, stream-format=(string)avc, alignment=(string)au, level=(string)3, profile=(string)constrained-baseline, codec_data=(buffer)0142c01effe100256742c01e925203c17fcbff800200018800001f480005dc07320092800c3540c2c0f162e48001000468ce32c8, width=(int)480, height=(int)360, pixel-aspect-ratio=(fraction)4/3, framerate=(fraction)47/2, interlace-mode=(string)progressive, parsed=(boolean)true" id="f49d93aa67485a428786d40153e50898f96d960963626eb89bc361ba90b971fb/video_00/002">
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, language-code=(string)apt, datetime=(datetime)2012-05-30T11:00:46Z, container-format=(string)&quot;ISO\ fMP4&quot;;"/>
</tags>
</stream>
</streams>
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, language-code=(string)apt, datetime=(datetime)2012-05-30T11:00:46Z, container-format=(string)&quot;ISO\ fMP4&quot;, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)119813, bitrate=(uint)65319;"/>
</tags>
</file>
\ No newline at end of file
# Ignore caps changing issues (aacparse and container reporting bogus streams)
change-severity, issue-id=caps::field-unexpected-value, new-severity=warning
<file duration="654000000000" frame-detection="0" uri="http://dash.akamaized.net/dash264/TestCases/1a/qualcomm/2/MultiRate.mpd" seekable="true">
<streams caps="application/dash+xml">
<stream type="video" caps="video/x-h264, stream-format=(string)avc, alignment=(string)au, level=(string)3.1, profile=(string)main, codec_data=(buffer)014d401fffe10019674d401feca02802dd80880000030008000003018078c18cb001000468ebecb2, width=(int)1280, height=(int)720, pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)100000/4201, interlace-mode=(string)progressive, parsed=(boolean)true" id="870b1e664380de860c75fcb2f0e000611647062e73a887332bafa76ef547cc23/video_00/001">
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)927346;"/>
</tags>
</stream>
<stream type="audio" caps="audio/mpeg, mpegversion=(int)4, framed=(boolean)true, stream-format=(string)raw, level=(string)1, codec_data=(buffer)eb098800, rate=(int)24000, channels=(int)1" id="870b1e664380de860c75fcb2f0e000611647062e73a887332bafa76ef547cc23/audio_00/001">
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32001, bitrate=(uint)32001, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)32943;"/>
</tags>
</stream>
</streams>
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32001, bitrate=(uint)32001, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)32943, video-codec=(string)&quot;H.264\ /\ AVC&quot;;"/>
</tags>
</file>
# Ignore caps changing issues (aacparse and container reporting bogus streams)
change-severity, issue-id=caps::field-unexpected-value, new-severity=warning
<file duration="654000000000" frame-detection="0" uri="http://dash.akamaized.net/dash264/TestCases/2a/qualcomm/1/MultiResMPEG2.mpd" seekable="true">
<streams caps="application/dash+xml">
<stream type="video" caps="video/x-h264, stream-format=(string)avc, alignment=(string)au, level=(string)3.1, profile=(string)main, codec_data=(buffer)014d401fffe10017674d401feca04012d808800000030080000018078c18cb01000468ebecb2, width=(int)512, height=(int)288, pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)47/2, interlace-mode=(string)progressive, parsed=(boolean)true" id="85ae0c35bbd331e8833c582a15a1d6b6a9fe447d1bbd39dba68f8bd54484380e/video_00/001">
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)601307;"/>
</tags>
</stream>
<stream type="audio" caps="audio/mpeg, mpegversion=(int)4, framed=(boolean)true, stream-format=(string)raw, level=(string)1, codec_data=(buffer)eb098800, rate=(int)24000, channels=(int)1" id="85ae0c35bbd331e8833c582a15a1d6b6a9fe447d1bbd39dba68f8bd54484380e/audio_00/001">
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32001, bitrate=(uint)32001, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)33210;"/>
</tags>
</stream>
</streams>
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32001, bitrate=(uint)32001, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)33210, video-codec=(string)&quot;H.264\ /\ AVC&quot;;"/>
</tags>
</file>
\ No newline at end of file
# Ignore caps changing issues (aacparse and container reporting bogus streams)
change-severity, issue-id=caps::field-unexpected-value, new-severity=warning
<file duration="704000000000" frame-detection="0" uri="http://dash.akamaized.net/dash264/TestCases/5c/nomor/4_1a.mpd" seekable="true">
<streams caps="application/dash+xml">
<stream type="video" caps="video/x-h264, stream-format=(string)avc, alignment=(string)au, level=(string)3.1, profile=(string)main, codec_data=(buffer)014d401fffe10019674d401feca02802dd80880000030008000003018078c18cb001000468ebecb2, width=(int)1280, height=(int)720, pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)47/2, interlace-mode=(string)progressive, parsed=(boolean)true" id="3a8db876ee14a383b9a67c22e5145db48296a592e7928ed95406b1c4ba8ecdec/video_00/001">
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)1622956;"/>
</tags>
</stream>
<stream type="audio" caps="audio/mpeg, mpegversion=(int)4, framed=(boolean)true, stream-format=(string)raw, level=(string)1, codec_data=(buffer)eb098800, rate=(int)48000, channels=(int)2" id="3a8db876ee14a383b9a67c22e5145db48296a592e7928ed95406b1c4ba8ecdec/audio_00/001">
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32002, bitrate=(uint)32002, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)34344;"/>
</tags>
</stream>
</streams>
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)32002, bitrate=(uint)32002, encoder=(string)Lavf55.16.101, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)34344, video-codec=(string)&quot;H.264\ /\ AVC&quot;;"/>
</tags>
</file>
\ No newline at end of file
# Ignore caps changing issues (aacparse and container reporting bogus streams)
change-severity, issue-id=caps::field-unexpected-value, new-severity=warning
<file duration="1491220992000000000" frame-detection="0" uri="http://vm2.dashif.org/livesim/testpic_2s/Manifest.mpd#t=posix:now" seekable="false">
<streams caps="application/dash+xml">
<stream type="audio" caps="audio/mpeg, mpegversion=(int)4, framed=(boolean)true, stream-format=(string)raw, level=(string)2, base-profile=(string)lc, profile=(string)lc, codec_data=(buffer)1190, rate=(int)48000, channels=(int)2" id="d1358c4723f528093c4780910d03b80d31a2e7e0abe0d072edccbc09a727f987/audio_00/001">
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, datetime=(datetime)2015-01-26T10:48:56Z, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)52692;"/>
</tags>
</stream>
<stream type="video" caps="video/x-h264, stream-format=(string)avc, alignment=(string)au, level=(string)3, profile=(string)high, codec_data=(buffer)0164001effe100196764001eacd940a02ff9610000030001000003003c8f162d9601000568ebecb22c, width=(int)640, height=(int)360, pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)59/2, interlace-mode=(string)progressive, parsed=(boolean)true" id="d1358c4723f528093c4780910d03b80d31a2e7e0abe0d072edccbc09a727f987/video_00/002">
<tags>
<tag content="taglist, video-codec=(string)&quot;H.264\ /\ AVC&quot;, language-code=(string)en, datetime=(datetime)2015-01-26T10:48:57Z, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)156456;"/>
</tags>
</stream>
</streams>
<tags>
<tag content="taglist, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, datetime=(datetime)2015-01-26T10:48:56Z, container-format=(string)&quot;ISO\ fMP4&quot;, nominal-bitrate=(uint)52692, video-codec=(string)&quot;H.264\ /\ AVC&quot;, language-code=(string)en;"/>
</tags>
</file>
\ No newline at end of file
# Pastebin PJeVxRyU
# -*- Mode: Python -*- vi:si:et:sw=4:sts=4:ts=4:syntax=python
#
# Copyright (c) 2014 Thibault Saunier <thibault.saunier@collabora.com>
# Copyright (c) 2017 Sebastian Droege <sebastian@centricular.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.
"""
The GstValidate DASH-IF test-vectors testsuite
"""
import os
from launcher.baseclasses import MediaFormatCombination
TEST_MANAGER = "validate"
BLACKLIST = [
('validate.dash.playback.trick_mode_seeks.DASHIF_TestCases_2a_qualcomm_1_MultiResMPEG2',
'https://bugzilla.gnome.org/show_bug.cgi?id=781313'),
('validate.dash.playback.trick_mode_seeks.DASHIF_TestCases_1a_netflix_exMPD_BIP_TC1',
'https://bugzilla.gnome.org/show_bug.cgi?id=781313'),
]
def setup_tests(test_manager, options):
print("Setting up GstValidate DASH-IF test-vectors tests")
assets_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "medias", "dash-if"))
options.add_paths(assets_dir)
test_manager.set_default_blacklist(BLACKLIST)
#test_manager.add_expected_issues(EXPECTED_ISSUES)
test_manager.register_defaults()
scenarios = test_manager.get_scenarios()
# Don't test generic tests that are already covered by the base validate runner
scenarios.remove("change_state_intensive")
# Scrubbing is a tad pointless/intensive for these suites
scenarios.remove("scrub_forward_seeking")
# Add keyunit trick mode testing
scenarios.append("trick_mode_seeks")
test_manager.set_scenarios(scenarios)
return True
# Pastebin PJeVxRyU
# -*- Mode: Python -*- vi:si:et:sw=4:sts=4:ts=4:syntax=python
#
# Copyright (c) 2014 Thibault Saunier <thibault.saunier@collabora.com>
# Copyright (c) 2017 Sebastian Droege <sebastian@centricular.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.
"""
The GstValidate DASH-IF test-vectors testsuite
"""
import json, requests
import sys
import xml.etree.ElementTree as ET
TESTVECTOR_URL = "http://testassets.dashif.org:3000/v1/testvectors"
def cleanup_entries(data):
# we need to do some minor cleanup of the entries:
# * Replace <a href=..>something</a>
# * Add __ID corresponding to the testvector unique key on the website
d = data["data"]
for entry in d:
# "testvector": "<a href=\"#testvector/details/58a5e0707459f8cb201b8ceb\">W3C Clear Key - audio only variant</a>",
testvector = entry["testvector"]
tmp = ET.XML(testvector)
_id = tmp.attrib["href"].split('/')[-1]
_desc = tmp.text
# use a key that is almost certain to be the first one. Ensures entries are sorted by ID
entry["A__ID"] = _id
entry["testvector_description"] = _desc
# "url": "<a href=https://media.axprod.net/TestVectors/v7-MultiDRM-MultiKey-MultiPeriod/Manifest_1080p_ClearKey.mpd>Link</a>"
# badly formed html, let's just use splitters
url = entry["url"]
_url = url.split("<a href=")[1].split(">Link<")[0]
entry["url"] = _url
def update_testvector_list(outputfile=None):
# download and cleanup testvector json
resp = requests.get(url=TESTVECTOR_URL)
data = json.loads(resp.text)
cleanup_entries(data)
if outputfile is None:
print(json.dumps(data["data"], indent=4, sort_keys=True))
else:
json.dump(data["data"], fp=open(outputfile, "w"), indent=4, sort_keys=True)
if __name__ == "__main__":
if len(sys.argv) > 1:
update_testvector_list(sys.argv[1])
else:
update_testvector_list()
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment