Skip to content

Commit

Permalink
COM-12448: Add H264 MVC test suite (#200)
Browse files Browse the repository at this point in the history 
* COM-12448: Add H264 MVC test suite

- added test suite generator code for MVC
- added MVC test suite
- generated output checksums with reference decoder

* COM-12448: Refactor, correct syntax, generate H264 test suites

- refactored gen_jvt.py
- regenerated JVT-AVC, JVT-MVC, JVT-Professional_profiles and
  JVT-SVC test suites
  • Loading branch information
@mdimopoulos
mdimopoulos authored Nov 7, 2024
1 parent 5771c37 commit 2cc85c3
Show file tree
Hide file tree
Showing 6 changed files with 266 additions and 58 deletions.
7 changes: 4 additions & 3 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,11 @@ packages = ["fluster", "fluster.decoders"]
"test_suites/av1/CHROMIUM-8bit-AV1-TEST-VECTORS.json"
]
"share/fluster/test_suites/h264" = [
"test_suites/h264/JVT-AVC_V1.json",
"test_suites/h264/JVT-AVC_v1.json",
"test_suites/h264/JVT-FR-EXT.json",
"test_suites/h264/JVT-Professional_profiles_V1.json",
"test_suites/h264/JVT-SVC_V1.json"
"test_suites/h264/JVT-MVC.json",
"test_suites/h264/JVT-Professional_profiles.json",
"test_suites/h264/JVT-SVC.json"
]
"share/fluster/test_suites/h265" = [
"test_suites/h265/JCT-VC-3D-HEVC.json",
Expand Down
106 changes: 73 additions & 33 deletions scripts/gen_jvt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,18 @@

BASE_URL = "https://www.itu.int/"
H264_URL = BASE_URL + "wftp3/av-arch/jvt-site/draft_conformance/"
BITSTREAM_EXTS = (
BITSTREAM_EXTS = [
".264",
".h264",
".jsv",
".jvt",
".avc",
".26l",
".bits",
)
MD5_EXTS = ("yuv_2.md5", "yuv.md5", ".md5", "md5.txt", "md5sum.txt")
MD5_EXCLUDES = (".bin.md5", "bit.md5")
RAW_EXTS = ("nogray.yuv", ".yuv", ".qcif")
]
MD5_EXTS = ["yuv_2.md5", "yuv.md5", ".md5", "md5.txt", "md5sum.txt"]
MD5_EXCLUDES = [".bin.md5", "bit.md5"]
RAW_EXTS = ["nogray.yuv", ".yuv", ".qcif"]


class HREFParser(HTMLParser):
Expand Down Expand Up @@ -81,7 +81,7 @@ def __init__(
codec: Codec,
description: str,
site: str,
use_ffprobe: bool = False
use_ffprobe: bool = False,
):
self.name = name
self.suite_name = suite_name
Expand Down Expand Up @@ -113,10 +113,15 @@ def generate(self, download, jobs):
# The first item in the AVCv1 list is a readme file
if "00readme_H" in url:
continue
# MVC contains files marked as old, we want to skip those
if "_old" in url:
continue
file_url = os.path.basename(url)
name = os.path.splitext(file_url)[0]
file_input = f"{name}.bin"
test_vector = TestVector(name, url, "__skip__", file_input, OutputFormat.YUV420P, "")
test_vector = TestVector(
name, url, "__skip__", file_input, OutputFormat.YUV420P, ""
)
test_suite.test_vectors[name] = test_vector

if download:
Expand Down Expand Up @@ -146,11 +151,19 @@ def generate(self, download, jobs):
raise Exception(f"Bitstream file not found in {dest_dir}")
test_vector.source_checksum = utils.file_checksum(dest_path)
if self.use_ffprobe:
ffprobe = utils.normalize_binary_cmd('ffprobe')
command = [ffprobe, '-v', 'error', '-select_streams', 'v:0',
'-show_entries', 'stream=pix_fmt', '-of',
'default=nokey=1:noprint_wrappers=1',
absolute_input_path]
ffprobe = utils.normalize_binary_cmd("ffprobe")
command = [
ffprobe,
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream=pix_fmt",
"-of",
"default=nokey=1:noprint_wrappers=1",
absolute_input_path,
]

result = utils.run_command_with_output(command).splitlines()
pix_fmt = result[0]
Expand All @@ -173,18 +186,27 @@ def generate(self, download, jobs):
"PPH444P6_Mitsubishi_A": OutputFormat.NONE,
"PPH444P7_Mitsubishi_A": OutputFormat.NONE,
"PPH444P8_Mitsubishi_A": OutputFormat.NONE,
"PPH444P9_Mitsubishi_A": OutputFormat.NONE
"PPH444P9_Mitsubishi_A": OutputFormat.NONE,
}
if test_vector.name in exceptions.keys():
test_vector.output_format = exceptions[test_vector.name]
else:
raise key_err

if self.name != "Professional_profiles": # result md5 generated from h264_reference_decoder
if self.name == "SVC": # result md5 generated for different Lines (L0, L1...)
new_vectors = self._fill_checksum_h264_multiple(test_vector, dest_dir)
if self.name not in (
"Professional_profiles",
"MVC",
): # result md5 generated from h264_reference_decoder
if (
self.name == "SVC"
): # result md5 generated for different Lines (L0, L1...)
new_vectors = self._fill_checksum_h264_multiple(
test_vector, dest_dir
)
new_test_vectors.extend(new_vectors)
test_suite.test_vectors = {vector.name: vector for vector in new_test_vectors}
test_suite.test_vectors = {
vector.name: vector for vector in new_test_vectors
}
else:
self._fill_checksum_h264(test_vector, dest_dir)

Expand All @@ -201,25 +223,31 @@ def _fill_checksum_h264(test_vector, dest_dir):
@staticmethod
def _fill_checksum_h264_multiple(test_vector, dest_dir):
def remove_r1_from_path(path):
parts = path.split('/')
parts = path.split("/")
if len(parts) >= 2:
parts[-2] = re.sub(r'-r1', '', parts[-2])
parts[-1] = re.sub(r'-r1', '', parts[-1])
return '/'.join(parts)
parts[-2] = re.sub(r"-r1", "", parts[-2])
parts[-1] = re.sub(r"-r1", "", parts[-1])
return "/".join(parts)

multiple_test_vectors = []

for suffix in [f"-L{i}" for i in range(8)]: # L0 ... L7
new_vector = copy.deepcopy(test_vector)
new_vector.name = test_vector.name + suffix

input_file_path = os.path.join(dest_dir, test_vector.name, f"{test_vector.name}{suffix}.264")
result_file_path = os.path.join(dest_dir, test_vector.name, f"{test_vector.name}{suffix}.yuv")
input_file_path = os.path.join(
dest_dir, test_vector.name, f"{test_vector.name}{suffix}.264"
)
result_file_path = os.path.join(
dest_dir, test_vector.name, f"{test_vector.name}{suffix}.yuv"
)

corrected_input_path = remove_r1_from_path(input_file_path)
corrected_result_path = remove_r1_from_path(result_file_path)

if os.path.exists(corrected_input_path) and os.path.exists(corrected_result_path):
if os.path.exists(corrected_input_path) and os.path.exists(
corrected_result_path
):
new_vector.input_file = os.path.relpath(corrected_input_path, dest_dir)
new_vector.result = utils.file_checksum(corrected_result_path)

Expand All @@ -244,31 +272,43 @@ def remove_r1_from_path(path):
default=2 * multiprocessing.cpu_count(),
)
args = parser.parse_args()

generator = JVTGenerator(
"AVCv1",
"JVT-AVC_V1",
"JVT-AVC_v1",
Codec.H264,
"JVT AVC version 1",
H264_URL
"JVT Advanced Video Coding v1 test suite",
H264_URL,
True,
)
generator.generate(not args.skip_download, args.jobs)

generator = JVTGenerator(
"MVC",
"JVT-MVC",
Codec.H264,
"JVT Multiview Video Coding test suite",
H264_URL,
True,
)
generator.generate(not args.skip_download, args.jobs)

generator = JVTGenerator(
"SVC",
"JVT-SVC_V1",
"JVT-SVC",
Codec.H264,
"JVT SVC version 1",
"JVT Scalable Video Coding test suite",
H264_URL,
True
True,
)
generator.generate(not args.skip_download, args.jobs)

generator = JVTGenerator(
"Professional_profiles",
"JVT-Professional_profiles_V1",
"JVT-Professional_profiles",
Codec.H264,
"JVT professional profiles version 1",
"JVT Professional Profiles test suite",
H264_URL,
True
True,
)
generator.generate(not args.skip_download, args.jobs)
6 changes: 3 additions & 3 deletions test_suites/h264/JVT-AVC_V1.json → test_suites/h264/JVT-AVC_v1.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "JVT-AVC_V1",
"name": "JVT-AVC_v1",
"codec": "H.264",
"description": "JVT AVC version 1",
"description": "JVT Advanced Video Coding v1 test suite",
"test_vectors": [
{
"name": "AUD_MW_E",
Expand Down Expand Up @@ -1084,4 +1084,4 @@
"result": "b47e932d436288013b8453d9a1d0f60d"
}
]
}
}
167 changes: 167 additions & 0 deletions test_suites/h264/JVT-MVC.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
{
"name": "JVT-MVC",
"codec": "H.264",
"description": "JVT Multiview Video Coding test suite",
"test_vectors": [
{
"name": "MVCDS-4",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS-4.264",
"source_checksum": "ed8c2beae88bf2a52416b32683972871",
"input_file": "MVCDS-4.264",
"output_format": "yuv420p",
"result": "4a15b286d65dc66099398e5ac008c06f"
},
{
"name": "MVCDS-5",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS-5.264",
"source_checksum": "e4c19397f32be3c9d713b595763bffc0",
"input_file": "MVCDS-5.264",
"output_format": "yuv420p",
"result": "4a15b286d65dc66099398e5ac008c06f"
},
{
"name": "MVCDS-6",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS-6.264",
"source_checksum": "943bcbb29c20c24d33fe3445206f12f8",
"input_file": "MVCDS-6.264",
"output_format": "yuv420p",
"result": "4a15b286d65dc66099398e5ac008c06f"
},
{
"name": "MVCDS1",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS1.264",
"source_checksum": "387c5cdeaa86d1ae3ecf58a8fa74f795",
"input_file": "MVCDS1.264",
"output_format": "yuv420p",
"result": "4e93953a232fa6f4cd8b72339ce8a2f4"
},
{
"name": "MVCDS2",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS2.264",
"source_checksum": "94eee17fc3733a85372a06ad877e850b",
"input_file": "MVCDS2.264",
"output_format": "yuv420p",
"result": "3d279d41cf52d70ca9af8ee7e950b504"
},
{
"name": "MVCDS3",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCDS3.264",
"source_checksum": "8db1c1e4148a7c989598c2dd03e69a09",
"input_file": "MVCDS3.264",
"output_format": "yuv420p",
"result": "505cb8ad90da9a59e42d0a08569806e9"
},
{
"name": "MVCICT-1",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCICT-1.264",
"source_checksum": "0948882f217b35fcbc2f2cb82a29faf9",
"input_file": "MVCICT-1.264",
"output_format": "yuv420p",
"result": "cfb27cf69ad9147bff1b6b528552fdb4"
},
{
"name": "MVCICT-2",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCICT-2.264",
"source_checksum": "ef6b4c8a10743180eeff103f63e7165f",
"input_file": "MVCICT-2.264",
"output_format": "yuv420p",
"result": "140bb029c34692b67d15d4c5c957df7b"
},
{
"name": "MVCNV-2",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCNV-2.264",
"source_checksum": "fa7418ba47995d6971243933eb2143c2",
"input_file": "MVCNV-2.264",
"output_format": "yuv420p",
"result": "4a15b286d65dc66099398e5ac008c06f"
},
{
"name": "MVCNV-3",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCNV-3.264",
"source_checksum": "ef9aa6f176bb4706d4a6dbee5676b2e2",
"input_file": "MVCNV-3.264",
"output_format": "yuv420p",
"result": "4a15b286d65dc66099398e5ac008c06f"
},
{
"name": "MVCNV1",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCNV1.264",
"source_checksum": "883d85865bbd6df9bf4ae3d3dae039de",
"input_file": "MVCNV1.264",
"output_format": "yuv420p",
"result": "3d279d41cf52d70ca9af8ee7e950b504"
},
{
"name": "MVCNV4",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCNV4.264",
"source_checksum": "754cf17d50f76811142c826137f4280d",
"input_file": "MVCNV4.264",
"output_format": "yuv420p",
"result": "505cb8ad90da9a59e42d0a08569806e9"
},
{
"name": "MVCRP_1",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_1.264",
"source_checksum": "5f0b327fdd075a3d7e5f201ca86e291e",
"input_file": "MVCRP_1.264",
"output_format": "yuv420p",
"result": "1312ce1b578bdd66dbfaed67d3b13678"
},
{
"name": "MVCRP_2",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_2.264",
"source_checksum": "9f46037181092e5167efa27bcdbdfc34",
"input_file": "MVCRP_2.264",
"output_format": "yuv420p",
"result": "b89fdf72007d76e4611237ca113a9e47"
},
{
"name": "MVCRP_3",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_3.264",
"source_checksum": "019c9c799983e858a71faf4324ee3a5e",
"input_file": "MVCRP_3.264",
"output_format": "yuv420p",
"result": "b89fdf72007d76e4611237ca113a9e47"
},
{
"name": "MVCRP_4",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_4.264",
"source_checksum": "142e45e69f11787ffaef2df578177f44",
"input_file": "MVCRP_4.264",
"output_format": "yuv420p",
"result": "198afdb2a23f707b36be0c3e6e1dde96"
},
{
"name": "MVCRP_5",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_5.264",
"source_checksum": "142e45e69f11787ffaef2df578177f44",
"input_file": "MVCRP_5.264",
"output_format": "yuv420p",
"result": "198afdb2a23f707b36be0c3e6e1dde96"
},
{
"name": "MVCRP_6",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCRP_6.264",
"source_checksum": "019c9c799983e858a71faf4324ee3a5e",
"input_file": "MVCRP_6.264",
"output_format": "yuv420p",
"result": "b89fdf72007d76e4611237ca113a9e47"
},
{
"name": "MVCSPS-1",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCSPS-1.264",
"source_checksum": "85db58f59ab490b0e84c241b33a7da20",
"input_file": "MVCSPS-1.264",
"output_format": "yuv420p",
"result": "3db35ab5bd1b052b456f2e7ea7f3fb74"
},
{
"name": "MVCSPS-2",
"source": "https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/MVC/MVCSPS-2.264",
"source_checksum": "17c86dbe49306710455dead0bc8afad0",
"input_file": "MVCSPS-2.264",
"output_format": "yuv420p",
"result": "3db35ab5bd1b052b456f2e7ea7f3fb74"
}
]
}
Loading

0 comments on commit 2cc85c3

Please sign in to comment.