From b2790afe0a27907d1c63ffad4ea08f49fb7a9179 Mon Sep 17 00:00:00 2001 From: Nigel Megitt Date: Mon, 2 Dec 2019 11:10:17 +0000 Subject: [PATCH] Tt1 to tt3/mspa 705 smpte (#43) * Validate bindings for EBU-TT-1 documents by dynamically setting superseding class on document creation * MSPA-702: Check existence of Styling and Layout elements for EBU-TT-1 * MSPA-702: EBU-TT-1 document must contain a tt:region element * MSPA-702: body/dur attribute is not allowed in EBU-TT-1 * MSPA-702: Check that smpte timeBase is acceptable in EBU-TT-1 * Validate bindings for EBU-TT-1 documents by dynamically setting superseding class on document creation * MSPA-702: EBU-TT-1 document must contain a tt:region element * tt3 to ttd conversion (#41) MSPA-728 ebu-tt-3 to ebu-tt-d conversion * rebased and fixed ebuttd test files * extracting ebutt1object base into generic ebuttdocumentbase class * Add EBU-TT 1 to EBU-TT 3 conversion Does not handle time conversion. Sets `ebuttp:sequenceIdentifier` to the value of `tt/head/metadata/documentMetadata/documentIdentifier` if present, otherwise uses "TestConverter". Resets the `conformsToStandard` to say it is EBU-TT-3 conformant. Sets the `timeBase` to `media` whether you like it or not, but doesn't do any other conversions. * Address review comments Also allows for a setting that specifies whether or not to use `ebuttm:documentIdentifier` element value in the input as the `ebuttp:sequenceIdentifier` attribute value in the output. Adds a test for this. * Unit test EBUTT1 to EBUTT3 conversion * Fix validation error messages for unexpected attributes so it doesn't say they are missing. * Fix cloning of unknown element, and conversion of metadata * Make EBUTT1Document instantiatable by including required attributes and elements in the constructor * Add unit test cases for programmatical construction with smpte (skipped) and media timebase * Add unit test cases for from-document construction with smpte (skipped) and media timebase * Timedelta converter Creates a fixed offset SMPTE timecode converter that can return a timedelta equivalent for any valid SMPTE timecode value equal to or after the provided reference start point. Invalid timecodes throw errors. The timedelta returned is the difference between the input timecode and the reference timecode. * Reimplement ebu/ebu-tt-live-toolkit#516 See https://github.com/ebu/ebu-tt-live-toolkit/pull/516 for further details. Enables an `ebuttm:documentStartOfProgramme` element to be included in a document without causing a crash. * Make a smpte timecode converter Make a FixedOffsetSMPTEtoTimedeltaConverter to convert SMPTE times based on the `ebuttm:documentStartOfProgramme` element if present, or assuming `00:00:00:00` otherwise. * pep8 tidying, where possible * test smpte time conversion * Unskip tests that now pass because they're implemented * add a test to check that using the `ebuttm:documentStartOfProgramme` as the basis for time conversion works. * pep8 tidying * Discard elements that end up with negative times Unit test for this also added. Tidy out some print statements that had made their way in there. * Add some documentation The docs in current state don't build properly because autodoc throws a "cannot import name `EBUTT1EBUTT3Converter`" exception. This is possibly a problem with circular `import` references, which should be resolved before we merge. This can be demonstrated by adding `from . import ebutt1_ebutt3` to `ebu_tt_live/bindings/converters/__init__.py` and running `python -m ebu_tt_live.bindings.converters` which shows a stack trace illustrating the issue. * Resolve circular import loop This was preventing documentation from building correctly. * Prune empty output elements * Add documentation and fix node code * Add EBUTT1 to EBUTT3 producer node config parameters * Add example config file for EBUTT1 to EBUTT3 conversion Also a manifest file for its input, using existing test XML file. * Add documentation * Correctly format code blocks RST uses double backticks, not like markdown's single ones! * Allow start of programme timecode to be manually overridden Sometimes we might know better than the document what start of programe timecode to use, so allow any value specified in the EBU-TT 1 document to be overridden. * Address @danielthepope's excellent review comments * Tidy .gitignore * Fix up documentation - backticks, mainly * Fix/improve code comments * Use snake case for method name instead of camelCase, to match PEP8 * Test that negative times raise errors * Fix dropped frame algorithm typo - it really should check frame values! Update tests to be more rigorous in checking that too, since they shouldn't have passed before. --- .gitignore | 4 +- docs/source/configurator.rst | 5 + docs/source/conversion_from_ebutt.rst | 23 ++- .../ebu_tt_live.bindings.converters.rst | 34 ++++ docs/source/ebu_tt_live.bindings.rst | 1 + docs/source/ebu_tt_live.documents.rst | 9 - docs/source/ebu_tt_live.node.rst | 8 + docs/source/scripts_and_their_functions.rst | 19 ++ ebu_tt_live/bindings/_ebuttdt.py | 33 ++-- .../bindings/converters/ebutt1_ebutt3.py | 186 ++++++++++++++++-- .../converters/timedelta_converter.py | 154 +++++++++++++++ .../test/test_smpte_time_converter.py | 156 +++++++++++++++ ebu_tt_live/config/node.py | 7 +- ebu_tt_live/documents/__init__.py | 4 - ebu_tt_live/documents/converters.py | 52 ++++- ebu_tt_live/documents/ebutt1.py | 1 - ebu_tt_live/documents/ebutt3.py | 41 ++-- ...ricsson1_smpte_with_start_of_programme.xml | 70 +++++++ ...e_with_start_of_programme_and_sub_zero.xml | 75 +++++++ .../documents/test/manifest_EbuTT1.txt | 1 + ebu_tt_live/documents/test/test_converters.py | 70 +++++-- ebu_tt_live/errors.py | 4 + .../config/ebutt1_ebutt3_producer.json | 22 +++ ebu_tt_live/node/ebutt1_ebutt3_producer.py | 54 ++++- ebu_tt_live/strings.py | 3 + ebu_tt_live/xsd/ebutt_datatypes.xsd | 4 + ebu_tt_live/xsd/ebutt_metadata.xsd | 2 +- .../timeBase_timeformat_constraints.feature | 26 +++ testing/bdd/templates/timeBase_timeformat.xml | 6 +- testing/bdd/test_ebutt1_conversion.py | 24 ++- testing/bdd/test_timeBase_timeformat.py | 4 + 31 files changed, 996 insertions(+), 106 deletions(-) create mode 100644 docs/source/ebu_tt_live.bindings.converters.rst create mode 100644 ebu_tt_live/bindings/converters/timedelta_converter.py create mode 100644 ebu_tt_live/bindings/test/test_smpte_time_converter.py create mode 100644 ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme.xml create mode 100644 ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml create mode 100644 ebu_tt_live/documents/test/manifest_EbuTT1.txt create mode 100644 ebu_tt_live/examples/config/ebutt1_ebutt3_producer.json diff --git a/.gitignore b/.gitignore index 2e8263a8b..11ab94a07 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,9 @@ ENV/ out gen +# VSCode project files +.vscode/ + # Oxygen XML project files *.xpr @@ -134,4 +137,3 @@ export/ # Mac OS X files .DS_Store - diff --git a/docs/source/configurator.rst b/docs/source/configurator.rst index c3d29aba2..8828b9344 100644 --- a/docs/source/configurator.rst +++ b/docs/source/configurator.rst @@ -103,6 +103,11 @@ Node type dependent options for [nodeN] : :: └─clock └─type : ["local" (default) | "auto" | "clock"] + type="ebutt1-ebutt3-producer" + ├─sequence_identifier : sequence identifier, default "SequenceFromEBUTT1" + ├─use_doc_id_as_sequence_id : whether to use the ebuttm:documentIdentifier element contents as the output sequence identifier if it is present, default False + └─smpte_start_of_programme : start of programme timecode override in case you know better than the document start of programme metadata, default None + type="ebuttd-encoder" ├─media_time_zero : ["current" (default) | clock time at media time zero TODO: check format] ├─default_namespace : ["false" (default) | "true"] diff --git a/docs/source/conversion_from_ebutt.rst b/docs/source/conversion_from_ebutt.rst index c650bfd5d..a7314a478 100644 --- a/docs/source/conversion_from_ebutt.rst +++ b/docs/source/conversion_from_ebutt.rst @@ -23,8 +23,25 @@ have a sequence number. EBU-TT part 3 documents must have both. In order to set the sequence identifier the converter can be configured with the desired value, or it can be set to extract the -document identifier from the `ebuttm:documentIdentifier` element +document identifier from the ``ebuttm:documentIdentifier`` element and use it, if it exists. -TODO: how to convert smpte timebase time expressions into clock or media -timebase time expressions. \ No newline at end of file +If the EBU-TT part 1 document uses the ``smpte`` timebase, then all +the time expressions must be converted to some other timebase. +Currently they are all converted to ``media``, using a simple fixed +offset based conversion strategy, encapsulated in the utility class +:py:class:`ebu_tt_live.bindings.converters.timedelta_converter.FixedOffsetSMPTEtoTimedeltaConverter`. +This currently looks for the metadata attribute +``tt/head/metadata/ebuttm:documentMetadata/ebuttm:documentStartOfProgramme`` +and if it finds it, uses that as +the zero point, otherwise it uses ``00:00:00:00``. This can be +overridden by setting the ``smpte_start_of_programme`` parameter to the +start of programme timecode to use instead. + +The document's frame rate, frame rate multiplier and drop mode are taken into +account when doing the conversion. This means that illegal frame +values will cause a +:py:class:`ebu_tt_live.errors.TimeFormatError` exception to be raised. + +Elements with ``begin`` or ``end`` attributes that fall before the start of +programme are discarded. diff --git a/docs/source/ebu_tt_live.bindings.converters.rst b/docs/source/ebu_tt_live.bindings.converters.rst new file mode 100644 index 000000000..cae1bdac5 --- /dev/null +++ b/docs/source/ebu_tt_live.bindings.converters.rst @@ -0,0 +1,34 @@ +converters package +================== + +:mod:`converters` Package +------------------------- + +.. currentmodule:: ebu_tt_live.bindings.converters + +:mod:`ebutt3_ebuttd` Module +--------------------------- + +.. automodule:: ebu_tt_live.bindings.converters.ebutt3_ebuttd + :members: + :private-members: + :undoc-members: + :show-inheritance: + +:mod:`ebutt1_ebutt3` Module +--------------------------- + +.. automodule:: ebu_tt_live.bindings.converters.ebutt1_ebutt3 + :members: + :private-members: + :undoc-members: + :show-inheritance: + +:mod:`timedelta_converter` Module +--------------------------------- + +.. automodule:: ebu_tt_live.bindings.converters.timedelta_converter + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/ebu_tt_live.bindings.rst b/docs/source/ebu_tt_live.bindings.rst index f8c34ebe2..705fc3c8c 100644 --- a/docs/source/ebu_tt_live.bindings.rst +++ b/docs/source/ebu_tt_live.bindings.rst @@ -79,5 +79,6 @@ Subpackages .. toctree:: ebu_tt_live.bindings.raw + ebu_tt_live.bindings.converters ebu_tt_live.bindings.validation diff --git a/docs/source/ebu_tt_live.documents.rst b/docs/source/ebu_tt_live.documents.rst index 998fdd982..9619691ed 100644 --- a/docs/source/ebu_tt_live.documents.rst +++ b/docs/source/ebu_tt_live.documents.rst @@ -44,15 +44,6 @@ documents Package :undoc-members: :show-inheritance: -:mod:`bindings_converters` Module ---------------------------------- - -.. automodule:: ebu_tt_live.bindings.converters.ebutt3_ebuttd - :members: - :private-members: - :undoc-members: - :show-inheritance: - :mod:`document_converters` Module --------------------------------- diff --git a/docs/source/ebu_tt_live.node.rst b/docs/source/ebu_tt_live.node.rst index 2fac55c6d..261677637 100644 --- a/docs/source/ebu_tt_live.node.rst +++ b/docs/source/ebu_tt_live.node.rst @@ -41,6 +41,14 @@ node Package :undoc-members: :show-inheritance: +:mod:`ebutt1_ebutt3_producer` Module +------------------------------------ + +.. automodule:: ebu_tt_live.node.ebutt1_ebutt3_producer + :members: + :undoc-members: + :show-inheritance: + :mod:`encoder` Module --------------------- diff --git a/docs/source/scripts_and_their_functions.rst b/docs/source/scripts_and_their_functions.rst index e0ec41064..7288f6295 100644 --- a/docs/source/scripts_and_their_functions.rst +++ b/docs/source/scripts_and_their_functions.rst @@ -128,6 +128,25 @@ Retiming Delay Node is primarily intended for delaying explicitly timed documents. Use ``ebu-run`` to start this script, for example ``ebu-run --admin.conf=ebu_tt_live/examples/config/retiming_delay.json.`` +EBU-TT-1 Producer +----------------- +This script produces an EBU-TT Part 3 document from an EBU-TT Part 1 source. +If SMPTE timecode is used (``ttp:timeBase="smpte"``) then the script looks for +an ``ebuttm:documentStartOfProgramme`` element in the input document, and if +present, maps that to the zero media time, and discards any elements that +begin or end before that time. If that element is absent, then times are +converted assuming that media time zero is SMPTE timecode ``00:00:00:00``. +Alternatively both of those values can be overridden by specifying a +start of programme timecode to use with the ``smpte_start_of_programme`` +configuration parameter. +The timecode conversion currently assumes that +the timecode is continuous. + +The default output sequence identifier can be specified. There is also a +parameter to allow the value of the input ``ebuttm:documentIdentifier`` element +to be used as the output sequence identifier, if present, overriding the +specified default. + EBU-TT-D Encoder ---------------- This script is an extension of simple consumer and is responsible for diff --git a/ebu_tt_live/bindings/_ebuttdt.py b/ebu_tt_live/bindings/_ebuttdt.py index 0702f2eb0..000f61fd8 100644 --- a/ebu_tt_live/bindings/_ebuttdt.py +++ b/ebu_tt_live/bindings/_ebuttdt.py @@ -70,20 +70,25 @@ def _ConvertArguments_vx(cls, args, kw): # This means we are in XML parsing context. There should be a timeBase and a timing_attribute_name in the # context object. time_base = context['timeBase'] - timing_att_name = context['timing_attribute_name'] - if time_base not in cls._compatible_timebases[timing_att_name]: - log.debug(ERR_SEMANTIC_VALIDATION_TIMING_TYPE.format( - attr_name=timing_att_name, - attr_type=cls, - attr_value=args, - time_base=time_base - )) - raise pyxb.SimpleTypeValueError(ERR_SEMANTIC_VALIDATION_TIMING_TYPE.format( - attr_name=timing_att_name, - attr_type=cls, - attr_value=args, - time_base=time_base - )) + # It is possible for a timing type to exist as the value of an element not an attribute, + # in which case no timing_attribute_name is in the context; in that case don't attempt + # to validate the data against a timebase. At the moment this only affects the + # documentStartOfProgramme metadata element. + if 'timing_attribute_name' in context: + timing_att_name = context['timing_attribute_name'] + if time_base not in cls._compatible_timebases[timing_att_name]: + log.debug(ERR_SEMANTIC_VALIDATION_TIMING_TYPE.format( + attr_name=timing_att_name, + attr_type=cls, + attr_value=args, + time_base=time_base + )) + raise pyxb.SimpleTypeValueError(ERR_SEMANTIC_VALIDATION_TIMING_TYPE.format( + attr_name=timing_att_name, + attr_type=cls, + attr_value=args, + time_base=time_base + )) for item in args: if isinstance(item, timedelta): result.append(cls.from_timedelta(item)) diff --git a/ebu_tt_live/bindings/converters/ebutt1_ebutt3.py b/ebu_tt_live/bindings/converters/ebutt1_ebutt3.py index 6bb9a0482..c7ebdd2ea 100644 --- a/ebu_tt_live/bindings/converters/ebutt1_ebutt3.py +++ b/ebu_tt_live/bindings/converters/ebutt1_ebutt3.py @@ -1,11 +1,14 @@ from ebu_tt_live.bindings import tt, tt1_tt_type, tt1_body_type, \ body_type, div_type, tt1_head_type, tt1_layout_type, p_type, span_type, \ br_type, head_type, style_type, styling, layout, \ - region_type + region_type, load_types_for_document from ebu_tt_live.bindings._ebuttm import headMetadata_type, documentMetadata, \ metadataBase_type, divMetadata_type -from ebu_tt_live.documents import EBUTT3Document +from ebu_tt_live.bindings._ebuttdt import FullClockTimingType +from ebu_tt_live.errors import TimeNegativeError +from ebu_tt_live.strings import ERR_TIME_NEGATIVE from pyxb.binding.basis import NonElementContent, ElementContent +from datetime import timedelta import copy import logging @@ -13,15 +16,27 @@ class EBUTT1EBUTT3Converter(object): + """ + Class to convert EBU-TT-1 documents into EBU-TT-3 documents. + + Includes a setting to extract the document identifier from metadata + and use it as the output sequence identifier, or can just use a + provided sequence identifier. The sequence number is always 1. + """ _semantic_dataset = None _sequenceIdentifier = None _use_doc_id_as_seq_id = False def __init__(self, sequence_id, use_doc_id_as_seq_id=False): + """ + Construct a converter. + + :param sequence_id: + :param use_doc_id_as_seq_id: + """ self._sequenceIdentifier = sequence_id self._use_doc_id_as_seq_id = use_doc_id_as_seq_id - pass def map_type(self, in_element): if isinstance(in_element, tt1_tt_type): @@ -79,7 +94,7 @@ def convert_head(self, head_in, dataset): head_children = self.convert_children(head_in, dataset) for item in head_children: new_elem.append(item) - + return new_elem def convert_headMetadata(self, headMetadata_in, dataset): @@ -87,7 +102,8 @@ def convert_headMetadata(self, headMetadata_in, dataset): *self.convert_children(headMetadata_in, dataset) ) - # Special handling for conformsToStandard. Throw out the old, add a new. + # Special handling for conformsToStandard. Throw out the old, add a + # new. # TODO: When XSD updated to allow ebuttm document metadata directly in # head metadata, check for this by uncommenting the following lines: # if new_elem.conformsToStandard is not None: @@ -101,15 +117,18 @@ def convert_headMetadata(self, headMetadata_in, dataset): new_elem.documentMetadata.conformsToStandard = [ 'urn:ebu:tt:live:2017-05'] - # We want to remember the documentIdentifier and use it later for the + # We want to remember the documentIdentifier and use it later for the # sequence identifier # TODO: When XSD updated to allow ebuttm document metadata directly in # head metadata, check for this by uncommenting the following lines: # if new_elem.documentIdentifier is not None: # _rememberDocumentIdentifier(new_elem.documentIdentifier, dataset) - - if new_elem.documentMetadata and new_elem.documentMetadata.documentIdentifier is not None: - self._rememberDocumentIdentifier(new_elem.documentMetadata.documentIdentifier, dataset) + + if new_elem.documentMetadata and \ + new_elem.documentMetadata.documentIdentifier is not None: + self._rememberDocumentIdentifier( + new_elem.documentMetadata.documentIdentifier, + dataset) return new_elem @@ -183,22 +202,91 @@ def convert_region(self, region_in, dataset): ) return new_elem + def calculate_times(self, elem_in, dataset): + begin = elem_in.begin + end = elem_in.end + + if dataset['timeBase'] == 'smpte': + syncbase = dataset['syncbase'][-1] + + if begin is not None: + begin = dataset['smpte_to_timebase_converter'].timedelta(begin) + if begin < syncbase: + raise TimeNegativeError(ERR_TIME_NEGATIVE) + begin = begin - syncbase + if end is not None: + end = dataset['smpte_to_timebase_converter'].timedelta(end) + if end < syncbase: + raise TimeNegativeError(ERR_TIME_NEGATIVE) + end = end - syncbase + else: + if begin is not None: + begin = begin.timedelta + if end is not None: + end = end.timedelta + + return begin, end + + def push_syncbase(self, dataset, sync_delta): + if sync_delta is None: + dataset['syncbase'].append(dataset['syncbase'][-1]) + else: + dataset['syncbase'].append(dataset['syncbase'][-1] + sync_delta) + + def pop_syncbase(self, dataset): + dataset['syncbase'].pop() + def convert_body(self, body_in, dataset): if len(body_in.div) == 0: return None + + # Set up a syncbase list for use down the tree + dataset['syncbase'] = [timedelta(seconds=0)] + + try: + begin, end = self.calculate_times(body_in, dataset) + except TimeNegativeError: + return None + + self.push_syncbase(dataset, begin) + + if begin is not None: + begin = FullClockTimingType(begin) + if end is not None: + end = FullClockTimingType(end) + new_elem = body_type( *self.convert_children(body_in, dataset), agent=body_in.agent, role=body_in.role, style=body_in.style, - begin=body_in.begin, - end=body_in.end + begin=begin, + end=end ) + + self.pop_syncbase(dataset) + + if len(new_elem.orderedContent()) == 0: + new_elem = None + return new_elem def convert_div(self, div_in, dataset): if len(div_in.orderedContent()) == 0: return None + + try: + begin, end = self.calculate_times(div_in, dataset) + except TimeNegativeError: + return None + + self.push_syncbase(dataset, begin) + + if begin is not None: + begin = FullClockTimingType(begin) + if end is not None: + end = FullClockTimingType(end) + new_elem = div_type( *self.convert_children(div_in, dataset), id=div_in.id, @@ -206,12 +294,30 @@ def convert_div(self, div_in, dataset): style=div_in.style, lang=div_in.lang, agent=div_in.agent, - begin=div_in.begin, - end=div_in.end + begin=begin, + end=end ) + + self.pop_syncbase(dataset) + + if len(new_elem.orderedContent()) == 0: + new_elem = None + return new_elem def convert_p(self, p_in, dataset): + try: + begin, end = self.calculate_times(p_in, dataset) + except TimeNegativeError: + return None + + self.push_syncbase(dataset, begin) + + if begin is not None: + begin = FullClockTimingType(begin) + if end is not None: + end = FullClockTimingType(end) + new_elem = p_type( *self.convert_children(p_in, dataset), id=p_in.id, @@ -219,25 +325,49 @@ def convert_p(self, p_in, dataset): lang=p_in.lang, region=p_in.region, style=p_in.style, - begin=p_in.begin, - end=p_in.end, + begin=begin, + end=end, agent=p_in.agent, role=p_in.role ) + + self.pop_syncbase(dataset) + + if len(new_elem.orderedContent()) == 0: + new_elem = None + return new_elem def convert_span(self, span_in, dataset): + try: + begin, end = self.calculate_times(span_in, dataset) + except TimeNegativeError: + return None + + self.push_syncbase(dataset, begin) + + if begin is not None: + begin = FullClockTimingType(begin) + if end is not None: + end = FullClockTimingType(end) + new_elem = span_type( *self.convert_children(span_in, dataset), id=span_in.id, space=span_in.space, lang=span_in.lang, style=span_in.style, - begin=span_in.begin, - end=span_in.end, + begin=begin, + end=end, agent=span_in.agent, role=span_in.role ) + + self.pop_syncbase(dataset) + + if len(new_elem.orderedContent()) == 0: + new_elem = None + return new_elem def convert_br(self, br_in, dataset): @@ -249,7 +379,8 @@ def convert_unknown(self, element_in, dataset): def convert_children(self, element, dataset): """ - Recursive step + Recursive step to convert child elements. + :param element: :param dataset: :return: @@ -274,14 +405,29 @@ def convert_element(self, element, dataset): converter = self.map_type(element) return converter(element, dataset) - def convert_document(self, root_element, dataset=None): + def convert_document(self, root_element, dataset=None, + smpte_to_timedelta_converter=None): + """ + Convert the EBU-TT-1 Document to an EBU-TT-3 document. + + :param root_element: The binding class for the EBU-TT-1 document's root element. + :param dataset: An optional dataset for passing information into the conversion. + :param smpte_to_timedelta_converter ISMPTEtoTimedeltaConverter: for mapping SMPTE to media timebase. Required if SMPTE timecodes are present. + :return an EBU-TT-3 document's root tt element: + """ if dataset is None: self._semantic_dataset = {} else: self._semantic_dataset = dataset + self._semantic_dataset['smpte_to_timebase_converter'] = \ + smpte_to_timedelta_converter + # Make sure that any new elements we correct get the right bindings - EBUTT3Document.load_types_for_document() + # Ideally we'd use EBUTT3Document.load_types_for_document() but that + # causes a circular import loop, because EBUTT3Document imports + # parts of bindings that include this (not sure why) + load_types_for_document('ebutt3') converted_bindings = self.convert_element( root_element, self._semantic_dataset diff --git a/ebu_tt_live/bindings/converters/timedelta_converter.py b/ebu_tt_live/bindings/converters/timedelta_converter.py new file mode 100644 index 000000000..a9a577a75 --- /dev/null +++ b/ebu_tt_live/bindings/converters/timedelta_converter.py @@ -0,0 +1,154 @@ +from datetime import timedelta +import re +from math import floor +from ebu_tt_live.strings import ERR_TIME_NEGATIVE, \ + ERR_TIME_FRAMES_OUT_OF_RANGE, \ + ERR_TIME_FRAME_IS_DROPPED +from ebu_tt_live.errors import TimeNegativeError, TimeFormatError + + +class ISMPTEtoTimedeltaConverter(object): + + def __init__(self): + raise NotImplementedError() + + def timedelta(smpte_time): + raise NotImplementedError() + + def can_convert(smpte_time): + raise NotImplementedError() + + +class FixedOffsetSMPTEtoTimedeltaConverter(ISMPTEtoTimedeltaConverter): + """ + Converts SMPTE timecodes to timedeltas with a fixed offset. + + This converter utility class uses a strategy that assumes a fixed offset, + a reference SMPTE timecode value that is considered the zero point, and + a continuous set of SMPTE timecodes monotonically increasing (aside + from drop frames). It should not be used in cases where there may be + discontinuities in the timecode, since it will give incorrect results. + + The object + uses the ``frameRate``, ``frameRateMultiplier`` and ``dropMode`` to + calculate the equivalent timedelta output value for any + given input SMPTE timecode, and raises an exception if an attempt + is made to convert a timecode that is earlier than the zero point. + This can be avoided by calling :py:func:`can_convert()` to check first. + + Alternatively call :py:func:`timedelta()` directly in a ``try`` block + and catch the :py:class:`ebu_tt_live.errors.TimeNegativeError` instead, + which avoids essentially running the same code twice. + """ + + _smpteReferenceS = None + _frameRate = None + _effectiveFrameRate = None + _dropMode = None + + _frm_regex = re.compile('(?P\\d+)\\s(?P\\d+)') + _tc_regex = \ + re.compile('([0-9][0-9]):([0-5][0-9]):([0-5][0-9]):([0-9][0-9])') + + def __init__(self, smpteReference, frameRate, + frameRateMultiplier, dropMode): + self._frameRate = int(frameRate) + self._effectiveFrameRate = \ + self._calc_effective_frame_rate( + int(frameRate), frameRateMultiplier) + self._dropMode = dropMode + self._smpteReferenceS = self._calculate_s(smpteReference) + + def timedelta(self, smpte_time): + """ + Convert a timecode to a timedelta. + + :param smpte_time: The timecode value to convert + :return timedelta: The equivalent timedelta + :raises TimeNegativeError: if the timecode occurs before the reference zero point + :raises TimeFormatError: if the frames value is illegal + """ + s = self._calculate_s(smpte_time) + + if self._smpteReferenceS > s: + raise TimeNegativeError(ERR_TIME_NEGATIVE) + + return timedelta(seconds=s-self._smpteReferenceS) + + def can_convert(self, smpte_time): + """ + Check if a given timecode can successfully be converted to a timedelta. + + :param smpte_time: The test value + :return Boolean: True if the timecode can successfully be converted + :raises TimeFormatError: if the frames value is illegal + """ + s = self._calculate_s(smpte_time) + + return self._smpteReferenceS <= s + + @classmethod + def _calc_effective_frame_rate(cls, frameRate, frameRateMultiplier): + # See https://www.w3.org/TR/ttml1/#time-expression-semantics-smpte + # for the semantics of effective frame rate calculation + frm_numerator_s, frm_denominator_s = \ + cls._frm_regex.match(frameRateMultiplier).groups() + + return float(frameRate) * \ + float(frm_numerator_s) / \ + float(frm_denominator_s) + + def _dropped_frames(self, hours, minutes): + # See https://www.w3.org/TR/ttml1/#time-expression-semantics-smpte + # for the semantics of dropped frame calculation + dropped_frames = 0 + + if self._dropMode == 'dropNTSC': + dropped_frames = \ + (hours * 54 + minutes - floor(minutes/10)) * 2 + elif self._dropMode == 'dropPAL': + dropped_frames = \ + (hours * 27 + floor(minutes / 2) - floor(minutes / 20)) * 4 + + return dropped_frames + + def _counted_frames(self, hours, minutes, seconds, frames): + # See https://www.w3.org/TR/ttml1/#time-expression-semantics-smpte + # for the semantics of counted frame calculation + return (3600 * hours + 60 * minutes + seconds) * \ + self._frameRate + frames + + def _calculate_s(self, smpte_time): + # Thie method mplements + # https://www.w3.org/TR/ttml1/#time-expression-semantics-smpte + # which specifies the calculation of S + hours, minutes, seconds, frames = \ + [int(x) for x in self._tc_regex.match(smpte_time).groups()] + + if frames >= self._frameRate: + raise TimeFormatError(ERR_TIME_FRAMES_OUT_OF_RANGE) + + if self._is_dropped_frame(minutes, seconds, frames): + raise TimeFormatError(ERR_TIME_FRAME_IS_DROPPED) + + s = (self._counted_frames(hours, minutes, seconds, frames) - + self._dropped_frames(hours, minutes)) / \ + self._effectiveFrameRate + + return s + + def _is_dropped_frame(self, minutes, seconds, frames): + # This method implements + # https://www.w3.org/TR/ttml1/#parameter-attribute-dropMode + # which defines the rules for dropped frames. + is_dropped_frame = False + + if seconds == 0: # in NTSC and PAL frames are only dropped at 0s + if self._dropMode == 'dropNTSC' and \ + minutes not in [0, 10, 20, 30, 40, 50]: + is_dropped_frame = frames in [0, 1] + elif self._dropMode == 'dropPAL' and \ + minutes % 2 == 0 and minutes not in [0, 20, 40]: + is_dropped_frame = frames in [0, 1, 2, 3] + + return is_dropped_frame diff --git a/ebu_tt_live/bindings/test/test_smpte_time_converter.py b/ebu_tt_live/bindings/test/test_smpte_time_converter.py new file mode 100644 index 000000000..37751231c --- /dev/null +++ b/ebu_tt_live/bindings/test/test_smpte_time_converter.py @@ -0,0 +1,156 @@ + +from unittest import TestCase +from datetime import timedelta +from ebu_tt_live.bindings.converters.timedelta_converter import \ + FixedOffsetSMPTEtoTimedeltaConverter +from ebu_tt_live.errors import TimeFormatError, TimeNegativeError +from ebu_tt_live.strings import ERR_TIME_FRAMES_OUT_OF_RANGE, \ + ERR_TIME_FRAME_IS_DROPPED, ERR_TIME_NEGATIVE + + +class TestFixedOffsetSMPTEtoTimedeltaConverter(TestCase): + + def test_effective_framerate_rational(self): + efr = \ + FixedOffsetSMPTEtoTimedeltaConverter.\ + _calc_effective_frame_rate(frameRate=30, + frameRateMultiplier='998 1000') + + self.assertEqual(efr, 29.94) + assert efr == 29.94 + + def test_effective_framerate_irrational(self): + efr = \ + FixedOffsetSMPTEtoTimedeltaConverter.\ + _calc_effective_frame_rate(frameRate=30, + frameRateMultiplier='1000 1001') + + self.assertAlmostEqual(efr, 29.97002997) + + def test_dropped_frames_nondrop(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'nonDrop') + df = conv._dropped_frames(1, 1) + self.assertEqual(df, 0) + + def test_dropped_frames_ntsc(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'dropNTSC') + df = conv._dropped_frames(1, 1) + self.assertEqual(df, 110) + + def test_dropped_frames_pal(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'dropPAL') + df = conv._dropped_frames(1, 1) + self.assertEqual(df, 108) + + def test_counted_frames(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'nonDrop') + self.assertEqual(conv._counted_frames(3, 57, 12, 17), 426977) + + def test_convert_timecode_25_1_1(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '25', + '1 1', + 'nonDrop') + self.assertEqual(conv.timedelta('10:00:00:00'), timedelta(seconds=0)) + self.assertEqual(conv.timedelta('11:12:13:14'), + timedelta(seconds=4333.56)) + + def test_convert_timecode_30_1000_1001(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'dropNTSC') + self.assertEqual(conv.timedelta('10:00:00:00'), timedelta(seconds=0)) + self.assertAlmostEqual(conv.timedelta('11:12:13:14').total_seconds(), + 4333.4625, 4) + + def test_can_convert(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '25', + '1 1', + 'nonDrop') + self.assertTrue(conv.can_convert('10:00:00:24')) + self.assertFalse(conv.can_convert('09:59:59:24')) + + def test_raise_exception_if_too_many_frames(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '25', + '1 1', + 'nonDrop') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAMES_OUT_OF_RANGE): + conv.can_convert('10:00:00:25') + + def test_raise_exception_if_dropped_frame_NTSC(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'dropNTSC') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAME_IS_DROPPED): + conv.can_convert('10:01:00:00') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAME_IS_DROPPED): + conv.can_convert('10:01:00:01') + + # Check a not dropped frame value works fine too + conv.can_convert('10:01:00:02') + + pass + + def test_raise_exception_if_dropped_frame_PAL(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '30', + '1000 1001', + 'dropPAL') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAME_IS_DROPPED): + conv.can_convert('10:02:00:00') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAME_IS_DROPPED): + conv.can_convert('10:02:00:02') + with self.assertRaises(TimeFormatError, + msg=ERR_TIME_FRAME_IS_DROPPED): + conv.can_convert('10:02:00:03') + + # Check a not dropped frame value works fine too + conv.can_convert('10:02:00:04') + + pass + + def test_raise_exception_if_negative(self): + conv = \ + FixedOffsetSMPTEtoTimedeltaConverter('10:00:00:00', + '25', + '1 1', + 'nonDrop') + + # Should convert without raising an exception + self.assertEqual(conv.timedelta('10:00:00:24'), + timedelta(milliseconds=960)) + with self.assertRaises(TimeNegativeError, + msg=ERR_TIME_NEGATIVE): + conv.timedelta('09:59:59:24') + pass diff --git a/ebu_tt_live/config/node.py b/ebu_tt_live/config/node.py index e60b0fe8c..3e6cad244 100644 --- a/ebu_tt_live/config/node.py +++ b/ebu_tt_live/config/node.py @@ -264,11 +264,16 @@ class EBUTT1EBUTT3Producer(ProducerMixin, ConsumerMixin, NodeBase): required_config = Namespace() required_config.add_option('id', default='ebutt1-ebutt3-producer') required_config.add_option('sequence_identifier', default='SequenceFromEBUTT1') + required_config.add_option('use_doc_id_as_sequence_id', default=False) + required_config.add_option('smpte_start_of_programme', default=None) def _create_component(self, config): self.component = processing_node.EBUTT1EBUTT3ProducerNode( - node_id=self.config.id + node_id=self.config.id, + sequence_identifier=self.config.sequence_identifier, + use_document_identifier_as_sequence_identifier=self.config.use_doc_id_as_sequence_id, + smpte_start_of_programme=self.config.smpte_start_of_programme ) def __init__(self, config, local_config): diff --git a/ebu_tt_live/documents/__init__.py b/ebu_tt_live/documents/__init__.py index 7c59ce5ac..87fc7ee5f 100644 --- a/ebu_tt_live/documents/__init__.py +++ b/ebu_tt_live/documents/__init__.py @@ -3,7 +3,3 @@ from .ebutt3 import EBUTT3Document, EBUTT3DocumentSequence, EBUTTAuthorsGroupControlRequest, EBUTTLiveMessage from .ebuttd import EBUTTDDocument from .converters import ebutt3_to_ebuttd, EBUTT3EBUTTDConverter - -__all__ = [ - 'base', 'ebutt3', 'ebuttd', 'ebutt3_splicer', 'ebutt3_segmentation', 'converters' -] diff --git a/ebu_tt_live/documents/converters.py b/ebu_tt_live/documents/converters.py index c6413ff22..5b9961ea9 100644 --- a/ebu_tt_live/documents/converters.py +++ b/ebu_tt_live/documents/converters.py @@ -1,13 +1,12 @@ from ebu_tt_live.bindings.converters.ebutt3_ebuttd import EBUTT3EBUTTDConverter from ebu_tt_live.bindings.converters.ebutt1_ebutt3 import EBUTT1EBUTT3Converter +from ebu_tt_live.bindings.converters.timedelta_converter import \ + FixedOffsetSMPTEtoTimedeltaConverter from ebu_tt_live.documents.ebuttd import EBUTTDDocument from ebu_tt_live.documents.ebutt1 import EBUTT1Document from ebu_tt_live.documents.ebutt3 import EBUTT3Document -from subprocess import Popen, PIPE -import tempfile -import os import logging @@ -16,7 +15,8 @@ def ebutt3_to_ebuttd(ebutt3_in, media_clock): """ - This function takes an EBUTT3Document instance and returns the same document as an EBUTTDDocument instance. + This function takes an EBUTT3Document instance and returns the same + document as an EBUTTDDocument instance. :param ebutt3_in: :return: """ @@ -35,17 +35,49 @@ def ebutt3_to_ebuttd(ebutt3_in, media_clock): return ebuttd_document -def ebutt1_to_ebutt3(ebutt1_in, sequence_id, use_doc_id_as_seq_id): +def ebutt1_to_ebutt3(ebutt1_in, + sequence_id, + use_doc_id_as_seq_id, + smpte_start_of_programme=None): """ - This function takes an EBUTT1Document instance and returns the same document as an EBUTT3Document instance. - :param ebutt1_in: - :return: + Convert an EBUTT1Document instance to an EBUTT3Document instance. + + :param EBUTT1Document ebutt1_in: The EBU-TT Part 1 document to convert + :param string sequence_id: The default sequence identifier to use for the output + :param Boolean use_doc_id_as_seq_id: Whether to use the ebuttm:documentIdentifier element value as the output sequence identifier, if it exists + :param string smpte_start_of_programme: SMPTE timecode for the start of programme, if known - if present, will override the ebuttm:documentStartOfProgramme metadata. + :return EBUTT3Document: The converted EBU-TT Part 3 document """ - converter = EBUTT1EBUTT3Converter(sequence_id=sequence_id, + converter = EBUTT1EBUTT3Converter( + sequence_id=sequence_id, use_doc_id_as_seq_id=use_doc_id_as_seq_id) doc_xml = ebutt1_in.get_xml() ebutt1_doc = EBUTT1Document.create_from_xml(doc_xml) - ebutt3_bindings = converter.convert_document(ebutt1_doc.binding) + + smpte_converter = None + if ebutt1_doc.binding.timeBase == 'smpte': + start_of_programme = '00:00:00:00' + if smpte_start_of_programme is None: + head_metadata = ebutt1_doc.binding.head.metadata + if head_metadata: + doc_metadata = head_metadata.documentMetadata + if doc_metadata and doc_metadata.documentStartOfProgramme: + start_of_programme = doc_metadata.documentStartOfProgramme + else: + start_of_programme = smpte_start_of_programme + + smpte_converter = \ + FixedOffsetSMPTEtoTimedeltaConverter( + start_of_programme, + ebutt1_doc.binding.frameRate, + ebutt1_doc.binding.frameRateMultiplier, + ebutt1_doc.binding.dropMode + ) + + ebutt3_bindings = converter.convert_document( + ebutt1_doc.binding, + dataset=None, + smpte_to_timedelta_converter=smpte_converter) ebutt3_document = EBUTT3Document.create_from_raw_binding(ebutt3_bindings) ebutt3_document.validate() return ebutt3_document diff --git a/ebu_tt_live/documents/ebutt1.py b/ebu_tt_live/documents/ebutt1.py index 036e519a0..a9779fdec 100644 --- a/ebu_tt_live/documents/ebutt1.py +++ b/ebu_tt_live/documents/ebutt1.py @@ -5,7 +5,6 @@ from .base import TimeBase from pyxb import BIND - class EBUTT1Document(TimelineUtilMixin, SubtitleDocument, EBUTTDocumentBase): """ This class wraps the binding object representation of the XML and provides the features the applications in the diff --git a/ebu_tt_live/documents/ebutt3.py b/ebu_tt_live/documents/ebutt3.py index 2414c942a..8bb4c9c1e 100644 --- a/ebu_tt_live/documents/ebutt3.py +++ b/ebu_tt_live/documents/ebutt3.py @@ -1,24 +1,27 @@ import logging -from .base import SubtitleDocument, TimeBase, CloningDocumentSequence, EBUTTDocumentBase +from .base import SubtitleDocument, TimeBase, CloningDocumentSequence, \ + EBUTTDocumentBase from .ebutt3_segmentation import EBUTT3Segmenter from .ebutt3_splicer import EBUTT3Splicer -from ebu_tt_live import bindings -from ebu_tt_live.bindings import _ebuttm as metadata, TimingValidationMixin -from ebu_tt_live.bindings import _ebuttlm as ebuttlm +from ebu_tt_live.bindings import _ebuttm as metadata, _ebuttlm as ebuttlm, \ + tt, CreateFromDocument, load_types_for_document, p_type from ebu_tt_live.strings import ERR_DOCUMENT_SEQUENCE_MISMATCH, \ ERR_DOCUMENT_NOT_COMPATIBLE, ERR_DOCUMENT_NOT_PART_OF_SEQUENCE, \ - ERR_DOCUMENT_SEQUENCE_INCONSISTENCY, DOC_DISCARDED, DOC_TRIMMED, DOC_REQ_SEGMENT, DOC_SEQ_REQ_SEGMENT, \ - DOC_INSERTED, DOC_SEMANTIC_VALIDATION_SUCCESSFUL, ERR_SEQUENCE_FROM_DOCUMENT, \ + ERR_DOCUMENT_SEQUENCE_INCONSISTENCY, DOC_DISCARDED, DOC_TRIMMED, \ + DOC_REQ_SEGMENT, DOC_SEQ_REQ_SEGMENT, \ + DOC_INSERTED, DOC_SEMANTIC_VALIDATION_SUCCESSFUL, \ + ERR_SEQUENCE_FROM_DOCUMENT, \ ERR_DOCUMENT_SEQUENCENUMBER_COLLISION, ERR_AUTHORS_GROUP_MISMATCH -from ebu_tt_live.errors import IncompatibleSequenceError, DocumentDiscardedError, \ - SequenceOverridden, SequenceNumberCollisionError, UnexpectedAuthorsGroupError +from ebu_tt_live.errors import IncompatibleSequenceError, \ + DocumentDiscardedError, \ + SequenceOverridden, SequenceNumberCollisionError, \ + UnexpectedAuthorsGroupError from ebu_tt_live.clocks import get_clock_from_document from datetime import timedelta from pyxb import BIND from sortedcontainers import sortedset -from sortedcontainers import sortedlist -from ebu_tt_live.documents.time_utils import TimelineUtilMixin, TimingEventBegin, TimingEventEnd -import gc +from ebu_tt_live.documents.time_utils import TimelineUtilMixin, \ + TimingEventBegin, TimingEventEnd log = logging.getLogger(__name__) @@ -107,6 +110,7 @@ def create_from_raw_binding(cls, binding, availability_time=None, **kwargs): # so orderedContent is needed and indexing the first component. ) + # Register the class in the base class EBUTTDocumentBase.message_type_mapping[EBUTTAuthorsGroupControlRequest.message_type_id] = EBUTTAuthorsGroupControlRequest @@ -137,7 +141,7 @@ def __init__(self, time_base, sequence_number, sequence_identifier, lang, clock_ self.load_types_for_document() if not clock_mode and time_base is TimeBase.CLOCK: clock_mode = 'local' - self._ebutt3_content = bindings.tt( + self._ebutt3_content = tt( timeBase=time_base, clockMode=clock_mode, sequenceIdentifier=sequence_identifier, @@ -170,7 +174,7 @@ def create_from_raw_binding(cls, binding, availability_time=None): def create_from_xml(cls, xml, availability_time=None): cls.load_types_for_document() instance = cls.create_from_raw_binding( - binding=bindings.CreateFromDocument( + binding=CreateFromDocument( xml_text=xml ), availability_time=availability_time @@ -179,7 +183,7 @@ def create_from_xml(cls, xml, availability_time=None): @classmethod def load_types_for_document(cls): - bindings.load_types_for_document('ebutt3') + load_types_for_document('ebutt3') def _cmp_key(self): return self.sequence_number @@ -314,7 +318,7 @@ def content_to_string(self, begin=None, end=None): affected_elements = self.lookup_range_on_timeline(begin=begin, end=end) affected_paragraphs = [] for item in affected_elements: - if isinstance(item, bindings.p_type): + if isinstance(item, p_type): affected_paragraphs.append(item) if begin is not None and self.resolved_begin_time < begin: @@ -365,12 +369,15 @@ def validate(self): # This only changes if the body does not declare a begin time. # Same for end time. if self._ebutt3_content.body is not None: - self._computed_begin_time = self._ebutt3_content.body.computed_begin_time - self._computed_end_time = self._ebutt3_content.body.computed_end_time + self._computed_begin_time = \ + self._ebutt3_content.body.computed_begin_time + self._computed_end_time = \ + self._ebutt3_content.body.computed_end_time else: self._computed_begin_time = availability_time self._computed_end_time = availability_time + return result def add_div(self, div): body = self._ebutt3_content.body diff --git a/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme.xml b/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme.xml new file mode 100644 index 000000000..e57d8a751 --- /dev/null +++ b/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme.xml @@ -0,0 +1,70 @@ + + + + + + v1.0 + 3.12.0 + BBC + 300 + 4:3 + WSTTeletextSubtitles + tgv + tgv + 2016-09-06 + + 37 + 12:00:00:00 + UK + Ericsson and Redbee + prernaB + + + + + + + v1.0 + timeOfDay + TestSequence1#localhost_EbuTT3_TestSeq#647#20160906#12-11-53#EbuTT3#source + tgv#2016-09-5T23:00:00Z#TestSequence1 + TestSequence1 + + + + + Subito VX + + + + + + + + + + + + + + + + + + + + + + + + + Subtitle_Source_Facet is of type VOICE + + + This is a position and text color + + test. + + + + diff --git a/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml b/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml new file mode 100644 index 000000000..7132f71a6 --- /dev/null +++ b/ebu_tt_live/documents/test/converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml @@ -0,0 +1,75 @@ + + + + + + v1.0 + 3.12.0 + BBC + 300 + 4:3 + WSTTeletextSubtitles + tgv + tgv + 2016-09-06 + + 37 + 12:00:00:00 + UK + Ericsson and Redbee + prernaB + + + + + + + v1.0 + timeOfDay + TestSequence1#localhost_EbuTT3_TestSeq#647#20160906#12-11-53#EbuTT3#source + tgv#2016-09-5T23:00:00Z#TestSequence1 + TestSequence1 + + + + + Subito VX + + + + + + + + + + + + + + + + + + + + + + + + + Subtitle zero should be discarded + + + + + Subtitle_Source_Facet is of type VOICE + + + This is a position and text color + + test. + + + + diff --git a/ebu_tt_live/documents/test/manifest_EbuTT1.txt b/ebu_tt_live/documents/test/manifest_EbuTT1.txt new file mode 100644 index 000000000..a24f81ead --- /dev/null +++ b/ebu_tt_live/documents/test/manifest_EbuTT1.txt @@ -0,0 +1 @@ +00:00:00.0,converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml diff --git a/ebu_tt_live/documents/test/test_converters.py b/ebu_tt_live/documents/test/test_converters.py index 3e2eadc5b..0eb5b7297 100644 --- a/ebu_tt_live/documents/test/test_converters.py +++ b/ebu_tt_live/documents/test/test_converters.py @@ -1,4 +1,3 @@ - from unittest import TestCase from datetime import timedelta import os @@ -57,11 +56,12 @@ def test_ericsson_3(self): ebuttdt.LimitedClockTimingType('12:11:50.000').timedelta) document = EBUTT3Document.create_from_xml(xml_file) - cdoc = ebutt3_to_ebuttd(document, self._media_clock) + ebutt3_to_ebuttd(document, self._media_clock) + class TestEBUTT1ToEBUTT3Converter(TestCase): - + def setUp(self): self._seqId = 'testConverter' @@ -73,8 +73,6 @@ def _load_asset(self, file_name): def test_simple_smpte(self): - self.skipTest('SMPTE time conversion not yet supported') - div = div_type( p_type( span_type( @@ -117,7 +115,7 @@ def test_simple_smpte(self): use_doc_id_as_seq_id=True) def test_simple_media(self): - + div = div_type( p_type( span_type( @@ -141,7 +139,10 @@ def test_simple_media(self): style_type(id='s0') ), tt1_layout_type( - region_type(id='r0', origin='0% 0%', extent='100% 100%') + region_type( + id='r0', + origin='0% 0%', + extent='100% 100%') ) ) ) @@ -158,23 +159,66 @@ def test_simple_media(self): def test_ericsson_smpte(self): - self.skipTest('SMPTE time conversion not supported yet') - xml_file = self._load_asset('converter_ericsson1_smpte.xml') document = EBUTT1Document.create_from_xml(xml_file) - cdoc = ebutt1_to_ebutt3( + ebutt1_to_ebutt3( + document, + sequence_id=self._seqId, + use_doc_id_as_seq_id=True) + + def test_ericsson_smpte_with_start_of_programme(self): + + xml_file = self._load_asset( + 'converter_ericsson1_smpte_with_start_of_programme.xml') + + document = EBUTT1Document.create_from_xml(xml_file) + ebutt1_to_ebutt3( + document, + sequence_id=self._seqId, + use_doc_id_as_seq_id=True) + + def test_ericsson_smpte_with_start_of_programme_and_sub_zero(self): + + xml_file = self._load_asset( + 'converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml') + + document = EBUTT1Document.create_from_xml(xml_file) + ebutt1_to_ebutt3( document, sequence_id=self._seqId, use_doc_id_as_seq_id=True) + def test_ericsson_smpte_with_overridden_start_of_programme(self): + + xml_file = self._load_asset( + 'converter_ericsson1_smpte_with_start_of_programme.xml') + + document = EBUTT1Document.create_from_xml(xml_file) + ebutt1_to_ebutt3( + document, + sequence_id=self._seqId, + use_doc_id_as_seq_id=True, + smpte_start_of_programme='11:00:00:00') + + def test_ericsson_smpte_with_overridden_start_of_programme_and_sub_zero(self): + + xml_file = self._load_asset( + 'converter_ericsson1_smpte_with_start_of_programme_and_sub_zero.xml') + + document = EBUTT1Document.create_from_xml(xml_file) + ebutt1_to_ebutt3( + document, + sequence_id=self._seqId, + use_doc_id_as_seq_id=True, + smpte_start_of_programme='11:00:00:00') + def test_ericsson_media(self): xml_file = self._load_asset('converter_ericsson1_media.xml') document = EBUTT1Document.create_from_xml(xml_file) - cdoc = ebutt1_to_ebutt3( + ebutt1_to_ebutt3( document, - sequence_id=self._seqId, + sequence_id=self._seqId, use_doc_id_as_seq_id=True) - diff --git a/ebu_tt_live/errors.py b/ebu_tt_live/errors.py index c8968453e..44b169ad4 100644 --- a/ebu_tt_live/errors.py +++ b/ebu_tt_live/errors.py @@ -22,6 +22,10 @@ class TimeFormatOverflowError(Exception): pass +class TimeNegativeError(Exception): + pass + + class XMLParsingFailed(Exception): pass diff --git a/ebu_tt_live/examples/config/ebutt1_ebutt3_producer.json b/ebu_tt_live/examples/config/ebutt1_ebutt3_producer.json new file mode 100644 index 000000000..1a4b68af4 --- /dev/null +++ b/ebu_tt_live/examples/config/ebutt1_ebutt3_producer.json @@ -0,0 +1,22 @@ +{ + "nodes": { + "node1": { + "id": "producer1", + "type": "ebutt1-ebutt3-producer", + "sequence_identifier": "TestSequenceFromEBUTT1", + "use_doc_id_as_sequence_id": true, + "smpte_start_of_programme": "11:00:00:00", + "input": { + "carriage": { + "type": "filesystem", + "manifest_file": "ebu_tt_live/documents/test/manifest_EbuTT1.txt" + } + }, + "output": { + "carriage": { + "type": "filesystem" + } + } + } + } +} \ No newline at end of file diff --git a/ebu_tt_live/node/ebutt1_ebutt3_producer.py b/ebu_tt_live/node/ebutt1_ebutt3_producer.py index dfbfa7943..7347e4350 100644 --- a/ebu_tt_live/node/ebutt1_ebutt3_producer.py +++ b/ebu_tt_live/node/ebutt1_ebutt3_producer.py @@ -1,19 +1,42 @@ from ebu_tt_live.bindings.converters.ebutt1_ebutt3 import EBUTT1EBUTT3Converter from ebu_tt_live.documents.ebutt1 import EBUTT1Document from .base import AbstractCombinedNode -from ebu_tt_live.documents import EBUTT3Document +from ebu_tt_live.documents.ebutt3 import EBUTT3Document +from ebu_tt_live.bindings.converters.timedelta_converter \ + import FixedOffsetSMPTEtoTimedeltaConverter class EBUTT1EBUTT3ProducerNode(AbstractCombinedNode): + """ + Produce an EBU-TT-3 document from an EBU-TT-1 input document. + For conversion of SMPTE timecodes, where ``ttp:timeBase="smpte"``, + makes a + :py:class:`ebu_tt_live.bindings.converters.timedelta_converter.FixedOffsetSMPTEtoTimedeltaConverter` using ``00:00:00:00`` + as the start of programme zero point, or if available, uses the + ``ebuttm:documentStartOfProgramme`` metadata value as the zero point, + and discards everything before the value of that element. + The start of programme zero point can be overridden by specifying a + ``smpte_start_of_programme`` attribute in the constructor. + + The output sequence identifier can be specified. Alternatively, + if the ``use_document_identifier_as_sequence_identifier`` parameter + is ``True`` and the ``ebuttm:documentIdentifier`` element is present + then that element's value is + used as the output sequence identifier. + """ _ebutt3_converter = None _expects = EBUTT1Document _provides = EBUTT3Document + _smpte_start_of_programme = None - def __init__(self, node_id, consumer_carriage=None, + def __init__(self, + node_id, + consumer_carriage=None, producer_carriage=None, sequence_identifier=None, use_document_identifier_as_sequence_identifier=True, + smpte_start_of_programme=None, **kwargs): super(EBUTT1EBUTT3ProducerNode, self).__init__( node_id=node_id, @@ -24,11 +47,36 @@ def __init__(self, node_id, consumer_carriage=None, self._ebutt3_converter = EBUTT1EBUTT3Converter( sequence_id=sequence_identifier, use_doc_id_as_seq_id=use_document_identifier_as_sequence_identifier) + self._smpte_start_of_programme = smpte_start_of_programme def process_document(self, document, **kwargs): # Convert each receiver document into EBU-TT-3 if self.is_document(document): + smpte_converter = None + if document.binding.timeBase == 'smpte': + start_of_programme = '00:00:00:00' + if self._smpte_start_of_programme is None: + head_metadata = document.binding.head.metadata + if head_metadata: + doc_metadata = head_metadata.documentMetadata + if doc_metadata and doc_metadata.documentStartOfProgramme: + start_of_programme = \ + doc_metadata.documentStartOfProgramme + else: + start_of_programme = self._smpte_start_of_programme + + smpte_converter = \ + FixedOffsetSMPTEtoTimedeltaConverter( + smpteReference=start_of_programme, + frameRate=document.binding.frameRate, + frameRateMultiplier=document.binding. + frameRateMultiplier, + dropMode=document.binding.dropMode + ) converted_doc = EBUTT3Document.create_from_raw_binding( - self._ebutt3_converter.convert_document(document.binding) + self._ebutt3_converter.convert_document( + document.binding, + dataset=None, + smpte_to_timedelta_converter=smpte_converter) ) self.producer_carriage.emit_data(data=converted_doc, **kwargs) diff --git a/ebu_tt_live/strings.py b/ebu_tt_live/strings.py index ac95d34cc..4e3ae7c15 100644 --- a/ebu_tt_live/strings.py +++ b/ebu_tt_live/strings.py @@ -9,7 +9,10 @@ ERR_INCOMPATIBLE_DATA_CONVERSION = gettext('{provides} -> {expects} conversion not possible with converters: {data_adapters}') ERR_CONV_NO_INPUT = gettext('The converter has no input set') ERR_TIME_WRONG_FORMAT = gettext('Wrong time format. datetime.timedelta is expected') +ERR_TIME_NEGATIVE = gettext('Time value is negative') ERR_TIME_FORMAT_OVERFLOW = gettext('Time value is out of format range') +ERR_TIME_FRAMES_OUT_OF_RANGE = gettext('Time value has too many frames for the frame rate') +ERR_TIME_FRAME_IS_DROPPED = gettext('Time value has a frame value corresponding to a dropped frame') ERR_DOCUMENT_SEQUENCE_MISMATCH = gettext('sequenceIdentifier mismatch') ERR_AUTHORS_GROUP_MISMATCH = gettext('authorsGroupIdentifier in document: [{agid_doc}] does not match sequence: [{agid_seq}]') ERR_DOCUMENT_SEQUENCENUMBER_COLLISION = gettext('Sequence number: [{sequence_number}] already present in sequence: [{sequence_identifier}]') diff --git a/ebu_tt_live/xsd/ebutt_datatypes.xsd b/ebu_tt_live/xsd/ebutt_datatypes.xsd index 8cb5f8efb..343df1d17 100644 --- a/ebu_tt_live/xsd/ebutt_datatypes.xsd +++ b/ebu_tt_live/xsd/ebutt_datatypes.xsd @@ -150,6 +150,10 @@ Please note that the EBU-TT XML Schema is a helping document and NOT normative b + + + diff --git a/ebu_tt_live/xsd/ebutt_metadata.xsd b/ebu_tt_live/xsd/ebutt_metadata.xsd index 489c92408..b42a235d0 100644 --- a/ebu_tt_live/xsd/ebutt_metadata.xsd +++ b/ebu_tt_live/xsd/ebutt_metadata.xsd @@ -342,7 +342,7 @@ Please note that the EBU-TT XML Schema is a helping document and NOT normative b text row (MNC). - + The time code of the first frame of the recorded video signal which is intended for transmission. Note: When the referenced diff --git a/testing/bdd/features/validation/timeBase_timeformat_constraints.feature b/testing/bdd/features/validation/timeBase_timeformat_constraints.feature index 679e1a6b8..194e158f6 100644 --- a/testing/bdd/features/validation/timeBase_timeformat_constraints.feature +++ b/testing/bdd/features/validation/timeBase_timeformat_constraints.feature @@ -194,3 +194,29 @@ Feature: ttp:timeBase-related attribute constraints | timeBase_timeformat.xml | smpte | 11.11:11 | | | timeBase_timeformat.xml | smpte | 11.11 | | | timeBase_timeformat.xml | smpte | 11:11:11:111 | | + +Scenario: Times in documentStartOfProgramme do not cause processing or validation error + Given an xml file + When it has timeBase + And it has documentStartOfProgramme + Then document is valid + + Examples: + | xml_file | time_base | start_time | + | timeBase_timeformat.xml | media | 00:00:00.000 | + | timeBase_timeformat.xml | clock | 00:00:00.000 | + | timeBase_timeformat.xml | smpte | 10:00:00:00 | + +# Element based time validation is not yet implemented +@skip +Scenario: Times in documentStartOfProgramme do cause validation error + Given an xml file + When it has timeBase + And it has documentStartOfProgramme + Then document is invalid + + Examples: + | xml_file | time_base | start_time | + | timeBase_timeformat.xml | media | 00:00:00:00 | + | timeBase_timeformat.xml | clock | 00:00:60 | + | timeBase_timeformat.xml | smpte | 10:00:00.00 | \ No newline at end of file diff --git a/testing/bdd/templates/timeBase_timeformat.xml b/testing/bdd/templates/timeBase_timeformat.xml index 2e5730213..edc54ba83 100644 --- a/testing/bdd/templates/timeBase_timeformat.xml +++ b/testing/bdd/templates/timeBase_timeformat.xml @@ -23,7 +23,11 @@ xmlns:xml="http://www.w3.org/XML/1998/namespace"> - + + {% if start_of_programme %} + {{ start_of_programme }} + {% endif %} + diff --git a/testing/bdd/test_ebutt1_conversion.py b/testing/bdd/test_ebutt1_conversion.py index b3f9b3cc9..a5bcea098 100644 --- a/testing/bdd/test_ebutt1_conversion.py +++ b/testing/bdd/test_ebutt1_conversion.py @@ -1,25 +1,26 @@ -import pytest from pytest_bdd import parsers, scenarios, then, when -from pyxb.exceptions_ import (IncompleteElementContentError, - UnrecognizedAttributeError) from ebu_tt_live.documents import EBUTT1Document, EBUTT3Document from ebu_tt_live.bindings.converters.ebutt1_ebutt3 import EBUTT1EBUTT3Converter scenarios('features/ebutt1/ebutt1_conversion.feature') + @when(parsers.parse('the document contains a "{element}" element')) def when_document_contains_element(template_dict, element): template_dict[element] = True + @when(parsers.parse('the document head metadata contains a documentIdentifier element')) def when_document_head_metadata_contains_documentIdentifier(template_dict): template_dict['head_metadata_documentIdentifier'] = True + @when(parsers.parse('the documentMetadata contains a documentIdentifier element')) def when_documentMetadata_contains_documentIdentifier(template_dict): template_dict['doc_metadata_documentIdentifier'] = True + @when('the XML is parsed as a valid EBU-TT-1 document') def when_document_parsed_ebutt1(test_context, template_file, template_dict): xml_text = template_file.render(template_dict) @@ -27,18 +28,22 @@ def when_document_parsed_ebutt1(test_context, template_file, template_dict): ebutt1_document.validate() test_context['ebutt1_document'] = ebutt1_document + @when('the EBU-TT-1 converter is set to use the documentIdentifier as a sequenceIdentifier') def when_converter_uses_docId_as_seqId(test_context): test_context['use_doc_id_as_seq_id'] = True + @when('the EBU-TT-1 converter is set not to use the documentIdentifier as a sequenceIdentifier') -def when_converter_uses_docId_as_seqId(test_context): +def when_converter_does_not_use_docId_as_seqId(test_context): test_context['use_doc_id_as_seq_id'] = False + @when(parsers.parse('the EBU-TT-1 converter sequenceIdentifier is "{seq_id}"')) def when_converter_seq_id(test_context, seq_id): test_context['converter_seq_id'] = seq_id + @when('the EBU-TT-1 document is converted to EBU-TT-3') def when_ebutt1_converted_to_ebutt3(test_context, template_file, template_dict): use_doc_id_as_seq_id = False @@ -47,20 +52,23 @@ def when_ebutt1_converted_to_ebutt3(test_context, template_file, template_dict): seq_id = 'TestConverter' if 'converter_seq_id' in test_context: seq_id = test_context['converter_seq_id'] - ebutt1_converter = EBUTT1EBUTT3Converter(sequence_id = seq_id, - use_doc_id_as_seq_id = use_doc_id_as_seq_id) + ebutt1_converter = EBUTT1EBUTT3Converter( + sequence_id=seq_id, + use_doc_id_as_seq_id=use_doc_id_as_seq_id) doc_xml = test_context["ebutt1_document"].get_xml() ebutt1_doc = EBUTT1Document.create_from_xml(doc_xml) converted_bindings = ebutt1_converter.convert_document(ebutt1_doc.binding) - ebutt3_document = EBUTT3Document.create_from_raw_binding(converted_bindings) + ebutt3_document = EBUTT3Document.create_from_raw_binding( + converted_bindings) test_context['ebutt3_document'] = ebutt3_document + @then('the EBU-TT-3 document is valid') def then_ebutt3_doc_valid(test_context): test_context['ebutt3_document'].validate() assert isinstance(test_context['ebutt3_document'], EBUTT3Document) + @then(parsers.parse('the sequenceIdentifier is "{value}"')) def then_sequence_identifier_is_value(test_context, value): assert test_context['ebutt3_document'].sequence_identifier == value - diff --git a/testing/bdd/test_timeBase_timeformat.py b/testing/bdd/test_timeBase_timeformat.py index d8f72bf3c..67877c225 100644 --- a/testing/bdd/test_timeBase_timeformat.py +++ b/testing/bdd/test_timeBase_timeformat.py @@ -51,3 +51,7 @@ def when_span_begin(span_begin, template_dict): @when('it has span end time ') def when_span_end(span_end, template_dict): template_dict['span_end'] = span_end + +@when('it has documentStartOfProgramme ') +def when_documentStartOfProgramme(start_time, template_dict): + template_dict['start_of_programme'] = start_time