diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index d32e709c..613c91c2 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, unicode_literals +import six + import babel import babel.numbers import babel.plural @@ -9,7 +11,7 @@ from .builtins import BUILTINS from .prepare import Compiler -from .resolver import ResolverEnvironment, CurrentEnvironment +from .resolver import ResolverEnvironment, CurrentEnvironment, TextElement from .utils import native_to_fluent from .fallback import FluentLocalization, AbstractResourceLoader, FluentResourceLoader @@ -86,7 +88,10 @@ def _lookup(self, entry_id, term=False): self._compiled[compiled_id] = self._compiler(entry) return self._compiled[compiled_id] - def format_pattern(self, pattern, args=None): + def format_to_parts(self, pattern, errors, args=None): + if isinstance(pattern, TextElement): + yield pattern.value + return if args is not None: fluent_args = { argname: native_to_fluent(argvalue) @@ -95,17 +100,26 @@ def format_pattern(self, pattern, args=None): else: fluent_args = {} - errors = [] env = ResolverEnvironment(context=self, current=CurrentEnvironment(args=fluent_args), errors=errors) + for part in pattern(env): + yield part + + def format_pattern(self, pattern, args=None): + errors = [] try: - result = pattern(env) + result = ''.join(self.format_part(part) for part in self.format_to_parts(pattern, errors, args=args)) except ValueError as e: errors.append(e) result = '{???}' return [result, errors] + def format_part(self, fluentish): + if isinstance(fluentish, six.string_types): + return fluentish + return fluentish.format(self._babel_locale) + def _get_babel_locale(self): for l in self.locales: try: diff --git a/fluent.runtime/fluent/runtime/prepare.py b/fluent.runtime/fluent/runtime/prepare.py index c930236d..dce61f49 100644 --- a/fluent.runtime/fluent/runtime/prepare.py +++ b/fluent.runtime/fluent/runtime/prepare.py @@ -44,5 +44,5 @@ def compile_Pattern(self, _, elements, **kwargs): if len(elements) == 1: return elements[0] return resolver.TextElement( - ''.join(child(None) for child in elements) + ''.join(part for child in elements for part in child(None)) ) diff --git a/fluent.runtime/fluent/runtime/resolver.py b/fluent.runtime/fluent/runtime/resolver.py index ccafbb78..4b624dde 100644 --- a/fluent.runtime/fluent/runtime/resolver.py +++ b/fluent.runtime/fluent/runtime/resolver.py @@ -26,10 +26,6 @@ """ -# Prevent expansion of too long placeables, for memory DOS protection -MAX_PART_LENGTH = 2500 - - @attr.s class CurrentEnvironment(object): # The parts of ResolverEnvironment that we want to mutate (and restore) @@ -85,6 +81,19 @@ class BaseResolver(object): def __call__(self, env): raise NotImplementedError + def to_type(self, env): + """ + Resolve this to a FluentType. + + For simple resolvers of one part, this retains the type of that part. + For multi-part resolvers, format each part to a string and concat + the formatted parts. + """ + parts = list(self(env)) + if len(parts) == 1: + return parts[0] + return ''.join(env.context.format_part(part) for part in parts) + class Literal(BaseResolver): pass @@ -124,56 +133,43 @@ def __init__(self, *args, **kwargs): def __call__(self, env): if self in env.active_patterns: env.errors.append(FluentCyclicReferenceError("Cyclic reference")) - return FluentNone() + yield FluentNone() + return env.active_patterns.add(self) - elements = self.elements - remaining_parts = self.MAX_PARTS - env.part_count - if len(self.elements) > remaining_parts: - env.active_patterns.remove(self) - raise ValueError("Too many parts in message (> {0}), " - "aborting.".format(self.MAX_PARTS)) - retval = ''.join( - resolve(element(env), env) for element in elements - ) - env.part_count += len(elements) + for element in self.elements: + for part in element(env): + yield part + env.part_count += 1 + if env.part_count > self.MAX_PARTS: + raise ValueError("Too many parts in message (> {0}), " + "aborting.".format(self.MAX_PARTS)) env.active_patterns.remove(self) - return retval - - -def resolve(fluentish, env): - if isinstance(fluentish, FluentType): - return fluentish.format(env.context._babel_locale) - if isinstance(fluentish, six.string_types): - if len(fluentish) > MAX_PART_LENGTH: - raise ValueError( - "Too many characters in placeable " - "({}, max allowed is {})".format(len(fluentish), Pattern.MAX_PARTS) - ) - return fluentish class TextElement(FTL.TextElement, Literal): def __call__(self, env): - return self.value + yield self.value -class Placeable(FTL.Placeable, BaseResolver): +class NeverIsolatingPlaceable(FTL.Placeable, BaseResolver): def __call__(self, env): - inner = resolve(self.expression(env), env) - if not env.context.use_isolating: - return inner - return "\u2068" + inner + "\u2069" + for part in self.expression(env): + yield part -class NeverIsolatingPlaceable(FTL.Placeable, BaseResolver): +class Placeable(NeverIsolatingPlaceable): def __call__(self, env): - inner = resolve(self.expression(env), env) - return inner + if env.context.use_isolating: + yield "\u2068" + for part in self.expression(env): + yield part + if env.context.use_isolating: + yield "\u2069" class StringLiteral(FTL.StringLiteral, Literal): def __call__(self, env): - return self.parse()['value'] + yield self.parse()['value'] class NumberLiteral(FTL.NumberLiteral, BaseResolver): @@ -185,7 +181,7 @@ def __init__(self, value, **kwargs): self.value = FluentInt(self.value) def __call__(self, env): - return self.value + yield self.value class EntryReference(BaseResolver): @@ -196,11 +192,12 @@ def __call__(self, env): pattern = entry.attributes[self.attribute.name] else: pattern = entry.value - return pattern(env) + for part in pattern(env): + yield part except LookupError: ref_id = reference_to_id(self) env.errors.append(unknown_reference_error_obj(ref_id)) - return FluentNone('{{{}}}'.format(ref_id)) + yield FluentNone('{{{}}}'.format(ref_id)) class MessageReference(FTL.MessageReference, EntryReference): @@ -213,11 +210,12 @@ def __call__(self, env): if self.arguments.positional: env.errors.append(FluentFormatError("Ignored positional arguments passed to term '{0}'" .format(reference_to_id(self)))) - kwargs = {kwarg.name.name: kwarg.value(env) for kwarg in self.arguments.named} + kwargs = {kwarg.name.name: kwarg.value.to_type(env) for kwarg in self.arguments.named} else: kwargs = None with env.modified_for_term_reference(args=kwargs): - return super(TermReference, self).__call__(env) + for part in super(TermReference, self).__call__(env): + yield part class VariableReference(FTL.VariableReference, BaseResolver): @@ -229,13 +227,16 @@ def __call__(self, env): if env.current.error_for_missing_arg: env.errors.append( FluentReferenceError("Unknown external: {0}".format(name))) - return FluentNone(name) + yield FluentNone(name) + return if isinstance(arg_val, (FluentType, six.text_type)): - return arg_val - env.errors.append(TypeError("Unsupported external type: {0}, {1}" - .format(name, type(arg_val)))) - return FluentNone(name) + yield arg_val + else: + env.errors.append(TypeError( + "Unsupported external type: {0}, {1}".format(name, type(arg_val)) + )) + yield FluentNone(name) class Attribute(FTL.Attribute, BaseResolver): @@ -244,8 +245,9 @@ class Attribute(FTL.Attribute, BaseResolver): class SelectExpression(FTL.SelectExpression, BaseResolver): def __call__(self, env): - key = self.selector(env) - return self.select_from_select_expression(env, key=key) + key = self.selector.to_type(env) + for part in self.select_from_select_expression(env, key=key): + yield part def select_from_select_expression(self, env, key): default = None @@ -254,7 +256,7 @@ def select_from_select_expression(self, env, key): if variant.default: default = variant - if match(key, variant.key(env), env): + if match(key, variant.key, env): found = variant break @@ -283,12 +285,14 @@ def match(val1, val2, env): class Variant(FTL.Variant, BaseResolver): - pass + def __init__(self, key, value, default=False, **kwargs): + key = key.to_type(None) + super(Variant, self).__init__(key, value, default=default, **kwargs) class Identifier(FTL.Identifier, BaseResolver): def __call__(self, env): - return self.name + yield self.name class CallArguments(FTL.CallArguments, BaseResolver): @@ -297,21 +301,22 @@ class CallArguments(FTL.CallArguments, BaseResolver): class FunctionReference(FTL.FunctionReference, BaseResolver): def __call__(self, env): - args = [arg(env) for arg in self.arguments.positional] - kwargs = {kwarg.name.name: kwarg.value(env) for kwarg in self.arguments.named} + args = [arg.to_type(env) for arg in self.arguments.positional] + kwargs = {kwarg.name.name: kwarg.value.to_type(env) for kwarg in self.arguments.named} function_name = self.id.name try: function = env.context._functions[function_name] except LookupError: env.errors.append(FluentReferenceError("Unknown function: {0}" .format(function_name))) - return FluentNone(function_name + "()") + yield FluentNone(function_name + "()") + return try: - return function(*args, **kwargs) + yield function(*args, **kwargs) except Exception as e: env.errors.append(e) - return FluentNone(function_name + "()") + yield FluentNone(function_name + "()") class NamedArgument(FTL.NamedArgument, BaseResolver): diff --git a/fluent.runtime/tests/test_bomb.py b/fluent.runtime/tests/test_bomb.py index 8c00c740..0304c4b3 100644 --- a/fluent.runtime/tests/test_bomb.py +++ b/fluent.runtime/tests/test_bomb.py @@ -34,7 +34,7 @@ def test_max_length_protection(self): val, errs = self.ctx.format_pattern(self.ctx.get_message('lolz').value) self.assertEqual(val, '{???}') self.assertNotEqual(len(errs), 0) - self.assertIn('Too many characters', str(errs[-1])) + self.assertIn('Too many parts', str(errs[-1])) def test_max_expansions_protection(self): # Without protection, emptylolz will take a really long time to