diff --git a/fluent.runtime/CHANGELOG.rst b/fluent.runtime/CHANGELOG.rst index 4284b52a..a5f5bef5 100644 --- a/fluent.runtime/CHANGELOG.rst +++ b/fluent.runtime/CHANGELOG.rst @@ -8,6 +8,7 @@ fluent.runtime next ``fluent.runtime.FluentBundle.add_resource``. * Removed ``fluent.runtime.FluentBundle.add_messages``. * Replaced ``bundle.format()`` with ``bundle.format_pattern(bundle.get_message().value)``. +* Added ``fluent.runtime.FluentLocalization`` as main entrypoint for applications. fluent.runtime 0.2 (September 10, 2019) --------------------------------------- diff --git a/fluent.runtime/docs/conf.py b/fluent.runtime/docs/conf.py index e2ccce0d..9aa4b645 100644 --- a/fluent.runtime/docs/conf.py +++ b/fluent.runtime/docs/conf.py @@ -24,9 +24,9 @@ author = 'Luke Plant' # The short X.Y version -version = '0.1' +version = '0.3' # The full version, including alpha/beta/rc tags -release = '0.1' +release = '0.3' # -- General configuration --------------------------------------------------- diff --git a/fluent.runtime/docs/index.rst b/fluent.runtime/docs/index.rst index 98eaf1c1..0596c117 100644 --- a/fluent.runtime/docs/index.rst +++ b/fluent.runtime/docs/index.rst @@ -15,4 +15,5 @@ significant changes. installation usage + internals history diff --git a/fluent.runtime/docs/internals.rst b/fluent.runtime/docs/internals.rst new file mode 100644 index 00000000..e3dbed7a --- /dev/null +++ b/fluent.runtime/docs/internals.rst @@ -0,0 +1,139 @@ +Internals of fluent.runtime +=========================== + +The application-facing API for ``fluent.runtime`` is ``FluentLocalization``. +This is the binding providing basic functionality for using Fluent in a +project. ``FluentLocalization`` builds on top of ``FluentBundle`` on top of +``FluentResource``. + +``FluentLocalization`` handles + +* Basic binding as an application-level API +* Language fallback +* uses resource loaders like ``FluentResourceLoader`` to create ``FluentResource`` + +``FluentBundle`` handles + +* Internationalization with plurals, number formatting, and date formatting +* Aggregating multiple Fluent resources with message and term references +* Functions exposed to Select and Call Expressions + +``FluentResource`` handles parsing of Fluent syntax. + +Determining which language to use, and which languages to fall back to is +outside of the scope of the ``fluent.runtime`` package. A concrete +application stack might have functionality for that. Otherwise it needs to +be built, `Babel `_ has +`helpers `_ +for that. ``fluent.runtime`` uses Babel internally for the international +functionality. + + +These bindings benefit from being adapted to the stack. Say, +a Django project would configure the localization binding through +``django.conf.settings``, and load Fluent files from the installed apps. + +Subclassing FluentLocalization +------------------------------ + +In the :doc:`usage` documentation, we used ``DemoLocalization``, which we'll +use here to exemplify how to subclass ``FluentLocalization`` for the needs +of specific stacks. + +.. code-block:: python + + from fluent.runtime import FluentLocalization, FluentResource + class DemoLocalization(FluentLocalization): + def __init__(self, fluent_content, locale='en', functions=None): + # Call super() with one locale, no resources nor loader + super(DemoLocalization, self).__init__([locale], [], None, functions=functions) + self.resource = FluentResource(fluent_content) + +This set up the custom class, passing ``locale`` and ``functions`` to the +base implementation. What's left to do is to customize the resource loading. + +.. code-block:: python + + def _bundles(self): + bundle = self._create_bundle(self.locales) + bundle.add_resource(self.resource) + yield bundle + +That's all that we need for our demo purposes. + +Using FluentBundle +------------------ + +The actual interaction with Fluent content is implemented in ``FluentBundle``. +Optimizations between the parsed content in ``FluentResource`` and a +representation suitable for the resolving of Patterns is also handled inside +``FluentBundle``. + +.. code-block:: python + + >>> from fluent.runtime import FluentBundle, FluentResource + +You pass a list of locales to the constructor - the first being the +desired locale, with fallbacks after that: + +.. code-block:: python + + >>> bundle = FluentBundle(["en-US"]) + +The passed locales are used for internationalization purposes inside Fluent, +being plural forms, as well as formatting of values. The locales passed in +don't affect the loaded messages, handling multiple localizations and the +fallback from one to the other is done in the ``FluentLocalization`` class. + +You must then add messages. These would normally come from a ``.ftl`` +file stored on disk, here we will just add them directly: + +.. code-block:: python + + >>> resource = FluentResource(""" + ... welcome = Welcome to this great app! + ... greet-by-name = Hello, { $name }! + ... """) + >>> bundle.add_resource(resource) + +To generate translations, use the ``get_message`` method to retrieve +a message from the bundle. This returns an object with ``value`` and +``attributes`` properties. The ``value`` can be ``None`` or an abstract pattern. +``attributes`` is a dictionary mapping attribute names to abstract patterns. +If the the message ID is not found, a ``LookupError`` is raised. An abstract +pattern is an implementation-dependent representation of a Pattern in the +Fluent syntax. Then use the ``format_pattern`` method, passing the message value +or one of its attributes and an optional dictionary of substitution parameters. +You should only pass patterns to ``format_pattern`` that you got from that same +bundle. As per the Fluent philosophy, the implementation tries hard to recover +from any formatting errors and generate the most human readable representation +of the value. The ``format_pattern`` method thereforereturns a tuple containing +``(translated string, errors)``, as below. + +.. code-block:: python + + >>> welcome = bundle.get_message('welcome') + >>> translated, errs = bundle.format_pattern(welcome.value) + >>> translated + "Welcome to this great app!" + >>> errs + [] + + >>> greet = bundle.get_message('greet-by-name') + >>> translated, errs = bundle.format_pattern(greet.value, {'name': 'Jane'}) + >>> translated + 'Hello, \u2068Jane\u2069!' + + >>> translated, errs = bundle.format_pattern(greet.value, {}) + >>> translated + 'Hello, \u2068{$name}\u2069!' + >>> errs + [FluentReferenceError('Unknown external: name')] + +You will notice the extra characters ``\u2068`` and ``\u2069`` in the +output. These are Unicode bidi isolation characters that help to ensure +that the interpolated strings are handled correctly in the situation +where the text direction of the substitution might not match the text +direction of the localized text. These characters can be disabled if you +are sure that is not possible for your app by passing +``use_isolating=False`` to the ``FluentBundle`` constructor. diff --git a/fluent.runtime/docs/requirements.txt b/fluent.runtime/docs/requirements.txt new file mode 100644 index 00000000..4a5ffb0e --- /dev/null +++ b/fluent.runtime/docs/requirements.txt @@ -0,0 +1 @@ +fluent.pygments diff --git a/fluent.runtime/docs/usage.rst b/fluent.runtime/docs/usage.rst index 089b3a89..87419410 100644 --- a/fluent.runtime/docs/usage.rst +++ b/fluent.runtime/docs/usage.rst @@ -19,70 +19,66 @@ In order to use fluent.runtime, you will need to create FTL files. `Read the Fluent Syntax Guide `_ in order to learn more about the syntax. -Using FluentBundle ------------------- +Using FluentLocalization +------------------------ Once you have some FTL files, you can generate translations using the -``fluent.runtime`` package. You start with the ``FluentBundle`` class: +``fluent.runtime`` package. You start with the ``FluentLocalization`` class: .. code-block:: python - >>> from fluent.runtime import FluentBundle, FluentResource + >>> from fluent.runtime import FluentLocalization, FluentResourceLoader -You pass a list of locales to the constructor - the first being the -desired locale, with fallbacks after that: +The Fluent files of your application are loaded with a ``FluentResourceLoader``. .. code-block:: python - >>> bundle = FluentBundle(["en-US"]) + >>> loader = FluentResourceLoader("l10n/{locale}") -You must then add messages. These would normally come from a ``.ftl`` -file stored on disk, here we will just add them directly: +The main entrypoint for your application is a ``FluentLocalization``. +You pass a list of locales to the constructor - the first being the +desired locale, with fallbacks after that - as well as resource IDs and your +loader. .. code-block:: python - >>> resource = FluentResource(""" - ... welcome = Welcome to this great app! - ... greet-by-name = Hello, { $name }! - ... """) - >>> bundle.add_resource(resource) - -To generate translations, use the ``get_message`` method to retrieve -a message from the bundle. If the the message ID is not found, a -``LookupError`` is raised. Then use the ``format_pattern`` method, passing -the message value or one if its attributes and an optional dictionary of -substitution parameters. As per the Fluent philosophy, the implementation -tries hard to recover from any formatting errors and generate the most human -readable representation of the value. The ``format_pattern`` method therefore -returns a tuple containing ``(translated string, errors)``, as below. + >>> l10n = FluentLocalization(["de", "en-US"], ["main.ftl"], loader) + >>> val = l10n.format_value("my-first-string") + "Fluent can be easy" + +This assumes that you have a directory layout like so + +.. code-block:: + + l10n/ + de/ + main.ftl + en-US/ + main.ftl + +and ``l10n/de/main.ftl`` with: + +.. code-block:: fluent + + second-string = Eine Übersetzung + +as well as ``l10n/en-US/main.ftl`` with: + +.. code-block:: fluent + + my-first-string = Fluent can be easy + second-string = An original string + +As you can see, our first example returned the English string, as that's on +our fallback list. When retrieving an existing translation, you get the +translated results as expected: .. code-block:: python - >>> welcome = bundle.get_message('welcome') - >>> translated, errs = bundle.format_pattern(welcome.value) - >>> translated - "Welcome to this great app!" - >>> errs - [] - - >>> greet = bundle.get_message('greet-by-name') - >>> translated, errs = bundle.format_pattern(greet.value, {'name': 'Jane'}) - >>> translated - 'Hello, \u2068Jane\u2069!' - - >>> translated, errs = bundle.format_pattern(greet.value, {}) - >>> translated - 'Hello, \u2068{$name}\u2069!' - >>> errs - [FluentReferenceError('Unknown external: name')] - -You will notice the extra characters ``\u2068`` and ``\u2069`` in the -output. These are Unicode bidi isolation characters that help to ensure -that the interpolated strings are handled correctly in the situation -where the text direction of the substitution might not match the text -direction of the localized text. These characters can be disabled if you -are sure that is not possible for your app by passing -``use_isolating=False`` to the ``FluentBundle`` constructor. + >>> l10n.format_value("second-string") + "Eine Übersetzung" + + Python 2 ~~~~~~~~ @@ -97,6 +93,18 @@ module or the start of your repl session: from __future__ import unicode_literals +DemoLocalization +~~~~~~~~~~~~~~~~ + +To make the documentation easier to read, we're using a ``DemoLocalization``, +that just uses a single literal Fluent resource. Find out more about the +details in the :doc:`internals` section. + +.. code-block:: python + + >>> l10n = DemoLocalization("key = A localization") + >>> pl = DemoLocalization("key = A localization", locale="pl") + Numbers ~~~~~~~ @@ -105,11 +113,10 @@ When rendering translations, Fluent passes any numeric arguments (``int``, .. code-block:: python - >>> bundle.add_resource(FluentResource( + >>> l10n = DemoLocalization( ... "show-total-points = You have { $points } points." - ... )) - >>> total_points = bundle.get_message("show-total-points") - >>> val, errs = bundle.format_pattern(total_points.value, {'points': 1234567}) + ... ) + >>> val = l10n.format_value("show-total-points", {'points': 1234567}) >>> val 'You have 1,234,567 points.' @@ -121,15 +128,14 @@ by wrapping your numeric arguments with >>> from fluent.runtime.types import fluent_number >>> points = fluent_number(1234567, useGrouping=False) - >>> val, errs = bundle.format_pattern(total_points.value, {'points': points})[0] + >>> l10n.format_value("show-total-points", {'points': 1234567}) 'You have 1234567 points.' >>> amount = fluent_number(1234.56, style="currency", currency="USD") - >>> bundle.add_resource(FluentResource( + >>> l10n = DemoLocalization( ... "your-balance = Your balance is { $amount }" - ... )) - >>> balance = bundle.get_message("your-balance") - >>> bundle.format_pattern(balance.value, {'amount': amount})[0] + ... ) + >>> l10n.format_value(balance.value, {'amount': amount}) 'Your balance is $1,234.56' The options available are defined in the Fluent spec for @@ -137,7 +143,7 @@ The options available are defined in the Fluent spec for Some of these options can also be defined in the FTL files, as described in the Fluent spec, and the options will be merged. -Date and time +Date and Time ~~~~~~~~~~~~~ Python ``datetime.datetime`` and ``datetime.date`` objects are also @@ -146,9 +152,8 @@ passed through locale aware functions: .. code-block:: python >>> from datetime import date - >>> bundle.add_resource(FluentResource("today-is = Today is { $today }")) - >>> today_is = bundle.get_message("today-is") - >>> val, errs = bundle.format(today_is.value, {"today": date.today() }) + >>> l10n = DemoLocalization("today-is = Today is { $today }") + >>> val = bundle.format_value("today-is", {"today": date.today() }) >>> val 'Today is Jun 16, 2018' @@ -156,9 +161,9 @@ You can explicitly call the ``DATETIME`` builtin to specify options: .. code-block:: python - >>> bundle.add_resource(FluentResource( + >>> l10n = DemoLocalization( ... 'today-is = Today is { DATETIME($today, dateStyle: "short") }' - ... )) + ... ) See the `DATETIME docs `_. @@ -177,7 +182,7 @@ To specify options from Python code, use >>> from fluent.runtime.types import fluent_date >>> today = date.today() >>> short_today = fluent_date(today, dateStyle='short') - >>> val, errs = bundle.format_pattern(today_is, {"today": short_today }) + >>> val = l10n.format_value("today-is", {"today": short_today }) >>> val 'Today is 6/17/18' @@ -206,9 +211,8 @@ ways: >>> utcnow datetime.datetime(2018, 6, 17, 12, 15, 5, 677597) - >>> bundle.add_resource(FluentResource("now-is = Now is { $now }")) - >>> now_is = bundle.get_message("now-is") - >>> val, errs = bundle.format_pattern(now_is.value, + >>> l10n = DemoLocalization("now-is = Now is { $now }") + >>> val = bundle.format_pattern("now-is", ... {"now": fluent_date(utcnow, ... timeZone="Europe/Moscow", ... dateStyle="medium", @@ -220,7 +224,7 @@ Custom functions ~~~~~~~~~~~~~~~~ You can add functions to the ones available to FTL authors by passing a -``functions`` dictionary to the ``FluentBundle`` constructor: +``functions`` dictionary to the ``FluentLocalization`` constructor: .. code-block:: python @@ -231,18 +235,21 @@ You can add functions to the ones available to FTL authors by passing a ... 'Darwin': 'mac', ... 'Windows': 'windows'}.get(platform.system(), 'other') - >>> bundle = FluentBundle(['en-US'], functions={'OS': os_name}) - >>> bundle.add_resource(FluentResource(""" - ... welcome = { OS() -> - ... [linux] Welcome to Linux - ... [mac] Welcome to Mac - ... [windows] Welcome to Windows - ... *[other] Welcome - ... } - ... """)) - >>> print(bundle.format_pattern(bundle.get_message('welcome'))[0]) + >>> l10n = FluentLocalization(['en-US'], ['os.ftl'], loader, functions={'OS': os_name}) + >>> l10n.format_value('welcome') Welcome to Linux +That's with ``l10n/en-US/os.ftl`` as: + +.. code-block:: fluent + + welcome = { OS() -> + [linux] Welcome to Linux + [mac] Welcome to Mac + [windows] Welcome to Windows + *[other] Welcome + } + These functions can accept positional and keyword arguments (like the ``NUMBER`` and ``DATETIME`` builtins), and in this case must accept the following types of arguments: diff --git a/fluent.runtime/fluent/runtime/__init__.py b/fluent.runtime/fluent/runtime/__init__.py index ba20cad6..d32e709c 100644 --- a/fluent.runtime/fluent/runtime/__init__.py +++ b/fluent.runtime/fluent/runtime/__init__.py @@ -11,6 +11,16 @@ from .prepare import Compiler from .resolver import ResolverEnvironment, CurrentEnvironment from .utils import native_to_fluent +from .fallback import FluentLocalization, AbstractResourceLoader, FluentResourceLoader + + +__all__ = [ + 'FluentLocalization', + 'AbstractResourceLoader', + 'FluentResourceLoader', + 'FluentResource', + 'FluentBundle', +] def FluentResource(source): diff --git a/fluent.runtime/fluent/runtime/fallback.py b/fluent.runtime/fluent/runtime/fallback.py new file mode 100644 index 00000000..b21fdee4 --- /dev/null +++ b/fluent.runtime/fluent/runtime/fallback.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- + +import codecs +import os +import six + + +class FluentLocalization(object): + """ + Generic API for Fluent applications. + + This handles language fallback, bundle creation and string localization. + It uses the given resource loader to load and parse Fluent data. + """ + def __init__( + self, locales, resource_ids, resource_loader, + use_isolating=False, + bundle_class=None, functions=None, + ): + self.locales = locales + self.resource_ids = resource_ids + self.resource_loader = resource_loader + self.use_isolating = use_isolating + if bundle_class is None: + from fluent.runtime import FluentBundle + self.bundle_class = FluentBundle + else: + self.bundle_class = bundle_class + self.functions = functions + self._bundle_cache = [] + self._bundle_it = self._iterate_bundles() + + def format_value(self, msg_id, args=None): + for bundle in self._bundles(): + if not bundle.has_message(msg_id): + continue + msg = bundle.get_message(msg_id) + if not msg.value: + continue + val, errors = bundle.format_pattern(msg.value, args) + return val + return msg_id + + def _create_bundle(self, locales): + return self.bundle_class( + locales, functions=self.functions, use_isolating=self.use_isolating + ) + + def _bundles(self): + bundle_pointer = 0 + while True: + if bundle_pointer == len(self._bundle_cache): + try: + self._bundle_cache.append(next(self._bundle_it)) + except StopIteration: + return + yield self._bundle_cache[bundle_pointer] + bundle_pointer += 1 + + def _iterate_bundles(self): + for first_loc in range(0, len(self.locales)): + locs = self.locales[first_loc:] + for resources in self.resource_loader.resources(locs[0], self.resource_ids): + bundle = self._create_bundle(locs) + for resource in resources: + bundle.add_resource(resource) + yield bundle + + +class AbstractResourceLoader(object): + """ + Interface to implement for resource loaders. + """ + def resources(self, locale, resource_ids): + """ + Yield lists of FluentResource objects, corresponding to + each of the resource_ids. + If there are multiple locations, this may yield multiple lists. + If a resource isn't found in any location, yield a partial list, + but don't yield empty lists. + """ + raise NotImplementedError + + +class FluentResourceLoader(AbstractResourceLoader): + """ + Resource loader to read Fluent files from disk. + + Different locales are in different locations based on locale code. + The locale code should be encoded as `{locale}` in the roots, or in + the resource_ids. + This loader does not support loading resources for one bundle from + different roots. + """ + def __init__(self, roots): + """ + Create a resource loader. The roots may be a string for a single + location on disk, or a list of strings. + """ + self.roots = [roots] if isinstance(roots, six.text_type) else roots + from fluent.runtime import FluentResource + self.Resource = FluentResource + + def resources(self, locale, resource_ids): + for root in self.roots: + resources = [] + for resource_id in resource_ids: + path = self.localize_path(os.path.join(root, resource_id), locale) + if not os.path.isfile(path): + continue + content = codecs.open(path, 'r', 'utf-8').read() + resources.append(self.Resource(content)) + if resources: + yield resources + + def localize_path(self, path, locale): + return path.format(locale=locale) diff --git a/fluent.runtime/setup.py b/fluent.runtime/setup.py index 737a6789..1049c644 100755 --- a/fluent.runtime/setup.py +++ b/fluent.runtime/setup.py @@ -28,4 +28,7 @@ 'six', ], test_suite='tests', + tests_require=[ + 'mock', + ], ) diff --git a/fluent.runtime/tests/test_fallback.py b/fluent.runtime/tests/test_fallback.py new file mode 100644 index 00000000..af95bf2a --- /dev/null +++ b/fluent.runtime/tests/test_fallback.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + +import unittest +import mock +import os +import six + +from fluent.runtime import ( + FluentLocalization, + FluentResourceLoader, +) + + +ISFILE = os.path.isfile + + +class TestLocalization(unittest.TestCase): + def test_init(self): + l10n = FluentLocalization( + ['en'], ['file.ftl'], FluentResourceLoader('{locale}') + ) + self.assertTrue(callable(l10n.format_value)) + + @mock.patch('os.path.isfile') + @mock.patch('codecs.open') + def test_bundles(self, codecs_open, isfile): + data = { + 'de/one.ftl': 'one = in German', + 'de/two.ftl': 'two = in German', + 'fr/two.ftl': 'three = in French', + 'en/one.ftl': 'four = exists', + 'en/two.ftl': 'five = exists', + } + isfile.side_effect = lambda p: p in data or ISFILE(p) + codecs_open.side_effect = lambda p, _, __: six.StringIO(data[p]) + l10n = FluentLocalization( + ['de', 'fr', 'en'], + ['one.ftl', 'two.ftl'], + FluentResourceLoader('{locale}') + ) + bundles_gen = l10n._bundles() + bundle_de = next(bundles_gen) + self.assertEqual(bundle_de.locales[0], 'de') + self.assertTrue(bundle_de.has_message('one')) + self.assertTrue(bundle_de.has_message('two')) + bundle_fr = next(bundles_gen) + self.assertEqual(bundle_fr.locales[0], 'fr') + self.assertFalse(bundle_fr.has_message('one')) + self.assertTrue(bundle_fr.has_message('three')) + self.assertListEqual(list(l10n._bundles())[:2], [bundle_de, bundle_fr]) + bundle_en = next(bundles_gen) + self.assertEqual(bundle_en.locales[0], 'en') + self.assertEqual(l10n.format_value('one'), 'in German') + self.assertEqual(l10n.format_value('two'), 'in German') + self.assertEqual(l10n.format_value('three'), 'in French') + self.assertEqual(l10n.format_value('four'), 'exists') + self.assertEqual(l10n.format_value('five'), 'exists') + + +@mock.patch('os.path.isfile') +@mock.patch('codecs.open') +class TestResourceLoader(unittest.TestCase): + def test_all_exist(self, codecs_open, isfile): + data = { + 'en/one.ftl': 'one = exists', + 'en/two.ftl': 'two = exists', + } + isfile.side_effect = lambda p: p in data + codecs_open.side_effect = lambda p, _, __: six.StringIO(data[p]) + loader = FluentResourceLoader('{locale}') + resources_list = list(loader.resources('en', ['one.ftl', 'two.ftl'])) + self.assertEqual(len(resources_list), 1) + resources = resources_list[0] + self.assertEqual(len(resources), 2) + + def test_one_exists(self, codecs_open, isfile): + data = { + 'en/two.ftl': 'two = exists', + } + isfile.side_effect = lambda p: p in data + codecs_open.side_effect = lambda p, _, __: six.StringIO(data[p]) + loader = FluentResourceLoader('{locale}') + resources_list = list(loader.resources('en', ['one.ftl', 'two.ftl'])) + self.assertEqual(len(resources_list), 1) + resources = resources_list[0] + self.assertEqual(len(resources), 1) + + def test_none_exist(self, codecs_open, isfile): + data = {} + isfile.side_effect = lambda p: p in data + codecs_open.side_effect = lambda p, _, __: six.StringIO(data[p]) + loader = FluentResourceLoader('{locale}') + resources_list = list(loader.resources('en', ['one.ftl', 'two.ftl'])) + self.assertEqual(len(resources_list), 0) diff --git a/fluent.runtime/tox.ini b/fluent.runtime/tox.ini index 4d305fd7..bcf0e34f 100644 --- a/fluent.runtime/tox.ini +++ b/fluent.runtime/tox.ini @@ -10,6 +10,7 @@ deps = syntax0.17: fluent.syntax==0.17 attrs==19.1.0 babel==2.7.0 + mock==3.0.5 pytz==2019.2 six==1.12.0 commands = ./runtests.py @@ -25,4 +26,5 @@ deps = attrs babel pytz + mock six