L10n part4 and part 8 add all scripts needed (#2029)

* Part 4 - Get screenshots on CI

* changes to get part 4 and 8 implemented

* remove  reference

* remove Klar references for now

.cron.yml

@@ -1 +1,12 @@
-jobs: []
+# Definitions for jobs that run periodically.  For details on the format, see
+# `taskcluster/taskgraph/cron/schema.py`.  For documentation, see
+# `taskcluster/docs/cron.rst`.
+---
+
+jobs:
+    - name: l10-screenshots
+      job:
+          type: decision-task
+          target-tasks-method: l10n_screenshots
+          treeherder-symbol: l10-screenshots
+      when: []  # Manual trigger only

.taskcluster.yml

@@ -256,7 +256,7 @@ tasks:
                                             ~/.local/bin/taskgraph action-callback
                                         else: >
                                             PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
-                                            PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
+                                            PIP_IGNORE_INSTALLED=0 pip install --user chunkify &&
                                             ln -s /builds/worker/artifacts artifacts &&
                                             ~/.local/bin/taskgraph decision
                                             --pushlog-id='0'

bitrise.yml

@@ -367,13 +367,12 @@ workflows:
             set -e
             # debug log
             set -x
-            ./screenshots.sh --test-without-building $MOZ_LOCALES
+            ./l10n-screenshots.sh --test-without-building $MOZ_LOCALES
             mkdir -p artifacts
             for locale in $(echo $MOZ_LOCALES); do
-              for product in Focus Klar; do
-                zip -9 -j "$locale-$product.zip" "l10n-screenshots/$product/$locale/"*
-                mv "$locale-$product.zip" artifacts/
-              done
+              # Only Focus for now
+              zip -9 -j "$locale.zip" "l10n-screenshots/$locale/$locale/"*
+              mv "$locale.zip" artifacts/
             done
     - [email protected]:
         inputs:

screenshots.sh → l10n-screenshots.sh

@@ -33,18 +33,17 @@ fi
 DEVICE="iPhone 11"
 
 for lang in $LOCALES; do
-    for PRODUCT in Focus Klar; do
-        echo "Snapshotting $PRODUCT on $DEVICE"
-            DEVICEDIR="${DEVICE// /}"
-            mkdir -p "l10n-screenshots/$PRODUCT/$lang"
-            fastlane snapshot --project Blockzilla.xcodeproj --scheme "${PRODUCT}SnapshotTests" \
-              --derived_data_path l10n-screenshots-dd \
-              --skip_open_summary \
-              --erase_simulator --localize_simulator \
-              --devices "$DEVICE" \
-              --languages "$lang" \
-              --output_directory "l10n-screenshots/$PRODUCT/$lang" \
-              $EXTRA_FAST_LANE_ARGS
-        echo "Fastlane exited with code: $?"
-    done
+    # start simple with Focus only
+    echo "Snapshotting on $DEVICE"
+    mkdir -p "l10n-screenshots/$lang"
+    fastlane snapshot --project Blockzilla.xcodeproj --scheme "FocusSnapshotTests" \
+      --derived_data_path l10n-screenshots-dd \
+      --skip_open_summary \
+      --xcargs "-maximum-parallel-testing-workers 2" \
+      --erase_simulator --localize_simulator \
+      --devices "$DEVICE" \
+      --languages "$lang" \
+      --output_directory "l10n-screenshots/$lang" \
+      $EXTRA_FAST_LANE_ARGS
+    echo "Fastlane exited with code: $?"
 done

taskcluster/ci/docker-image/kind.yml

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    screenshots: {}

taskcluster/ci/generate-screenshots/kind.yml

@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: focusios_taskgraph.loader.screenshots_locales:loader
+
+transforms:
+    - focusios_taskgraph.transforms.bitrise:transforms
+    - focusios_taskgraph.transforms.secrets:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - build
+
+not-for-locales:
+    - en-US   # Already built as part of the build task
+
+locales-per-chunk: 6
+
+job-defaults:
+    bitrise-workflow: L10nScreenshotsTests
+    build-derived-data-path: {artifact-reference: '<build/public/build/target.zip>'}
+    description: Generate localized screenshots by delegating the work to bitrise.io
+    dependencies:
+        build: build-screenshots
+    index:
+        type: l10n-screenshots
+    run-on-tasks-for: []

taskcluster/focusios_taskgraph/loader/screenshots_locales.py

@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+
+from copy import deepcopy
+from chunkify import chunkify
+from math import log, ceil
+from taskgraph.loader.transform import loader as base_loader
+
+from ..screenshots_locales import get_screenshots_locales
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    not_for_locales = config.get("not-for-locales", [])
+    locales_per_chunk = config["locales-per-chunk"]
+
+    filtered_locales = [
+        locale for locale in get_screenshots_locales() if locale not in not_for_locales
+    ]
+    chunks, remainder = divmod(len(filtered_locales), locales_per_chunk)
+    if remainder:
+        # We need one last chunk to include locales in remainder
+        chunks = int(chunks + 1)
+
+    # Taskcluster sorts task names alphabetically, we need numbers to be zero-padded.
+    max_number_of_digits = _get_number_of_digits(chunks)
+
+    jobs = {
+        str(this_chunk).zfill(max_number_of_digits): {
+            "attributes": {
+                "chunk_locales": chunkify(filtered_locales, this_chunk, chunks),
+                "l10n_chunk": str(this_chunk),
+            }
+        }
+        # Chunks starts at 1 (and not 0)
+        for this_chunk in range(1, chunks + 1)
+    }
+
+    config["jobs"] = jobs
+
+    return base_loader(kind, path, config, params, loaded_tasks)
+
+
+def _get_number_of_digits(number):
+    # XXX We add 1 to number because `log(100)` returns 2 instead of 3
+    return int(ceil(log(number + 1, 10)))

taskcluster/focusios_taskgraph/routes.py

@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+
+from taskgraph.transforms.task import index_builder
+
+# Please ping the l10n team and contributors if these routes change.
+# In the future, notifying consumers may be easier (https://bugzilla.mozilla.org/show_bug.cgi?id=1548810), but
+# we need to remember to tell users for the time being
+SCREENSHOTS_ROUTE_TEMPLATES = [
+    "index.{trust-domain}.v2.{project}.{variant}.latest.{locale}",
+    "index.{trust-domain}.v2.{project}.{variant}.{build_date}.revision.{head_rev}.{locale}",
+    "index.{trust-domain}.v2.{project}.{variant}.{build_date}.latest.{locale}",
+    "index.{trust-domain}.v2.{project}.{variant}.revision.{head_rev}.{locale}",
+]
+
+
+@index_builder("l10n-screenshots")
+def add_signing_indexes(config, task):
+    if config.params["level"] != "3":
+        return task
+
+    subs = config.params.copy()
+    subs["build_date"] = time.strftime(
+        "%Y.%m.%d", time.gmtime(config.params["build_date"])
+    )
+    subs["trust-domain"] = config.graph_config["trust-domain"]
+    subs["variant"] = "l10n-screenshots"
+
+    routes = task.setdefault("routes", [])
+    for tpl in SCREENSHOTS_ROUTE_TEMPLATES:
+        for locale in task["attributes"]["chunk_locales"]:
+            subs["locale"] = locale
+            routes.append(tpl.format(**subs))
+
+    task["routes"] = _deduplicate_and_sort_sequence(routes)
+
+    return task
+
+
+def _deduplicate_and_sort_sequence(sequence):
+    return sorted(list(set(sequence)))

taskcluster/focusios_taskgraph/screenshots_locales.py

@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import yaml
+
+from taskgraph.util.memoize import memoize
+
+@memoize
+def get_screenshots_locales():
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    project_dir = os.path.realpath(os.path.join(current_dir, '..', '..'))
+
+    with open(os.path.join(project_dir, 'l10n-screenshots-config.yml')) as f:
+        config = yaml.safe_load(f)
+
+    return config["locales"]

taskcluster/focusios_taskgraph/target_tasks.py

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.target_tasks import _target_task, filter_for_tasks_for
+
+
+@_target_task('l10n_screenshots')
+def target_tasks_default(full_task_graph, parameters, graph_config):
+    """Target the tasks which have indicated they should be run on this project
+    via the `run_on_projects` attributes."""
+    def filter(task, parameters):
+        return task.kind == "generate-screenshots"
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]

taskcluster/focusios_taskgraph/transforms/bitrise.py

@@ -0,0 +1,102 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Resolve secrets and dummy secrets
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_run_config(config, tasks):
+    for task in tasks:
+        run = task.setdefault("run", {})
+        run.setdefault("using", "run-commands")
+        run.setdefault("use-caches", False)
+
+        run["secrets"] = {
+            "by-level": {
+                "3": [{
+                    "name": "project/mobile/focus-ios/bitrise",
+                    "key": "api_key",
+                    "path": ".bitrise_token",
+                }],
+                "default": [],
+            },
+        }
+
+        run["dummy-secrets"] = {
+            "by-level": {
+                "3": [],
+                "default": [{
+                    "content": "faketoken",
+                    "path": ".bitrise_token",
+                }],
+            },
+        }
+
+        yield task
+
+
+_ARTIFACTS_DIRECTORY = "/builds/worker/artifacts"
+
+
+@transforms.add
+def set_worker_config(config, tasks):
+    for task in tasks:
+        worker = task.setdefault("worker", {})
+        artifacts = worker.setdefault("artifacts", [])
+
+        artifacts.append({
+            "type": "file",
+            "name": "public/logs/bitrise.log",
+            "path": "{}/bitrise.log".format(_ARTIFACTS_DIRECTORY),
+        })
+
+        for locale in task["attributes"]["chunk_locales"]:
+            artifacts.append({
+                "type": "file",
+                "name": "public/screenshots/{}.zip".format(locale),
+                "path": "{}/{}.zip".format(_ARTIFACTS_DIRECTORY, locale),
+            })
+
+        worker.setdefault("docker-image", {"in-tree": "screenshots"})
+        worker.setdefault("max-run-time", 10800)
+
+        task.setdefault("worker-type", "bitrise")
+
+        yield task
+
+
+@transforms.add
+def add_bitrise_command(config, tasks):
+    for task in tasks:
+        commands = task["run"].setdefault("commands", [])
+        workflow = task.pop("bitrise-workflow")
+
+        command = [
+            "python3",
+            "taskcluster/scripts/bitrise-schedule.py",
+            "--token-file", ".bitrise_token",
+            "--branch", config.params["head_ref"],
+            "--commit", config.params["head_rev"],
+            "--workflow", workflow,
+            "--artifacts-directory", _ARTIFACTS_DIRECTORY
+        ]
+
+        for locale in task["attributes"]["chunk_locales"]:
+            command.extend(["--importLocales", locale])
+
+        derived_data_path = task.pop("build-derived-data-path", "")
+        if derived_data_path:
+            command.extend(["--derived-data-path", derived_data_path])
+
+        commands.append(command)
+
+        yield task

taskcluster/focusios_taskgraph/transforms/secrets.py

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Resolve secrets and dummy secrets
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.schema import resolve_keyed_by
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def resolve_keys(config, tasks):
+    for task in tasks:
+        for key in ("run.secrets", "run.dummy-secrets"):
+            resolve_keyed_by(
+                task,
+                key,
+                item_name=task["name"],
+                level=config.params["level"]
+            )
+        yield task