diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index df6b4bf39d626..7bb7d04a6fb54 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -1,9 +1,94 @@ -name: On Host Tests -description: Runs on-host tests. +name: On Device Test +description: Runs on-device tests. +inputs: + results_dir: + description: "Path to directory where test results are saved." + required: true + runs: using: "composite" steps: - - name: Run On-Device Tests + - name: Install Requirements + # TODO (b/388329764) - set up requirements file. + run: | + pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 + shell: bash + - name: Generate gRPC files + run: | + python -m grpc_tools.protoc -Itools/ --python_out=cobalt/tools/ --grpc_python_out=cobalt/tools/ cobalt/tools/on_device_tests_gateway.proto + shell: bash + - name: Set Up Cloud SDK + uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 + - name: Set env vars + run: | + echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV + # Test results and logs + echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV + # Dimension env + if [ "${{ matrix.dimension }}" != "null" ]; then + echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV + fi + shell: bash + - name: Run Tests on ${{ matrix.platform }} Platform + env: + GCS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} + GITHUB_ACTOR_ID: ${{ github.actor_id }} + GITHUB_REPO: ${{ github.repository }} + GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }} + GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }} + GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} + GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} + run: | + set -uxe + python3 -u cobalt/tools/on_device_tests_gateway_client.py \ + --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \ + --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \ + --token ${GITHUB_TOKEN} \ + --change_id ${GITHUB_PR_NUMBER:-postsubmit} \ + --tag cobalt_github_${GITHUB_EVENT_NAME} \ + --builder_name github_${{ matrix.platform }}_tests \ + --build_number ${GITHUB_RUN_NUMBER} \ + --builder_url ${GITHUB_RUN_URL} \ + --label github \ + --label ${GITHUB_EVENT_NAME} \ + --label ${GITHUB_WORKFLOW} \ + --label actor-${GITHUB_ACTOR} \ + --label actor_id-${GITHUB_ACTOR_ID} \ + --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ + --label sha-${GITHUB_SHA} \ + --label repository-${GITHUB_REPO} \ + --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ + --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} + ${DIMENSION:+"--dimension" "$DIMENSION"} \ + ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ + --archive_path "${GCS_PATH}" \ + --gcs_result_path "${GCS_RESULTS_PATH}" \ + trigger shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() + env: + RESULTS_DIR: ${{ inputs.results_dir }} run: | - echo "Nothing yet" + set -uxe + mkdir -p "${GITHUB_WORKSPACE}/${RESULTS_DIR}" + cd "${GITHUB_WORKSPACE}/${RESULTS_DIR}" + gsutil cp "${GCS_RESULTS_PATH}/" . + echo "TEST_LOG=${GITHUB_WORKSPACE}/${RESULTS_DIR}/test_results.txt" >> $GITHUB_ENV + shell: bash + - name: Archive Test Logs + uses: actions/upload-artifact@v3 + if: always() + with: + name: Test log + path: ${{ env.TEST_LOG }}/ + diff --git a/.github/config/android-arm.json b/.github/config/android-arm.json index e8be887ccd2aa..6bfc85207a7e0 100644 --- a/.github/config/android-arm.json +++ b/.github/config/android-arm.json @@ -21,6 +21,10 @@ "sql_unittests", "url_unittests" ], + "test_dimensions": { + "gtest_device": "sabrina", + "gtest_lab": "maneki" + }, "test_on_device": true, "includes": [ { diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 15405744faa1a..35b68bf3ca5ca 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -182,9 +182,9 @@ jobs: on_host: ${{ needs.initialize.outputs.test_on_host }} on_device: ${{ needs.initialize.outputs.test_on_device }} - test: + on-host-test: needs: [initialize, docker-build-image, build] - if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true' + if: needs.initialize.outputs.test_on_host == 'true' permissions: {} # TODO(b/372303096): Should have dedicated runner? runs-on: [self-hosted, chrobalt-linux-runner] @@ -224,3 +224,52 @@ jobs: test_results_key: ${{ env.TEST_RESULTS_KEY }} datadog_api_key: ${{ secrets.datadog_api_key }} continue-on-error: true + + # Runs on-device integration and unit tests. + on-device-test: + needs: [initialize, build] + # Run ODT when on_device label is applied on PR. + # Also, run ODT on push and schedule if not explicitly disabled via repo vars. + if: | + needs.initialize.outputs.test_on_device == 'true' && (( + github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'on_device') ) || (( + inputs.nightly == 'true' || github.event_name == 'schedule') && + vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') || + ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) ) + runs-on: [self-hosted, odt-runner] + name: ${{ matrix.name }}_on_device + permissions: {} + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.initialize.outputs.platforms) }} + config: [devel] + include: ${{ fromJson(needs.initialize.outputs.includes) }} + env: + TEST_RESULTS_DIR: ${{ matrix.name }}_test_results + steps: + - name: Checkout + uses: kaidokert/checkout@v3.5.999 + timeout-minutes: 30 + with: + fetch-depth: 1 + persist-credentials: false + - name: Run On-Device Tests (${{ matrix.shard }}) + id: on-device-tests + uses: ./.github/actions/on_device_tests + with: + results_dir: ${{ env.TEST_RESULTS_DIR }} + - name: Process Test Results + if: | + always() && + ( + steps.on-device-tests.outcome == 'success' || + steps.on-device-tests.outcome == 'failure' + ) + uses: ./src/.github/actions/process_test_results + with: + results_dir: ${{ env.TEST_RESULTS_DIR }} + datadog_api_key: ${{ secrets.DD_API_KEY }} + is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }} + continue-on-error: true diff --git a/cobalt/testing/android-arm/base_unittests_filter.json b/cobalt/testing/android-arm/base_unittests_filter.json new file mode 100644 index 0000000000000..fb7886542530b --- /dev/null +++ b/cobalt/testing/android-arm/base_unittests_filter.json @@ -0,0 +1,6 @@ +{ + "failing_tests": [ + "BreakIteratorTest.BreakCharacter", + "ValuesUtilTest.FilePath" + ] +} diff --git a/cobalt/tools/on_device_tests_gateway.proto b/cobalt/tools/on_device_tests_gateway.proto new file mode 100644 index 0000000000000..933b5818faa4e --- /dev/null +++ b/cobalt/tools/on_device_tests_gateway.proto @@ -0,0 +1,71 @@ +// Copyright 2022 The Cobalt Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package on_device_tests_gateway; + +// Interface exported by the server. +service on_device_tests_gateway { + // A dumb proxy RPC service that passes user defined command line options + // to the on-device tests gateway and streams back output in real time. + rpc exec_command (OnDeviceTestsCommand) returns (stream OnDeviceTestsResponse) { + } + + rpc exec_watch_command (OnDeviceTestsWatchCommand) returns (stream OnDeviceTestsResponse) { + } +} + +// Working directory and command line arguments to be passed to the gateway. +message OnDeviceTestsCommand { + string workdir = 1; + string token = 2; + string platform = 3; + string archive_path = 4; + repeated string labels = 5; + string change_id = 6; + bool dry_run = 7; + repeated string dimension = 8; + string test_attempts = 9; + string retry_level = 10; + string start_timeout = 11; + string test_timeout = 12; + string gcs_result_path = 13; + repeated ApkTest apk_tests = 14; +} + +// apk_test details +message ApkTest { + string test_target = 1; + string apk_path = 2; + string device_model = 3; + string device_pool = 4; + string gtest_filters = 5; +} + +// Working directory and command line arguments to be passed to the gateway. +message OnDeviceTestsWatchCommand { + // Next ID: 6 + string workdir = 1; + string token = 2; + string session_id = 3; + string change_id = 4; + bool dry_run = 5; +} + +// Response from the on-device tests. +message OnDeviceTestsResponse { + // Next ID: 2 + string response = 1; +} diff --git a/cobalt/tools/on_device_tests_gateway_client.py b/cobalt/tools/on_device_tests_gateway_client.py new file mode 100644 index 0000000000000..2cfad00758fce --- /dev/null +++ b/cobalt/tools/on_device_tests_gateway_client.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 The Cobalt Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""gRPC On-device Tests Gateway client.""" + +import argparse +import logging +import sys +import os +import json + +import grpc + +import on_device_tests_gateway_pb2 +import on_device_tests_gateway_pb2_grpc + +_WORK_DIR = '/on_device_tests_gateway' + +# Comment out the next three lines for local testing +_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ( + 'on-device-tests-gateway-service.on-device-tests.svc.cluster.local') +_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '50052' + +# Uncomment the next two lines for local testing +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST = ('localhost') +#_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT = '12345' + + +class OnDeviceTestsGatewayClient(): + """On-device tests Gateway Client class.""" + + def __init__(self): + self.channel = grpc.insecure_channel( + target=f'{_ON_DEVICE_TESTS_GATEWAY_SERVICE_HOST}:{_ON_DEVICE_TESTS_GATEWAY_SERVICE_PORT}', # pylint:disable=line-too-long + # These options need to match server settings. + options=[('grpc.keepalive_time_ms', 10000), + ('grpc.keepalive_timeout_ms', 5000), + ('grpc.keepalive_permit_without_calls', 1), + ('grpc.http2.max_pings_without_data', 0), + ('grpc.http2.min_time_between_pings_ms', 10000), + ('grpc.http2.min_ping_interval_without_data_ms', 5000)]) + self.stub = on_device_tests_gateway_pb2_grpc.on_device_tests_gatewayStub( + self.channel) + + def run_trigger_command(self, workdir: str, args: argparse.Namespace, apk_tests = None): + """Calls On-Device Tests service and passing given parameters to it. + + Args: + workdir (str): Current script workdir. + args (Namespace): Arguments passed in command line. + """ + for response_line in self.stub.exec_command( + on_device_tests_gateway_pb2.OnDeviceTestsCommand( + workdir=workdir, + token=args.token, + platform=args.platform, + archive_path=args.archive_path, + labels=args.label, + gcs_result_path=args.gcs_result_path, + change_id=args.change_id, + dry_run=args.dry_run, + dimension=args.dimension or [], + test_attempts=args.test_attempts, + retry_level=args.retry_level, + apk_tests=apk_tests, + )): + + print(response_line.response) + + def run_watch_command(self, workdir: str, args: argparse.Namespace): + """Calls On-Device Tests watch service and passing given parameters to it. + + Args: + workdir (str): Current script workdir. + args (Namespace): Arguments passed in command line. + """ + for response_line in self.stub.exec_watch_command( + on_device_tests_gateway_pb2.OnDeviceTestsWatchCommand( + workdir=workdir, + token=args.token, + change_id=args.change_id, + session_id=args.session_id, + )): + + print(response_line.response) + +def _read_json_config(filename): + """ + Reads and parses data from a JSON configuration file. + + Args: + filename: The name of the JSON configuration file. + + Returns: + A list of dictionaries, where each dictionary represents a test configuration. + """ + try: + with open(filename, 'r') as f: + data = json.load(f) + return data + except FileNotFoundError: + print(f" Config file '{filename}' not found.") + return None + except json.JSONDecodeError: + print(f" Invalid JSON format in '{filename}'.") + return None + +def _get_platform_json_file(platform): + """Constructs the path to the platform JSON configuration file. + + Args: + platform: The name of the platform. + + Returns: + The absolute path to the platform JSON file. + """ + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + ".github", + "config", + f"{platform}.json" + ) + +def _get_tests_filter_json_file(platform, gtest_target): + """Constructs the path to the gtest filter JSON file. + + Args: + platform: The name of the platform. + gtest_target: The name of the gtest target. + + Returns: + The absolute path to the gtest filter JSON file. + """ + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "cobalt", + "testing", + platform, + f"{gtest_target}_filter.json" + ) + +def _process_apk_tests(args): + """Processes APK tests based on provided arguments. + + Args: + args: An argparse.Namespace object containing the following attributes: + platform: The platform for the tests. + platform_json: Optional path to a platform JSON file. + blaze_targets: A list of Blaze targets. + archive_path: Path to the archive containing APKs. + + Returns: + A list of dictionaries, where each dictionary represents an APK test + with keys like "test_target", "device_model", "device_pool", + "apk_path", and "gtest_filters". + """ + + apk_tests = [] + platform_json_file = args.platform_json or _default_platform_json_file(args.platform) + print(f"The platform_json_file is '{platform_json_file}'") + platform_data = _read_json_config(platform_json_file) + print(f"Loaded platform data: {platform_data}") # Added verbosity + + for target in args.blaze_targets: + print(f"Processing Blaze target: {target}") # Added verbosity + for gtest_target in platform_data["gtest_targets"]: + print(f" Processing gtest_target: {gtest_target}") # Added verbosity + apk_test = { + "test_target": target, + "device_model": platform_data["gtest_device"], + "device_pool": platform_data["gtest_lab"], + "apk_path": f"{args.archive_path}/{gtest_target}-debug.apk", + "gtest_filters": _get_gtest_filters(args.platform, gtest_target) + } + apk_tests.append(apk_test) + print(f" Created apk_test: {apk_test}") # Added verbosity + + print(f"apk_tests: {apk_tests}") + return apk_tests + + +def _get_gtest_filters(platform, gtest_target): + """Retrieves gtest filters for a given target. + + Args: + platform: The platform for the tests. + gtest_target: The name of the gtest target. + + Returns: + A string representing the gtest filters. + """ + + gtest_filters = "*" + filter_json_file = _get_tests_filter_json_file(platform, gtest_target) + print(f" gtest_filter_json_file = {filter_json_file}") + filter_data = _read_json_config(filter_json_file) + if filter_data: + print(f" Loaded filter data: {filter_data}") # Added verbosity + failing_tests = ":".join(filter_data.get("failing_tests", [])) + if failing_tests: + gtest_filters += ":-" + failing_tests + print(f" gtest_filters = {gtest_filters}") + else: + print(f" This gtest_target does not have gtest_filters specified") + return gtest_filters + + +def main(): + """Main routine for the on-device tests gateway client.""" + + logging.basicConfig( + level=logging.INFO, + format='[%(filename)s:%(lineno)s] %(message)s' + ) + print('Starting main routine') + + parser = argparse.ArgumentParser( + description="Client for interacting with the On-Device Tests gateway.", + epilog=( + 'Example: ./on_device_tests_gateway_client.py trigger ' + '--token token1 ' + '--platform android-arm ' + '--archive_path /bigstore/yt-temp ' + '--blaze_targets //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_maneki_sabrina //experimental/cobalt/chrobalt_poc:chrobalt_unit_tests_shared_boreal ' + '--dimension host_name=regex:maneki-mhserver-05.*' + ), + formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # Authentication + parser.add_argument( + '-t', + '--token', + type=str, + required=True, + help='On Device Tests authentication token' + ) + + # General options + parser.add_argument( + '--dry_run', + action='store_true', + help='Show what would be done without actually doing it.' + ) + parser.add_argument( + '-i', + '--change_id', + type=str, + help='ChangeId that triggered this test, if any. Saved with performance test results.' + ) + + subparsers = parser.add_subparsers( + dest='action', + help='On-Device tests commands', + required=True + ) + + # Trigger command + trigger_parser = subparsers.add_parser( + 'trigger', + help='Trigger On-Device tests', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + trigger_parser.add_argument( + '-p', + '--platform', + type=str, + required=True, + help='Platform this test was built for.' + ) + trigger_parser.add_argument( + '-pf', + '--platform_json', + type=str, + help='Platform-specific JSON file containing the list of target tests.' + ) + trigger_parser.add_argument( + '-a', + '--archive_path', + type=str, + required=True, + help='Path to Chrobalt archive to be tested. Must be on GCS.' + ) + trigger_parser.add_argument( + '-l', + '--label', + type=str, + action='append', + default=[], + help='Additional labels to assign to the test.' + ) + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS URL where test result files should be uploaded.' + ) + trigger_parser.add_argument( + '--dimension', + type=str, + action='append', + help=( + 'On-Device Tests dimension used to select a device. ' + 'Must have the following form: =. ' + 'E.G. "release_version=regex:10.*"' + ) + ) + trigger_parser.add_argument( + '--test_attempts', + type=str, + default='1', + help='The maximum number of times a test could retry.' + ) + trigger_parser.add_argument( + '--blaze_targets', + nargs="+", + type=str, + required=True, + help='A list of Blaze targets to run.' + ) + trigger_parser.add_argument( + '--retry_level', + type=str, + default='ERROR', + choices=['ERROR', 'FAIL'], + help=( + 'The retry level of Mobile Harness job. ' + 'ERROR to retry for MH errors, ' + 'FAIL to retry for failing tests (and MH errors).' + ) + ) + + # Watch command + watch_parser = subparsers.add_parser( + 'watch', + help='Watch a previously triggered On-Device test' + ) + watch_parser.add_argument( + 'session_id', + type=str, + help=( + 'Session ID of a previously triggered Mobile Harness test. ' + 'The test will be watched until it completes.' + ) + ) + + args = parser.parse_args() + apk_tests = _process_apk_tests(args) + + client = OnDeviceTestsGatewayClient() + try: + if args.action == 'trigger': + client.run_trigger_command(workdir=_WORK_DIR, args=args, apk_tests=apk_tests) + else: + client.run_watch_command(workdir=_WORK_DIR, args=args) + except grpc.RpcError as e: + print(e) + return e.code().value + + +if __name__ == '__main__': + sys.exit(main())