Chrobalt on-device test work for Android

.github/actions/on_device_tests/action.yaml

@@ -1,9 +1,94 @@
-name: On Host Tests
-description: Runs on-host tests.
+name: On Device Test
+description: Runs on-device tests.
+inputs:
+  results_dir:
+    description: "Path to directory where test results are saved."
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Run On-Device Tests
+    - name: Install Requirements
+      # TODO (b/388329764) - set up requirements file.
+      run: |
+        pip3 install grpcio==1.38.0 grpcio-tools==1.38.0
+      shell: bash
+    - name: Generate gRPC files
+      run: |
+        python -m grpc_tools.protoc -Itools/ --python_out=cobalt/tools/ --grpc_python_out=cobalt/tools/ cobalt/tools/on_device_tests_gateway.proto
+      shell: bash
+    - name: Set Up Cloud SDK
+      uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1
+    - name: Set env vars
+      run: |
+        echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV
+        # Test results and logs
+        echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}" >> $GITHUB_ENV
+        # Dimension env
+        if [ "${{ matrix.dimension }}" != "null" ]; then
+          echo "DIMENSION=${{ matrix.dimension }}" >> $GITHUB_ENV
+        fi
+      shell: bash
+    - name: Run Tests on ${{ matrix.platform }} Platform
+      env:
+        GCS_PATH: /bigstore/${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }}
+        GITHUB_SHA: ${{ github.sha }}
+        GITHUB_TOKEN: ${{ github.token }}
+        GITHUB_EVENT_NAME: ${{ github.event_name }}
+        GITHUB_ACTOR: ${{ github.actor }}
+        GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        GITHUB_ACTOR_ID: ${{ github.actor_id }}
+        GITHUB_REPO: ${{ github.repository }}
+        GITHUB_PR_HEAD_USER_LOGIN: ${{ github.event.pull_request.head.user.login }}
+        GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }}
+        GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }}
+        GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }}
+        GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+        GITHUB_RUN_NUMBER: ${{ github.run_number }}
+        GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        GITHUB_WORKFLOW: ${{ github.workflow }} 
+      run: |
+        set -uxe
+        python3 -u cobalt/tools/on_device_tests_gateway_client.py \
+          --platform_json "${GITHUB_WORKSPACE}/src/.github/config/${{ matrix.platform}}.json" \
+          --filter_json_dir "${GITHUB_WORKSPACE}/src/cobalt/testing/${{ matrix.platform}}" \
+          --token ${GITHUB_TOKEN} \
+          --change_id ${GITHUB_PR_NUMBER:-postsubmit} \
+          --tag cobalt_github_${GITHUB_EVENT_NAME} \
+          --builder_name github_${{ matrix.platform }}_tests \
+          --build_number ${GITHUB_RUN_NUMBER} \
+          --builder_url ${GITHUB_RUN_URL} \
+          --label github \
+          --label ${GITHUB_EVENT_NAME} \
+          --label ${GITHUB_WORKFLOW} \
+          --label actor-${GITHUB_ACTOR} \
+          --label actor_id-${GITHUB_ACTOR_ID} \
+          --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \
+          --label sha-${GITHUB_SHA} \
+          --label repository-${GITHUB_REPO} \
+          --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \
+          --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL}
+          ${DIMENSION:+"--dimension" "$DIMENSION"} \
+          ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \
+          --archive_path "${GCS_PATH}" \
+          --gcs_result_path "${GCS_RESULTS_PATH}" \
+          trigger
       shell: bash
+    - name: Download ${{ matrix.platform }} Test Results
+      if: always()
+      env:
+        RESULTS_DIR: ${{ inputs.results_dir }}
       run: |
-        echo "Nothing yet"
+        set -uxe
+        mkdir -p "${GITHUB_WORKSPACE}/${RESULTS_DIR}"
+        cd "${GITHUB_WORKSPACE}/${RESULTS_DIR}"
+        gsutil cp "${GCS_RESULTS_PATH}/" .
+        echo "TEST_LOG=${GITHUB_WORKSPACE}/${RESULTS_DIR}/test_results.txt" >> $GITHUB_ENV
+      shell: bash
+    - name: Archive Test Logs
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: Test log
+        path: ${{ env.TEST_LOG }}/
+

.github/config/android-arm.json

@@ -21,6 +21,10 @@
         "sql_unittests",
         "url_unittests"
     ],
+    "test_dimensions": {
+        "gtest_device": "sabrina",
+        "gtest_lab": "maneki"
+    },
     "test_on_device": true,
     "includes": [
         {

.github/workflows/main.yaml

@@ -182,9 +182,9 @@ jobs:
           on_host: ${{ needs.initialize.outputs.test_on_host }}
           on_device: ${{ needs.initialize.outputs.test_on_device }}
 
-  test:
+  on-host-test:
     needs: [initialize, docker-build-image, build]
-    if: needs.initialize.outputs.test_on_host == 'true' || needs.initialize.outputs.test_on_device == 'true'
+    if: needs.initialize.outputs.test_on_host == 'true'
     permissions: {}
     # TODO(b/372303096): Should have dedicated runner?
     runs-on: [self-hosted, chrobalt-linux-runner]
@@ -224,3 +224,52 @@ jobs:
           test_results_key: ${{ env.TEST_RESULTS_KEY }}
           datadog_api_key: ${{ secrets.datadog_api_key }}
         continue-on-error: true
+
+  # Runs on-device integration and unit tests.
+  on-device-test:
+    needs: [initialize, build]
+    # Run ODT when on_device label is applied on PR.
+    # Also, run ODT on push and schedule if not explicitly disabled via repo vars.
+    if: |
+      needs.initialize.outputs.test_on_device == 'true' && ((
+        github.event_name == 'pull_request' &&
+        contains(github.event.pull_request.labels.*.name, 'on_device') ) || ((
+            inputs.nightly == 'true' || github.event_name == 'schedule') &&
+            vars.RUN_ODT_TESTS_ON_NIGHTLY != 'False') ||
+          ( github.event_name == 'push' && vars.RUN_ODT_TESTS_ON_POSTSUBMIT != 'False' ) )
+    runs-on: [self-hosted, odt-runner]
+    name: ${{ matrix.name }}_on_device
+    permissions: {}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.initialize.outputs.platforms) }}
+        config: [devel]
+        include: ${{ fromJson(needs.initialize.outputs.includes) }}
+    env:
+      TEST_RESULTS_DIR: ${{ matrix.name }}_test_results
+    steps:
+      - name: Checkout
+        uses: kaidokert/[email protected]
+        timeout-minutes: 30
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+      - name: Run On-Device Tests (${{ matrix.shard }})
+        id: on-device-tests
+        uses: ./.github/actions/on_device_tests
+        with:
+          results_dir: ${{ env.TEST_RESULTS_DIR }}
+      - name: Process Test Results
+        if: |
+          always() &&
+          (
+            steps.on-device-tests.outcome == 'success' ||
+            steps.on-device-tests.outcome == 'failure'
+          )
+        uses: ./src/.github/actions/process_test_results
+        with:
+          results_dir: ${{ env.TEST_RESULTS_DIR }}
+          datadog_api_key: ${{ secrets.DD_API_KEY }}
+          is_postsubmit: ${{ github.event_name == 'schedule' || github.event_name == 'push' }}
+        continue-on-error: true

cobalt/testing/android-arm/base_unittests_filter.json

@@ -0,0 +1,6 @@
+{
+  "failing_tests": [
+    "BreakIteratorTest.BreakCharacter",
+    "ValuesUtilTest.FilePath"
+  ]
+}

cobalt/tools/on_device_tests_gateway.proto

@@ -0,0 +1,71 @@
+// Copyright 2022 The Cobalt Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package on_device_tests_gateway;
+
+// Interface exported by the server.
+service on_device_tests_gateway {
+  // A dumb proxy RPC service that passes user defined command line options
+  // to the on-device tests gateway and streams back output in real time.
+  rpc exec_command (OnDeviceTestsCommand) returns (stream OnDeviceTestsResponse) {
+  }
+
+  rpc exec_watch_command (OnDeviceTestsWatchCommand) returns (stream OnDeviceTestsResponse) {
+  }
+}
+
+// Working directory and command line arguments to be passed to the gateway.
+message OnDeviceTestsCommand {
+  string workdir = 1;
+  string token = 2;
+  string platform = 3;
+  string archive_path = 4;
+  repeated string labels = 5;
+  string change_id = 6;
+  bool dry_run = 7;
+  repeated string dimension = 8;
+  string test_attempts = 9;
+  string retry_level = 10;
+  string start_timeout = 11;
+  string test_timeout = 12;
+  string gcs_result_path = 13;
+  repeated ApkTest apk_tests = 14;
+}
+
+// apk_test details
+message ApkTest {
+  string test_target = 1;
+  string apk_path = 2;
+  string device_model = 3;
+  string device_pool = 4;
+  string gtest_filters = 5;
+}
+
+// Working directory and command line arguments to be passed to the gateway.
+message OnDeviceTestsWatchCommand {
+  // Next ID: 6
+  string workdir = 1;
+  string token = 2;
+  string session_id = 3;
+  string change_id = 4;
+  bool dry_run = 5;
+}
+
+// Response from the on-device tests.
+message OnDeviceTestsResponse {
+  // Next ID: 2
+  string response = 1;
+}