From 35fc53c21daf711d4151dc2ac10ac01dbcc720ca Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 10 Oct 2023 08:37:47 -0700 Subject: [PATCH 1/5] Change build workflow to run on production docker image Signed-off-by: Tyler Ohlsen --- .../workflows/test_build_multi_platform.yml | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index db0e1dfc0..dc8bf4c67 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -8,12 +8,35 @@ on: - "*" jobs: + Get-CI-Image-Tag: + runs-on: ubuntu-latest + outputs: + ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} + steps: + - name: Install crane + uses: iarekylew00t/crane-installer@v1 + with: + crane-release: v0.15.2 + - name: Checkout opensearch-build repository + uses: actions/checkout@v2 + with: + repository: "opensearch-project/opensearch-build" + ref: "main" + path: "opensearch-build" + - name: Get ci image version from opensearch-build repository scripts + id: step-ci-image-version-linux + run: | + crane version + CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p centos7 -u opensearch -t build | head -1` + echo $CI_IMAGE_VERSION + echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT Build-ad-windows: strategy: matrix: - java: [ 11, 17, 20 ] + java: [11, 17, 20] name: Build and Test Anomaly Detection Plugin on Windows runs-on: windows-latest + needs: Get-CI-Image-Tag env: JENKINS_URL: build.ci.opensearch.org steps: @@ -41,12 +64,12 @@ jobs: Build-ad: strategy: matrix: - java: [11,17,20] + java: [11, 17, 20] os: [ubuntu-latest, macos-latest] fail-fast: false - name: Build and Test Anomaly detection Plugin runs-on: ${{ matrix.os }} + needs: Get-CI-Image-Tag env: JENKINS_URL: build.ci.opensearch.org From c4b1e83223f7c128889cbe86db7017cf4a370392 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Tue, 10 Oct 2023 08:44:45 -0700 Subject: [PATCH 2/5] Fix setup-java Signed-off-by: Tyler Ohlsen --- .github/workflows/benchmark.yml | 3 ++- .github/workflows/test_build_multi_platform.yml | 2 ++ .github/workflows/test_bwc.yml | 3 ++- .github/workflows/test_security.yml | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5caeedd0..aa5a833d1 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,6 +20,7 @@ jobs: steps: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 + distribution: temurin with: java-version: ${{ matrix.java }} @@ -30,4 +31,4 @@ jobs: - name: Build and Run Tests run: | ./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true - ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true \ No newline at end of file + ./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index dc8bf4c67..b65a9a4e2 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -43,6 +43,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: temurin java-version: ${{ matrix.java }} - name: Checkout Anomaly Detection uses: actions/checkout@v4 @@ -77,6 +78,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: temurin java-version: ${{ matrix.java }} - name: Checkout AD diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml index 3c86f238a..8965c9fd3 100644 --- a/.github/workflows/test_bwc.yml +++ b/.github/workflows/test_bwc.yml @@ -11,7 +11,7 @@ jobs: Build-ad: strategy: matrix: - java: [11,17,20] + java: [11, 17, 20] fail-fast: false name: Test Anomaly detection BWC @@ -21,6 +21,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: temurin java-version: ${{ matrix.java }} # anomaly-detection diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 74229918d..515a56ec4 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -11,7 +11,7 @@ jobs: Build-ad: strategy: matrix: - java: [11,17,20] + java: [11, 17, 20] fail-fast: false name: Security test workflow for Anomaly Detection @@ -21,6 +21,7 @@ jobs: - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v3 with: + distribution: temurin java-version: ${{ matrix.java }} # time-series-analytics From d2fcfdbc72b7f71d76043d3c777fca53a12d4b55 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 18 Oct 2023 17:51:31 -0700 Subject: [PATCH 3/5] Update to latest strategy Signed-off-by: Tyler Ohlsen --- .../workflows/test_build_multi_platform.yml | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index b65a9a4e2..bba5d2999 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -9,34 +9,59 @@ on: jobs: Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + Build-ad-linux: + needs: Get-CI-Image-Tag + strategy: + matrix: + java: [11, 17, 20] + fail-fast: false + name: Build and Test Anomaly detection Plugin runs-on: ubuntu-latest - outputs: - ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }} + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: - - name: Install crane - uses: iarekylew00t/crane-installer@v1 - with: - crane-release: v0.15.2 - - name: Checkout opensearch-build repository - uses: actions/checkout@v2 + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 with: - repository: "opensearch-project/opensearch-build" - ref: "main" - path: "opensearch-build" - - name: Get ci image version from opensearch-build repository scripts - id: step-ci-image-version-linux + distribution: temurin + java-version: ${{ matrix.java }} + + - name: Checkout AD + uses: actions/checkout@v4 + + - name: Assemble anomaly-detection run: | - crane version - CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p centos7 -u opensearch -t build | head -1` - echo $CI_IMAGE_VERSION - echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT + ./gradlew assemble + - name: Build and Run Tests + run: | + ./gradlew build + - name: Publish to Maven Local + run: | + ./gradlew publishToMavenLocal + - name: Multi Nodes Integration Testing + run: | + ./gradlew integTest -PnumNodes=3 + - name: Upload Coverage Report + uses: codecov/codecov-action@v3 + with: + file: ./build/reports/jacoco/test/jacocoTestReport.xml + flags: plugin + Build-ad-windows: strategy: matrix: java: [11, 17, 20] name: Build and Test Anomaly Detection Plugin on Windows runs-on: windows-latest - needs: Get-CI-Image-Tag env: JENKINS_URL: build.ci.opensearch.org steps: @@ -62,15 +87,14 @@ jobs: with: file: ./build/reports/jacoco/test/jacocoTestReport.xml flags: plugin - Build-ad: + + Build-ad-macos: strategy: matrix: java: [11, 17, 20] - os: [ubuntu-latest, macos-latest] fail-fast: false name: Build and Test Anomaly detection Plugin - runs-on: ${{ matrix.os }} - needs: Get-CI-Image-Tag + runs-on: macos-latest env: JENKINS_URL: build.ci.opensearch.org From 90427c40d8c7494131f3d4cafa6d3fcaada6e220 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 18 Oct 2023 18:04:13 -0700 Subject: [PATCH 4/5] add container shell configs Signed-off-by: Tyler Ohlsen --- .github/workflows/test_build_multi_platform.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index bba5d2999..da0982186 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -75,13 +75,16 @@ jobs: - name: Build and Run Tests run: | - ./gradlew build + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew build - name: Publish to Maven Local run: | - ./gradlew publishToMavenLocal + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew publishToMavenLocal - name: Multi Nodes Integration Testing run: | - ./gradlew integTest -PnumNodes=3 + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -PnumNodes=3 - name: Upload Coverage Report uses: codecov/codecov-action@v3 with: From 0f487052f2f392aa7235320732947b5f40ed8307 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 18 Oct 2023 18:09:59 -0700 Subject: [PATCH 5/5] update linux only Signed-off-by: Tyler Ohlsen --- .../workflows/test_build_multi_platform.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml index da0982186..82cc8c779 100644 --- a/.github/workflows/test_build_multi_platform.yml +++ b/.github/workflows/test_build_multi_platform.yml @@ -40,16 +40,20 @@ jobs: - name: Assemble anomaly-detection run: | - ./gradlew assemble + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew assemble - name: Build and Run Tests run: | - ./gradlew build + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew build - name: Publish to Maven Local run: | - ./gradlew publishToMavenLocal + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew publishToMavenLocal - name: Multi Nodes Integration Testing run: | - ./gradlew integTest -PnumNodes=3 + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -PnumNodes=3 - name: Upload Coverage Report uses: codecov/codecov-action@v3 with: @@ -75,16 +79,13 @@ jobs: - name: Build and Run Tests run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew build + ./gradlew build - name: Publish to Maven Local run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew publishToMavenLocal + ./gradlew publishToMavenLocal - name: Multi Nodes Integration Testing run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -PnumNodes=3 + ./gradlew integTest -PnumNodes=3 - name: Upload Coverage Report uses: codecov/codecov-action@v3 with: