From 04cc8a644b382fee49c17370338e386c1100b6f2 Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Thu, 7 Jan 2021 17:37:36 +0100 Subject: [PATCH 1/8] DB-11134 testing upgrades in spliceengine --- check_upgrade.sh | 57 ++++++++++++++ .../catalog/SYSNATURALNUMBERSRowFactory.java | 2 +- .../sql/execute/operations/UpgradeTestIT.java | 77 +++++++++++++++++++ .../access/hbase/HBasePartitionAdmin.java | 10 ++- .../derby/test/framework/SpliceWatcher.java | 9 +++ 5 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 check_upgrade.sh create mode 100644 hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java diff --git a/check_upgrade.sh b/check_upgrade.sh new file mode 100644 index 00000000000..3f51ea803d9 --- /dev/null +++ b/check_upgrade.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +VERSION=3.1.0.1971 + +# stop current cluster +./start-splice-cluster -k + +# clean up platform_it +cd platform_it +git clean -dfx +cd .. + +# download the previous standalone data +aws s3 cp s3://splice-snapshots/upgrade_tests/platform_it_${VERSION}.tar.gz . +tar -xzvf platform_it_${VERSION}.tar.gz +rm platform_it_${VERSION}.tar.gz + +# restart cluster +./start-splice-cluster -l + +# test +if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then + echo "UPGRADE SUCCEEDED" + cat platform_it/splice.log | grep 'upgrade scripts' + cat platform_it/splice.log | grep 'Running upgrade script' +else + echo "!!! UPGRADE FAILED !!!" +fi + +exit + +# create a .tar.gz to upgrade from +VERSION=3.1.0.1971 + + + +############ + +# creates a file platform_it_${VERSION}.tar.gz +function create_upgrade_targz +{ + git checkout tags/${VERSION} + cd platform_it + git clean -dfx + cd .. + + ./start-splice-cluster + + rm -rf upgrade_test_TMP + mkdir -p upgrade_test_TMP/platform_it/target + cd upgrade_test_TMP + cp -r ../platform_it/target/hbase platform_it/target/. + cp -r ../platform_it/target/zookeeper platform_it/target/. + tar -czvf ../platform_it_${VERSION}.tar.gz platform_it + cd .. + rm -rf upgrade_test_TMP +} diff --git a/db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java b/db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java index b3efaa3489c..ce373a9cb4e 100644 --- a/db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java +++ b/db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java @@ -54,7 +54,7 @@ public class SYSNATURALNUMBERSRowFactory extends CatalogRowFactory private static final int SYSNATURALNUMBERS_COLUMN_COUNT = 1; private static final int SYSNATURALNUMBERS_N = 1; - private static final int MAX_NUMBER = 2048; + public static final int MAX_NUMBER = 2048; private static final String[] uuids = { diff --git a/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java b/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java new file mode 100644 index 00000000000..e08cfa36d38 --- /dev/null +++ b/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2012 - 2020 Splice Machine, Inc. + * + * This file is part of Splice Machine. + * Splice Machine is free software: you can redistribute it and/or modify it under the terms of the + * GNU Affero General Public License as published by the Free Software Foundation, either + * version 3, or (at your option) any later version. + * Splice Machine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * You should have received a copy of the GNU Affero General Public License along with Splice Machine. + * If not, see . + */ + +package com.splicemachine.derby.impl.sql.execute.operations; + +import com.splicemachine.access.hbase.HBasePartitionAdmin; +import com.splicemachine.db.impl.sql.catalog.SYSNATURALNUMBERSRowFactory; +import com.splicemachine.derby.test.framework.SpliceSchemaWatcher; +import com.splicemachine.derby.test.framework.SpliceUnitTest; +import com.splicemachine.derby.test.framework.SpliceWatcher; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.junit.*; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +public class UpgradeTestIT extends SpliceUnitTest { + + private static final String SCHEMA_NAME = UpgradeTestIT.class.getSimpleName().toUpperCase(); + private static final SpliceWatcher spliceClassWatcher = new SpliceWatcher(SCHEMA_NAME); + private static final SpliceSchemaWatcher spliceSchemaWatcher = new SpliceSchemaWatcher(SCHEMA_NAME); + @Rule + public SpliceWatcher methodWatcher = new SpliceWatcher(SCHEMA_NAME); + + @ClassRule + public static TestRule chain = RuleChain.outerRule(spliceClassWatcher) + .around(spliceSchemaWatcher); + + @Test // 1983 + public void testUpgradeScriptToAddBaseTableSchemaColumnsToSysTablesInSYSIBM() throws Exception { + String expected = + "NAME | CREATOR |TYPE |COLCOUNT |KEYCOLUMNS | KEYUNIQUE |CODEPAGE | BASE_NAME | BASE_SCHEMA |\n" + + "-------------------------------------------------------------------------------------------------\n" + + "SYSTABLES | SYSIBM | V | 9 | 0 | 0 | 1208 | NULL | NULL |"; + SpliceUnitTest.sqlExpectToString(methodWatcher, + "select * from sysibm.SYSTABLES WHERE NAME='SYSTABLES' AND CREATOR='SYSIBM'", + expected, false); + } + + @Test // 1985 + public void testUpgradeScriptToAddSysNaturalNumbersTable() throws Exception { + Assert.assertEquals(SYSNATURALNUMBERSRowFactory.MAX_NUMBER, + methodWatcher.executeGetInt("select count(*) from SYS.SYSNATURALNUMBERS", 1) ); + } + + @Test // 1989 + public void testUpgradeScriptToAddIndexColUseViewInSYSCAT() throws Exception { + SpliceUnitTest.sqlExpectToString(methodWatcher, + "select TABLENAME from sysvw.systablesview WHERE TABLENAME='INDEXCOLUSE'", + "TABLENAME |\n" + + "-------------\n" + + "INDEXCOLUSE |", + false); + } + + @Test // 1992 + public void testUpgradeScriptForTablePriorities() throws Exception { + try(Connection conn = ConnectionFactory.createConnection(new Configuration()); + Admin admin= conn.getAdmin()) { + Assert.assertEquals(0, HBasePartitionAdmin.getToUpgradeStream(admin.listTableDescriptors()).count()); + } + } + +} diff --git a/hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java b/hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java index fe653499f05..d4ae2cd35c3 100644 --- a/hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java +++ b/hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java @@ -24,6 +24,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import splice.com.google.common.collect.Lists; import com.google.protobuf.InvalidProtocolBufferException; @@ -764,6 +765,11 @@ public static void setHTablePriority(Admin admin, TableName tn, admin.enableTable(tn); } + public static Stream getToUpgradeStream( + List tableDescriptors) { + return tableDescriptors.stream().filter( td -> td.getPriority() != getPriorityShouldHave(td)); + } + public static int upgradeTablePrioritiesFromList(Admin admin, List tableDescriptors) throws Exception @@ -771,9 +777,7 @@ public static int upgradeTablePrioritiesFromList(Admin admin, final int NUM_THREADS = 10; ExecutorService executor = null; try { - - List> upgradeTasks = tableDescriptors.stream() - .filter( td -> td.getPriority() != getPriorityShouldHave(td)) + List> upgradeTasks = getToUpgradeStream(tableDescriptors) .map( td -> (Callable) () -> { setHTablePriority(admin, td.getTableName(), td, getPriorityShouldHave(td)); return null; diff --git a/splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java b/splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java index 521de5c1c99..1de1641677c 100644 --- a/splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java +++ b/splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java @@ -472,4 +472,13 @@ public void assertStrResult(String res, String query, Boolean sort) throws Excep Assert.assertEquals( "failed asserting the results of sql\n" + query, res, executeToString(query, sort)); } + + public int executeGetInt(String sql, int index) throws SQLException { + try( Statement s = getOrCreateConnection().createStatement(); + ResultSet rs = s.executeQuery(sql)) + { + Assert.assertTrue(rs.next()); + return rs.getInt(index); + } + } } From 82a64dc86928c5fc5a23ec21d03da84f00493baf Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Tue, 20 Jul 2021 13:08:23 +0200 Subject: [PATCH 2/8] DB-11134 start-splice-cluster to abort when start fails, added pipelines/upgrade-testing - check_upgrade.sh {VERSION} {additional start-splice-cluster parameters}" - create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters} --- check_upgrade.sh | 57 ------------------- pipelines/upgrade-testing/check_upgrade.sh | 42 ++++++++++++++ .../upgrade-testing/create_upgrade_targz.sh | 45 +++++++++++++++ start-splice-cluster | 34 +++++++++-- 4 files changed, 117 insertions(+), 61 deletions(-) delete mode 100644 check_upgrade.sh create mode 100644 pipelines/upgrade-testing/check_upgrade.sh create mode 100644 pipelines/upgrade-testing/create_upgrade_targz.sh diff --git a/check_upgrade.sh b/check_upgrade.sh deleted file mode 100644 index 3f51ea803d9..00000000000 --- a/check_upgrade.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -VERSION=3.1.0.1971 - -# stop current cluster -./start-splice-cluster -k - -# clean up platform_it -cd platform_it -git clean -dfx -cd .. - -# download the previous standalone data -aws s3 cp s3://splice-snapshots/upgrade_tests/platform_it_${VERSION}.tar.gz . -tar -xzvf platform_it_${VERSION}.tar.gz -rm platform_it_${VERSION}.tar.gz - -# restart cluster -./start-splice-cluster -l - -# test -if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then - echo "UPGRADE SUCCEEDED" - cat platform_it/splice.log | grep 'upgrade scripts' - cat platform_it/splice.log | grep 'Running upgrade script' -else - echo "!!! UPGRADE FAILED !!!" -fi - -exit - -# create a .tar.gz to upgrade from -VERSION=3.1.0.1971 - - - -############ - -# creates a file platform_it_${VERSION}.tar.gz -function create_upgrade_targz -{ - git checkout tags/${VERSION} - cd platform_it - git clean -dfx - cd .. - - ./start-splice-cluster - - rm -rf upgrade_test_TMP - mkdir -p upgrade_test_TMP/platform_it/target - cd upgrade_test_TMP - cp -r ../platform_it/target/hbase platform_it/target/. - cp -r ../platform_it/target/zookeeper platform_it/target/. - tar -czvf ../platform_it_${VERSION}.tar.gz platform_it - cd .. - rm -rf upgrade_test_TMP -} diff --git a/pipelines/upgrade-testing/check_upgrade.sh b/pipelines/upgrade-testing/check_upgrade.sh new file mode 100644 index 00000000000..33fcf25b7a7 --- /dev/null +++ b/pipelines/upgrade-testing/check_upgrade.sh @@ -0,0 +1,42 @@ +#!/bin/bash +UPGRADE_URL=s3://splice-snapshots/upgrade_tests + +if [ $# -lt 2 ] +then + echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}" + echo "------------------------------------------------------------------------------------------" + echo "uses a previously created tar.gz to test upgrade" + echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16" + echo "make sure you current branch has already been build" + echo "don't use -b, since we are deleting some files in platform_it/target" + + exit 1 +fi + +VERSION=${1} +shift + +# stop current cluster +./start-splice-cluster -k + +# clean up platform_it +cd platform_it +git clean -dfx +cd .. + +# download the previous standalone data +aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz . +tar -xzvf platform_it_${VERSION}.tar.gz +rm platform_it_${VERSION}.tar.gz + +# restart cluster +./start-splice-cluster -l $* + +# test +if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then + echo "UPGRADE SUCCEEDED" + cat platform_it/splice.log | grep 'upgrade scripts' + cat platform_it/splice.log | grep 'Running upgrade script' +else + echo "!!! UPGRADE FAILED !!!" +fi diff --git a/pipelines/upgrade-testing/create_upgrade_targz.sh b/pipelines/upgrade-testing/create_upgrade_targz.sh new file mode 100644 index 00000000000..d51f4469fa8 --- /dev/null +++ b/pipelines/upgrade-testing/create_upgrade_targz.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +UPGRADE_URL=s3://splice-snapshots/upgrade_tests + + +if [ $# -lt 2 ] +then + echo "usage: bash platform_it/src/test/bin/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}" + echo "----------------------------------------------------------------------------------------------------------------------------------" + echo "creates a tar.gz of a spliceengine standalone cluster that can be used to test upgrade" + echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -T 16 -pcore,cdh6.3.0" + exit 1 +fi + +VERSION=${1} # e.g. 3.1.0.1971 +MODE=${2} # test or upload +shift 2 + + +PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD` +git stash + +# creates a file platform_it_${VERSION}.tar.gz +git checkout tags/${VERSION} +cd platform_it +rm -rf target *.log snappy*.jnilib +cd .. + +./start-splice-cluster $* + +rm -rf upgrade_test_TMP +mkdir -p upgrade_test_TMP/platform_it/target +cd upgrade_test_TMP +cp -r ../platform_it/target/hbase platform_it/target/. +cp -r ../platform_it/target/zookeeper platform_it/target/. +tar -czvf ../platform_it_${VERSION}.tar.gz platform_it +cd .. +rm -rf upgrade_test_TMP + +if [[ $MODE == "upload" ]]; then + aws s3 cp platform_it_${VERSION}.tar.gz ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz +fi + +git checkout ${PREVIOUS_BRANCH} +git stash pop diff --git a/start-splice-cluster b/start-splice-cluster index edf7452911a..75b0a03e8c5 100755 --- a/start-splice-cluster +++ b/start-splice-cluster @@ -64,18 +64,24 @@ function is_port_open } ################################################################################## -# print message ${1} and wait until port {2} is open +# print message ${1} and wait until port {2} is open, for max {3}*2 seconds ################################################################################## function _wait_for_port { msg=${1} port=${2} + wait_max=${3} echo -n ${msg} echo -n " " counter=0 until is_port_open localhost ${port}; do echo -n "${counter} - " counter=$((counter+1)) + + if [ ${counter} -ge ${wait_max} ]; then + echo "Waited too long, aborting." + exit 2 + fi sleep 2 done echo "done!" @@ -175,7 +181,7 @@ function _start_region_servers { function _wait_for_region_servers { if [[ ${MEMBERS} -gt 0 ]]; then for (( MEMBER=1; MEMBER<${MEMBERS}; MEMBER++ )); do - _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} )) + _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} )) 60 done fi } @@ -198,7 +204,27 @@ function _start_master { } function _wait_for_master { - _wait_for_port "Waiting for Master to be ready ..." 1527 + + msg="Waiting for Master to be ready ..." + port=1527 + wait_max=60 + echo -n ${msg} + echo -n " " + counter=0 + until is_port_open localhost ${port}; do + echo -n "${counter} - " + counter=$((counter+1)) + if [ ${counter} -ge ${wait_max} ]; then + echo "Waited too long, aborting." + exit 2 + fi + if cat ${SPLICE_LOG} | grep "[ERROR] Failed to execute goal" ; then + echo "ERROR: Spliceengine couldn't start." + exit 2 + fi + sleep 2 + done + echo "done!" } @@ -234,7 +260,7 @@ function _start_mem function _wait_for_mem { - _wait_for_port "Waiting until Mem Platform is ready ..." 1527 + _wait_for_port "Waiting until Mem Platform is ready ..." 1527 20 } export -f _kill_em_all From 057563ef07a09de4855f07a2b6b40a2587b596cb Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Tue, 20 Jul 2021 13:33:27 +0200 Subject: [PATCH 3/8] fix1 --- .../{operations => upgrade}/UpgradeTestIT.java | 16 +++++++++++++++- platform_it/pom.xml | 6 ++++++ start-splice-cluster | 7 ++++--- 3 files changed, 25 insertions(+), 4 deletions(-) rename hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/{operations => upgrade}/UpgradeTestIT.java (83%) diff --git a/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java b/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java similarity index 83% rename from hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java rename to hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java index e08cfa36d38..1a505edd1a4 100644 --- a/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/operations/UpgradeTestIT.java +++ b/hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java @@ -12,7 +12,7 @@ * If not, see . */ -package com.splicemachine.derby.impl.sql.execute.operations; +package com.splicemachine.derby.impl.sql.execute.upgrade; import com.splicemachine.access.hbase.HBasePartitionAdmin; import com.splicemachine.db.impl.sql.catalog.SYSNATURALNUMBERSRowFactory; @@ -27,6 +27,20 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +/** + * Note: These tests need to be excluded in platform_it/pom.xml, + * as they are only mean to be run with the check_upgrade.sh script. + * + * e.g. + * bash pipelines/upgrade-testing/check_upgrade.sh 3.1.0.2000 -T 4 + * this downloads a tar.gz (here https://splice-snapshots.s3.amazonaws.com/upgrade_tests/platform_it_3.1.0.2000.tar.gz) + * extracts that, then starts the cluster with that data (without clean), and then executes the UpgradeTestIT + * + * see also + * pipelines/upgrade-testing/check_upgrade.sh + * pipelines/upgrade-testing/create_upgrade_targz.sh + * https://s3.console.aws.amazon.com/s3/buckets/splice-snapshots?region=us-east-1&prefix=upgrade_tests/&showversions=false + */ public class UpgradeTestIT extends SpliceUnitTest { private static final String SCHEMA_NAME = UpgradeTestIT.class.getSimpleName().toUpperCase(); diff --git a/platform_it/pom.xml b/platform_it/pom.xml index ea7f2391e11..36c7c61b46f 100644 --- a/platform_it/pom.xml +++ b/platform_it/pom.xml @@ -460,6 +460,8 @@ com/splicemachine/derby/impl/sql/execute/operations/window/test/framework/** + com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java + alphabetical @@ -495,6 +497,10 @@ **/*IT.java + + com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java + + alphabetical async diff --git a/start-splice-cluster b/start-splice-cluster index 75b0a03e8c5..4207416f60e 100755 --- a/start-splice-cluster +++ b/start-splice-cluster @@ -24,7 +24,8 @@ CHAOS="false" MEMBERS=2 DEBUG_PATH="" - +MAX_WAIT_REGION_SERVERS=100 # *2 seconds +MAX_WAIT_MASTER=200 # *2 seconds ################################################################################## # Function to kill all splice test processes - zoo, SpliceTestPlatform, YARN, and @@ -181,7 +182,7 @@ function _start_region_servers { function _wait_for_region_servers { if [[ ${MEMBERS} -gt 0 ]]; then for (( MEMBER=1; MEMBER<${MEMBERS}; MEMBER++ )); do - _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} )) 60 + _wait_for_port "Waiting for Region Server $(($MEMBER +1)) to be ready ..." $(( 1527 + ${MEMBER} )) ${MAX_WAIT_REGION_SERVERS} done fi } @@ -207,7 +208,7 @@ function _wait_for_master { msg="Waiting for Master to be ready ..." port=1527 - wait_max=60 + wait_max=${MAX_WAIT_MASTER} echo -n ${msg} echo -n " " counter=0 From 269d7fbf4dd8ca6b95505ee528eb450263c39e98 Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Tue, 20 Jul 2021 13:46:19 +0200 Subject: [PATCH 4/8] fix2 --- start-splice-cluster | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/start-splice-cluster b/start-splice-cluster index 4207416f60e..d7333e4f3fc 100755 --- a/start-splice-cluster +++ b/start-splice-cluster @@ -76,7 +76,11 @@ function _wait_for_port echo -n " " counter=0 until is_port_open localhost ${port}; do - echo -n "${counter} - " + if [ ${JENKINS_MODE} ]; then + echo "${msg} ${counter}" + else + echo -n "${counter} - " + fi counter=$((counter+1)) if [ ${counter} -ge ${wait_max} ]; then @@ -213,13 +217,17 @@ function _wait_for_master { echo -n " " counter=0 until is_port_open localhost ${port}; do - echo -n "${counter} - " + if [ ${JENKINS_MODE} ]; then + echo "${msg} ${counter}" + else + echo -n "${counter} - " + fi counter=$((counter+1)) if [ ${counter} -ge ${wait_max} ]; then echo "Waited too long, aborting." exit 2 fi - if cat ${SPLICE_LOG} | grep "[ERROR] Failed to execute goal" ; then + if cat ${SPLICE_LOG} | grep "\[ERROR\] Failed to execute goal" ; then echo "ERROR: Spliceengine couldn't start." exit 2 fi @@ -290,6 +298,7 @@ function usage echo " -K like k, but will also reset database to clean. Needs some rebuild afterwards (use -l)." echo " -T use -T with maven (build with n threads)." echo " -h => print this message" + echo " -L use linefeeds when waiting (jenkins mode)" } function verify_profiles @@ -313,7 +322,7 @@ function verify_profiles done } -while getopts "chkp:s:bld:fT:Ke" flag ; do +while getopts "chkp:s:bld:fT:KeL" flag ; do case $flag in h* | \?) usage @@ -366,6 +375,9 @@ while getopts "chkp:s:bld:fT:Ke" flag ; do # number of threads MVN_THREADS="-T ${OPTARG}" ;; + L) + JENKINS_MODE=1 + ;; e) # enterprise edition ENTERPRISE_EDITION=1 From 95dbd0d3ad0e72204bbc849d37059ad1ae6e5826 Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Tue, 20 Jul 2021 14:14:50 +0200 Subject: [PATCH 5/8] fix3 --- pipelines/upgrade-testing/check_upgrade.sh | 41 +++++++++++------- .../upgrade-testing/check_upgrade_fast.sh | 42 +++++++++++++++++++ .../upgrade-testing/create_upgrade_targz.sh | 1 + 3 files changed, 69 insertions(+), 15 deletions(-) create mode 100644 pipelines/upgrade-testing/check_upgrade_fast.sh diff --git a/pipelines/upgrade-testing/check_upgrade.sh b/pipelines/upgrade-testing/check_upgrade.sh index 33fcf25b7a7..85836089d33 100644 --- a/pipelines/upgrade-testing/check_upgrade.sh +++ b/pipelines/upgrade-testing/check_upgrade.sh @@ -1,31 +1,42 @@ #!/bin/bash + UPGRADE_URL=s3://splice-snapshots/upgrade_tests -if [ $# -lt 2 ] -then - echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}" - echo "------------------------------------------------------------------------------------------" - echo "uses a previously created tar.gz to test upgrade" - echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16" - echo "make sure you current branch has already been build" - echo "don't use -b, since we are deleting some files in platform_it/target" - - exit 1 -fi +VERSION=${1} # e.g. 3.1.0.1971 +shift 1 + +PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD` -VERSION=${1} -shift +# creates a file platform_it_${VERSION}.tar.gz +git checkout tags/${VERSION} +cd platform_it +rm -rf target *.log snappy*.jnilib +cd .. -# stop current cluster +./start-splice-cluster $* ./start-splice-cluster -k +rm -rf upgrade_test_TMP +mkdir -p upgrade_test_TMP/platform_it/target +cd upgrade_test_TMP +cp -r ../platform_it/target/hbase platform_it/target/. +cp -r ../platform_it/target/zookeeper platform_it/target/. +tar -czvf ../platform_it_${VERSION}.tar.gz platform_it +cd .. +rm -rf upgrade_test_TMP + +git checkout ${PREVIOUS_BRANCH} + + +# restart on that version +./start-splice-cluster $* + # clean up platform_it cd platform_it git clean -dfx cd .. # download the previous standalone data -aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz . tar -xzvf platform_it_${VERSION}.tar.gz rm platform_it_${VERSION}.tar.gz diff --git a/pipelines/upgrade-testing/check_upgrade_fast.sh b/pipelines/upgrade-testing/check_upgrade_fast.sh new file mode 100644 index 00000000000..33fcf25b7a7 --- /dev/null +++ b/pipelines/upgrade-testing/check_upgrade_fast.sh @@ -0,0 +1,42 @@ +#!/bin/bash +UPGRADE_URL=s3://splice-snapshots/upgrade_tests + +if [ $# -lt 2 ] +then + echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}" + echo "------------------------------------------------------------------------------------------" + echo "uses a previously created tar.gz to test upgrade" + echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16" + echo "make sure you current branch has already been build" + echo "don't use -b, since we are deleting some files in platform_it/target" + + exit 1 +fi + +VERSION=${1} +shift + +# stop current cluster +./start-splice-cluster -k + +# clean up platform_it +cd platform_it +git clean -dfx +cd .. + +# download the previous standalone data +aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz . +tar -xzvf platform_it_${VERSION}.tar.gz +rm platform_it_${VERSION}.tar.gz + +# restart cluster +./start-splice-cluster -l $* + +# test +if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then + echo "UPGRADE SUCCEEDED" + cat platform_it/splice.log | grep 'upgrade scripts' + cat platform_it/splice.log | grep 'Running upgrade script' +else + echo "!!! UPGRADE FAILED !!!" +fi diff --git a/pipelines/upgrade-testing/create_upgrade_targz.sh b/pipelines/upgrade-testing/create_upgrade_targz.sh index d51f4469fa8..15567529d73 100644 --- a/pipelines/upgrade-testing/create_upgrade_targz.sh +++ b/pipelines/upgrade-testing/create_upgrade_targz.sh @@ -27,6 +27,7 @@ rm -rf target *.log snappy*.jnilib cd .. ./start-splice-cluster $* +./start-splice-cluster -k rm -rf upgrade_test_TMP mkdir -p upgrade_test_TMP/platform_it/target From d607b8a056e6eb60c48fc237733c3ba3ee8bcf5a Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Tue, 20 Jul 2021 14:22:38 +0200 Subject: [PATCH 6/8] fix4 --- pipelines/upgrade-testing/check_upgrade.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/upgrade-testing/check_upgrade.sh b/pipelines/upgrade-testing/check_upgrade.sh index 85836089d33..24432b14bac 100644 --- a/pipelines/upgrade-testing/check_upgrade.sh +++ b/pipelines/upgrade-testing/check_upgrade.sh @@ -6,6 +6,7 @@ VERSION=${1} # e.g. 3.1.0.1971 shift 1 PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD` +git stash # creates a file platform_it_${VERSION}.tar.gz git checkout tags/${VERSION} @@ -26,7 +27,7 @@ cd .. rm -rf upgrade_test_TMP git checkout ${PREVIOUS_BRANCH} - +git stash pop # restart on that version ./start-splice-cluster $* From fce3f9ee09cde83d2490c64aa8e2a26d0e6934f0 Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Wed, 21 Jul 2021 12:08:48 +0200 Subject: [PATCH 7/8] jenkins file --- pipelines/upgrade-testing/jenkins.sh | 106 +++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 pipelines/upgrade-testing/jenkins.sh diff --git a/pipelines/upgrade-testing/jenkins.sh b/pipelines/upgrade-testing/jenkins.sh new file mode 100644 index 00000000000..516327684c9 --- /dev/null +++ b/pipelines/upgrade-testing/jenkins.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +export platform=${ghprbCommentBody##*@} +platform="$( sed 's/\\n//g' <<<"$platform" )" +platform="$( sed 's/\\r//g' <<<"$platform" )" + +platform=($(echo $platform | awk '{print $1}')) + +if [[ ${platform} =~ ^mem$ ]] ; then + profiles="core,mem" +elif [[ ${platform} =~ ^cdh ]] ; then + profiles="mem,core,${platform},ee,parcel" +elif [[ ${platform} =~ ^hdp ]] ; then + profiles="core,${platform},ee,hdp_service" +elif [[ ${platform} =~ ^hdp ]] ; then + profiles="core,${platform},ee,hdp_service" +else + profiles=mem,"core,${platform},ee,installer" +fi + +echo "building ${profiles} for ${platform}" + +MAVEN_OPTS="-Xms64m -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=30 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=YYYY-MM-dd|HH:mm:ss,SSS" + +function test_upgrade +{ + OLDVERSION=$1 + echo "" + echo " testing $OLDVERSION upgrade" + echo "--------------------------------------------------------------------------------" + echo "" + pushd spliceengine + + UPGRADE_URL=s3://splice-snapshots/upgrade_tests + + # stop current cluster + ./start-splice-cluster -k + + # clean up platform_it + cd platform_it + git clean -dfx + cd .. + + echo "" + echo " download ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz" + echo "--------------------------------------------------------------------------------" + echo "" + # download the previous standalone data + aws s3 cp ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz . + tar -xzvf platform_it_${OLDVERSION}.tar.gz + rm platform_it_${OLDVERSION}.tar.gz + + echo "" + echo " ./start-splice-cluster -l" + echo "--------------------------------------------------------------------------------" + echo "" + + # restart cluster + ./start-splice-cluster -l #-p${profiles},ee + + # test + if mvn -B -e surefire:test -Pcore,cdh6.3.0 \ + -Dtest='UpgradeTestIT#*' -DskipServerStart \ + -DfailIfNoTests=false; \ + -Dmaven.test.redirectTestOutputToFile=true + then + echo "UPGRADE SUCCEEDED" + cat platform_it/splice.log | grep 'upgrade scripts' + cat platform_it/splice.log | grep 'Running upgrade script' + else + echo "!!! UPGRADE FAILED !!!" + exit 2 + fi + + echo "--------------------------------------------------------------------------------" + echo "" + popd # spliceengine +} + +echo "ghprbCommentBody = ${ghprbCommentBody}" + +if [[ -f spliceengine/pipelines/upgrade-testing/check_upgrade.sh && ${platform} =~ ^cdh6.3.0 ]]; +then + echo "upgrade file there" + upgrade=${ghprbCommentBody##*#upgrade:} + if [[ $ghprbCommentBody != $upgrade ]] ; then + echo "upgrade comment there" + upgrades=($(echo $upgrade | tr ":" "\n")) + for OLDVERSION in "${upgrades[@]}" + do + test_upgrade $OLDVERSION + done + exit 0 + fi +fi + +mvn \ + -B \ + -f spliceengine/pom.xml \ + -P${profiles} \ + -D${platform} \ + -Djenkins.build.number=${BUILD_NUMBER} \ + -Dexcluded.categories=com.splicemachine.test.SlowTest \ + -Dmaven.test.redirectTestOutputToFile=true \ + clean install + From 47e73d76eadeb9e64a975e0cbde8786bed886bfa Mon Sep 17 00:00:00 2001 From: Martin Rupp Date: Wed, 21 Jul 2021 14:05:13 +0200 Subject: [PATCH 8/8] address review --- pipelines/upgrade-testing/check_upgrade.sh | 115 ++++++++++-------- .../upgrade-testing/check_upgrade_fast.sh | 42 ------- .../upgrade-testing/create_upgrade_targz.sh | 5 +- pipelines/upgrade-testing/jenkins.sh | 2 + 4 files changed, 69 insertions(+), 95 deletions(-) delete mode 100644 pipelines/upgrade-testing/check_upgrade_fast.sh diff --git a/pipelines/upgrade-testing/check_upgrade.sh b/pipelines/upgrade-testing/check_upgrade.sh index 24432b14bac..0e7251b9f85 100644 --- a/pipelines/upgrade-testing/check_upgrade.sh +++ b/pipelines/upgrade-testing/check_upgrade.sh @@ -1,54 +1,69 @@ #!/bin/bash - UPGRADE_URL=s3://splice-snapshots/upgrade_tests -VERSION=${1} # e.g. 3.1.0.1971 -shift 1 - -PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD` -git stash - -# creates a file platform_it_${VERSION}.tar.gz -git checkout tags/${VERSION} -cd platform_it -rm -rf target *.log snappy*.jnilib -cd .. - -./start-splice-cluster $* -./start-splice-cluster -k - -rm -rf upgrade_test_TMP -mkdir -p upgrade_test_TMP/platform_it/target -cd upgrade_test_TMP -cp -r ../platform_it/target/hbase platform_it/target/. -cp -r ../platform_it/target/zookeeper platform_it/target/. -tar -czvf ../platform_it_${VERSION}.tar.gz platform_it -cd .. -rm -rf upgrade_test_TMP - -git checkout ${PREVIOUS_BRANCH} -git stash pop - -# restart on that version -./start-splice-cluster $* - -# clean up platform_it -cd platform_it -git clean -dfx -cd .. - -# download the previous standalone data -tar -xzvf platform_it_${VERSION}.tar.gz -rm platform_it_${VERSION}.tar.gz - -# restart cluster -./start-splice-cluster -l $* - -# test -if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then - echo "UPGRADE SUCCEEDED" - cat platform_it/splice.log | grep 'upgrade scripts' - cat platform_it/splice.log | grep 'Running upgrade script' -else - echo "!!! UPGRADE FAILED !!!" +if [ $# -lt 2 ] +then + echo "usage: bash pipelines/upgrade-testing/check_upgrade.sh {VERSION} {additional start-splice-cluster parameters}" + echo "------------------------------------------------------------------------------------------" + echo "uses a previously created tar.gz to test upgrade" + echo "e.g. bash check_upgrade.sh 3.2.2021" + + exit 1 fi + +function test_upgrade +{ + OLDVERSION=$1 + shift + echo "" + echo " testing $OLDVERSION upgrade" + echo "--------------------------------------------------------------------------------" + echo "" + + UPGRADE_URL=s3://splice-snapshots/upgrade_tests + + # stop current cluster + ./start-splice-cluster -k + + # clean up platform_it + cd platform_it + git clean -dfx + cd .. + + echo "" + echo " download ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz" + echo "--------------------------------------------------------------------------------" + echo "" + # download the previous standalone data + aws s3 cp ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz . + tar -xzf platform_it_${OLDVERSION}.tar.gz + rm platform_it_${OLDVERSION}.tar.gz + + echo "" + echo " ./start-splice-cluster -l $*" + echo "--------------------------------------------------------------------------------" + echo "" + + # restart cluster + ./start-splice-cluster -l $* + + # test + if mvn -B -e surefire:test -Pcore,cdh6.3.0 \ + -Dtest='UpgradeTestIT#*' -DskipServerStart \ + -DfailIfNoTests=false; \ + -Dmaven.test.redirectTestOutputToFile=true + then + echo "UPGRADE SUCCEEDED" + cat platform_it/splice.log | grep 'upgrade scripts' + cat platform_it/splice.log | grep 'Running upgrade script' + else + echo "!!! UPGRADE FAILED !!!" + exit 2 + fi + + echo "--------------------------------------------------------------------------------" + echo "" +} + + +test_upgrade $* \ No newline at end of file diff --git a/pipelines/upgrade-testing/check_upgrade_fast.sh b/pipelines/upgrade-testing/check_upgrade_fast.sh deleted file mode 100644 index 33fcf25b7a7..00000000000 --- a/pipelines/upgrade-testing/check_upgrade_fast.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -UPGRADE_URL=s3://splice-snapshots/upgrade_tests - -if [ $# -lt 2 ] -then - echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}" - echo "------------------------------------------------------------------------------------------" - echo "uses a previously created tar.gz to test upgrade" - echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16" - echo "make sure you current branch has already been build" - echo "don't use -b, since we are deleting some files in platform_it/target" - - exit 1 -fi - -VERSION=${1} -shift - -# stop current cluster -./start-splice-cluster -k - -# clean up platform_it -cd platform_it -git clean -dfx -cd .. - -# download the previous standalone data -aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz . -tar -xzvf platform_it_${VERSION}.tar.gz -rm platform_it_${VERSION}.tar.gz - -# restart cluster -./start-splice-cluster -l $* - -# test -if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then - echo "UPGRADE SUCCEEDED" - cat platform_it/splice.log | grep 'upgrade scripts' - cat platform_it/splice.log | grep 'Running upgrade script' -else - echo "!!! UPGRADE FAILED !!!" -fi diff --git a/pipelines/upgrade-testing/create_upgrade_targz.sh b/pipelines/upgrade-testing/create_upgrade_targz.sh index 15567529d73..a377dbad662 100644 --- a/pipelines/upgrade-testing/create_upgrade_targz.sh +++ b/pipelines/upgrade-testing/create_upgrade_targz.sh @@ -5,10 +5,10 @@ UPGRADE_URL=s3://splice-snapshots/upgrade_tests if [ $# -lt 2 ] then - echo "usage: bash platform_it/src/test/bin/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}" + echo "usage: bash pipelines/upgrade-testing/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}" echo "----------------------------------------------------------------------------------------------------------------------------------" echo "creates a tar.gz of a spliceengine standalone cluster that can be used to test upgrade" - echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -T 16 -pcore,cdh6.3.0" + echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -pcore,cdh6.3.0" exit 1 fi @@ -16,7 +16,6 @@ VERSION=${1} # e.g. 3.1.0.1971 MODE=${2} # test or upload shift 2 - PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD` git stash diff --git a/pipelines/upgrade-testing/jenkins.sh b/pipelines/upgrade-testing/jenkins.sh index 516327684c9..9300e13b15c 100644 --- a/pipelines/upgrade-testing/jenkins.sh +++ b/pipelines/upgrade-testing/jenkins.sh @@ -1,3 +1,5 @@ +# JENKINS FILE USED FOR UPGRADE-TESTING + #!/bin/bash export platform=${ghprbCommentBody##*@}