Db 11134 4

db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java

@@ -54,7 +54,7 @@ public class SYSNATURALNUMBERSRowFactory extends CatalogRowFactory
     private static final int SYSNATURALNUMBERS_COLUMN_COUNT = 1;
     private static final int SYSNATURALNUMBERS_N = 1;
 
-    private static final int MAX_NUMBER = 2048;
+    public static final int MAX_NUMBER = 2048;
 
     private static final String[] uuids =
     {

hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2012 - 2020 Splice Machine, Inc.
+ *
+ * This file is part of Splice Machine.
+ * Splice Machine is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either
+ * version 3, or (at your option) any later version.
+ * Splice Machine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Affero General Public License for more details.
+ * You should have received a copy of the GNU Affero General Public License along with Splice Machine.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.splicemachine.derby.impl.sql.execute.upgrade;
+
+import com.splicemachine.access.hbase.HBasePartitionAdmin;
+import com.splicemachine.db.impl.sql.catalog.SYSNATURALNUMBERSRowFactory;
+import com.splicemachine.derby.test.framework.SpliceSchemaWatcher;
+import com.splicemachine.derby.test.framework.SpliceUnitTest;
+import com.splicemachine.derby.test.framework.SpliceWatcher;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.junit.*;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+/**
+ * Note: These tests need to be excluded in platform_it/pom.xml,
+ * as they are only mean to be run with the check_upgrade.sh script.
+ *
+ * e.g.
+ * bash pipelines/upgrade-testing/check_upgrade.sh 3.1.0.2000 -T 4
+ * this downloads a tar.gz (here https://splice-snapshots.s3.amazonaws.com/upgrade_tests/platform_it_3.1.0.2000.tar.gz)
+ * extracts that, then starts the cluster with that data (without clean), and then executes the UpgradeTestIT
+ *
+ * see also
+ * pipelines/upgrade-testing/check_upgrade.sh
+ * pipelines/upgrade-testing/create_upgrade_targz.sh
+ * https://s3.console.aws.amazon.com/s3/buckets/splice-snapshots?region=us-east-1&prefix=upgrade_tests/&showversions=false
+ */
+public class UpgradeTestIT extends SpliceUnitTest {
+
+    private static final String SCHEMA_NAME = UpgradeTestIT.class.getSimpleName().toUpperCase();
+    private static final SpliceWatcher spliceClassWatcher = new SpliceWatcher(SCHEMA_NAME);
+    private static final SpliceSchemaWatcher spliceSchemaWatcher = new SpliceSchemaWatcher(SCHEMA_NAME);
+    @Rule
+    public SpliceWatcher methodWatcher = new SpliceWatcher(SCHEMA_NAME);
+
+    @ClassRule
+    public static TestRule chain = RuleChain.outerRule(spliceClassWatcher)
+            .around(spliceSchemaWatcher);
+
+    @Test // 1983
+    public void testUpgradeScriptToAddBaseTableSchemaColumnsToSysTablesInSYSIBM() throws Exception {
+        String expected =
+                "NAME    | CREATOR |TYPE |COLCOUNT |KEYCOLUMNS | KEYUNIQUE |CODEPAGE | BASE_NAME | BASE_SCHEMA |\n" +
+                "-------------------------------------------------------------------------------------------------\n" +
+                "SYSTABLES | SYSIBM  |  V  |    9    |     0     |     0     |  1208   |   NULL    |    NULL     |";
+        SpliceUnitTest.sqlExpectToString(methodWatcher,
+                "select * from sysibm.SYSTABLES WHERE NAME='SYSTABLES' AND CREATOR='SYSIBM'",
+                expected, false);
+    }
+
+    @Test // 1985
+    public void testUpgradeScriptToAddSysNaturalNumbersTable() throws Exception {
+        Assert.assertEquals(SYSNATURALNUMBERSRowFactory.MAX_NUMBER,
+                methodWatcher.executeGetInt("select count(*) from SYS.SYSNATURALNUMBERS", 1) );
+    }
+
+    @Test // 1989
+    public void testUpgradeScriptToAddIndexColUseViewInSYSCAT() throws Exception {
+        SpliceUnitTest.sqlExpectToString(methodWatcher,
+                "select TABLENAME  from sysvw.systablesview WHERE TABLENAME='INDEXCOLUSE'",
+                        "TABLENAME  |\n" +
+                        "-------------\n" +
+                        "INDEXCOLUSE |",
+                false);
+    }
+
+    @Test // 1992
+    public void testUpgradeScriptForTablePriorities() throws Exception {
+        try(Connection conn = ConnectionFactory.createConnection(new Configuration());
+                Admin admin= conn.getAdmin()) {
+            Assert.assertEquals(0, HBasePartitionAdmin.getToUpgradeStream(admin.listTableDescriptors()).count());
+        }
+    }
+
+}

hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java

@@ -24,6 +24,7 @@
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import splice.com.google.common.collect.Lists;
 import com.google.protobuf.InvalidProtocolBufferException;
@@ -764,16 +765,19 @@ public static void setHTablePriority(Admin admin, TableName tn,
         admin.enableTable(tn);
     }
 
+    public static Stream<org.apache.hadoop.hbase.client.TableDescriptor> getToUpgradeStream(
+            List<org.apache.hadoop.hbase.client.TableDescriptor> tableDescriptors) {
+        return tableDescriptors.stream().filter( td -> td.getPriority() != getPriorityShouldHave(td));
+    }
+
     public static int upgradeTablePrioritiesFromList(Admin admin,
                                                      List<org.apache.hadoop.hbase.client.TableDescriptor> tableDescriptors)
             throws Exception
     {
         final int NUM_THREADS = 10;
         ExecutorService executor = null;
         try {
-
-            List<Callable<Void>> upgradeTasks = tableDescriptors.stream()
-                    .filter( td -> td.getPriority() != getPriorityShouldHave(td))
+            List<Callable<Void>> upgradeTasks = getToUpgradeStream(tableDescriptors)
                     .map( td -> (Callable<Void>) () -> {
                         setHTablePriority(admin, td.getTableName(), td, getPriorityShouldHave(td));
                         return null;

pipelines/upgrade-testing/check_upgrade.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+if [ $# -lt 2 ]
+then
+	echo "usage: bash pipelines/upgrade-testing/check_upgrade.sh {VERSION} {additional start-splice-cluster parameters}"
+	echo "------------------------------------------------------------------------------------------"
+	echo "uses a previously created tar.gz to test upgrade"
+	echo "e.g. bash check_upgrade.sh 3.2.2021"
+
+    exit 1
+fi
+
+function test_upgrade
+{
+	OLDVERSION=$1
+	shift
+	echo ""
+	echo "                        testing $OLDVERSION upgrade"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+
+	UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+    # stop current cluster
+    ./start-splice-cluster -k
+
+    # clean up platform_it
+    cd platform_it
+    git clean -dfx
+    cd ..
+
+	echo ""
+  	echo "   download ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+	# download the previous standalone data
+	aws s3 cp ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz .
+	tar -xzf platform_it_${OLDVERSION}.tar.gz
+    rm platform_it_${OLDVERSION}.tar.gz
+
+	echo ""
+  	echo "	./start-splice-cluster -l $*"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+
+	# restart cluster
+	./start-splice-cluster -l $*
+
+	# test
+	if mvn -B -e surefire:test -Pcore,cdh6.3.0 \
+    	-Dtest='UpgradeTestIT#*' -DskipServerStart \
+        -DfailIfNoTests=false; \
+        -Dmaven.test.redirectTestOutputToFile=true
+   then
+	    echo "UPGRADE SUCCEEDED"
+	    cat platform_it/splice.log | grep 'upgrade scripts'
+	    cat platform_it/splice.log | grep 'Running upgrade script'	
+	else
+		echo "!!! UPGRADE FAILED !!!"
+		exit 2
+	fi
+
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+}
+
+
+test_upgrade $*

pipelines/upgrade-testing/create_upgrade_targz.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+
+if [ $# -lt 2 ]
+then
+    echo "usage: bash pipelines/upgrade-testing/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}"
+    echo "----------------------------------------------------------------------------------------------------------------------------------"
+    echo "creates a tar.gz of a spliceengine standalone cluster that can be used to test upgrade"
+    echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -pcore,cdh6.3.0"
+    exit 1
+fi
+
+VERSION=${1} # e.g. 3.1.0.1971
+MODE=${2} # test or upload
+shift 2
+
+PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD`
+git stash
+
+# creates a file platform_it_${VERSION}.tar.gz
+git checkout tags/${VERSION}
+cd platform_it
+rm -rf target *.log snappy*.jnilib
+cd ..
+
+./start-splice-cluster $*
+./start-splice-cluster -k
+
+rm -rf upgrade_test_TMP
+mkdir -p upgrade_test_TMP/platform_it/target
+cd upgrade_test_TMP
+cp -r ../platform_it/target/hbase platform_it/target/.
+cp -r ../platform_it/target/zookeeper platform_it/target/.
+tar -czvf ../platform_it_${VERSION}.tar.gz platform_it
+cd ..
+rm -rf upgrade_test_TMP
+
+if [[ $MODE == "upload" ]]; then
+    aws s3 cp platform_it_${VERSION}.tar.gz ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz
+fi
+
+git checkout ${PREVIOUS_BRANCH}
+git stash pop

pipelines/upgrade-testing/jenkins.sh

@@ -0,0 +1,108 @@
+# JENKINS FILE USED FOR UPGRADE-TESTING
+
+#!/bin/bash
+
+export platform=${ghprbCommentBody##*@}
+platform="$( sed 's/\\n//g' <<<"$platform" )"
+platform="$( sed 's/\\r//g' <<<"$platform" )"
+
+platform=($(echo $platform | awk '{print $1}'))
+
+if [[ ${platform} =~ ^mem$ ]] ; then
+  profiles="core,mem"
+elif [[ ${platform} =~ ^cdh ]] ; then
+  profiles="mem,core,${platform},ee,parcel"
+elif [[ ${platform} =~ ^hdp ]] ; then
+  profiles="core,${platform},ee,hdp_service"
+elif [[ ${platform} =~ ^hdp ]] ; then
+  profiles="core,${platform},ee,hdp_service"
+else
+  profiles=mem,"core,${platform},ee,installer"
+fi
+
+echo "building ${profiles} for ${platform}"
+
+MAVEN_OPTS="-Xms64m -XX:MinHeapFreeRatio=10 -XX:MaxHeapFreeRatio=30 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=YYYY-MM-dd|HH:mm:ss,SSS"
+
+function test_upgrade
+{
+	OLDVERSION=$1
+	echo ""
+	echo "                        testing $OLDVERSION upgrade"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+	pushd spliceengine
+
+	UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+    # stop current cluster
+    ./start-splice-cluster -k
+
+    # clean up platform_it
+    cd platform_it
+    git clean -dfx
+    cd ..
+
+	echo ""
+  	echo "   download ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+	# download the previous standalone data
+	aws s3 cp ${UPGRADE_URL}/platform_it_${OLDVERSION}.tar.gz .
+	tar -xzvf platform_it_${OLDVERSION}.tar.gz
+    rm platform_it_${OLDVERSION}.tar.gz
+
+	echo ""
+  	echo "	./start-splice-cluster -l"
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+
+	# restart cluster
+	./start-splice-cluster -l #-p${profiles},ee 
+
+	# test
+	if mvn -B -e surefire:test -Pcore,cdh6.3.0 \
+    	-Dtest='UpgradeTestIT#*' -DskipServerStart \
+        -DfailIfNoTests=false; \
+        -Dmaven.test.redirectTestOutputToFile=true
+   then
+	    echo "UPGRADE SUCCEEDED"
+	    cat platform_it/splice.log | grep 'upgrade scripts'
+	    cat platform_it/splice.log | grep 'Running upgrade script'	
+	else
+		echo "!!! UPGRADE FAILED !!!"
+		exit 2
+	fi
+
+	echo "--------------------------------------------------------------------------------"
+    echo ""
+	popd # spliceengine
+}
+
+echo "ghprbCommentBody = ${ghprbCommentBody}"
+
+if [[ -f spliceengine/pipelines/upgrade-testing/check_upgrade.sh && ${platform} =~ ^cdh6.3.0 ]];
+then
+	echo "upgrade file there"
+	upgrade=${ghprbCommentBody##*#upgrade:}
+	if [[ $ghprbCommentBody != $upgrade ]] ; then
+		echo "upgrade comment there"
+		upgrades=($(echo $upgrade | tr ":" "\n"))
+		for OLDVERSION in "${upgrades[@]}"
+		do
+			test_upgrade $OLDVERSION
+		done
+		exit 0
+	fi	
+fi
+
+mvn \
+  -B \
+  -f spliceengine/pom.xml \
+  -P${profiles} \
+  -D${platform} \
+  -Djenkins.build.number=${BUILD_NUMBER} \
+  -Dexcluded.categories=com.splicemachine.test.SlowTest \
+  -Dmaven.test.redirectTestOutputToFile=true \
+  clean install
+

platform_it/pom.xml

@@ -460,6 +460,8 @@
                             <excludes>
                                 <exclude>com/splicemachine/derby/impl/sql/execute/operations/window/test/framework/**
                                 </exclude>
+                                <exclude>com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java
+                                </exclude>
                             </excludes>
                             <runOrder>alphabetical</runOrder>
                             <systemPropertyVariables>
@@ -495,6 +497,10 @@
                             <includes>
                                 <include>**/*IT.java</include>
                             </includes>
+                            <excludes>
+                                <exclude>com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java
+                                </exclude>
+                            </excludes>
                             <runOrder>alphabetical</runOrder>
                             <systemPropertyVariables>
                                 <splice.schemaCleanup>async</splice.schemaCleanup>

splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java

@@ -472,4 +472,13 @@ public void assertStrResult(String res, String query, Boolean sort) throws Excep
         Assert.assertEquals( "failed asserting the results of sql\n" + query,
                 res, executeToString(query, sort));
     }
+
+    public int executeGetInt(String sql, int index) throws SQLException {
+        try( Statement s = getOrCreateConnection().createStatement();
+             ResultSet rs = s.executeQuery(sql))
+        {
+            Assert.assertTrue(rs.next());
+            return rs.getInt(index);
+        }
+    }
 }