Db 11134 4

db-engine/src/main/java/com/splicemachine/db/impl/sql/catalog/SYSNATURALNUMBERSRowFactory.java

@@ -54,7 +54,7 @@ public class SYSNATURALNUMBERSRowFactory extends CatalogRowFactory
     private static final int SYSNATURALNUMBERS_COLUMN_COUNT = 1;
     private static final int SYSNATURALNUMBERS_N = 1;
 
-    private static final int MAX_NUMBER = 2048;
+    public static final int MAX_NUMBER = 2048;
 
     private static final String[] uuids =
     {

hbase_sql/src/test/java/com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2012 - 2020 Splice Machine, Inc.
+ *
+ * This file is part of Splice Machine.
+ * Splice Machine is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either
+ * version 3, or (at your option) any later version.
+ * Splice Machine is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Affero General Public License for more details.
+ * You should have received a copy of the GNU Affero General Public License along with Splice Machine.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.splicemachine.derby.impl.sql.execute.upgrade;
+
+import com.splicemachine.access.hbase.HBasePartitionAdmin;
+import com.splicemachine.db.impl.sql.catalog.SYSNATURALNUMBERSRowFactory;
+import com.splicemachine.derby.test.framework.SpliceSchemaWatcher;
+import com.splicemachine.derby.test.framework.SpliceUnitTest;
+import com.splicemachine.derby.test.framework.SpliceWatcher;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.junit.*;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+/**
+ * Note: These tests need to be excluded in platform_it/pom.xml,
+ * as they are only mean to be run with the check_upgrade.sh script.
+ *
+ * e.g.
+ * bash pipelines/upgrade-testing/check_upgrade.sh 3.1.0.2000 -T 4
+ * this downloads a tar.gz (here https://splice-snapshots.s3.amazonaws.com/upgrade_tests/platform_it_3.1.0.2000.tar.gz)
+ * extracts that, then starts the cluster with that data (without clean), and then executes the UpgradeTestIT
+ *
+ * see also
+ * pipelines/upgrade-testing/check_upgrade.sh
+ * pipelines/upgrade-testing/create_upgrade_targz.sh
+ * https://s3.console.aws.amazon.com/s3/buckets/splice-snapshots?region=us-east-1&prefix=upgrade_tests/&showversions=false
+ */
+public class UpgradeTestIT extends SpliceUnitTest {
+
+    private static final String SCHEMA_NAME = UpgradeTestIT.class.getSimpleName().toUpperCase();
+    private static final SpliceWatcher spliceClassWatcher = new SpliceWatcher(SCHEMA_NAME);
+    private static final SpliceSchemaWatcher spliceSchemaWatcher = new SpliceSchemaWatcher(SCHEMA_NAME);
+    @Rule
+    public SpliceWatcher methodWatcher = new SpliceWatcher(SCHEMA_NAME);
+
+    @ClassRule
+    public static TestRule chain = RuleChain.outerRule(spliceClassWatcher)
+            .around(spliceSchemaWatcher);
+
+    @Test // 1983
+    public void testUpgradeScriptToAddBaseTableSchemaColumnsToSysTablesInSYSIBM() throws Exception {
+        String expected =
+                "NAME    | CREATOR |TYPE |COLCOUNT |KEYCOLUMNS | KEYUNIQUE |CODEPAGE | BASE_NAME | BASE_SCHEMA |\n" +
+                "-------------------------------------------------------------------------------------------------\n" +
+                "SYSTABLES | SYSIBM  |  V  |    9    |     0     |     0     |  1208   |   NULL    |    NULL     |";
+        SpliceUnitTest.sqlExpectToString(methodWatcher,
+                "select * from sysibm.SYSTABLES WHERE NAME='SYSTABLES' AND CREATOR='SYSIBM'",
+                expected, false);
+    }
+
+    @Test // 1985
+    public void testUpgradeScriptToAddSysNaturalNumbersTable() throws Exception {
+        Assert.assertEquals(SYSNATURALNUMBERSRowFactory.MAX_NUMBER,
+                methodWatcher.executeGetInt("select count(*) from SYS.SYSNATURALNUMBERS", 1) );
+    }
+
+    @Test // 1989
+    public void testUpgradeScriptToAddIndexColUseViewInSYSCAT() throws Exception {
+        SpliceUnitTest.sqlExpectToString(methodWatcher,
+                "select TABLENAME  from sysvw.systablesview WHERE TABLENAME='INDEXCOLUSE'",
+                        "TABLENAME  |\n" +
+                        "-------------\n" +
+                        "INDEXCOLUSE |",
+                false);
+    }
+
+    @Test // 1992
+    public void testUpgradeScriptForTablePriorities() throws Exception {
+        try(Connection conn = ConnectionFactory.createConnection(new Configuration());
+                Admin admin= conn.getAdmin()) {
+            Assert.assertEquals(0, HBasePartitionAdmin.getToUpgradeStream(admin.listTableDescriptors()).count());
+        }
+    }
+
+}

hbase_storage/src/main/java/com/splicemachine/access/hbase/HBasePartitionAdmin.java

@@ -24,6 +24,7 @@
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import splice.com.google.common.collect.Lists;
 import com.google.protobuf.InvalidProtocolBufferException;
@@ -764,16 +765,19 @@ public static void setHTablePriority(Admin admin, TableName tn,
         admin.enableTable(tn);
     }
 
+    public static Stream<org.apache.hadoop.hbase.client.TableDescriptor> getToUpgradeStream(
+            List<org.apache.hadoop.hbase.client.TableDescriptor> tableDescriptors) {
+        return tableDescriptors.stream().filter( td -> td.getPriority() != getPriorityShouldHave(td));
+    }
+
     public static int upgradeTablePrioritiesFromList(Admin admin,
                                                      List<org.apache.hadoop.hbase.client.TableDescriptor> tableDescriptors)
             throws Exception
     {
         final int NUM_THREADS = 10;
         ExecutorService executor = null;
         try {
-
-            List<Callable<Void>> upgradeTasks = tableDescriptors.stream()
-                    .filter( td -> td.getPriority() != getPriorityShouldHave(td))
+            List<Callable<Void>> upgradeTasks = getToUpgradeStream(tableDescriptors)
                     .map( td -> (Callable<Void>) () -> {
                         setHTablePriority(admin, td.getTableName(), td, getPriorityShouldHave(td));
                         return null;

pipelines/upgrade-testing/check_upgrade.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+VERSION=${1} # e.g. 3.1.0.1971
+shift 1
+
+PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD`
+git stash
+
+# creates a file platform_it_${VERSION}.tar.gz
+git checkout tags/${VERSION}
+cd platform_it
+rm -rf target *.log snappy*.jnilib
+cd ..
+
+./start-splice-cluster $*
+./start-splice-cluster -k
+
+rm -rf upgrade_test_TMP
+mkdir -p upgrade_test_TMP/platform_it/target
+cd upgrade_test_TMP
+cp -r ../platform_it/target/hbase platform_it/target/.
+cp -r ../platform_it/target/zookeeper platform_it/target/.
+tar -czvf ../platform_it_${VERSION}.tar.gz platform_it
+cd ..
+rm -rf upgrade_test_TMP
+
+git checkout ${PREVIOUS_BRANCH}
+git stash pop
+
+# restart on that version
+./start-splice-cluster $*
+
+# clean up platform_it
+cd platform_it
+git clean -dfx
+cd ..
+
+# download the previous standalone data
+tar -xzvf platform_it_${VERSION}.tar.gz
+rm platform_it_${VERSION}.tar.gz
+
+# restart cluster
+./start-splice-cluster -l $*
+
+# test
+if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then
+    echo "UPGRADE SUCCEEDED"
+    cat platform_it/splice.log | grep 'upgrade scripts'
+    cat platform_it/splice.log | grep 'Running upgrade script'	
+else
+	echo "!!! UPGRADE FAILED !!!"
+fi

pipelines/upgrade-testing/check_upgrade_fast.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+if [ $# -lt 2 ]
+then
+	echo "usage: bash create_upgrade_targz.sh {VERSION} {additional start-splice-cluster parameters}"
+	echo "------------------------------------------------------------------------------------------"
+	echo "uses a previously created tar.gz to test upgrade"
+	echo "e.g. bash create_upgrade_targz.sh 3.2.2021 -T 16"
+	echo "make sure you current branch has already been build"
+	echo "don't use -b, since we are deleting some files in platform_it/target"
+
+    exit 1
+fi
+
+VERSION=${1}
+shift
+
+# stop current cluster
+./start-splice-cluster -k
+
+# clean up platform_it
+cd platform_it
+git clean -dfx
+cd ..
+
+# download the previous standalone data
+aws s3 cp ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz .
+tar -xzvf platform_it_${VERSION}.tar.gz
+rm platform_it_${VERSION}.tar.gz
+
+# restart cluster
+./start-splice-cluster -l $*
+
+# test
+if mvn -B -e surefire:test -Pcore,cdh6.3.0 -Dtest='UpgradeTestIT#*' -DskipServerStart -DfailIfNoTests=false; then
+    echo "UPGRADE SUCCEEDED"
+    cat platform_it/splice.log | grep 'upgrade scripts'
+    cat platform_it/splice.log | grep 'Running upgrade script'	
+else
+	echo "!!! UPGRADE FAILED !!!"
+fi

pipelines/upgrade-testing/create_upgrade_targz.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+UPGRADE_URL=s3://splice-snapshots/upgrade_tests
+
+
+if [ $# -lt 2 ]
+then
+    echo "usage: bash platform_it/src/test/bin/create_upgrade_targz.sh {VERSION} {test/upload} {additional start-splice-cluster parameters}"
+    echo "----------------------------------------------------------------------------------------------------------------------------------"
+    echo "creates a tar.gz of a spliceengine standalone cluster that can be used to test upgrade"
+    echo "e.g. bash create_upgrade_targz.sh 3.1.0.1971 test -T 16 -pcore,cdh6.3.0"
+    exit 1
+fi
+
+VERSION=${1} # e.g. 3.1.0.1971
+MODE=${2} # test or upload
+shift 2
+
+
+PREVIOUS_BRANCH=`git rev-parse --abbrev-ref HEAD`
+git stash
+
+# creates a file platform_it_${VERSION}.tar.gz
+git checkout tags/${VERSION}
+cd platform_it
+rm -rf target *.log snappy*.jnilib
+cd ..
+
+./start-splice-cluster $*
+./start-splice-cluster -k
+
+rm -rf upgrade_test_TMP
+mkdir -p upgrade_test_TMP/platform_it/target
+cd upgrade_test_TMP
+cp -r ../platform_it/target/hbase platform_it/target/.
+cp -r ../platform_it/target/zookeeper platform_it/target/.
+tar -czvf ../platform_it_${VERSION}.tar.gz platform_it
+cd ..
+rm -rf upgrade_test_TMP
+
+if [[ $MODE == "upload" ]]; then
+    aws s3 cp platform_it_${VERSION}.tar.gz ${UPGRADE_URL}/platform_it_${VERSION}.tar.gz
+fi
+
+git checkout ${PREVIOUS_BRANCH}
+git stash pop

platform_it/pom.xml

@@ -460,6 +460,8 @@
                             <excludes>
                                 <exclude>com/splicemachine/derby/impl/sql/execute/operations/window/test/framework/**
                                 </exclude>
+                                <exclude>com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java
+                                </exclude>
                             </excludes>
                             <runOrder>alphabetical</runOrder>
                             <systemPropertyVariables>
@@ -495,6 +497,10 @@
                             <includes>
                                 <include>**/*IT.java</include>
                             </includes>
+                            <excludes>
+                                <exclude>com/splicemachine/derby/impl/sql/execute/upgrade/UpgradeTestIT.java
+                                </exclude>
+                            </excludes>
                             <runOrder>alphabetical</runOrder>
                             <systemPropertyVariables>
                                 <splice.schemaCleanup>async</splice.schemaCleanup>

splice_machine/src/test/java/com/splicemachine/derby/test/framework/SpliceWatcher.java

@@ -472,4 +472,13 @@ public void assertStrResult(String res, String query, Boolean sort) throws Excep
         Assert.assertEquals( "failed asserting the results of sql\n" + query,
                 res, executeToString(query, sort));
     }
+
+    public int executeGetInt(String sql, int index) throws SQLException {
+        try( Statement s = getOrCreateConnection().createStatement();
+             ResultSet rs = s.executeQuery(sql))
+        {
+            Assert.assertTrue(rs.next());
+            return rs.getInt(index);
+        }
+    }
 }