PHOENIX-2365 Remove Pherf uber jar and create python scripts to execu…

…te from Phoenix bin directory

bin/pherf-cluster.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+# This script is intended for use where HBase/Phoenix is loaded from HBase classpath
+# therefore HBASE_DIR environment variable needs to be configured for this script to execute
+
+import os
+import subprocess
+import sys
+import phoenix_utils
+
+phoenix_utils.setPath()
+
+args = phoenix_utils.shell_quote(sys.argv[1:])
+
+# HBase configuration folder path (where hbase-site.xml reside) for
+# HBase/Phoenix client side property override
+hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir)
+hbase_path = os.getenv('HBASE_DIR')
+
+java_home = os.getenv('JAVA_HOME')
+
+# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
+hbase_env_path = None
+hbase_env_cmd  = None
+if os.name == 'posix':
+    hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
+    hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
+elif os.name == 'nt':
+    hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
+    hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
+if not hbase_env_path or not hbase_env_cmd:
+    print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+    sys.exit(-1)
+
+hbase_env = {}
+if os.path.isfile(hbase_env_path):
+    p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
+    for x in p.stdout:
+        (k, _, v) = x.partition('=')
+        hbase_env[k.strip()] = v.strip()
+
+if hbase_env.has_key('JAVA_HOME'):
+    java_home = hbase_env['JAVA_HOME']
+
+if java_home:
+    java = os.path.join(java_home, 'bin', 'java')
+else:
+    java = 'java'
+
+print "HBASE_DIR environment variable is currently set to: " + hbase_path
+
+# Get the HBase classpath
+hbasecp, stderr = subprocess.Popen(hbase_path + "/bin/hbase classpath",
+                                  shell=True,
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE).communicate()
+
+java_cmd = java +' -cp "' + hbasecp + os.pathsep + phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_pherf_jar + \
+    '" -Dlog4j.configuration=file:' + \
+    os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
+    " org.apache.phoenix.pherf.Pherf " + args 
+
+exitcode = subprocess.call(java_cmd, shell=True)
+sys.exit(exitcode)

bin/pherf-standalone.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+import os
+import subprocess
+import sys
+import phoenix_utils
+
+phoenix_utils.setPath()
+
+args = phoenix_utils.shell_quote(sys.argv[1:])
+
+# HBase configuration folder path (where hbase-site.xml reside) for
+# HBase/Phoenix client side property override
+hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir)
+
+java_home = os.getenv('JAVA_HOME')
+
+# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
+hbase_env_path = None
+hbase_env_cmd  = None
+if os.name == 'posix':
+    hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
+    hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
+elif os.name == 'nt':
+    hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
+    hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
+if not hbase_env_path or not hbase_env_cmd:
+    print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name
+    sys.exit(-1)
+
+hbase_env = {}
+if os.path.isfile(hbase_env_path):
+    p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
+    for x in p.stdout:
+        (k, _, v) = x.partition('=')
+        hbase_env[k.strip()] = v.strip()
+
+if hbase_env.has_key('JAVA_HOME'):
+    java_home = hbase_env['JAVA_HOME']
+
+if java_home:
+    java = os.path.join(java_home, 'bin', 'java')
+else:
+    java = 'java'
+
+java_cmd = java +' -Xms512m -Xmx3072m  -cp "' + phoenix_utils.pherf_conf_path + os.pathsep + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.phoenix_pherf_jar + \
+    '" -Dlog4j.configuration=file:' + \
+    os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
+    " org.apache.phoenix.pherf.Pherf " + args 
+
+exitcode = subprocess.call(java_cmd, shell=True)
+sys.exit(exitcode)

bin/phoenix_utils.py

@@ -70,6 +70,7 @@ def setPath():
     PHOENIX_QUERYSERVER_JAR_PATTERN = "phoenix-server-*-runnable.jar"
     PHOENIX_TRACESERVER_JAR_PATTERN = "phoenix-tracing-webapp-*-runnable.jar"
     PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar"
+    PHOENIX_PHERF_JAR_PATTERN = "phoenix-pherf-*-minimal*.jar"
 
     # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH
     global phoenix_class_path
@@ -93,6 +94,12 @@ def setPath():
     global current_dir
     current_dir = os.path.dirname(os.path.abspath(__file__))
 
+    global pherf_conf_path
+    pherf_conf_path = os.path.join(current_dir, "config")
+    pherf_properties_file = find("pherf.properties", pherf_conf_path)
+    if pherf_properties_file == "":
+        pherf_conf_path = os.path.join(current_dir, "..", "phoenix-pherf", "config")
+
     global phoenix_jar_path
     phoenix_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
 
@@ -148,6 +155,12 @@ def setPath():
     if phoenix_traceserver_jar == "":
         phoenix_traceserver_jar = findFileInPathWithoutRecursion(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, ".."))
 
+    global phoenix_pherf_jar
+    phoenix_pherf_jar = find(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-pherf", "target", "*"))
+    if phoenix_pherf_jar == "":
+        phoenix_pherf_jar = findFileInPathWithoutRecursion(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, "..", "lib"))
+    if phoenix_pherf_jar == "":
+        phoenix_pherf_jar = findFileInPathWithoutRecursion(PHOENIX_PHERF_JAR_PATTERN, os.path.join(current_dir, ".."))
 
     global phoenix_thin_client_jar
     phoenix_thin_client_jar = find(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server-client", "target", "*"))

dev/make_rc.sh

@@ -37,6 +37,7 @@ REL_SRC=phoenix-$PHOENIX-src
 DIR_REL_SRC_TAR_PATH=$DIR_REL_ROOT/src
 DIR_REL_BIN_TAR_PATH=$DIR_REL_ROOT/bin
 DIR_BIN=$DIR_REL_BIN_PATH/bin
+DIR_PHERF_CONF=phoenix-pherf/config
 DIR_EXAMPLES=$DIR_REL_BIN_PATH/examples
 DIR_DOCS=dev/release_files
 
@@ -75,12 +76,9 @@ rm -rf $(find . -type d -name archive-tmp);
 phx_jars=$(find -iname phoenix-*.jar)
 cp $phx_jars $DIR_REL_BIN_PATH;
 
-# Exclude uber jars as it exceeds max upload limit for Apache SVN until PHOENIX-2248 is fixed
-rm $DIR_REL_BIN_PATH/*webapp*runnable*
-rm $DIR_REL_BIN_PATH/*pherf*dependencies*
-
 # Copy bin
 cp bin/* $DIR_BIN;
+cp -R $DIR_PHERF_CONF $DIR_BIN;
 
 # Copy release docs
 

phoenix-pherf/README.md

@@ -100,6 +100,6 @@ Default quorum is localhost. If you want to override set the system variable.
 
 Run unit tests: `mvn test -DZK_QUORUM=localhost`<br />
 Run a specific method: `mvn -Dtest=ClassName#methodName test` <br />
-To test on a real cluster: `./pherf.sh -drop all -l -q -z localhost -schemaFile .*user_defined_schema.sql -scenarioFile .*user_defined_scenario.xml`
+To test: `./pherf-standalone.py -drop all -l -q -z localhost -schemaFile .*user_defined_schema.sql -scenarioFile .*user_defined_scenario.xml`
 
 More to come...

phoenix-pherf/config/pherf.properties

@@ -36,4 +36,4 @@ pherf.default.results.dir=RESULTS
 pherf.default.summary.file=RESULTS/summary.html
 
 # Threshold for comparator to fail. ex. 0.5 equates to 50%
-pherf.default.comparison.threshold=0.45
+pherf.default.comparison.threshold=0.45

phoenix-pherf/pom.xml

@@ -108,6 +108,10 @@
 			</resource>
 			<resource>
 				<directory>config</directory>
+				<excludes>
+					<exclude>**/*.sql</exclude>
+					<exclude>**/*.xml</exclude>
+				</excludes>
 			</resource>
 		</resources>
 		<testResources>
@@ -177,11 +181,6 @@
 							<descriptors>
 								<!-- Produces minimal Pherf jar -->
 								<descriptor>src/main/assembly/minimal.xml</descriptor>
-								<!-- Produces standalone zip that bundles all required dependencies -->
-								<descriptor>src/main/assembly/standalone.xml</descriptor>
-								<!-- Produces cluster zip with minimal Pherf jar. Setting HBase classpath 
-									in env.sh is required for this configuration -->
-								<descriptor>src/main/assembly/cluster.xml</descriptor>
 							</descriptors>
 						</configuration>
 					</execution>

phoenix-pherf/src/main/java/org/apache/phoenix/pherf/PherfConstants.java

@@ -37,7 +37,7 @@ public enum GeneratePhoenixStats {
     public static final String RESOURCE_SCENARIO = "/scenario";
     public static final String
             SCENARIO_ROOT_PATTERN =
-            ".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*";
+            		".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*" + PherfConstants.RESOURCE_SCENARIO.substring(1) + ".*";
     public static final String SCHEMA_ROOT_PATTERN = ".*";
     public static final String PHERF_PROPERTIES = "pherf.properties";
 

phoenix-pherf/src/main/java/org/apache/phoenix/pherf/rules/RulesApplier.java

@@ -130,7 +130,7 @@ public DataValue getDataValue(Column column) throws Exception{
             prefix = column.getPrefix();
         }
 
-        if (prefix.length() >= length) {
+        if ((prefix.length() >= length) && (length > 0)) {
             logger.warn("You are attempting to generate data with a prefix (" + prefix + ") "
                     + "That is longer than expected overall field length (" + length + "). "
                     + "This will certainly lead to unexpected data values.");