Merge pull request DataDog#1410 from DataDog/yann/jmx-default-heap-sizes

[jmx] default heap size options

conf.d/activemq.yaml.example

@@ -8,6 +8,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: activemq_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

conf.d/activemq_58.yaml.example

@@ -8,6 +8,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: activemq_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

conf.d/cassandra.yaml.example

@@ -8,6 +8,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: cassandra_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

conf.d/jmx.yaml.example

@@ -10,6 +10,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: jmx_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

conf.d/solr.yaml.example

@@ -8,6 +8,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: solr_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

conf.d/tomcat.yaml.example

@@ -8,6 +8,7 @@ instances:
   #   tools_jar_path: /usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar # To be set when process_name_regex is set
   #   name: tomcat_instance
   #   # java_bin_path: /path/to/java # Optional, should be set if the agent cannot find your java executable
+  #   # java_options: "-Xmx200m -Xms50m" # Optional, Java JVM options
   #   # trust_store_path: /path/to/trustStore.jks # Optional, should be set if ssl is enabled
   #   # trust_store_password: password
   #   tags:

jmxfetch.py

@@ -26,6 +26,8 @@
 }
 
 JMX_FETCH_JAR_NAME = "jmxfetch-0.4.1-jar-with-dependencies.jar"
+_JVM_DEFAULT_MAX_MEMORY_ALLOCATION = " -Xmx200m"
+_JVM_DEFAULT_INITIAL_MEMORY_ALLOCATION = " -Xms50m"
 JMXFETCH_MAIN_CLASS = "org.datadog.jmxfetch.App"
 JMX_CHECKS = [
     'activemq',
@@ -347,14 +349,19 @@ def start(cls, confd_path, agentConfig, logging_config, path_to_java, java_run_o
                 command, # Name of the command
             ]
 
-
             subprocess_args.insert(4, '--check')
             for check in jmx_checks:
                 subprocess_args.insert(5, check)
 
-            if java_run_opts:
-                for opt in java_run_opts.split():
-                    subprocess_args.insert(1,opt)
+            # Specify a maximum memory allocation pool for the JVM
+            if "Xmx" not in java_run_opts and "XX:MaxHeapSize" not in java_run_opts:
+                java_run_opts += _JVM_DEFAULT_MAX_MEMORY_ALLOCATION
+            # Specify the initial memory allocation pool for the JVM
+            if "Xms" not in java_run_opts and "XX:InitialHeapSize" not in java_run_opts:
+                java_run_opts += _JVM_DEFAULT_INITIAL_MEMORY_ALLOCATION
+
+            for opt in java_run_opts.split():
+                subprocess_args.insert(1, opt)
 
             log.info("Running %s" % " ".join(subprocess_args))
             if reporter != "console":

tests/common.py

@@ -170,7 +170,6 @@ def _candidates_size_assert(self, candidates, count=None, tolerance=1):
             self.print_current_state()
             raise
 
-
     def assertMetric(self, metric_name, metric_value=None, tags=None, count=None):
         candidates = []
         for m_name, ts, val, mdata in self.metrics:
@@ -184,7 +183,6 @@ def assertMetric(self, metric_name, metric_value=None, tags=None, count=None):
 
         self._candidates_size_assert(candidates, count=count)
 
-
     def assertMetricTagPrefix(self, metric_name, tag_prefix, count=None):
         candidates = []
         for m_name, ts, val, mdata in self.metrics:
@@ -219,3 +217,9 @@ def assertServiceCheck(self, service_check_name, status=None, tags=None, count=N
                 candidates.append(sc)
 
         self._candidates_size_assert(candidates, count=count)
+
+    def assertIn(self, first, second):
+        self.assertTrue(first in second, "{0} not in {1}".format(first, second))
+
+    def assertNotIn(self, first, second):
+        self.assertTrue(first not in second, "{0} in {1}".format(first, second))

tests/test_java_jmx.py

@@ -1,15 +1,26 @@
+# stdlib
 import unittest
-from nose.plugins.attrib import attr
 import time
 import threading
+import os
+import tempfile
+from nose.plugins.attrib import attr
+from mock import patch
+
+# datadog
 from aggregator import MetricsAggregator
-from dogstatsd import Dogstatsd, init, Server
+from dogstatsd import Server
 from util import PidFile
-import os
 from config import get_logging_config
 from jmxfetch import JMXFetch, JMX_COLLECT_COMMAND
+from common import AgentCheckTest
+
+# 3rd party
+import yaml
 
 STATSD_PORT = 8129
+
+
 class DummyReporter(threading.Thread):
     def __init__(self, metrics_aggregator):
         threading.Thread.__init__(self)
@@ -20,7 +31,6 @@ def __init__(self, metrics_aggregator):
         self.finished = False
         self.start()
 
-
     def run(self):
         while not self.finished:
             time.sleep(self.interval)
@@ -31,6 +41,59 @@ def flush(self):
         if metrics:
             self.metrics = metrics
 
+
+@attr('local')
+class JMXInitTest(AgentCheckTest):
+    CHECK_NAME = "java_jmx"
+
+    @patch("subprocess.call")
+    def _get_jmxfetch_subprocess_args(self, yaml_jmx_conf, mock_subprocess_call):
+        # Helper function
+        # Returns the Java JMX subprocess_args called from a YAML configuration
+        tmp_dir = tempfile.mkdtemp()
+        filename = "jmx.yaml"
+        with open(os.path.join(tmp_dir, filename), 'wb') as temp_file:
+            temp_file.write(yaml.dump(yaml_jmx_conf))
+
+        JMXFetch.init(tmp_dir, {}, {}, 15, None, reporter="console")
+        return mock_subprocess_call.call_args[0][0]
+
+    def _get_jmx_conf(self, java_options):
+        return {
+            'instances': [{
+                'host': "localhost",
+                'port': 7199,
+                'java_options': java_options
+            }]
+        }
+
+    def assertJavaRunsWith(self, yaml_conf, include=[], exclude=[]):
+        subprocess_args = self._get_jmxfetch_subprocess_args(yaml_conf)
+        for i in include:
+            self.assertIn(i, subprocess_args)
+        for e in exclude:
+            self.assertNotIn(e, subprocess_args)
+
+    def test_jmx_start(self):
+        # Empty java_options
+        jmx_conf = self._get_jmx_conf("")
+        self.assertJavaRunsWith(jmx_conf, ['-Xms50m', '-Xmx200m'])
+
+        # Specified initial memory allocation pool for the JVM
+        jmx_conf = self._get_jmx_conf("-Xms10m")
+        self.assertJavaRunsWith(jmx_conf, ['-Xms10m', '-Xmx200m'], ['-Xms50m'])
+
+        jmx_conf = self._get_jmx_conf("-XX:InitialHeapSize=128m")
+        self.assertJavaRunsWith(jmx_conf, ['-XX:InitialHeapSize=128m', '-Xmx200m'], ['-Xms50m'])
+
+        # Specified maximum memory allocation pool for the JVM
+        jmx_conf = self._get_jmx_conf("-Xmx500m")
+        self.assertJavaRunsWith(jmx_conf, ['-Xms50m', '-Xmx500m'], ['-Xmx200m'])
+
+        jmx_conf = self._get_jmx_conf("-XX:MaxHeapSize=500m")
+        self.assertJavaRunsWith(jmx_conf, ['-Xms50m', '-XX:MaxHeapSize=500m'], ['-Xmx200m'])
+
+
 @attr(requires='tomcat')
 class JMXTestCase(unittest.TestCase):
     def setUp(self):
@@ -45,13 +108,11 @@ def setUp(self):
         confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
         JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
 
-
     def tearDown(self):
         self.server.stop()
         self.reporter.finished = True
         JMXFetch.stop()
 
-
     def testCustomJMXMetric(self):
         count = 0
         while self.reporter.metrics is None: