Merge pull request #216 from Helene/profiler

Add profiler module

source/analytics.py

@@ -35,3 +35,6 @@
 
 global cherrypy_internal_stats
 cherrypy_internal_stats = False
+
+global runtime_profiling
+runtime_profiling = False

source/collector.py

@@ -30,7 +30,7 @@
 from threading import Thread
 from metadata import MetadataHandler
 from bridgeLogger import getBridgeLogger
-from utils import classattributes, cond_execution_time
+from utils import classattributes, cond_execution_time, get_runtime_statistics
 
 
 local_cache = set()
@@ -86,6 +86,7 @@ def parse_tags(self, filtersMap):
             else:
                 self.tags[_key] = _values.pop()
 
+    @get_runtime_statistics(enabled=analytics.runtime_profiling)
     def reduce_dps_to_first_not_none(self, reverse_order=False):
         """Reduce multiple data points(dps) of a single
            TimeSeries to the first non null value in a sorted order.

source/profiler.py

@@ -0,0 +1,85 @@
+'''
+##############################################################################
+# Copyright 2024 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##############################################################################
+
+Created on May 15, 2024
+
+@author: HWASSMAN
+'''
+
+import cherrypy
+import io
+import os
+from cProfile import Profile
+from pstats import SortKey, Stats
+from metaclasses import Singleton
+
+
+class Profiler(metaclass=Singleton):
+    exposed = True
+
+    def __init__(self, path=None):
+        if not path:
+            path = os.path.join(os.path.dirname(__file__), 'profile')
+        self.path = path
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    def run(self, func, *args, **kwargs):
+        """Dump profile data into self.path."""
+        with Profile() as profile:
+            filename = f"profiling_{func.__name__}.prof"
+            result = func(*args, **kwargs)
+            (
+                Stats(profile)
+                .strip_dirs()
+                .sort_stats(SortKey.CALLS)
+                .dump_stats(os.path.join(self.path, filename))
+            )
+        return result
+
+    def statfiles(self):
+        """ Returns a list of available profiling files"""
+        return [f for f in os.listdir(self.path)
+                if f.startswith('profiling_') and f.endswith('.prof')]
+
+    def stats(self, filename, sortby='cumulative'):
+        """ Returns output of print_stats() for the given profiling file"""
+        sio = io.StringIO()
+        s = Stats(os.path.join(self.path, filename), stream=sio)
+        s.strip_dirs()
+        s.sort_stats(sortby)
+        s.print_stats()
+        response = sio.getvalue().splitlines()
+        sio.close()
+        return response
+
+    def GET(self, **params):
+        """ Forward GET REST HTTP/s API incoming requests to Profiler
+            available endpoints:
+                            /profiling
+        """
+        resp = []
+        # /profiling
+        if '/profiling' == cherrypy.request.script_name:
+            del cherrypy.response.headers['Content-Type']
+            outp = []
+            runs = self.statfiles()
+            for name in runs:
+                outp.extend(self.stats(filename=name))
+            resp = '\n'.join(outp) + '\n'
+        cherrypy.response.headers['Content-Type'] = 'text/plain'
+        return resp

source/queryHandler/QueryHandler.py

@@ -28,7 +28,7 @@
 from collections import namedtuple, defaultdict
 from itertools import chain
 from typing import NamedTuple, Tuple
-from utils import cond_execution_time
+from utils import cond_execution_time, get_runtime_statistics
 
 from .PerfmonRESTclient import perfHTTPrequestHelper, createRequestDataObj, getAuthHandler
 
@@ -176,10 +176,12 @@ def __init__(self, query, res_json):
             for row in self.rows:
                 self._add_calculated_row_data(calc, row)
 
+    @get_runtime_statistics(enabled=analytics.runtime_profiling)
     def __parseHeader(self):
         item = self.json['header']
         return HeaderData(**item)
 
+    @get_runtime_statistics(enabled=analytics.runtime_profiling)
     def __parseLegend(self):
         legendItems = self.json['legend']
         columnInfos = []

source/utils.py

@@ -25,6 +25,7 @@
 from typing import Callable, TypeVar, Any
 from functools import wraps
 from messages import MSG
+from profiler import Profiler
 
 T = TypeVar('T')
 
@@ -87,6 +88,22 @@ def no_outer(f: Callable[..., T]) -> Callable[..., T]:
     return outer if enabled else no_outer
 
 
+def get_runtime_statistics(enabled: bool = False) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """ Conditionally executes the passed through function f with profiling."""
+
+    def outer(f: Callable[..., T]) -> Callable[..., T]:
+        @wraps(f)
+        def wrapper(*args: Any, **kwargs: Any) -> T:
+            profiler = Profiler()
+            result = profiler.run(f, *args, **kwargs)
+            return result
+        return wrapper
+
+    def no_outer(f: Callable[..., T]) -> Callable[..., T]:
+        return f
+    return outer if enabled else no_outer
+
+
 def classattributes(default_attr: dict, more_allowed_attr: list):
     """ class __init__decorator
         Parses kwargs attributes, for optional arguments uses default values,

source/zimonGrafanaIntf.py

@@ -38,6 +38,7 @@
 from metadata import MetadataHandler
 from opentsdb import OpenTsdbApi
 from prometheus import PrometheusExporter
+from profiler import Profiler
 from watcher import ConfigWatcher
 from cherrypy import _cperror
 from cherrypy.lib.cpstats import StatsPage
@@ -342,6 +343,13 @@ def main(argv):
                                      }
                                     )
         registered_apps.append("Prometheus Exporter Api listening on Prometheus requests")
+    profiler = Profiler(args.get('logPath'))
+    # query for list configured zimon sensors
+    cherrypy.tree.mount(profiler, '/profiling',
+                        {'/':
+                         {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
+                         }
+                        )
     if analytics.cherrypy_internal_stats:
         cherrypy.tree.mount(StatsPage(), '/cherrypy_internal_stats')
     logger.info("%s", MSG['sysStart'].format(sys.version, cherrypy.__version__))