python: tools: Add a tool for displaying a PSI list

Add a tool for displaying the PSI usage (sorted descending) of each
cgroup on the system.

Examples:

$ ./libcg-psilist.py -c cpu -f some-total -l 10
some-total PSI CGROUP
     870310746 /user.slice/user-1000.slice/[email protected]/session.slice/pipewire.service
     822350996 /user.slice/user-1000.slice
     822317743 /user.slice
     769340896 /user.slice/user-1000.slice/[email protected]/session.slice
     709259036 /user.slice/user-1000.slice/session-2.scope
     651866476 /user.slice/user-1000.slice/[email protected]
     359063900 /user.slice/user-1000.slice/[email protected]/session.slice/pipewire-pulse.service
     118937224 /system.slice
      89869438 /system.slice/lightdm.service
      74812217 /user.slice/user-1000.slice/[email protected]/app.slice

$ ./libcg-psilist.py -c cpu -f some-avg10 -l 10 -t 0.0
some-avg10 PSI CGROUP
          0.00 /sys-fs-fuse-connections.mount
          0.00 /sys-kernel-config.mount
          0.00 /sys-kernel-debug.mount
          0.00 /dev-mqueue.mount
          0.00 /user.slice
          0.00 /user.slice/user-1000.slice
          0.00 /user.slice/user-1000.slice/[email protected]
          0.00 /user.slice/user-1000.slice/[email protected]/session.slice
          0.00 /user.slice/user-1000.slice/[email protected]/session.slice/gvfs-goa-volume-monitor.service
          0.00 /user.slice/user-1000.slice/[email protected]/session.slice/xdg-permission-store.service

Signed-off-by: Tom Hromatka <[email protected]>

src/python/Makefile.am

@@ -24,7 +24,7 @@ PY_INSTALL = ${PY_DISTUTILS} install
 
 EXTRA_DIST = cgroup.pxd.m4 libcgroup.pyx.m4 setup.py.m4 \
 	     cgroup.pxd libcgroup.pyx setup.py \
-	     libcgrouputils.py libcgrouptree.py
+	     libcgrouputils.py libcgrouptree.py libcgrouplist.py
 
 if WITH_SYSTEMD
 M4_FLAGS = -D WITH_SYSTEMD

src/python/libcgrouplist.py

@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: LGPL-2.1-only
+#
+# Libcgroup list class
+#
+# Copyright (c) 2021-2024 Oracle and/or its affiliates.
+# Author: Tom Hromatka <[email protected]>
+#
+
+#
+#!/usr/bin/env python3
+#
+# Libcgroup list class
+#
+# Copyright (c) 2021-2024 Oracle and/or its affiliates.
+# Author: Tom Hromatka <[email protected]>
+#
+
+# pip install treelib
+# https://treelib.readthedocs.io/en/latest/
+from treelib import Node, Tree
+import os
+
+from libcgroup import Version
+
+from libcgrouptree import LibcgroupTree
+from libcgrouputils import LibcgroupPid
+
+float_metrics = ['%usr', '%system', '%guest', '%wait', '%CPU', '%MEM', 'minflt/s', 'majflt/s']
+int_metrics = ['Time', 'UID', 'PID', 'CPU', 'RSS', 'threads', 'fd-nr']
+str_metrics = ['Command']
+
+class LibcgroupList(LibcgroupTree):
+    def __init__(self, name, version=Version.CGROUP_V2, controller='cpu', depth=None,
+                 metric='%CPU', threshold=1.0, limit=None):
+        super().__init__(name, version, controller, depth=depth, files=False)
+
+        self.metric = metric
+        self.threshold = threshold
+        self.cgpid_list = list()
+        self.limit = limit
+
+    def walk_action(self, cg):
+        cg.get_pids()
+
+        for pid in cg.pids:
+            cgpid = LibcgroupPid.create_from_pidstat(pid)
+
+            try:
+                cgpid.cgroup = cg.path[len(self.start_path):]
+
+                if self.metric in float_metrics:
+                   if float(cgpid.pidstats[self.metric]) >= self.threshold:
+                        self.cgpid_list.append(cgpid)
+
+                elif self.metric in int_metrics:
+                    if int(cgpid.pidstats[self.metric]) >= self.threshold:
+                        self.cgpid_list.append(cgpid)
+
+                else:
+                    self.cgpid_list.append(cgpid)
+
+            except AttributeError:
+                # The pid could have been deleted between when we read cgroup.procs
+                # and when we ran pidstat.  Ignore it and move on
+                pass
+
+    def sort(self):
+        if self.metric in float_metrics:
+            self.cgpid_list = sorted(self.cgpid_list, reverse=True,
+                                     key=lambda cgpid: float(cgpid.pidstats[self.metric]))
+
+        elif self.metric in int_metrics:
+            self.cgpid_list = sorted(self.cgpid_list, reverse=True,
+                                     key=lambda cgpid: int(cgpid.pidstats[self.metric]))
+
+        else:
+            self.cgpid_list = sorted(self.cgpid_list, reverse=True,
+                                     key=lambda cgpid: cgpid.pidstats[self.metric])
+
+    def show(self, sort=True):
+        if sort:
+            self.sort()
+
+        print('{0: >10} {1: >16}  {2: >8} {3: <50}'.format(
+            'PID', 'COMMAND', self.metric, 'CGROUP'))
+
+        for i, cgpid in enumerate(self.cgpid_list):
+            if self.limit and i >= self.limit:
+                break
+
+            if self.metric in float_metrics:
+                print('{0: >10} {1: >16} {2: 9.2f} {3: <50}'.format(cgpid.pid,
+                      cgpid.pidstats['Command'], float(cgpid.pidstats[self.metric]),
+                      cgpid.cgroup))
+            elif self.metric in int_metrics:
+                print('{0: >10} {1: >16}    {2: 7d} {3: <50}'.format(cgpid.pid,
+                      cgpid.pidstats['Command'], int(cgpid.pidstats[self.metric]),
+                      cgpid.cgroup))
+            else:
+                print('{0: >10} {1: >16}    {2: >6} {3: <50}'.format(cgpid.pid,
+                      cgpid.pidstats['Command'], cgpid.pidstats[self.metric],
+                      cgpid.cgroup))
+
+class LibcgroupPsiList(LibcgroupTree):
+    def __init__(self, name, controller='cpu', depth=None, psi_field='some-avg10',
+                 threshold=None, limit=None):
+        super().__init__(name, version=Version.CGROUP_V2, controller=controller,
+                         depth=depth)
+
+        self.controller = controller
+        self.psi_field = psi_field
+        self.cglist = list()
+        self.threshold = threshold
+        self.limit = limit
+
+    def walk_action(self, cg):
+        cg.get_psi(self.controller)
+
+        if not self.threshold:
+            self.cglist.append(cg)
+        elif cg.psi[self.psi_field] >= self.threshold:
+            self.cglist.append(cg)
+
+    def sort(self):
+        self.cglist = sorted(self.cglist, reverse=True,
+                             key=lambda cg: cg.psi[self.psi_field])
+
+    def _show_float(self):
+        print('{0: >10} {1: >3} {2: <16}'.format(self.psi_field, 'PSI', 'CGROUP'))
+
+        for i, cg in enumerate(self.cglist):
+            if self.limit and i >= self.limit:
+                break
+
+            print('        {0: 6.2f} {1: <16}'.format(cg.psi[self.psi_field],
+                                                      cg.path[len(self.start_path):]))
+
+    def _show_int(self):
+        print('{0: >10} {1: >3} {2: <16}'.format(self.psi_field, 'PSI', 'CGROUP'))
+
+        for i, cg in enumerate(self.cglist):
+            if self.limit and i >= self.limit:
+                break
+
+            print('   {0: 11d} {1: <16}'.format(cg.psi[self.psi_field],
+                                                cg.path[len(self.start_path):]))
+
+    def show(self, sort=True):
+        if sort:
+            self.sort()
+
+        if 'total' in self.psi_field:
+            self._show_int()
+        else:
+            self._show_float()

src/python/libcgrouputils.py

@@ -11,6 +11,54 @@
 
 import subprocess
 
+class LibcgroupPid(object):
+    def __init__(self, pid, command=None):
+        self.pid = pid
+        self.command = command
+        self.pidstats = dict()
+
+    def __str__(self):
+        out_str = 'LibcgroupPid: {}'.format(self.pid)
+        out_str += '\n\tcommand = {}'.format(self.command)
+        for key, value in self.pidstats.items():
+            out_str += '\n\tpidstats[{}] = {}'.format(key, value)
+
+        return out_str
+
+    @staticmethod
+    def create_from_pidstat(pid):
+        cmd = list()
+        cmd.append('pidstat')
+        cmd.append('-H')
+        cmd.append('-h')
+        cmd.append('-r')
+        cmd.append('-u')
+        cmd.append('-v')
+        cmd.append('-p')
+        cmd.append('{}'.format(pid))
+
+        out = run(cmd)
+
+        for line in out.splitlines():
+            if not len(line.strip()):
+                continue
+            if line.startswith('Linux'):
+                # ignore the kernel info
+                continue
+            if line.startswith('#'):
+                line = line.lstrip('#')
+                keys = line.split()
+                continue
+
+            # the last line of pidstat is information regarding the pid
+            values = line.split()
+
+            cgpid = LibcgroupPid(pid)
+            for i, key in enumerate(keys):
+                cgpid.pidstats[key] = values[i]
+
+            return cgpid
+
 def run(command, run_in_shell=False):
     if run_in_shell:
         if isinstance(command, str):

src/tools/Makefile.am

@@ -71,6 +71,7 @@ cgsnapshot_CFLAGS = $(CODE_COVERAGE_CFLAGS) $(EXTRA_CFLAGS)
 
 if ENABLE_PYTHON
 EXTRA_DIST = \
+	cgpsilist.py \
 	cgpsitree.py
 endif
 

src/tools/cgpsilist.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: LGPL-2.1-only
+#
+# Display a list of cgroups and their specified PSI metrics
+#
+# Copyright (c) 2021-2024 Oracle and/or its affiliates.
+# Author: Tom Hromatka <[email protected]>
+#
+
+from libcgrouplist import LibcgroupPsiList
+import argparse
+import os
+
+def parse_args():
+    parser = argparse.ArgumentParser('Libcgroup PSI List')
+    parser.add_argument('-C', '--cgroup', type=str, required=False, default=None,
+                        help='Relative path to the cgroup of interest, e.g. machine.slice/foo.scope')
+    parser.add_argument('-c', '--controller', required=True,
+                        help='PSI controller data to display.  cpu, io, or memory')
+    parser.add_argument('-f', '--field', required=False, default='some-avg10',
+                        help='Which PSI field to display, e.g. some-avg10, full-avg60, ...')
+    parser.add_argument('-d', '--depth', type=int, required=False, default=None,
+                        help='Depth to recurse into the cgroup path.  0 == only this cgroup, 1 == this cgroup and its children, ...')
+    parser.add_argument('-t', '--threshold', type=float, required=False, default=1.0,
+                        help='Only list cgroups whose PSI exceeds this percentage')
+    parser.add_argument('-l', '--limit', type=int, required=False, default=None,
+                        help='Only display the first N cgroups. If not provided, all cgroups that match are displayed')
+
+    args = parser.parse_args()
+
+    return args
+
+def main(args):
+    cglist = LibcgroupPsiList(args.cgroup, controller=args.controller, depth=args.depth,
+                              psi_field=args.field, threshold=args.threshold, limit=args.limit)
+
+    cglist.walk()
+    cglist.show()
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(args)