Merge branch 'refs/heads/upstream-HEAD' into repo-HEAD

.github/workflows/ci.yml

@@ -39,9 +39,9 @@ jobs:
       CC: ${{ matrix.cc }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
@@ -67,9 +67,9 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install dependencies

.github/workflows/vmtest-build.yml

@@ -21,7 +21,7 @@ jobs:
     env:
       GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           sudo apt-get update
@@ -31,9 +31,9 @@ jobs:
         run: python3 -m vmtest.manage --kernel-directory build/vmtest/linux.git --build-directory build/vmtest/kbuild -K -a ${{ matrix.arch }} -f ${{ matrix.flavor }}
       - name: Upload kernel build logs
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: kernel-build-logs
+          name: kernel-build-logs-${{ matrix.arch }}-${{ matrix.flavor }}
           path: build/vmtest/kbuild/*.log
           if-no-files-found: ignore
   test:

_drgn.pyi

@@ -460,6 +460,33 @@ class Program:
             return an :class:`Object` or ``None`` if not found.
         """
         ...
+    def add_symbol_finder(
+        self, fn: Callable[[Optional[str], Optional[int], bool], Sequence[Symbol]]
+    ) -> None:
+        """
+        Register a callback for finding symbols in the program.
+
+        The callback should take three arguments: a search name, a search
+        address, and a boolean flag 'one' indicating whether to return only
+        the single best match. When the 'one' flag is True, the callback should
+        return a list containing at most one :class:`Symbol`. When the flag is
+        False, the callback should return a list of all matching
+        :class:`Symbol`\\ s. Both the name and address arguments are optional.
+        If both are provided, then the result(s) should match both. If neither
+        are provided, the finder should return all available symbols. If no
+        result is found, the return should be an empty list.
+
+        Callbacks are called in reverse order of the order they were added
+        (i.e,, the most recently added callback is called first). When the
+        'one' flag is set, the search will short-circuit after the first
+        finder which returns a result, and subsequent finders will not be
+        called. Otherwise, all callbacks will be called, and all results will be
+        returned.
+
+        :param fn: Callable taking name, address, and 'one' flag, and
+            returning a sequence of :class:`Symbol`\\ s.
+        """
+        ...
     def set_core_dump(self, path: Union[Path, int]) -> None:
         """
         Set the program to a core dump.

tools/bpf_inspect.py → contrib/bpf_inspect.py

@@ -2,7 +2,7 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-# type: ignore
+"""List BPF programs or maps and their properties unavailable via kernel API."""
 
 import argparse
 

docs/exts/drgndoc/format.py

@@ -39,10 +39,16 @@ def __init__(
         self._parts: List[str] = []
 
     def visit(  # type: ignore[override]  # This is intentionally incompatible with the supertype.
-        self, node: ast.AST, rst: bool, qualify_typing: bool
+        self,
+        node: ast.AST,
+        *,
+        rst: bool,
+        qualify_typing: bool,
+        qualify_typeshed: bool,
     ) -> str:
         self._rst = rst
         self._qualify_typing = qualify_typing
+        self._qualify_typeshed = qualify_typeshed
         super().visit(node)
         ret = "".join(self._parts)
         self._parts.clear()
@@ -97,6 +103,8 @@ def _append_resolved_name(self, name: str) -> None:
         title = target
         if not self._qualify_typing and title.startswith("typing."):
             title = title[len("typing.") :]
+        elif not self._qualify_typeshed and title.startswith("_typeshed."):
+            title = title[len("_typeshed.") :]
         elif self._context_module and title.startswith(self._context_module + "."):
             title = title[len(self._context_module) + 1 :]
             if self._context_class and title.startswith(self._context_class + "."):
@@ -284,12 +292,23 @@ def visit_arg(
             default_sep = "="
             if arg.annotation:
                 signature.append(": ")
-                signature.append(visitor.visit(arg.annotation, False, rst))
+                signature.append(
+                    visitor.visit(
+                        arg.annotation,
+                        rst=False,
+                        qualify_typing=rst,
+                        qualify_typeshed=False,
+                    )
+                )
                 default_sep = " = "
 
             if default:
                 signature.append(default_sep)
-                signature.append(visitor.visit(default, False, True))
+                signature.append(
+                    visitor.visit(
+                        default, rst=False, qualify_typing=True, qualify_typeshed=True
+                    )
+                )
             need_comma = True
 
         posonlyargs = getattr(node.args, "posonlyargs", [])
@@ -328,7 +347,11 @@ def visit_arg(
 
         if want_rtype and node.returns:
             signature.append(" -> ")
-            signature.append(visitor.visit(node.returns, False, rst))
+            signature.append(
+                visitor.visit(
+                    node.returns, rst=False, qualify_typing=rst, qualify_typeshed=False
+                )
+            )
 
         return "".join(signature), lines
 
@@ -356,7 +379,12 @@ def _format_class(
                 context_module,
                 context_class,
             )
-            bases = [visitor.visit(base, rst, False) for base in node.bases]
+            bases = [
+                visitor.visit(
+                    base, rst=rst, qualify_typing=False, qualify_typeshed=False
+                )
+                for base in node.bases
+            ]
             if lines:
                 lines.append("")
             lines.append(("    " if rst else "") + "Bases: " + ", ".join(bases))
@@ -489,7 +517,13 @@ def _format_variable(
             lines = [f".. {directive}:: {name}"]
             if node.annotation:
                 lines.append(
-                    "    :type: " + visitor.visit(node.annotation, False, True)
+                    "    :type: "
+                    + visitor.visit(
+                        node.annotation,
+                        rst=False,
+                        qualify_typing=True,
+                        qualify_typeshed=False,
+                    )
                 )
             if docstring_lines:
                 lines.append("")
@@ -500,7 +534,15 @@ def _format_variable(
             if node.annotation:
                 if docstring_lines:
                     docstring_lines.insert(0, "")
-                docstring_lines.insert(0, visitor.visit(node.annotation, False, False))
+                docstring_lines.insert(
+                    0,
+                    visitor.visit(
+                        node.annotation,
+                        rst=False,
+                        qualify_typing=False,
+                        qualify_typeshed=False,
+                    ),
+                )
             return docstring_lines
 
     def format(

docs/release_highlights.rst

@@ -6,6 +6,7 @@ from the full `release notes <https://github.com/osandov/drgn/releases>`_.
 
 .. toctree::
 
+   release_highlights/0.0.26.rst
    release_highlights/0.0.25.rst
    release_highlights/0.0.24.rst
    release_highlights/0.0.23.rst

docs/release_highlights/0.0.26.rst

@@ -0,0 +1,100 @@
+0.0.26 (Released March 11th, 2024)
+====================================
+
+These are some of the highlights of drgn 0.0.26. See the `GitHub release
+<https://github.com/osandov/drgn/releases/tag/v0.0.26>`_ for the full release
+notes, including more improvements and bug fixes.
+
+.. highlight:: pycon
+
+Miscellaneous Helpers
+---------------------
+
+This release added several new Linux kernel helpers with no particular theme:
+
+- :func:`~drgn.helpers.linux.printk.print_dmesg()`, a shortcut for printing the
+  kernel log buffer.
+- :func:`~drgn.helpers.linux.idr.idr_for_each_entry()`, a shortcut for
+  iterating over an IDR and casting its entries to a specific type.
+- :func:`~drgn.helpers.linux.stackdepot.stack_depot_fetch()` for getting stack
+  traces from the storage used by KASAN and other kernel debugging tools. This
+  was contributed by Peter Collingbourne.
+- :func:`~drgn.helpers.linux.plist.plist_head_empty()`,
+  :func:`~drgn.helpers.linux.plist.plist_node_empty()`,
+  :func:`~drgn.helpers.linux.plist.plist_first_entry()`,
+  :func:`~drgn.helpers.linux.plist.plist_last_entry()`,
+  :func:`~drgn.helpers.linux.plist.plist_for_each()`, and
+  :func:`~drgn.helpers.linux.plist.plist_for_each_entry()`, helpers for working
+  with the kernel's priority-sorted lists.
+
+``fsrefs.py`` Tool
+------------------
+
+The ``fsrefs.py`` tool was added to the ``tools`` directory. It prints
+information about everything that is referencing a file or filesystem. This is
+similar to :manpage:`fuser(1)` and :manpage:`lsof(8)`, but it can find more
+since it has access to kernel internals.
+
+.. code-block:: console
+
+    $ ./tools/fsrefs.py --inode /dev/urandom
+    pid 1349 (bluetoothd) fd 16 (struct file *)0xffff8881458cf000
+    pid 1368 (udisksd) fd 15 (struct file *)0xffff888145c13100
+    ...
+    $ ./tools/fsrefs.py --super-block /run
+    mount /run (struct mount *)0xffff8881015cc140
+    pid 1 (systemd) fd 256 (struct file *)0xffff8881012f3d00 /run/initctl
+    pid 1 (systemd) fd 380 (struct file *)0xffff88810bf88800 /run/dmeventd-server
+    pid 1 (systemd) fd 385 (struct file *)0xffff88810bf88f00 /run/dmeventd-client
+    mount /run (mount namespace 4026532545) (struct mount *)0xffff8881474028c0
+    pid 2135770 (systemd-journal) vma 0x7f7d94f2a000-0x7f7d94f2b000 (struct file *)0xffff88813925bf00 /run/systemd/journal/kernel-seqnum
+    pid 2135770 (systemd-journal) vma 0x7f7d94f2b000-0x7f7d94f2c000 (struct file *)0xffff88813925a100 /run/systemd/journal/seqnum
+    ...
+
+``fsrefs.py`` currently checks:
+
+- File descriptors
+- Task working directories
+- Task root directories
+- Memory mappings
+- Filesystem mounts
+- `binfmt_misc <https://docs.kernel.org/admin-guide/binfmt-misc.html>`_
+- :manpage:`loop(4)` devices
+- Swap files
+- `uprobes <https://docs.kernel.org/trace/uprobetracer.html>`_
+
+It will be extended to check more as the need arises, so feel free to report
+anything it missed.
+
+(Note that as opposed to the ``contrib`` directory, scripts in the ``tools``
+directory are regularly maintained and tested.)
+
+DWARF Package Files
+-------------------
+
+drgn now supports split DWARF package (.dwp) files. These are generated by the
+``dwp`` and ``llvm-dwp`` tools.
+
+Linux 6.8 Support
+-----------------
+
+Linux 6.8 changed some filesystem internals in a way that broke a couple of
+drgn helpers. Here are some errors you might see with older versions of drgn
+that are fixed in this release.
+
+From :func:`~drgn.helpers.linux.fs.path_lookup()` or
+:func:`~drgn.helpers.linux.fs.for_each_mount()` (fixed by Johannes Thumshirn)::
+
+    AttributeError: 'struct mnt_namespace' has no member 'list'
+
+From :func:`~drgn.helpers.linux.fs.path_lookup()`::
+
+    AttributeError: 'struct dentry' has no member 'd_subdirs'
+
+Python 3.13 Support
+-------------------
+
+Python 3.13, currently in alpha, removed or changed some private APIs
+(``_PyDict_GetItemIdWithError()``, ``_PyDict_SetItemId()``, and
+``_PyLong_AsByteArray()``) that drgn depended on, which caused build failures.
+This was fixed by using public APIs instead.

drgn/helpers/linux/printk.py

@@ -9,14 +9,19 @@
 kernel log buffer.
 """
 
-from typing import Dict, List, NamedTuple, Optional, Tuple
+import sys
+from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple
+
+if TYPE_CHECKING:
+    from _typeshed import SupportsWrite
 
 from drgn import Object, Program, cast, sizeof
 from drgn.helpers.common.prog import takes_program_or_default
 
 __all__ = (
     "get_dmesg",
     "get_printk_records",
+    "print_dmesg",
 )
 
 
@@ -243,6 +248,8 @@ def get_dmesg(prog: Program) -> bytes:
     Get the contents of the kernel log buffer formatted like
     :manpage:`dmesg(1)`.
 
+    If you just want to print the log buffer, use :func:`print_dmesg()`.
+
     The format of each line is:
 
     .. code-block::
@@ -263,3 +270,21 @@ def get_dmesg(prog: Program) -> bytes:
     ]
     lines.append(b"")  # So we get a trailing newline.
     return b"\n".join(lines)
+
+
+@takes_program_or_default
+def print_dmesg(prog: Program, *, file: "Optional[SupportsWrite[str]]" = None) -> None:
+    """
+    Print the contents of the kernel log buffer.
+
+    >>> print_dmesg()
+    [    0.000000] Linux version 6.8.0-vmtest28.1default (drgn@drgn) (x86_64-linux-gcc (GCC) 12.2.0, GNU ld (GNU Binutils) 2.39) #1 SMP PREEMPT_DYNAMIC Mon Mar 11 06:38:45 UTC 2024
+    [    0.000000] Command line: rootfstype=9p rootflags=trans=virtio,cache=loose,msize=1048576 ro console=ttyS0,115200 panic=-1 crashkernel=256M init=/tmp/drgn-vmtest-rudzppeo/init
+    [    0.000000] BIOS-provided physical RAM map:
+    ...
+
+    :param file: File to print to. Defaults to :data:`sys.stdout`.
+    """
+    (sys.stdout if file is None else file).write(
+        get_dmesg(prog).decode(errors="replace")
+    )

libdrgn/configure.ac

@@ -1,7 +1,7 @@
 dnl Copyright (c) Meta Platforms, Inc. and affiliates.
 dnl SPDX-License-Identifier: LGPL-2.1-or-later
 
-AC_INIT([libdrgn], [0.0.25],
+AC_INIT([libdrgn], [0.0.26],
 	[https://github.com/osandov/drgn/issues],,
 	[https://github.com/osandov/drgn])