Use _Py_DebugOffsets instead of hardcoded offsets when possible

.github/workflows/build_wheels.yml

@@ -152,6 +152,49 @@ jobs:
           PYTHON_TEST_VERSION: "auto"
         run: python${{matrix.python_version}} -m pytest tests -k 'not 2.7' -n auto -vvv
 
+  test_free_threading:
+    needs: [build_wheels]
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{matrix.python_version}}-dev"
+      - uses: actions/download-artifact@v4
+        with:
+          name: "manylinux_x86_64-wheels"
+          path: dist
+      - name: Set up dependencies
+        run: |
+          sudo add-apt-repository ppa:deadsnakes/ppa
+          sudo apt-get update
+          sudo apt-get install -qy \
+            gdb \
+            python${{matrix.python_version}}-dev \
+            python${{matrix.python_version}}-nogil \
+            python${{matrix.python_version}}-venv
+      - name: Install Python dependencies
+        run: |
+          python${{matrix.python_version}} -m pip install --upgrade pip
+          python${{matrix.python_version}} -m pip install -r requirements-test.txt
+          python${{matrix.python_version}} -m pip install --no-index --find-links=dist/ --only-binary=pystack pystack
+      - name: Install setuptools for the free-threading version
+        run: |
+          python${{matrix.python_version}}t -m venv --system-site-packages /tmp/pip${{matrix.python_version}}
+          /tmp/pip${{matrix.python_version}}/bin/pip install --user setuptools
+      - name: Disable ptrace security restrictions
+        run: |
+          echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+      - name: Run pytest
+        env:
+          PYTHON_TEST_VERSION: "${{matrix.python_version}}t"
+        run: python${{matrix.python_version}} -m pytest tests -k 'not 2.7' -n auto -vvv
+
   test_in_alpine:
     needs: [build_wheels]
     runs-on: ubuntu-latest

news/206.feature.rst

@@ -0,0 +1 @@
+Support debugging free-threading (a.k.a. "nogil") Python 3.13 builds. Note that PyStack can't itself be run with ``python3.13t``, it can only attach to a ``python3.13t`` process or core file from another interpreter.

pyproject.toml

@@ -116,3 +116,8 @@ omit = [
 
 [tool.coverage.report]
 show_missing = true
+
+[tool.pytest.ini_options]
+# pytest retains all temp files from the last 3 test suite runs by default.
+# Keep only ones for failed tests to avoid filling up a disk.
+tmp_path_retention_policy = "failed"

src/pystack/_pystack.pyi

@@ -36,6 +36,7 @@ class StackMethod(enum.Enum):
     ELF_DATA: int
     HEAP: int
     SYMBOLS: int
+    DEBUG_OFFSETS: int
 
 class ProcessManager: ...
 

src/pystack/_pystack.pyx

@@ -77,7 +77,8 @@ class StackMethod(enum.Enum):
     BSS = 1 << 2
     ANONYMOUS_MAPS = 1 << 3
     HEAP = 1 << 4
-    AUTO = ELF_DATA | SYMBOLS | BSS
+    DEBUG_OFFSETS = 1 << 5
+    AUTO = DEBUG_OFFSETS | ELF_DATA | SYMBOLS | BSS
     ALL = AUTO | ANONYMOUS_MAPS | HEAP
 
 
@@ -307,6 +308,7 @@ cdef class ProcessManager:
             )
         )
 
+        native_manager.get().setPythonVersionFromDebugOffsets()
         python_version = native_manager.get().findPythonVersion()
         if python_version == (-1, -1):
             python_version = get_python_version_for_process(pid, map_info)
@@ -353,10 +355,12 @@ cdef class ProcessManager:
             make_shared[CoreFileProcessManager](pid, analyzer, maps, native_map_info)
         )
 
+        native_manager.get().setPythonVersionFromDebugOffsets()
         python_version = native_manager.get().findPythonVersion()
         if python_version == (-1, -1):
             python_version = get_python_version_for_core(core_file, executable, map_info)
         native_manager.get().setPythonVersion(python_version)
+
         cdef ProcessManager new_manager = cls(
             pid, python_version, virtual_maps, map_info
         )
@@ -527,6 +531,7 @@ cdef remote_addr_t _get_interpreter_state_addr(
 ) except*:
     cdef remote_addr_t head = 0
     possible_methods = [
+        StackMethod.DEBUG_OFFSETS,
         StackMethod.ELF_DATA,
         StackMethod.SYMBOLS,
         StackMethod.BSS,
@@ -539,7 +544,10 @@ cdef remote_addr_t _get_interpreter_state_addr(
             continue
 
         try:
-            if possible_method == StackMethod.ELF_DATA:
+            if possible_method == StackMethod.DEBUG_OFFSETS:
+                how = "using debug offsets data"
+                head = manager.findInterpreterStateFromDebugOffsets()
+            elif possible_method == StackMethod.ELF_DATA:
                 how = "using ELF data"
                 head = manager.findInterpreterStateFromElfData()
             elif possible_method == StackMethod.SYMBOLS:

src/pystack/_pystack/cpython/code.h

@@ -197,14 +197,4 @@ typedef struct
 } PyCodeObject;
 }  // namespace Python3_13
 
-typedef union {
-    Python2::PyCodeObject v2;
-    Python3_3::PyCodeObject v3_3;
-    Python3_6::PyCodeObject v3_6;
-    Python3_8::PyCodeObject v3_8;
-    Python3_11::PyCodeObject v3_11;
-    Python3_12::PyCodeObject v3_12;
-    Python3_13::PyCodeObject v3_13;
-} PyCodeObject;
-
 }  // namespace pystack

src/pystack/_pystack/cpython/dict.h

@@ -29,7 +29,7 @@ typedef struct _dictobject
 
 namespace Python3 {
 typedef Py_ssize_t (*dict_lookup_func)(void* mp, PyObject* key, Py_hash_t hash, PyObject** value_addr);
-union PyDictKeysObject;
+struct PyDictKeysObject;
 
 typedef struct
 {
@@ -82,7 +82,7 @@ typedef struct _dictkeysobject
     uint32_t dk_version;
     Py_ssize_t dk_usable;
     Py_ssize_t dk_nentries;
-    char dk_indices[]; /* char is required to avoid strict aliasing. */
+    char dk_indices[1]; /* char is required to avoid strict aliasing. */
 } PyDictKeysObject;
 }  // namespace Python3_11
 
@@ -99,14 +99,4 @@ typedef struct _dictvalues
 
 }  // namespace Python3_13
 
-typedef union {
-    Python3_3::PyDictKeysObject v3_3;
-    Python3_11::PyDictKeysObject v3_11;
-} PyDictKeysObject;
-
-typedef union {
-    Python3::PyDictValuesObject v3_3;
-    Python3_13::PyDictValuesObject v3_13;
-} PyDictValuesObject;
-
 }  // namespace pystack

src/pystack/_pystack/cpython/frame.h

@@ -39,7 +39,7 @@ namespace Python3_7 {
 typedef struct _pyframeobject
 {
     PyObject_VAR_HEAD struct _pyframeobject* f_back;
-    PyCodeObject* f_code;
+    PyObject* f_code;
     PyObject* f_builtins;
     PyObject* f_globals;
     PyObject* f_locals;
@@ -64,7 +64,7 @@ typedef signed char PyFrameState;
 typedef struct _pyframeobject
 {
     PyObject_VAR_HEAD struct _pyframeobject* f_back;
-    PyCodeObject* f_code;
+    PyObject* f_code;
     PyObject* f_builtins;
     PyObject* f_globals;
     PyObject* f_locals;
@@ -126,12 +126,4 @@ typedef struct _interpreter_frame
 
 }  // namespace Python3_12
 
-typedef union {
-    Python2::PyFrameObject v2;
-    Python3_7::PyFrameObject v3_7;
-    Python3_10::PyFrameObject v3_10;
-    Python3_11::PyFrameObject v3_11;
-    Python3_12::PyFrameObject v3_12;
-} PyFrameObject;
-
 }  // namespace pystack

src/pystack/_pystack/cpython/gc.h

@@ -106,10 +106,4 @@ struct _gc_runtime_state
 };
 
 }  // namespace Python3_13
-
-typedef union {
-    struct Python3_7::_gc_runtime_state v3_7;
-    struct Python3_8::_gc_runtime_state v3_8;
-    struct Python3_13::_gc_runtime_state v3_13;
-} GCRuntimeState;
 }  // namespace pystack

src/pystack/_pystack/cpython/interpreter.h

@@ -338,15 +338,4 @@ typedef struct _is
     struct _import_state imports;
 } PyInterpreterState;
 }  // namespace Python3_13
-
-typedef union {
-    Python2::PyInterpreterState v2;
-    Python3_5::PyInterpreterState v3_5;
-    Python3_7::PyInterpreterState v3_7;
-    Python3_8::PyInterpreterState v3_8;
-    Python3_9::PyInterpreterState v3_9;
-    Python3_11::PyInterpreterState v3_11;
-    Python3_12::PyInterpreterState v3_12;
-    Python3_13::PyInterpreterState v3_13;
-} PyInterpreterState;
 }  // namespace pystack

src/pystack/_pystack/cpython/object.h

@@ -205,12 +205,6 @@ typedef struct _typeobject
 } PyTypeObject;
 }  // namespace Python3_8
 
-typedef union {
-    Python2::PyTypeObject v2;
-    Python3_3::PyTypeObject v3_3;
-    Python3_8::PyTypeObject v3_8;
-} PyTypeObject;
-
 /* These flags are used to determine if a type is a subclass. */
 constexpr long Pystack_TPFLAGS_INT_SUBCLASS = 1ul << 23u;
 constexpr long Pystack_TPFLAGS_LONG_SUBCLASS = 1ul << 24u;

src/pystack/_pystack/cpython/runtime.h

@@ -105,13 +105,15 @@ struct _ceval_runtime_state
     struct _gil_runtime_state gil;
 };
 
+struct PyThreadState;
+
 typedef struct pyruntimestate
 {
     int preinitializing;
     int preinitialized;
     int core_initialized;
     int initialized;
-    PyThreadState* finalizing;
+    void* finalizing;
 
     struct pyinterpreters
     {
@@ -171,7 +173,7 @@ typedef struct pyruntimestate
     int preinitialized;
     int core_initialized;
     int initialized;
-    PyThreadState* finalizing;
+    void* finalizing;
 
     struct pyinterpreters
     {

src/pystack/_pystack/cpython/string.h

@@ -109,14 +109,4 @@ typedef struct
 
 }  // namespace Python3_12
 
-typedef union {
-    Python3::PyBytesObject v3;
-} PyBytesObject;
-
-typedef union {
-    Python2::PyUnicodeObject v2;
-    Python3::PyUnicodeObject v3;
-    Python3_12::PyUnicodeObject v3_12;
-} PyUnicodeObject;
-
 }  // namespace pystack

src/pystack/_pystack/cpython/thread.h

@@ -295,18 +295,4 @@ typedef struct _pythreadstate
 } PyThreadState;
 }  // namespace Python3_13
 
-typedef union {
-    Python2::PyThreadState v2;
-    Python3_4::PyThreadState v3_4;
-    Python3_7::PyThreadState v3_7;
-    Python3_11::PyThreadState v3_11;
-    Python3_12::PyThreadState v3_12;
-    Python3_13::PyThreadState v3_13;
-} PyThreadState;
-
-union CFrame {
-    Python3_11::CFrame v3_11;
-    Python3_12::CFrame v3_12;
-};
-
 }  // namespace pystack

src/pystack/_pystack/mem.cpp

@@ -398,7 +398,7 @@ CorefileRemoteMemoryManager::StatusCode
 CorefileRemoteMemoryManager::getMemoryLocationFromCore(remote_addr_t addr, off_t* offset_in_file) const
 {
     auto corefile_it = std::find_if(d_vmaps.cbegin(), d_vmaps.cend(), [&](auto& map) {
-        return (map.Start() <= addr && addr <= map.End()) && (map.FileSize() != 0 && map.Offset() != 0);
+        return (map.Start() <= addr && addr < map.End()) && (map.FileSize() != 0 && map.Offset() != 0);
     });
     if (corefile_it == d_vmaps.cend()) {
         return StatusCode::ERROR;