Update kdmp-parser to v0.7.1 (#190)

The latest version of `kdmp-parser` now supports parsing the new kernel dumps types (8, 9, 10) that recent WinDbg generates.
0vercl0k · Dec 20, 2023 · ad37f7c · ad37f7c
1 parent 3106447
commit ad37f7c
Show file tree

Hide file tree

Showing 36 changed files with 2,020 additions and 1,292 deletions.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -7,7 +7,7 @@ set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
 project(wtf)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/bochscpu-bins/include)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/robin-map/include)
-include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/kdmp-parser/src/lib)
+add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/libs/kdmp-parser/)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/libfuzzer)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/readerwriterqueue)
 include_directories(${CMAKE_CURRENT_LIST_DIR}/libs/json/single_include)
@@ -22,12 +22,6 @@ file(
     wtf/*.cc
 )
 
-file(
-    GLOB_RECURSE
-    kdmp_srcfiles
-    ${CMAKE_CURRENT_LIST_DIR}/libs/kdmp-parser/src/lib/*.cc
-)
-
 file(
     GLOB_RECURSE
     libfuzzer_srcfiles
@@ -44,7 +38,6 @@ set(
 add_executable(
     wtf
     ${wtf_srcfiles}
-    ${kdmp_srcfiles}
     ${libfuzzer_srcfiles}
     ${blake3_srcfiles}
 )
@@ -131,4 +124,5 @@ target_precompile_headers(
 target_link_libraries(
     wtf
     ${CMAKE_CURRENT_LIST_DIR}/libs/bochscpu-bins/lib/${CMAKE_SHARED_MODULE_PREFIX}bochscpu_ffi${CMAKE_STATIC_LIBRARY_SUFFIX}
+    kdmp-parser
 )
diff --git a/src/libs/kdmp-parser/.github/Invoke-VisualStudio.ps1 b/src/libs/kdmp-parser/.github/Invoke-VisualStudio.ps1
@@ -0,0 +1,24 @@
+Function Invoke-CmdScript {
+    param(
+        [String] $scriptName
+    )
+    $cmdLine = """$scriptName"" $args & set"
+    & $env:SystemRoot\system32\cmd.exe /c $cmdLine |
+    Select-String '^([^=]*)=(.*)$' | ForEach-Object {
+        $varName = $_.Matches[0].Groups[1].Value
+        $varValue = $_.Matches[0].Groups[2].Value
+        Set-Item Env:$varName $varValue
+    }
+}
+
+Function Invoke-VisualStudio2022win32 {
+    Invoke-CmdScript "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars32.bat"
+}
+
+Function Invoke-VisualStudio2022x64 {
+    Invoke-CmdScript "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+}
+
+Function Invoke-VisualStudio2022arm64 {
+    Invoke-CmdScript "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvarsamd64_arm64.bat"
+}
diff --git a/src/libs/kdmp-parser/.github/workflows/kdmp-parser.yml b/src/libs/kdmp-parser/.github/workflows/kdmp-parser.yml
@@ -1,94 +1,185 @@
-# Axel '0vercl0k' Souchet - July 26 2020
 name: Builds
 
 on: [push, pull_request]
 
 jobs:
-  Ubuntu2004:
+  testdatas:
+    env:
+      TESTDATA_URL: https://github.com/0vercl0k/kdmp-parser/releases/download/v0.1/testdatas.7z
+
+    name: Fetch Test Data
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cache Artifacts
+      id: cache-testdatas
+      uses: actions/cache@v3
+      with:
+        key: kdmp-parser-testdatas-cache
+        path: .
+    - if: steps.cache-testdatas.outputs.cache-hit != 'true'
+      run: |
+        sudo apt-get -y update; sudo apt-get install -y p7zip-full;
+        curl ${{ env.TESTDATA_URL }}  -O -L
+        7z x testdatas.7z; rm testdatas.7z
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        if-no-files-found: error
+        name: kdmp-parser-testdatas-cache
+        path: .
+
+  parser:
+    needs: testdatas
     strategy:
       fail-fast: false
       matrix:
-        # Available versions: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
-        # We don't test below 3.6 because f'strings were implemented in 3.6 and we use them in build scripts, etc.
-        pyver: ['3.6', '3.7', '3.8', '3.9', '3.x']
-        compiler: ['clang', 'gcc']
+        variant:
+          - {os: windows-latest, generator: msvc, arch: x64, config: RelWithDebInfo}
+          - {os: windows-latest, generator: ninja, arch: x64, config: RelWithDebInfo}
+          - {os: windows-latest, generator: msvc, arch: win32, config: RelWithDebInfo}
+          # - {os: windows-latest, generator: msvc, arch: arm64, config: RelWithDebInfo}
+          - {os: ubuntu-latest, generator: gcc, arch: x64, config: RelWithDebInfo}
+          - {os: ubuntu-latest, generator: clang, arch: x64, config: RelWithDebInfo}
+          - {os: macos-latest, generator: clang, arch: x64, config: Release}
+    runs-on: ${{ matrix.variant.os }}
+    name: parser / ${{ matrix.variant.os }} / ${{ matrix.variant.generator }} / ${{ matrix.variant.arch }}
+    env:
+      CMAKE_FLAGS: "-DBUILD_PARSER:BOOL=ON -DBUILD_TESTS:BOOL=ON -DBUILD_PYTHON_BINDING:BOOL=OFF"
+      CMAKE_ARCH: ""
 
-    name: Ubuntu 20.04 - Py${{ matrix.pyver }} / ${{ matrix.compiler }}
-    runs-on: ubuntu-20.04
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
-    - name: Setup Python3
-      uses: actions/setup-python@v2
+    - name: Retrieve testdatas
+      uses: actions/download-artifact@v3
       with:
-        python-version: ${{ matrix.pyver }}
-        architecture: 'x64'
+        name: kdmp-parser-testdatas-cache
+        path: .
 
-    - name: Install dependencies
+    - name: Environment Setup (Windows)
+      if: matrix.variant.os == 'windows-latest'
+      run: |
+        Import-Module .\.github\Invoke-VisualStudio.ps1
+        Invoke-VisualStudio2022${{ matrix.variant.arch }}
+        echo "CMAKE_ARCH='-A ${{ matrix.variant.arch }}'" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+
+    - name: Environment Setup (Linux)
+      if: matrix.variant.os == 'ubuntu-latest'
+      run: |
+        sudo apt update
+
+    - name: Build (Linux/GCC)
+      if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'gcc'
+      run: |
+        sudo apt install -y g++
+        echo CC=gcc >> $GITHUB_ENV
+        echo CXX=g++ >> $GITHUB_ENV
+
+    - name: Environment Setup (Linux/CLang)
+      if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'clang'
       run: |
-        sudo apt-get -y update
-        sudo apt install -y g++-10-multilib g++-10 ninja-build
         sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+        echo CC=clang >> $GITHUB_ENV
+        echo CXX=clang++ >> $GITHUB_ENV
 
-    - name: Build with gcc
-      if: matrix.compiler == 'gcc'
-      run: python3 builder.py --run-tests
-      env:
-        CC: gcc-10
-        CXX: g++-10
-
-    - name: Build with clang
-      if: matrix.compiler == 'clang'
-      run: python3 builder.py --run-tests
-      env:
-        CC: clang-11
-        CXX: clang++-11
-
-    - name: Upload artifacts for rel/x64
-      uses: actions/upload-artifact@v2
-      with:
-        name: linx64-RelWithDebInfo-py${{ matrix.pyver }}
-        path: bin/linx64-RelWithDebInfo
+    - name: Build
+      run: |
+        mkdir build
+        mkdir artifact
+        cmake -S . -B ./build ${{ env.CMAKE_ARCH }} ${{ env.CMAKE_FLAGS }}
+        cmake --build ./build --verbose --config ${{ matrix.variant.config }}
+        cmake --install ./build --config ${{ matrix.variant.config }} --prefix ./artifact
+
+    - name: Tests
+      run: |
+        mv *.dmp ./build/src/tests/
+        ctest --progress --build-config ${{ matrix.variant.config }} -T test --test-dir ./build/src/tests/
 
-    - name: Upload artifacts for dbg/x64
-      uses: actions/upload-artifact@v2
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        name: linx64-Debug-py${{ matrix.pyver }}
-        path: bin/linx64-Debug
+        name: parser-${{ matrix.variant.os }}.${{ matrix.variant.generator }}-${{ matrix.variant.arch }}.${{ matrix.variant.config }}-${{ github.sha }}
+        path: artifact/
 
-  Windows:
+  bindings:
+    needs: testdatas
     strategy:
       fail-fast: false
       matrix:
-        pyver: ['3.6', '3.7', '3.8', '3.9', '3.x']
-
-    name: Windows latest - Py${{ matrix.pyver }}
-    runs-on: windows-latest
+        # nanobind does not support Python < 3.8.
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        variant:
+          - {os: windows-latest, generator: msvc, arch: x64, config: RelWithDebInfo, py-arch: x64}
+          - {os: windows-latest, generator: msvc, arch: win32, config: RelWithDebInfo, py-arch: x86}
+          # - {os: windows-latest, generator: msvc, arch: arm64, config: RelWithDebInfo, py-arch: x64} # Unsupported (see https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json)
+          - {os: ubuntu-latest, generator: gcc, arch: x64, config: RelWithDebInfo, py-arch: x64}
+          - {os: ubuntu-latest, generator: clang, arch: x64, config: RelWithDebInfo, py-arch: x64}
+          - {os: macos-latest, generator: clang, arch: x64, config: Release, py-arch: x64}
+    runs-on: ${{ matrix.variant.os }}
+    name: bindings / ${{ matrix.variant.os }} / ${{ matrix.variant.generator }} / ${{ matrix.python-version }} / ${{ matrix.variant.arch }}
+    env:
+      CMAKE_FLAGS: "-DBUILD_PARSER:BOOL=OFF -DBUILD_TESTS:BOOL=OFF -DBUILD_PYTHON_BINDING:BOOL=ON"
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
-    - name: Setup Python3
-      uses: actions/setup-python@v2
+    - name: Retrieve testdatas
+      uses: actions/download-artifact@v3
       with:
-        python-version: ${{ matrix.pyver }}
-        architecture: 'x64'
+        name: kdmp-parser-testdatas-cache
+        path: .
 
-    - name: Setup vs prompt
-      uses: ilammy/msvc-dev-cmd@v1
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: ${{ matrix.variant.py-arch }}
 
-    - name: Build
-      run: python builder.py --run-tests
+    - name: Environment Setup (Windows)
+      if: matrix.variant.os == 'windows-latest'
+      run: |
+        Import-Module .\.github\Invoke-VisualStudio.ps1
+        Invoke-VisualStudio2022${{ matrix.variant.arch }}
 
-    - name: Upload artifacts for rel/x64
-      uses: actions/upload-artifact@v2
-      with:
-        name: winx64-RelWithDebInfo-py${{ matrix.pyver }}
-        path: bin/winx64-RelWithDebInfo
+    - name: Environment Setup (Linux)
+      if: matrix.variant.os == 'ubuntu-latest'
+      run: |
+        sudo apt-get -y update
+
+    - name: Environment Setup (OSX)
+      if: matrix.variant.os == 'macos-latest'
+      run: |
+        echo
+
+    - name: Environment Setup (Linux/GCC)
+      if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'gcc'
+      run: |
+        sudo apt install -y g++
+        echo CC=gcc >> $GITHUB_ENV
+        echo CXX=g++ >> $GITHUB_ENV
+
+    - name: Environment Setup (Linux/CLang)
+      if: matrix.variant.os == 'ubuntu-latest' && matrix.variant.generator == 'clang'
+      run: |
+        sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+        echo CC=clang >> $GITHUB_ENV
+        echo CXX=clang++ >> $GITHUB_ENV
+
+    - name: Build wheel
+      run: |
+        mkdir wheel
+        pip wheel src/python -w ./wheel
+
+    - name: Python Binding Tests
+      run: |
+        pip install --upgrade pip setuptools wheel
+        pip install -U -r src/python/tests/requirements.txt
+        pip install -U --user src/python
+        pytest -vvv src/python/tests
 
-    - name: Upload artifacts for dbg/x64
-      uses: actions/upload-artifact@v2
+    - name: Upload wheel
+      uses: actions/upload-artifact@v3
       with:
-        name: winx64-Debug
-        path: bin/winx64-Debug
+        name: wheels
+        path: wheel/*.whl
diff --git a/src/libs/kdmp-parser/.gitignore b/src/libs/kdmp-parser/.gitignore
@@ -41,6 +41,7 @@
 
 # vs directory
 **/.vs
+**/.vscode
 
 # user files
 *.vcxproj.user
@@ -49,4 +50,18 @@
 bin/
 
 # Build output
-build/
+build/
+wheel/
+
+# Cache folders
+__pycache__/
+.pytest_cache/
+
+# Test dumps
+*.dmp
+
+# Version files
+src/lib/kdmp-parser-version.h
+
+kdmp-parser-testdatas
+
diff --git a/src/libs/kdmp-parser/CMakeLists.txt b/src/libs/kdmp-parser/CMakeLists.txt
@@ -2,24 +2,35 @@
 # CMakeList.txt : CMake project for kdmp-parser, include source and define
 # project specific logic here.
 #
-cmake_minimum_required (VERSION 3.8)
+cmake_minimum_required(VERSION 3.21)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_POSITION_INDEPENDENT_CODE True)
 
-OPTION(BUILD_TESTS "Build and run tests" OFF)
-
-project(kdmp-parser)
-add_compile_options(
-    $<$<CXX_COMPILER_ID:MSVC>:/W4$<SEMICOLON>/WX>
-    $<$<CXX_COMPILER_ID:GNU>:-Wall$<SEMICOLON>-Wextra$<SEMICOLON>-pedantic$<SEMICOLON>-Werror>
-    $<$<CXX_COMPILER_ID:Clang>:-Wall$<SEMICOLON>-Wextra$<SEMICOLON>-pedantic$<SEMICOLON>-Werror>
+project(
+    kdmp-parser
+    DESCRIPTION "A Cross-Platform C++ parser library for Windows kernel minidumps."
+    HOMEPAGE_URL https://github.com/0vercl0k/kdmp-parser
+    VERSION 0.7.1
 )
 
+set(PROJECT_AUTHOR 0vercl0k)
+set(PROJECT_LICENSE MIT)
+
+option(BUILD_PARSER "Build the parser executable for kdmp-parser" ON)
+option(BUILD_TESTS "Build the test suite for kdmp-parser" OFF)
+option(BUILD_PYTHON_BINDING "Build the Python bindings for kdmp-parser" OFF)
+
 add_subdirectory(src/lib)
-add_subdirectory(src/parser)
-add_subdirectory(src/testapp)
-add_subdirectory(src/python)
+
+if(BUILD_PARSER)
+    add_subdirectory(src/parser)
+endif(BUILD_PARSER)
+
+if(BUILD_PYTHON_BINDING)
+    add_subdirectory(src/python)
+endif(BUILD_PYTHON_BINDING)
+
 if(BUILD_TESTS)
     add_subdirectory(src/tests)
-endif()
+endif(BUILD_TESTS)