From d1bdec31fa41db74450c8ffa4dcea779609b4dee Mon Sep 17 00:00:00 2001
From: Habip Hakan Isler <habiphakan.isler@gmail.com>
Date: Tue, 4 Feb 2025 12:32:19 +0300
Subject: [PATCH 1/4] Feat: Add Kafka integration for Parseable server #936 .
 (#1047)

This pull request implements the Kafka connector for Parseable by
introducing better stream management, modularizing the code, and
integrating Prometheus metrics. It also adds groundwork for dynamic configurations.

Partition Management: The connector dynamically manages streams for
each partition using StreamState. Each Kafka message is routed to the
corresponding PartitionStreamReceiver for processing.

Rebalance Handling:
  - On partition assignment: New streams are created for untracked partitions.
  - On partition revocation: Streams for revoked partitions are terminated and cleaned up.

Stream Processing: The StreamWorker processes messages in batches,
ensuring backpressure handling with configurable buffer_size and buffer_timeout.

Also exposed metrics via Prometheus include: Partition lag, consumer
throughput, etc provided by rd-kafka statistics.

Part of #936
---------

Signed-off-by: Nitish Tiwari <nitish@parseable.com>
Signed-off-by: Nikhil Sinha <131262146+nikhilsinhaparseable@users.noreply.github.com>
Co-authored-by: Devdutt Shenoi <devdutt@parseable.com>
Co-authored-by: Nitish Tiwari <nitish@parseable.com>
---
 .github/workflows/build-push-edge-kafka.yaml  |   45 +
 .github/workflows/build.yaml                  |  165 +-
 .github/workflows/coverage.yaml               |   14 +-
 .github/workflows/integration-test.yaml       |   32 +-
 .gitignore                                    |    1 +
 Cargo.lock                                    | 2112 ++++++++++-------
 Cargo.toml                                    |   51 +-
 Dockerfile                                    |    5 +-
 Dockerfile.debug                              |    4 +-
 Dockerfile.dev                                |    4 +-
 Dockerfile.kafka                              |   66 +
 build.rs                                      |   20 +-
 ...r-compose-distributed-test-with-kafka.yaml |  366 +++
 docker-compose-distributed-test.yaml          |   20 +-
 docker-compose-local.yaml                     |   41 +
 docker-compose-test-with-kafka.yaml           |  164 ++
 docker-compose-test.yaml                      |   36 +-
 parseable-ingest-haproxy.cfg                  |   45 +
 scripts/Dockerfile                            |   29 +
 scripts/kafka_log_stream_generator.py         |  237 ++
 src/cli.rs                                    |   60 +-
 src/connectors/common/mod.rs                  |  111 +
 src/connectors/common/processor.rs            |   29 +
 src/connectors/common/shutdown.rs             |  132 ++
 src/connectors/kafka/config.rs                | 1008 ++++++++
 src/connectors/kafka/consumer.rs              |  247 ++
 src/connectors/kafka/metrics.rs               |  874 +++++++
 src/connectors/kafka/mod.rs                   |  266 +++
 src/connectors/kafka/partition_stream.rs      |  110 +
 src/connectors/kafka/processor.rs             |  181 ++
 src/connectors/kafka/rebalance_listener.rs    |   87 +
 src/connectors/kafka/sink.rs                  |   94 +
 src/connectors/kafka/state.rs                 |   68 +
 src/connectors/mod.rs                         |  100 +
 src/handlers/http/modal/ingest_server.rs      |   29 +-
 src/handlers/http/modal/mod.rs                |    6 +-
 src/handlers/http/modal/query_server.rs       |   14 +-
 src/handlers/http/modal/server.rs             |   12 +-
 src/kafka.rs                                  |  255 --
 src/lib.rs                                    |    7 +-
 src/main.rs                                   |   68 +-
 src/metadata.rs                               |   15 +
 src/option.rs                                 |   30 +-
 43 files changed, 5923 insertions(+), 1337 deletions(-)
 create mode 100644 .github/workflows/build-push-edge-kafka.yaml
 create mode 100644 Dockerfile.kafka
 create mode 100644 docker-compose-distributed-test-with-kafka.yaml
 create mode 100644 docker-compose-local.yaml
 create mode 100644 docker-compose-test-with-kafka.yaml
 create mode 100644 parseable-ingest-haproxy.cfg
 create mode 100644 scripts/Dockerfile
 create mode 100644 scripts/kafka_log_stream_generator.py
 create mode 100644 src/connectors/common/mod.rs
 create mode 100644 src/connectors/common/processor.rs
 create mode 100644 src/connectors/common/shutdown.rs
 create mode 100644 src/connectors/kafka/config.rs
 create mode 100644 src/connectors/kafka/consumer.rs
 create mode 100644 src/connectors/kafka/metrics.rs
 create mode 100644 src/connectors/kafka/mod.rs
 create mode 100644 src/connectors/kafka/partition_stream.rs
 create mode 100644 src/connectors/kafka/processor.rs
 create mode 100644 src/connectors/kafka/rebalance_listener.rs
 create mode 100644 src/connectors/kafka/sink.rs
 create mode 100644 src/connectors/kafka/state.rs
 create mode 100644 src/connectors/mod.rs
 delete mode 100644 src/kafka.rs

diff --git a/.github/workflows/build-push-edge-kafka.yaml b/.github/workflows/build-push-edge-kafka.yaml
new file mode 100644
index 000000000..1131d2fa4
--- /dev/null
+++ b/.github/workflows/build-push-edge-kafka.yaml
@@ -0,0 +1,45 @@
+name: Build and push edge kafka tag
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - 'helm/**'
+      - 'assets/**'
+      - '**.md'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: parseable/parseable
+
+      - name: Build and push
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          file: ./Dockerfile.kafka
+          push: true
+          tags: parseable/parseable:edge-kafka
+          platforms: linux/amd64,linux/arm64
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index b7e3195c5..080a9ba68 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -1,3 +1,5 @@
+name: Ensure parseable builds on all release targets
+
 on:
   pull_request:
     paths-ignore:
@@ -6,68 +8,141 @@ on:
       - "assets/**"
       - "**.md"
 
-name: Ensure parseable builds on all release targets
 jobs:
-  build-linux:
-    name: Build for ${{matrix.target}}
-    runs-on: ubuntu-latest
+  # Default build without Kafka
+  build-default:
+    name: Build Default ${{matrix.target}}
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        target:
-          - aarch64-unknown-linux-gnu # linux(arm)
-          - x86_64-unknown-linux-gnu # linux(64 bit)
+        include:
+          # Linux builds
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+          # macOS builds  
+          - os: macos-latest
+            target: x86_64-apple-darwin
+          - os: macos-latest
+            target: aarch64-apple-darwin
+          # Windows build
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.target }}
 
-      - uses: actions-rs/toolchain@v1
+      - name: Cache dependencies
+        uses: actions/cache@v4
         with:
-          toolchain: stable
-          profile: minimal # minimal component installation (ie, no documentation)
-          target: ${{ matrix.target }}
-          override: true
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ matrix.target }}-default-${{ hashFiles('**/Cargo.lock') }}
 
-      - uses: actions-rs/cargo@v1
+      - name: Build
+        uses: actions-rs/cargo@v1
         with:
-          use-cross: true
+          use-cross: ${{ runner.os == 'Linux' }}
           command: build
-          args: --target ${{matrix.target}}
+          args: --target ${{ matrix.target }} --release
 
-  build-windows:
-    name: Build for windows
-    runs-on: windows-latest
+  # Kafka build for supported platforms
+  build-kafka:
+    name: Build Kafka ${{matrix.target}}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Linux builds
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            target: aarch64-apple-darwin
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal # minimal component installation (ie, no documentation)
-          default: true
-          override: true
+      # Linux-specific dependencies
+      - name: Install Linux dependencies
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            build-essential \
+            pkg-config \
+            cmake \
+            clang \
+            zlib1g-dev \
+            libzstd-dev \
+            liblz4-dev \
+            libssl-dev \
+            libsasl2-dev \
+            python3 \
+            gcc-aarch64-linux-gnu \
+            g++-aarch64-linux-gnu
+          
+          # Install cross-compilation specific packages
+          if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then
+            sudo apt-get install -y \
+              gcc-aarch64-linux-gnu \
+              g++-aarch64-linux-gnu \
+              libc6-dev-arm64-cross \
+              libsasl2-dev:arm64 \
+              libssl-dev:arm64 \
+              pkg-config-aarch64-linux-gnu
+          fi
 
-      - name: Build on windows
-        run: cargo build --target x86_64-pc-windows-msvc
 
-  build-macos:
-    name: Build for ${{matrix.target}}
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        target:
-          - aarch64-apple-darwin # macos(arm)
-          - x86_64-apple-darwin # macos(intel 64 bit)
+      # macOS-specific dependencies
+      - name: Install macOS dependencies
+        if: runner.os == 'macOS'
+        run: |
+          brew install \
+            cmake \
+            llvm \
+            pkg-config \
+            zstd \
+            lz4 \
+            openssl@3.0 \
+            cyrus-sasl \
+            python3
 
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: stable
-          profile: minimal
-          target: ${{ matrix.target }}
-          override: true
+          targets: ${{ matrix.target }}
 
-      - name: Build on ${{ matrix.target }}
-        run: cargo build --target ${{ matrix.target }}
+      - name: Cache dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ matrix.target }}-kafka-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Build with Kafka
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ runner.os == 'Linux' }}
+          command: build
+          args: --target ${{ matrix.target }} --features kafka --release
+        env:
+          LIBRDKAFKA_SSL_VENDORED: 1
+          PKG_CONFIG_ALLOW_CROSS: "1"
+          PKG_CONFIG_PATH: "/usr/lib/aarch64-linux-gnu/pkgconfig"
+          SASL2_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_ROOT_DIR: "/usr/lib/aarch64-linux-gnu"
+          OPENSSL_STATIC: "1"
+          SASL2_STATIC: "0"
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
index 53d2f3464..b2ff61cb9 100644
--- a/.github/workflows/coverage.yaml
+++ b/.github/workflows/coverage.yaml
@@ -11,7 +11,7 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
@@ -24,13 +24,23 @@ jobs:
         with:
           tool: cargo-hack, cargo-llvm-cov, nextest
 
+      - name: Install System Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y \
+            libsasl2-dev \
+            libssl-dev \
+            pkg-config \
+            build-essential
+        if: runner.os == 'Linux'
+
       - name: Check with clippy
         run: cargo hack clippy --verbose --each-feature --no-dev-deps -- -D warnings
 
       - name: Tests
         run: cargo hack --each-feature llvm-cov --no-report nextest
 
-      - name: Genrate coverage report
+      - name: Generate coverage report
         run: cargo llvm-cov report --lcov --output-path coverage.lcov
 
       - name: Upload Coverage Report
diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
index a605bfcb9..cb8df920e 100644
--- a/.github/workflows/integration-test.yaml
+++ b/.github/workflows/integration-test.yaml
@@ -14,21 +14,45 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Start compose
         run: docker compose -f docker-compose-test.yaml up --build --exit-code-from quest
       - name: Stop compose
         if: always()
-        run: docker compose -f docker-compose-test.yaml down
+        run: docker compose -f docker-compose-test.yaml down -v
 
   docker-compose-distributed-test:
     name: Quest Smoke and Load Tests for Distributed deployments
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Start compose
         run: docker compose -f docker-compose-distributed-test.yaml up --build --exit-code-from quest
       - name: Stop compose
         if: always()
-        run: docker compose -f docker-compose-distributed-test.yaml down
+        run: docker compose -f docker-compose-distributed-test.yaml down -v
+
+  docker-compose-test-with-kafka:
+    name: Quest Smoke and Load Tests for Standalone deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Start compose
+        run: docker compose -f docker-compose-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-test-with-kafka.yaml down -v
+
+  docker-compose-distributed-test-with-kafka:
+    name: Quest Smoke and Load Tests for Distributed deployments with Kafka
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Start compose
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml up --build --exit-code-from quest
+      - name: Stop compose
+        if: always()
+        run: docker compose -f docker-compose-distributed-test-with-kafka.yaml down -v
diff --git a/.gitignore b/.gitignore
index 22df045e3..853182f54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ parseable
 parseable_*
 parseable-env-secret
 cache
+.idea
diff --git a/Cargo.lock b/Cargo.lock
index 00a83ec92..34da482d7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8,7 +8,7 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.8.0",
  "bytes",
  "futures-core",
  "futures-sink",
@@ -27,7 +27,7 @@ checksum = "f9e772b3bcafe335042b5db010ab7c09013dad6eac4915c91d8d50902769f331"
 dependencies = [
  "actix-utils",
  "actix-web",
- "derive_more",
+ "derive_more 0.99.18",
  "futures-util",
  "log",
  "once_cell",
@@ -46,12 +46,12 @@ dependencies = [
  "actix-tls",
  "actix-utils",
  "ahash",
- "base64 0.22.0",
- "bitflags 2.5.0",
+ "base64 0.22.1",
+ "bitflags 2.8.0",
  "brotli 6.0.0",
  "bytes",
  "bytestring",
- "derive_more",
+ "derive_more 0.99.18",
  "encoding_rs",
  "flate2",
  "futures-core",
@@ -81,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
 dependencies = [
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -101,9 +101,9 @@ dependencies = [
 
 [[package]]
 name = "actix-rt"
-version = "2.9.0"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d"
+checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208"
 dependencies = [
  "futures-core",
  "tokio",
@@ -111,16 +111,16 @@ dependencies = [
 
 [[package]]
 name = "actix-server"
-version = "2.3.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3eb13e7eef0423ea6eab0e59f6c72e7cb46d33691ad56a726b3cd07ddec2c2d4"
+checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894"
 dependencies = [
  "actix-rt",
  "actix-service",
  "actix-utils",
  "futures-core",
  "futures-util",
- "mio 0.8.11",
+ "mio",
  "socket2",
  "tokio",
  "tracing",
@@ -187,7 +187,7 @@ dependencies = [
  "bytestring",
  "cfg-if",
  "cookie 0.16.2",
- "derive_more",
+ "derive_more 0.99.18",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -218,18 +218,18 @@ dependencies = [
  "actix-router",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "actix-web-httpauth"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d613edf08a42ccc6864c941d30fe14e1b676a77d16f1dbadc1174d065a0a775"
+checksum = "456348ed9dcd72a13a1f4a660449fafdecee9ac8205552e286809eb5b0b29bd3"
 dependencies = [
  "actix-utils",
  "actix-web",
- "base64 0.21.7",
+ "base64 0.22.1",
  "futures-core",
  "futures-util",
  "log",
@@ -248,7 +248,7 @@ dependencies = [
  "pin-project",
  "prometheus",
  "quanta",
- "thiserror 1.0.64",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -258,26 +258,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adf6d1ef6d7a60e084f9e0595e2a5234abda14e76c105ecf8e2d0e8800c41a1f"
 dependencies = [
  "actix-web",
- "derive_more",
+ "derive_more 0.99.18",
  "futures-util",
  "static-files",
 ]
 
 [[package]]
 name = "addr2line"
-version = "0.21.0"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
+checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
 dependencies = [
  "gimli",
 ]
 
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
 [[package]]
 name = "adler2"
 version = "2.0.0"
@@ -292,7 +286,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
  "const-random",
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -324,9 +318,9 @@ dependencies = [
 
 [[package]]
 name = "allocator-api2"
-version = "0.2.18"
+version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
 [[package]]
 name = "android-tzdata"
@@ -345,66 +339,68 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.13"
+version = "0.6.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
 dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
  "anstyle-wincon",
  "colorchoice",
+ "is_terminal_polyfill",
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle"
-version = "1.0.6"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.3"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.0.2"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.2"
+version = "3.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
 dependencies = [
  "anstyle",
- "windows-sys 0.52.0",
+ "once_cell",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.82"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
+checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 dependencies = [
  "backtrace",
 ]
 
 [[package]]
 name = "arbitrary"
-version = "1.3.2"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
+checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
 dependencies = [
  "derive_arbitrary",
 ]
@@ -423,21 +419,21 @@ dependencies = [
 
 [[package]]
 name = "arrayref"
-version = "0.3.7"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
 
 [[package]]
 name = "arrayvec"
-version = "0.7.4"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d"
+checksum = "eaf3437355979f1e93ba84ba108c38be5767713051f3c8ffbf07c094e2e61f9f"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -456,9 +452,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f"
+checksum = "31dce77d2985522288edae7206bffd5fc4996491841dda01a13a58415867e681"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -471,9 +467,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f"
+checksum = "2d45fe6d3faed0435b7313e59a02583b14c6c6339fa7729e94c32a20af319a79"
 dependencies = [
  "ahash",
  "arrow-buffer",
@@ -488,9 +484,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16"
+checksum = "2b02656a35cc103f28084bc80a0159668e0a680d919cef127bd7e0aaccb06ec1"
 dependencies = [
  "bytes",
  "half",
@@ -499,9 +495,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd"
+checksum = "c73c6233c5b5d635a56f6010e6eb1ab9e30e94707db21cea03da317f67d84cf3"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -509,7 +505,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "atoi",
- "base64 0.22.0",
+ "base64 0.22.1",
  "chrono",
  "comfy-table",
  "half",
@@ -520,9 +516,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97"
+checksum = "ec222848d70fea5a32af9c3602b08f5d740d5e2d33fbd76bf6fd88759b5b13a7"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -539,9 +535,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623"
+checksum = "b7f2861ffa86f107b8ab577d86cff7c7a490243eabe961ba1e1af4f27542bb79"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -551,16 +547,16 @@ dependencies = [
 
 [[package]]
 name = "arrow-flight"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c09b331887a526f203f2123444792aee924632bd08b9940435070901075832e"
+checksum = "3ab7635558f3f803b492eae56c03cde97ea5f85a1c768f94181cb7db69cd81be"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-ipc",
  "arrow-schema",
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
  "futures",
  "paste",
@@ -572,9 +568,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8"
+checksum = "0270dc511f11bb5fa98a25020ad51a99ca5b08d8a8dfbd17503bb9dba0388f0b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -588,9 +584,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85"
+checksum = "0eff38eeb8a971ad3a4caf62c5d57f0cff8a48b64a55e3207c4fd696a9234aad"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -599,7 +595,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "lexical-core",
  "num",
  "serde",
@@ -608,9 +604,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd"
+checksum = "c6f202a879d287099139ff0d121e7f55ae5e0efe634b8cf2106ebc27a8715dee"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -623,9 +619,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff"
+checksum = "a8f936954991c360ba762dff23f5dda16300774fafd722353d9683abd97630ae"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -637,18 +633,18 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678"
+checksum = "9579b9d8bce47aa41389fe344f2c6758279983b7c0ebb4013e283e3e91bb450e"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "arrow-select"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722"
+checksum = "7471ba126d0b0aaa24b50a36bc6c25e4e74869a1fd1a5553357027a0b1c8d1f1"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -660,9 +656,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf"
+checksum = "72993b01cb62507b06f1fb49648d7286c8989ecfabdb7b77a750fcb54410731b"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -677,11 +673,11 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.8"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60"
+checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522"
 dependencies = [
- "brotli 4.0.0",
+ "brotli 7.0.0",
  "bzip2 0.4.4",
  "flate2",
  "futures-core",
@@ -695,9 +691,9 @@ dependencies = [
 
 [[package]]
 name = "async-stream"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
 dependencies = [
  "async-stream-impl",
  "futures-core",
@@ -706,24 +702,24 @@ dependencies = [
 
 [[package]]
 name = "async-stream-impl"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.82"
+version = "0.1.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
+checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -743,22 +739,22 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
 
 [[package]]
 name = "autocfg"
-version = "1.2.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "axum"
-version = "0.7.5"
+version = "0.7.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
+checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
 dependencies = [
  "async-trait",
  "axum-core",
  "bytes",
  "futures-util",
- "http 1.1.0",
- "http-body 1.0.0",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
  "itoa",
  "matchit",
@@ -768,53 +764,47 @@ dependencies = [
  "pin-project-lite",
  "rustversion",
  "serde",
- "sync_wrapper 1.0.1",
- "tower",
+ "sync_wrapper 1.0.2",
+ "tower 0.5.2",
  "tower-layer",
  "tower-service",
 ]
 
 [[package]]
 name = "axum-core"
-version = "0.4.4"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
 dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http 1.1.0",
- "http-body 1.0.0",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
  "mime",
  "pin-project-lite",
  "rustversion",
- "sync_wrapper 1.0.1",
+ "sync_wrapper 1.0.2",
  "tower-layer",
  "tower-service",
 ]
 
 [[package]]
 name = "backtrace"
-version = "0.3.71"
+version = "0.3.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
 dependencies = [
  "addr2line",
- "cc",
  "cfg-if",
  "libc",
- "miniz_oxide 0.7.2",
+ "miniz_oxide",
  "object",
  "rustc-demangle",
+ "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "base64"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
-
 [[package]]
 name = "base64"
 version = "0.21.7"
@@ -823,9 +813,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64"
-version = "0.22.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "base64ct"
@@ -870,9 +860,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
 [[package]]
 name = "blake2"
@@ -885,9 +875,9 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.5.1"
+version = "1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52"
+checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -905,17 +895,6 @@ dependencies = [
  "generic-array",
 ]
 
-[[package]]
-name = "brotli"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
- "brotli-decompressor 3.0.0",
-]
-
 [[package]]
 name = "brotli"
 version = "6.0.0"
@@ -924,7 +903,7 @@ checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
- "brotli-decompressor 4.0.1",
+ "brotli-decompressor",
 ]
 
 [[package]]
@@ -935,24 +914,14 @@ checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
- "brotli-decompressor 4.0.1",
+ "brotli-decompressor",
 ]
 
 [[package]]
 name = "brotli-decompressor"
-version = "3.0.0"
+version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525"
-dependencies = [
- "alloc-no-stdlib",
- "alloc-stdlib",
-]
-
-[[package]]
-name = "brotli-decompressor"
-version = "4.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
+checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -960,9 +929,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.16.0"
+version = "3.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
 
 [[package]]
 name = "byteorder"
@@ -972,15 +941,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.6.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
 
 [[package]]
 name = "bytestring"
-version = "1.3.1"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72"
+checksum = "e465647ae23b2823b0753f50decb2d5a86d2bb2cac04788fafd1f80e45378e5f"
 dependencies = [
  "bytes",
 ]
@@ -1018,41 +987,41 @@ dependencies = [
 
 [[package]]
 name = "camino"
-version = "1.1.6"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
+checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "cargo-platform"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc"
+checksum = "e35af189006b9c0f00a064685c727031e3ed2d8020f7ba284d78cc2671bd36ea"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "cargo_metadata"
-version = "0.18.1"
+version = "0.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
+checksum = "8769706aad5d996120af43197bf46ef6ad0fda35216b4505f926a365a232d924"
 dependencies = [
  "camino",
  "cargo-platform",
  "semver",
  "serde",
  "serde_json",
- "thiserror 1.0.64",
+ "thiserror 2.0.11",
 ]
 
 [[package]]
 name = "cargo_toml"
-version = "0.20.1"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35a60491a82bdc0440640298990087ac1625e23c2feacd584eb33775903d5bb3"
+checksum = "5fbd1fe9db3ebf71b89060adaf7b0504c2d6a425cf061313099547e382c2e472"
 dependencies = [
  "serde",
  "toml",
@@ -1060,9 +1029,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.23"
+version = "1.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3bbb537bb4a30b90362caddba8f360c0a56bc13d3a5570028e7197204cb54a17"
+checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229"
 dependencies = [
  "jobserver",
  "libc",
@@ -1075,6 +1044,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
 [[package]]
 name = "change-detection"
 version = "1.2.0"
@@ -1087,9 +1062,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.38"
+version = "0.4.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1111,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "chrono-tz"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6"
+checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f"
 dependencies = [
  "chrono",
  "chrono-tz-build",
@@ -1132,9 +1107,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.4"
+version = "4.5.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0"
+checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1142,9 +1117,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.2"
+version = "4.5.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
+checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1153,21 +1128,21 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.4"
+version = "4.5.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64"
+checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c"
 dependencies = [
- "heck 0.5.0",
+ "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.7.0"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
 
 [[package]]
 name = "clokwerk"
@@ -1178,17 +1153,26 @@ dependencies = [
  "chrono",
 ]
 
+[[package]]
+name = "cmake"
+version = "0.1.53"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "colorchoice"
-version = "1.0.0"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
 [[package]]
 name = "comfy-table"
-version = "7.1.1"
+version = "7.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7"
+checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9"
 dependencies = [
  "strum",
  "strum_macros",
@@ -1210,16 +1194,16 @@ version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
  "once_cell",
  "tiny-keccak",
 ]
 
 [[package]]
 name = "constant_time_eq"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
 
 [[package]]
 name = "convert_case"
@@ -1227,6 +1211,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
 
+[[package]]
+name = "convert_case"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
+dependencies = [
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "cookie"
 version = "0.16.2"
@@ -1258,17 +1251,27 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.12"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
 dependencies = [
  "libc",
 ]
@@ -1284,9 +1287,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
 dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
@@ -1303,9 +1306,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.20"
+version = "0.8.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
 
 [[package]]
 name = "crossterm"
@@ -1313,11 +1316,11 @@ version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.8.0",
  "crossterm_winapi",
- "mio 1.0.2",
+ "mio",
  "parking_lot",
- "rustix 0.38.34",
+ "rustix",
  "signal-hook",
  "signal-hook-mio",
  "winapi",
@@ -1334,9 +1337,9 @@ dependencies = [
 
 [[package]]
 name = "crunchy"
-version = "0.2.2"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
 
 [[package]]
 name = "crypto-common"
@@ -1350,9 +1353,9 @@ dependencies = [
 
 [[package]]
 name = "csv"
-version = "1.3.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
 dependencies = [
  "csv-core",
  "itoa",
@@ -1371,9 +1374,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.8"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -1381,27 +1384,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.8"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.8"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -1420,9 +1423,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.5.0"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
+checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f"
 
 [[package]]
 name = "datafusion"
@@ -1504,7 +1507,7 @@ dependencies = [
  "arrow-schema",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "libc",
  "log",
  "object_store",
@@ -1565,7 +1568,7 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-functions-window-common",
  "datafusion-physical-expr-common",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "paste",
  "recursive",
  "serde_json",
@@ -1591,7 +1594,7 @@ checksum = "fd197f3b2975424d3a4898ea46651be855a46721a56727515dbd5c9e2fb597da"
 dependencies = [
  "arrow",
  "arrow-buffer",
- "base64 0.22.0",
+ "base64 0.22.1",
  "blake2",
  "blake3",
  "chrono",
@@ -1720,7 +1723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c"
 dependencies = [
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -1734,7 +1737,7 @@ dependencies = [
  "datafusion-common",
  "datafusion-expr",
  "datafusion-physical-expr",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "itertools 0.13.0",
  "log",
  "recursive",
@@ -1760,7 +1763,7 @@ dependencies = [
  "datafusion-physical-expr-common",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "itertools 0.13.0",
  "log",
  "paste",
@@ -1822,7 +1825,7 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "itertools 0.13.0",
  "log",
  "parking_lot",
@@ -1842,7 +1845,7 @@ dependencies = [
  "bigdecimal",
  "datafusion-common",
  "datafusion-expr",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "log",
  "recursive",
  "regex",
@@ -1860,13 +1863,44 @@ dependencies = [
 
 [[package]]
 name = "derive_arbitrary"
-version = "1.3.2"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
+checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -1875,11 +1909,33 @@ version = "0.99.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
 dependencies = [
- "convert_case",
+ "convert_case 0.4.0",
  "proc-macro2",
  "quote",
  "rustc_version",
- "syn 2.0.87",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "derive_more"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
+dependencies = [
+ "derive_more-impl",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
+dependencies = [
+ "convert_case 0.6.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+ "unicode-xid",
 ]
 
 [[package]]
@@ -1901,20 +1957,32 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "duct"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c"
+dependencies = [
+ "libc",
+ "once_cell",
+ "os_pipe",
+ "shared_child",
 ]
 
 [[package]]
 name = "either"
-version = "1.11.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.34"
+version = "0.8.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
 dependencies = [
  "cfg-if",
 ]
@@ -1927,12 +1995,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "errno"
-version = "0.3.8"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1946,9 +2014,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.0.2"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "fixedbitset"
@@ -1958,9 +2026,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flatbuffers"
-version = "24.3.25"
+version = "24.12.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f"
+checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
 dependencies = [
  "bitflags 1.3.2",
  "rustc_version",
@@ -1968,12 +2036,12 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.34"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
+checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
 dependencies = [
  "crc32fast",
- "miniz_oxide 0.8.0",
+ "miniz_oxide",
 ]
 
 [[package]]
@@ -1999,9 +2067,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "futures"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2014,9 +2082,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -2024,15 +2092,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -2041,9 +2109,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
 name = "futures-lite"
@@ -2062,26 +2130,26 @@ dependencies = [
 
 [[package]]
 name = "futures-macro"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "futures-sink"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
 
 [[package]]
 name = "futures-task"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
 [[package]]
 name = "futures-timer"
@@ -2091,9 +2159,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
 
 [[package]]
 name = "futures-util"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2119,9 +2187,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -2130,17 +2198,29 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.13.3+wasi-0.2.2",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "gimli"
-version = "0.28.1"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
+checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
 [[package]]
 name = "glob"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
 
 [[package]]
 name = "h2"
@@ -2154,7 +2234,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "slab",
  "tokio",
  "tokio-util",
@@ -2163,17 +2243,17 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.5"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
+checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
 dependencies = [
  "atomic-waker",
  "bytes",
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.1.0",
- "indexmap 2.5.0",
+ "http 1.2.0",
+ "indexmap 2.7.1",
  "slab",
  "tokio",
  "tokio-util",
@@ -2213,12 +2293,6 @@ version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -2261,9 +2335,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
 dependencies = [
  "bytes",
  "fnv",
@@ -2272,11 +2346,11 @@ dependencies = [
 
 [[package]]
 name = "http-auth-basic"
-version = "0.3.3"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd2e17aacf7f4a2428def798e2ff4f4f883c0987bdaf47dd5c8bc027bc9f1ebc"
+checksum = "0e0c088bddfd73005b09807131224ad12c302655436b1270c8346a3ae8aaa37a"
 dependencies = [
- "base64 0.13.1",
+ "base64 0.22.1",
 ]
 
 [[package]]
@@ -2292,32 +2366,32 @@ dependencies = [
 
 [[package]]
 name = "http-body"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.1.0",
+ "http 1.2.0",
 ]
 
 [[package]]
 name = "http-body-util"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
 dependencies = [
  "bytes",
- "futures-core",
- "http 1.1.0",
- "http-body 1.0.0",
+ "futures-util",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "pin-project-lite",
 ]
 
 [[package]]
 name = "httparse"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
+checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a"
 
 [[package]]
 name = "httpdate"
@@ -2349,9 +2423,9 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "0.14.30"
+version = "0.14.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"
+checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -2373,16 +2447,16 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "1.4.1"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
 dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "h2 0.4.5",
- "http 1.1.0",
- "http-body 1.0.0",
+ "h2 0.4.7",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "httparse",
  "httpdate",
  "itoa",
@@ -2400,7 +2474,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
  "futures-util",
  "http 0.2.12",
- "hyper 0.14.30",
+ "hyper 0.14.32",
  "rustls 0.21.12",
  "tokio",
  "tokio-rustls 0.24.1",
@@ -2408,30 +2482,30 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.27.3"
+version = "0.27.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
+checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
 dependencies = [
  "futures-util",
- "http 1.1.0",
- "hyper 1.4.1",
+ "http 1.2.0",
+ "hyper 1.6.0",
  "hyper-util",
- "rustls 0.23.13",
+ "rustls 0.23.22",
  "rustls-native-certs",
  "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
  "tower-service",
- "webpki-roots 0.26.1",
+ "webpki-roots 0.26.7",
 ]
 
 [[package]]
 name = "hyper-timeout"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
 dependencies = [
- "hyper 1.4.1",
+ "hyper 1.6.0",
  "hyper-util",
  "pin-project-lite",
  "tokio",
@@ -2440,29 +2514,28 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.6"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956"
+checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
 dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http 1.1.0",
- "http-body 1.0.0",
- "hyper 1.4.1",
+ "http 1.2.0",
+ "http-body 1.0.1",
+ "hyper 1.6.0",
  "pin-project-lite",
  "socket2",
  "tokio",
- "tower",
  "tower-service",
  "tracing",
 ]
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.60"
+version = "0.1.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -2482,118 +2555,262 @@ dependencies = [
 ]
 
 [[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
-[[package]]
-name = "idna"
-version = "0.5.0"
+name = "icu_collections"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
 dependencies = [
- "unicode-bidi",
- "unicode-normalization",
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+ "zerovec",
 ]
 
 [[package]]
-name = "impl-more"
-version = "0.1.6"
+name = "icu_locid"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d"
+checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+dependencies = [
+ "displaydoc",
+ "litemap",
+ "tinystr",
+ "writeable",
+ "zerovec",
+]
 
 [[package]]
-name = "indexmap"
-version = "1.9.3"
+name = "icu_locid_transform"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
 dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
+ "displaydoc",
+ "icu_locid",
+ "icu_locid_transform_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
 ]
 
 [[package]]
-name = "indexmap"
-version = "2.5.0"
+name = "icu_locid_transform_data"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
-dependencies = [
- "equivalent",
- "hashbrown 0.14.5",
-]
+checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
 
 [[package]]
-name = "instant"
-version = "0.1.12"
+name = "icu_normalizer"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
 dependencies = [
- "cfg-if",
+ "displaydoc",
+ "icu_collections",
+ "icu_normalizer_data",
+ "icu_properties",
+ "icu_provider",
+ "smallvec",
+ "utf16_iter",
+ "utf8_iter",
+ "write16",
+ "zerovec",
 ]
 
 [[package]]
-name = "integer-encoding"
-version = "3.0.4"
+name = "icu_normalizer_data"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
+checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
 
 [[package]]
-name = "io-lifetimes"
-version = "1.0.11"
+name = "icu_properties"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
+checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
 dependencies = [
- "hermit-abi",
- "libc",
- "windows-sys 0.48.0",
+ "displaydoc",
+ "icu_collections",
+ "icu_locid_transform",
+ "icu_properties_data",
+ "icu_provider",
+ "tinystr",
+ "zerovec",
 ]
 
 [[package]]
-name = "ipnet"
-version = "2.9.0"
+name = "icu_properties_data"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
 
 [[package]]
-name = "itertools"
-version = "0.12.1"
+name = "icu_provider"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
 dependencies = [
- "either",
+ "displaydoc",
+ "icu_locid",
+ "icu_provider_macros",
+ "stable_deref_trait",
+ "tinystr",
+ "writeable",
+ "yoke",
+ "zerofrom",
+ "zerovec",
 ]
 
 [[package]]
-name = "itertools"
-version = "0.13.0"
+name = "icu_provider_macros"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
 dependencies = [
- "either",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "idna"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "idna"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
+dependencies = [
+ "idna_adapter",
+ "smallvec",
+ "utf8_iter",
+]
+
+[[package]]
+name = "idna_adapter"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
+dependencies = [
+ "icu_normalizer",
+ "icu_properties",
+]
+
+[[package]]
+name = "impl-more"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.15.2",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "integer-encoding"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
+
+[[package]]
+name = "ipnet"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
 [[package]]
 name = "jobserver"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.69"
+version = "0.3.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
 dependencies = [
+ "once_cell",
  "wasm-bindgen",
 ]
 
@@ -2605,9 +2822,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388"
 
 [[package]]
 name = "lazy_static"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "lexical-core"
@@ -2675,27 +2892,39 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.153"
+version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
 
 [[package]]
 name = "libm"
-version = "0.2.8"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
+checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 
 [[package]]
-name = "linux-raw-sys"
-version = "0.1.4"
+name = "libz-sys"
+version = "1.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
+checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.13"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
+[[package]]
+name = "litemap"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
+checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
 
 [[package]]
 name = "local-channel"
@@ -2716,9 +2945,9 @@ checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487"
 
 [[package]]
 name = "lock_api"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
 dependencies = [
  "autocfg",
  "scopeguard",
@@ -2732,9 +2961,9 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
 
 [[package]]
 name = "log"
-version = "0.4.21"
+version = "0.4.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
 
 [[package]]
 name = "lz4_flex"
@@ -2804,9 +3033,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "mime_guess"
-version = "2.0.4"
+version = "2.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
 dependencies = [
  "mime",
  "unicase",
@@ -2820,41 +3049,19 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "miniz_oxide"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924"
 dependencies = [
  "adler2",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
-dependencies = [
- "libc",
- "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.48.0",
-]
-
-[[package]]
-name = "mio"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
- "hermit-abi",
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
@@ -2892,9 +3099,9 @@ dependencies = [
 
 [[package]]
 name = "num"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41"
+checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
 dependencies = [
  "num-bigint",
  "num-complex",
@@ -2906,20 +3113,19 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.4"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
 dependencies = [
- "autocfg",
  "num-integer",
  "num-traits",
 ]
 
 [[package]]
 name = "num-complex"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
 dependencies = [
  "num-traits",
 ]
@@ -2941,9 +3147,9 @@ dependencies = [
 
 [[package]]
 name = "num-iter"
-version = "0.1.44"
+version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9"
+checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -2952,11 +3158,10 @@ dependencies = [
 
 [[package]]
 name = "num-rational"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
 dependencies = [
- "autocfg",
  "num-bigint",
  "num-integer",
  "num-traits",
@@ -2964,9 +3169,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
 dependencies = [
  "autocfg",
  "libm",
@@ -3000,7 +3205,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -3014,9 +3219,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.32.2"
+version = "0.36.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
+checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
 dependencies = [
  "memchr",
 ]
@@ -3028,20 +3233,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf"
 dependencies = [
  "async-trait",
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
  "chrono",
  "futures",
  "httparse",
  "humantime",
- "hyper 1.4.1",
+ "hyper 1.6.0",
  "itertools 0.13.0",
  "md-5",
  "parking_lot",
  "percent-encoding",
  "quick-xml",
  "rand",
- "reqwest 0.12.8",
+ "reqwest 0.12.12",
  "ring",
  "serde",
  "serde_json",
@@ -3054,9 +3259,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
 [[package]]
 name = "openid"
@@ -3064,24 +3269,46 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "627898ab5b3fff5e5f1dc0e404bafdbb87a4337d815e86149f53640380946ccc"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "biscuit",
  "chrono",
  "lazy_static",
  "mime",
- "reqwest 0.12.8",
+ "reqwest 0.12.12",
  "serde",
  "serde_json",
- "thiserror 1.0.64",
+ "thiserror 1.0.69",
  "url",
  "validator",
 ]
 
 [[package]]
 name = "openssl-probe"
-version = "0.1.5"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
+
+[[package]]
+name = "openssl-src"
+version = "300.4.1+3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741"
+dependencies = [
+ "cc",
+ "libc",
+ "openssl-src",
+ "pkg-config",
+ "vcpkg",
+]
 
 [[package]]
 name = "opentelemetry"
@@ -3092,7 +3319,7 @@ dependencies = [
  "futures-sink",
  "js-sys",
  "pin-project-lite",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "tracing",
 ]
 
@@ -3123,7 +3350,7 @@ dependencies = [
  "percent-encoding",
  "rand",
  "serde_json",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
 ]
 
 [[package]]
@@ -3135,6 +3362,16 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "os_pipe"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -3143,15 +3380,15 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
 [[package]]
 name = "parking"
-version = "2.2.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
 [[package]]
 name = "parking_lot"
-version = "0.12.1"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
 dependencies = [
  "lock_api",
  "parking_lot_core",
@@ -3159,22 +3396,22 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.9"
+version = "0.9.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
 name = "parquet"
-version = "53.3.0"
+version = "53.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191"
+checksum = "8957c0c95a6a1804f3e51a18f69df29be53856a8c5768cc9b6d00fcafcd2917c"
 dependencies = [
  "ahash",
  "arrow-array",
@@ -3184,7 +3421,7 @@ dependencies = [
  "arrow-ipc",
  "arrow-schema",
  "arrow-select",
- "base64 0.22.0",
+ "base64 0.22.1",
  "brotli 7.0.0",
  "bytes",
  "chrono",
@@ -3208,9 +3445,9 @@ dependencies = [
 
 [[package]]
 name = "parse-zoneinfo"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41"
+checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24"
 dependencies = [
  "regex",
 ]
@@ -3234,7 +3471,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "async-trait",
- "base64 0.22.0",
+ "base64 0.22.1",
  "byteorder",
  "bytes",
  "cargo_toml",
@@ -3245,7 +3482,7 @@ dependencies = [
  "cookie 0.18.1",
  "crossterm",
  "datafusion",
- "derive_more",
+ "derive_more 1.0.0",
  "fs_extra",
  "futures",
  "futures-util",
@@ -3256,7 +3493,7 @@ dependencies = [
  "human-size",
  "humantime",
  "humantime-serde",
- "itertools 0.13.0",
+ "itertools 0.14.0",
  "lazy_static",
  "nom",
  "num_cpus",
@@ -3275,7 +3512,8 @@ dependencies = [
  "reqwest 0.11.27",
  "rstest",
  "rustls 0.22.4",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.2.0",
+ "sasl2-sys",
  "semver",
  "serde",
  "serde_json",
@@ -3285,9 +3523,10 @@ dependencies = [
  "static-files",
  "sysinfo",
  "temp-dir",
- "thiserror 2.0.9",
+ "thiserror 2.0.11",
  "tokio",
  "tokio-stream",
+ "tokio-util",
  "tonic",
  "tonic-web",
  "tower-http 0.6.2",
@@ -3298,6 +3537,7 @@ dependencies = [
  "ureq",
  "url",
  "vergen",
+ "vergen-gitcl",
  "xxhash-rust",
  "zip",
 ]
@@ -3348,28 +3588,28 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "petgraph"
-version = "0.6.4"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
+checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
 ]
 
 [[package]]
 name = "phf"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
 dependencies = [
  "phf_shared",
 ]
 
 [[package]]
 name = "phf_codegen"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a"
+checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
 dependencies = [
  "phf_generator",
  "phf_shared",
@@ -3377,9 +3617,9 @@ dependencies = [
 
 [[package]]
 name = "phf_generator"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
 dependencies = [
  "phf_shared",
  "rand",
@@ -3387,38 +3627,38 @@ dependencies = [
 
 [[package]]
 name = "phf_shared"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
 dependencies = [
  "siphasher",
 ]
 
 [[package]]
 name = "pin-project"
-version = "1.1.5"
+version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3"
+checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.5"
+version = "1.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
+checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.14"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
 
 [[package]]
 name = "pin-utils"
@@ -3428,9 +3668,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "powerfmt"
@@ -3440,9 +3680,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.17"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
 
 [[package]]
 name = "proc-macro-crate"
@@ -3479,31 +3722,41 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.86"
+version = "1.0.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "procfs"
-version = "0.14.2"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69"
+checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
 dependencies = [
- "bitflags 1.3.2",
- "byteorder",
+ "bitflags 2.8.0",
  "hex",
  "lazy_static",
- "rustix 0.36.17",
+ "procfs-core",
+ "rustix",
+]
+
+[[package]]
+name = "procfs-core"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
+dependencies = [
+ "bitflags 2.8.0",
+ "hex",
 ]
 
 [[package]]
 name = "prometheus"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
+checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
 dependencies = [
  "cfg-if",
  "fnv",
@@ -3513,7 +3766,7 @@ dependencies = [
  "parking_lot",
  "procfs",
  "protobuf",
- "thiserror 1.0.64",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -3530,9 +3783,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f"
+checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -3540,22 +3793,22 @@ dependencies = [
 
 [[package]]
 name = "prost-derive"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5"
+checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "prost-types"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670"
+checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc"
 dependencies = [
  "prost",
 ]
@@ -3603,57 +3856,61 @@ dependencies = [
 
 [[package]]
 name = "quinn"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
+checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
 dependencies = [
  "bytes",
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
  "rustc-hash",
- "rustls 0.23.13",
+ "rustls 0.23.22",
  "socket2",
- "thiserror 1.0.64",
+ "thiserror 2.0.11",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.8"
+version = "0.11.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
+checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
 dependencies = [
  "bytes",
+ "getrandom 0.2.15",
  "rand",
  "ring",
  "rustc-hash",
- "rustls 0.23.13",
+ "rustls 0.23.22",
+ "rustls-pki-types",
  "slab",
- "thiserror 1.0.64",
+ "thiserror 2.0.11",
  "tinyvec",
  "tracing",
+ "web-time",
 ]
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.4"
+version = "0.5.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285"
+checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904"
 dependencies = [
+ "cfg_aliases",
  "libc",
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
 dependencies = [
  "proc-macro2",
 ]
@@ -3685,7 +3942,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
 ]
 
 [[package]]
@@ -3719,9 +3976,9 @@ dependencies = [
 
 [[package]]
 name = "rdkafka"
-version = "0.36.2"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1beea247b9a7600a81d4cc33f659ce1a77e1988323d7d2809c7ed1c21f4c316d"
+checksum = "14b52c81ac3cac39c9639b95c20452076e74b8d9a71bc6fc4d83407af2ea6fff"
 dependencies = [
  "futures-channel",
  "futures-util",
@@ -3733,6 +3990,7 @@ dependencies = [
  "serde_json",
  "slab",
  "tokio",
+ "tracing",
 ]
 
 [[package]]
@@ -3741,9 +3999,13 @@ version = "4.8.0+2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced38182dc436b3d9df0c77976f37a67134df26b050df1f0006688e46fc4c8be"
 dependencies = [
+ "cmake",
  "libc",
+ "libz-sys",
  "num_enum",
+ "openssl-sys",
  "pkg-config",
+ "sasl2-sys",
 ]
 
 [[package]]
@@ -3763,27 +4025,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
 dependencies = [
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.4.1"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.8.0",
 ]
 
 [[package]]
 name = "regex"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.8",
+ "regex-automata 0.4.9",
  "regex-syntax 0.8.5",
 ]
 
@@ -3798,9 +4060,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -3849,7 +4111,7 @@ dependencies = [
  "h2 0.3.26",
  "http 0.2.12",
  "http-body 0.4.6",
- "hyper 0.14.30",
+ "hyper 0.14.32",
  "hyper-rustls 0.24.2",
  "ipnet",
  "js-sys",
@@ -3879,20 +4141,20 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.12.8"
+version = "0.12.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
+checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
  "futures-core",
  "futures-util",
- "h2 0.4.5",
- "http 1.1.0",
- "http-body 1.0.0",
+ "h2 0.4.7",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
- "hyper 1.4.1",
- "hyper-rustls 0.27.3",
+ "hyper 1.6.0",
+ "hyper-rustls 0.27.5",
  "hyper-util",
  "ipnet",
  "js-sys",
@@ -3902,24 +4164,25 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "quinn",
- "rustls 0.23.13",
+ "rustls 0.23.22",
  "rustls-native-certs",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sync_wrapper 1.0.1",
+ "sync_wrapper 1.0.2",
  "tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
  "tokio-util",
+ "tower 0.5.2",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots 0.26.1",
+ "webpki-roots 0.26.7",
  "windows-registry",
 ]
 
@@ -3931,7 +4194,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom",
+ "getrandom 0.2.15",
  "libc",
  "spin",
  "untrusted",
@@ -3964,21 +4227,21 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.87",
+ "syn 2.0.96",
  "unicode-ident",
 ]
 
 [[package]]
 name = "rustc-demangle"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
 [[package]]
 name = "rustc-hash"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
+checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497"
 
 [[package]]
 name = "rustc_version"
@@ -3991,29 +4254,15 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.36.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "305efbd14fde4139eb501df5f136994bb520b033fa9fbdce287507dc23b8c7ed"
-dependencies = [
- "bitflags 1.3.2",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.1.4",
- "windows-sys 0.45.0",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.34"
+version = "0.38.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.8.0",
  "errno",
  "libc",
- "linux-raw-sys 0.4.13",
- "windows-sys 0.52.0",
+ "linux-raw-sys",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4044,9 +4293,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.13"
+version = "0.23.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8"
+checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7"
 dependencies = [
  "log",
  "once_cell",
@@ -4059,12 +4308,11 @@ dependencies = [
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
+checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.2",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -4081,19 +4329,21 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.1.2"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
 dependencies = [
- "base64 0.22.0",
  "rustls-pki-types",
 ]
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.9.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55"
+checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
+dependencies = [
+ "web-time",
+]
 
 [[package]]
 name = "rustls-webpki"
@@ -4118,15 +4368,15 @@ dependencies = [
 
 [[package]]
 name = "rustversion"
-version = "1.0.15"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47"
+checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
 
 [[package]]
 name = "ryu"
-version = "1.0.17"
+version = "1.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
 
 [[package]]
 name = "same-file"
@@ -4137,13 +4387,25 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "sasl2-sys"
+version = "0.1.22+2.1.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f2a7f7efd9fc98b3a9033272df10709f5ee3fa0eabbd61a527a3a1ed6bd3c6"
+dependencies = [
+ "cc",
+ "duct",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "schannel"
-version = "0.1.23"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4164,12 +4426,12 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.10.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6"
+checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
 dependencies = [
- "bitflags 1.3.2",
- "core-foundation",
+ "bitflags 2.8.0",
+ "core-foundation 0.10.0",
  "core-foundation-sys",
  "libc",
  "security-framework-sys",
@@ -4177,9 +4439,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.10.0"
+version = "2.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef"
+checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -4187,9 +4449,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.22"
+version = "1.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
+checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03"
 dependencies = [
  "serde",
 ]
@@ -4202,32 +4464,33 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.198"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.198"
+version = "1.0.217"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.116"
+version = "1.0.138"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813"
+checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
@@ -4240,7 +4503,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -4277,9 +4540,9 @@ dependencies = [
 
 [[package]]
 name = "sha1_smol"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
 
 [[package]]
 name = "sha2"
@@ -4301,6 +4564,16 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "shared_child"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c"
+dependencies = [
+ "libc",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -4324,7 +4597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
 dependencies = [
  "libc",
- "mio 1.0.2",
+ "mio",
  "signal-hook",
 ]
 
@@ -4345,9 +4618,9 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
 
 [[package]]
 name = "siphasher"
-version = "0.3.11"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "slab"
@@ -4379,10 +4652,10 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
 dependencies = [
- "heck 0.5.0",
+ "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -4393,9 +4666,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.5.6"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871"
+checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
@@ -4425,27 +4698,33 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "stacker"
-version = "0.1.15"
+version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce"
+checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b"
 dependencies = [
  "cc",
  "cfg-if",
  "libc",
  "psm",
- "winapi",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "static-files"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64712ea1e3e140010e1d9605872ba205afa2ab5bd38191cc6ebd248ae1f6a06b"
+checksum = "4e8590e848e1c53be9258210bcd4a8f4118e08988f03a4e2d63b62e4ad9f7ced"
 dependencies = [
  "change-detection",
  "mime_guess",
@@ -4460,34 +4739,34 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "strsim"
-version = "0.10.0"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "strum"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
 
 [[package]]
 name = "strum_macros"
-version = "0.26.2"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
 dependencies = [
- "heck 0.4.1",
+ "heck",
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "subtle"
-version = "2.5.0"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
@@ -4501,9 +4780,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.87"
+version = "2.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
+checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4518,18 +4797,29 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
 name = "sync_wrapper"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
 dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
 [[package]]
 name = "sysinfo"
-version = "0.31.4"
+version = "0.33.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "355dbe4f8799b304b05e1b0f05fc59b2a18d36645cf169607da45bde2f69a1be"
+checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -4546,7 +4836,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
 dependencies = [
  "bitflags 1.3.2",
- "core-foundation",
+ "core-foundation 0.9.4",
  "system-configuration-sys",
 ]
 
@@ -4568,54 +4858,56 @@ checksum = "bc1ee6eef34f12f765cb94725905c6312b6610ab2b0940889cfe58dae7bc3c72"
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
 dependencies = [
  "cfg-if",
- "fastrand 2.0.2",
- "rustix 0.38.34",
- "windows-sys 0.52.0",
+ "fastrand 2.3.0",
+ "getrandom 0.3.1",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.64"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl 1.0.64",
+ "thiserror-impl 1.0.69",
 ]
 
 [[package]]
 name = "thiserror"
-version = "2.0.9"
+version = "2.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
 dependencies = [
- "thiserror-impl 2.0.9",
+ "thiserror-impl 2.0.11",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.64"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.9"
+version = "2.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -4641,9 +4933,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.36"
+version = "0.3.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
 dependencies = [
  "deranged",
  "itoa",
@@ -4664,9 +4956,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
 dependencies = [
  "num-conv",
  "time-core",
@@ -4681,11 +4973,21 @@ dependencies = [
  "crunchy",
 ]
 
+[[package]]
+name = "tinystr"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+dependencies = [
+ "displaydoc",
+ "zerovec",
+]
+
 [[package]]
 name = "tinyvec"
-version = "1.6.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -4698,32 +5000,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.37.0"
+version = "1.43.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
 dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio 0.8.11",
- "num_cpus",
+ "mio",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "tokio-macros"
-version = "2.2.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -4749,20 +5050,19 @@ dependencies = [
 
 [[package]]
 name = "tokio-rustls"
-version = "0.26.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
 dependencies = [
- "rustls 0.23.13",
- "rustls-pki-types",
+ "rustls 0.23.22",
  "tokio",
 ]
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.16"
+version = "0.1.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
+checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -4771,23 +5071,22 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.10"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
 dependencies = [
  "bytes",
  "futures-core",
  "futures-sink",
  "pin-project-lite",
  "tokio",
- "tracing",
 ]
 
 [[package]]
 name = "toml"
-version = "0.8.12"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -4806,11 +5105,11 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.22.22"
+version = "0.22.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
+checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -4826,25 +5125,25 @@ dependencies = [
  "async-stream",
  "async-trait",
  "axum",
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
  "flate2",
- "h2 0.4.5",
- "http 1.1.0",
- "http-body 1.0.0",
+ "h2 0.4.7",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
- "hyper 1.4.1",
+ "hyper 1.6.0",
  "hyper-timeout",
  "hyper-util",
  "percent-encoding",
  "pin-project",
  "prost",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.2.0",
  "socket2",
  "tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
  "tokio-stream",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -4857,10 +5156,10 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5299dd20801ad736dccb4a5ea0da7376e59cd98f213bf1c3d478cf53f4834b58"
 dependencies = [
- "base64 0.22.0",
+ "base64 0.22.1",
  "bytes",
- "http 1.1.0",
- "http-body 1.0.0",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
  "pin-project",
  "tokio-stream",
@@ -4891,16 +5190,31 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tower"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper 1.0.2",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-http"
 version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.8.0",
  "bytes",
- "http 1.1.0",
- "http-body 1.0.0",
+ "http 1.2.0",
+ "http-body 1.0.1",
  "http-body-util",
  "pin-project-lite",
  "tower-layer",
@@ -4913,9 +5227,9 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.8.0",
  "bytes",
- "http 1.1.0",
+ "http 1.2.0",
  "pin-project-lite",
  "tower-layer",
  "tower-service",
@@ -4929,9 +5243,9 @@ checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
@@ -4953,7 +5267,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -4979,9 +5293,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
 dependencies = [
  "matchers",
  "nu-ansi-term",
@@ -4990,6 +5304,7 @@ dependencies = [
  "sharded-slab",
  "smallvec",
  "thread_local",
+ "time",
  "tracing",
  "tracing-core",
  "tracing-log",
@@ -5019,11 +5334,10 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "ulid"
-version = "1.1.2"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34778c17965aa2a08913b57e1f34db9b4a63f5de31768b55bf20d2795f921259"
+checksum = "f294bff79170ed1c5633812aff1e565c35d993a36e757f9bc0accf5eec4e6045"
 dependencies = [
- "getrandom",
  "rand",
  "serde",
  "web-time",
@@ -5031,45 +5345,48 @@ dependencies = [
 
 [[package]]
 name = "unicase"
-version = "2.7.0"
+version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
-dependencies = [
- "version_check",
-]
+checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.15"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.13"
+version = "1.0.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
 dependencies = [
  "tinyvec",
 ]
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 
 [[package]]
 name = "unicode-width"
-version = "0.1.11"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
 [[package]]
 name = "untrusted"
@@ -5079,57 +5396,68 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "uptime_lib"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4e71ddbefed856d5881821d6ada4e606bbb91fd332296963ed596e2ad2100f3"
+checksum = "9e64b558561f12a171bbea5325c3f24f129db371adee1d7ae93b6e310bd69192"
 dependencies = [
  "libc",
- "thiserror 1.0.64",
- "windows 0.52.0",
+ "thiserror 1.0.69",
+ "windows 0.57.0",
 ]
 
 [[package]]
 name = "ureq"
-version = "2.9.6"
+version = "2.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35"
+checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.22.1",
  "flate2",
  "log",
  "once_cell",
- "rustls 0.22.4",
+ "rustls 0.23.22",
  "rustls-pki-types",
- "rustls-webpki 0.102.8",
  "url",
- "webpki-roots 0.26.1",
+ "webpki-roots 0.26.7",
 ]
 
 [[package]]
 name = "url"
-version = "2.5.0"
+version = "2.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
 dependencies = [
  "form_urlencoded",
- "idna",
+ "idna 1.0.3",
  "percent-encoding",
  "serde",
 ]
 
+[[package]]
+name = "utf16_iter"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+
+[[package]]
+name = "utf8_iter"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
+
 [[package]]
 name = "utf8parse"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.8.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
+checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b"
 dependencies = [
- "getrandom",
+ "getrandom 0.2.15",
 ]
 
 [[package]]
@@ -5138,7 +5466,7 @@ version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "db79c75af171630a3148bd3e6d7c4f42b6a9a014c2945bc5ed0020cbb8d9478e"
 dependencies = [
- "idna",
+ "idna 0.5.0",
  "once_cell",
  "regex",
  "serde",
@@ -5150,49 +5478,83 @@ dependencies = [
 
 [[package]]
 name = "validator_derive"
-version = "0.18.1"
+version = "0.18.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55591299b7007f551ed1eb79a684af7672c19c3193fb9e0a31936987bb2438ec"
+checksum = "df0bcf92720c40105ac4b2dda2a4ea3aa717d4d6a862cc217da653a4bd5c6b10"
 dependencies = [
  "darling",
  "once_cell",
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
 name = "valuable"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vergen"
-version = "8.3.1"
+version = "9.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525"
+checksum = "e0d2f179f8075b805a43a2a21728a46f0cc2921b3c58695b28fa8817e103cd9a"
 dependencies = [
  "anyhow",
  "cargo_metadata",
- "cfg-if",
+ "derive_builder",
  "regex",
+ "rustc_version",
  "rustversion",
+ "sysinfo",
  "time",
+ "vergen-lib",
+]
+
+[[package]]
+name = "vergen-gitcl"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2f89d70a58a4506a6079cedf575c64cf51649ccbb4e02a63dac539b264b7711"
+dependencies = [
+ "anyhow",
+ "derive_builder",
+ "rustversion",
+ "time",
+ "vergen",
+ "vergen-lib",
+]
+
+[[package]]
+name = "vergen-lib"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166"
+dependencies = [
+ "anyhow",
+ "derive_builder",
+ "rustversion",
 ]
 
 [[package]]
 name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "waker-fn"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690"
+checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
 
 [[package]]
 name = "walkdir"
@@ -5225,48 +5587,59 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasi"
+version = "0.13.3+wasi-0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
+dependencies = [
+ "wit-bindgen-rt",
+]
+
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
 dependencies = [
  "cfg-if",
+ "once_cell",
+ "rustversion",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
 dependencies = [
  "bumpalo",
  "log",
- "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.42"
+version = "0.4.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
 dependencies = [
  "cfg-if",
  "js-sys",
+ "once_cell",
  "wasm-bindgen",
  "web-sys",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -5274,28 +5647,31 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+dependencies = [
+ "unicode-ident",
+]
 
 [[package]]
 name = "wasm-streams"
-version = "0.4.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
 dependencies = [
  "futures-util",
  "js-sys",
@@ -5306,9 +5682,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.69"
+version = "0.3.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -5332,9 +5708,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.1"
+version = "0.26.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009"
+checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -5357,11 +5733,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.7"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "134306a13c5647ad6453e8deaec55d3a44d6021970129e6188735e74bf546697"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5419,7 +5795,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -5430,7 +5806,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
 ]
 
 [[package]]
@@ -5472,15 +5848,6 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
-[[package]]
-name = "windows-sys"
-version = "0.45.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
-dependencies = [
- "windows-targets 0.42.2",
-]
-
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -5500,18 +5867,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "windows-targets"
-version = "0.42.2"
+name = "windows-sys"
+version = "0.59.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
 dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -5545,12 +5906,6 @@ dependencies = [
  "windows_x86_64_msvc 0.52.6",
 ]
 
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
-
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.48.5"
@@ -5563,12 +5918,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
-
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.5"
@@ -5581,12 +5930,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
-
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.5"
@@ -5605,12 +5948,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
-
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.5"
@@ -5623,12 +5960,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
-
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.5"
@@ -5641,12 +5972,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
-
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.48.5"
@@ -5659,12 +5984,6 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
-
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.5"
@@ -5679,9 +5998,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.6.20"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
+checksum = "7e49d2d35d3fad69b39b94139037ecfb4f359f08958b9c11e7315ce770462419"
 dependencies = [
  "memchr",
 ]
@@ -5696,11 +6015,32 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "wit-bindgen-rt"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
+dependencies = [
+ "bitflags 2.8.0",
+]
+
+[[package]]
+name = "write16"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
+
+[[package]]
+name = "writeable"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+
 [[package]]
 name = "xxhash-rust"
-version = "0.8.10"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
 
 [[package]]
 name = "xz2"
@@ -5711,46 +6051,114 @@ dependencies = [
  "lzma-sys",
 ]
 
+[[package]]
+name = "yoke"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+ "synstructure",
+]
+
 [[package]]
 name = "zerocopy"
-version = "0.7.32"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
 dependencies = [
+ "byteorder",
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.32"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
+
+[[package]]
+name = "zerofrom"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
+checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.87",
+ "syn 2.0.96",
+ "synstructure",
 ]
 
 [[package]]
 name = "zeroize"
-version = "1.7.0"
+version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
+checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+
+[[package]]
+name = "zerovec"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+dependencies = [
+ "yoke",
+ "zerofrom",
+ "zerovec-derive",
+]
+
+[[package]]
+name = "zerovec-derive"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.96",
+]
 
 [[package]]
 name = "zip"
-version = "2.2.0"
+version = "2.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494"
+checksum = "ae9c1ea7b3a5e1f4b922ff856a129881167511563dc219869afe3787fc0c1a45"
 dependencies = [
  "arbitrary",
  "crc32fast",
  "crossbeam-utils",
  "displaydoc",
  "flate2",
- "indexmap 2.5.0",
+ "indexmap 2.7.1",
  "memchr",
- "thiserror 1.0.64",
+ "thiserror 2.0.11",
  "zopfli",
 ]
 
@@ -5770,27 +6178,27 @@ dependencies = [
 
 [[package]]
 name = "zstd"
-version = "0.13.1"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
+checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
 dependencies = [
  "zstd-safe",
 ]
 
 [[package]]
 name = "zstd-safe"
-version = "7.1.0"
+version = "7.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
+checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
 dependencies = [
  "zstd-sys",
 ]
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.10+zstd.1.5.6"
+version = "2.0.13+zstd.1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
+checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
 dependencies = [
  "cc",
  "pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index b565185d5..4f789ba04 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,6 +32,10 @@ tonic-web = "0.12.3"
 tower-http = { version = "0.6.1", features = ["cors"] }
 url = "2.4.0"
 
+# Connectors dependencies
+rdkafka = { version = "0.37", optional = true, features = ["cmake-build", "tracing", "libz-static"] }
+sasl2-sys = { version = "0.1.22", optional = true, features = ["vendored"] }
+
 # Authentication and Security
 argon2 = "0.5.0"
 base64 = "0.22.0"
@@ -49,22 +53,24 @@ serde_json = "1.0"
 serde_repr = "0.1.17"
 
 # Async and Runtime
-async-trait = "0.1.82"
+async-trait = "0.1"
 futures = "0.3"
-futures-util = "0.3.28"
-tokio = { version = "1.28", default-features = false, features = [
+futures-util = "0.3"
+tokio = { version = "^1.43", default-features = false, features = [
     "sync",
     "macros",
     "fs",
+    "rt-multi-thread",
 ] }
 tokio-stream = { version = "0.1", features = ["fs"] }
+tokio-util = { version = "0.7" }
 
 # Logging and Metrics
 opentelemetry-proto = { git = "https://github.com/parseablehq/opentelemetry-rust", branch = "fix-metrics-u64-serialization" }
 prometheus = { version = "0.13", features = ["process"] }
 prometheus-parse = "0.2.5"
-tracing = "0.1.41"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "time"] }
 
 # Time and Date
 chrono = "0.4"
@@ -78,7 +84,7 @@ path-clean = "1.0.1"
 relative-path = { version = "1.7", features = ["serde"] }
 
 # CLI and System
-clap = { version = "4.1", default-features = false, features = [
+clap = { version = "4.5", default-features = false, features = [
     "std",
     "color",
     "help",
@@ -91,18 +97,18 @@ crossterm = "0.28.1"
 hostname = "0.4.0"
 human-size = "0.4"
 num_cpus = "1.15"
-sysinfo = "0.31.4"
+sysinfo = "0.33.1"
 uptime_lib = "0.3.0"
 
 # Utility Libraries
 anyhow = { version = "1.0", features = ["backtrace"] }
 bytes = "1.4"
 clokwerk = "0.4"
-derive_more = "0.99.18"
-itertools = "0.13.0"
+derive_more = { version = "1", features = ["full"] }
+itertools = "0.14"
 lazy_static = "1.4"
 nom = "7.1.3"
-once_cell = "1.17.1"
+once_cell = "1.20"
 rand = "0.8.5"
 regex = "1.7.3"
 reqwest = { version = "0.11.27", default-features = false, features = [
@@ -113,18 +119,20 @@ reqwest = { version = "0.11.27", default-features = false, features = [
 ] } # cannot update cause rustls is not latest `see rustls`
 semver = "1.0"
 static-files = "0.2"
-thiserror = "2.0.0"
+thiserror = "2.0"
 ulid = { version = "1.0", features = ["serde"] }
 xxhash-rust = { version = "0.8", features = ["xxh3"] }
 
 [build-dependencies]
-cargo_toml = "0.20.1"
+cargo_toml = "0.21"
 sha1_smol = { version = "1.0", features = ["std"] }
 static-files = "0.2"
-ureq = "2.6"
-url = "2.4.0"
-vergen = { version = "8.1", features = ["build", "git", "cargo", "gitcl"] }
-zip = { version = "2.2.0", default-features = false, features = ["deflate"] }
+ureq = "2.12"
+url = "2.5"
+vergen = { version = "9.0", features = ["build", "cargo", "rustc", "si"] }
+vergen-gitcl = { version = "1.0", features = ["build", "cargo", "rustc", "si"] }
+zip = { version = "2.2", default-features = false, features = ["deflate"] }
+anyhow = "1.0"
 
 [dev-dependencies]
 rstest = "0.23.0"
@@ -135,14 +143,11 @@ temp-dir = "0.1.14"
 assets-url = "https://github.com/parseablehq/console/releases/download/v0.9.18/build.zip"
 assets-sha1 = "4516db38c8e556707b29b33569f9b1e53d5165f2"
 
+[features]
+debug = []
+kafka = ["rdkafka", "sasl2-sys", "sasl2-sys/vendored", "rdkafka/ssl-vendored", "rdkafka/ssl", "rdkafka/sasl"]
+
 [profile.release-lto]
 inherits = "release"
 lto = "fat"
 codegen-units = 1
-
-# adding rdkafka here because, for unsupported platforms, cargo skips other deps which come after this
-[target.'cfg(all(target_os = "linux", target_arch = "x86_64"))'.dependencies]
-rdkafka = { version = "0.36.2", default-features = false, features = ["tokio"] }
-
-[target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dependencies]
-rdkafka = { version = "0.36.2", default-features = false, features = ["tokio"] }
diff --git a/Dockerfile b/Dockerfile
index 5b7c71af3..a9980e8d9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,7 +14,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # build stage
-FROM rust:1.83.0-bookworm AS builder
+FROM  rust:1.84.0-bookworm AS builder
+
 
 LABEL org.opencontainers.image.title="Parseable"
 LABEL maintainer="Parseable Team <hi@parseable.io>"
@@ -24,7 +25,7 @@ LABEL org.opencontainers.image.licenses="AGPL-3.0"
 WORKDIR /parseable
 
 # Cache dependencies
-COPY Cargo.toml Cargo.lock build.rs .git ./
+COPY Cargo.toml Cargo.lock build.rs ./
 RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build --release && rm -rf src
 
 # Build the actual binary
diff --git a/Dockerfile.debug b/Dockerfile.debug
index 02a450e4f..1d21291f0 100644
--- a/Dockerfile.debug
+++ b/Dockerfile.debug
@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # build stage
-FROM rust:1.83.0-bookworm as builder
+FROM rust:1.84.0-bookworm AS builder
 
 LABEL org.opencontainers.image.title="Parseable"
 LABEL maintainer="Parseable Team <hi@parseable.io>"
@@ -24,7 +24,7 @@ LABEL org.opencontainers.image.licenses="AGPL-3.0"
 WORKDIR /parseable
 
 # Cache dependencies
-COPY Cargo.toml Cargo.lock build.rs .git ./
+COPY Cargo.toml Cargo.lock build.rs ./
 RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build && rm -rf src
 
 # Build the actual binary
diff --git a/Dockerfile.dev b/Dockerfile.dev
index b55866caa..227ade1ce 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -14,7 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # build stage
-FROM rust:1.83.0-bookworm AS builder
+FROM rust:1.84.0-bookworm AS builder
 
 LABEL org.opencontainers.image.title="Parseable"
 LABEL maintainer="Parseable Team <hi@parseable.io>"
@@ -34,7 +34,7 @@ FROM gcr.io/distroless/cc-debian12:latest
 
 WORKDIR /parseable
 
-# Copy the static shell into base image.
+# Copy the Parseable binary from builder
 COPY --from=builder /parseable/target/release/parseable /usr/bin/parseable
 
 CMD ["/usr/bin/parseable"]
diff --git a/Dockerfile.kafka b/Dockerfile.kafka
new file mode 100644
index 000000000..0c45fe7d6
--- /dev/null
+++ b/Dockerfile.kafka
@@ -0,0 +1,66 @@
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# build stage
+FROM  rust:1.84.0-bookworm AS builder
+
+LABEL org.opencontainers.image.title="Parseable"
+LABEL maintainer="Parseable Team <hi@parseable.io>"
+LABEL org.opencontainers.image.vendor="Parseable Inc"
+LABEL org.opencontainers.image.licenses="AGPL-3.0"
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y \
+    cmake \
+    clang \
+    librdkafka-dev \
+    ca-certificates \
+    build-essential \
+    libsasl2-dev \
+    libssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /parseable
+COPY Cargo.toml Cargo.lock build.rs ./
+
+# Create a dummy main.rs to pre-cache dependencies
+RUN mkdir src && echo "fn main() {}" > src/main.rs && \
+    cargo build --release --features kafka && \
+    rm -rf src
+
+# Copy the actual source code
+COPY src ./src
+
+# Build the actual binary with kafka feature
+RUN cargo build --release --features kafka
+
+# final stage
+FROM gcr.io/distroless/cc-debian12:latest
+
+WORKDIR /parseable
+
+# Copy the Parseable binary from builder
+COPY --from=builder /parseable/target/release/parseable /usr/bin/parseable
+
+# Copy only the libraries that binary needs since kafka is statically linked
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libsasl2.so.2 /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libssl.so.3   /usr/lib/x86_64-linux-gnu/
+COPY --from=builder /usr/lib/x86_64-linux-gnu/libcrypto.so.3 /usr/lib/x86_64-linux-gnu/
+
+# Copy CA certificates
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+
+
+CMD ["/usr/bin/parseable"]
diff --git a/build.rs b/build.rs
index fa773d2c8..e73bf7082 100644
--- a/build.rs
+++ b/build.rs
@@ -16,17 +16,21 @@
  *
  */
 
-use std::error::Error;
-use vergen::EmitBuilder;
+use anyhow::Result;
+use vergen_gitcl::{
+    BuildBuilder, CargoBuilder, Emitter, GitclBuilder, RustcBuilder, SysinfoBuilder,
+};
 
-fn main() -> Result<(), Box<dyn Error>> {
-    ui::setup().unwrap();
+pub fn main() -> Result<()> {
+    ui::setup()?;
 
     // Init vergen
-    EmitBuilder::builder()
-        .all_build()
-        .all_cargo()
-        .git_sha(true)
+    Emitter::default()
+        .add_instructions(&BuildBuilder::all_build()?)?
+        .add_instructions(&CargoBuilder::all_cargo()?)?
+        .add_instructions(&GitclBuilder::default().all().sha(true).build()?)?
+        .add_instructions(&RustcBuilder::all_rustc()?)?
+        .add_instructions(&SysinfoBuilder::all_sysinfo()?)?
         .emit()?;
 
     Ok(())
diff --git a/docker-compose-distributed-test-with-kafka.yaml b/docker-compose-distributed-test-with-kafka.yaml
new file mode 100644
index 000000000..d65f7390b
--- /dev/null
+++ b/docker-compose-distributed-test-with-kafka.yaml
@@ -0,0 +1,366 @@
+networks:
+  parseable-internal:
+
+services:
+  # HAProxy Load Balancer
+  parseable-ingest-haproxy:
+    image: haproxy:3.0.7-alpine3.21
+    ports:
+      - "9001:9001"  # HAProxy stats
+      - "8001:8001"  # Load balanced ingestion endpoint
+    volumes:
+      - ./parseable-ingest-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    depends_on:
+      - parseable-ingest-one
+      - parseable-ingest-two
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg" ]
+      interval: 15s
+      timeout: 10s
+      retries: 3
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  # minio
+  minio:
+    image: minio/minio:RELEASE.2023-02-10T18-48-39Z
+    entrypoint:
+      - sh
+      - -euc
+      - |
+        mkdir -p /tmp/minio/parseable && \
+        minio server /tmp/minio
+    environment:
+      - MINIO_ROOT_USER=parseable
+      - MINIO_ROOT_PASSWORD=supersecret
+      - MINIO_UPDATE=off
+    ports:
+      - "9000:9000"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    networks:
+      - parseable-internal
+    volumes:
+      - minio_data:/tmp/minio
+
+  # query server
+  parseable-query:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    command: [ "parseable", "s3-store" ]
+    ports:
+      - "8000:8000"
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=query
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+    volumes:
+      - parseable_query_data:/tmp/data
+
+  # ingest server one
+  parseable-ingest-one:
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
+    platform: linux/amd64
+    command: [ "parseable", "s3-store", ]
+    expose:
+      - "8000"
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
+      - P_KAFKA_CONSUMER_TOPICS=dist-test-logs-stream
+      - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
+      - P_KAFKA_PARTITION_LISTENER_CONCURRENCY=3
+      # additional settings like security, tuning, etc.
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+      - kafka-0
+      - kafka-1
+      - kafka-2
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+    volumes:
+      - parseable_ingest_one_data:/tmp/data
+
+  parseable-ingest-two:
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
+    platform: linux/amd64
+    command: [ "parseable", "s3-store", ]
+    expose:
+      - "8000"
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_MODE=ingest
+      - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000
+      - P_KAFKA_CONSUMER_TOPICS=dist-test-logs-stream
+      - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
+      - P_KAFKA_PARTITION_LISTENER_CONCURRENCY=3
+      # additional settings like security, tuning, etc.
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    depends_on:
+      - parseable-query
+      - minio
+      - kafka-0
+      - kafka-1
+      - kafka-2
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+    volumes:
+      - parseable_ingest_two_data:/tmp/data
+
+  quest:
+    platform: linux/amd64
+    image: ghcr.io/parseablehq/quest:main
+    command:
+      [
+        "load",
+        "http://parseable-query:8000",
+        "parseableadmin",
+        "parseableadmin",
+        "20",
+        "10",
+        "5m",
+        "minio:9000",
+        "parseable",
+        "supersecret",
+        "parseable",
+        "http://parseable-ingest-haproxy:8001",
+        "parseableadmin",
+        "parseableadmin",
+      ]
+    networks:
+      - parseable-internal
+    depends_on:
+      - parseable-query
+      - parseable-ingest-haproxy
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  kafka-0:
+    image: docker.io/bitnami/kafka:3.9
+    ports:
+      - "9092"
+    environment:
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=0
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
+      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
+      # Listeners
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+      # Clustering
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
+    volumes:
+      - kafka_0_data:/bitnami/kafka
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-1:
+    image: docker.io/bitnami/kafka:3.9
+    ports:
+      - "9092"
+    environment:
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=1
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
+      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
+      # Listeners
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+      # Clustering
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
+    volumes:
+      - kafka_1_data:/bitnami/kafka
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-2:
+    image: docker.io/bitnami/kafka:3.9
+    ports:
+      - "9092"
+    environment:
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=2
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
+      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
+      # Listeners
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+      # Clustering
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
+    volumes:
+      - kafka_2_data:/bitnami/kafka
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-ui:
+    platform: linux/amd64
+    image: provectuslabs/kafka-ui:latest
+    environment:
+      KAFKA_CLUSTERS_0_NAME: dist-test
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-0:9092,kafka-1:9092,kafka-2:9092
+      KAFKA_CLUSTERS_0_METRICS_PORT: 9101
+      DYNAMIC_CONFIG_ENABLED: "true"
+      JAVA_OPTS: -Xms256m -Xmx512m -XX:+UseG1GC
+    networks:
+      - parseable-internal
+    depends_on:
+      - kafka-0
+      - kafka-1
+      - kafka-2
+    ports:
+      - "8080:8080"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  kafka-log-generator:
+    build:
+      context: ./scripts
+      dockerfile: Dockerfile
+    environment:
+      - KAFKA_BROKERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
+      - KAFKA_TOPIC=dist-test-logs-stream
+      - LOG_RATE=5000
+      - TOTAL_LOGS=1_000_000
+      - REPLICATION_FACTOR=3
+    depends_on:
+      - kafka-0
+      - kafka-1
+      - kafka-2
+    networks:
+      - parseable-internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+volumes:
+  minio_data:
+    driver: local
+  parseable_query_data:
+    driver: local
+  parseable_ingest_one_data:
+    driver: local
+  parseable_ingest_two_data:
+    driver: local
+  kafka_0_data:
+    driver: local
+  kafka_1_data:
+    driver: local
+  kafka_2_data:
+    driver: local
diff --git a/docker-compose-distributed-test.yaml b/docker-compose-distributed-test.yaml
index 06cfd585b..ca79941f3 100644
--- a/docker-compose-distributed-test.yaml
+++ b/docker-compose-distributed-test.yaml
@@ -1,6 +1,6 @@
-version: "3.7"
 networks:
   parseable-internal:
+
 services:
   # minio
   minio:
@@ -16,9 +16,9 @@ services:
       - MINIO_ROOT_PASSWORD=supersecret
       - MINIO_UPDATE=off
     ports:
-      - 9000:9000
+      - "9000:9000"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
       interval: 15s
       timeout: 20s
       retries: 5
@@ -29,9 +29,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: ["parseable", "s3-store"]
+    platform: linux/amd64
+    command: [ "parseable", "s3-store" ]
     ports:
-      - 8000:8000
+      - "8000:8000"
     environment:
       - P_S3_URL=http://minio:9000
       - P_S3_ACCESS_KEY=parseable
@@ -47,7 +48,7 @@ services:
     networks:
       - parseable-internal
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
       interval: 15s
       timeout: 20s
       retries: 5
@@ -63,9 +64,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: ["parseable", "s3-store"]
+    platform: linux/amd64
+    command: [ "parseable", "s3-store", ]
     ports:
-      - 8000
+      - "8000"
     environment:
       - P_S3_URL=http://minio:9000
       - P_S3_ACCESS_KEY=parseable
@@ -82,7 +84,7 @@ services:
     networks:
       - parseable-internal
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness"]
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
       interval: 15s
       timeout: 20s
       retries: 5
diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml
new file mode 100644
index 000000000..c44283dd3
--- /dev/null
+++ b/docker-compose-local.yaml
@@ -0,0 +1,41 @@
+services:
+  kafka:
+    image: docker.io/bitnami/kafka:3.9
+    ports:
+      - "9092:9092"
+      - "29092:29092"
+    volumes:
+      - "kafka_data:/bitnami"
+    environment:
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=0
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
+      # Listeners for internal and external communication
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://0.0.0.0:9092,PLAINTEXT_INTERNAL://0.0.0.0:29092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://kafka:29092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT_INTERNAL
+
+  kafka-ui:
+    platform: linux/amd64
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - "8080:8080"
+    depends_on:
+      - kafka
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
+      KAFKA_CLUSTERS_0_METRICS_PORT: 9101
+      DYNAMIC_CONFIG_ENABLED: "true"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+volumes:
+  kafka_data:
+    driver: local
diff --git a/docker-compose-test-with-kafka.yaml b/docker-compose-test-with-kafka.yaml
new file mode 100644
index 000000000..0e747c097
--- /dev/null
+++ b/docker-compose-test-with-kafka.yaml
@@ -0,0 +1,164 @@
+networks:
+  parseable-internal:
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2023-02-10T18-48-39Z
+    entrypoint:
+      - sh
+      - -euc
+      - |
+        mkdir -p /tmp/minio/parseable && \
+        minio server /tmp/minio
+    environment:
+      - MINIO_ROOT_USER=parseable
+      - MINIO_ROOT_PASSWORD=supersecret
+      - MINIO_UPDATE=off
+    ports:
+      - "9000:9000"
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    networks:
+      - parseable-internal
+
+  parseable:
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
+    platform: linux/amd64
+    command: [ "parseable", "s3-store", ]
+    ports:
+      - "8000:8000"
+    environment:
+      - P_S3_URL=http://minio:9000
+      - P_S3_ACCESS_KEY=parseable
+      - P_S3_SECRET_KEY=supersecret
+      - P_S3_REGION=us-east-1
+      - P_S3_BUCKET=parseable
+      - P_STAGING_DIR=/tmp/data
+      - P_USERNAME=parseableadmin
+      - P_PASSWORD=parseableadmin
+      - P_CHECK_UPDATE=false
+      - P_PARQUET_COMPRESSION_ALGO=snappy
+      - P_KAFKA_CONSUMER_TOPICS=test-logs-stream
+      - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092
+      # additional settings like security, tuning, etc.
+    depends_on:
+      - minio
+      - kafka-0
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
+      interval: 15s
+      timeout: 20s
+      retries: 5
+    networks:
+      - parseable-internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  quest:
+    image: ghcr.io/parseablehq/quest:main
+    platform: linux/amd64
+    command: [
+      "load",
+      "http://parseable:8000",
+      "parseableadmin",
+      "parseableadmin",
+      "20",
+      "10",
+      "5m",
+      "minio:9000",
+      "parseable",
+      "supersecret",
+      "parseable"
+    ]
+    depends_on:
+      - parseable
+    networks:
+      - parseable-internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  kafka-0:
+    image: docker.io/bitnami/kafka:3.9
+    ports:
+      - "9092:9092"
+      - "11001:11001"
+    environment:
+      # KRaft settings
+      - KAFKA_CFG_NODE_ID=0
+      - KAFKA_CFG_PROCESS_ROLES=controller,broker
+      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093
+      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
+      # Listeners
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
+      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
+      # Clustering
+      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
+    volumes:
+      - kafka_0_data:/bitnami/kafka
+    networks:
+      - parseable-internal
+    healthcheck:
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka-ui:
+    platform: linux/amd64
+    image: provectuslabs/kafka-ui:latest
+    ports:
+      - "8080:8080"
+    depends_on:
+      - kafka-0
+    environment:
+      KAFKA_CLUSTERS_0_NAME: test
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-0:9092
+      KAFKA_CLUSTERS_0_METRICS_PORT: 11001
+      DYNAMIC_CONFIG_ENABLED: "true"
+      JAVA_OPTS: -Xms256m -Xmx512m -XX:+UseG1GC
+    networks:
+      - parseable-internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+  kafka-log-generator:
+    build:
+      context: ./scripts
+      dockerfile: Dockerfile
+    environment:
+      - KAFKA_BROKERS=kafka-0:9092
+      - KAFKA_TOPIC=test-logs-stream
+      - LOG_RATE=5000
+      - TOTAL_LOGS=500_000
+    depends_on:
+      - kafka-0
+    networks:
+      - parseable-internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 20s
+        max_attempts: 3
+
+volumes:
+  kafka_0_data:
+    driver: local
diff --git a/docker-compose-test.yaml b/docker-compose-test.yaml
index 59b323c78..0a81d3c8c 100644
--- a/docker-compose-test.yaml
+++ b/docker-compose-test.yaml
@@ -1,5 +1,3 @@
-version: "3.7"
-
 networks:
   parseable-internal:
 
@@ -17,7 +15,7 @@ services:
       - MINIO_ROOT_PASSWORD=supersecret
       - MINIO_UPDATE=off
     ports:
-      - 9000
+      - "9000:9000"
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
       interval: 15s
@@ -30,9 +28,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: ["parseable", "s3-store"]
+    platform: linux/amd64
+    command: [ "parseable", "s3-store", ]
     ports:
-      - 8000
+      - "8000:8000"
     environment:
       - P_S3_URL=http://minio:9000
       - P_S3_ACCESS_KEY=parseable
@@ -44,15 +43,15 @@ services:
       - P_PASSWORD=parseableadmin
       - P_CHECK_UPDATE=false
       - P_PARQUET_COMPRESSION_ALGO=snappy
-    networks:
-      - parseable-internal
+    depends_on:
+      - minio
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
       interval: 15s
       timeout: 20s
       retries: 5
-    depends_on:
-      - minio
+    networks:
+      - parseable-internal
     deploy:
       restart_policy:
         condition: on-failure
@@ -61,11 +60,24 @@ services:
 
   quest:
     image: ghcr.io/parseablehq/quest:main
-    command: ["load", "http://parseable:8000", "parseableadmin", "parseableadmin", "20", "10", "5m", "minio:9000", "parseable", "supersecret", "parseable"]
-    networks:
-      - parseable-internal
+    platform: linux/amd64
+    command: [
+      "load",
+      "http://parseable:8000",
+      "parseableadmin",
+      "parseableadmin",
+      "20",
+      "10",
+      "5m",
+      "minio:9000",
+      "parseable",
+      "supersecret",
+      "parseable"
+    ]
     depends_on:
       - parseable
+    networks:
+      - parseable-internal
     deploy:
       restart_policy:
         condition: on-failure
diff --git a/parseable-ingest-haproxy.cfg b/parseable-ingest-haproxy.cfg
new file mode 100644
index 000000000..073150996
--- /dev/null
+++ b/parseable-ingest-haproxy.cfg
@@ -0,0 +1,45 @@
+global
+    log stdout format raw local0
+    maxconn 60000
+    daemon
+
+defaults
+    log     global
+    mode    http
+    option  httplog
+    option  dontlognull
+    timeout connect 5000
+    timeout client  50000
+    timeout server  50000
+
+frontend stats
+    bind *:9001
+    stats enable
+    stats uri /
+    stats refresh 30s
+    stats admin if TRUE
+
+frontend ingestion_frontend
+    bind *:8001
+    mode http
+    default_backend ingestion_backend
+
+backend ingestion_backend
+    mode http
+    balance roundrobin
+    option forwardfor
+
+    # Health check configuration
+    option httpchk GET /api/v1/liveness
+    http-check expect status 200
+
+    # Backend servers
+    server ingest1 parseable-ingest-one:8000 check inter 5s rise 2 fall 3
+    server ingest2 parseable-ingest-two:8000 check inter 5s rise 2 fall 3
+
+    # Session persistence
+    hash-type consistent
+
+    # Retry configuration
+    retries 3
+    option redispatch
diff --git a/scripts/Dockerfile b/scripts/Dockerfile
new file mode 100644
index 000000000..8a0a9d27d
--- /dev/null
+++ b/scripts/Dockerfile
@@ -0,0 +1,29 @@
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+FROM python:3.13-slim-bookworm
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    librdkafka-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN pip install confluent-kafka
+RUN pip install faker
+
+WORKDIR /app
+COPY kafka_log_stream_generator.py /app/
+
+CMD ["python", "/app/kafka_log_stream_generator.py"]
diff --git a/scripts/kafka_log_stream_generator.py b/scripts/kafka_log_stream_generator.py
new file mode 100644
index 000000000..ab6cf89ae
--- /dev/null
+++ b/scripts/kafka_log_stream_generator.py
@@ -0,0 +1,237 @@
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import logging
+import os
+import random
+import sys
+import time
+import uuid
+from datetime import datetime, timezone
+from typing import Dict, Any
+
+from confluent_kafka import Producer
+from confluent_kafka.admin import AdminClient
+from confluent_kafka.cimpl import NewTopic
+from faker import Faker
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[logging.StreamHandler(sys.stdout)]
+)
+
+logger = logging.getLogger(__name__)
+fake = Faker()
+
+# Kafka Configuration
+KAFKA_BROKERS = os.getenv("KAFKA_BROKERS", "localhost:9092")
+KAFKA_TOPIC = os.getenv("KAFKA_TOPIC", "local-logs-stream")
+NUM_PARTITIONS = int(os.getenv("NUM_PARTITIONS", "6"))  # Default partitions
+REPLICATION_FACTOR = int(os.getenv("REPLICATION_FACTOR", "1"))  # Default RF
+TOTAL_LOGS = int(os.getenv("TOTAL_LOGS", "100"))  # Total logs to produce
+LOG_RATE = int(os.getenv("LOG_RATE", "50"))  # Logs per second
+REPORT_EVERY = 5_000  # Progress report frequency
+
+# Kubernetes Configuration
+K8S_NAMESPACES = ["default", "kube-system", "monitoring", "logging", "app"]
+CONTAINER_IMAGES = [
+    "parseable/parseable:v1.8.1",
+    "parseable/query-service:v1.8.1",
+    "parseable/ingester:v1.8.1",
+    "parseable/frontend:v1.8.1"
+]
+NODE_TYPES = ["compute", "storage", "ingestion"]
+COMPONENTS = ["query", "storage", "ingestion", "frontend"]
+
+producer_conf = {
+    "bootstrap.servers": KAFKA_BROKERS,
+    "queue.buffering.max.messages": 200_000,
+    "queue.buffering.max.ms": 100,  # Up to 100ms linger
+    "batch.num.messages": 10_000,
+    "compression.type": "lz4",  # Compression (lz4, snappy, zstd, gzip)
+    "message.send.max.retries": 3,
+    "reconnect.backoff.ms": 100,
+    "reconnect.backoff.max.ms": 3600000,
+    # "acks": "all",  # Safer but can reduce throughput if replication is slow
+}
+
+admin_client = AdminClient({"bootstrap.servers": KAFKA_BROKERS})
+producer = Producer(producer_conf)
+
+
+def generate_kubernetes_metadata() -> Dict[str, str]:
+    namespace = random.choice(K8S_NAMESPACES)
+    sts_name = f"parseable-{random.choice(COMPONENTS)}"
+    pod_index = str(random.randint(0, 5))
+    pod_name = f"{sts_name}-{pod_index}"
+
+    return {
+        "kubernetes_namespace_name": namespace,
+        "kubernetes_pod_name": pod_name,
+        "kubernetes_pod_id": str(uuid.uuid4()),
+        "kubernetes_pod_ip": f"10.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}",
+        "kubernetes_host": f"ip-10-0-{random.randint(0, 255)}-{random.randint(0, 255)}.ec2.internal",
+        "kubernetes_container_name": random.choice(COMPONENTS),
+        "kubernetes_container_image": random.choice(CONTAINER_IMAGES),
+        "kubernetes_container_hash": fake.sha256(),
+        "kubernetes_docker_id": fake.sha256()[:12],
+        "kubernetes_labels_app": "parseable",
+        "kubernetes_labels_component": random.choice(COMPONENTS),
+        "kubernetes_labels_pbc_nodetype": random.choice(NODE_TYPES),
+        "kubernetes_labels_spot": random.choice(["true", "false"]),
+        "kubernetes_labels_sts_name": sts_name,
+        "kubernetes_labels_statefulset.kubernetes.io/pod-name": pod_name,
+        "kubernetes_labels_apps.kubernetes.io/pod-index": pod_index,
+        "kubernetes_labels_controller-revision-hash": fake.sha256()[:10],
+        "kubernetes_labels_original_sts_name": sts_name,
+        "kubernetes_labels_parseable_cr": "parseable-cluster"
+    }
+
+
+def generate_log_entry() -> Dict[str, Any]:
+    now = datetime.now(timezone.utc)
+
+    # Generate request-related data
+    status_code = random.choice([200, 200, 200, 201, 400, 401, 403, 404, 500])
+    response_time = random.randint(10, 2000)
+
+    # Basic log structure
+    log_entry = {
+        "app_meta": json.dumps({"version": "v0.8.0", "component": random.choice(COMPONENTS)}),
+        "device_id": random.randint(1000, 9999),
+        "host": f"ip-{random.randint(0, 255)}-{random.randint(0, 255)}-{random.randint(0, 255)}-{random.randint(0, 255)}",
+        "level": random.choice(["INFO", "INFO", "INFO", "WARN", "ERROR"]),
+        "location": fake.city(),
+        "message": fake.sentence(),
+        "os": random.choice(["linux/amd64", "linux/arm64"]),
+        "process_id": random.randint(1, 65535),
+        "request_body": json.dumps({"query": "SELECT * FROM logs LIMIT 100"}),
+        "response_time": response_time,
+        "runtime": "python3.9",
+        "session_id": str(uuid.uuid4()),
+        "source": "application",
+        "source_time": now.isoformat(),
+        "status_code": status_code,
+        "stream": "stdout",
+        "time": int(now.timestamp() * 1000),
+        "timezone": "UTC",
+        "user_agent": fake.user_agent(),
+        "user_id": random.randint(1000, 9999),
+        "uuid": str(uuid.uuid4()),
+        "version": "v0.8.0"
+    }
+
+    # Add Kubernetes metadata
+    log_entry.update(generate_kubernetes_metadata())
+
+    return log_entry
+
+
+def create_topic(topic_name: str, num_partitions: int, replication_factor: int) -> None:
+    new_topic = NewTopic(
+        topic=topic_name,
+        num_partitions=num_partitions,
+        replication_factor=replication_factor
+    )
+
+    logger.info(f"Creating topic '{topic_name}' with {num_partitions} partitions and RF {replication_factor}...")
+    fs = admin_client.create_topics([new_topic])
+
+    for topic, f in fs.items():
+        try:
+            f.result()
+            logger.info(f"Topic '{topic}' created successfully.")
+        except Exception as e:
+            if "TopicExistsError" in str(e):
+                logger.warning(f"Topic '{topic}' already exists.")
+            else:
+                logger.error(f"Failed to create topic '{topic}': {e}")
+
+
+def delivery_report(err, msg):
+    if err:
+        logger.error(f"Delivery failed for message {msg.key()}: {err}")
+    else:
+        logger.debug(f"Message delivered to {msg.topic()} [{msg.partition()}]")
+
+
+def main():
+    logger.info("Starting continuous log producer...")
+    create_topic(KAFKA_TOPIC, NUM_PARTITIONS, REPLICATION_FACTOR)
+    logger.info(f"Broker: {KAFKA_BROKERS}, Topic: {KAFKA_TOPIC}, Rate: {LOG_RATE} logs/sec")
+
+    message_count = 0
+    start_time = time.time()
+    batch_start_time = time.time()
+    limit_reached = False
+
+    try:
+        while True:
+            current_time = time.time()
+
+            if not limit_reached:
+                if message_count < TOTAL_LOGS:
+                    log_data = generate_log_entry()
+                    log_str = json.dumps(log_data)
+
+                    # Send to Kafka
+                    producer.produce(
+                        topic=KAFKA_TOPIC,
+                        value=log_str,
+                        callback=delivery_report
+                    )
+
+                    message_count += 1
+
+                    if message_count % REPORT_EVERY == 0:
+                        batch_elapsed = current_time - batch_start_time
+                        total_elapsed = current_time - start_time
+
+                        logger.info(f"Batch of {REPORT_EVERY} messages produced in {batch_elapsed:.2f}s")
+                        logger.info(f"Total messages: {message_count}, Running time: {total_elapsed:.2f}s")
+                        logger.info(f"Current rate: ~{REPORT_EVERY / batch_elapsed:,.0f} logs/sec")
+
+                        producer.flush()
+                        batch_start_time = current_time
+
+                elif not limit_reached:
+                    logger.info(
+                        f"Reached TOTAL_LOGS limit of {TOTAL_LOGS}. Continuing to run without producing messages...")
+                    producer.flush()
+                    limit_reached = True
+
+            if limit_reached:
+                time.sleep(5)
+            else:
+                # Sleep to maintain the logs/second rate
+                time.sleep(1 / LOG_RATE)
+
+    except KeyboardInterrupt:
+        logger.warning("Interrupted by user! Flushing remaining messages...")
+        producer.flush()
+
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+
+    finally:
+        logger.info("Flushing producer...")
+        producer.flush()
+        logger.info("Generator stopped.")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/cli.rs b/src/cli.rs
index 727ed4337..a331cc581 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -21,6 +21,9 @@ use std::path::PathBuf;
 
 use url::Url;
 
+#[cfg(feature = "kafka")]
+use crate::connectors::kafka::config::KafkaConfig;
+
 use crate::{
     oidc::{self, OpenidConfig},
     option::{validation, Compression, Mode},
@@ -84,6 +87,9 @@ pub struct LocalStoreArgs {
     pub options: Options,
     #[command(flatten)]
     pub storage: FSConfig,
+    #[cfg(feature = "kafka")]
+    #[command(flatten)]
+    pub kafka: KafkaConfig,
 }
 
 #[derive(Parser)]
@@ -92,6 +98,9 @@ pub struct S3StoreArgs {
     pub options: Options,
     #[command(flatten)]
     pub storage: S3Config,
+    #[cfg(feature = "kafka")]
+    #[command(flatten)]
+    pub kafka: KafkaConfig,
 }
 
 #[derive(Parser)]
@@ -100,6 +109,9 @@ pub struct BlobStoreArgs {
     pub options: Options,
     #[command(flatten)]
     pub storage: AzureBlobConfig,
+    #[cfg(feature = "kafka")]
+    #[command(flatten)]
+    pub kafka: KafkaConfig,
 }
 
 #[derive(Parser, Debug, Default)]
@@ -285,54 +297,6 @@ pub struct Options {
     #[command(flatten)]
     pub oidc: Option<OidcConfig>,
 
-    // Kafka configuration (conditionally compiled)
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(long, env = "P_KAFKA_TOPICS", help = "Kafka topics to subscribe to")]
-    pub kafka_topics: Option<String>,
-
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(long, env = "P_KAFKA_HOST", help = "Address and port for Kafka server")]
-    pub kafka_host: Option<String>,
-
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(long, env = "P_KAFKA_GROUP", help = "Kafka group")]
-    pub kafka_group: Option<String>,
-
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(long, env = "P_KAFKA_CLIENT_ID", help = "Kafka client id")]
-    pub kafka_client_id: Option<String>,
-
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(
-        long,
-        env = "P_KAFKA_SECURITY_PROTOCOL",
-        value_parser = validation::kafka_security_protocol,
-        help = "Kafka security protocol"
-    )]
-    pub kafka_security_protocol: Option<crate::kafka::SslProtocol>,
-
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    #[arg(long, env = "P_KAFKA_PARTITIONS", help = "Kafka partitions")]
-    pub kafka_partitions: Option<String>,
-
     // Audit logging
     #[arg(
         long,
diff --git a/src/connectors/common/mod.rs b/src/connectors/common/mod.rs
new file mode 100644
index 000000000..cb77d983c
--- /dev/null
+++ b/src/connectors/common/mod.rs
@@ -0,0 +1,111 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use clap::ValueEnum;
+use rdkafka::error::{KafkaError, RDKafkaErrorCode};
+use std::str::FromStr;
+use thiserror::Error;
+use tokio::runtime;
+use tokio::runtime::Builder;
+
+pub mod processor;
+pub mod shutdown;
+
+#[derive(Debug, Error)]
+pub enum ConnectorError {
+    #[error("Kafka error: {0}")]
+    Kafka(KafkaError),
+
+    #[error("Connection error: {0}")]
+    Connection(String),
+
+    #[error("Fatal error: {0}")]
+    Fatal(String),
+
+    #[error("Processing error: {0}")]
+    Processing(#[from] anyhow::Error),
+
+    #[error("State error: {0}")]
+    State(String),
+
+    #[error("Authentication error: {0}")]
+    Auth(String),
+}
+
+impl From<KafkaError> for ConnectorError {
+    fn from(error: KafkaError) -> Self {
+        if let Some(code) = error.rdkafka_error_code() {
+            match code {
+                RDKafkaErrorCode::BrokerTransportFailure
+                | RDKafkaErrorCode::NetworkException
+                | RDKafkaErrorCode::AllBrokersDown => ConnectorError::Connection(error.to_string()),
+
+                RDKafkaErrorCode::Fatal | RDKafkaErrorCode::CriticalSystemResource => {
+                    ConnectorError::Fatal(error.to_string())
+                }
+
+                RDKafkaErrorCode::Authentication | RDKafkaErrorCode::SaslAuthenticationFailed => {
+                    ConnectorError::Auth(error.to_string())
+                }
+
+                _ => ConnectorError::Kafka(error),
+            }
+        } else {
+            ConnectorError::Kafka(error)
+        }
+    }
+}
+impl ConnectorError {
+    pub fn is_fatal(&self) -> bool {
+        matches!(
+            self,
+            ConnectorError::Fatal(_) | ConnectorError::Auth(_) | ConnectorError::State(_)
+        )
+    }
+}
+
+#[derive(ValueEnum, Default, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum BadData {
+    #[default]
+    Fail,
+    Drop,
+    Dlt, //TODO: Implement Dead Letter Topic support when needed
+}
+
+impl FromStr for BadData {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_lowercase().as_str() {
+            "drop" => Ok(BadData::Drop),
+            "fail" => Ok(BadData::Fail),
+            "dlt" => Ok(BadData::Dlt),
+            _ => Err(format!("Invalid bad data policy: {}", s)),
+        }
+    }
+}
+
+pub fn build_runtime(worker_threads: usize, thread_name: &str) -> anyhow::Result<runtime::Runtime> {
+    Builder::new_multi_thread()
+        .enable_all()
+        .thread_name(thread_name)
+        .worker_threads(worker_threads)
+        .max_blocking_threads(worker_threads)
+        .build()
+        .map_err(|e| anyhow::anyhow!(e))
+}
diff --git a/src/connectors/common/processor.rs b/src/connectors/common/processor.rs
new file mode 100644
index 000000000..cce0fe1cc
--- /dev/null
+++ b/src/connectors/common/processor.rs
@@ -0,0 +1,29 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait Processor<IN, OUT>: Send + Sync + Sized + 'static {
+    async fn process(&self, records: IN) -> anyhow::Result<OUT>;
+
+    #[allow(unused_variables)]
+    async fn post_stream(&self) -> anyhow::Result<()> {
+        Ok(())
+    }
+}
diff --git a/src/connectors/common/shutdown.rs b/src/connectors/common/shutdown.rs
new file mode 100644
index 000000000..03517a326
--- /dev/null
+++ b/src/connectors/common/shutdown.rs
@@ -0,0 +1,132 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use tokio::sync::mpsc;
+use tokio_util::sync::CancellationToken;
+use tracing::{info, warn};
+
+#[derive(Debug)]
+pub struct Shutdown {
+    cancel_token: CancellationToken,
+    shutdown_complete_tx: mpsc::Sender<()>,
+    shutdown_complete_rx: Option<mpsc::Receiver<()>>,
+}
+
+impl Shutdown {
+    pub fn start(&self) {
+        self.cancel_token.cancel();
+    }
+
+    pub async fn recv(&self) {
+        self.cancel_token.cancelled().await;
+    }
+
+    pub async fn signal_listener(&self) {
+        let ctrl_c_signal = tokio::signal::ctrl_c();
+        #[cfg(unix)]
+        let mut sigterm_signal =
+            tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).unwrap();
+        #[cfg(unix)]
+        tokio::select! {
+            _ = ctrl_c_signal => {},
+            _ = sigterm_signal.recv() => {}
+        }
+        #[cfg(windows)]
+        let _ = ctrl_c_signal.await;
+
+        warn!("Shutdown signal received!");
+        self.start();
+    }
+
+    pub async fn complete(self) {
+        drop(self.shutdown_complete_tx);
+        self.shutdown_complete_rx.unwrap().recv().await;
+        info!("Shutdown complete!")
+    }
+}
+
+impl Default for Shutdown {
+    fn default() -> Self {
+        let cancel_token = CancellationToken::new();
+        let (shutdown_complete_tx, shutdown_complete_rx) = mpsc::channel(1);
+        Self {
+            cancel_token,
+            shutdown_complete_tx,
+            shutdown_complete_rx: Some(shutdown_complete_rx),
+        }
+    }
+}
+
+impl Clone for Shutdown {
+    fn clone(&self) -> Self {
+        Self {
+            cancel_token: self.cancel_token.clone(),
+            shutdown_complete_tx: self.shutdown_complete_tx.clone(),
+            shutdown_complete_rx: None,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::{Arc, Mutex};
+
+    use super::*;
+    use tokio::time::Duration;
+
+    #[tokio::test]
+    async fn test_shutdown_recv() {
+        let shutdown = Shutdown::default();
+        let shutdown_clone = shutdown.clone();
+        // receive shutdown task
+        let task = tokio::spawn(async move {
+            shutdown_clone.recv().await;
+            1
+        });
+        // start shutdown task after 200 ms
+        tokio::spawn(async move {
+            tokio::time::sleep(Duration::from_millis(200)).await;
+            shutdown.start();
+        });
+        // if shutdown is not received within 5 seconds, fail test
+        let check_value = tokio::select! {
+            _ = tokio::time::sleep(Duration::from_secs(5)) => panic!("Shutdown not received within 5 seconds"),
+            v = task => v.unwrap(),
+        };
+        assert_eq!(check_value, 1);
+    }
+
+    #[tokio::test]
+    async fn test_shutdown_wait_for_complete() {
+        let shutdown = Shutdown::default();
+        let shutdown_clone = shutdown.clone();
+        let check_value: Arc<Mutex<bool>> = Arc::new(Mutex::new(false));
+        let check_value_clone = Arc::clone(&check_value);
+        // receive shutdown task
+        tokio::spawn(async move {
+            shutdown_clone.recv().await;
+            tokio::time::sleep(Duration::from_millis(200)).await;
+            let mut check: std::sync::MutexGuard<'_, bool> = check_value_clone.lock().unwrap();
+            *check = true;
+        });
+        shutdown.start();
+        shutdown.complete().await;
+        let check = check_value.lock().unwrap();
+        assert!(*check, "shutdown did not successfully wait for complete");
+    }
+}
diff --git a/src/connectors/kafka/config.rs b/src/connectors/kafka/config.rs
new file mode 100644
index 000000000..ee7e77446
--- /dev/null
+++ b/src/connectors/kafka/config.rs
@@ -0,0 +1,1008 @@
+use crate::connectors::common::BadData;
+use clap::{Args, Parser, ValueEnum};
+use rdkafka::{ClientConfig, Offset};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+use std::time::Duration;
+
+#[derive(Debug, Clone, Parser)]
+pub struct KafkaConfig {
+    #[arg(
+        long = "bootstrap-servers",
+        env = "P_KAFKA_BOOTSTRAP_SERVERS",
+        value_name = "bootstrap-servers",
+        required = false,
+        help = "Comma-separated list of Kafka bootstrap servers"
+    )]
+    pub bootstrap_servers: Option<String>,
+
+    #[arg(
+        long = "client-id",
+        env = "P_KAFKA_CLIENT_ID",
+        required = false,
+        default_value_t = String::from("parseable-connect"),
+        value_name = "client_id",
+        help = "Client ID for Kafka connection"
+    )]
+    pub client_id: String,
+
+    #[arg(
+        long = "partition-listener-concurrency",
+        env = "P_KAFKA_PARTITION_LISTENER_CONCURRENCY",
+        value_name = "concurrency",
+        required = false,
+        default_value_t = 2,
+        help = "Number of parallel threads for Kafka partition listeners. Each partition gets processed on a dedicated thread."
+    )]
+    pub partition_listener_concurrency: usize,
+
+    #[command(flatten)]
+    pub consumer: Option<ConsumerConfig>,
+
+    #[command(flatten)]
+    pub producer: Option<ProducerConfig>,
+
+    #[command(flatten)]
+    pub security: Option<SecurityConfig>,
+
+    #[arg(
+        value_enum,
+        long = "bad-data-policy",
+        required = false,
+        default_value_t = BadData::Fail,
+        env = "P_CONNECTOR_BAD_DATA_POLICY",
+        help = "Policy for handling bad data"
+    )]
+    pub bad_data: BadData,
+}
+
+#[derive(Debug, Clone, Args)]
+pub struct ConsumerConfig {
+    #[arg(
+        long = "consumer-topics",
+        env = "P_KAFKA_CONSUMER_TOPICS",
+        value_name = "consumer-topics",
+        required = false,
+        value_delimiter = ',',
+        help = "Comma-separated list of topics"
+    )]
+    pub topics: Vec<String>,
+
+    #[arg(
+        long = "consumer-group-id",
+        env = "P_KAFKA_CONSUMER_GROUP_ID",
+        value_name = "id",
+        required = false,
+        default_value_t = String::from("parseable-connect-cg"),
+        help = "Consumer group ID"
+    )]
+    pub group_id: String,
+
+    // uses per partition stream micro-batch buffer size
+    #[arg(
+        long = "buffer-size",
+        env = "P_KAFKA_CONSUMER_BUFFER_SIZE",
+        value_name = "size",
+        required = false,
+        default_value_t = 10000,
+        help = "Size of the buffer for batching records"
+    )]
+    pub buffer_size: usize,
+
+    // uses per partition stream micro-batch buffer timeout
+    #[clap(
+        value_parser = humantime::parse_duration,
+        default_value= "10000ms",
+        long = "buffer-timeout",
+        env = "P_KAFKA_CONSUMER_BUFFER_TIMEOUT",
+        value_name = "timeout_ms",
+        required = false,
+        help = "Timeout for buffer flush in milliseconds"
+    )]
+    pub buffer_timeout: Duration,
+
+    #[arg(
+        long = "consumer-group-instance-id",
+        required = false,
+        env = "P_KAFKA_CONSUMER_GROUP_INSTANCE_ID",
+             default_value_t = format!("parseable-connect-cg-ii-{}", rand::random::<u8>()).to_string(),
+        help = "Group instance ID for static membership"
+    )]
+    pub group_instance_id: String,
+
+    #[arg(
+        long = "consumer-partition-strategy",
+        env = "P_KAFKA_CONSUMER_PARTITION_STRATEGY",
+        required = false,
+        default_value_t = String::from("roundrobin,range"),
+        help = "Partition assignment strategy"
+    )]
+    pub partition_assignment_strategy: String,
+
+    #[arg(
+        long = "consumer-session-timeout",
+        env = "P_KAFKA_CONSUMER_SESSION_TIMEOUT",
+        required = false,
+        default_value_t = 60000,
+        help = "Session timeout in milliseconds"
+    )]
+    pub session_timeout_ms: u32,
+
+    #[arg(
+        long = "consumer-heartbeat-interval",
+        env = "P_KAFKA_CONSUMER_HEARTBEAT_INTERVAL",
+        required = false,
+        default_value_t = 3000,
+        help = "Heartbeat interval in milliseconds"
+    )]
+    pub heartbeat_interval_ms: u32,
+
+    #[arg(
+        long = "consumer-max-poll-interval",
+        env = "P_KAFKA_CONSUMER_MAX_POLL_INTERVAL",
+        required = false,
+        default_value_t = 300000,
+        help = "Maximum poll interval in milliseconds"
+    )]
+    pub max_poll_interval_ms: u32,
+
+    #[arg(
+        long = "consumer-enable-auto-offset-store",
+        env = "P_KAFKA_CONSUMER_ENABLE_AUTO_OFFSET_STORE",
+        required = false,
+        default_value_t = true,
+        help = "Enable auto offset store"
+    )]
+    pub enable_auto_offset_store: bool,
+
+    #[clap(
+        value_enum,
+        long = "consumer-auto-offset-reset",
+        required = false,
+        env = "P_KAFKA_CONSUMER_AUTO_OFFSET_RESET",
+        default_value_t = SourceOffset::Earliest,
+        help = "Auto offset reset behavior"
+    )]
+    pub auto_offset_reset: SourceOffset,
+
+    #[arg(
+        long = "consumer-fetch-min-bytes",
+        env = "P_KAFKA_CONSUMER_FETCH_MIN_BYTES",
+        default_value_t = 1,
+        required = false,
+        help = "Minimum bytes to fetch"
+    )]
+    pub fetch_min_bytes: u32,
+
+    #[arg(
+        long = "consumer-fetch-max-bytes",
+        env = "P_KAFKA_CONSUMER_FETCH_MAX_BYTES",
+        default_value_t = 52428800,
+        required = false,
+        help = "Maximum bytes to fetch"
+    )]
+    pub fetch_max_bytes: u32,
+
+    #[arg(
+        long = "consumer-fetch-max-wait",
+        env = "P_KAFKA_CONSUMER_FETCH_MAX_WAIT",
+        default_value_t = 500,
+        required = false,
+        help = "Maximum wait time for fetch in milliseconds"
+    )]
+    pub fetch_max_wait_ms: u32,
+
+    #[arg(
+        long = "consumer-max-partition-fetch-bytes",
+        env = "P_KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES",
+        required = false,
+        default_value_t = 1048576,
+        help = "Maximum bytes to fetch per partition"
+    )]
+    pub max_partition_fetch_bytes: u32,
+
+    #[arg(
+        long = "consumer-queued-min-messages",
+        env = "P_KAFKA_CONSUMER_QUEUED_MIN_MESSAGES",
+        required = false,
+        default_value_t = 100000,
+        help = "Minimum messages to queue"
+    )]
+    pub queued_min_messages: u32,
+
+    #[arg(
+        long = "consumer-queued-max-messages-kbytes",
+        env = "P_KAFKA_CONSUMER_QUEUED_MAX_MESSAGES_KBYTES",
+        required = false,
+        default_value_t = 65536,
+        help = "Maximum message queue size in KBytes"
+    )]
+    pub queued_max_messages_kbytes: u32,
+
+    #[arg(
+        long = "consumer-enable-partition-eof",
+        env = "P_KAFKA_CONSUMER_ENABLE_PARTITION_EOF",
+        required = false,
+        default_value_t = false,
+        help = "Enable partition EOF"
+    )]
+    pub enable_partition_eof: bool,
+
+    #[arg(
+        long = "consumer-check-crcs",
+        env = "P_KAFKA_CONSUMER_CHECK_CRCS",
+        required = false,
+        default_value_t = false,
+        help = "Check CRCs on messages"
+    )]
+    pub check_crcs: bool,
+
+    #[arg(
+        long = "consumer-isolation-level",
+        env = "P_KAFKA_CONSUMER_ISOLATION_LEVEL",
+        required = false,
+        default_value_t = String::from("read_committed"),
+        help = "Transaction isolation level"
+    )]
+    pub isolation_level: String,
+
+    #[arg(
+        long = "consumer-fetch-message-max-bytes",
+        env = "P_KAFKA_CONSUMER_FETCH_MESSAGE_MAX_BYTES",
+        required = false,
+        default_value_t = 1048576,
+        help = "Maximum bytes per message"
+    )]
+    pub fetch_message_max_bytes: u64,
+
+    #[arg(
+        long = "consumer-stats-interval",
+        env = "P_KAFKA_CONSUMER_STATS_INTERVAL",
+        required = false,
+        default_value_t = 10000,
+        help = "Statistics interval in milliseconds"
+    )]
+    pub stats_interval_ms: u64,
+}
+
+#[derive(Debug, Clone, Args)]
+pub struct ProducerConfig {
+    #[arg(
+        long = "producer-acks",
+        env = "P_KAFKA_PRODUCER_ACKS",
+        required = false,
+        default_value_t = String::from("all"),
+        value_parser = ["0", "1", "all"],
+        help = "Number of acknowledgments the producer requires"
+    )]
+    pub acks: String,
+
+    #[arg(
+        long = "producer-compression-type",
+        env = "P_KAFKA_PRODUCER_COMPRESSION_TYPE",
+        required = false,
+        default_value_t= String::from("lz4"),
+        value_parser = ["none", "gzip", "snappy", "lz4", "zstd"],
+        help = "Compression type for messages"
+    )]
+    pub compression_type: String,
+
+    #[arg(
+        long = "producer-batch-size",
+        env = "P_KAFKA_PRODUCER_BATCH_SIZE",
+        required = false,
+        default_value_t = 16384,
+        help = "Maximum size of a request in bytes"
+    )]
+    pub batch_size: u32,
+
+    #[arg(
+        long = "producer-linger-ms",
+        env = "P_KAFKA_PRODUCER_LINGER_MS",
+        required = false,
+        default_value_t = 5,
+        help = "Delay to wait for more messages in the same batch"
+    )]
+    pub linger_ms: u32,
+
+    #[arg(
+        long = "producer-message-timeout-ms",
+        env = "P_KAFKA_PRODUCER_MESSAGE_TIMEOUT_MS",
+        required = false,
+        default_value_t = 120000,
+        help = "Local message timeout"
+    )]
+    pub message_timeout_ms: u32,
+
+    #[arg(
+        long = "producer-max-inflight",
+        env = "P_KAFKA_PRODUCER_MAX_INFLIGHT",
+        required = false,
+        default_value_t = 5,
+        help = "Maximum number of in-flight requests per connection"
+    )]
+    pub max_in_flight_requests_per_connection: u32,
+
+    #[arg(
+        long = "producer-message-max-bytes",
+        env = "P_KAFKA_PRODUCER_MESSAGE_MAX_BYTES",
+        required = false,
+        default_value_t = 1048576,
+        help = "Maximum size of a message in bytes"
+    )]
+    pub message_max_bytes: u32,
+
+    #[arg(
+        long = "producer-enable-idempotence",
+        env = "P_KAFKA_PRODUCER_ENABLE_IDEMPOTENCE",
+        required = false,
+        default_value_t = true,
+        help = "Enable idempotent producer"
+    )]
+    pub enable_idempotence: bool,
+
+    #[arg(
+        long = "producer-transaction-timeout-ms",
+        env = "P_KAFKA_PRODUCER_TRANSACTION_TIMEOUT_MS",
+        required = false,
+        default_value_t = 60000,
+        help = "Transaction timeout"
+    )]
+    pub transaction_timeout_ms: u64,
+
+    #[arg(
+        long = "producer-buffer-memory",
+        env = "P_KAFKA_PRODUCER_BUFFER_MEMORY",
+        required = false,
+        default_value_t = 33554432,
+        help = "Total bytes of memory the producer can use"
+    )]
+    pub buffer_memory: u32,
+
+    #[arg(
+        long = "producer-retry-backoff-ms",
+        env = "P_KAFKA_PRODUCER_RETRY_BACKOFF_MS",
+        required = false,
+        default_value_t = 100,
+        help = "Time to wait before retrying a failed request"
+    )]
+    pub retry_backoff_ms: u32,
+
+    #[arg(
+        long = "producer-request-timeout-ms",
+        env = "P_KAFKA_PRODUCER_REQUEST_TIMEOUT_MS",
+        required = false,
+        default_value_t = 30000,
+        help = "Time to wait for a response from brokers"
+    )]
+    pub request_timeout_ms: u32,
+
+    #[arg(
+        long = "producer-queue-buffering-max-messages",
+        env = "P_KAFKA_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES",
+        required = false,
+        default_value_t = 100000,
+        help = "Maximum number of messages allowed on the producer queue"
+    )]
+    pub queue_buffering_max_messages: u32,
+
+    #[arg(
+        long = "producer-queue-buffering-max-kbytes",
+        env = "P_KAFKA_PRODUCER_QUEUE_BUFFERING_MAX_KBYTES",
+        required = false,
+        default_value_t = 1048576,
+        help = "Maximum total message size sum allowed on the producer queue"
+    )]
+    pub queue_buffering_max_kbytes: u32,
+
+    #[arg(
+        long = "producer-delivery-timeout-ms",
+        env = "P_KAFKA_PRODUCER_DELIVERY_TIMEOUT_MS",
+        required = false,
+        default_value_t = 120000,
+        help = "Maximum time to report success or failure after send"
+    )]
+    pub delivery_timeout_ms: u32,
+
+    #[arg(
+        long = "producer-max-retries",
+        env = "P_KAFKA_PRODUCER_MAX_RETRIES",
+        required = false,
+        default_value_t = 2147483647,
+        help = "Maximum number of retries per message"
+    )]
+    pub max_retries: u32,
+
+    #[arg(
+        long = "producer-retry-backoff-max-ms",
+        env = "P_KAFKA_PRODUCER_RETRY_BACKOFF_MAX_MS",
+        required = false,
+        default_value_t = 1000,
+        help = "Maximum back-off time between retries"
+    )]
+    pub retry_backoff_max_ms: u32,
+}
+
+#[derive(Debug, Clone, Args)]
+pub struct SecurityConfig {
+    #[clap(
+        value_enum,
+        long = "security-protocol",
+        env = "P_KAFKA_SECURITY_PROTOCOL",
+        required = false,
+        default_value_t = SecurityProtocol::Plaintext,
+        help = "Security protocol"
+    )]
+    pub protocol: SecurityProtocol,
+
+    // SSL Configuration
+    #[arg(
+        long = "ssl-ca-location",
+        env = "P_KAFKA_SSL_CA_LOCATION",
+        required = false,
+        help = "CA certificate file path"
+    )]
+    pub ssl_ca_location: Option<PathBuf>,
+
+    #[arg(
+        long = "ssl-certificate-location",
+        env = "P_KAFKA_SSL_CERTIFICATE_LOCATION",
+        required = false,
+        help = "Client certificate file path"
+    )]
+    pub ssl_certificate_location: Option<PathBuf>,
+
+    #[arg(
+        long = "ssl-key-location",
+        env = "P_KAFKA_SSL_KEY_LOCATION",
+        required = false,
+        help = "Client key file path"
+    )]
+    pub ssl_key_location: Option<PathBuf>,
+
+    // SASL Configuration
+    #[arg(
+        value_enum,
+        long = "sasl-mechanism",
+        env = "P_KAFKA_SASL_MECHANISM",
+        required = false,
+        help = "SASL mechanism"
+    )]
+    pub sasl_mechanism: Option<SaslMechanism>,
+
+    #[arg(
+        long = "sasl-username",
+        env = "P_KAFKA_SASL_USERNAME",
+        required = false,
+        help = "SASL username"
+    )]
+    pub sasl_username: Option<String>,
+
+    #[arg(
+        long = "sasl-password",
+        env = "P_KAFKA_SASL_PASSWORD",
+        required = false,
+        help = "SASL password"
+    )]
+    pub sasl_password: Option<String>,
+
+    #[arg(
+        long = "ssl-key-password",
+        env = "P_KAFKA_SSL_KEY_PASSWORD",
+        required = false,
+        help = "SSL key password"
+    )]
+    pub ssl_key_password: Option<String>,
+
+    // Kerberos configuration fields
+    #[arg(
+        long = "kerberos-service-name",
+        env = "P_KAFKA_KERBEROS_SERVICE_NAME",
+        required = false,
+        help = "Kerberos service name"
+    )]
+    pub kerberos_service_name: Option<String>,
+
+    #[arg(
+        long = "kerberos-principal",
+        env = "P_KAFKA_KERBEROS_PRINCIPAL",
+        required = false,
+        help = "Kerberos principal"
+    )]
+    pub kerberos_principal: Option<String>,
+
+    #[arg(
+        long = "kerberos-keytab",
+        env = "P_KAFKA_KERBEROS_KEYTAB",
+        required = false,
+        help = "Path to Kerberos keytab file"
+    )]
+    pub kerberos_keytab: Option<PathBuf>,
+}
+
+impl KafkaConfig {
+    pub fn to_rdkafka_consumer_config(&self) -> ClientConfig {
+        let mut config = ClientConfig::new();
+
+        // Basic configuration
+        config
+            .set(
+                "bootstrap.servers",
+                self.bootstrap_servers
+                    .as_ref()
+                    .expect("Bootstrap servers must not be empty"),
+            )
+            .set("client.id", &self.client_id);
+
+        // Consumer configuration
+        if let Some(consumer) = &self.consumer {
+            consumer.apply_to_config(&mut config);
+        }
+
+        // Security configuration
+        if let Some(security) = &self.security {
+            security.apply_to_config(&mut config);
+        } else {
+            config.set("security.protocol", "PLAINTEXT");
+        }
+
+        config
+    }
+
+    pub fn to_rdkafka_producer_config(&self) -> ClientConfig {
+        let mut config = ClientConfig::new();
+
+        // Basic configuration
+        config
+            .set(
+                "bootstrap.servers",
+                self.bootstrap_servers
+                    .as_ref()
+                    .expect("Bootstrap servers must not be empty"),
+            )
+            .set("client.id", &self.client_id);
+
+        // Producer configuration
+        if let Some(producer) = &self.producer {
+            producer.apply_to_config(&mut config);
+        }
+
+        // Security configuration
+        if let Some(security) = &self.security {
+            security.apply_to_config(&mut config);
+        } else {
+            config.set("security.protocol", "PLAINTEXT");
+        }
+
+        config
+    }
+
+    pub fn consumer(&self) -> Option<&ConsumerConfig> {
+        self.consumer.as_ref()
+    }
+
+    pub fn producer(&self) -> Option<&ProducerConfig> {
+        self.producer.as_ref()
+    }
+
+    pub fn security(&self) -> Option<&SecurityConfig> {
+        self.security.as_ref()
+    }
+
+    pub fn validate(&self) -> anyhow::Result<()> {
+        if self.bootstrap_servers.is_none() {
+            anyhow::bail!("Bootstrap servers must not be empty");
+        }
+
+        if let Some(consumer) = &self.consumer {
+            consumer.validate()?;
+        }
+
+        if let Some(producer) = &self.producer {
+            producer.validate()?;
+        }
+
+        if let Some(security) = &self.security {
+            security.validate()?;
+        }
+
+        Ok(())
+    }
+}
+
+impl ConsumerConfig {
+    pub fn validate(&self) -> anyhow::Result<()> {
+        if self.group_id.is_empty() {
+            anyhow::bail!("Consumer group ID must not be empty");
+        }
+        if self.topics.is_empty() {
+            anyhow::bail!("At least one topic must be specified");
+        }
+        Ok(())
+    }
+
+    fn apply_to_config(&self, config: &mut ClientConfig) {
+        config
+            .set("group.id", &self.group_id)
+            .set(
+                "partition.assignment.strategy",
+                &self.partition_assignment_strategy,
+            )
+            .set("session.timeout.ms", self.session_timeout_ms.to_string())
+            .set(
+                "heartbeat.interval.ms",
+                self.heartbeat_interval_ms.to_string(),
+            )
+            .set(
+                "max.poll.interval.ms",
+                self.max_poll_interval_ms.to_string(),
+            )
+            .set("enable.auto.commit", "false")
+            .set("fetch.min.bytes", self.fetch_min_bytes.to_string())
+            .set("fetch.max.bytes", self.fetch_max_bytes.to_string())
+            .set(
+                "max.partition.fetch.bytes",
+                self.max_partition_fetch_bytes.to_string(),
+            )
+            .set("isolation.level", self.isolation_level.to_string())
+            .set("group.instance.id", self.group_instance_id.to_string())
+            .set("statistics.interval.ms", self.stats_interval_ms.to_string());
+    }
+
+    pub fn topics(&self) -> Vec<&str> {
+        self.topics.iter().map(|t| t.as_str()).collect()
+    }
+
+    pub fn buffer_config(&self) -> BufferConfig {
+        BufferConfig {
+            buffer_size: self.buffer_size,
+            buffer_timeout: self.buffer_timeout,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct BufferConfig {
+    pub buffer_size: usize,
+    pub buffer_timeout: Duration,
+}
+
+impl Default for BufferConfig {
+    fn default() -> Self {
+        Self {
+            buffer_size: 10000,
+            buffer_timeout: Duration::from_millis(10000),
+        }
+    }
+}
+
+impl ProducerConfig {
+    fn apply_to_config(&self, config: &mut ClientConfig) {
+        config
+            .set("acks", self.acks.to_string())
+            .set("compression.type", self.compression_type.to_string())
+            .set("batch.size", self.batch_size.to_string())
+            .set("linger.ms", self.linger_ms.to_string())
+            .set("enable.idempotence", self.enable_idempotence.to_string())
+            .set(
+                "max.in.flight.requests.per.connection",
+                self.max_in_flight_requests_per_connection.to_string(),
+            )
+            .set("delivery.timeout.ms", self.delivery_timeout_ms.to_string())
+            .set("retry.backoff.ms", self.retry_backoff_ms.to_string())
+            .set(
+                "transaction.timeout.ms",
+                self.transaction_timeout_ms.to_string(),
+            )
+            .set("request.timeout.ms", self.request_timeout_ms.to_string())
+            .set("max.retries", self.max_retries.to_string())
+            .set(
+                "retry.backoff.max.ms",
+                self.retry_backoff_max_ms.to_string(),
+            )
+            .set("buffer.memory", self.buffer_memory.to_string())
+            .set("message.timeout.ms", self.message_timeout_ms.to_string())
+            .set("message.max.bytes", self.message_max_bytes.to_string());
+    }
+
+    fn validate(&self) -> anyhow::Result<()> {
+        if self.batch_size == 0 {
+            anyhow::bail!("Batch size must be greater than 0");
+        }
+
+        if self.linger_ms > self.delivery_timeout_ms {
+            anyhow::bail!("Linger time cannot be greater than delivery timeout");
+        }
+
+        Ok(())
+    }
+}
+
+impl SecurityConfig {
+    fn apply_to_config(&self, config: &mut ClientConfig) {
+        // Set security protocol
+        config.set("security.protocol", self.protocol.to_string());
+
+        // Configure SSL if enabled
+        if matches!(
+            self.protocol,
+            SecurityProtocol::Ssl | SecurityProtocol::SaslSsl
+        ) {
+            if let Some(ref path) = self.ssl_ca_location {
+                config.set("ssl.ca.location", path.to_string_lossy().to_string());
+            }
+            if let Some(ref path) = self.ssl_certificate_location {
+                config.set(
+                    "ssl.certificate.location",
+                    path.to_string_lossy().to_string(),
+                );
+            }
+            if let Some(ref path) = self.ssl_key_location {
+                config.set("ssl.key.location", path.to_string_lossy().to_string());
+            }
+            if let Some(ref password) = self.ssl_key_password {
+                config.set("ssl.key.password", password);
+            }
+        }
+
+        // Configure SASL if enabled
+        if matches!(
+            self.protocol,
+            SecurityProtocol::SaslSsl | SecurityProtocol::SaslPlaintext
+        ) {
+            if let Some(ref mechanism) = self.sasl_mechanism {
+                config.set("sasl.mechanism", mechanism.to_string());
+            }
+            if let Some(ref username) = self.sasl_username {
+                config.set("sasl.username", username);
+            }
+            if let Some(ref password) = self.sasl_password {
+                config.set("sasl.password", password);
+            }
+
+            // Configure Kerberos settings if using GSSAPI
+            if matches!(self.sasl_mechanism, Some(SaslMechanism::Gssapi)) {
+                if let Some(ref service) = self.kerberos_service_name {
+                    config.set("sasl.kerberos.service.name", service);
+                }
+                if let Some(ref principal) = self.kerberos_principal {
+                    config.set("sasl.kerberos.principal", principal);
+                }
+                if let Some(ref keytab) = self.kerberos_keytab {
+                    config.set("sasl.kerberos.keytab", keytab.to_string_lossy().to_string());
+                }
+            }
+
+            // TODO: Implement OAuthBearer mechanism for SASL when needed. This depends on the vendor (on-prem, Confluent Kafka, AWS MSK, etc.).
+        }
+    }
+
+    fn validate(&self) -> anyhow::Result<()> {
+        match self.protocol {
+            SecurityProtocol::Ssl | SecurityProtocol::SaslSsl => {
+                if self.ssl_ca_location.is_none() {
+                    anyhow::bail!("CA certificate location is required for SSL");
+                }
+                if self.ssl_certificate_location.is_none() {
+                    anyhow::bail!("Client certificate location is required for SSL");
+                }
+                if self.ssl_key_location.is_none() {
+                    anyhow::bail!("Client key location is required for SSL");
+                }
+            }
+            SecurityProtocol::SaslPlaintext => {
+                if self.sasl_mechanism.is_none() {
+                    anyhow::bail!("SASL mechanism is required when SASL is enabled");
+                }
+                if self.sasl_username.is_none() || self.sasl_password.is_none() {
+                    anyhow::bail!("SASL username and password are required");
+                }
+
+                if matches!(self.sasl_mechanism, Some(SaslMechanism::Gssapi))
+                    && self.kerberos_service_name.is_none()
+                {
+                    anyhow::bail!("Kerberos service name is required for GSSAPI");
+                }
+            }
+            SecurityProtocol::Plaintext => {} // No validation needed for PLAINTEXT
+        }
+        Ok(())
+    }
+}
+
+#[derive(ValueEnum, Debug, Clone, Serialize, Deserialize)]
+pub enum SecurityProtocol {
+    Plaintext,
+    Ssl,
+    SaslSsl,
+    SaslPlaintext,
+}
+
+impl std::str::FromStr for SecurityProtocol {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_uppercase().as_str() {
+            "PLAINTEXT" => Ok(SecurityProtocol::Plaintext),
+            "SSL" => Ok(SecurityProtocol::Ssl),
+            "SASL_SSL" => Ok(SecurityProtocol::SaslSsl),
+            "SASL_PLAINTEXT" => Ok(SecurityProtocol::SaslPlaintext),
+            _ => Err(format!("Invalid security protocol: {}", s)),
+        }
+    }
+}
+
+impl std::fmt::Display for SecurityProtocol {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SecurityProtocol::Plaintext => write!(f, "PLAINTEXT"),
+            SecurityProtocol::Ssl => write!(f, "SSL"),
+            SecurityProtocol::SaslSsl => write!(f, "SASL_SSL"),
+            SecurityProtocol::SaslPlaintext => write!(f, "SASL_PLAINTEXT"),
+        }
+    }
+}
+
+#[derive(ValueEnum, Debug, Clone, Serialize, Deserialize)]
+pub enum SaslMechanism {
+    Plain,
+    ScramSha256,
+    ScramSha512,
+    Gssapi,
+    OAuthBearer,
+}
+
+impl Default for KafkaConfig {
+    fn default() -> Self {
+        Self {
+            // Common configuration with standard broker port
+            bootstrap_servers: Some("localhost:9092".to_string()),
+            client_id: "parseable-connect".to_string(),
+            // Listener for all assigned partitions
+            partition_listener_concurrency: 2,
+            // Component-specific configurations with production-ready defaults
+            consumer: Some(ConsumerConfig::default()),
+            producer: Some(ProducerConfig::default()),
+            // Security configuration with plaintext protocol
+            security: Some(SecurityConfig::default()),
+            bad_data: BadData::default(),
+        }
+    }
+}
+
+impl std::fmt::Display for SaslMechanism {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SaslMechanism::Plain => write!(f, "PLAIN"),
+            SaslMechanism::ScramSha256 => write!(f, "SCRAM-SHA-256"),
+            SaslMechanism::ScramSha512 => write!(f, "SCRAM-SHA-512"),
+            SaslMechanism::Gssapi => write!(f, "GSSAPI"),
+            SaslMechanism::OAuthBearer => write!(f, "OAUTHBEARER"),
+        }
+    }
+}
+impl Default for ProducerConfig {
+    fn default() -> Self {
+        Self {
+            acks: "all".to_string(),
+            compression_type: "lz4".to_string(),
+            batch_size: 16384,           // 16KB default batch size
+            linger_ms: 5,                // Small latency for better batching
+            delivery_timeout_ms: 120000, // 2 minute delivery timeout
+            max_retries: 20,
+            max_in_flight_requests_per_connection: 5,
+            message_max_bytes: 1048576,    // 1MB maximum message size
+            enable_idempotence: true,      // Ensure exactly-once delivery
+            transaction_timeout_ms: 60000, // 1 minute transaction timeout
+            queue_buffering_max_messages: 100000, // Producer queue size
+            retry_backoff_ms: 100,         // Backoff time between retries
+            message_timeout_ms: 120000,    // 2 minute message timeout
+            buffer_memory: 33554432,       // 32MB buffer memory
+            request_timeout_ms: 60000,     // 60 second request timeout
+            queue_buffering_max_kbytes: 1048576, // 1MB maximum queue size
+            retry_backoff_max_ms: 1000,    // Maximum backoff time between retries
+        }
+    }
+}
+
+impl Default for ConsumerConfig {
+    fn default() -> Self {
+        Self {
+            topics: vec![],
+            group_id: "parseable-connect-cg".to_string(),
+            buffer_size: 10_000,
+            buffer_timeout: Duration::from_millis(10000),
+            group_instance_id: "parseable-cg-ii".to_string(),
+            // NOTE: cooperative-sticky does not work well in rdkafka when using manual commit.
+            // @see https://github.com/confluentinc/librdkafka/issues/4629
+            // @see https://github.com/confluentinc/librdkafka/issues/4368
+            partition_assignment_strategy: "roundrobin,range".to_string(),
+            session_timeout_ms: 60000,
+            heartbeat_interval_ms: 3000,
+            max_poll_interval_ms: 300000,
+            enable_auto_offset_store: true,
+            auto_offset_reset: SourceOffset::Earliest,
+            fetch_min_bytes: 1,
+            fetch_max_bytes: 52428800,
+            fetch_max_wait_ms: 500,
+            max_partition_fetch_bytes: 1048576,
+            queued_min_messages: 100000,
+            queued_max_messages_kbytes: 65536,
+            enable_partition_eof: false,
+            check_crcs: false,
+            isolation_level: "read_committed".to_string(),
+            fetch_message_max_bytes: 1048576,
+            stats_interval_ms: 10000,
+        }
+    }
+}
+
+impl Default for SecurityConfig {
+    fn default() -> Self {
+        Self {
+            protocol: SecurityProtocol::Plaintext,
+            ssl_ca_location: None,
+            ssl_certificate_location: None,
+            ssl_key_location: None,
+            sasl_mechanism: None,
+            sasl_username: None,
+            sasl_password: None,
+            ssl_key_password: None,
+            kerberos_service_name: None,
+            kerberos_principal: None,
+            kerberos_keytab: None,
+        }
+    }
+}
+
+impl std::str::FromStr for SaslMechanism {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_uppercase().as_str() {
+            "PLAIN" => Ok(SaslMechanism::Plain),
+            "SCRAM-SHA-256" => Ok(SaslMechanism::ScramSha256),
+            "SCRAM-SHA-512" => Ok(SaslMechanism::ScramSha512),
+            "GSSAPI" => Ok(SaslMechanism::Gssapi),
+            "OAUTHBEARER" => Ok(SaslMechanism::OAuthBearer),
+            _ => Err(format!("Invalid SASL mechanism: {}", s)),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Acks {
+    None,
+    Leader,
+    All,
+}
+
+impl std::str::FromStr for Acks {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "0" => Ok(Acks::None),
+            "1" => Ok(Acks::Leader),
+            "all" => Ok(Acks::All),
+            _ => Err(format!("Invalid acks value: {}", s)),
+        }
+    }
+}
+
+#[derive(ValueEnum, Debug, Clone)]
+pub enum SourceOffset {
+    Earliest,
+    Latest,
+    Group,
+}
+
+impl SourceOffset {
+    pub fn get_offset(&self) -> Offset {
+        match self {
+            SourceOffset::Earliest => Offset::Beginning,
+            SourceOffset::Latest => Offset::End,
+            SourceOffset::Group => Offset::Stored,
+        }
+    }
+}
diff --git a/src/connectors/kafka/consumer.rs b/src/connectors/kafka/consumer.rs
new file mode 100644
index 000000000..3bfc01702
--- /dev/null
+++ b/src/connectors/kafka/consumer.rs
@@ -0,0 +1,247 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::common::shutdown::Shutdown;
+use crate::connectors::common::ConnectorError;
+use crate::connectors::kafka::partition_stream::{PartitionStreamReceiver, PartitionStreamSender};
+use crate::connectors::kafka::state::StreamState;
+use crate::connectors::kafka::{
+    partition_stream, ConsumerRecord, KafkaContext, StreamConsumer, TopicPartition,
+};
+use futures_util::FutureExt;
+use rdkafka::consumer::Consumer;
+use rdkafka::message::BorrowedMessage;
+use rdkafka::Statistics;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::{mpsc, RwLock};
+use tokio_stream::wrappers::ReceiverStream;
+use tracing::{error, info, warn};
+
+pub struct KafkaStreams {
+    consumer: Arc<StreamConsumer>,
+    stream_state: Arc<RwLock<StreamState>>,
+    statistics: Arc<std::sync::RwLock<Statistics>>,
+    shutdown_handle: Shutdown,
+}
+
+impl KafkaStreams {
+    pub fn init(
+        context: KafkaContext,
+        stream_state: Arc<RwLock<StreamState>>,
+        shutdown_handle: Shutdown,
+    ) -> anyhow::Result<KafkaStreams> {
+        info!("Initializing KafkaStreams...");
+        let statistics = Arc::clone(&context.statistics);
+        let consumer = KafkaStreams::create_consumer(context);
+        info!("KafkaStreams initialized successfully.");
+
+        Ok(Self {
+            consumer,
+            stream_state,
+            statistics,
+            shutdown_handle,
+        })
+    }
+
+    pub fn consumer(&self) -> Arc<StreamConsumer> {
+        Arc::clone(&self.consumer)
+    }
+
+    pub fn statistics(&self) -> Arc<std::sync::RwLock<Statistics>> {
+        Arc::clone(&self.statistics)
+    }
+
+    pub fn state(&self) -> Arc<RwLock<StreamState>> {
+        Arc::clone(&self.stream_state)
+    }
+
+    /// Manages Kafka partition streams manually due to limitations in `rust-rdkafka`'s `split_partition_queue`.
+    ///
+    /// This method continuously listens incoming Kafka messages, dynamically creating
+    /// or updating streams for each partition. It is implemented using a separate standard thread to avoid
+    /// potential deadlocks and long-running task issues encountered with `tokio::spawn`.
+    ///
+    /// Steps:
+    /// 1. Consumes Kafka messages in a loop, processes each message to identify the associated partition.
+    /// 2. Dynamically creates a new stream for untracked partitions, allowing for isolated processing.
+    /// 3. Updates existing streams when new messages arrive for already tracked partitions.
+    /// 4. Listens for shutdown signals and gracefully terminates all partition streams, unsubscribing the consumer.
+    ///
+    /// Limitations and References:
+    ///   - Issues with `split_partition_queue` in rust-rdkafka:
+    ///   - https://github.com/fede1024/rust-rdkafka/issues/535
+    ///   - https://github.com/confluentinc/librdkafka/issues/4059
+    ///   - https://github.com/confluentinc/librdkafka/issues/4059
+    ///   - https://github.com/fede1024/rust-rdkafka/issues/654
+    ///   - https://github.com/fede1024/rust-rdkafka/issues/651
+    ///   - https://github.com/fede1024/rust-rdkafka/issues/604
+    ///   - https://github.com/fede1024/rust-rdkafka/issues/564
+    ///
+    /// - Potential deadlocks and long-running task issues with `tokio::spawn`:
+    /// - Details on blocking vs. async design choices:
+    ///   - https://ryhl.io/blog/async-what-is-blocking/
+    ///
+    /// Returns:
+    /// A `ReceiverStream` that produces `PartitionStreamReceiver` for each active partition.
+    pub fn partitioned(&self) -> ReceiverStream<PartitionStreamReceiver> {
+        let (stream_tx, stream_rx) = mpsc::channel(100);
+        let consumer = self.consumer();
+        let stream_state = self.state();
+        let tokio_handle = tokio::runtime::Handle::current();
+        let shutdown_handle = self.shutdown_handle.clone();
+
+        std::thread::Builder::new().name("kafka-streams-thread".to_string()).spawn(move || {
+            tokio_handle.block_on(async move {
+                loop {
+                    let result: Result<(), ConnectorError> = tokio::select! {
+                        result = consumer.recv() => {
+                            match result {
+                                Ok(msg) => KafkaStreams::handle_message(msg, &stream_state, &stream_tx).await.map_err(Into::into),
+                                Err(e) => Err(e.into())
+                            }
+                        }
+                        _ = shutdown_handle.recv() => {
+                            KafkaStreams::shutdown(&consumer, &stream_state).await;
+                            break;
+                        }
+                    };
+
+                    match result {
+                        Ok(_) => continue,
+                        Err(error) => match &error {
+                            ConnectorError::Connection(msg) => {
+                                error!("Connection error: {}", msg);
+                                tokio::time::sleep(Duration::from_secs(1)).await;
+                            }
+                            ConnectorError::Fatal(msg) => {
+                                error!("Fatal error: {}", msg);
+                                break;
+                            }
+                            ConnectorError::Auth(msg) => {
+                                error!("Authentication error: {}", msg);
+                                break;
+                            }
+                            error => {
+                                warn!("Non-fatal error: {}", error);
+                            }
+                        },
+                    }
+                }
+
+                info!("Kafka stream processing terminated");
+            });
+        }).expect("Failed to spawn Kafka partitioned stream thread");
+
+        ReceiverStream::new(stream_rx)
+    }
+
+    /// Handle individual Kafka message and route it to the proper partition stream
+    async fn handle_message(
+        msg: BorrowedMessage<'_>,
+        stream_state: &RwLock<StreamState>,
+        stream_tx: &mpsc::Sender<PartitionStreamReceiver>,
+    ) -> anyhow::Result<()> {
+        let mut state = stream_state.write().await;
+        let tp = TopicPartition::from_kafka_msg(&msg);
+        let consumer_record = ConsumerRecord::from_borrowed_msg(msg);
+
+        let partition_stream_tx =
+            KafkaStreams::get_or_create_partition_stream(&mut state, stream_tx, tp).await;
+        partition_stream_tx.send(consumer_record).await;
+
+        Ok(())
+    }
+
+    async fn get_or_create_partition_stream(
+        state: &mut StreamState,
+        stream_tx: &mpsc::Sender<PartitionStreamReceiver>,
+        tp: TopicPartition,
+    ) -> PartitionStreamSender {
+        if let Some(ps_tx) = state.get_partition_sender(&tp) {
+            ps_tx.clone()
+        } else {
+            Self::create_new_partition_stream(state, stream_tx, tp).await
+        }
+    }
+
+    async fn create_new_partition_stream(
+        state: &mut StreamState,
+        stream_tx: &mpsc::Sender<PartitionStreamReceiver>,
+        tp: TopicPartition,
+    ) -> PartitionStreamSender {
+        info!("Creating new stream for {:?}", tp);
+
+        let (ps_tx, ps_rx) = partition_stream::bounded(100_000, tp.clone());
+        state.insert_partition_sender(tp.clone(), ps_tx.clone());
+
+        if let Err(e) = stream_tx.send(ps_rx).await {
+            error!(
+                "Failed to send partition stream receiver for {:?}: {:?}",
+                tp, e
+            );
+        }
+
+        ps_tx
+    }
+
+    async fn shutdown(consumer: &Arc<StreamConsumer>, stream_state: &RwLock<StreamState>) {
+        info!("Gracefully stopping kafka partition streams!");
+        let mut state = stream_state.write().await;
+        state.clear();
+        consumer.unsubscribe();
+    }
+
+    fn create_consumer(context: KafkaContext) -> Arc<StreamConsumer> {
+        let kafka_config = &context.config;
+        let consumer_config = kafka_config.to_rdkafka_consumer_config();
+        info!(
+            "Creating Kafka consumer from configs: {:#?}",
+            &consumer_config
+        );
+
+        let consumer: StreamConsumer = consumer_config
+            .create_with_context(context.clone())
+            .expect("Consumer creation failed");
+
+        if consumer.recv().now_or_never().is_some() {
+            panic!("Consumer should not have any messages");
+        }
+
+        let consumer = Arc::new(consumer);
+        let topics = kafka_config
+            .consumer()
+            .expect("Consumer config is missing")
+            .topics();
+
+        KafkaStreams::subscribe(&consumer, &topics);
+
+        consumer
+    }
+
+    fn subscribe(consumer: &Arc<StreamConsumer>, topics: &[&str]) {
+        match consumer.subscribe(topics) {
+            Ok(_) => {
+                info!("Subscribed to topics: {:?}", topics);
+            }
+            Err(e) => {
+                error!("Error subscribing to topics: {:?} {:?}", topics, e);
+            }
+        };
+    }
+}
diff --git a/src/connectors/kafka/metrics.rs b/src/connectors/kafka/metrics.rs
new file mode 100644
index 000000000..45a5f52d4
--- /dev/null
+++ b/src/connectors/kafka/metrics.rs
@@ -0,0 +1,874 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use prometheus::core::{Collector, Desc};
+use prometheus::{
+    proto, Histogram, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge,
+    IntGaugeVec, Opts,
+};
+use rdkafka::Statistics;
+use std::sync::{Arc, RwLock};
+
+#[derive(Debug)]
+pub struct KafkaMetricsCollector {
+    stats: Arc<RwLock<Statistics>>,
+    descs: Vec<Desc>,
+    core_metrics: CoreMetrics,
+    broker_metrics: BrokerMetrics,
+    topic_metrics: TopicMetrics,
+    partition_metrics: PartitionMetrics,
+    consumer_metrics: ConsumerGroupMetrics,
+    eos_metrics: EosMetrics,
+}
+
+#[derive(Debug)]
+struct CoreMetrics {
+    msg_cnt: IntGauge,
+    msg_size: IntGauge,
+    msg_max: IntGauge,
+    msg_size_max: IntGauge,
+    metadata_cache_cnt: IntGauge,
+    tx: IntCounter,
+    tx_bytes: IntCounter,
+    rx: IntCounter,
+    rx_bytes: IntCounter,
+    txmsgs: IntCounter,
+    txmsg_bytes: IntCounter,
+    rxmsgs: IntCounter,
+    rxmsg_bytes: IntCounter,
+}
+
+#[derive(Debug)]
+struct BrokerMetrics {
+    state_cnt: IntGauge,
+    outbuf_cnt: IntGauge,
+    outbuf_msg_cnt: IntGauge,
+    waitresp_cnt: IntGauge,
+    waitresp_msg_cnt: IntGauge,
+    tx: IntCounter,
+    tx_bytes: IntCounter,
+    tx_errs: IntCounter,
+    tx_retries: IntCounter,
+    req_timeouts: IntCounter,
+    rx: IntCounter,
+    rx_bytes: IntCounter,
+    rx_errs: IntCounter,
+    rx_corrid_errs: IntCounter,
+    rx_partial: IntCounter,
+    connects: IntCounter,
+    disconnects: IntCounter,
+    int_latency: Histogram,
+    outbuf_latency: Histogram,
+    rtt: Histogram,
+    throttle: Histogram,
+}
+
+#[derive(Debug)]
+struct TopicMetrics {
+    metadata_age: IntGaugeVec,
+    batchsize: HistogramVec,
+    batchcnt: HistogramVec,
+}
+
+#[derive(Debug)]
+struct PartitionMetrics {
+    msgq_cnt: IntGaugeVec,
+    msgq_bytes: IntGaugeVec,
+    xmit_msgq_cnt: IntGaugeVec,
+    xmit_msgq_bytes: IntGaugeVec,
+    fetchq_cnt: IntGaugeVec,
+    fetchq_size: IntGaugeVec,
+    query_offset: IntGaugeVec,
+    next_offset: IntGaugeVec,
+    app_offset: IntGaugeVec,
+    stored_offset: IntGaugeVec,
+    committed_offset: IntGaugeVec,
+    eof_offset: IntGaugeVec,
+    lo_offset: IntGaugeVec,
+    hi_offset: IntGaugeVec,
+    consumer_lag: IntGaugeVec,
+    consumer_lag_stored: IntGaugeVec,
+    txmsgs: IntCounterVec,
+    txbytes: IntCounterVec,
+    rxmsgs: IntCounterVec,
+    rxbytes: IntCounterVec,
+    msgs: IntCounterVec,
+    rx_ver_drops: IntCounterVec,
+    msgs_inflight: IntGaugeVec,
+}
+
+#[derive(Debug)]
+struct ConsumerGroupMetrics {
+    rebalance_cnt: IntCounter,
+    rebalance_age: IntGauge,
+    assignment_size: IntGauge,
+}
+
+#[derive(Debug)]
+struct EosMetrics {
+    epoch_cnt: IntCounter,
+    producer_id: IntGauge,
+    producer_epoch: IntGauge,
+}
+
+impl CoreMetrics {
+    fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            msg_cnt: IntGauge::new(
+                "kafka_msg_cnt",
+                "Current number of messages in producer queues",
+            )?,
+            msg_size: IntGauge::new(
+                "kafka_msg_size",
+                "Current total size of messages in producer queues",
+            )?,
+            msg_max: IntGauge::new(
+                "kafka_msg_max",
+                "Maximum number of messages allowed in producer queues",
+            )?,
+            msg_size_max: IntGauge::new(
+                "kafka_msg_size_max",
+                "Maximum total size of messages allowed in producer queues",
+            )?,
+            metadata_cache_cnt: IntGauge::new(
+                "kafka_metadata_cache_cnt",
+                "Number of topics in metadata cache",
+            )?,
+            tx: IntCounter::new("kafka_tx_total", "Total number of transmissions")?,
+            tx_bytes: IntCounter::new("kafka_tx_bytes_total", "Total number of bytes transmitted")?,
+            rx: IntCounter::new("kafka_rx_total", "Total number of receptions")?,
+            rx_bytes: IntCounter::new("kafka_rx_bytes_total", "Total number of bytes received")?,
+            txmsgs: IntCounter::new("kafka_txmsgs_total", "Total number of messages transmitted")?,
+            txmsg_bytes: IntCounter::new(
+                "kafka_txmsg_bytes_total",
+                "Total number of message bytes transmitted",
+            )?,
+            rxmsgs: IntCounter::new("kafka_rxmsgs_total", "Total number of messages received")?,
+            rxmsg_bytes: IntCounter::new(
+                "kafka_rxmsg_bytes_total",
+                "Total number of message bytes received",
+            )?,
+        })
+    }
+}
+
+impl BrokerMetrics {
+    fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            state_cnt: IntGauge::new("kafka_broker_state", "Broker connection state")?,
+            outbuf_cnt: IntGauge::new(
+                "kafka_broker_outbuf_cnt",
+                "Number of requests awaiting transmission",
+            )?,
+            outbuf_msg_cnt: IntGauge::new(
+                "kafka_broker_outbuf_msg_cnt",
+                "Number of messages awaiting transmission",
+            )?,
+            waitresp_cnt: IntGauge::new(
+                "kafka_broker_waitresp_cnt",
+                "Number of requests in-flight",
+            )?,
+            waitresp_msg_cnt: IntGauge::new(
+                "kafka_broker_waitresp_msg_cnt",
+                "Number of messages in-flight",
+            )?,
+            tx: IntCounter::new("kafka_broker_tx_total", "Total broker transmissions")?,
+            tx_bytes: IntCounter::new(
+                "kafka_broker_tx_bytes_total",
+                "Total broker bytes transmitted",
+            )?,
+            tx_errs: IntCounter::new(
+                "kafka_broker_tx_errs_total",
+                "Total broker transmission errors",
+            )?,
+            tx_retries: IntCounter::new(
+                "kafka_broker_tx_retries_total",
+                "Total broker transmission retries",
+            )?,
+            req_timeouts: IntCounter::new(
+                "kafka_broker_req_timeouts_total",
+                "Total broker request timeouts",
+            )?,
+            rx: IntCounter::new("kafka_broker_rx_total", "Total broker receptions")?,
+            rx_bytes: IntCounter::new(
+                "kafka_broker_rx_bytes_total",
+                "Total broker bytes received",
+            )?,
+            rx_errs: IntCounter::new(
+                "kafka_broker_rx_errs_total",
+                "Total broker reception errors",
+            )?,
+            rx_corrid_errs: IntCounter::new(
+                "kafka_broker_rx_corrid_errs_total",
+                "Total broker correlation ID errors",
+            )?,
+            rx_partial: IntCounter::new(
+                "kafka_broker_rx_partial_total",
+                "Total broker partial message sets",
+            )?,
+            connects: IntCounter::new(
+                "kafka_broker_connects_total",
+                "Total broker connection attempts",
+            )?,
+            disconnects: IntCounter::new(
+                "kafka_broker_disconnects_total",
+                "Total broker disconnections",
+            )?,
+            int_latency: Histogram::with_opts(HistogramOpts::new(
+                "kafka_broker_int_latency",
+                "Internal broker latency",
+            ))?,
+            outbuf_latency: Histogram::with_opts(HistogramOpts::new(
+                "kafka_broker_outbuf_latency",
+                "Outbuf latency",
+            ))?,
+            rtt: Histogram::with_opts(HistogramOpts::new(
+                "kafka_broker_rtt",
+                "Broker round-trip time",
+            ))?,
+            throttle: Histogram::with_opts(HistogramOpts::new(
+                "kafka_broker_throttle",
+                "Broker throttle time",
+            ))?,
+        })
+    }
+}
+
+impl TopicMetrics {
+    fn new(labels: &[&str], descs: &mut Vec<Desc>) -> Self {
+        Self {
+            metadata_age: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_topic_metadata_age",
+                "Age of topic metadata",
+                labels,
+                descs,
+            ),
+            batchsize: KafkaMetricsCollector::create_histogram_vec(
+                "kafka_topic_batchsize",
+                "Topic batch sizes",
+                labels,
+                descs,
+            ),
+            batchcnt: KafkaMetricsCollector::create_histogram_vec(
+                "kafka_topic_batchcnt",
+                "Topic batch counts",
+                labels,
+                descs,
+            ),
+        }
+    }
+}
+
+impl PartitionMetrics {
+    fn new(labels: &[&str], descs: &mut Vec<Desc>) -> Self {
+        Self {
+            msgq_cnt: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_msgq_cnt",
+                "Messages in partition queue",
+                labels,
+                descs,
+            ),
+            msgq_bytes: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_msgq_bytes",
+                "Bytes in partition queue",
+                labels,
+                descs,
+            ),
+            xmit_msgq_cnt: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_xmit_msgq_cnt",
+                "Messages in partition transmit queue",
+                labels,
+                descs,
+            ),
+            xmit_msgq_bytes: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_xmit_msgq_bytes",
+                "Bytes in partition transmit queue",
+                labels,
+                descs,
+            ),
+            fetchq_cnt: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_fetchq_cnt",
+                "Messages in partition fetch queue",
+                labels,
+                descs,
+            ),
+            fetchq_size: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_fetchq_size",
+                "Size of partition fetch queue",
+                labels,
+                descs,
+            ),
+            query_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_query_offset",
+                "Current partition query offset",
+                labels,
+                descs,
+            ),
+            next_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_next_offset",
+                "Next partition offset",
+                labels,
+                descs,
+            ),
+            app_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_app_offset",
+                "Application partition offset",
+                labels,
+                descs,
+            ),
+            stored_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_stored_offset",
+                "Stored partition offset",
+                labels,
+                descs,
+            ),
+            committed_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_committed_offset",
+                "Committed partition offset",
+                labels,
+                descs,
+            ),
+            eof_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_eof_offset",
+                "EOF partition offset",
+                labels,
+                descs,
+            ),
+            lo_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_lo_offset",
+                "Low watermark partition offset",
+                labels,
+                descs,
+            ),
+            hi_offset: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_hi_offset",
+                "High watermark partition offset",
+                labels,
+                descs,
+            ),
+            consumer_lag: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_consumer_lag",
+                "Consumer lag",
+                labels,
+                descs,
+            ),
+            consumer_lag_stored: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_consumer_lag_stored",
+                "Stored consumer lag",
+                labels,
+                descs,
+            ),
+            txmsgs: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_txmsgs_total",
+                "Total partition messages transmitted",
+                labels,
+                descs,
+            ),
+            txbytes: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_txbytes_total",
+                "Total partition bytes transmitted",
+                labels,
+                descs,
+            ),
+            rxmsgs: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_rxmsgs_total",
+                "Total partition messages received",
+                labels,
+                descs,
+            ),
+            rxbytes: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_rxbytes_total",
+                "Total partition bytes received",
+                labels,
+                descs,
+            ),
+            msgs: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_msgs_total",
+                "Total partition messages",
+                labels,
+                descs,
+            ),
+            rx_ver_drops: KafkaMetricsCollector::create_counter_vec(
+                "kafka_partition_rx_ver_drops_total",
+                "Total partition version drops",
+                labels,
+                descs,
+            ),
+            msgs_inflight: KafkaMetricsCollector::create_gauge_vec(
+                "kafka_partition_msgs_inflight",
+                "Messages in flight",
+                labels,
+                descs,
+            ),
+        }
+    }
+
+    fn collect_metrics(
+        &self,
+        topic_name: &str,
+        partition_id: &i32,
+        partition: &rdkafka::statistics::Partition,
+    ) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+        let labels = &[topic_name, &partition_id.to_string()];
+
+        self.set_gauges(labels, partition);
+        self.set_counters(labels, partition);
+
+        self.collect_all_metrics(&mut mfs);
+
+        mfs
+    }
+
+    fn set_gauges(&self, labels: &[&str], partition: &rdkafka::statistics::Partition) {
+        self.msgq_cnt
+            .with_label_values(labels)
+            .set(partition.msgq_cnt);
+        self.msgq_bytes
+            .with_label_values(labels)
+            .set(partition.msgq_bytes as i64);
+        self.xmit_msgq_cnt
+            .with_label_values(labels)
+            .set(partition.xmit_msgq_cnt);
+        self.xmit_msgq_bytes
+            .with_label_values(labels)
+            .set(partition.xmit_msgq_bytes as i64);
+        self.fetchq_cnt
+            .with_label_values(labels)
+            .set(partition.fetchq_cnt);
+        self.fetchq_size
+            .with_label_values(labels)
+            .set(partition.fetchq_size as i64);
+        self.query_offset
+            .with_label_values(labels)
+            .set(partition.query_offset);
+        self.next_offset
+            .with_label_values(labels)
+            .set(partition.next_offset);
+        self.app_offset
+            .with_label_values(labels)
+            .set(partition.app_offset);
+        self.stored_offset
+            .with_label_values(labels)
+            .set(partition.stored_offset);
+        self.committed_offset
+            .with_label_values(labels)
+            .set(partition.committed_offset);
+        self.eof_offset
+            .with_label_values(labels)
+            .set(partition.eof_offset);
+        self.lo_offset
+            .with_label_values(labels)
+            .set(partition.lo_offset);
+        self.hi_offset
+            .with_label_values(labels)
+            .set(partition.hi_offset);
+        self.consumer_lag
+            .with_label_values(labels)
+            .set(partition.consumer_lag);
+        self.consumer_lag_stored
+            .with_label_values(labels)
+            .set(partition.consumer_lag_stored);
+        self.msgs_inflight
+            .with_label_values(labels)
+            .set(partition.msgs_inflight);
+    }
+
+    fn set_counters(&self, labels: &[&str], partition: &rdkafka::statistics::Partition) {
+        self.txmsgs
+            .with_label_values(labels)
+            .inc_by(partition.txmsgs);
+        self.txbytes
+            .with_label_values(labels)
+            .inc_by(partition.txbytes);
+        self.rxmsgs
+            .with_label_values(labels)
+            .inc_by(partition.rxmsgs);
+        self.rxbytes
+            .with_label_values(labels)
+            .inc_by(partition.rxbytes);
+        self.msgs.with_label_values(labels).inc_by(partition.msgs);
+        self.rx_ver_drops
+            .with_label_values(labels)
+            .inc_by(partition.rx_ver_drops);
+    }
+
+    fn collect_all_metrics(&self, mfs: &mut Vec<proto::MetricFamily>) {
+        // Collect gauges
+        mfs.extend(self.msgq_cnt.collect());
+        mfs.extend(self.msgq_bytes.collect());
+        mfs.extend(self.xmit_msgq_cnt.collect());
+        mfs.extend(self.xmit_msgq_bytes.collect());
+        mfs.extend(self.fetchq_cnt.collect());
+        mfs.extend(self.fetchq_size.collect());
+        mfs.extend(self.query_offset.collect());
+        mfs.extend(self.next_offset.collect());
+        mfs.extend(self.app_offset.collect());
+        mfs.extend(self.stored_offset.collect());
+        mfs.extend(self.committed_offset.collect());
+        mfs.extend(self.eof_offset.collect());
+        mfs.extend(self.lo_offset.collect());
+        mfs.extend(self.hi_offset.collect());
+        mfs.extend(self.consumer_lag.collect());
+        mfs.extend(self.consumer_lag_stored.collect());
+        mfs.extend(self.msgs_inflight.collect());
+
+        // Collect counters
+        mfs.extend(self.txmsgs.collect());
+        mfs.extend(self.txbytes.collect());
+        mfs.extend(self.rxmsgs.collect());
+        mfs.extend(self.rxbytes.collect());
+        mfs.extend(self.msgs.collect());
+        mfs.extend(self.rx_ver_drops.collect());
+    }
+}
+
+impl BrokerMetrics {
+    fn collect_metrics(&self, broker: &rdkafka::statistics::Broker) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+
+        self.state_cnt.set(match broker.state.as_str() {
+            "UP" => 1,
+            "DOWN" => 0,
+            _ => -1,
+        });
+
+        self.set_gauges(broker);
+        self.set_counters(broker);
+        self.set_latency_metrics(broker);
+        self.collect_all_metrics(&mut mfs);
+
+        mfs
+    }
+
+    fn set_gauges(&self, broker: &rdkafka::statistics::Broker) {
+        self.outbuf_cnt.set(broker.outbuf_cnt);
+        self.outbuf_msg_cnt.set(broker.outbuf_msg_cnt);
+        self.waitresp_cnt.set(broker.waitresp_cnt);
+        self.waitresp_msg_cnt.set(broker.waitresp_msg_cnt);
+    }
+
+    fn set_counters(&self, broker: &rdkafka::statistics::Broker) {
+        self.tx.inc_by(broker.tx);
+        self.tx_bytes.inc_by(broker.txbytes);
+        self.tx_errs.inc_by(broker.txerrs);
+        self.tx_retries.inc_by(broker.txretries);
+        self.req_timeouts.inc_by(broker.req_timeouts);
+        self.rx.inc_by(broker.rx);
+        self.rx_bytes.inc_by(broker.rxbytes);
+        self.rx_errs.inc_by(broker.rxerrs);
+        self.rx_corrid_errs.inc_by(broker.rxcorriderrs);
+        self.rx_partial.inc_by(broker.rxpartial);
+
+        if let Some(connects) = broker.connects {
+            self.connects.inc_by(connects as u64);
+        }
+        if let Some(disconnects) = broker.disconnects {
+            self.disconnects.inc_by(disconnects as u64);
+        }
+    }
+
+    fn set_latency_metrics(&self, broker: &rdkafka::statistics::Broker) {
+        if let Some(ref latency) = broker.int_latency {
+            self.int_latency.observe(latency.avg as f64);
+        }
+        if let Some(ref latency) = broker.outbuf_latency {
+            self.outbuf_latency.observe(latency.avg as f64);
+        }
+        if let Some(ref rtt) = broker.rtt {
+            self.rtt.observe(rtt.avg as f64);
+        }
+        if let Some(ref throttle) = broker.throttle {
+            self.throttle.observe(throttle.avg as f64);
+        }
+    }
+
+    fn collect_all_metrics(&self, mfs: &mut Vec<proto::MetricFamily>) {
+        mfs.extend(self.state_cnt.collect());
+        mfs.extend(self.outbuf_cnt.collect());
+        mfs.extend(self.outbuf_msg_cnt.collect());
+        mfs.extend(self.waitresp_cnt.collect());
+        mfs.extend(self.waitresp_msg_cnt.collect());
+        mfs.extend(self.tx.collect());
+        mfs.extend(self.tx_bytes.collect());
+        mfs.extend(self.tx_errs.collect());
+        mfs.extend(self.tx_retries.collect());
+        mfs.extend(self.req_timeouts.collect());
+        mfs.extend(self.rx.collect());
+        mfs.extend(self.rx_bytes.collect());
+        mfs.extend(self.rx_errs.collect());
+        mfs.extend(self.rx_corrid_errs.collect());
+        mfs.extend(self.rx_partial.collect());
+        mfs.extend(self.connects.collect());
+        mfs.extend(self.disconnects.collect());
+        mfs.extend(self.int_latency.collect());
+        mfs.extend(self.outbuf_latency.collect());
+        mfs.extend(self.rtt.collect());
+        mfs.extend(self.throttle.collect());
+    }
+}
+
+impl TopicMetrics {
+    fn collect_metrics(
+        &self,
+        topic_name: &str,
+        topic: &rdkafka::statistics::Topic,
+    ) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+        let labels = &[topic_name];
+
+        self.metadata_age
+            .with_label_values(labels)
+            .set(topic.metadata_age);
+        self.batchsize
+            .with_label_values(labels)
+            .observe(topic.batchsize.avg as f64);
+        self.batchcnt
+            .with_label_values(labels)
+            .observe(topic.batchcnt.avg as f64);
+
+        mfs.extend(self.metadata_age.collect());
+        mfs.extend(self.batchsize.collect());
+        mfs.extend(self.batchcnt.collect());
+
+        mfs
+    }
+}
+
+impl ConsumerGroupMetrics {
+    fn collect_metrics(
+        &self,
+        cgrp: &rdkafka::statistics::ConsumerGroup,
+    ) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+
+        self.rebalance_cnt.inc_by(cgrp.rebalance_cnt as u64);
+        self.rebalance_age.set(cgrp.rebalance_age);
+        self.assignment_size.set(cgrp.assignment_size as i64);
+
+        mfs.extend(self.rebalance_cnt.collect());
+        mfs.extend(self.rebalance_age.collect());
+        mfs.extend(self.assignment_size.collect());
+
+        mfs
+    }
+}
+
+impl EosMetrics {
+    fn collect_metrics(
+        &self,
+        eos: &rdkafka::statistics::ExactlyOnceSemantics,
+    ) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+
+        self.epoch_cnt.inc_by(eos.epoch_cnt as u64);
+        self.producer_id.set(eos.producer_id);
+        self.producer_epoch.set(eos.producer_epoch);
+
+        mfs.extend(self.epoch_cnt.collect());
+        mfs.extend(self.producer_id.collect());
+        mfs.extend(self.producer_epoch.collect());
+
+        mfs
+    }
+}
+
+impl CoreMetrics {
+    fn collect_metrics(&self, stats: &Statistics) -> Vec<proto::MetricFamily> {
+        let mut mfs = Vec::new();
+
+        self.msg_cnt.set(stats.msg_cnt as i64);
+        self.msg_size.set(stats.msg_size as i64);
+        self.msg_max.set(stats.msg_max as i64);
+        self.msg_size_max.set(stats.msg_size_max as i64);
+        self.metadata_cache_cnt.set(stats.metadata_cache_cnt);
+        self.tx.inc_by(stats.tx as u64);
+        self.tx_bytes.inc_by(stats.tx_bytes as u64);
+        self.rx.inc_by(stats.rx as u64);
+        self.rx_bytes.inc_by(stats.rx_bytes as u64);
+        self.txmsgs.inc_by(stats.txmsgs as u64);
+        self.txmsg_bytes.inc_by(stats.txmsg_bytes as u64);
+        self.rxmsgs.inc_by(stats.rxmsgs as u64);
+        self.rxmsg_bytes.inc_by(stats.rxmsg_bytes as u64);
+
+        mfs.extend(self.msg_cnt.collect());
+        mfs.extend(self.msg_size.collect());
+        mfs.extend(self.msg_max.collect());
+        mfs.extend(self.msg_size_max.collect());
+        mfs.extend(self.metadata_cache_cnt.collect());
+        mfs.extend(self.tx.collect());
+        mfs.extend(self.tx_bytes.collect());
+        mfs.extend(self.rx.collect());
+        mfs.extend(self.rx_bytes.collect());
+        mfs.extend(self.txmsgs.collect());
+        mfs.extend(self.txmsg_bytes.collect());
+        mfs.extend(self.rxmsgs.collect());
+        mfs.extend(self.rxmsg_bytes.collect());
+
+        mfs
+    }
+}
+
+impl ConsumerGroupMetrics {
+    fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            rebalance_cnt: IntCounter::new("kafka_cgrp_rebalance_total", "Total rebalances")?,
+            rebalance_age: IntGauge::new("kafka_cgrp_rebalance_age", "Rebalance age")?,
+            assignment_size: IntGauge::new("kafka_cgrp_assignment_size", "Assignment size")?,
+        })
+    }
+}
+
+impl EosMetrics {
+    fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            epoch_cnt: IntCounter::new("kafka_eos_epoch_total", "Total number of epochs")?,
+            producer_id: IntGauge::new("kafka_eos_producer_id", "Producer ID")?,
+            producer_epoch: IntGauge::new("kafka_eos_producer_epoch", "Producer epoch")?,
+        })
+    }
+}
+
+impl KafkaMetricsCollector {
+    pub fn new(stats: Arc<RwLock<Statistics>>) -> anyhow::Result<KafkaMetricsCollector> {
+        let mut descs = Vec::new();
+        let topic_labels = &["topic"];
+        let partition_labels = &["topic", "partition"];
+
+        let core_metrics = CoreMetrics::new()?;
+        let broker_metrics = BrokerMetrics::new()?;
+        let topic_metrics = TopicMetrics::new(topic_labels, &mut descs);
+        let partition_metrics = PartitionMetrics::new(partition_labels, &mut descs);
+        let consumer_metrics = ConsumerGroupMetrics::new()?;
+        let eos_metrics = EosMetrics::new()?;
+
+        Ok(KafkaMetricsCollector {
+            stats,
+            descs,
+            core_metrics,
+            broker_metrics,
+            topic_metrics,
+            partition_metrics,
+            consumer_metrics,
+            eos_metrics,
+        })
+    }
+
+    fn create_gauge_vec(
+        name: &str,
+        help: &str,
+        labels: &[&str],
+        descs: &mut Vec<Desc>,
+    ) -> IntGaugeVec {
+        let gauge = IntGaugeVec::new(Opts::new(name, help), labels).unwrap();
+        descs.extend(gauge.clone().desc().into_iter().cloned());
+        gauge
+    }
+
+    fn create_counter_vec(
+        name: &str,
+        help: &str,
+        labels: &[&str],
+        descs: &mut Vec<Desc>,
+    ) -> IntCounterVec {
+        let counter = IntCounterVec::new(Opts::new(name, help), labels).unwrap();
+        descs.extend(counter.clone().desc().into_iter().cloned());
+        counter
+    }
+
+    fn create_histogram_vec(
+        name: &str,
+        help: &str,
+        labels: &[&str],
+        descs: &mut Vec<Desc>,
+    ) -> HistogramVec {
+        let histogram = HistogramVec::new(HistogramOpts::new(name, help), labels).unwrap();
+        descs.extend(histogram.clone().desc().into_iter().cloned());
+        histogram
+    }
+}
+
+impl Collector for KafkaMetricsCollector {
+    fn desc(&self) -> Vec<&Desc> {
+        self.descs.iter().collect()
+    }
+
+    fn collect(&self) -> Vec<proto::MetricFamily> {
+        let stats = match self.stats.read() {
+            Ok(stats) => stats,
+            Err(_) => return vec![],
+        };
+
+        let mut mfs = Vec::new();
+
+        // Collect core metrics
+        mfs.extend(self.core_metrics.collect_metrics(&stats));
+
+        // Collect broker metrics
+        for (_broker_id, broker) in stats.brokers.iter() {
+            mfs.extend(self.broker_metrics.collect_metrics(broker));
+        }
+
+        // Collect topic and partition metrics
+        for (topic_name, topic) in stats.topics.iter() {
+            mfs.extend(self.topic_metrics.collect_metrics(topic_name, topic));
+
+            // Collect partition metrics for each topic
+            for (partition_id, partition) in topic.partitions.iter() {
+                mfs.extend(self.partition_metrics.collect_metrics(
+                    topic_name,
+                    partition_id,
+                    partition,
+                ));
+            }
+        }
+
+        // Collect consumer group metrics
+        if let Some(ref cgrp) = stats.cgrp {
+            mfs.extend(self.consumer_metrics.collect_metrics(cgrp));
+        }
+
+        // Collect EOS metrics
+        if let Some(ref eos) = stats.eos {
+            mfs.extend(self.eos_metrics.collect_metrics(eos));
+        }
+
+        mfs
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use prometheus::Registry;
+
+    #[test]
+    fn test_kafka_metrics_collector() {
+        let stats = Arc::new(RwLock::new(Statistics::default()));
+        let collector = KafkaMetricsCollector::new(stats).unwrap();
+
+        let descs = collector.desc();
+        assert!(!descs.is_empty());
+
+        let mfs = collector.collect();
+        assert!(!mfs.is_empty());
+
+        let registry = Registry::new();
+        assert!(registry.register(Box::new(collector)).is_ok());
+    }
+}
diff --git a/src/connectors/kafka/mod.rs b/src/connectors/kafka/mod.rs
new file mode 100644
index 000000000..211371b44
--- /dev/null
+++ b/src/connectors/kafka/mod.rs
@@ -0,0 +1,266 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::kafka::config::KafkaConfig;
+use derive_more::Constructor;
+use rdkafka::client::OAuthToken;
+use rdkafka::consumer::{ConsumerContext, Rebalance};
+use rdkafka::error::KafkaResult;
+use rdkafka::message::{BorrowedMessage, Headers};
+use rdkafka::producer::ProducerContext;
+use rdkafka::topic_partition_list::TopicPartitionListElem;
+use rdkafka::{ClientContext, Message, Offset, Statistics};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::error::Error;
+use std::sync::{Arc, RwLock};
+use tokio::sync::mpsc;
+use tokio::sync::mpsc::Receiver;
+use tracing::{error, info, warn};
+
+pub mod config;
+pub mod consumer;
+pub mod metrics;
+mod partition_stream;
+pub mod processor;
+pub mod rebalance_listener;
+pub mod sink;
+pub mod state;
+#[allow(dead_code)]
+type BaseConsumer = rdkafka::consumer::BaseConsumer<KafkaContext>;
+#[allow(dead_code)]
+type FutureProducer = rdkafka::producer::FutureProducer<KafkaContext>;
+type StreamConsumer = rdkafka::consumer::StreamConsumer<KafkaContext>;
+
+#[derive(Clone, Debug)]
+pub struct KafkaContext {
+    config: Arc<KafkaConfig>,
+    statistics: Arc<RwLock<Statistics>>,
+    rebalance_tx: mpsc::Sender<RebalanceEvent>,
+}
+
+impl KafkaContext {
+    pub fn new(config: Arc<KafkaConfig>) -> (Self, Receiver<RebalanceEvent>) {
+        let (rebalance_tx, rebalance_rx) = mpsc::channel(10);
+        let statistics = Arc::new(RwLock::new(Statistics::default()));
+        (
+            Self {
+                config,
+                statistics,
+                rebalance_tx,
+            },
+            rebalance_rx,
+        )
+    }
+
+    pub fn notify(&self, rebalance_event: RebalanceEvent) {
+        let rebalance_sender = self.rebalance_tx.clone();
+        std::thread::spawn(move || {
+            info!("Sending RebalanceEvent to listener...");
+            if let Err(e) = rebalance_sender.blocking_send(rebalance_event) {
+                warn!("Rebalance event receiver is closed! {:?}", e);
+            } else {
+                info!("RebalanceEvent sent successfully!");
+            }
+        });
+    }
+
+    pub fn config(&self) -> Arc<KafkaConfig> {
+        Arc::clone(&self.config)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum RebalanceEvent {
+    Assign(TopicPartitionList),
+    Revoke(TopicPartitionList, std::sync::mpsc::Sender<()>),
+}
+
+impl RebalanceEvent {
+    pub fn get_assignment(&self) -> &TopicPartitionList {
+        match self {
+            RebalanceEvent::Assign(tpl) => tpl,
+            RebalanceEvent::Revoke(tpl, _) => tpl,
+        }
+    }
+}
+
+#[derive(Constructor, Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq, Hash)]
+pub struct TopicPartition {
+    pub topic: String,
+    pub partition: i32,
+}
+
+impl TopicPartition {
+    pub fn from_kafka_msg(msg: &BorrowedMessage) -> Self {
+        Self::new(msg.topic().to_owned(), msg.partition())
+    }
+
+    pub fn from_tp_elem(elem: &TopicPartitionListElem<'_>) -> Self {
+        Self::new(elem.topic().to_owned(), elem.partition())
+    }
+}
+
+#[derive(Constructor, Serialize, Deserialize, Debug, Default, Clone, PartialEq, Eq, Hash)]
+pub struct TopicPartitionList {
+    pub tpl: Vec<TopicPartition>,
+}
+
+impl TopicPartitionList {
+    pub fn from_rdkafka_tpl(tpl: &rdkafka::topic_partition_list::TopicPartitionList) -> Self {
+        let elements = tpl.elements();
+        let mut tp_vec = Vec::with_capacity(elements.len());
+        for ref element in elements {
+            let tp = TopicPartition::from_tp_elem(element);
+            tp_vec.push(tp);
+        }
+        Self::new(tp_vec)
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.tpl.is_empty()
+    }
+}
+
+#[derive(Constructor, Debug, Hash, Eq, PartialEq)]
+pub struct ConsumerRecord {
+    pub payload: Option<Vec<u8>>,
+    pub key: Option<Vec<u8>>,
+    pub topic: String,
+    pub partition: i32,
+    pub offset: i64,
+    pub timestamp: Option<i64>,
+    //pub headers: Option<HashMap<String, Option<String>>>,
+}
+
+impl ConsumerRecord {
+    pub fn from_borrowed_msg(msg: BorrowedMessage) -> Self {
+        Self {
+            key: msg.key().map(|k| k.to_vec()),
+            payload: msg.payload().map(|p| p.to_vec()),
+            topic: msg.topic().to_owned(),
+            partition: msg.partition(),
+            offset: msg.offset(),
+            timestamp: msg.timestamp().to_millis(),
+            //headers: extract_headers(&msg),
+        }
+    }
+
+    pub fn key_str(&self) -> String {
+        self.key.clone().map_or_else(
+            || String::from("null"),
+            |k| String::from_utf8_lossy(k.as_ref()).to_string(),
+        )
+    }
+
+    pub fn offset_to_commit(&self) -> KafkaResult<rdkafka::TopicPartitionList> {
+        let mut offset_to_commit = rdkafka::TopicPartitionList::new();
+        offset_to_commit.add_partition_offset(
+            &self.topic,
+            self.partition,
+            Offset::Offset(self.offset + 1),
+        )?;
+        Ok(offset_to_commit)
+    }
+}
+
+#[allow(unused)]
+fn extract_headers(msg: &BorrowedMessage<'_>) -> Option<HashMap<String, Option<String>>> {
+    msg.headers().map(|headers| {
+        headers
+            .iter()
+            .map(|header| {
+                (
+                    header.key.to_string(),
+                    header.value.map(|v| String::from_utf8_lossy(v).to_string()),
+                )
+            })
+            .collect()
+    })
+}
+
+impl ConsumerContext for KafkaContext {
+    fn pre_rebalance(
+        &self,
+        _base_consumer: &rdkafka::consumer::BaseConsumer<Self>,
+        rebalance: &Rebalance<'_>,
+    ) {
+        info!("Running pre-rebalance with {:?}", rebalance);
+        match rebalance {
+            Rebalance::Revoke(tpl) => {
+                let (pq_waiter_tx, pq_waiter_rx) = std::sync::mpsc::channel();
+
+                let tpl = TopicPartitionList::from_rdkafka_tpl(tpl);
+                self.notify(RebalanceEvent::Revoke(tpl, pq_waiter_tx));
+
+                if pq_waiter_rx.recv().is_err() {
+                    warn!("Queue termination sender dropped");
+                }
+                info!("Rebalance Revoke started");
+            }
+            Rebalance::Assign(tpl) => {
+                let tpl = TopicPartitionList::from_rdkafka_tpl(tpl);
+                self.notify(RebalanceEvent::Assign(tpl));
+            }
+
+            Rebalance::Error(err) => error!("Error occurred during rebalance {:?}", err),
+        };
+    }
+
+    fn post_rebalance(
+        &self,
+        _base_consumer: &rdkafka::consumer::BaseConsumer<Self>,
+        rebalance: &Rebalance<'_>,
+    ) {
+        info!("Running post-rebalance with {:?}", rebalance);
+    }
+}
+
+impl ProducerContext for KafkaContext {
+    type DeliveryOpaque = ();
+    fn delivery(
+        &self,
+        _delivery_result: &rdkafka::message::DeliveryResult<'_>,
+        _delivery_opaque: Self::DeliveryOpaque,
+    ) {
+    }
+}
+
+impl ClientContext for KafkaContext {
+    // TODO: when implementing OAuth, set this to true
+    const ENABLE_REFRESH_OAUTH_TOKEN: bool = false;
+
+    fn stats(&self, new_stats: Statistics) {
+        match self.statistics.write() {
+            Ok(mut stats) => {
+                *stats = new_stats;
+            }
+            Err(e) => {
+                error!("Cannot write to kafka statistics from RwLock. Error: {}", e)
+            }
+        };
+    }
+
+    fn generate_oauth_token(
+        &self,
+        _oauthbearer_config: Option<&str>,
+    ) -> Result<OAuthToken, Box<dyn Error>> {
+        // TODO Implement OAuth token generation when needed
+        Err("OAuth token generation is not implemented".into())
+    }
+}
diff --git a/src/connectors/kafka/partition_stream.rs b/src/connectors/kafka/partition_stream.rs
new file mode 100644
index 000000000..f1a6ca8bc
--- /dev/null
+++ b/src/connectors/kafka/partition_stream.rs
@@ -0,0 +1,110 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::kafka::{ConsumerRecord, TopicPartition};
+use std::sync::Arc;
+use tokio::sync::{mpsc, Notify};
+use tokio_stream::wrappers::ReceiverStream;
+use tracing::{error, info};
+
+#[derive(Clone)]
+pub struct PartitionStreamSender {
+    inner: mpsc::Sender<ConsumerRecord>,
+    notify: Arc<Notify>,
+}
+
+impl PartitionStreamSender {
+    fn new(inner: mpsc::Sender<ConsumerRecord>, notify: Arc<Notify>) -> Self {
+        Self { inner, notify }
+    }
+
+    pub fn terminate(&self) {
+        self.notify.notify_waiters();
+    }
+
+    pub async fn send(&self, consumer_record: ConsumerRecord) {
+        if let Err(e) = self.inner.send(consumer_record).await {
+            error!("Failed to send message to partition stream: {:?}", e);
+        }
+    }
+
+    pub fn sender(&self) -> mpsc::Sender<ConsumerRecord> {
+        self.inner.clone()
+    }
+}
+
+pub struct PartitionStreamReceiver {
+    inner: ReceiverStream<ConsumerRecord>,
+    topic_partition: TopicPartition,
+    notify: Arc<Notify>,
+}
+
+impl PartitionStreamReceiver {
+    fn new(
+        receiver: mpsc::Receiver<ConsumerRecord>,
+        topic_partition: TopicPartition,
+        notify: Arc<Notify>,
+    ) -> Self {
+        Self {
+            inner: ReceiverStream::new(receiver),
+            topic_partition,
+            notify,
+        }
+    }
+
+    /// Processes the stream with a provided callback and listens for termination.
+    ///
+    /// # Parameters
+    /// - `invoke`: A callback function that processes the `ReceiverStream<ConsumerRecord>`.
+    ///
+    /// # Behavior
+    /// - The callback runs until either the stream is completed or a termination signal is received.
+    pub async fn run_drain<Fut, F>(self, f: F)
+    where
+        F: Fn(ReceiverStream<ConsumerRecord>) -> Fut,
+        Fut: futures_util::Future<Output = ()>,
+    {
+        let notify = self.notify.clone();
+
+        tokio::select! {
+            _ = f(self.inner) => {
+                info!("PartitionStreamReceiver completed processing for {:?}.", self.topic_partition);
+            }
+            _ = notify.notified() => {
+                info!("Received termination signal for {:?}.", self.topic_partition);
+            }
+        }
+    }
+
+    pub fn topic_partition(&self) -> &TopicPartition {
+        &self.topic_partition
+    }
+}
+
+pub fn bounded(
+    size: usize,
+    topic_partition: TopicPartition,
+) -> (PartitionStreamSender, PartitionStreamReceiver) {
+    let (tx, rx) = mpsc::channel(size);
+    let notify = Arc::new(Notify::new());
+
+    let sender = PartitionStreamSender::new(tx, notify.clone());
+    let receiver = PartitionStreamReceiver::new(rx, topic_partition, notify);
+
+    (sender, receiver)
+}
diff --git a/src/connectors/kafka/processor.rs b/src/connectors/kafka/processor.rs
new file mode 100644
index 000000000..e06835121
--- /dev/null
+++ b/src/connectors/kafka/processor.rs
@@ -0,0 +1,181 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::common::processor::Processor;
+use crate::connectors::kafka::config::BufferConfig;
+use crate::connectors::kafka::{ConsumerRecord, StreamConsumer, TopicPartition};
+use crate::event::format::EventFormat;
+use crate::event::format::{self, LogSource};
+use crate::event::Event as ParseableEvent;
+use crate::handlers::http::ingest::create_stream_if_not_exists;
+use crate::metadata::STREAM_INFO;
+use crate::storage::StreamType;
+use async_trait::async_trait;
+use chrono::Utc;
+use futures_util::StreamExt;
+use rdkafka::consumer::{CommitMode, Consumer};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio_stream::wrappers::ReceiverStream;
+use tracing::{debug, error};
+
+#[derive(Default, Debug, Clone)]
+pub struct ParseableSinkProcessor;
+
+impl ParseableSinkProcessor {
+    async fn build_event_from_chunk(
+        &self,
+        records: &[ConsumerRecord],
+    ) -> anyhow::Result<ParseableEvent> {
+        let stream_name = records
+            .first()
+            .map(|r| r.topic.as_str())
+            .unwrap_or_default();
+
+        create_stream_if_not_exists(stream_name, StreamType::UserDefined, LogSource::Json).await?;
+
+        let schema = STREAM_INFO.schema_raw(stream_name)?;
+        let time_partition = STREAM_INFO.get_time_partition(stream_name)?;
+        let static_schema_flag = STREAM_INFO.get_static_schema_flag(stream_name)?;
+        let schema_version = STREAM_INFO.get_schema_version(stream_name)?;
+
+        let (json_vec, total_payload_size) = Self::json_vec(records);
+        let batch_json_event = format::json::Event {
+            data: Value::Array(json_vec.to_vec()),
+        };
+
+        let (rb, is_first) = batch_json_event.into_recordbatch(
+            &schema,
+            static_schema_flag,
+            time_partition.as_ref(),
+            schema_version,
+        )?;
+
+        let p_event = ParseableEvent {
+            rb,
+            stream_name: stream_name.to_string(),
+            origin_format: "json",
+            origin_size: total_payload_size,
+            is_first_event: is_first,
+            parsed_timestamp: Utc::now().naive_utc(),
+            time_partition: None,
+            custom_partition_values: HashMap::new(),
+            stream_type: StreamType::UserDefined,
+        };
+
+        Ok(p_event)
+    }
+
+    fn json_vec(records: &[ConsumerRecord]) -> (Vec<Value>, u64) {
+        let mut json_vec = Vec::with_capacity(records.len());
+        let mut total_payload_size = 0u64;
+
+        for record in records.iter().filter_map(|r| r.payload.as_ref()) {
+            total_payload_size += record.len() as u64;
+            if let Ok(value) = serde_json::from_slice::<Value>(record) {
+                json_vec.push(value);
+            }
+        }
+
+        (json_vec, total_payload_size)
+    }
+}
+
+#[async_trait]
+impl Processor<Vec<ConsumerRecord>, ()> for ParseableSinkProcessor {
+    async fn process(&self, records: Vec<ConsumerRecord>) -> anyhow::Result<()> {
+        let len = records.len();
+        debug!("Processing {} records", len);
+
+        self.build_event_from_chunk(&records)
+            .await?
+            .process()
+            .await?;
+
+        debug!("Processed {} records", len);
+        Ok(())
+    }
+}
+
+#[derive(Clone)]
+pub struct StreamWorker<P>
+where
+    P: Processor<Vec<ConsumerRecord>, ()>,
+{
+    processor: Arc<P>,
+    consumer: Arc<StreamConsumer>,
+    buffer_config: BufferConfig,
+}
+
+impl<P> StreamWorker<P>
+where
+    P: Processor<Vec<ConsumerRecord>, ()> + Send + Sync + 'static,
+{
+    pub fn new(processor: Arc<P>, consumer: Arc<StreamConsumer>) -> Self {
+        let buffer_config = consumer
+            .context()
+            .config()
+            .consumer()
+            .expect("Consumer config is missing")
+            .buffer_config();
+
+        Self {
+            processor,
+            consumer,
+            buffer_config,
+        }
+    }
+
+    pub async fn process_partition(
+        &self,
+        tp: TopicPartition,
+        record_stream: ReceiverStream<ConsumerRecord>,
+    ) -> anyhow::Result<()> {
+        let chunked_stream = tokio_stream::StreamExt::chunks_timeout(
+            record_stream,
+            self.buffer_config.buffer_size,
+            self.buffer_config.buffer_timeout,
+        );
+
+        chunked_stream
+            .for_each_concurrent(None, |records| async {
+                if let Some(last_record) = records.iter().max_by_key(|r| r.offset) {
+                    let tpl = last_record.offset_to_commit().unwrap();
+
+                    if let Err(e) = self.processor.process(records).await {
+                        error!("Failed to process records for {:?}: {:?}", tp, e);
+                    }
+
+                    //CommitMode::Async race condition.
+                    //@see https://github.com/confluentinc/librdkafka/issues/4534
+                    //@see https://github.com/confluentinc/librdkafka/issues/4059
+                    if let Err(e) = self.consumer.commit(&tpl, CommitMode::Sync) {
+                        error!(error = %e, "Failed to commit offsets for {:?}", tpl);
+                    } else {
+                        debug!("Committed offsets for {:?}", tpl);
+                    }
+                }
+            })
+            .await;
+
+        self.processor.post_stream().await?;
+
+        Ok(())
+    }
+}
diff --git a/src/connectors/kafka/rebalance_listener.rs b/src/connectors/kafka/rebalance_listener.rs
new file mode 100644
index 000000000..8372e1b5b
--- /dev/null
+++ b/src/connectors/kafka/rebalance_listener.rs
@@ -0,0 +1,87 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::common::shutdown::Shutdown;
+use crate::connectors::kafka::state::StreamState;
+use crate::connectors::kafka::RebalanceEvent;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tokio::{runtime::Handle, sync::mpsc::Receiver};
+use tracing::{info, warn};
+
+pub struct RebalanceListener {
+    rebalance_rx: Receiver<RebalanceEvent>,
+    stream_state: Arc<RwLock<StreamState>>,
+    shutdown_handle: Shutdown,
+}
+
+impl RebalanceListener {
+    pub fn new(
+        rebalance_rx: Receiver<RebalanceEvent>,
+        stream_state: Arc<RwLock<StreamState>>,
+        shutdown_handle: Shutdown,
+    ) -> Self {
+        Self {
+            rebalance_rx,
+            stream_state,
+            shutdown_handle,
+        }
+    }
+
+    pub fn start(self) {
+        let mut rebalance_receiver = self.rebalance_rx;
+        let stream_state = self.stream_state.clone();
+        let shutdown_handle = self.shutdown_handle.clone();
+        let tokio_runtime_handle = Handle::current();
+
+        std::thread::Builder::new().name("rebalance-listener-thread".to_string()).spawn(move || {
+            tokio_runtime_handle.block_on(async move {
+                loop {
+                    tokio::select! {
+                        rebalance = rebalance_receiver.recv() => {
+                            match rebalance  {
+                                Some(RebalanceEvent::Assign(tpl)) => info!("RebalanceEvent Assign: {:?}", tpl),
+                                Some(RebalanceEvent::Revoke(tpl, callback)) => {
+                                    info!("RebalanceEvent Revoke: {:?}", tpl);
+                                    if let Ok(mut stream_state) = stream_state.try_write() {
+                                        stream_state.terminate_partition_streams(tpl).await;
+                                        drop(stream_state);
+                                    } else {
+                                        warn!("Stream state lock is busy, skipping rebalance revoke for {:?}", tpl);
+                                    }
+                                    if let Err(err) = callback.send(()) {
+                                        warn!("Error during sending response to context. Cause: {:?}", err);
+                                    }
+                                    info!("Finished Rebalance Revoke");
+                                }
+                                None => {
+                                    info!("Rebalance event sender is closed!");
+                                    break
+                                }
+                            }
+                        },
+                        _ = shutdown_handle.recv() => {
+                            info!("Gracefully stopping rebalance listener!");   
+                            break;
+                        },
+                    }
+                }
+            })
+        }).expect("Failed to start rebalance listener thread");
+    }
+}
diff --git a/src/connectors/kafka/sink.rs b/src/connectors/kafka/sink.rs
new file mode 100644
index 000000000..447955686
--- /dev/null
+++ b/src/connectors/kafka/sink.rs
@@ -0,0 +1,94 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+use crate::connectors::common::build_runtime;
+use crate::connectors::common::processor::Processor;
+use crate::connectors::kafka::consumer::KafkaStreams;
+use crate::connectors::kafka::processor::StreamWorker;
+use crate::connectors::kafka::ConsumerRecord;
+use anyhow::Result;
+use futures_util::StreamExt;
+use rdkafka::consumer::Consumer;
+use std::sync::Arc;
+use tokio::runtime::Runtime;
+use tracing::{error, info};
+
+pub struct KafkaSinkConnector<P>
+where
+    P: Processor<Vec<ConsumerRecord>, ()>,
+{
+    streams: KafkaStreams,
+    stream_processor: Arc<StreamWorker<P>>,
+    runtime: Runtime,
+}
+
+impl<P> KafkaSinkConnector<P>
+where
+    P: Processor<Vec<ConsumerRecord>, ()> + Send + Sync + 'static,
+{
+    pub fn new(kafka_streams: KafkaStreams, processor: P) -> Self {
+        let consumer = kafka_streams.consumer();
+        let stream_processor = Arc::new(StreamWorker::new(
+            Arc::new(processor),
+            Arc::clone(&consumer),
+        ));
+
+        let runtime = build_runtime(
+            consumer.context().config.partition_listener_concurrency,
+            "kafka-sink-worker",
+        )
+        .expect("Failed to build runtime");
+        let _ = runtime.enter();
+
+        Self {
+            streams: kafka_streams,
+            stream_processor,
+            runtime,
+        }
+    }
+
+    pub async fn run(self) -> Result<()> {
+        self.streams
+            .partitioned()
+            .map(|partition_stream| {
+                let worker = Arc::clone(&self.stream_processor);
+                let tp = partition_stream.topic_partition().clone();
+                self.runtime.spawn(async move {
+                    partition_stream
+                        .run_drain(|partition_records| async {
+                            info!("Starting task for partition: {:?}", tp);
+
+                            worker
+                                .process_partition(tp.clone(), partition_records)
+                                .await
+                                .unwrap();
+                        })
+                        .await;
+
+                    info!("Task completed for partition: {:?}", tp);
+                })
+            })
+            .for_each_concurrent(None, |task| async {
+                if let Err(e) = task.await {
+                    error!("Task failed: {:?}", e);
+                }
+            })
+            .await;
+
+        Ok(())
+    }
+}
diff --git a/src/connectors/kafka/state.rs b/src/connectors/kafka/state.rs
new file mode 100644
index 000000000..7b20ae762
--- /dev/null
+++ b/src/connectors/kafka/state.rs
@@ -0,0 +1,68 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::kafka::partition_stream::PartitionStreamSender;
+use crate::connectors::kafka::{TopicPartition, TopicPartitionList};
+use std::collections::HashMap;
+use tracing::info;
+
+pub struct StreamState {
+    partition_senders: HashMap<TopicPartition, PartitionStreamSender>,
+}
+
+impl StreamState {
+    pub fn new(capacity: usize) -> Self {
+        Self {
+            partition_senders: HashMap::with_capacity(capacity),
+        }
+    }
+
+    pub fn insert_partition_sender(
+        &mut self,
+        tp: TopicPartition,
+        sender: PartitionStreamSender,
+    ) -> Option<PartitionStreamSender> {
+        self.partition_senders.insert(tp, sender)
+    }
+
+    pub fn get_partition_sender(&self, tp: &TopicPartition) -> Option<&PartitionStreamSender> {
+        self.partition_senders.get(tp)
+    }
+
+    pub async fn terminate_partition_streams(&mut self, tpl: TopicPartitionList) {
+        info!("Terminating streams: {:?}", tpl);
+
+        for tp in tpl.tpl {
+            if let Some(sender) = self.partition_senders.remove(&tp) {
+                info!("Terminating stream for {:?}", tp);
+                sender.terminate();
+                drop(sender);
+                info!("Stream terminated for {:?}", tp);
+            } else {
+                info!("Stream already completed for {:?}", tp);
+            }
+        }
+
+        info!("All streams terminated!");
+    }
+
+    pub fn clear(&mut self) {
+        info!("Clearing all stream states...");
+        self.partition_senders.clear();
+    }
+}
diff --git a/src/connectors/mod.rs b/src/connectors/mod.rs
new file mode 100644
index 000000000..757c49a97
--- /dev/null
+++ b/src/connectors/mod.rs
@@ -0,0 +1,100 @@
+/*
+ * Parseable Server (C) 2022 - 2024 Parseable, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use crate::connectors::common::processor::Processor;
+use crate::connectors::common::shutdown::Shutdown;
+use crate::connectors::kafka::config::KafkaConfig;
+use crate::connectors::kafka::consumer::KafkaStreams;
+use crate::connectors::kafka::metrics::KafkaMetricsCollector;
+use crate::connectors::kafka::processor::ParseableSinkProcessor;
+use crate::connectors::kafka::rebalance_listener::RebalanceListener;
+use crate::connectors::kafka::sink::KafkaSinkConnector;
+use crate::connectors::kafka::state::StreamState;
+use crate::connectors::kafka::{ConsumerRecord, KafkaContext};
+use crate::option::{Mode, CONFIG};
+use actix_web_prometheus::PrometheusMetrics;
+use prometheus::Registry;
+use std::sync::Arc;
+use tokio::sync::RwLock;
+use tracing::{info, warn};
+
+pub mod common;
+pub mod kafka;
+
+pub async fn init(prometheus: &PrometheusMetrics) -> anyhow::Result<()> {
+    if matches!(CONFIG.options.mode, Mode::Ingest | Mode::All) {
+        match CONFIG.kafka_config.validate() {
+            Err(e) => {
+                warn!("Kafka connector configuration invalid. {}", e);
+            }
+            Ok(_) => {
+                let config = CONFIG.kafka_config.clone();
+                let shutdown_handle = Shutdown::default();
+                let registry = prometheus.registry.clone();
+                let processor = ParseableSinkProcessor;
+
+                tokio::spawn({
+                    let shutdown_handle = shutdown_handle.clone();
+                    async move {
+                        shutdown_handle.signal_listener().await;
+                        info!("Connector received shutdown signal!");
+                    }
+                });
+
+                run_kafka2parseable(config, registry, processor, shutdown_handle).await?;
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn run_kafka2parseable<P>(
+    config: KafkaConfig,
+    registry: Registry,
+    processor: P,
+    shutdown_handle: Shutdown,
+) -> anyhow::Result<()>
+where
+    P: Processor<Vec<ConsumerRecord>, ()> + Send + Sync + 'static,
+{
+    info!("Initializing KafkaSink connector...");
+
+    let kafka_config = Arc::new(config.clone());
+    let (kafka_context, rebalance_rx) = KafkaContext::new(kafka_config);
+
+    //TODO: fetch topics metadata from kafka then give dynamic value to StreamState
+    let stream_state = Arc::new(RwLock::new(StreamState::new(60)));
+    let rebalance_listener = RebalanceListener::new(
+        rebalance_rx,
+        Arc::clone(&stream_state),
+        shutdown_handle.clone(),
+    );
+
+    let kafka_streams = KafkaStreams::init(kafka_context, stream_state, shutdown_handle.clone())?;
+
+    let stats = kafka_streams.statistics();
+    registry.register(Box::new(KafkaMetricsCollector::new(stats)?))?;
+
+    let kafka_parseable_sink_connector = KafkaSinkConnector::new(kafka_streams, processor);
+
+    rebalance_listener.start();
+    kafka_parseable_sink_connector.run().await?;
+
+    Ok(())
+}
diff --git a/src/handlers/http/modal/ingest_server.rs b/src/handlers/http/modal/ingest_server.rs
index bafb58080..4cdfb6428 100644
--- a/src/handlers/http/modal/ingest_server.rs
+++ b/src/handlers/http/modal/ingest_server.rs
@@ -31,7 +31,6 @@ use crate::handlers::http::logstream;
 use crate::handlers::http::middleware::DisAllowRootUser;
 use crate::handlers::http::middleware::RouteExt;
 use crate::handlers::http::role;
-use crate::metrics;
 use crate::migration;
 use crate::migration::metadata_migration::migrate_ingester_metadata;
 use crate::rbac::role::Action;
@@ -44,9 +43,10 @@ use crate::sync;
 use crate::utils::get_ingestor_id;
 use crate::utils::get_url;
 use crate::{handlers::http::base_path, option::CONFIG};
+
 use actix_web::web;
-use actix_web::web::resource;
 use actix_web::Scope;
+use actix_web_prometheus::PrometheusMetrics;
 use async_trait::async_trait;
 use base64::Engine;
 use bytes::Bytes;
@@ -177,8 +177,8 @@ impl ParseableServer for IngestServer {
         // parseable can't use local storage for persistence when running a distributed setup
         if CONFIG.get_storage_mode_string() == "Local drive" {
             return Err(anyhow::Error::msg(
-                 "This instance of the Parseable server has been configured to run in a distributed setup, it doesn't support local storage.",
-             ));
+                "This instance of the Parseable server has been configured to run in a distributed setup, it doesn't support local storage.",
+            ));
         }
 
         // check for querier state. Is it there, or was it there in the past
@@ -190,9 +190,12 @@ impl ParseableServer for IngestServer {
     }
 
     /// configure the server and start an instance to ingest data
-    async fn init(&self, shutdown_rx: oneshot::Receiver<()>) -> anyhow::Result<()> {
-        let prometheus = metrics::build_metrics_handler();
-        CONFIG.storage().register_store_metrics(&prometheus);
+    async fn init(
+        &self,
+        prometheus: &PrometheusMetrics,
+        shutdown_rx: oneshot::Receiver<()>,
+    ) -> anyhow::Result<()> {
+        CONFIG.storage().register_store_metrics(prometheus);
 
         migration::run_migration(&CONFIG).await?;
 
@@ -207,7 +210,7 @@ impl ParseableServer for IngestServer {
         set_ingestor_metadata().await?;
 
         // Ingestors shouldn't have to deal with OpenId auth flow
-        let app = self.start(shutdown_rx, prometheus, None);
+        let app = self.start(shutdown_rx, prometheus.clone(), None);
 
         tokio::pin!(app);
         loop {
@@ -237,7 +240,7 @@ impl ParseableServer for IngestServer {
                     (remote_sync_handler, remote_sync_outbox, remote_sync_inbox) = sync::object_store_sync().await;
                 }
 
-            };
+            }
         }
     }
 }
@@ -258,21 +261,21 @@ impl IngestServer {
     pub fn get_user_role_webscope() -> Scope {
         web::scope("/role")
             // GET Role List
-            .service(resource("").route(web::get().to(role::list).authorize(Action::ListRole)))
+            .service(web::resource("").route(web::get().to(role::list).authorize(Action::ListRole)))
             .service(
                 // PUT and GET Default Role
-                resource("/default")
+                web::resource("/default")
                     .route(web::put().to(role::put_default).authorize(Action::PutRole))
                     .route(web::get().to(role::get_default).authorize(Action::GetRole)),
             )
             .service(
                 // PUT, GET, DELETE Roles
-                resource("/{name}")
+                web::resource("/{name}")
                     .route(web::delete().to(role::delete).authorize(Action::DeleteRole))
                     .route(web::get().to(role::get).authorize(Action::GetRole)),
             )
             .service(
-                resource("/{name}/sync")
+                web::resource("/{name}/sync")
                     .route(web::put().to(ingestor_role::put).authorize(Action::PutRole)),
             )
     }
diff --git a/src/handlers/http/modal/mod.rs b/src/handlers/http/modal/mod.rs
index d36372b2b..f791d16b2 100644
--- a/src/handlers/http/modal/mod.rs
+++ b/src/handlers/http/modal/mod.rs
@@ -68,7 +68,11 @@ pub trait ParseableServer {
     async fn load_metadata(&self) -> anyhow::Result<Option<Bytes>>;
 
     /// code that describes starting and setup procedures for each type of server
-    async fn init(&self, shutdown_rx: oneshot::Receiver<()>) -> anyhow::Result<()>;
+    async fn init(
+        &self,
+        prometheus: &PrometheusMetrics,
+        shutdown_rx: oneshot::Receiver<()>,
+    ) -> anyhow::Result<()>;
 
     /// configure the server
     async fn start(
diff --git a/src/handlers/http/modal/query_server.rs b/src/handlers/http/modal/query_server.rs
index 4c77e226c..83b4206dd 100644
--- a/src/handlers/http/modal/query_server.rs
+++ b/src/handlers/http/modal/query_server.rs
@@ -30,9 +30,10 @@ use crate::rbac::role::Action;
 use crate::sync;
 use crate::users::dashboards::DASHBOARDS;
 use crate::users::filters::FILTERS;
-use crate::{analytics, metrics, migration, storage};
+use crate::{analytics, migration, storage};
 use actix_web::web::{resource, ServiceConfig};
 use actix_web::{web, Scope};
+use actix_web_prometheus::PrometheusMetrics;
 use async_trait::async_trait;
 use bytes::Bytes;
 use tokio::sync::oneshot;
@@ -89,9 +90,12 @@ impl ParseableServer for QueryServer {
     }
 
     /// initialize the server, run migrations as needed and start an instance
-    async fn init(&self, shutdown_rx: oneshot::Receiver<()>) -> anyhow::Result<()> {
-        let prometheus = metrics::build_metrics_handler();
-        CONFIG.storage().register_store_metrics(&prometheus);
+    async fn init(
+        &self,
+        prometheus: &PrometheusMetrics,
+        shutdown_rx: oneshot::Receiver<()>,
+    ) -> anyhow::Result<()> {
+        CONFIG.storage().register_store_metrics(prometheus);
 
         migration::run_migration(&CONFIG).await?;
 
@@ -135,7 +139,7 @@ impl ParseableServer for QueryServer {
             sync::object_store_sync().await;
 
         tokio::spawn(airplane::server());
-        let app = self.start(shutdown_rx, prometheus, CONFIG.options.openid());
+        let app = self.start(shutdown_rx, prometheus.clone(), CONFIG.options.openid());
 
         tokio::pin!(app);
         loop {
diff --git a/src/handlers/http/modal/server.rs b/src/handlers/http/modal/server.rs
index c7ee3963f..0b66a743a 100644
--- a/src/handlers/http/modal/server.rs
+++ b/src/handlers/http/modal/server.rs
@@ -39,6 +39,7 @@ use actix_web::web;
 use actix_web::web::resource;
 use actix_web::Resource;
 use actix_web::Scope;
+use actix_web_prometheus::PrometheusMetrics;
 use actix_web_static_files::ResourceFiles;
 use async_trait::async_trait;
 use bytes::Bytes;
@@ -99,9 +100,12 @@ impl ParseableServer for Server {
     }
 
     // configure the server and start an instance of the single server setup
-    async fn init(&self, shutdown_rx: oneshot::Receiver<()>) -> anyhow::Result<()> {
-        let prometheus = metrics::build_metrics_handler();
-        CONFIG.storage().register_store_metrics(&prometheus);
+    async fn init(
+        &self,
+        prometheus: &PrometheusMetrics,
+        shutdown_rx: oneshot::Receiver<()>,
+    ) -> anyhow::Result<()> {
+        CONFIG.storage().register_store_metrics(prometheus);
 
         migration::run_migration(&CONFIG).await?;
 
@@ -138,7 +142,7 @@ impl ParseableServer for Server {
         tokio::spawn(handlers::livetail::server());
         tokio::spawn(handlers::airplane::server());
 
-        let app = self.start(shutdown_rx, prometheus, CONFIG.options.openid());
+        let app = self.start(shutdown_rx, prometheus.clone(), CONFIG.options.openid());
 
         tokio::pin!(app);
 
diff --git a/src/kafka.rs b/src/kafka.rs
deleted file mode 100644
index 423d227ee..000000000
--- a/src/kafka.rs
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Parseable Server (C) 2022 - 2024 Parseable, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-use arrow_schema::Field;
-use chrono::Utc;
-use futures_util::StreamExt;
-use itertools::Itertools;
-use rdkafka::config::ClientConfig;
-use rdkafka::consumer::stream_consumer::StreamConsumer;
-use rdkafka::consumer::Consumer;
-use rdkafka::error::{KafkaError as NativeKafkaError, RDKafkaError};
-use rdkafka::message::BorrowedMessage;
-use rdkafka::util::Timeout;
-use rdkafka::{Message, TopicPartitionList};
-use serde::{Deserialize, Serialize};
-use std::num::ParseIntError;
-use std::sync::Arc;
-use std::{collections::HashMap, fmt::Debug};
-use tracing::{debug, error, info, warn};
-
-use crate::audit::AuditLogBuilder;
-use crate::event::format::LogSource;
-use crate::option::CONFIG;
-use crate::{
-    event::{
-        self,
-        error::EventError,
-        format::{self, EventFormat},
-    },
-    handlers::http::ingest::{create_stream_if_not_exists, PostError},
-    metadata::{error::stream_info::MetadataError, STREAM_INFO},
-    storage::StreamType,
-};
-
-#[derive(Debug, Deserialize, Serialize, Clone, Copy)]
-pub enum SslProtocol {
-    Plaintext,
-    Ssl,
-    SaslPlaintext,
-    SaslSsl,
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum KafkaError {
-    #[error("Please set env var {0} (To use Kafka integration env vars P_KAFKA_TOPICS, P_KAFKA_HOST, and P_KAFKA_GROUP are mandatory)")]
-    NoVarError(&'static str),
-
-    #[error("Kafka error {0}")]
-    NativeError(#[from] NativeKafkaError),
-    #[error("RDKafka error {0}")]
-    RDKError(#[from] RDKafkaError),
-
-    #[error("Error parsing int {1} for environment variable {0}")]
-    ParseIntError(&'static str, ParseIntError),
-    #[error("Error parsing duration int {1} for environment variable {0}")]
-    ParseDurationError(&'static str, ParseIntError),
-
-    #[error("Stream not found: #{0}")]
-    StreamNotFound(String),
-    #[error("Post error: #{0}")]
-    PostError(#[from] PostError),
-    #[error("Metadata error: #{0}")]
-    MetadataError(#[from] MetadataError),
-    #[error("Event error: #{0}")]
-    EventError(#[from] EventError),
-    #[error("JSON error: #{0}")]
-    JsonError(#[from] serde_json::Error),
-    #[error("Invalid group offset storage: #{0}")]
-    InvalidGroupOffsetStorage(String),
-
-    #[error("Invalid SSL protocol: #{0}")]
-    InvalidSslProtocolError(String),
-    #[error("Invalid unicode for environment variable {0}")]
-    EnvNotUnicode(&'static str),
-    #[error("")]
-    DoNotPrintError,
-}
-
-fn setup_consumer() -> Result<(StreamConsumer, Vec<String>), KafkaError> {
-    if let Some(topics) = &CONFIG.options.kafka_topics {
-        // topics can be a comma separated list of topics to subscribe to
-        let topics = topics.split(',').map(|v| v.to_owned()).collect_vec();
-
-        let host = if CONFIG.options.kafka_host.is_some() {
-            CONFIG.options.kafka_host.as_ref()
-        } else {
-            return Err(KafkaError::NoVarError("P_KAKFA_HOST"));
-        };
-
-        let group = if CONFIG.options.kafka_group.is_some() {
-            CONFIG.options.kafka_group.as_ref()
-        } else {
-            return Err(KafkaError::NoVarError("P_KAKFA_GROUP"));
-        };
-
-        let mut conf = ClientConfig::new();
-        conf.set("bootstrap.servers", host.unwrap());
-        conf.set("group.id", group.unwrap());
-
-        if let Some(val) = CONFIG.options.kafka_client_id.as_ref() {
-            conf.set("client.id", val);
-        }
-
-        if let Some(ssl_protocol) = CONFIG.options.kafka_security_protocol.as_ref() {
-            conf.set("security.protocol", serde_json::to_string(&ssl_protocol)?);
-        }
-
-        let consumer: StreamConsumer = conf.create()?;
-        consumer.subscribe(&topics.iter().map(|v| v.as_str()).collect_vec())?;
-
-        if let Some(vals_raw) = CONFIG.options.kafka_partitions.as_ref() {
-            // partitions is a comma separated pairs of topic:partitions
-            let mut topic_partition_pairs = Vec::new();
-            let mut set = true;
-            for vals in vals_raw.split(',') {
-                let intermediate = vals.split(':').collect_vec();
-                if intermediate.len() != 2 {
-                    warn!(
-                        "Value for P_KAFKA_PARTITIONS is incorrect! Skipping setting partitions!"
-                    );
-                    set = false;
-                    break;
-                }
-                topic_partition_pairs.push(intermediate);
-            }
-
-            if set {
-                let mut parts = TopicPartitionList::new();
-                for pair in topic_partition_pairs {
-                    let topic = pair[0];
-                    match pair[1].parse::<i32>() {
-                        Ok(partition) => {
-                            parts.add_partition(topic, partition);
-                        }
-                        Err(_) => warn!("Skipping setting partition for topic- {topic}"),
-                    }
-                }
-                consumer.seek_partitions(parts, Timeout::Never)?;
-            }
-        }
-        Ok((consumer, topics.clone()))
-    } else {
-        // if the user hasn't even set KAFKA_TOPICS
-        // then they probably don't want to use the integration
-        // send back the DoNotPrint error
-        Err(KafkaError::DoNotPrintError)
-    }
-}
-
-fn resolve_schema(stream_name: &str) -> Result<HashMap<String, Arc<Field>>, KafkaError> {
-    let hash_map = STREAM_INFO.read().unwrap();
-    let raw = hash_map
-        .get(stream_name)
-        .ok_or_else(|| KafkaError::StreamNotFound(stream_name.to_owned()))?;
-    Ok(raw.schema.clone())
-}
-
-async fn ingest_message(msg: BorrowedMessage<'_>) -> Result<(), KafkaError> {
-    let Some(payload) = msg.payload() else {
-        debug!("No payload received");
-        return Ok(());
-    };
-
-    let msg = msg.detach();
-    let stream_name = msg.topic();
-
-    // stream should get created only if there is an incoming event, not before that
-    create_stream_if_not_exists(stream_name, StreamType::UserDefined, LogSource::default()).await?;
-
-    let schema = resolve_schema(stream_name)?;
-    let event = format::json::Event {
-        data: serde_json::from_slice(payload)?,
-    };
-
-    let time_partition = STREAM_INFO.get_time_partition(stream_name)?;
-    let static_schema_flag = STREAM_INFO.get_static_schema_flag(stream_name)?;
-    let schema_version = STREAM_INFO.get_schema_version(stream_name)?;
-
-    let (rb, is_first) = event
-        .into_recordbatch(
-            &schema,
-            static_schema_flag,
-            time_partition.as_ref(),
-            schema_version,
-        )
-        .map_err(|err| KafkaError::PostError(PostError::CustomError(err.to_string())))?;
-
-    event::Event {
-        rb,
-        stream_name: stream_name.to_string(),
-        origin_format: "json",
-        origin_size: payload.len() as u64,
-        is_first_event: is_first,
-        parsed_timestamp: Utc::now().naive_utc(),
-        time_partition: None,
-        custom_partition_values: HashMap::new(),
-        stream_type: StreamType::UserDefined,
-    }
-    .process()
-    .await?;
-
-    Ok(())
-}
-
-pub async fn setup_integration() {
-    let (consumer, stream_names) = match setup_consumer() {
-        Ok(c) => c,
-        Err(err) => {
-            match err {
-                KafkaError::DoNotPrintError => {
-                    debug!("P_KAFKA_TOPICS not set, skipping kafka integration");
-                }
-                _ => {
-                    error!("{err}");
-                }
-            }
-            return;
-        }
-    };
-
-    info!("Setup kafka integration for {stream_names:?}");
-    let mut stream = consumer.stream();
-
-    while let Ok(curr) = stream.next().await.unwrap() {
-        // TODO: maybe we should not constructs an audit log for each kafka message, but do so at the batch level
-        let log_builder = AuditLogBuilder::default()
-            .with_host(CONFIG.options.kafka_host.as_deref().unwrap_or(""))
-            .with_user_agent("Kafka Client")
-            .with_protocol("Kafka")
-            .with_stream(curr.topic());
-
-        let Err(err) = ingest_message(curr).await else {
-            log_builder.with_status(200).send().await;
-            continue;
-        };
-        error!("Unable to ingest incoming kafka message- {err}");
-
-        log_builder.with_status(500).with_error(err).send().await;
-    }
-}
diff --git a/src/lib.rs b/src/lib.rs
index 51b0d45ef..2a20b2bb0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -23,15 +23,12 @@ pub mod audit;
 pub mod banner;
 mod catalog;
 mod cli;
+#[cfg(feature = "kafka")]
+pub mod connectors;
 pub mod correlation;
 mod event;
 pub mod handlers;
 pub mod hottier;
-#[cfg(any(
-    all(target_os = "linux", target_arch = "x86_64"),
-    all(target_os = "macos", target_arch = "aarch64")
-))]
-pub mod kafka;
 mod livetail;
 mod metadata;
 pub mod metrics;
diff --git a/src/main.rs b/src/main.rs
index 069306951..82776ed43 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -15,29 +15,24 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
-
+#[cfg(feature = "kafka")]
+use parseable::connectors;
 use parseable::{
-    banner,
+    banner, metrics,
     option::{Mode, CONFIG},
     rbac, storage, IngestServer, ParseableServer, QueryServer, Server,
 };
 use tokio::signal::ctrl_c;
 use tokio::sync::oneshot;
-use tracing::info;
-use tracing_subscriber::EnvFilter;
-
-#[cfg(any(
-    all(target_os = "linux", target_arch = "x86_64"),
-    all(target_os = "macos", target_arch = "aarch64")
-))]
-use parseable::kafka;
+use tracing::level_filters::LevelFilter;
+use tracing::{info, warn};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{fmt, EnvFilter, Registry};
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt()
-        .with_env_filter(EnvFilter::from_default_env())
-        .compact()
-        .init();
+    init_logger(LevelFilter::DEBUG);
 
     // these are empty ptrs so mem footprint should be minimal
     let server: Box<dyn ParseableServer> = match CONFIG.options.mode {
@@ -55,30 +50,53 @@ async fn main() -> anyhow::Result<()> {
     // keep metadata info in mem
     metadata.set_global();
 
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    // load kafka server
-    if CONFIG.options.mode != Mode::Query {
-        tokio::task::spawn(kafka::setup_integration());
-    }
-
     // Spawn a task to trigger graceful shutdown on appropriate signal
     let (shutdown_trigger, shutdown_rx) = oneshot::channel::<()>();
     tokio::spawn(async move {
         block_until_shutdown_signal().await;
 
         // Trigger graceful shutdown
-        println!("Received shutdown signal, notifying server to shut down...");
+        warn!("Received shutdown signal, notifying server to shut down...");
         shutdown_trigger.send(()).unwrap();
     });
 
-    server.init(shutdown_rx).await?;
+    let prometheus = metrics::build_metrics_handler();
+    // Start servers
+    #[cfg(feature = "kafka")]
+    {
+        let parseable_server = server.init(&prometheus, shutdown_rx);
+        let connectors = connectors::init(&prometheus);
+
+        tokio::try_join!(parseable_server, connectors)?;
+    }
+
+    #[cfg(not(feature = "kafka"))]
+    {
+        let parseable_server = server.init(&prometheus, shutdown_rx);
+        parseable_server.await?;
+    }
 
     Ok(())
 }
 
+pub fn init_logger(default_level: LevelFilter) {
+    let filter_layer = EnvFilter::try_from_default_env()
+        .unwrap_or_else(|_| EnvFilter::new(default_level.to_string()));
+
+    let fmt_layer = fmt::layer()
+        .with_thread_names(true)
+        .with_thread_ids(true)
+        .with_line_number(true)
+        .with_timer(tracing_subscriber::fmt::time::UtcTime::rfc_3339())
+        .with_target(true)
+        .compact();
+
+    Registry::default()
+        .with(filter_layer)
+        .with(fmt_layer)
+        .init();
+}
+
 #[cfg(windows)]
 /// Asynchronously blocks until a shutdown signal is received
 pub async fn block_until_shutdown_signal() {
diff --git a/src/metadata.rs b/src/metadata.rs
index 66fae17a3..e98d7e52a 100644
--- a/src/metadata.rs
+++ b/src/metadata.rs
@@ -163,6 +163,21 @@ impl StreamInfo {
         Ok(Arc::new(schema))
     }
 
+    #[allow(dead_code)]
+    pub fn schema_raw(
+        &self,
+        stream_name: &str,
+    ) -> Result<HashMap<String, Arc<Field>>, MetadataError> {
+        let map = self.read().expect(LOCK_EXPECT);
+
+        let schema = map
+            .get(stream_name)
+            .ok_or(MetadataError::StreamMetaNotFound(stream_name.to_string()))
+            .map(|metadata| metadata.schema.clone())?;
+
+        Ok(schema)
+    }
+
     pub fn set_retention(
         &self,
         stream_name: &str,
diff --git a/src/option.rs b/src/option.rs
index c6f185397..11aea57a4 100644
--- a/src/option.rs
+++ b/src/option.rs
@@ -17,6 +17,8 @@
  */
 
 use crate::cli::{Cli, Options, StorageOptions, DEFAULT_PASSWORD, DEFAULT_USERNAME};
+#[cfg(feature = "kafka")]
+use crate::connectors::kafka::config::KafkaConfig;
 use crate::storage::object_storage::parseable_json_path;
 use crate::storage::{ObjectStorageError, ObjectStorageProvider};
 use bytes::Bytes;
@@ -37,6 +39,8 @@ pub struct Config {
     pub options: Options,
     storage: Arc<dyn ObjectStorageProvider>,
     pub storage_name: &'static str,
+    #[cfg(feature = "kafka")]
+    pub kafka_config: KafkaConfig,
 }
 
 impl Config {
@@ -63,17 +67,23 @@ impl Config {
                     options: args.options,
                     storage: Arc::new(args.storage),
                     storage_name: "drive",
+                    #[cfg(feature = "kafka")]
+                    kafka_config: args.kafka,
                 }
             }
             StorageOptions::S3(args) => Config {
                 options: args.options,
                 storage: Arc::new(args.storage),
                 storage_name: "s3",
+                #[cfg(feature = "kafka")]
+                kafka_config: args.kafka,
             },
             StorageOptions::Blob(args) => Config {
                 options: args.options,
                 storage: Arc::new(args.storage),
                 storage_name: "blob_store",
+                #[cfg(feature = "kafka")]
+                kafka_config: args.kafka,
             },
         }
     }
@@ -198,12 +208,6 @@ pub mod validation {
 
     use path_clean::PathClean;
 
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    use crate::kafka::SslProtocol;
-
     use super::{Compression, Mode};
 
     pub fn file_path(s: &str) -> Result<PathBuf, String> {
@@ -248,20 +252,6 @@ pub mod validation {
         url::Url::parse(s).map_err(|_| "Invalid URL provided".to_string())
     }
 
-    #[cfg(any(
-        all(target_os = "linux", target_arch = "x86_64"),
-        all(target_os = "macos", target_arch = "aarch64")
-    ))]
-    pub fn kafka_security_protocol(s: &str) -> Result<SslProtocol, String> {
-        match s {
-            "plaintext" => Ok(SslProtocol::Plaintext),
-            "ssl" => Ok(SslProtocol::Ssl),
-            "sasl_plaintext" => Ok(SslProtocol::SaslPlaintext),
-            "sasl_ssl" => Ok(SslProtocol::SaslSsl),
-            _ => Err("Invalid Kafka Security Protocol provided".to_string()),
-        }
-    }
-
     pub fn mode(s: &str) -> Result<Mode, String> {
         match s {
             "query" => Ok(Mode::Query),

From 5fd5b0de4594253496aae8149d396d61ebcc57d3 Mon Sep 17 00:00:00 2001
From: Devdutt Shenoi <devdutt@parseable.com>
Date: Wed, 5 Feb 2025 07:32:19 +0530
Subject: [PATCH 2/4] fix: build push kafka edge (#1159)

Signed-off-by: Devdutt Shenoi <devdutt@parseable.com>
---
 .github/workflows/build-push-edge-kafka.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build-push-edge-kafka.yaml b/.github/workflows/build-push-edge-kafka.yaml
index 1131d2fa4..afa866d0b 100644
--- a/.github/workflows/build-push-edge-kafka.yaml
+++ b/.github/workflows/build-push-edge-kafka.yaml
@@ -19,6 +19,8 @@ jobs:
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v8.1.5
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

From 1d43901de008e7e93a1d697d1e044568f3ff1252 Mon Sep 17 00:00:00 2001
From: Devdutt Shenoi <devdutt@parseable.com>
Date: Wed, 5 Feb 2025 17:08:32 +0530
Subject: [PATCH 3/4] log: default to warn level in release builds (#1160)

---
 src/main.rs | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 82776ed43..492121490 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -24,7 +24,7 @@ use parseable::{
 };
 use tokio::signal::ctrl_c;
 use tokio::sync::oneshot;
-use tracing::level_filters::LevelFilter;
+use tracing::Level;
 use tracing::{info, warn};
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
@@ -32,7 +32,7 @@ use tracing_subscriber::{fmt, EnvFilter, Registry};
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
-    init_logger(LevelFilter::DEBUG);
+    init_logger();
 
     // these are empty ptrs so mem footprint should be minimal
     let server: Box<dyn ParseableServer> = match CONFIG.options.mode {
@@ -79,9 +79,15 @@ async fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
-pub fn init_logger(default_level: LevelFilter) {
-    let filter_layer = EnvFilter::try_from_default_env()
-        .unwrap_or_else(|_| EnvFilter::new(default_level.to_string()));
+pub fn init_logger() {
+    let filter_layer = EnvFilter::try_from_default_env().unwrap_or_else(|_| {
+        let default_level = if cfg!(debug_assertions) {
+            Level::DEBUG
+        } else {
+            Level::WARN
+        };
+        EnvFilter::new(default_level.to_string())
+    });
 
     let fmt_layer = fmt::layer()
         .with_thread_names(true)

From 1e795773577007996288ab1ee6c48089ce5a9055 Mon Sep 17 00:00:00 2001
From: Arpit Saxena <arpit.saxena2000@yahoo.in>
Date: Wed, 5 Feb 2025 20:52:06 +0530
Subject: [PATCH 4/4] fix: quick start links to docker image and executable
 binary (#1162)

Signed-off-by: Arpit Saxena <arpit.saxena2000@yahoo.in>
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7a40adf78..86bbe23a1 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ To experience Parseable UI, checkout [demo.parseable.com ↗︎](https://demo.pa
 ### Run Parseable
 
 <details>
-<summary><a href="https://www.parseable.com/docs/docker-quick-start">Docker Image</a></summary>
+<summary><a href="https://www.parseable.com/docs/server/get-started/docker-quick-start">Docker Image</a></summary>
 <p>
 
 Get started with Parseable Docker with a single command:
@@ -41,7 +41,7 @@ docker run -p 8000:8000 \
 </details>
 
 <details>
-<summary><a href="https://www.parseable.com/docs/docker-quick-start">Executable Binary</a></summary>
+<summary><a href="https://www.parseable.com/docs/server/get-started/binary-quick-start">Executable Binary</a></summary>
 <p>
 
 Download and run the Parseable binary on your laptop: