Merge from main

.github/workflows/audit.yml

@@ -22,7 +22,7 @@ jobs:
       pull-requests: read
       contents: read
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: cargo audit
         run: |

.github/workflows/check.yml

@@ -31,14 +31,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/rm
 
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy,rustfmt
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/bin/
@@ -59,6 +59,10 @@ jobs:
         if: ${{ success() || failure() }}
         run: cargo clippy --tests --features shuttle
 
+      - name: Clippy web
+        if: ${{ success() || failure() }}
+        run: cargo clippy --no-default-features --features "cli web-app real-world-infra test-fixture descriptive-gate"
+
       - name: Build
         if: ${{ success() || failure() }}
         run: cargo build --tests
@@ -78,14 +82,14 @@ jobs:
     env:
       RUSTFLAGS: -C target-cpu=native
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/rm
 
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy,rustfmt
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/bin/
@@ -112,14 +116,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/rm
 
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy,rustfmt
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/bin/
@@ -155,7 +159,8 @@ jobs:
     env:
       TARGET: x86_64-unknown-linux-gnu
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/rm
       - uses: dtolnay/rust-toolchain@nightly
       - name: Add Rust sources
         run: rustup component add rust-src
@@ -167,7 +172,8 @@ jobs:
     env:
       TARGET: x86_64-unknown-linux-gnu
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    - uses: ./.github/actions/rm
     - uses: dtolnay/rust-toolchain@nightly
     - name: Add Miri
       run: rustup component add miri
@@ -184,6 +190,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
+      - uses: ./.github/actions/rm
 
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -197,7 +204,7 @@ jobs:
         run: ./scripts/coverage-ci --lcov --output-path ipa.cov
 
       - name: Report to codecov.io
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ipa.cov

.github/workflows/docker.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: "Checkout"
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: "Setup Docker Buildx"
       uses: docker/setup-buildx-action@v2

Cargo.toml

@@ -6,12 +6,12 @@ members = ["ipa-core", "ipa-macros"]
 incremental = true
 lto = "thin"
 
-[profile.bench]
-debug-assertions = true
+[profile.release-max]
+inherits = "release"
+codegen-units = 1
 
 [profile.bench-dhat]
 inherits = "bench"
-debug-assertions = false
 incremental = true
 lto = "thin"
 debug = 1

ipa-core/Cargo.toml

@@ -18,7 +18,6 @@ default = [
     "ipa-prf",
 ]
 cli = ["comfy-table", "clap"]
-enable-serde = ["serde", "serde_json"]
 disable-metrics = []
 # TODO move web-app to a separate crate. It adds a lot of build time to people who mostly write protocols
 # TODO Consider moving out benches as well
@@ -28,7 +27,6 @@ web-app = [
     "base64",
     "clap",
     "comfy-table",
-    "enable-serde",
     "hyper",
     "hyper-rustls",
     "rcgen",
@@ -40,7 +38,7 @@ web-app = [
     "tower",
     "tower-http",
 ]
-test-fixture = ["enable-serde", "weak-field"]
+test-fixture = ["weak-field"]
 # Include observability instruments that detect lack of progress inside MPC. If there is a bug that leads to helper
 # miscommunication, this feature helps to detect it. Turning it on has some cost.
 # If "shuttle" feature is enabled, turning this on has no effect.
@@ -63,6 +61,10 @@ descriptive-gate = []
 compact-gate = ["ipa-macros/compact-gate"]
 # Enable using more than one thread for protocol execution. Most of the parallelism occurs at parallel/seq_join operations
 multi-threading = ["async-scoped"]
+# Enable tokio task profiling. Requires tokio_unstable flag to be passed to the compiler.
+# RUSTFLAGS="--cfg tokio_unstable" cargo run ... --features="tokio-console ...".
+# Note that if there are other flags enabled on your platform in .cargo/config.toml, you need to include them as well.
+tokio-console = ["console-subscriber", "tokio/tracing"]
 
 # Standalone aggregation protocol. We use IPA infra for communication
 # but it has nothing to do with IPA.
@@ -74,7 +76,7 @@ ipa-prf = []
 ipa-macros = { version = "*", path = "../ipa-macros" }
 
 aes = "0.8.3"
-async-trait = "0.1.68"
+async-trait = "0.1.79"
 async-scoped = { version = "0.9.0", features = ["use-tokio"], optional = true }
 axum = { version = "0.5.17", optional = true, features = ["http2"] }
 axum-server = { version = "0.5.1", optional = true, features = [
@@ -87,7 +89,8 @@ bitvec = "1.0"
 bytes = "1.4"
 clap = { version = "4.3.2", optional = true, features = ["derive"] }
 comfy-table = { version = "7.0", optional = true }
-config = "0.13.2"
+config = "0.14"
+console-subscriber = {  version = "0.2", optional = true }
 criterion = { version = "0.5.1", optional = true, default-features = false, features = [
     "async_tokio",
     "plotters",
@@ -129,8 +132,8 @@ rustls-pemfile = { version = "1", optional = true }
 # we can remove pinning
 rustls-webpki = "^0.101.4"
 # TODO consider using zerocopy or serde_bytes or in-house serialization
-serde = { version = "1.0", optional = true, features = ["derive"] }
-serde_json = { version = "1.0", optional = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
 sha2 = "0.10"
 shuttle-crate = { package = "shuttle", version = "0.6.1", optional = true }
 thiserror = "1.0"
@@ -144,7 +147,7 @@ tower = { version = "0.4.13", optional = true }
 tower-http = { version = "0.4.0", optional = true, features = ["trace"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-typenum = "1.16"
+typenum = { version = "1.17", features = ["i128"] }
 # hpke is pinned to it
 x25519-dalek = "2.0.0-rc.3"
 
@@ -174,7 +177,7 @@ bench = false
 [[bin]]
 name = "ipa_bench"
 path = "src/bin/ipa_bench/ipa_bench.rs"
-required-features = ["cli", "enable-serde"]
+required-features = ["cli"]
 bench = false
 
 [[bin]]
@@ -206,16 +209,15 @@ harness = false
 required-features = ["enable-benches", "descriptive-gate"]
 
 [[bench]]
-name = "oneshot_sort"
-path = "benches/oneshot/sort.rs"
+name = "oneshot_ipa"
+path = "benches/oneshot/ipa.rs"
 harness = false
 required-features = ["enable-benches", "descriptive-gate"]
 
 [[bench]]
-name = "oneshot_ipa"
-path = "benches/oneshot/ipa.rs"
+name = "transpose"
 harness = false
-required-features = ["enable-benches", "descriptive-gate"]
+required-features = ["enable-benches"]
 
 [[test]]
 name = "helper_networks"

ipa-core/benches/ct/arithmetic_circuit.rs

@@ -1,8 +1,52 @@
 use criterion::{
-    black_box, criterion_group, criterion_main, BenchmarkId, Criterion, SamplingMode, Throughput,
+    black_box, criterion_group, criterion_main, measurement::Measurement, BatchSize,
+    BenchmarkGroup, BenchmarkId, Criterion, SamplingMode, Throughput,
 };
-use ipa_core::{ff::Fp31, test_fixture::circuit};
-use tokio::runtime::Builder;
+use ipa_core::{
+    ff::{Field, Fp31, Fp32BitPrime, U128Conversions},
+    protocol::{basics::SecureMul, context::SemiHonestContext},
+    secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, FieldSimd, IntoShares},
+    test_fixture::circuit,
+};
+use rand::distributions::{Distribution, Standard};
+use tokio::runtime::{Builder, Runtime};
+
+fn do_benchmark<M, F, const N: usize>(
+    rt: &Runtime,
+    group: &mut BenchmarkGroup<M>,
+    width: u32,
+    depth: u16,
+    active_work: usize,
+) where
+    M: Measurement,
+    F: Field + FieldSimd<N> + U128Conversions,
+    for<'a> Replicated<F, N>: SecureMul<SemiHonestContext<'a>>,
+    [F; N]: IntoShares<Replicated<F, N>>,
+    Standard: Distribution<F>,
+{
+    group.throughput(Throughput::Elements((width * depth as u32) as u64));
+    group.bench_with_input(
+        BenchmarkId::new(
+            "circuit",
+            format!("{width}:{depth}:{active_work}:{}x{}", F::NAME, N),
+        ),
+        &(width, depth),
+        |b, &(width, depth)| {
+            b.to_async(rt).iter_batched(
+                || circuit::arithmetic_setup(width, depth),
+                |input| {
+                    circuit::arithmetic::<F, N>(
+                        black_box(width),
+                        black_box(depth),
+                        active_work,
+                        input,
+                    )
+                },
+                BatchSize::PerIteration,
+            );
+        },
+    );
+}
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let rt = Builder::new_multi_thread()
@@ -16,18 +60,25 @@ pub fn criterion_benchmark(c: &mut Criterion) {
     group.sample_size(10);
     group.sampling_mode(SamplingMode::Flat);
 
-    for width in [5_000u32, 50_000, 500_000, 1_000_000] {
-        for depth in [1u8, 10, 64] {
-            group.throughput(Throughput::Elements((width * depth as u32) as u64));
-            group.bench_with_input(
-                BenchmarkId::new("circuit", format!("{width}:{depth}")),
-                &(width, depth),
-                |b, &(width, depth)| {
-                    b.to_async(&rt)
-                        .iter(|| circuit::arithmetic::<Fp31>(black_box(width), black_box(depth)));
-                },
-            );
-        }
+    // Note that the width parameter (3rd-to-last argument to do_benchmark) must
+    // be a multiple of the vectorization width.
+
+    #[cfg(not(coverage))]
+    {
+        do_benchmark::<_, Fp31, 1>(&rt, &mut group, 4_096, 64, 1024);
+        do_benchmark::<_, Fp31, 1>(&rt, &mut group, 1_024, 256, 1024);
+
+        do_benchmark::<_, Fp32BitPrime, 1>(&rt, &mut group, 4_096, 64, 1024);
+        do_benchmark::<_, Fp32BitPrime, 1>(&rt, &mut group, 1_024, 256, 1024);
+        do_benchmark::<_, Fp32BitPrime, 32>(&rt, &mut group, 4_096, 64, 32);
+        do_benchmark::<_, Fp32BitPrime, 32>(&rt, &mut group, 1_024, 256, 32);
+    }
+
+    #[cfg(coverage)]
+    {
+        do_benchmark::<_, Fp31, 1>(&rt, &mut group, 256, 64, 32);
+        do_benchmark::<_, Fp32BitPrime, 1>(&rt, &mut group, 256, 64, 32);
+        do_benchmark::<_, Fp32BitPrime, 32>(&rt, &mut group, 256, 64, 32);
     }
 }
 

ipa-core/benches/iai/arithmetic_circuit.rs

@@ -10,10 +10,18 @@ pub fn iai_benchmark() {
         .expect("Creating runtime failed");
 
     const CIRCUIT_WIDTH: u32 = 500_000;
-    const CIRCUIT_DEPTH: u8 = 1;
+    const CIRCUIT_DEPTH: u16 = 1;
 
+    tracing::warn!("test data generation may skew results of this benchmark");
     rt.block_on(async {
-        circuit::arithmetic::<Fp31>(black_box(CIRCUIT_WIDTH), black_box(CIRCUIT_DEPTH)).await;
+        let input = circuit::arithmetic_setup(CIRCUIT_WIDTH, CIRCUIT_DEPTH);
+        circuit::arithmetic::<Fp31, 1>(
+            black_box(CIRCUIT_WIDTH),
+            black_box(CIRCUIT_DEPTH),
+            1024,
+            input,
+        )
+        .await;
     })
 }
 

ipa-core/benches/oneshot/arithmetic_circuit.rs

@@ -14,7 +14,7 @@ pub struct CircuitArgs {
     pub width: u32,
 
     #[arg(short, long, help = "depth of the circuit", default_value_t = 10)]
-    pub depth: u8,
+    pub depth: u16,
 
     /// Cargo passes the bench argument
     /// https://doc.rust-lang.org/cargo/commands/cargo-bench.html
@@ -33,8 +33,9 @@ pub async fn main() {
         println!("benchmark parameters: Field size: {field_size} bits, circuit width: {width}, depth: {depth}");
     }
 
+    let input = circuit::arithmetic_setup(args.width, args.depth);
     let start = Instant::now();
-    circuit::arithmetic::<Fp31>(args.width, args.depth).await;
+    circuit::arithmetic::<Fp31, 1>(args.width, args.depth, 1024, input).await;
     let duration = start.elapsed().as_secs_f32();
 
     println!("benchmark complete after {duration}s");