diff --git a/algebraic/src/arch/x86_64/avx2_field_gl.rs b/algebraic/src/arch/x86_64/avx2_field_gl.rs index 2e959677..1dadce6d 100644 --- a/algebraic/src/arch/x86_64/avx2_field_gl.rs +++ b/algebraic/src/arch/x86_64/avx2_field_gl.rs @@ -39,7 +39,7 @@ impl Avx2GoldilocksField { } #[inline] pub fn reduce(x: __m256i, y: __m256i) -> Avx2GoldilocksField { - Self::new(unsafe {reduce128((x,y))}) + Self::new(unsafe { reduce128((x, y)) }) } } diff --git a/starky/src/arch/x86_64/avx2_poseidon_gl.rs b/starky/src/arch/x86_64/avx2_poseidon_gl.rs index 763f574b..2e4df243 100644 --- a/starky/src/arch/x86_64/avx2_poseidon_gl.rs +++ b/starky/src/arch/x86_64/avx2_poseidon_gl.rs @@ -65,12 +65,21 @@ impl Poseidon { } // #[inline(always)] - // unsafe fn extract_u64s_from_m256i(value: __m256i) -> [u64; 4] { + // unsafe fn _extract_u64s_from_m256i(value: __m256i) -> [u64; 4] { // mem::transmute(value) // } #[inline(always)] - fn pow7_avx2( + fn pow7(x: &mut Avx2GoldilocksField) { + let aux = *x; + *x = x.square(); + *x *= aux; + *x = x.square(); + *x *= aux; + } + + #[inline(always)] + fn pow7_triple( st0: &mut Avx2GoldilocksField, st1: &mut Avx2GoldilocksField, st2: &mut Avx2GoldilocksField, @@ -99,12 +108,24 @@ impl Poseidon { st2: &mut Avx2GoldilocksField, c: Vec, ) { - let c0 = Avx2GoldilocksField::pack_slice(&c[0..4])[0]; - let c1 = Avx2GoldilocksField::pack_slice(&c[4..8])[0]; - let c2 = Avx2GoldilocksField::pack_slice(&c[8..12])[0]; - *st0 = *st0 + c0; - *st1 = *st1 + c1; - *st2 = *st2 + c2; + let c = Avx2GoldilocksField::pack_slice(&c); + *st0 = *st0 + c[0]; + *st1 = *st1 + c[1]; + *st2 = *st2 + c[2]; + } + + #[inline(always)] + fn mult_add_avx( + st0: &mut Avx2GoldilocksField, + st1: &mut Avx2GoldilocksField, + st2: &mut Avx2GoldilocksField, + s0: Avx2GoldilocksField, + s: Vec, + ) { + let s = Avx2GoldilocksField::pack_slice(&s); + *st0 = *st0 + s[0] * s0; + *st1 = *st1 + s[1] * s0; + *st2 = *st2 + s[2] * s0; } #[inline(always)] @@ -175,10 +196,8 @@ impl Poseidon { st2: Avx2GoldilocksField, m: Vec, ) { - let m0 = Avx2GoldilocksField::pack_slice(&m[0..4])[0]; - let m1 = Avx2GoldilocksField::pack_slice(&m[4..8])[0]; - let m2 = Avx2GoldilocksField::pack_slice(&m[8..12])[0]; - *r = (st0 * m0) + (st1 * m1) + (st2 * m2) + let m = Avx2GoldilocksField::pack_slice(&m); + *r = (st0 * m[0]) + (st1 * m[1]) + (st2 * m[2]) } #[inline(always)] @@ -249,18 +268,16 @@ impl Poseidon { st2: Avx2GoldilocksField, m: Vec, ) { - let m0 = Avx2GoldilocksField::pack_slice(&m[0..4])[0]; - let m1 = Avx2GoldilocksField::pack_slice(&m[4..8])[0]; - let m2 = Avx2GoldilocksField::pack_slice(&m[8..12])[0]; + let m = Avx2GoldilocksField::pack_slice(&m); let mut c0_h = Avx2GoldilocksField::ZEROS; let mut c0_l = Avx2GoldilocksField::ZEROS; let mut c1_h = Avx2GoldilocksField::ZEROS; let mut c1_l = Avx2GoldilocksField::ZEROS; let mut c2_h = Avx2GoldilocksField::ZEROS; let mut c2_l = Avx2GoldilocksField::ZEROS; - Self::mult_avx_72(&mut c0_h, &mut c0_l, st0, m0); - Self::mult_avx_72(&mut c1_h, &mut c1_l, st1, m1); - Self::mult_avx_72(&mut c2_h, &mut c2_l, st2, m2); + Self::mult_avx_72(&mut c0_h, &mut c0_l, st0, m[0]); + Self::mult_avx_72(&mut c1_h, &mut c1_l, st1, m[1]); + Self::mult_avx_72(&mut c2_h, &mut c2_l, st2, m[2]); let c_h = c0_h + c1_h + c2_h; let c_l = c0_l + c1_l + c2_l; *r = Avx2GoldilocksField::reduce(c_h.get(), c_l.get()) @@ -302,13 +319,8 @@ impl Poseidon { *c_h = Avx2GoldilocksField::new(_mm256_srli_epi64(r0, 32)); } - pub unsafe fn hash( - &self, - inp: &Vec, - init_state: &[FGL], - out: usize, - ) -> Result, String> { - self.hash_inner(inp, init_state, out) + pub fn hash(&self, inp: &Vec, init_state: &[FGL], out: usize) -> Result, String> { + unsafe { self.hash_inner(inp, init_state, out) } } unsafe fn hash_inner( @@ -336,18 +348,15 @@ impl Poseidon { _state[8..].clone_from_slice(init_state); let state: Vec<_> = _state.iter().map(|x| x.into_repr()).collect(); - - let mut part0 = state[0..4].to_vec(); - let mut part1 = state[4..8].to_vec(); - let mut part2 = state[8..12].to_vec(); - - let mut st0 = Avx2GoldilocksField::pack_slice_mut(&mut part0)[0]; - let mut st1 = Avx2GoldilocksField::pack_slice_mut(&mut part1)[0]; - let mut st2 = Avx2GoldilocksField::pack_slice_mut(&mut part2)[0]; + let mut state_vec = state.to_vec(); + let st = Avx2GoldilocksField::pack_slice_mut(&mut state_vec); + let mut st0 = st[0]; + let mut st1 = st[1]; + let mut st2 = st[2]; Self::add_avx(&mut st0, &mut st1, &mut st2, (&C[0..12]).to_vec()); for r in 0..(n_rounds_f / 2 - 1) { - Self::pow7_avx2(&mut st0, &mut st1, &mut st2); + Self::pow7_triple(&mut st0, &mut st1, &mut st2); Self::add_avx( &mut st0, &mut st1, @@ -356,168 +365,78 @@ impl Poseidon { ); Self::mmult_avx_8(&mut st0, &mut st1, &mut st2, (&M[0..144]).to_vec()); } - Self::pow7_avx2(&mut st0, &mut st1, &mut st2); + Self::pow7_triple(&mut st0, &mut st1, &mut st2); Self::add_avx(&mut st0, &mut st1, &mut st2, (&C[48..60]).to_vec()); Self::mmult_avx(&mut st0, &mut st1, &mut st2, (&P[0..144]).to_vec()); - let state_u64s = (unsafe { Self::extract_u64s_from_m256i(st0.get()) }); - println!("ok! pow7_u64s- {:?}", state_u64s); - - Ok(_state[0..out].to_vec()) - - // let mut tmp_state = vec![FGL::ZERO; t]; - // for r in 0..(n_rounds_f / 2 - 1) { - // state.iter_mut().for_each(Self::pow7); - // - // println!("pow7[{}]: {:?}", r, state); - // state.iter_mut().enumerate().for_each(|(i, a)| { - // a.add_assign(&C[(r + 1) * t + i]); - // }); - - // let sz = state.len(); - // tmp_state.iter_mut().enumerate().for_each(|(i, out)| { - // let mut acc = FGL::ZERO; - // for j in 0..sz { - // let mut tmp = M[j][i]; - // tmp.mul_assign(&state[j]); - // acc.add_assign(&tmp); - // } - // *out = acc; - // }); - // state - // .iter_mut() - // .zip(tmp_state.iter()) - // .for_each(|(out, inp)| { - // *out = *inp; - // }); - // } - // // println!("0- {:?}", state); - // // state.iter_mut().for_each(Self::pow7); - // state.chunks_exact_mut(4).for_each(|chunk| { - // let mut field_chunk = Avx2GoldilocksField([ - // chunk[0].into_repr(), - // chunk[1].into_repr(), - // chunk[2].into_repr(), - // chunk[3].into_repr(), - // ]); - - // Self::pow7_avx2(&mut field_chunk); - - // for (i, field) in field_chunk.0.iter().enumerate() { - // chunk[i] = FGL::from_repr(*field).unwrap(); - // } - // }); - // println!("00- {:?}", state); - // state.iter_mut().enumerate().for_each(|(i, a)| { - // a.add_assign(&C[(n_rounds_f / 2 - 1 + 1) * t + i]); - // }); //opt - // // println!("000- {:?}", state); - // let sz = state.len(); - // tmp_state.iter_mut().enumerate().for_each(|(i, out)| { - // let mut acc = FGL::ZERO; - // for j in 0..sz { - // let mut tmp = P[j][i]; - // tmp.mul_assign(&state[j]); - // acc.add_assign(&tmp); - // } - // *out = acc; - // }); - // // println!("0000- {:?}", state); - // state - // .iter_mut() - // .zip(tmp_state.iter()) - // .for_each(|(out, inp)| { - // *out = *inp; - // }); - // // println!("1- {:?}", state); - - // for r in 0..n_rounds_p { - // Self::pow7(&mut state[0]); - // state[0].add_assign(&C[(n_rounds_f / 2 + 1) * t + r]); - - // let sz = state.len(); - // let mut s0 = FGL::ZERO; - // for j in 0..sz { - // let mut tmp = S[(t * 2 - 1) * r + j]; - // tmp.mul_assign(&state[j]); - // s0.add_assign(&tmp); - // } - - // for k in 1..t { - // let mut tmp = S[(t * 2 - 1) * r + t + k - 1]; - // tmp.mul_assign(&state[0]); - // state[k].add_assign(&tmp); - // } - - // state[0] = s0; - // } - // // println!("2- {:?}", state); - // for r in 0..(n_rounds_f / 2 - 1) { - // // state.iter_mut().for_each(Self::pow7); - // state.chunks_exact_mut(4).for_each(|chunk| { - // let mut field_chunk = Avx2GoldilocksField([ - // chunk[0].into_repr(), - // chunk[1].into_repr(), - // chunk[2].into_repr(), - // chunk[3].into_repr(), - // ]); - - // Self::pow7_avx2(&mut field_chunk); - - // for (i, field) in field_chunk.0.iter().enumerate() { - // chunk[i] = FGL::from_repr(*field).unwrap(); - // } - // }); - // state.iter_mut().enumerate().for_each(|(i, a)| { - // a.add_assign(&C[(n_rounds_f / 2 + 1) * t + n_rounds_p + r * t + i]); - // }); - - // let sz = state.len(); - // tmp_state.iter_mut().enumerate().for_each(|(i, out)| { - // let mut acc = FGL::ZERO; - // for j in 0..sz { - // let mut tmp = M[j][i]; - // tmp.mul_assign(&state[j]); - // acc.add_assign(&tmp); - // } - // *out = acc; - // }); - // state - // .iter_mut() - // .zip(tmp_state.iter()) - // .for_each(|(out, inp)| { - // *out = *inp; - // }); - // } - - // // state.iter_mut().for_each(Self::pow7); - // state.chunks_exact_mut(4).for_each(|chunk| { - // let mut field_chunk = Avx2GoldilocksField([ - // chunk[0].into_repr(), - // chunk[1].into_repr(), - // chunk[2].into_repr(), - // chunk[3].into_repr(), - // ]); - - // Self::pow7_avx2(&mut field_chunk); - - // for (i, field) in field_chunk.0.iter().enumerate() { - // chunk[i] = FGL::from_repr(*field).unwrap(); - // } - // }); - // let sz = state.len(); - // tmp_state.iter_mut().enumerate().for_each(|(i, out)| { - // let mut acc = FGL::ZERO; - // for j in 0..sz { - // let mut tmp = M[j][i]; - // tmp.mul_assign(&state[j]); - // acc.add_assign(&tmp); - // } - // *out = acc; - // }); - // state = tmp_state; - - // Ok(state[0..out].to_vec()) + for r in 0..n_rounds_p { + let st0_slice = st0.as_slice_mut(); + let mut s_arr = { [st0_slice[0], FrRepr([0]), FrRepr([0]), FrRepr([0])] }; + let mut _st0 = Avx2GoldilocksField::from_slice_mut(&mut s_arr); + + Self::pow7(&mut _st0); + let c_arr = { [C[(4 + 1) * 12 + r], FrRepr([0]), FrRepr([0]), FrRepr([0])] }; + let c = Avx2GoldilocksField::from_slice(&c_arr); + *_st0 = *_st0 + *c; + let st0_slice = st0.as_slice_mut(); + st0_slice[0] = _st0.as_slice_mut()[0]; + + let mut tmp = Avx2GoldilocksField::ZEROS; + Self::spmv_avx_4x12( + &mut tmp, + st0, + st1, + st2, + S[12 * 2 * r..(12 * 2 * r + 12)].to_vec(), + ); + let tmp_slice = tmp.as_slice_mut(); + let sum = FGL::from_repr(tmp_slice[0]).unwrap() + + FGL::from_repr(tmp_slice[1]).unwrap() + + FGL::from_repr(tmp_slice[2]).unwrap() + + FGL::from_repr(tmp_slice[3]).unwrap(); + + let tmp_arr = { + [ + _st0.as_slice_mut()[0], + _st0.as_slice_mut()[0], + _st0.as_slice_mut()[0], + _st0.as_slice_mut()[0], + ] + }; + let s0 = Avx2GoldilocksField::from_slice(&tmp_arr); + Self::mult_add_avx( + &mut st0, + &mut st1, + &mut st2, + *s0, + (&S[(12 * (2 * r + 1))..(12 * (2 * r + 2))]).to_vec(), + ); + + let st0_slice = st0.as_slice_mut(); + st0_slice[0] = sum.into_repr(); + } + + for r in 0..(n_rounds_f / 2 - 1) { + Self::pow7_triple(&mut st0, &mut st1, &mut st2); + Self::add_avx( + &mut st0, + &mut st1, + &mut st2, + (&C[((n_rounds_f / 2 + 1) * t + n_rounds_p + r * t) + ..((n_rounds_f / 2 + 1) * t + n_rounds_p + r * t + 12)]) + .to_vec(), + ); + Self::mmult_avx_8(&mut st0, &mut st1, &mut st2, (&M[0..144]).to_vec()); + } + Self::pow7_triple(&mut st0, &mut st1, &mut st2); + Self::mmult_avx(&mut st0, &mut st1, &mut st2, (&M[0..144]).to_vec()); + + let st0_slice = st0.as_slice(); + + let mut result_vec: Vec = Vec::new(); + result_vec.extend(st0_slice.iter().map(|&repr| FGL::from_repr(repr).unwrap())); + + Ok(result_vec) } } @@ -528,64 +447,50 @@ mod tests { use algebraic::packed::PackedField; use plonky::field_gl::Fr as FGL; use plonky::PrimeField; - use rand::Rand; - - // #[test] - // fn test_pow7_avx2() { - // let mut rng = rand::thread_rng(); - // let x = FGL::rand(&mut rng); - // let x7 = x * x * x * x * x * x * x; - // let a_arr = [x.into_repr(), x.into_repr(), x.into_repr(), x.into_repr()]; - // let packed_a = Avx2GoldilocksField::from_slice(&a_arr); - // let mut x = *packed_a; - // Poseidon::pow7_avx2(&mut x); - // let arr_res = x.as_slice(); - // assert_eq!(x7.into_repr(), arr_res[0]); - // } #[test] fn test_poseidon_opt_hash_all_0_avx() { let poseidon = Poseidon::new(); let input = vec![FGL::ZERO; 8]; let state = vec![FGL::ZERO; 4]; - let res = unsafe { poseidon.hash(&input, &state, 4).unwrap() }; + let res = poseidon.hash(&input, &state, 4).unwrap(); let expected = vec![ FGL::from(0x3c18a9786cb0b359u64), FGL::from(0xc4055e3364a246c3u64), FGL::from(0x7953db0ab48808f4u64), FGL::from(0xc71603f33a1144cau64), ]; - // assert_eq!(res, expected); + assert_eq!(res, expected); } - // #[test] - // fn test_poseidon_opt_hash_1_11() { - // let poseidon = Poseidon::new(); - // let input = (0u64..8).map(FGL::from).collect::>(); - // let state = (8u64..12).map(FGL::from).collect::>(); - // let res = poseidon.hash(&input, &state, 4).unwrap(); - // let expected = vec![ - // FGL::from(0xd64e1e3efc5b8e9eu64), - // FGL::from(0x53666633020aaa47u64), - // FGL::from(0xd40285597c6a8825u64), - // FGL::from(0x613a4f81e81231d2u64), - // ]; - // assert_eq!(res, expected); - // } + #[test] + fn test_poseidon_opt_hash_1_11_avx() { + let poseidon = Poseidon::new(); + let input = (0u64..8).map(FGL::from).collect::>(); + let state = (8u64..12).map(FGL::from).collect::>(); + let res = poseidon.hash(&input, &state, 4).unwrap(); + let expected = vec![ + FGL::from(0xd64e1e3efc5b8e9eu64), + FGL::from(0x53666633020aaa47u64), + FGL::from(0xd40285597c6a8825u64), + FGL::from(0x613a4f81e81231d2u64), + ]; + assert_eq!(res, expected); + } - // #[test] - // fn test_poseidon_opt_hash_all_neg_1() { - // let poseidon = Poseidon::new(); - // let init = FGL::ZERO - FGL::ONE; - // let input = vec![init; 8]; - // let state = vec![init; 4]; - // let res = poseidon.hash(&input, &state, 4).unwrap(); - // let expected = vec![ - // FGL::from(0xbe0085cfc57a8357u64), - // FGL::from(0xd95af71847d05c09u64), - // FGL::from(0xcf55a13d33c1c953u64), - // FGL::from(0x95803a74f4530e82u64), - // ]; - // assert_eq!(res, expected); - // } + #[test] + fn test_poseidon_opt_hash_all_neg_1_avx() { + let poseidon = Poseidon::new(); + let init = FGL::ZERO - FGL::ONE; + let input = vec![init; 8]; + let state = vec![init; 4]; + let res = poseidon.hash(&input, &state, 4).unwrap(); + let expected = vec![ + FGL::from(0xbe0085cfc57a8357u64), + FGL::from(0xd95af71847d05c09u64), + FGL::from(0xcf55a13d33c1c953u64), + FGL::from(0x95803a74f4530e82u64), + ]; + assert_eq!(res, expected); + } } diff --git a/starky/src/linearhash.rs b/starky/src/linearhash.rs index be9d9759..26465564 100644 --- a/starky/src/linearhash.rs +++ b/starky/src/linearhash.rs @@ -1,8 +1,8 @@ #![allow(non_snake_case)] -// #[cfg(target_feature = "avx2")] -// use crate::arch::x86_64::avx2_poseidon_gl::Poseidon; +#[cfg(target_feature = "avx2")] +use crate::arch::x86_64::avx2_poseidon_gl::Poseidon; use crate::errors::Result; -// #[cfg(not(target_feature = "avx2"))] +#[cfg(not(target_feature = "avx2"))] use crate::poseidon_opt::Poseidon; use crate::traits::MTNodeType; use crate::ElementDigest; diff --git a/starky/src/merklehash.rs b/starky/src/merklehash.rs index 05bd26b3..12b0f43f 100644 --- a/starky/src/merklehash.rs +++ b/starky/src/merklehash.rs @@ -1,12 +1,12 @@ #![allow(dead_code)] -// #[cfg(target_feature = "avx2")] -// use crate::arch::x86_64::avx2_poseidon_gl::Poseidon; +#[cfg(target_feature = "avx2")] +use crate::arch::x86_64::avx2_poseidon_gl::Poseidon; use crate::constant::{get_max_workers, MAX_OPS_PER_THREAD, MIN_OPS_PER_THREAD}; use crate::digest::ElementDigest; use crate::errors::{EigenError, Result}; use crate::f3g::F3G; use crate::linearhash::LinearHash; -// #[cfg(not(target_feature = "avx2"))] +#[cfg(not(target_feature = "avx2"))] use crate::poseidon_opt::Poseidon; use crate::traits::MTNodeType; use crate::traits::MerkleTree; diff --git a/starky/src/poseidon_constants_avx.rs b/starky/src/poseidon_constants_avx.rs index d50694f3..ae066e99 100644 --- a/starky/src/poseidon_constants_avx.rs +++ b/starky/src/poseidon_constants_avx.rs @@ -290,6 +290,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x463f9ee03d290810, 0xc810936e64982542, 0x43b1c289f7bc3ac, + 0x0, 0x94877900674181c3, 0xc6c67cc37a2a2bbd, 0xd667c2055387940f, @@ -313,6 +314,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x6591b02092d671bb, 0xe18c71963dd1b7, 0x8a21bcd24a14218a, + 0x0, 0xadef3740e71c726, 0xa37bf67c6f986559, 0xc6b16f7ed4fa1b00, @@ -336,6 +338,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x3c667a1d833a3cca, 0xda6f61838efa1ffe, 0xe8f749470bd7c446, + 0x0, 0x481ac7746b159c67, 0xe367de32f108e278, 0x73f260087ad28bec, @@ -359,6 +362,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x8584defff7589bd7, 0x3c5fe4aeb1fb52ba, 0x9e7cd88acf543a5e, + 0x0, 0xb22d2432b72d5098, 0x9e18a487f44d2fe4, 0x4b39e14ce22abd3c, @@ -382,6 +386,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x2c08893f0d1580e2, 0xed3cbcff6fcc5ba, 0xc82f510ecf81f6d0, + 0x0, 0x11ba9a1b81718c2a, 0x9f7d798a3323410c, 0xa821855c8c1cf5e5, @@ -405,6 +410,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x30276f1221ace5fa, 0x7935dd342764a144, 0xeac6db520bb03708, + 0x0, 0x37f4e36af6073c6e, 0x4edc0918210800e9, 0xc44998e99eae4188, @@ -428,6 +434,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x83329c90f04233ce, 0xb5b99e6664a0a3ee, 0x6b0731849e200a7f, + 0x0, 0x577f9a9e7ee3f9c2, 0x88c522b949ace7b1, 0x82f07007c8b72106, @@ -451,6 +458,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xaf947c59af5e4047, 0x4653fb0685084ef2, 0x57fde2062ae35bf, + 0x0, 0xf02a3ac068ee110b, 0xa3630dafb8ae2d7, 0xce0dc874eaf9b55c, @@ -474,6 +482,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x2d3c5f42a39c91a0, 0x811719919351ae8, 0xf669de0add993131, + 0x0, 0xab1cbd41d8c1e335, 0x9322ed4c0bc2df01, 0x51c3c0983d4284e5, @@ -497,6 +506,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xb0dc3ecd724bb076, 0x5e34d8554a6452ba, 0x4f78fd8c1fdcc5f, + 0x0, 0x3d4eab2b8ef5f796, 0xcfff421583896e22, 0x4143cb32d39ac3d9, @@ -520,6 +530,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x9bb6c88de8cd178, 0xdc05b676564f538a, 0x60192d883e473fee, + 0x0, 0x1183dfce7c454afd, 0x21cea4aa3d3ed949, 0xfce6f70303f2304, @@ -543,6 +554,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x77dc8d856c05a44a, 0x87948589e4f243fd, 0x7e5217af969952c2, + 0x0, 0x84d1ecc4d53d2ff1, 0xd8af8b9ceb4e11b6, 0x335856bb527b52f4, @@ -566,6 +578,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xeb9de00d594828e6, 0x88c5f20df9e5c26, 0xf555f4112b19781f, + 0x0, 0x238aa6daa612186d, 0x9137a5c630bad4b4, 0xc7db3817870c5eda, @@ -589,6 +602,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xbe4430134182978b, 0x3771e82493ab262d, 0xa671690d8095ce82, + 0x0, 0xd6e15ffc055e154e, 0xec67881f381a32bf, 0xfbb1196092bf409c, @@ -612,6 +626,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x662b329b01e7bb38, 0x8aa674b36144d9a9, 0xcbabf78f97f95e65, + 0x0, 0xbd87ad390420258, 0xad8617bca9e33c8, 0xc00ad377a1e2666, @@ -635,6 +650,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xf5e6c40f1621c299, 0xcec0e58c34cb64b1, 0xa868ea113387939f, + 0x0, 0xcf29427ff7c58, 0xbd9b3cf49eec8, 0xd1dc8aa81fb26, @@ -658,6 +674,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xf464864ad6f2bb93, 0x2d55e52a5d44414, 0xdd8de62487c40925, + 0x0, 0xe24c99adad8, 0xcf389ed4bc8, 0xe580cbf6966, @@ -681,6 +698,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xe1adf8b84437180, 0xd593a5e584af47b, 0xa023d94c56e151c7, + 0x0, 0xf7157bc98, 0xe3006d948, 0xfa65811e6, @@ -704,6 +722,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0x2fe9d756c9f12d1, 0xe9633210630cbf12, 0x1ffea9fe85a0b0b1, + 0x0, 0x11131738, 0xf56d588, 0x11050f86, @@ -727,6 +746,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xa4e6f0a8c33348a6, 0xc0a26efc7be5669b, 0xa6b6582c547d0d60, + 0x0, 0x11f718, 0x10b6c8, 0x134a96, @@ -750,6 +770,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xac9ea09074e3e150, 0x8f0fa011a2035fb0, 0x1a37905d8450904a, + 0x0, 0x1300, 0x1750, 0x114e, @@ -773,6 +794,7 @@ pub fn constants() -> (Vec, Vec, Vec, Vec) { 0xb69a0fa70aea684a, 0x9584acaa6e062a0, 0xbc051640145b19b, + 0x0, 0x14, 0x22, 0x12,