Skip to content

Commit

Permalink
Pull update
Browse files Browse the repository at this point in the history 
Signed-off-by: Basil Hess <[email protected]>
  • Loading branch information
@bhess
bhess committed Jan 21, 2025
1 parent 274d30c commit 959c697
Show file tree
Hide file tree
Showing 304 changed files with 14,028 additions and 11,358 deletions.
18 changes: 9 additions & 9 deletions src/kem/ml_kem/CMakeLists.txt
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions src/kem/ml_kem/mlkem-native_ml-kem-1024_aarch64/aarch64/src/arith_native_aarch64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,14 @@ void poly_tobytes_asm_clean(uint8_t *r, const int16_t *a);
void poly_tobytes_asm_opt(uint8_t *r, const int16_t *a);

#define polyvec_basemul_acc_montgomery_cached_asm_clean \
MLKEM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_asm_clean)
MLKEM_NAMESPACE_K(polyvec_basemul_acc_montgomery_cached_asm_clean)
void polyvec_basemul_acc_montgomery_cached_asm_clean(int16_t *r,
const int16_t *a,
const int16_t *b,
const int16_t *b_cache);

#define polyvec_basemul_acc_montgomery_cached_asm_opt \
MLKEM_NAMESPACE(polyvec_basemul_acc_montgomery_cached_asm_opt)
MLKEM_NAMESPACE_K(polyvec_basemul_acc_montgomery_cached_asm_opt)
void polyvec_basemul_acc_montgomery_cached_asm_opt(int16_t *r, const int16_t *a,
const int16_t *b,
const int16_t *b_cache);
Expand Down
1 change: 0 additions & 1 deletion src/kem/ml_kem/mlkem-native_ml-kem-1024_aarch64/aarch64/src/clean_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ static INLINE void ntt_native(poly *data)
aarch64_ntt_zetas_layer56);
}

#define INVNTT_BOUND_NATIVE (8 * MLKEM_Q)
static INLINE void intt_native(poly *data)
{
intt_asm_clean(data->coeffs, aarch64_invntt_zetas_layer01234,
Expand Down
85 changes: 55 additions & 30 deletions src/kem/ml_kem/mlkem-native_ml-kem-1024_aarch64/aarch64/src/intt_clean.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@

inp .req x3
count .req x4
xtmp .req x5
wtmp .req w5

data0 .req v8
data1 .req v9
Expand Down Expand Up @@ -193,40 +193,20 @@
t3 .req v28

ninv .req v29
q_ninv .req q29
ninv_tw .req v30
q_ninv_tw .req q30

/* Literal pool */
.macro dup8h c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.endm

.p2align 4
c_consts: .short 3329
.short 20159
.short 0
.short 0
.short 0
.short 0
.short 0
.short 0
c_ninv: dup8h 512
c_ninv_tw: dup8h 5040

MLKEM_ASM_NAMESPACE(intt_asm_clean):
push_stack

ldr q_consts, c_consts
ldr q_ninv, c_ninv
ldr q_ninv_tw, c_ninv_tw
// Setup constants
mov wtmp, #3329
mov consts.h[0], wtmp
mov wtmp, #20159
mov consts.h[1], wtmp
mov wtmp, #512
dup ninv.8h, wtmp
mov wtmp, #5040
dup ninv_tw.8h, wtmp

mov inp, in
mov count, #8
Expand Down Expand Up @@ -361,4 +341,49 @@ layer012_start:
pop_stack
ret

/****************** REGISTER DEALLOCATIONS *******************/
.unreq in
.unreq r01234_ptr
.unreq r56_ptr
.unreq inp
.unreq count
.unreq wtmp
.unreq data0
.unreq data1
.unreq data2
.unreq data3
.unreq data4
.unreq data5
.unreq data6
.unreq data7
.unreq q_data0
.unreq q_data1
.unreq q_data2
.unreq q_data3
.unreq q_data4
.unreq q_data5
.unreq q_data6
.unreq q_data7
.unreq root0
.unreq root1
.unreq root2
.unreq root0_tw
.unreq root1_tw
.unreq root2_tw
.unreq consts
.unreq q_consts
.unreq q_root0
.unreq q_root1
.unreq q_root2
.unreq q_root0_tw
.unreq q_root1_tw
.unreq q_root2_tw
.unreq tmp
.unreq t0
.unreq t1
.unreq t2
.unreq t3
.unreq ninv
.unreq ninv_tw

#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */
85 changes: 55 additions & 30 deletions src/kem/ml_kem/mlkem-native_ml-kem-1024_aarch64/aarch64/src/intt_opt.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@

inp .req x3
count .req x4
xtmp .req x5
wtmp .req w5

data0 .req v8
data1 .req v9
Expand Down Expand Up @@ -193,40 +193,20 @@
t3 .req v28

ninv .req v29
q_ninv .req q29
ninv_tw .req v30
q_ninv_tw .req q30

/* Literal pool */
.macro dup8h c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.short \c
.endm

.p2align 4
c_consts: .short 3329
.short 20159
.short 0
.short 0
.short 0
.short 0
.short 0
.short 0
c_ninv: dup8h 512
c_ninv_tw: dup8h 5040

MLKEM_ASM_NAMESPACE(intt_asm_opt):
push_stack

ldr q_consts, c_consts
ldr q_ninv, c_ninv
ldr q_ninv_tw, c_ninv_tw
// Setup constants
mov wtmp, #3329
mov consts.h[0], wtmp
mov wtmp, #20159
mov consts.h[1], wtmp
mov wtmp, #512
dup ninv.8h, wtmp
mov wtmp, #5040
dup ninv_tw.8h, wtmp

mov inp, in
mov count, #8
Expand Down Expand Up @@ -1017,4 +997,49 @@ layer012_start:
pop_stack
ret

/****************** REGISTER DEALLOCATIONS *******************/
.unreq in
.unreq r01234_ptr
.unreq r56_ptr
.unreq inp
.unreq count
.unreq wtmp
.unreq data0
.unreq data1
.unreq data2
.unreq data3
.unreq data4
.unreq data5
.unreq data6
.unreq data7
.unreq q_data0
.unreq q_data1
.unreq q_data2
.unreq q_data3
.unreq q_data4
.unreq q_data5
.unreq q_data6
.unreq q_data7
.unreq root0
.unreq root1
.unreq root2
.unreq root0_tw
.unreq root1_tw
.unreq root2_tw
.unreq consts
.unreq q_consts
.unreq q_root0
.unreq q_root1
.unreq q_root2
.unreq q_root0_tw
.unreq q_root1_tw
.unreq q_root2_tw
.unreq tmp
.unreq t0
.unreq t1
.unreq t2
.unreq t3
.unreq ninv
.unreq ninv_tw

#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_OPT */
63 changes: 48 additions & 15 deletions src/kem/ml_kem/mlkem-native_ml-kem-1024_aarch64/aarch64/src/ntt_clean.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@

inp .req x3
count .req x4
xtmp .req x5
wtmp .req w5

data0 .req v8
data1 .req v9
Expand Down Expand Up @@ -156,7 +156,6 @@
q_root2_tw .req q6

consts .req v7
q_consts .req q7

tmp .req v24
t0 .req v25
Expand All @@ -167,21 +166,13 @@
.text
.global MLKEM_ASM_NAMESPACE(ntt_asm_clean)

/* Literal pool */
.p2align 4
c_consts:
.short 3329
.short 20159
.short 0
.short 0
.short 0
.short 0
.short 0
.short 0

MLKEM_ASM_NAMESPACE(ntt_asm_clean):
push_stack
ldr q_consts, c_consts

mov wtmp, #3329
mov consts.h[0], wtmp
mov wtmp, #20159
mov consts.h[1], wtmp

mov inp, in
mov count, #4
Expand Down Expand Up @@ -280,4 +271,46 @@ layer3456_start:
pop_stack
ret

/****************** REGISTER DEALLOCATIONS *******************/
.unreq in
.unreq r01234_ptr
.unreq r56_ptr
.unreq inp
.unreq count
.unreq wtmp
.unreq data0
.unreq data1
.unreq data2
.unreq data3
.unreq data4
.unreq data5
.unreq data6
.unreq data7
.unreq q_data0
.unreq q_data1
.unreq q_data2
.unreq q_data3
.unreq q_data4
.unreq q_data5
.unreq q_data6
.unreq q_data7
.unreq root0
.unreq root1
.unreq root2
.unreq root0_tw
.unreq root1_tw
.unreq root2_tw
.unreq q_root0
.unreq q_root1
.unreq q_root2
.unreq q_root0_tw
.unreq q_root1_tw
.unreq q_root2_tw
.unreq consts
.unreq tmp
.unreq t0
.unreq t1
.unreq t2
.unreq t3

#endif /* MLKEM_NATIVE_ARITH_BACKEND_AARCH64_CLEAN */
Loading

0 comments on commit 959c697

Please sign in to comment.