Skip to content

Commit

Permalink
chore(ci): update scripts and Makefile for future forward compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
IceTDrinker committed Jan 31, 2024
1 parent 76665ab commit f9669c3
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 17 deletions.
30 changes: 20 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ FAST_TESTS?=FALSE
FAST_BENCH?=FALSE
BENCH_OP_FLAVOR?=DEFAULT
NODE_VERSION=20
FORWARD_COMPAT?=OFF
# sed: -n, do not print input stream, -e means a script/expression
# 1,/version/ indicates from the first line, to the line matching version at the start of the line
# p indicates to print, so we keep only the start of the Cargo.toml until we hit the first version
Expand Down Expand Up @@ -49,6 +50,12 @@ else
COVERAGE_ONLY=
endif

ifeq ($(FORWARD_COMPAT),ON)
FORWARD_COMPAT_FEATURE=forward_compatibility
else
FORWARD_COMPAT_FEATURE=
endif

# Variables used only for regex_engine example
REGEX_STRING?=''
REGEX_PATTERN?=''
Expand Down Expand Up @@ -286,15 +293,16 @@ symlink_c_libs_without_fingerprint:
.PHONY: build_c_api # Build the C API for boolean, shortint and integer
build_c_api: install_rs_check_toolchain
RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) build --profile $(CARGO_PROFILE) \
--features=$(TARGET_ARCH_FEATURE),boolean-c-api,shortint-c-api,high-level-c-api \
--features=$(TARGET_ARCH_FEATURE),boolean-c-api,shortint-c-api,high-level-c-api,$(FORWARD_COMPAT_FEATURE) \
-p $(TFHE_SPEC)
@"$(MAKE)" symlink_c_libs_without_fingerprint

.PHONY: build_c_api_experimental_deterministic_fft # Build the C API for boolean, shortint and integer with experimental deterministic FFT
build_c_api_experimental_deterministic_fft: install_rs_check_toolchain
RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_CHECK_TOOLCHAIN) build --profile $(CARGO_PROFILE) \
--features=$(TARGET_ARCH_FEATURE),boolean-c-api,shortint-c-api,high-level-c-api,experimental-force_fft_algo_dif4 \
--features=$(TARGET_ARCH_FEATURE),boolean-c-api,shortint-c-api,high-level-c-api,experimental-force_fft_algo_dif4,$(FORWARD_COMPAT_FEATURE) \
-p $(TFHE_SPEC)
@"$(MAKE)" symlink_c_libs_without_fingerprint

.PHONY: build_web_js_api # Build the js API targeting the web browser
build_web_js_api: install_rs_build_toolchain install_wasm_pack
Expand Down Expand Up @@ -397,14 +405,14 @@ test_shortint_ci: install_rs_build_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/shortint-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)"
--cargo-profile "$(CARGO_PROFILE)" --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_shortint_multi_bit_ci # Run the tests for shortint ci running only multibit tests
test_shortint_multi_bit_ci: install_rs_build_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/shortint-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --multi-bit
--cargo-profile "$(CARGO_PROFILE)" --multi-bit --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_shortint # Run all the tests for shortint
test_shortint: install_rs_build_toolchain
Expand All @@ -424,46 +432,48 @@ test_integer_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)"
--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \
--tfhe-package "$(TFHE_SPEC)"

.PHONY: test_unsigned_integer_ci # Run the tests for unsigned integer ci
test_unsigned_integer_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \
--unsigned-only
--unsigned-only --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_signed_integer_ci # Run the tests for signed integer ci
test_signed_integer_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --avx512-support "$(AVX512_SUPPORT)" \
--signed-only
--signed-only --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_integer_multi_bit_ci # Run the tests for integer ci running only multibit tests
test_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)"
--cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \
--tfhe-package "$(TFHE_SPEC)"

.PHONY: test_unsigned_integer_multi_bit_ci # Run the tests for nsigned integer ci running only multibit tests
test_unsigned_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \
--unsigned-only
--unsigned-only --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_signed_integer_multi_bit_ci # Run the tests for nsigned integer ci running only multibit tests
test_signed_integer_multi_bit_ci: install_rs_check_toolchain install_cargo_nextest
BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
FAST_TESTS="$(FAST_TESTS)" \
./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_CHECK_TOOLCHAIN) \
--cargo-profile "$(CARGO_PROFILE)" --multi-bit --avx512-support "$(AVX512_SUPPORT)" \
--signed-only
--signed-only --tfhe-package "$(TFHE_SPEC)"

.PHONY: test_safe_deserialization # Run the tests for safe deserialization
test_safe_deserialization: install_rs_build_toolchain install_cargo_nextest
Expand Down
11 changes: 9 additions & 2 deletions scripts/integer-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ function usage() {
echo "--signed-only Run only signed integer tests, by default both signed and unsigned tests are run"
echo "--cargo-profile The cargo profile used to build tests"
echo "--avx512-support Set to ON to enable avx512"
echo "--tfhe-package The package spec like [email protected], default=tfhe"
echo
}

Expand All @@ -23,6 +24,7 @@ signed=""
not_signed=""
cargo_profile="release"
avx512_feature=""
tfhe_package="tfhe"

while [ -n "$1" ]
do
Expand Down Expand Up @@ -64,6 +66,11 @@ do
fi
;;

"--tfhe-package" )
shift
tfhe_package="$1"
;;

*)
echo "Unknown param : $1"
exit 1
Expand Down Expand Up @@ -153,7 +160,7 @@ fi
cargo "${RUST_TOOLCHAIN}" nextest run \
--tests \
--cargo-profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--profile ci \
--features="${ARCH_FEATURE}",integer,internal-keycache,"${avx512_feature}" \
--test-threads "${test_threads}" \
Expand All @@ -162,7 +169,7 @@ cargo "${RUST_TOOLCHAIN}" nextest run \
if [[ "${multi_bit}" == "" ]]; then
cargo "${RUST_TOOLCHAIN}" test \
--profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--features="${ARCH_FEATURE}",integer,internal-keycache,"${avx512_feature}" \
--doc \
-- --test-threads="${doctest_threads}" integer::
Expand Down
17 changes: 12 additions & 5 deletions scripts/shortint-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ function usage() {
echo "--rust-toolchain The toolchain to run the tests with default: stable"
echo "--multi-bit Run multi-bit tests only: default off"
echo "--cargo-profile The cargo profile used to build tests"
echo "--tfhe-package The package spec like [email protected], default=tfhe"
echo
}

RUST_TOOLCHAIN="+stable"
multi_bit=""
cargo_profile="release"
tfhe_package="tfhe"

while [ -n "$1" ]
do
Expand All @@ -38,6 +40,11 @@ do
cargo_profile="$1"
;;

"--tfhe-package" )
shift
tfhe_package="$1"
;;

*)
echo "Unknown param : $1"
exit 1
Expand Down Expand Up @@ -111,7 +118,7 @@ and not test(~smart_add_and_mul)""" # This test is too slow
cargo "${RUST_TOOLCHAIN}" nextest run \
--tests \
--cargo-profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--profile ci \
--features="${ARCH_FEATURE}",shortint,internal-keycache \
--test-threads "${n_threads_small}" \
Expand All @@ -128,7 +135,7 @@ and not test(~smart_add_and_mul)"""
cargo "${RUST_TOOLCHAIN}" nextest run \
--tests \
--cargo-profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--profile ci \
--features="${ARCH_FEATURE}",shortint,internal-keycache \
--test-threads "${n_threads_big}" \
Expand All @@ -137,7 +144,7 @@ and not test(~smart_add_and_mul)"""
if [[ "${multi_bit}" == "" ]]; then
cargo "${RUST_TOOLCHAIN}" test \
--profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--features="${ARCH_FEATURE}",shortint,internal-keycache \
--doc \
-- shortint::
Expand Down Expand Up @@ -177,7 +184,7 @@ and not test(~smart_add_and_mul)""" # This test is too slow
cargo "${RUST_TOOLCHAIN}" nextest run \
--tests \
--cargo-profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--profile ci \
--features="${ARCH_FEATURE}",shortint,internal-keycache \
--test-threads "$(${nproc_bin})" \
Expand All @@ -186,7 +193,7 @@ and not test(~smart_add_and_mul)""" # This test is too slow
if [[ "${multi_bit}" == "" ]]; then
cargo "${RUST_TOOLCHAIN}" test \
--profile "${cargo_profile}" \
--package tfhe \
--package "${tfhe_package}" \
--features="${ARCH_FEATURE}",shortint,internal-keycache \
--doc \
-- --test-threads="$(${nproc_bin})" shortint::
Expand Down

0 comments on commit f9669c3

Please sign in to comment.