chore(tfhe): add multi bit shortint and integer tests

- default tests do not run multi bit PBS as it's not yet deterministic - only radix parallel currently use multi bit pbs in integer - remove determinism checks for some unchecked ops - 4_4 multi bit parameters are disabled for now as they seem to introduce too much noise
ratik · Jun 2, 2023 · d1cb55b · d1cb55b
1 parent 2b9a49d
commit d1cb55b
Show file tree

Hide file tree

Showing 9 changed files with 940 additions and 390 deletions.
diff --git a/.github/workflows/aws_tfhe_multi_bit_tests.yml b/.github/workflows/aws_tfhe_multi_bit_tests.yml
@@ -1,4 +1,4 @@
-name: AWS Tests on CPU
+name: AWS Multi Bit Tests on CPU
 
 env:
   CARGO_TERM_COLOR: always
@@ -48,3 +48,43 @@ jobs:
           echo "Request ID: ${{ inputs.request_id }}"
           echo "Fork repo: ${{ inputs.fork_repo }}"
           echo "Fork git sha: ${{ inputs.fork_git_sha }}"
+
+      - name: Checkout tfhe-rs
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
+        with:
+          repository: ${{ inputs.fork_repo }}
+          ref: ${{ inputs.fork_git_sha }}
+
+      - name: Set up home
+        run: |
+          echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
+
+      - name: Install latest stable
+        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af
+        with:
+          toolchain: stable
+          default: true
+
+      - name: Gen Keys if required
+        run: |
+          MULTI_BIT_ONLY=TRUE make gen_key_cache
+
+      - name: Run shortint multi-bit tests
+        run: |
+          make test_shortint_multi_bit_ci
+
+      - name: Run integer multi-bit tests
+        run: |
+          make test_integer_multi_bit_ci
+
+      - name: Slack Notification
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
+        env:
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
+          SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
+          SLACK_MESSAGE: "Shortint tests finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
+          SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
diff --git a/.github/workflows/trigger_aws_tests_on_pr.yml b/.github/workflows/trigger_aws_tests_on_pr.yml
@@ -16,3 +16,4 @@ jobs:
           message: |
             @slab-ci cpu_test
             @slab-ci cpu_integer_test
+            @slab-ci cpu_multi_bit_test
diff --git a/Makefile b/Makefile
@@ -142,9 +142,13 @@ clippy_fast: clippy clippy_all_targets clippy_c_api clippy_js_wasm_api clippy_ta
 
 .PHONY: gen_key_cache # Run the script to generate keys and cache them for shortint tests
 gen_key_cache: install_rs_build_toolchain
+	if [[ "$${MULTI_BIT_ONLY}" == TRUE ]]; then \
+		multi_bit_flag="--multi-bit-only"; \
+	fi && \
 	RUSTFLAGS="$(RUSTFLAGS)" cargo $(CARGO_RS_BUILD_TOOLCHAIN) run --profile $(CARGO_PROFILE) \
 		--example generates_test_keys \
-		--features=$(TARGET_ARCH_FEATURE),shortint,internal-keycache -p tfhe
+		--features=$(TARGET_ARCH_FEATURE),shortint,internal-keycache -p tfhe -- \
+		$${multi_bit_flag:+"$${multi_bit_flag}"}
 
 .PHONY: build_core # Build core_crypto without experimental features
 build_core: install_rs_build_toolchain install_rs_check_toolchain
@@ -234,7 +238,12 @@ test_c_api: build_c_api
 .PHONY: test_shortint_ci # Run the tests for shortint ci
 test_shortint_ci: install_rs_build_toolchain install_cargo_nextest
 	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
-		./scripts/shortint-tests.sh $(CARGO_RS_BUILD_TOOLCHAIN)
+		./scripts/shortint-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN)
+
+.PHONY: test_shortint_multi_bit_ci # Run the tests for shortint ci running only multibit tests
+test_shortint_multi_bit_ci: install_rs_build_toolchain install_cargo_nextest
+	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
+		./scripts/shortint-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN) --multi-bit
 
 .PHONY: test_shortint # Run all the tests for shortint
 test_shortint: install_rs_build_toolchain
@@ -244,7 +253,12 @@ test_shortint: install_rs_build_toolchain
 .PHONY: test_integer_ci # Run the tests for integer ci
 test_integer_ci: install_rs_build_toolchain install_cargo_nextest
 	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
-		./scripts/integer-tests.sh $(CARGO_RS_BUILD_TOOLCHAIN)
+		./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN)
+
+.PHONY: test_integer_multi_bit_ci # Run the tests for integer ci running only multibit tests
+test_integer_multi_bit_ci: install_rs_build_toolchain install_cargo_nextest
+	BIG_TESTS_INSTANCE="$(BIG_TESTS_INSTANCE)" \
+		./scripts/integer-tests.sh --rust-toolchain $(CARGO_RS_BUILD_TOOLCHAIN) --multi-bit
 
 .PHONY: test_integer # Run all the tests for integer
 test_integer: install_rs_build_toolchain

diff --git a/ci/slab.toml b/ci/slab.toml
@@ -18,6 +18,11 @@ workflow = "aws_tfhe_integer_tests.yml"
 profile = "cpu-big"
 check_run_name = "CPU Integer AWS Tests"
 
+[command.cpu_multi_bit_test]
+workflow = "aws_tfhe_multi_bit_tests.yml"
+profile = "cpu-big"
+check_run_name = "CPU AWS Multi Bit Tests"
+
 [command.integer_bench]
 workflow = "integer_benchmark.yml"
 profile = "bench"

diff --git a/scripts/integer-tests.sh b/scripts/integer-tests.sh
@@ -2,6 +2,49 @@
 
 set -e
 
+function usage() {
+    echo "$0: shortint test runner"
+    echo
+    echo "--help                    Print this message"
+    echo "--rust-toolchain          The toolchain to run the tests with default: stable"
+    echo "--multi-bit               Run multi-bit tests only: default off"
+    echo
+}
+
+RUST_TOOLCHAIN="+stable"
+multi_bit=""
+not_multi_bit="_multi_bit"
+
+while [ -n "$1" ]
+do
+   case "$1" in
+        "--help" | "-h" )
+            usage
+            exit 0
+            ;;
+
+        "--rust-toolchain" )
+            shift
+            RUST_TOOLCHAIN="$1"
+            ;;
+
+        "--multi-bit" )
+            multi_bit="_multi_bit"
+            not_multi_bit=""
+            ;;
+
+        *)
+            echo "Unknown param : $1"
+            exit 1
+            ;;
+   esac
+   shift
+done
+
+if [[ "${RUST_TOOLCHAIN::1}" != "+" ]]; then
+    RUST_TOOLCHAIN="+${RUST_TOOLCHAIN}"
+fi
+
 CURR_DIR="$(dirname "$0")"
 ARCH_FEATURE="$("${CURR_DIR}/get_arch_feature.sh")"
 
@@ -29,14 +72,15 @@ if [[ "${BIG_TESTS_INSTANCE}" != TRUE ]]; then
     # mul_crt_4_4 is extremely flaky (~80% failure)
     # test_wopbs_bivariate_crt_wopbs_param_message generate tables that are too big at the moment
     # test_integer_smart_mul_param_message_4_carry_4 is too slow
-    filter_expression=''\
-'test(/^integer::.*$/)'\
-'and not test(/.*_block_pbs(_base)?_param_message_[34]_carry_[34]$/)'\
-'and not test(~mul_crt_param_message_4_carry_4)'\
-'and not test(/.*test_wopbs_bivariate_crt_wopbs_param_message_[34]_carry_[34]$/)'\
-'and not test(/.*test_integer_smart_mul_param_message_4_carry_4$/)'
-
-    cargo ${1:+"${1}"} nextest run \
+    filter_expression="""\
+test(/^integer::.*${multi_bit}/) \
+${not_multi_bit:+"and not test(~${not_multi_bit})"} \
+and not test(/.*_block_pbs(_base)?_param_message_[34]_carry_[34]$/) \
+and not test(~mul_crt_param_message_4_carry_4) \
+and not test(/.*test_wopbs_bivariate_crt_wopbs_param_message_[34]_carry_[34]$/) \
+and not test(/.*test_integer_smart_mul_param_message_4_carry_4$/)"""
+
+    cargo "${RUST_TOOLCHAIN}" nextest run \
         --tests \
         --release \
         --package tfhe \
@@ -45,27 +89,30 @@ if [[ "${BIG_TESTS_INSTANCE}" != TRUE ]]; then
         --test-threads "${n_threads}" \
         -E "$filter_expression"
 
-    cargo ${1:+"${1}"} test \
-        --release \
-        --package tfhe \
-        --features="${ARCH_FEATURE}",integer,internal-keycache \
-        --doc \
-        integer::
+    if [[ "${multi_bit}" == "" ]]; then
+        cargo "${RUST_TOOLCHAIN}" test \
+            --release \
+            --package tfhe \
+            --features="${ARCH_FEATURE}",integer,internal-keycache \
+            --doc \
+            integer::
+    fi
 else
     # block pbs are too slow for high params
     # mul_crt_4_4 is extremely flaky (~80% failure)
     # test_wopbs_bivariate_crt_wopbs_param_message generate tables that are too big at the moment
     # test_integer_smart_mul_param_message_4_carry_4 is too slow
-    filter_expression=''\
-'test(/^integer::.*$/)'\
-'and not test(/.*_block_pbs(_base)?_param_message_[34]_carry_[34]$/)'\
-'and not test(~mul_crt_param_message_4_carry_4)'\
-'and not test(/.*test_wopbs_bivariate_crt_wopbs_param_message_[34]_carry_[34]$/)'\
-'and not test(/.*test_integer_smart_mul_param_message_4_carry_4$/)'
+    filter_expression="""\
+test(/^integer::.*${multi_bit}/) \
+${not_multi_bit:+"and not test(~${not_multi_bit})"} \
+and not test(/.*_block_pbs(_base)?_param_message_[34]_carry_[34]$/) \
+and not test(~mul_crt_param_message_4_carry_4) \
+and not test(/.*test_wopbs_bivariate_crt_wopbs_param_message_[34]_carry_[34]$/) \
+and not test(/.*test_integer_smart_mul_param_message_4_carry_4$/)"""
 
     num_cpu_threads="$(${nproc_bin})"
     num_threads=$((num_cpu_threads * 2 / 3))
-    cargo ${1:+"${1}"} nextest run \
+    cargo "${RUST_TOOLCHAIN}" nextest run \
         --tests \
         --release \
         --package tfhe \
@@ -74,12 +121,14 @@ else
         --test-threads $num_threads \
         -E "$filter_expression"
 
-    cargo ${1:+"${1}"} test \
-        --release \
-        --package tfhe \
-        --features="${ARCH_FEATURE}",integer,internal-keycache \
-        --doc \
-        integer:: -- --test-threads="$(${nproc_bin})"
+    if [[ "${multi_bit}" == "" ]]; then
+        cargo "${RUST_TOOLCHAIN}" test \
+            --release \
+            --package tfhe \
+            --features="${ARCH_FEATURE}",integer,internal-keycache \
+            --doc \
+            integer:: -- --test-threads="$(${nproc_bin})"
+    fi
 fi
 
 echo "Test ran in $SECONDS seconds"