libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / libtommath / bn_mp_reduce_2k_setup.c
blob2eaf7addf6c7a1d128af2ce863cbb8c5b7954342
1 #include "tommath_private.h"
2 #ifdef BN_MP_REDUCE_2K_SETUP_C
3 /* LibTomMath, multiple-precision integer library -- Tom St Denis */
4 /* SPDX-License-Identifier: Unlicense */
6 /* determines the setup value */
7 mp_err mp_reduce_2k_setup(const mp_int *a, mp_digit *d)
9 mp_err err;
10 mp_int tmp;
11 int p;
13 if ((err = mp_init(&tmp)) != MP_OKAY) {
14 return err;
17 p = mp_count_bits(a);
18 if ((err = mp_2expt(&tmp, p)) != MP_OKAY) {
19 mp_clear(&tmp);
20 return err;
23 if ((err = s_mp_sub(&tmp, a, &tmp)) != MP_OKAY) {
24 mp_clear(&tmp);
25 return err;
28 *d = tmp.dp[0];
29 mp_clear(&tmp);
30 return MP_OKAY;
32 #endif