libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / libtommath / bn_mp_reduce_is_2k.c
bloba9f4f9f2b6a1d7da0b7fe6471e1758a132d246c2
1 #include "tommath_private.h"
2 #ifdef BN_MP_REDUCE_IS_2K_C
3 /* LibTomMath, multiple-precision integer library -- Tom St Denis */
4 /* SPDX-License-Identifier: Unlicense */
6 /* determines if mp_reduce_2k can be used */
7 mp_bool mp_reduce_is_2k(const mp_int *a)
9 int ix, iy, iw;
10 mp_digit iz;
12 if (a->used == 0) {
13 return MP_NO;
14 } else if (a->used == 1) {
15 return MP_YES;
16 } else if (a->used > 1) {
17 iy = mp_count_bits(a);
18 iz = 1;
19 iw = 1;
21 /* Test every bit from the second digit up, must be 1 */
22 for (ix = MP_DIGIT_BIT; ix < iy; ix++) {
23 if ((a->dp[iw] & iz) == 0u) {
24 return MP_NO;
26 iz <<= 1;
27 if (iz > MP_DIGIT_MAX) {
28 ++iw;
29 iz = 1;
32 return MP_YES;
33 } else {
34 return MP_YES;
38 #endif