libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / libtommath / bn_mp_sqr.c
blobe0d0a73e448326b99e79f146247ea39de2285be3
1 #include "tommath_private.h"
2 #ifdef BN_MP_SQR_C
3 /* LibTomMath, multiple-precision integer library -- Tom St Denis */
4 /* SPDX-License-Identifier: Unlicense */
6 /* computes b = a*a */
7 mp_err mp_sqr(const mp_int *a, mp_int *b)
9 mp_err err;
10 if (MP_HAS(S_MP_TOOM_SQR) && /* use Toom-Cook? */
11 (a->used >= MP_TOOM_SQR_CUTOFF)) {
12 err = s_mp_toom_sqr(a, b);
13 } else if (MP_HAS(S_MP_KARATSUBA_SQR) && /* Karatsuba? */
14 (a->used >= MP_KARATSUBA_SQR_CUTOFF)) {
15 err = s_mp_karatsuba_sqr(a, b);
16 } else if (MP_HAS(S_MP_SQR_FAST) && /* can we use the fast comba multiplier? */
17 (((a->used * 2) + 1) < MP_WARRAY) &&
18 (a->used < (MP_MAXFAST / 2))) {
19 err = s_mp_sqr_fast(a, b);
20 } else if (MP_HAS(S_MP_SQR)) {
21 err = s_mp_sqr(a, b);
22 } else {
23 err = MP_VAL;
25 b->sign = MP_ZPOS;
26 return err;
28 #endif