libtommath: Fix possible integer overflow CVE-2023-36328
[heimdal.git] / lib / hcrypto / libtommath / bn_mp_to_sbin.c
blobdbaf53e1f9875bdf354425dc367e0c8796188ab5
1 #include "tommath_private.h"
2 #ifdef BN_MP_TO_SBIN_C
3 /* LibTomMath, multiple-precision integer library -- Tom St Denis */
4 /* SPDX-License-Identifier: Unlicense */
6 /* store in signed [big endian] format */
7 mp_err mp_to_sbin(const mp_int *a, unsigned char *buf, size_t maxlen, size_t *written)
9 mp_err err;
10 if (maxlen == 0u) {
11 return MP_BUF;
13 if ((err = mp_to_ubin(a, buf + 1, maxlen - 1u, written)) != MP_OKAY) {
14 return err;
16 if (written != NULL) {
17 (*written)++;
19 buf[0] = (a->sign == MP_ZPOS) ? (unsigned char)0 : (unsigned char)1;
20 return MP_OKAY;
22 #endif