lib/arctwo.c

   1 /* arctwo.c --- The RC2 cipher as described in RFC 2268.
   2  * Copyright (C) 2003-2006, 2008-2025 Free Software Foundation, Inc.
   3  *
   4  * This file is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU Lesser General Public License as
   6  * published by the Free Software Foundation; either version 2.1 of the
   7  * License, or (at your option) any later version.
   8  *
   9  * This file is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public License
  15  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16  */
  17
  18 /* Code from GnuTLS/Libgcrypt adapted for gnulib by Simon Josefsson. */
  19
  20 /* This implementation was written by Nikos Mavroyanopoulos for GNUTLS
  21  * as a Libgcrypt module (gnutls/lib/x509/rc2.c) and later adapted for
  22  * direct use by Libgcrypt by Werner Koch.  This implementation is
  23  * only useful for pkcs#12 decryption.
  24  *
  25  * The implementation here is based on Peter Gutmann's RRC.2 paper.
  26  */
  27
  28 #include <config.h>
  29
  30 #include "arctwo.h"
  31
  32 #include "bitrotate.h"
  33
  34 static const uint8_t arctwo_sbox[] = {
  35   217, 120, 249, 196, 25, 221, 181, 237,
  36   40, 233, 253, 121, 74, 160, 216, 157,
  37   198, 126, 55, 131, 43, 118, 83, 142,
  38   98, 76, 100, 136, 68, 139, 251, 162,
  39   23, 154, 89, 245, 135, 179, 79, 19,
  40   97, 69, 109, 141, 9, 129, 125, 50,
  41   189, 143, 64, 235, 134, 183, 123, 11,
  42   240, 149, 33, 34, 92, 107, 78, 130,
  43   84, 214, 101, 147, 206, 96, 178, 28,
  44   115, 86, 192, 20, 167, 140, 241, 220,
  45   18, 117, 202, 31, 59, 190, 228, 209,
  46   66, 61, 212, 48, 163, 60, 182, 38,
  47   111, 191, 14, 218, 70, 105, 7, 87,
  48   39, 242, 29, 155, 188, 148, 67, 3,
  49   248, 17, 199, 246, 144, 239, 62, 231,
  50   6, 195, 213, 47, 200, 102, 30, 215,
  51   8, 232, 234, 222, 128, 82, 238, 247,
  52   132, 170, 114, 172, 53, 77, 106, 42,
  53   150, 26, 210, 113, 90, 21, 73, 116,
  54   75, 159, 208, 94, 4, 24, 164, 236,
  55   194, 224, 65, 110, 15, 81, 203, 204,
  56   36, 145, 175, 80, 161, 244, 112, 57,
  57   153, 124, 58, 133, 35, 184, 180, 122,
  58   252, 2, 54, 91, 37, 85, 151, 49,
  59   45, 93, 250, 152, 227, 138, 146, 174,
  60   5, 223, 41, 16, 103, 108, 186, 201,
  61   211, 0, 230, 207, 225, 158, 168, 44,
  62   99, 22, 1, 63, 88, 226, 137, 169,
  63   13, 56, 52, 27, 171, 51, 255, 176,
  64   187, 72, 12, 95, 185, 177, 205, 46,
  65   197, 243, 219, 71, 229, 165, 156, 119,
  66   10, 166, 32, 104, 254, 127, 193, 173
  67 };
  68
  69 /* C89 compliant way to cast 'char' to 'unsigned char'. */
  70 static unsigned char
  71 to_uchar (char ch)
  72 {
  73   return ch;
  74 }
  75
  76 void
  77 arctwo_encrypt (arctwo_context *context, const char *inbuf,
  78                 char *outbuf, size_t length)
  79 {
  80   for (; length >= ARCTWO_BLOCK_SIZE; length -= ARCTWO_BLOCK_SIZE,
  81          inbuf += ARCTWO_BLOCK_SIZE, outbuf += ARCTWO_BLOCK_SIZE)
  82     {
  83       size_t i, j;
  84       uint16_t word0 = 0, word1 = 0, word2 = 0, word3 = 0;
  85
  86       word0 = (word0 << 8) | to_uchar (inbuf[1]);
  87       word0 = (word0 << 8) | to_uchar (inbuf[0]);
  88       word1 = (word1 << 8) | to_uchar (inbuf[3]);
  89       word1 = (word1 << 8) | to_uchar (inbuf[2]);
  90       word2 = (word2 << 8) | to_uchar (inbuf[5]);
  91       word2 = (word2 << 8) | to_uchar (inbuf[4]);
  92       word3 = (word3 << 8) | to_uchar (inbuf[7]);
  93       word3 = (word3 << 8) | to_uchar (inbuf[6]);
  94
  95       for (i = 0; i < 16; i++)
  96         {
  97           j = i * 4;
  98           /* For some reason I cannot combine those steps. */
  99           word0 += (word1 & ~word3) + (word2 & word3) + context->S[j];
 100           word0 = rotl16 (word0, 1);
 101
 102           word1 += (word2 & ~word0) + (word3 & word0) + context->S[j + 1];
 103           word1 = rotl16 (word1, 2);
 104
 105           word2 += (word3 & ~word1) + (word0 & word1) + context->S[j + 2];
 106           word2 = rotl16 (word2, 3);
 107
 108           word3 += (word0 & ~word2) + (word1 & word2) + context->S[j + 3];
 109           word3 = rotl16 (word3, 5);
 110
 111           if (i == 4 || i == 10)
 112             {
 113               word0 += context->S[word3 & 63];
 114               word1 += context->S[word0 & 63];
 115               word2 += context->S[word1 & 63];
 116               word3 += context->S[word2 & 63];
 117             }
 118         }
 119
 120       outbuf[0] = word0 & 255;
 121       outbuf[1] = word0 >> 8;
 122       outbuf[2] = word1 & 255;
 123       outbuf[3] = word1 >> 8;
 124       outbuf[4] = word2 & 255;
 125       outbuf[5] = word2 >> 8;
 126       outbuf[6] = word3 & 255;
 127       outbuf[7] = word3 >> 8;
 128     }
 129 }
 130
 131 void
 132 arctwo_decrypt (arctwo_context *context, const char *inbuf,
 133                 char *outbuf, size_t length)
 134 {
 135   for (; length >= ARCTWO_BLOCK_SIZE; length -= ARCTWO_BLOCK_SIZE,
 136          inbuf += ARCTWO_BLOCK_SIZE, outbuf += ARCTWO_BLOCK_SIZE)
 137     {
 138       size_t i, j;
 139       uint16_t word0 = 0, word1 = 0, word2 = 0, word3 = 0;
 140
 141       word0 = (word0 << 8) | to_uchar (inbuf[1]);
 142       word0 = (word0 << 8) | to_uchar (inbuf[0]);
 143       word1 = (word1 << 8) | to_uchar (inbuf[3]);
 144       word1 = (word1 << 8) | to_uchar (inbuf[2]);
 145       word2 = (word2 << 8) | to_uchar (inbuf[5]);
 146       word2 = (word2 << 8) | to_uchar (inbuf[4]);
 147       word3 = (word3 << 8) | to_uchar (inbuf[7]);
 148       word3 = (word3 << 8) | to_uchar (inbuf[6]);
 149
 150       for (i = 16; i > 0; i--)
 151         {
 152           j = (i - 1) * 4;
 153
 154           word3 = rotr16 (word3, 5);
 155           word3 -= (word0 & ~word2) + (word1 & word2) + context->S[j + 3];
 156
 157           word2 = rotr16 (word2, 3);
 158           word2 -= (word3 & ~word1) + (word0 & word1) + context->S[j + 2];
 159
 160           word1 = rotr16 (word1, 2);
 161           word1 -= (word2 & ~word0) + (word3 & word0) + context->S[j + 1];
 162
 163           word0 = rotr16 (word0, 1);
 164           word0 -= (word1 & ~word3) + (word2 & word3) + context->S[j];
 165
 166           if (i == 6 || i == 12)
 167             {
 168               word3 = word3 - context->S[word2 & 63];
 169               word2 = word2 - context->S[word1 & 63];
 170           word1 = word1 - context->S[word0 & 63];
 171           word0 = word0 - context->S[word3 & 63];
 172             }
 173         }
 174
 175       outbuf[0] = word0 & 255;
 176       outbuf[1] = word0 >> 8;
 177       outbuf[2] = word1 & 255;
 178       outbuf[3] = word1 >> 8;
 179       outbuf[4] = word2 & 255;
 180       outbuf[5] = word2 >> 8;
 181       outbuf[6] = word3 & 255;
 182       outbuf[7] = word3 >> 8;
 183     }
 184 }
 185
 186 void
 187 arctwo_setkey_ekb (arctwo_context *context,
 188                    size_t keylen, const char *key, size_t effective_keylen)
 189 {
 190   size_t i;
 191   uint8_t *S, x;
 192
 193   if (keylen < 40 / 8 || effective_keylen > 1024)
 194     return;
 195
 196   S = (uint8_t *) context->S;
 197
 198   for (i = 0; i < keylen; i++)
 199     S[i] = (uint8_t) key[i];
 200
 201   for (i = keylen; i < 128; i++)
 202     S[i] = arctwo_sbox[(S[i - keylen] + S[i - 1]) & 255];
 203
 204   S[0] = arctwo_sbox[S[0]];
 205
 206   /* Phase 2 - reduce effective key size to "bits". This was not
 207    * discussed in Gutmann's paper. I've copied that from the public
 208    * domain code posted in sci.crypt. */
 209   if (effective_keylen)
 210     {
 211       size_t len = (effective_keylen + 7) >> 3;
 212       i = 128 - len;
 213       x = arctwo_sbox[S[i] & (255 >> (7 & -effective_keylen))];
 214       S[i] = x;
 215
 216       while (i--)
 217         {
 218           x = arctwo_sbox[x ^ S[i + len]];
 219           S[i] = x;
 220         }
 221     }
 222
 223   /* Make the expanded key, endian independent. */
 224   for (i = 0; i < 64; i++)
 225     context->S[i] = ((uint16_t) S[i * 2] | (((uint16_t) S[i * 2 + 1]) << 8));
 226 }