Merge pull request #268619 from tweag/lib-descriptions
[NixPkgs.git] / pkgs / applications / networking / mumble / 0001-BUILD-crypto-Migrate-to-OpenSSL-3.0-compatible-API.patch
blob3f2adc8d2a63246c4fc794370521e3adc88aeae5
1 From f4cea62ed95e4967d8591f25e903f5e8fc2e2a30 Mon Sep 17 00:00:00 2001
2 From: Terry Geng <terry@terriex.com>
3 Date: Mon, 6 Dec 2021 10:45:11 -0500
4 Subject: [PATCH] BUILD(crypto): Migrate to OpenSSL 3.0-compatible API
6 OpenSSL 3.0 deprecated several low-level APIs and the usage of them
7 caused errors/warnings that prevent the binary from being built against
8 OpenSSL 3.0.
9 Some primitive efforts have been made in #5317 but were incomplete.
10 This commit follows https://www.openssl.org/docs/man3.0/man7/migration_guide.html,
11 https://code.woboq.org/qt6/qtopcua/src/opcua/x509/qopcuakeypair_openssl.cpp.html,
12 and clears all errors/warnings related to the usage of deprecated APIs.
14 Fixes #5277
15 Fixes #4266
16 ---
17 src/SelfSignedCertificate.cpp | 235 +++++++++++-----------------------
18 src/SelfSignedCertificate.h | 5 +
19 src/crypto/CryptStateOCB2.cpp | 53 +++++---
20 src/crypto/CryptStateOCB2.h | 9 +-
21 4 files changed, 121 insertions(+), 181 deletions(-)
23 diff --git a/src/SelfSignedCertificate.cpp b/src/SelfSignedCertificate.cpp
24 index a77e5fad9..ea0dec4cc 100644
25 --- a/src/SelfSignedCertificate.cpp
26 +++ b/src/SelfSignedCertificate.cpp
27 @@ -5,8 +5,6 @@
29 #include "SelfSignedCertificate.h"
31 -#include <openssl/x509v3.h>
33 #define SSL_STRING(x) QString::fromLatin1(x).toUtf8().data()
35 static int add_ext(X509 *crt, int nid, char *value) {
36 @@ -28,108 +26,86 @@ static int add_ext(X509 *crt, int nid, char *value) {
37 return 1;
40 -bool SelfSignedCertificate::generate(CertificateType certificateType, QString clientCertName, QString clientCertEmail,
41 - QSslCertificate &qscCert, QSslKey &qskKey) {
42 - bool ok = true;
43 - X509 *x509 = nullptr;
44 - EVP_PKEY *pkey = nullptr;
45 - RSA *rsa = nullptr;
46 - BIGNUM *e = nullptr;
47 - X509_NAME *name = nullptr;
48 - ASN1_INTEGER *serialNumber = nullptr;
49 - ASN1_TIME *notBefore = nullptr;
50 - ASN1_TIME *notAfter = nullptr;
51 - QString commonName;
52 - bool isServerCert = certificateType == CertificateTypeServerCertificate;
54 - if (CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON) == -1) {
55 - ok = false;
56 - goto out;
57 +EVP_PKEY *SelfSignedCertificate::generate_rsa_keypair() {
58 + EVP_PKEY *pkey = EVP_PKEY_new();
59 + if (!pkey) {
60 + return nullptr;
63 - x509 = X509_new();
64 - if (!x509) {
65 - ok = false;
66 - goto out;
67 +#if OPENSSL_VERSION_NUMBER >= 0x10100000L
68 + EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr);
69 + if (!ctx) {
70 + return nullptr;
73 - pkey = EVP_PKEY_new();
74 - if (!pkey) {
75 - ok = false;
76 - goto out;
77 + if (EVP_PKEY_keygen_init(ctx) <= 0) {
78 + return nullptr;
81 - rsa = RSA_new();
82 + if (EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, 2048) <= 0) {
83 + return nullptr;
84 + }
85 + if (EVP_PKEY_keygen(ctx, &pkey) <= 0) {
86 + return nullptr;
87 + }
88 + EVP_PKEY_CTX_free(ctx);
89 +#else
90 + RSA *rsa = RSA_new();
91 + BIGNUM *e = BN_new();
92 if (!rsa) {
93 - ok = false;
94 - goto out;
95 + return nullptr;
98 - e = BN_new();
99 if (!e) {
100 - ok = false;
101 - goto out;
102 + return nullptr;
104 if (BN_set_word(e, 65537) == 0) {
105 - ok = false;
106 - goto out;
107 + return nullptr;
110 if (RSA_generate_key_ex(rsa, 2048, e, nullptr) == 0) {
111 - ok = false;
112 - goto out;
113 + return nullptr;
116 if (EVP_PKEY_assign_RSA(pkey, rsa) == 0) {
117 - ok = false;
118 - goto out;
119 + return nullptr;
121 + BN_free(e);
122 + RSA_free(rsa);
123 +#endif
124 + return pkey;
127 - if (X509_set_version(x509, 2) == 0) {
128 - ok = false;
129 - goto out;
130 +#define CHECK(statement) \
131 + if (!(statement)) { \
132 + ok = false; \
133 + goto out; \
136 - serialNumber = X509_get_serialNumber(x509);
137 - if (!serialNumber) {
138 - ok = false;
139 - goto out;
141 - if (ASN1_INTEGER_set(serialNumber, 1) == 0) {
142 - ok = false;
143 - goto out;
146 - notBefore = X509_get_notBefore(x509);
147 - if (!notBefore) {
148 - ok = false;
149 - goto out;
151 - if (!X509_gmtime_adj(notBefore, 0)) {
152 - ok = false;
153 - goto out;
155 +bool SelfSignedCertificate::generate(CertificateType certificateType, QString clientCertName, QString clientCertEmail,
156 + QSslCertificate &qscCert, QSslKey &qskKey) {
157 + bool ok = true;
158 + EVP_PKEY *pkey = nullptr;
159 + X509 *x509 = nullptr;
160 + X509_NAME *name = nullptr;
161 + ASN1_INTEGER *serialNumber = nullptr;
162 + ASN1_TIME *notBefore = nullptr;
163 + ASN1_TIME *notAfter = nullptr;
164 + QString commonName;
165 + bool isServerCert = certificateType == CertificateTypeServerCertificate;
167 - notAfter = X509_get_notAfter(x509);
168 - if (!notAfter) {
169 - ok = false;
170 - goto out;
172 - if (!X509_gmtime_adj(notAfter, 60 * 60 * 24 * 365 * 20)) {
173 - ok = false;
174 - goto out;
176 + // In Qt 5.15, a class was added to wrap up the procedures of generating a self-signed certificate.
177 + // See https://doc.qt.io/qt-5/qopcuax509certificatesigningrequest.html.
178 + // We should consider migrating to this class after switching to Qt 5.15.
180 - if (X509_set_pubkey(x509, pkey) == 0) {
181 - ok = false;
182 - goto out;
184 + CHECK(pkey = generate_rsa_keypair());
186 - name = X509_get_subject_name(x509);
187 - if (!name) {
188 - ok = false;
189 - goto out;
191 + CHECK(x509 = X509_new());
192 + CHECK(X509_set_version(x509, 2));
193 + CHECK(serialNumber = X509_get_serialNumber(x509));
194 + CHECK(ASN1_INTEGER_set(serialNumber, 1));
195 + CHECK(notBefore = X509_get_notBefore(x509));
196 + CHECK(X509_gmtime_adj(notBefore, 0));
197 + CHECK(notAfter = X509_get_notAfter(x509));
198 + CHECK(X509_gmtime_adj(notAfter, 60 * 60 * 24 * 365 * 20))
199 + CHECK(X509_set_pubkey(x509, pkey));
200 + CHECK(name = X509_get_subject_name(x509));
202 if (isServerCert) {
203 commonName = QLatin1String("Murmur Autogenerated Certificate v2");
204 @@ -141,120 +117,63 @@ bool SelfSignedCertificate::generate(CertificateType certificateType, QString cl
208 - if (X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_UTF8,
209 - reinterpret_cast< unsigned char * >(commonName.toUtf8().data()), -1, -1, 0)
210 - == 0) {
211 - ok = false;
212 - goto out;
214 + CHECK(X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_UTF8,
215 + reinterpret_cast< unsigned char * >(commonName.toUtf8().data()), -1, -1, 0));
217 - if (X509_set_issuer_name(x509, name) == 0) {
218 - ok = false;
219 - goto out;
221 + CHECK(X509_set_issuer_name(x509, name));
223 - if (add_ext(x509, NID_basic_constraints, SSL_STRING("critical,CA:FALSE")) == 0) {
224 - ok = false;
225 - goto out;
227 + CHECK(add_ext(x509, NID_basic_constraints, SSL_STRING("critical,CA:FALSE")));
229 if (isServerCert) {
230 - if (add_ext(x509, NID_ext_key_usage, SSL_STRING("serverAuth,clientAuth")) == 0) {
231 - ok = false;
232 - goto out;
234 + CHECK(add_ext(x509, NID_ext_key_usage, SSL_STRING("serverAuth,clientAuth")))
235 } else {
236 - if (add_ext(x509, NID_ext_key_usage, SSL_STRING("clientAuth")) == 0) {
237 - ok = false;
238 - goto out;
240 + CHECK(add_ext(x509, NID_ext_key_usage, SSL_STRING("clientAuth")));
243 - if (add_ext(x509, NID_subject_key_identifier, SSL_STRING("hash")) == 0) {
244 - ok = false;
245 - goto out;
247 + CHECK(add_ext(x509, NID_subject_key_identifier, SSL_STRING("hash")));
249 if (isServerCert) {
250 - if (add_ext(x509, NID_netscape_comment, SSL_STRING("Generated from murmur")) == 0) {
251 - ok = false;
252 - goto out;
254 + CHECK(add_ext(x509, NID_netscape_comment, SSL_STRING("Generated from murmur")));
255 } else {
256 - if (add_ext(x509, NID_netscape_comment, SSL_STRING("Generated by Mumble")) == 0) {
257 - ok = false;
258 - goto out;
260 + CHECK(add_ext(x509, NID_netscape_comment, SSL_STRING("Generated by Mumble")));
263 if (!isServerCert) {
264 if (!clientCertEmail.trimmed().isEmpty()) {
265 - if (add_ext(x509, NID_subject_alt_name,
266 - QString::fromLatin1("email:%1").arg(clientCertEmail).toUtf8().data())
267 - == 0) {
268 - ok = false;
269 - goto out;
271 + CHECK(add_ext(x509, NID_subject_alt_name,
272 + QString::fromLatin1("email:%1").arg(clientCertEmail).toUtf8().data()));
276 - if (X509_sign(x509, pkey, EVP_sha1()) == 0) {
277 - ok = false;
278 - goto out;
280 + CHECK(X509_sign(x509, pkey, EVP_sha1()));
283 QByteArray crt;
284 int len = i2d_X509(x509, nullptr);
285 - if (len <= 0) {
286 - ok = false;
287 - goto out;
289 + CHECK(len > 0);
290 crt.resize(len);
292 unsigned char *dptr = reinterpret_cast< unsigned char * >(crt.data());
293 - if (i2d_X509(x509, &dptr) != len) {
294 - ok = false;
295 - goto out;
297 + CHECK(i2d_X509(x509, &dptr) == len);
299 qscCert = QSslCertificate(crt, QSsl::Der);
300 - if (qscCert.isNull()) {
301 - ok = false;
302 - goto out;
304 + CHECK(!qscCert.isNull());
308 QByteArray key;
309 int len = i2d_PrivateKey(pkey, nullptr);
310 - if (len <= 0) {
311 - ok = false;
312 - goto out;
314 + CHECK(len > 0);
315 key.resize(len);
317 unsigned char *dptr = reinterpret_cast< unsigned char * >(key.data());
318 - if (i2d_PrivateKey(pkey, &dptr) != len) {
319 - ok = false;
320 - goto out;
322 + CHECK(i2d_PrivateKey(pkey, &dptr) == len);
324 qskKey = QSslKey(key, QSsl::Rsa, QSsl::Der);
325 - if (qskKey.isNull()) {
326 - ok = false;
327 - goto out;
329 + CHECK(!qskKey.isNull());
332 out:
333 - if (e) {
334 - BN_free(e);
336 - // We only need to free the pkey pointer,
337 - // not the RSA pointer. We have assigned
338 - // our RSA key to pkey, and it will be freed
339 - // once we free pkey.
340 if (pkey) {
341 EVP_PKEY_free(pkey);
343 diff --git a/src/SelfSignedCertificate.h b/src/SelfSignedCertificate.h
344 index b85a8752b..7c5f59e9c 100644
345 --- a/src/SelfSignedCertificate.h
346 +++ b/src/SelfSignedCertificate.h
347 @@ -6,6 +6,10 @@
348 #ifndef MUMBLE_SELFSIGNEDCERTIFICATE_H_
349 #define MUMBLE_SELFSIGNEDCERTIFICATE_H_
351 +#include <openssl/evp.h>
352 +#include <openssl/rsa.h>
353 +#include <openssl/x509v3.h>
355 #include <QtCore/QString>
356 #include <QtNetwork/QSslCertificate>
357 #include <QtNetwork/QSslKey>
358 @@ -16,6 +20,7 @@ class SelfSignedCertificate {
359 private:
360 static bool generate(CertificateType certificateType, QString clientCertName, QString clientCertEmail,
361 QSslCertificate &qscCert, QSslKey &qskKey);
362 + static EVP_PKEY *generate_rsa_keypair();
364 public:
365 static bool generateMumbleCertificate(QString name, QString email, QSslCertificate &qscCert, QSslKey &qskKey);
366 diff --git a/src/crypto/CryptStateOCB2.cpp b/src/crypto/CryptStateOCB2.cpp
367 index 2176d6488..640fdedac 100644
368 --- a/src/crypto/CryptStateOCB2.cpp
369 +++ b/src/crypto/CryptStateOCB2.cpp
370 @@ -30,7 +30,7 @@
371 #include <cstring>
372 #include <openssl/rand.h>
374 -CryptStateOCB2::CryptStateOCB2() : CryptState() {
375 +CryptStateOCB2::CryptStateOCB2() : CryptState(), enc_ctx(EVP_CIPHER_CTX_new()), dec_ctx(EVP_CIPHER_CTX_new()) {
376 for (int i = 0; i < 0x100; i++)
377 decrypt_history[i] = 0;
378 memset(raw_key, 0, AES_KEY_SIZE_BYTES);
379 @@ -38,6 +38,11 @@ CryptStateOCB2::CryptStateOCB2() : CryptState() {
380 memset(decrypt_iv, 0, AES_BLOCK_SIZE);
383 +CryptStateOCB2::~CryptStateOCB2() noexcept {
384 + EVP_CIPHER_CTX_free(enc_ctx);
385 + EVP_CIPHER_CTX_free(dec_ctx);
388 bool CryptStateOCB2::isValid() const {
389 return bInit;
391 @@ -46,8 +51,6 @@ void CryptStateOCB2::genKey() {
392 CryptographicRandom::fillBuffer(raw_key, AES_KEY_SIZE_BYTES);
393 CryptographicRandom::fillBuffer(encrypt_iv, AES_BLOCK_SIZE);
394 CryptographicRandom::fillBuffer(decrypt_iv, AES_BLOCK_SIZE);
395 - AES_set_encrypt_key(raw_key, AES_KEY_SIZE_BITS, &encrypt_key);
396 - AES_set_decrypt_key(raw_key, AES_KEY_SIZE_BITS, &decrypt_key);
397 bInit = true;
400 @@ -56,8 +59,6 @@ bool CryptStateOCB2::setKey(const std::string &rkey, const std::string &eiv, con
401 memcpy(raw_key, rkey.data(), AES_KEY_SIZE_BYTES);
402 memcpy(encrypt_iv, eiv.data(), AES_BLOCK_SIZE);
403 memcpy(decrypt_iv, div.data(), AES_BLOCK_SIZE);
404 - AES_set_encrypt_key(raw_key, AES_KEY_SIZE_BITS, &encrypt_key);
405 - AES_set_decrypt_key(raw_key, AES_KEY_SIZE_BITS, &decrypt_key);
406 bInit = true;
407 return true;
409 @@ -256,10 +257,24 @@ static void inline ZERO(keyblock &block) {
410 block[i] = 0;
413 -#define AESencrypt(src, dst, key) \
414 - AES_encrypt(reinterpret_cast< const unsigned char * >(src), reinterpret_cast< unsigned char * >(dst), key);
415 -#define AESdecrypt(src, dst, key) \
416 - AES_decrypt(reinterpret_cast< const unsigned char * >(src), reinterpret_cast< unsigned char * >(dst), key);
417 +#define AESencrypt(src, dst, key) \
418 + { \
419 + int outlen = 0; \
420 + EVP_EncryptInit_ex(enc_ctx, EVP_aes_128_ecb(), NULL, key, NULL); \
421 + EVP_CIPHER_CTX_set_padding(enc_ctx, 0); \
422 + EVP_EncryptUpdate(enc_ctx, reinterpret_cast< unsigned char * >(dst), &outlen, \
423 + reinterpret_cast< const unsigned char * >(src), AES_BLOCK_SIZE); \
424 + EVP_EncryptFinal_ex(enc_ctx, reinterpret_cast< unsigned char * >(dst + outlen), &outlen); \
426 +#define AESdecrypt(src, dst, key) \
427 + { \
428 + int outlen = 0; \
429 + EVP_DecryptInit_ex(dec_ctx, EVP_aes_128_ecb(), NULL, key, NULL); \
430 + EVP_CIPHER_CTX_set_padding(dec_ctx, 0); \
431 + EVP_DecryptUpdate(dec_ctx, reinterpret_cast< unsigned char * >(dst), &outlen, \
432 + reinterpret_cast< const unsigned char * >(src), AES_BLOCK_SIZE); \
433 + EVP_DecryptFinal_ex(dec_ctx, reinterpret_cast< unsigned char * >(dst + outlen), &outlen); \
436 bool CryptStateOCB2::ocb_encrypt(const unsigned char *plain, unsigned char *encrypted, unsigned int len,
437 const unsigned char *nonce, unsigned char *tag, bool modifyPlainOnXEXStarAttack) {
438 @@ -267,7 +282,7 @@ bool CryptStateOCB2::ocb_encrypt(const unsigned char *plain, unsigned char *encr
439 bool success = true;
441 // Initialize
442 - AESencrypt(nonce, delta, &encrypt_key);
443 + AESencrypt(nonce, delta, raw_key);
444 ZERO(checksum);
446 while (len > AES_BLOCK_SIZE) {
447 @@ -299,7 +314,7 @@ bool CryptStateOCB2::ocb_encrypt(const unsigned char *plain, unsigned char *encr
448 if (flipABit) {
449 *reinterpret_cast< unsigned char * >(tmp) ^= 1;
451 - AESencrypt(tmp, tmp, &encrypt_key);
452 + AESencrypt(tmp, tmp, raw_key);
453 XOR(reinterpret_cast< subblock * >(encrypted), delta, tmp);
454 XOR(checksum, checksum, reinterpret_cast< const subblock * >(plain));
455 if (flipABit) {
456 @@ -315,7 +330,7 @@ bool CryptStateOCB2::ocb_encrypt(const unsigned char *plain, unsigned char *encr
457 ZERO(tmp);
458 tmp[BLOCKSIZE - 1] = SWAPPED(len * 8);
459 XOR(tmp, tmp, delta);
460 - AESencrypt(tmp, pad, &encrypt_key);
461 + AESencrypt(tmp, pad, raw_key);
462 memcpy(tmp, plain, len);
463 memcpy(reinterpret_cast< unsigned char * >(tmp) + len, reinterpret_cast< const unsigned char * >(pad) + len,
464 AES_BLOCK_SIZE - len);
465 @@ -325,7 +340,7 @@ bool CryptStateOCB2::ocb_encrypt(const unsigned char *plain, unsigned char *encr
467 S3(delta);
468 XOR(tmp, delta, checksum);
469 - AESencrypt(tmp, tag, &encrypt_key);
470 + AESencrypt(tmp, tag, raw_key);
472 return success;
474 @@ -336,13 +351,13 @@ bool CryptStateOCB2::ocb_decrypt(const unsigned char *encrypted, unsigned char *
475 bool success = true;
477 // Initialize
478 - AESencrypt(nonce, delta, &encrypt_key);
479 + AESencrypt(nonce, delta, raw_key);
480 ZERO(checksum);
482 while (len > AES_BLOCK_SIZE) {
483 S2(delta);
484 XOR(tmp, delta, reinterpret_cast< const subblock * >(encrypted));
485 - AESdecrypt(tmp, tmp, &decrypt_key);
486 + AESdecrypt(tmp, tmp, raw_key);
487 XOR(reinterpret_cast< subblock * >(plain), delta, tmp);
488 XOR(checksum, checksum, reinterpret_cast< const subblock * >(plain));
489 len -= AES_BLOCK_SIZE;
490 @@ -354,7 +369,7 @@ bool CryptStateOCB2::ocb_decrypt(const unsigned char *encrypted, unsigned char *
491 ZERO(tmp);
492 tmp[BLOCKSIZE - 1] = SWAPPED(len * 8);
493 XOR(tmp, tmp, delta);
494 - AESencrypt(tmp, pad, &encrypt_key);
495 + AESencrypt(tmp, pad, raw_key);
496 memset(tmp, 0, AES_BLOCK_SIZE);
497 memcpy(tmp, encrypted, len);
498 XOR(tmp, tmp, pad);
499 @@ -372,7 +387,7 @@ bool CryptStateOCB2::ocb_decrypt(const unsigned char *encrypted, unsigned char *
501 S3(delta);
502 XOR(tmp, delta, checksum);
503 - AESencrypt(tmp, tag, &encrypt_key);
504 + AESencrypt(tmp, tag, raw_key);
506 return success;
508 @@ -381,5 +396,5 @@ bool CryptStateOCB2::ocb_decrypt(const unsigned char *encrypted, unsigned char *
509 #undef SHIFTBITS
510 #undef SWAPPED
511 #undef HIGHBIT
512 -#undef AES_encrypt
513 -#undef AES_decrypt
514 +#undef AESencrypt
515 +#undef AESdecrypt
516 diff --git a/src/crypto/CryptStateOCB2.h b/src/crypto/CryptStateOCB2.h
517 index 53d4b4b6a..cc3f1c0bc 100644
518 --- a/src/crypto/CryptStateOCB2.h
519 +++ b/src/crypto/CryptStateOCB2.h
520 @@ -8,8 +8,9 @@
522 #include "CryptState.h"
524 -#include <openssl/aes.h>
525 +#include <openssl/evp.h>
527 +#define AES_BLOCK_SIZE 16
528 #define AES_KEY_SIZE_BITS 128
529 #define AES_KEY_SIZE_BYTES (AES_KEY_SIZE_BITS / 8)
531 @@ -17,7 +18,7 @@
532 class CryptStateOCB2 : public CryptState {
533 public:
534 CryptStateOCB2();
535 - ~CryptStateOCB2(){};
536 + ~CryptStateOCB2() noexcept override;
538 virtual bool isValid() const Q_DECL_OVERRIDE;
539 virtual void genKey() Q_DECL_OVERRIDE;
540 @@ -43,8 +44,8 @@ private:
541 unsigned char decrypt_iv[AES_BLOCK_SIZE];
542 unsigned char decrypt_history[0x100];
544 - AES_KEY encrypt_key;
545 - AES_KEY decrypt_key;
546 + EVP_CIPHER_CTX *enc_ctx;
547 + EVP_CIPHER_CTX *dec_ctx;
552 2.38.4