Fix build against OpenSSL 1.1

Closes: #3196

Telegram/SourceFiles/base/openssl_help.h

@@ -55,8 +55,7 @@ private:
 
 class BigNum {
 public:
-	BigNum() {
-		BN_init(raw());
+	BigNum() : _data(BN_new()) {
 	}
 	BigNum(const BigNum &other) : BigNum() {
 		*this = other;
@@ -176,10 +175,10 @@ public:
 	}
 
 	BIGNUM *raw() {
-		return &_data;
+		return _data;
 	}
 	const BIGNUM *raw() const {
-		return &_data;
+		return _data;
 	}
 
 	bool failed() const {
@@ -193,7 +192,7 @@ public:
 	}
 
 private:
-	BIGNUM _data;
+	BIGNUM *_data = nullptr;
 	mutable bool _failed = false;
 
 };

Telegram/SourceFiles/mtproto/auth_key.cpp

@@ -21,6 +21,9 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
 #include "mtproto/auth_key.h"
 
 #include <openssl/aes.h>
+extern "C" {
+#include <openssl/modes.h>
+}
 
 namespace MTP {
 
@@ -109,7 +112,7 @@ void aesCtrEncrypt(void *data, quint32 len, const void *key, CTRState *state) {
 	static_assert(CTRState::IvecSize == AES_BLOCK_SIZE, "Wrong size of ctr ivec!");
 	static_assert(CTRState::EcountSize == AES_BLOCK_SIZE, "Wrong size of ctr ecount!");
 
-	AES_ctr128_encrypt(static_cast<const uchar*>(data), static_cast<uchar*>(data), len, &aes, state->ivec, state->ecount, &state->num);
+	CRYPTO_ctr128_encrypt(static_cast<const uchar*>(data), static_cast<uchar*>(data), len, &aes, state->ivec, state->ecount, &state->num, (block128_f) AES_encrypt);
 }
 
 } // namespace MTP

Telegram/SourceFiles/mtproto/rsa_public_key.cpp

@@ -29,6 +29,45 @@ Copyright (c) 2014-2017 John Preston, https://desktop.telegram.org
 using std::string;
 
 namespace MTP {
+namespace {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+
+// This is a key setter for compatibility with OpenSSL 1.0
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
+	if ((r->n == nullptr && n == nullptr) || (r->e == nullptr && e == nullptr)) {
+		return false;
+	}
+	if (n != nullptr) {
+		BN_free(r->n);
+		r->n = n;
+	}
+	if (e != nullptr) {
+		BN_free(r->e);
+		r->e = e;
+	}
+	if (d != nullptr) {
+		BN_free(r->d);
+		r->d = d;
+	}
+	return true;
+}
+
+// This is a key getter for compatibility with OpenSSL 1.0
+void RSA_get0_key(const RSA *r, const BIGNUM **n, const BIGNUM **e, const BIGNUM **d) {
+	if (n != nullptr) {
+		*n = r->n;
+	}
+	if (e != nullptr) {
+		*e = r->e;
+	}
+	if (d != nullptr) {
+		*d = r->d;
+	}
+}
+
+#endif
+}
+
 namespace internal {
 
 class RSAPublicKey::Private {
@@ -40,9 +79,10 @@ public:
 	}
 	Private(base::const_byte_span nBytes, base::const_byte_span eBytes) : _rsa(RSA_new()) {
 		if (_rsa) {
-			_rsa->n = BN_dup(openssl::BigNum(nBytes).raw());
-			_rsa->e = BN_dup(openssl::BigNum(eBytes).raw());
-			if (!_rsa->n || !_rsa->e) {
+			BIGNUM *n = BN_dup(openssl::BigNum(nBytes).raw());
+			BIGNUM *e = BN_dup(openssl::BigNum(eBytes).raw());
+			RSA_set0_key(_rsa, n, e, nullptr);
+			if (!n || !e) {
 				RSA_free(base::take(_rsa));
 			} else {
 				computeFingerprint();
@@ -51,11 +91,15 @@ public:
 	}
 	base::byte_vector getN() const {
 		Expects(isValid());
-		return toBytes(_rsa->n);
+		const BIGNUM *n;
+		RSA_get0_key(_rsa, &n, nullptr, nullptr);
+		return toBytes(n);
 	}
 	base::byte_vector getE() const {
 		Expects(isValid());
-		return toBytes(_rsa->e);
+		const BIGNUM *e;
+		RSA_get0_key(_rsa, nullptr, &e, nullptr);
+		return toBytes(e);
 	}
 	quint64 getFingerPrint() const {
 		return _fingerprint;
@@ -105,14 +149,16 @@ private:
 	void computeFingerprint() {
 		Expects(isValid());
 
+		const BIGNUM *n, *e;
 		mtpBuffer string;
-		MTP_bytes(toBytes(_rsa->n)).write(string);
-		MTP_bytes(toBytes(_rsa->e)).write(string);
+		RSA_get0_key(_rsa, &n, &e, nullptr);
+		MTP_bytes(toBytes(n)).write(string);
+		MTP_bytes(toBytes(e)).write(string);
 
 		uchar sha1Buffer[20];
 		_fingerprint = *(quint64*)(hashSha1(&string[0], string.size() * sizeof(mtpPrime), sha1Buffer) + 3);
 	}
-	static base::byte_vector toBytes(BIGNUM *number) {
+	static base::byte_vector toBytes(const BIGNUM *number) {
 		auto size = BN_num_bytes(number);
 		auto result = base::byte_vector(size, gsl::byte {});
 		BN_bn2bin(number, reinterpret_cast<unsigned char*>(result.data()));