Fix crash in RSA public key wrapper.

Telegram/SourceFiles/base/openssl_help.h

@@ -83,11 +83,18 @@ public:
 		}
 	}
 	void setBytes(base::const_byte_span bytes) {
-		if (!BN_bin2bn(reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size(), raw())) {
+		if (!BN_bin2bn(
+				reinterpret_cast<const unsigned char*>(bytes.data()),
+				bytes.size(),
+				raw())) {
 			_failed = true;
 		}
 	}
-	void setModExp(const BigNum &a, const BigNum &p, const BigNum &m, const Context &context = Context()) {
+	void setModExp(
+			const BigNum &a,
+			const BigNum &p,
+			const BigNum &m,
+			const Context &context = Context()) {
 		if (a.failed() || p.failed() || m.failed()) {
 			_failed = true;
 		} else if (a.isNegative() || p.isNegative() || m.isNegative()) {
@@ -134,7 +141,11 @@ public:
 			return false;
 		}
 		constexpr auto kMillerRabinIterationCount = 30;
-		auto result = BN_is_prime_ex(raw(), kMillerRabinIterationCount, context.raw(), NULL);
+		auto result = BN_is_prime_ex(
+			raw(),
+			kMillerRabinIterationCount,
+			context.raw(),
+			NULL);
 		if (result == 1) {
 			return true;
 		} else if (result != 0) {
@@ -169,7 +180,9 @@ public:
 		}
 		auto length = BN_num_bytes(raw());
 		auto result = base::byte_vector(length, gsl::byte());
-		auto resultSize = BN_bn2bin(raw(), reinterpret_cast<unsigned char*>(result.data()));
+		auto resultSize = BN_bn2bin(
+			raw(),
+			reinterpret_cast<unsigned char*>(result.data()));
 		Assert(resultSize == length);
 		return result;
 	}
@@ -180,6 +193,9 @@ public:
 	const BIGNUM *raw() const {
 		return _data;
 	}
+	BIGNUM *takeRaw() {
+		return base::take(_data);
+	}
 
 	bool failed() const {
 		return _failed;

Telegram/SourceFiles/mtproto/connection.cpp

@@ -154,7 +154,10 @@ bool IsPrimeAndGood(base::const_byte_span primeBytes, int g) {
 	return IsPrimeAndGoodCheck(openssl::BigNum(primeBytes), g);
 }
 
-std::vector<gsl::byte> CreateAuthKey(base::const_byte_span firstBytes, base::const_byte_span randomBytes, base::const_byte_span primeBytes) {
+std::vector<gsl::byte> CreateAuthKey(
+		base::const_byte_span firstBytes,
+		base::const_byte_span randomBytes,
+		base::const_byte_span primeBytes) {
 	using openssl::BigNum;
 	BigNum first(firstBytes);
 	BigNum prime(primeBytes);
@@ -165,7 +168,10 @@ std::vector<gsl::byte> CreateAuthKey(base::const_byte_span firstBytes, base::con
 	return BigNum::ModExp(first, BigNum(randomBytes), prime).getBytes();
 }
 
-ModExpFirst CreateModExp(int g, base::const_byte_span primeBytes, base::const_byte_span randomSeed) {
+ModExpFirst CreateModExp(
+		int g,
+		base::const_byte_span primeBytes,
+		base::const_byte_span randomSeed) {
 	Expects(randomSeed.size() == ModExpFirst::kRandomPowerSize);
 
 	using namespace openssl;

Telegram/SourceFiles/mtproto/rsa_public_key.cpp

@@ -35,7 +35,7 @@ namespace {
 // This is a key setter for compatibility with OpenSSL 1.0
 int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
 	if ((r->n == nullptr && n == nullptr) || (r->e == nullptr && e == nullptr)) {
-		return false;
+		return 0;
 	}
 	if (n != nullptr) {
 		BN_free(r->n);
@@ -49,7 +49,7 @@ int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
 		BN_free(r->d);
 		r->d = d;
 	}
-	return true;
+	return 1;
 }
 
 // This is a key getter for compatibility with OpenSSL 1.0
@@ -72,17 +72,21 @@ namespace internal {
 
 class RSAPublicKey::Private {
 public:
-	Private(base::const_byte_span key) : _rsa(PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<gsl::byte*>(key.data()), key.size()), 0, 0, 0)) {
+	Private(base::const_byte_span key)
+	: _rsa(PEM_read_bio_RSAPublicKey(BIO_new_mem_buf(const_cast<gsl::byte*>(key.data()), key.size()), 0, 0, 0)) {
 		if (_rsa) {
 			computeFingerprint();
 		}
 	}
-	Private(base::const_byte_span nBytes, base::const_byte_span eBytes) : _rsa(RSA_new()) {
+	Private(base::const_byte_span nBytes, base::const_byte_span eBytes)
+	: _rsa(RSA_new()) {
 		if (_rsa) {
-			BIGNUM *n = openssl::BigNum(nBytes).raw();
-			BIGNUM *e = openssl::BigNum(eBytes).raw();
-			RSA_set0_key(_rsa, n, e, nullptr);
-			if (!n || !e) {
+			auto n = openssl::BigNum(nBytes).takeRaw();
+			auto e = openssl::BigNum(eBytes).takeRaw();
+			auto valid = (n != nullptr) && (e != nullptr);
+			// We still pass both values to RSA_set0_key() so that even
+			// if only one of them is valid RSA would take ownership of it.
+			if (!RSA_set0_key(_rsa, n, e, nullptr) || !valid) {
 				RSA_free(base::take(_rsa));
 			} else {
 				computeFingerprint();