Added test and patch for VLA stack overflow vuln.

Also added and used the new crypto_malloc and crypto_free.

The latter also zeroes out the memory safely. The former only exists for
symmetry (static analysis can detect asymmetric usages).

2 files changed, 70 insertions, 6 deletions

diff --git a/auto_tests/encryptsave_test.c b/auto_tests/encryptsave_test.c
index 906bf3f8..19574d16 100644
--- a/auto_tests/encryptsave_test.c
+++ b/auto_tests/encryptsave_test.c
@@ -142,7 +142,8 @@ static void test_keys(void)
     Tox_Err_Encryption encerr;
     Tox_Err_Decryption decerr;
     Tox_Err_Key_Derivation keyerr;
-    Tox_Pass_Key *key = tox_pass_key_derive((const uint8_t *)"123qweasdzxc", 12, &keyerr);
+    const uint8_t *key_char = (const uint8_t *)"123qweasdzxc";
+    Tox_Pass_Key *key = tox_pass_key_derive(key_char, 12, &keyerr);
     ck_assert_msg(key != nullptr, "generic failure 1: %d", keyerr);
     const uint8_t *string = (const uint8_t *)"No Patrick, mayonnaise is not an instrument."; // 44
 
@@ -150,8 +151,27 @@ static void test_keys(void)
     bool ret = tox_pass_key_encrypt(key, string, 44, encrypted, &encerr);
     ck_assert_msg(ret, "generic failure 2: %d", encerr);
 
+    // Testing how tox handles encryption of large messages.
+    int size_large = 30 * 1024 * 1024;
+    int ciphertext_length2a = size_large + TOX_PASS_ENCRYPTION_EXTRA_LENGTH;
+    int plaintext_length2a = size_large;
+    uint8_t *encrypted2a = (uint8_t *)malloc(ciphertext_length2a);
+    uint8_t *in_plaintext2a = (uint8_t *)malloc(plaintext_length2a);
+    ret = tox_pass_encrypt(in_plaintext2a, plaintext_length2a, key_char, 12, encrypted2a, &encerr);
+    ck_assert_msg(ret, "tox_pass_encrypt failure 2a: %d", encerr);
+
+    // Decryption of same message.
+    uint8_t *out_plaintext2a = (uint8_t *) malloc(plaintext_length2a);
+    ret = tox_pass_decrypt(encrypted2a, ciphertext_length2a, key_char, 12, out_plaintext2a, &decerr);
+    ck_assert_msg(ret, "tox_pass_decrypt failure 2a: %d", decerr);
+    ck_assert_msg(memcmp(in_plaintext2a, out_plaintext2a, plaintext_length2a) == 0, "Large message decryption failed");
+    free(encrypted2a);
+    free(in_plaintext2a);
+    free(out_plaintext2a);
+
+
     uint8_t encrypted2[44 + TOX_PASS_ENCRYPTION_EXTRA_LENGTH];
-    ret = tox_pass_encrypt(string, 44, (const uint8_t *)"123qweasdzxc", 12, encrypted2, &encerr);
+    ret = tox_pass_encrypt(string, 44, key_char, 12, encrypted2, &encerr);
     ck_assert_msg(ret, "generic failure 3: %d", encerr);
 
     uint8_t out1[44 + TOX_PASS_ENCRYPTION_EXTRA_LENGTH];
diff --git a/toxcore/crypto_core.c b/toxcore/crypto_core.c
index 1fd62866..0e53100a 100644
--- a/toxcore/crypto_core.c
+++ b/toxcore/crypto_core.c
@@ -30,6 +30,7 @@
 #include "ccompat.h"
 #include "crypto_core.h"
 
+#include <stdlib.h>
 #include <string.h>
 
 #ifndef VANILLA_NACL
@@ -82,6 +83,20 @@
 #error "CRYPTO_PUBLIC_KEY_SIZE is required to be 32 bytes for public_key_cmp to work,"
 #endif
 
+static uint8_t *crypto_malloc(size_t bytes)
+{
+    return (uint8_t *)malloc(bytes);
+}
+
+static void crypto_free(uint8_t *ptr, size_t bytes)
+{
+    if (ptr != nullptr) {
+        crypto_memzero(ptr, bytes);
+    }
+
+    free(ptr);
+}
+
 int32_t public_key_cmp(const uint8_t *pk1, const uint8_t *pk2)
 {
     return crypto_verify_32(pk1, pk2);
@@ -142,8 +157,17 @@ int32_t encrypt_data_symmetric(const uint8_t *secret_key, const uint8_t *nonce,
         return -1;
     }
 
-    VLA(uint8_t, temp_plain, length + crypto_box_ZEROBYTES);
-    VLA(uint8_t, temp_encrypted, length + crypto_box_MACBYTES + crypto_box_BOXZEROBYTES);
+    const size_t size_temp_plain = length + crypto_box_ZEROBYTES;
+    const size_t size_temp_encrypted = length + crypto_box_MACBYTES + crypto_box_BOXZEROBYTES;
+
+    uint8_t *temp_plain = crypto_malloc(size_temp_plain);
+    uint8_t *temp_encrypted = crypto_malloc(size_temp_encrypted);
+
+    if (temp_plain == nullptr || temp_encrypted == nullptr) {
+        crypto_free(temp_plain, size_temp_plain);
+        crypto_free(temp_encrypted, size_temp_encrypted);
+        return -1;
+    }
 
     memset(temp_plain, 0, crypto_box_ZEROBYTES);
     // Pad the message with 32 0 bytes.
@@ -151,11 +175,17 @@ int32_t encrypt_data_symmetric(const uint8_t *secret_key, const uint8_t *nonce,
 
     if (crypto_box_afternm(temp_encrypted, temp_plain, length + crypto_box_ZEROBYTES, nonce,
                            secret_key) != 0) {
+        crypto_free(temp_plain, size_temp_plain);
+        crypto_free(temp_encrypted, size_temp_encrypted);
         return -1;
     }
 
     // Unpad the encrypted message.
     memcpy(encrypted, temp_encrypted + crypto_box_BOXZEROBYTES, length + crypto_box_MACBYTES);
+
+    crypto_free(temp_plain, size_temp_plain);
+    crypto_free(temp_encrypted, size_temp_encrypted);
+
     return length + crypto_box_MACBYTES;
 }
 
@@ -166,8 +196,17 @@ int32_t decrypt_data_symmetric(const uint8_t *secret_key, const uint8_t *nonce,
         return -1;
     }
 
-    VLA(uint8_t, temp_plain, length + crypto_box_ZEROBYTES);
-    VLA(uint8_t, temp_encrypted, length + crypto_box_BOXZEROBYTES);
+    const size_t size_temp_plain = length + crypto_box_ZEROBYTES;
+    const size_t size_temp_encrypted = length + crypto_box_BOXZEROBYTES;
+
+    uint8_t *temp_plain = crypto_malloc(size_temp_plain);
+    uint8_t *temp_encrypted = crypto_malloc(size_temp_encrypted);
+
+    if (temp_plain == nullptr || temp_encrypted == nullptr) {
+        crypto_free(temp_plain, size_temp_plain);
+        crypto_free(temp_encrypted, size_temp_encrypted);
+        return -1;
+    }
 
     memset(temp_encrypted, 0, crypto_box_BOXZEROBYTES);
     // Pad the message with 16 0 bytes.
@@ -175,10 +214,15 @@ int32_t decrypt_data_symmetric(const uint8_t *secret_key, const uint8_t *nonce,
 
     if (crypto_box_open_afternm(temp_plain, temp_encrypted, length + crypto_box_BOXZEROBYTES, nonce,
                                 secret_key) != 0) {
+        crypto_free(temp_plain, size_temp_plain);
+        crypto_free(temp_encrypted, size_temp_encrypted);
         return -1;
     }
 
     memcpy(plain, temp_plain + crypto_box_ZEROBYTES, length - crypto_box_MACBYTES);
+
+    crypto_free(temp_plain, size_temp_plain);
+    crypto_free(temp_encrypted, size_temp_encrypted);
     return length - crypto_box_MACBYTES;
 }