Stricter checks (RSA OAEP, X25519, X448) - based on wycheproof results

src/pk/rsa/rsa_decrypt_key.c

@@ -25,6 +25,7 @@ int rsa_decrypt_key_v2(const unsigned char *in,             unsigned long  inlen
                              int           *stat,     const rsa_key       *key)
 {
   int           err;
+  unsigned char zero, one;
   unsigned char *tmp;
   unsigned long modulus_bitlen, modulus_bytelen, x;
   ltc_rsa_op_checked op_checked = ltc_rsa_op_checked_init(key, params);
@@ -51,6 +52,20 @@ int rsa_decrypt_key_v2(const unsigned char *in,             unsigned long  inlen
      return CRYPT_INVALID_PACKET;
   }
 
+  /* SP 800-56B Rev. 2 Section 7.1.2.1 says to reject ciphertext values 0 and 1 */
+  if (params->padding == LTC_PKCS_1_OAEP) {
+     zero = one = 0;
+     for (x = 0; x < inlen; ++x) {
+        zero |= in[x];
+        if (x == inlen - 1) {
+           one |= (unsigned char)(in[x] ^ 0x01);
+        } else {
+           one |= in[x];
+        }
+     }
+     if (zero == 0 || one == 0) return CRYPT_INVALID_PACKET;
+  }
+
   /* allocate ram */
   tmp = XMALLOC(inlen);
   if (tmp == NULL) {

src/pk/x25519/x25519_shared_secret.c

@@ -21,6 +21,9 @@ int x25519_shared_secret(const    curve25519_key *private_key,
                          const    curve25519_key *public_key,
                                unsigned char *out, unsigned long *outlen)
 {
+   unsigned char nonzero = 0;
+   unsigned long x;
+
    LTC_ARGCHK(private_key        != NULL);
    LTC_ARGCHK(public_key         != NULL);
    LTC_ARGCHK(out                != NULL);
@@ -37,6 +40,10 @@ int x25519_shared_secret(const    curve25519_key *private_key,
    tweetnacl_crypto_scalarmult(out, private_key->priv, public_key->pub);
    *outlen = 32uL;
 
+   /* Reject all-zero shared secrets; RFC 7748 Section 6.1 says peers MAY check for this */
+   for (x = 0; x < *outlen; ++x) nonzero |= out[x];
+   if (nonzero == 0) return CRYPT_INVALID_PACKET;
+
    return CRYPT_OK;
 }
 

src/pk/x448/x448_shared_secret.c

@@ -21,6 +21,9 @@ int x448_shared_secret(const    curve448_key *private_key,
                        const    curve448_key *public_key,
                              unsigned char *out, unsigned long *outlen)
 {
+   unsigned char nonzero = 0;
+   unsigned long x;
+
    LTC_ARGCHK(private_key        != NULL);
    LTC_ARGCHK(public_key         != NULL);
    LTC_ARGCHK(out                != NULL);
@@ -37,6 +40,10 @@ int x448_shared_secret(const    curve448_key *private_key,
    ec448_scalarmult_internal(out, private_key->priv, public_key->pub);
    *outlen = 56uL;
 
+   /* Reject all-zero shared secrets; RFC 7748 Section 6.2 says peers MAY check for this */
+   for (x = 0; x < *outlen; ++x) nonzero |= out[x];
+   if (nonzero == 0) return CRYPT_INVALID_PACKET;
+
    return CRYPT_OK;
 }
 

tests/rsa_test.c

@@ -521,6 +521,51 @@ static int s_rsa_pss_test(void)
    return CRYPT_OK;
 }
 
+static int s_rsa_oaep_small_ciphertext_test(int prng_idx)
+{
+   rsa_key key;
+   unsigned char ciphertext[256], plaintext[256], ct_zero[256], ct_one[256];
+   unsigned long ciphertext_len, plaintext_len, modulus_len;
+   int hash_idx, stat;
+   const unsigned char msg[] = "hello strict-mode roundtrip";
+   ltc_rsa_op_parameters rsa_params = {
+      .padding = LTC_PKCS_1_OAEP,
+   };
+
+   hash_idx = find_hash("sha256");
+   if (hash_idx == -1) return CRYPT_NOP;
+
+   rsa_params.prng = &yarrow_prng;
+   rsa_params.wprng = prng_idx;
+   rsa_params.params.hash_idx = hash_idx;
+   rsa_params.params.mgf1_hash_idx = hash_idx;
+
+   DO(rsa_make_key(&yarrow_prng, prng_idx, 1024/8, 65537, &key));
+   modulus_len = (unsigned long)rsa_get_size(&key);
+   ENSURE(modulus_len <= sizeof(ciphertext));
+
+   zeromem(ct_zero, modulus_len);
+   zeromem(ct_one, modulus_len);
+   ct_one[modulus_len - 1] = 1;
+
+   plaintext_len = sizeof(plaintext);
+   SHOULD_FAIL_WITH(rsa_decrypt_key_v2(ct_zero, modulus_len, plaintext, &plaintext_len, &rsa_params, &stat, &key), CRYPT_INVALID_PACKET);
+
+   plaintext_len = sizeof(plaintext);
+   SHOULD_FAIL_WITH(rsa_decrypt_key_v2(ct_one, modulus_len, plaintext, &plaintext_len, &rsa_params, &stat, &key), CRYPT_INVALID_PACKET);
+
+   ciphertext_len = sizeof(ciphertext);
+   DO(rsa_encrypt_key_v2(msg, sizeof(msg) - 1, ciphertext, &ciphertext_len, &rsa_params, &key));
+
+   plaintext_len = sizeof(plaintext);
+   DO(rsa_decrypt_key_v2(ciphertext, ciphertext_len, plaintext, &plaintext_len, &rsa_params, &stat, &key));
+   ENSURE(stat == 1);
+   COMPARE_TESTVECTOR(plaintext, plaintext_len, msg, sizeof(msg) - 1, "rsa oaep roundtrip", 0);
+
+   rsa_free(&key);
+   return CRYPT_OK;
+}
+
 int rsa_test(void)
 {
    unsigned char in[1024], out[1024], tmp[3072];
@@ -564,6 +609,7 @@ int rsa_test(void)
    DO(s_rsa_cryptx_issue_69());
    DO(s_rsa_issue_301(prng_idx));
    DO(s_rsa_public_ubin_e(prng_idx));
+   DO(s_rsa_oaep_small_ciphertext_test(prng_idx));
 
    /* make 10 random key */
    for (cnt = 0; cnt < 10; cnt++) {

tests/x25519_test.c

@@ -218,6 +218,30 @@ static int s_x25519_compat_test(void)
    return CRYPT_OK;
 }
 
+static int s_x25519_zero_shared_secret_test(void)
+{
+   curve25519_key alice, bob, zero_pub;
+   unsigned char shared[32], zeros[32];
+   unsigned long len;
+   int prng_idx = find_prng("yarrow");
+
+   zeromem(zeros, sizeof(zeros));
+
+   DO(x25519_make_key(&yarrow_prng, prng_idx, &alice));
+   DO(x25519_make_key(&yarrow_prng, prng_idx, &bob));
+
+   len = sizeof(shared);
+   DO(x25519_shared_secret(&alice, &bob, shared, &len));
+   ENSURE(len == sizeof(shared));
+
+   DO(x25519_import_raw(zeros, sizeof(zeros), PK_PUBLIC, &zero_pub));
+
+   len = sizeof(shared);
+   SHOULD_FAIL_WITH(x25519_shared_secret(&alice, &zero_pub, shared, &len), CRYPT_INVALID_PACKET);
+
+   return CRYPT_OK;
+}
+
 /**
   Test the x25519 system
   @return CRYPT_OK if successful
@@ -226,6 +250,7 @@ int x25519_test(void)
 {
    DO(s_rfc_7748_5_2_test());
    DO(s_rfc_7748_6_test());
+   DO(s_x25519_zero_shared_secret_test());
    return CRYPT_OK;
 }
 

tests/x448_test.c

@@ -227,12 +227,37 @@ static int s_x448_compat_test(void)
    return CRYPT_OK;
 }
 
+static int s_x448_zero_shared_secret_test(void)
+{
+   curve448_key alice, bob, zero_pub;
+   unsigned char shared[56], zeros[56];
+   unsigned long len;
+   int prng_idx = find_prng("yarrow");
+
+   zeromem(zeros, sizeof(zeros));
+
+   DO(x448_make_key(&yarrow_prng, prng_idx, &alice));
+   DO(x448_make_key(&yarrow_prng, prng_idx, &bob));
+
+   len = sizeof(shared);
+   DO(x448_shared_secret(&alice, &bob, shared, &len));
+   ENSURE(len == sizeof(shared));
+
+   DO(x448_import_raw(zeros, sizeof(zeros), PK_PUBLIC, &zero_pub));
+
+   len = sizeof(shared);
+   SHOULD_FAIL_WITH(x448_shared_secret(&alice, &zero_pub, shared, &len), CRYPT_INVALID_PACKET);
+
+   return CRYPT_OK;
+}
+
 int x448_test(void)
 {
    DO(s_x448_rfc7748_scalarmult_test());
    DO(s_x448_rfc7748_iter_test());
    DO(s_x448_keygen_dh_test());
    DO(s_x448_wycheproof_special_test());
+   DO(s_x448_zero_shared_secret_test());
    return CRYPT_OK;
 }