Skip to content

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
noiser7 wants to merge 42 commits into
base: main
Choose a base branch
from
Open

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
36c5c16
Add OAEP parameter passing
Dec 24, 2025
5bd133f
Add Asymmetric encrypt/decrypt parameter
Dec 24, 2025
73e5fe2
Refactor RSA/ add OAEP encrypt/decrypt parameters
Dec 24, 2025
31b359a
Fix RSA encrypt-decrypt tests
Dec 24, 2025
9edab6c
Fix memory leaks and messages
Dec 25, 2025
9a03665
Add BuildRSAOAEPParam fuction
Dec 25, 2025
eb02aa2
Add OpenSSL 1 support
Dec 26, 2025
bb16236
Remove legacy code
Dec 26, 2025
69d3118
Fix RSA_set_method call
Dec 26, 2025
6c1e1d0
Add PSS salt length setting in VerifyFinal
Dec 26, 2025
530cba7
Add salt length setting in one-stage verify
Dec 26, 2025
65c0423
Add OAEP encrypt-decrypt parameters for Botan
Dec 30, 2025
dd25216
Add asymmetric encrypt-decrypt parameters passing
Dec 30, 2025
2f850fc
Fix sourceData pointer check
Dec 30, 2025
51450ed
Fix maximum length validation for RSA encryption
Dec 30, 2025
49df263
Revert codestyle changes
Jan 29, 2026
477e16d
Revert code style changes
Jan 29, 2026
e3f93b7
Merge branch 'main' into Refactor-RSA/Add-modern-hash-on-OAEP
noiser7 Jan 29, 2026
dfddb57
fix pointer calculation
Jan 29, 2026
80f9daa
Fix memory leak on *parameters
Feb 2, 2026
deadbc1
Fix OSSL_PARAM_BLD_to_param calling on error
Feb 2, 2026
139e2c2
Fix inRSA NULL check before dereference
Feb 2, 2026
c7bbcf7
Fix bug wth pointer check
Feb 2, 2026
f85d283
Fix maximum key length checking
Feb 2, 2026
934ce4d
Check OAEP label length to prevent buffer overflow
Feb 2, 2026
65ec656
Check input parameter in setFromOSSL
Feb 2, 2026
bc008af
Fix BN free if RSA_new fails
Feb 3, 2026
1574ae4
Text format changed
Mar 10, 2026
8c8f9a3
Text format changed, debug artifacts removed
Mar 10, 2026
254c779
Merge branch 'main' into Refactor-RSA/Add-modern-hash-on-OAEP
noiser7 Mar 17, 2026
7153b77
Fix OSSLMLDSA encrypt/decrypt
Mar 17, 2026
9a8fd9d
Fixed NITs
Mar 25, 2026
b1173b3
Refactor RSA OAEP to *MechanismParam parameters tranfer
Mar 26, 2026
ebb28c2
Refactor RSA OAEP to *MechanismParam parameters tranfer for Botan
Apr 3, 2026
dc65043
Delete RSA_PKCS_OAEP_PARAM
Apr 14, 2026
0b1563d
Refactor asymmetric signing parameter pass from void pointer to Mecha…
Apr 15, 2026
f7c5a78
Fix .gitignore
Apr 16, 2026
a09d289
Merge branch 'Refacor-RSA/refactor-RSA-PSS-signing-to-RSAMechanismPar…
Apr 16, 2026
854de26
Fix BotanDH
Apr 16, 2026
08d48c1
Merge branch 'Refacor-RSA/refactor-RSA-PSS-signing-to-RSAMechanismPar…
Apr 16, 2026
ee2c9e9
Add unit test for check label usage with RSA encryption
Apr 16, 2026
83103b2
Fix variable type for 64-bit
Apr 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 172 additions & 43 deletions src/lib/SoftHSM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2514,16 +2514,34 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
}
else
{
return CKR_MECHANISM_INVALID;
}
return CKR_MECHANISM_INVALID;
}
// set mechanism parameters
void *parameters = NULL;
size_t paramLen = 0;
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
&parameters, &paramLen);
if (rv != CKR_OK)
{
asymCrypto->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(asymCrypto);
return rv;
}
}

session->setOpType(SESSION_OP_ENCRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPublicKey(publicKey);

if (parameters != NULL)
{
session->setParameters(parameters, paramLen);
free(parameters);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return CKR_OK;
}

Expand Down Expand Up @@ -2620,6 +2638,8 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
AsymmetricAlgorithm* asymCrypto = session->getAsymmetricCryptoOp();
AsymMech::Type mechanism = session->getMechanism();
PublicKey* publicKey = session->getPublicKey();
size_t paramLen = 0;
void* parameters = session->getParameters(paramLen);
if (asymCrypto == NULL || !session->getAllowSinglePartOp() || publicKey == NULL)
{
session->resetOp();
Expand Down Expand Up @@ -2653,8 +2673,8 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen

data += ByteString(pData, ulDataLen);

// Encrypt the data
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism))
// Encrypt the data
Comment thread
noiser7 marked this conversation as resolved.
Outdated
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism, parameters, paramLen))
{
session->resetOp();
return CKR_GENERAL_ERROR;
Expand Down Expand Up @@ -3256,14 +3276,31 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
{
session->setReAuthentication(true);
}

// set mechanism parameters
void *parameters = NULL;
size_t paramLen = 0;
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
&parameters, &paramLen);
if (rv != CKR_OK)
{
asymCrypto->recyclePrivateKey(privateKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(asymCrypto);
return rv;
}
}
session->setOpType(SESSION_OP_DECRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPrivateKey(privateKey);

if (parameters != NULL)
{
session->setParameters(parameters,paramLen);
free(parameters);
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return CKR_OK;
}

Expand Down Expand Up @@ -3352,6 +3389,8 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG
AsymmetricAlgorithm* asymCrypto = session->getAsymmetricCryptoOp();
AsymMech::Type mechanism = session->getMechanism();
PrivateKey* privateKey = session->getPrivateKey();
size_t paramLen = 0;
void* parameters = session->getParameters(paramLen);
if (asymCrypto == NULL || !session->getAllowSinglePartOp() || privateKey == NULL)
{
session->resetOp();
Expand Down Expand Up @@ -3385,7 +3424,7 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG
ByteString data;

// Decrypt the data
if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism))
if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism,parameters,paramLen))
{
session->resetOp();
return CKR_ENCRYPTED_DATA_INVALID;
Expand Down Expand Up @@ -6438,9 +6477,13 @@ CK_RV SoftHSM::WrapKeyAsym
ByteString& wrapped
)
{
CK_RV rv = CKR_OK;
const size_t bb = 8;
AsymAlgo::Type algo = AsymAlgo::Unknown;
AsymMech::Type mech = AsymMech::Unknown;
void *parameters = NULL;
size_t paramLen = 0;
size_t hashLen = 0;

CK_ULONG modulus_length;
switch(pMechanism->mechanism) {
Expand Down Expand Up @@ -6470,7 +6513,7 @@ CK_RV SoftHSM::WrapKeyAsym
mech = AsymMech::RSA_PKCS_OAEP;
// SHA-1 is the only supported option
// PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen
if (keydata.size() > modulus_length - 2 - 2 * 160 / 8)
if (keydata.size() > modulus_length - 2 - (2 * hashLen))
return CKR_KEY_SIZE_RANGE;
break;

Expand Down Expand Up @@ -6500,20 +6543,29 @@ CK_RV SoftHSM::WrapKeyAsym
break;

default:
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_MECHANISM_INVALID;
}
// Wrap the key
if (!cipher->wrapKey(publicKey, keydata, wrapped, mech))
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_GENERAL_ERROR;
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
&parameters,&paramLen,&hashLen);
if (rv != CKR_OK)
{
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
}
}

// Wrap the key
if (!cipher->wrapKey(publicKey, keydata, wrapped, mech,parameters,paramLen))
rv = CKR_GENERAL_ERROR;
free(parameters);
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);

return CKR_OK;
return rv;
}

// Internal: Wrap with mechanism RSA_AES_KEY_WRAP
Expand Down Expand Up @@ -7029,6 +7081,7 @@ CK_RV SoftHSM::UnwrapKeyAsym
// Get the symmetric algorithm matching the mechanism
AsymAlgo::Type algo = AsymAlgo::Unknown;
AsymMech::Type mode = AsymMech::Unknown;

switch(pMechanism->mechanism) {
case CKM_RSA_PKCS:
algo = AsymAlgo::RSA;
Expand Down Expand Up @@ -7065,12 +7118,28 @@ CK_RV SoftHSM::UnwrapKeyAsym
break;

default:
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_MECHANISM_INVALID;
}

void *parameters = NULL;
size_t paramLen = 0;
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
&parameters,&paramLen);
if (rv != CKR_OK)
{
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
}
}
// Unwrap the key
if (!cipher->unwrapKey(unwrappingkey, wrapped, keydata, mode))
if (!cipher->unwrapKey(unwrappingkey, wrapped, keydata, mode, parameters, paramLen ))
rv = CKR_GENERAL_ERROR;
free(parameters);
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
Expand Down Expand Up @@ -13166,31 +13235,16 @@ CK_RV SoftHSM::MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism)
ERROR_MSG("pParameter must be of type CK_RSA_PKCS_OAEP_PARAMS");
return CKR_ARGUMENTS_BAD;
}

CK_RSA_PKCS_OAEP_PARAMS_PTR params = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter;
if (params->hashAlg != CKM_SHA_1)
{
ERROR_MSG("hashAlg must be CKM_SHA_1");
return CKR_ARGUMENTS_BAD;
}
if (params->mgf != CKG_MGF1_SHA1)
{
ERROR_MSG("mgf must be CKG_MGF1_SHA1");
return CKR_ARGUMENTS_BAD;
}

if (params->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if (params->pSourceData != NULL)
{
ERROR_MSG("pSourceData must be NULL");
return CKR_ARGUMENTS_BAD;
}
if (params->ulSourceDataLen != 0)
if ((params-> pSourceData == NULL)&&(params->ulSourceDataLen != 0))
{
ERROR_MSG("ulSourceDataLen must be 0");
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
return CKR_OK;
Expand Down Expand Up @@ -13222,27 +13276,102 @@ CK_RV SoftHSM::MechParamCheckRSAAESKEYWRAP(CK_MECHANISM_PTR pMechanism)
ERROR_MSG("pOAEPParams must be of type CK_RSA_PKCS_OAEP_PARAMS");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->mgf < 1UL || params->pOAEPParams->mgf > 5UL)
if (params->pOAEPParams->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("mgf not supported");
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->source != CKZ_DATA_SPECIFIED)
if ((params->pOAEPParams->pSourceData == NULL) && (params->pOAEPParams->ulSourceDataLen != 0))
{
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
return CKR_OK;
}

CK_RV SoftHSM::BuildRSAOAEPParam(const CK_RSA_PKCS_OAEP_PARAMS *params,
void **parameters,
size_t *paramLen,
size_t *hashLen)
{
RSA_PKCS_OAEP_PARAMS oaep_param;
if (params == NULL)
{
ERROR_MSG("parameters is NULL for RSA OAEP encryption");
return CKR_ARGUMENTS_BAD;
}
if (params->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if ((params->pSourceData == NULL) && (params->ulSourceDataLen != 0))
{
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->pSourceData != NULL)
switch (params->hashAlg)
{
ERROR_MSG("pSourceData must be NULL");
case CKM_SHA_1:
oaep_param.hashAlg = HashAlgo::SHA1;
if (hashLen) *hashLen = 20;
break;
case CKM_SHA224:
oaep_param.hashAlg = HashAlgo::SHA224;
if (hashLen) *hashLen = 28;
break;
case CKM_SHA256:
oaep_param.hashAlg = HashAlgo::SHA256;
if (hashLen) *hashLen = 32;
break;
case CKM_SHA384:
oaep_param.hashAlg = HashAlgo::SHA384;
if (hashLen) *hashLen = 48;
break;
case CKM_SHA512:
oaep_param.hashAlg = HashAlgo::SHA512;
if (hashLen) *hashLen = 64;
break;
default:
ERROR_MSG("hash algorithm not supported for OAEP");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->ulSourceDataLen != 0)
switch (params->mgf)
{
ERROR_MSG("ulSourceDataLen must be 0");
case CKG_MGF1_SHA1:
oaep_param.mgf = AsymRSAMGF::MGF1_SHA1;
break;
case CKG_MGF1_SHA224:
oaep_param.mgf = AsymRSAMGF::MGF1_SHA224;
break;
case CKG_MGF1_SHA256:
oaep_param.mgf = AsymRSAMGF::MGF1_SHA256;
break;
case CKG_MGF1_SHA384:
oaep_param.mgf = AsymRSAMGF::MGF1_SHA384;
break;
case CKG_MGF1_SHA512:
oaep_param.mgf = AsymRSAMGF::MGF1_SHA512;
break;
default:
ERROR_MSG("mgf algorithm not supported for OAEP");
return CKR_ARGUMENTS_BAD;
}

// need copy parameters to session context
// label source data will be copyed to end of parameter block
size_t bufLen = sizeof(RSA_PKCS_OAEP_PARAMS) + params->ulSourceDataLen;
void *paramBuf = malloc(bufLen);
if (paramBuf == NULL)
{
return CKR_HOST_MEMORY;
}
// copy label data to end of parameter block
oaep_param.sourceData = (char*)paramBuf + sizeof(RSA_PKCS_OAEP_PARAMS);
oaep_param.sourceDataLen = params->ulSourceDataLen;
memcpy(paramBuf, &oaep_param, sizeof(RSA_PKCS_OAEP_PARAMS));
memcpy(oaep_param.sourceData, params->pSourceData, params->ulSourceDataLen);
*parameters = paramBuf;
Comment thread
noiser7 marked this conversation as resolved.
Outdated
*paramLen = bufLen;
return CKR_OK;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

Expand Down
4 changes: 3 additions & 1 deletion src/lib/SoftHSM.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,9 @@ class SoftHSM

CK_RV MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism);
CK_RV MechParamCheckRSAAESKEYWRAP(CK_MECHANISM_PTR pMechanism);

CK_RV BuildRSAOAEPParam(const CK_RSA_PKCS_OAEP_PARAMS* par,
void** parameters,size_t* paramLen,
size_t* hashLen = NULL);
bool isMechanismPermitted(OSObject* key, CK_MECHANISM_TYPE mechanism);
void prepareSupportedMechanisms(std::map<std::string, CK_MECHANISM_TYPE> &t);
bool detectFork(void);
Expand Down
10 changes: 6 additions & 4 deletions src/lib/crypto/AsymmetricAlgorithm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,22 @@ bool AsymmetricAlgorithm::isWrappingMech(AsymMech::Type padding)
}

// Wrap/Unwrap keys
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding)
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param, size_t paramLen)
{
if (!isWrappingMech(padding))
return false;

return encrypt(publicKey, data, encryptedData, padding);
return encrypt(publicKey, data, encryptedData, padding, param, paramLen);
}

bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding)
bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param, size_t paramLen)
{
if (!isWrappingMech(padding))
return false;

return decrypt(privateKey, encryptedData, data, padding);
return decrypt(privateKey, encryptedData, data, padding, param, paramLen);
}


Expand Down
Loading