Skip to content

Add support for missing OAEP algorithms #869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luxifer wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Add support for missing OAEP algorithms #869

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
79d7439
implement OAEP encrypt/decrypt with more hashing algs
luxifer Apr 28, 2026
732f541
set session parameters with OAEP param for encryption
luxifer Apr 28, 2026
37a7a50
export compile commands, useful for clangd
luxifer Apr 28, 2026
fa26a5c
add decrypt and check for all hashing algs
luxifer Apr 29, 2026
8a508e3
move compile commands cmake flag to the top
luxifer Apr 29, 2026
346bee8
alloc whole oaep params instead of only subfield
luxifer Apr 29, 2026
d2735a9
remove unused parameter
luxifer Apr 29, 2026
ec56f58
add missing parameters to asym decrypt
luxifer Apr 29, 2026
7d937ce
add checks on OAEP source data length before allocating memory
luxifer Apr 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ Botan-*
ROOT
*.cmake
CMakeFiles
compile_commands.json

# Specifics
softhsm2.module
Expand Down
3 changes: 3 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,3 +146,6 @@ set(CPACK_SOURCE_GENERATOR "TGZ")
set(CPACK_SOURCE_IGNORE_FILES "build/*;\.git/*")

include(CPack)

# Compiler commands
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
77 changes: 47 additions & 30 deletions src/lib/SoftHSM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
The implementation of the SoftHSM's main class
*****************************************************************************/

#include "HashAlgorithm.h"
#include "config.h"
#include "log.h"
#include "access.h"
Expand Down Expand Up @@ -2521,6 +2522,11 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec

// Get the asymmetric algorithm matching the mechanism
AsymMech::Type mechanism;
void* param = NULL;
size_t paramLen = 0;
MechanismParam* mechanismParam = NULL;
CK_RSA_PKCS_OAEP_PARAMS_PTR pOaepParams;
RSA_PKCS_OAEP_PARAMS oaepParam;
bool isRSA = false;
switch(pMechanism->mechanism) {
case CKM_RSA_PKCS:
Expand All @@ -2538,11 +2544,38 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
case CKM_RSA_PKCS_OAEP:
if (keyType != CKK_RSA)
return CKR_KEY_TYPE_INCONSISTENT;

rv = MechParamCheckRSAPKCSOAEP(pMechanism);
if (rv != CKR_OK)
return rv;

mechanism = AsymMech::RSA_PKCS_OAEP;
unsigned long allowedMgf;
pOaepParams = CK_RSA_PKCS_OAEP_PARAMS_PTR(pMechanism->pParameter);

switch (pOaepParams->hashAlg) {
case CKM_SHA_1:
oaepParam.hashAlg = HashAlgo::SHA1;
oaepParam.mgf = AsymRSAMGF::MGF1_SHA1;
allowedMgf = CKG_MGF1_SHA1;
break;
}

if (pOaepParams->mgf != allowedMgf) {
ERROR_MSG("Hash and MGF don't match");
return CKR_ARGUMENTS_BAD;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

oaepParam.pSourceData = malloc(pOaepParams->ulSourceDataLen);
if (oaepParam.pSourceData == NULL)
{
return CKR_HOST_MEMORY;
}
memcpy((void*)oaepParam.pSourceData, pOaepParams->pSourceData, pOaepParams->ulSourceDataLen);
oaepParam.ulSourceDataLen = pOaepParams->ulSourceDataLen;

param = &oaepParam;
paramLen = sizeof(oaepParam);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
isRSA = true;
break;
default:
Expand Down Expand Up @@ -2578,6 +2611,7 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
session->setOpType(SESSION_OP_ENCRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setParameters(param, paramLen);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPublicKey(publicKey);
Expand Down Expand Up @@ -2678,6 +2712,9 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
AsymmetricAlgorithm* asymCrypto = session->getAsymmetricCryptoOp();
AsymMech::Type mechanism = session->getMechanism();
PublicKey* publicKey = session->getPublicKey();
size_t paramLen;
void* param = session->getParameters(paramLen);
MechanismParam* mechanismParam = session->getMechanismParam();
if (asymCrypto == NULL || !session->getAllowSinglePartOp() || publicKey == NULL)
{
session->resetOp();
Expand Down Expand Up @@ -2712,7 +2749,7 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
data += ByteString(pData, ulDataLen);

// Encrypt the data
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism))
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism,param, paramLen, mechanismParam))
{
session->resetOp();
return CKR_GENERAL_ERROR;
Expand Down Expand Up @@ -7203,7 +7240,7 @@ CK_RV SoftHSM::UnwrapKeySym
SymWrap::Type mode = SymWrap::Unknown;
size_t bb = 8;
size_t blocksize = 0;

switch(pMechanism->mechanism) {
#ifdef HAVE_AES_KEY_WRAP
case CKM_AES_KEY_WRAP:
Expand Down Expand Up @@ -7249,14 +7286,14 @@ CK_RV SoftHSM::UnwrapKeySym
ByteString iv;
ByteString decryptedFinal;
CK_RV rv = CKR_OK;

switch(pMechanism->mechanism) {

case CKM_AES_CBC_PAD:
case CKM_DES3_CBC_PAD:
iv.resize(blocksize);
memcpy(&iv[0], pMechanism->pParameter, blocksize);

if (!cipher->decryptInit(unwrappingkey, SymMode::CBC, iv, false))
{
cipher->recycleKey(unwrappingkey);
Expand Down Expand Up @@ -7285,7 +7322,7 @@ CK_RV SoftHSM::UnwrapKeySym
return CKR_GENERAL_ERROR; // TODO should be another error
}
break;

default:
// Unwrap the key
rv = CKR_OK;
Expand Down Expand Up @@ -7576,7 +7613,7 @@ CK_RV SoftHSM::C_UnwrapKey
pMechanism->ulParameterLen != 8)
return CKR_ARGUMENTS_BAD;
break;

default:
return CKR_MECHANISM_INVALID;
}
Expand Down Expand Up @@ -7620,7 +7657,7 @@ CK_RV SoftHSM::C_UnwrapKey
if (pMechanism->mechanism == CKM_DES3_CBC && (unwrapKey->getUnsignedLongValue(CKA_KEY_TYPE, CKK_VENDOR_DEFINED) != CKK_DES2 ||
unwrapKey->getUnsignedLongValue(CKA_KEY_TYPE, CKK_VENDOR_DEFINED) != CKK_DES3))
return CKR_WRAPPING_KEY_TYPE_INCONSISTENT;

// Check if the unwrapping key can be used for unwrapping
if (unwrapKey->getBooleanValue(CKA_UNWRAP, false) == false)
return CKR_KEY_FUNCTION_NOT_PERMITTED;
Expand Down Expand Up @@ -8431,11 +8468,11 @@ CK_RV SoftHSM::generateAES
if (rv == CKR_OK)
{
OSObject* osobject = (OSObject*)handleManager->getObject(*phKey);
if (osobject == NULL_PTR || !osobject->isValid())
if (osobject == NULL_PTR || !osobject->isValid())
{
rv = CKR_FUNCTION_FAILED;
}
else if (osobject->startTransaction())
}
else if (osobject->startTransaction())
{
bool bOK = true;

Expand Down Expand Up @@ -13721,31 +13758,11 @@ CK_RV SoftHSM::MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism)
}

CK_RSA_PKCS_OAEP_PARAMS_PTR params = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter;
if (params->hashAlg != CKM_SHA_1)
{
ERROR_MSG("hashAlg must be CKM_SHA_1");
return CKR_ARGUMENTS_BAD;
}
if (params->mgf != CKG_MGF1_SHA1)
{
ERROR_MSG("mgf must be CKG_MGF1_SHA1");
return CKR_ARGUMENTS_BAD;
}
if (params->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if (params->pSourceData != NULL)
{
ERROR_MSG("pSourceData must be NULL");
return CKR_ARGUMENTS_BAD;
}
if (params->ulSourceDataLen != 0)
{
ERROR_MSG("ulSourceDataLen must be 0");
return CKR_ARGUMENTS_BAD;
}
return CKR_OK;
}

Expand Down
9 changes: 4 additions & 5 deletions src/lib/crypto/AsymmetricAlgorithm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,20 +158,20 @@ bool AsymmetricAlgorithm::isWrappingMech(AsymMech::Type padding)
}

// Wrap/Unwrap keys
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding)
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const void* param, const size_t paramLen, const MechanismParam* mechanismParam)
{
if (!isWrappingMech(padding))
return false;

return encrypt(publicKey, data, encryptedData, padding);
return encrypt(publicKey, data, encryptedData, padding, param, paramLen, mechanismParam);
}

bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding)
bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const void* param, const size_t paramLen, const MechanismParam* mechanismParam)
{
if (!isWrappingMech(padding))
return false;

return decrypt(privateKey, encryptedData, data, padding);
return decrypt(privateKey, encryptedData, data, padding, param, paramLen, mechanismParam);
}


Expand Down Expand Up @@ -220,4 +220,3 @@ void AsymmetricAlgorithm::recycleSymmetricKey(SymmetricKey* toRecycle)
{
delete toRecycle;
}

27 changes: 22 additions & 5 deletions src/lib/crypto/AsymmetricAlgorithm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,24 @@ struct RSA_PKCS_PSS_PARAMS
size_t sLen;
};

struct AsymOAEPSource
{
enum Type
{
Unknown,
DATA_SPECIFIED,
};
};

struct RSA_PKCS_OAEP_PARAMS
{
HashAlgo::Type hashAlg;
AsymRSAMGF::Type mgf;
AsymOAEPSource::Type source;
void *pSourceData;
size_t ulSourceDataLen;
};

class AsymmetricAlgorithm
{
public:
Expand All @@ -144,14 +162,14 @@ class AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding) = 0;
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const void* param = NULL, const size_t paramLen = 0, const MechanismParam* mechanismParam = NULL);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding) = 0;
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const void* param = NULL, const size_t paramLen = 0, const MechanismParam* mechanismParam = NULL);

// Wrap/Unwrap keys
bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
bool unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding, const void* param = NULL, const size_t paramLen = 0, const MechanismParam* mechanismParam = NULL);
bool unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding, const void* param = NULL, const size_t paramLen = 0, const MechanismParam* mechanismParam = NULL);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL) = 0;
Expand Down Expand Up @@ -194,4 +212,3 @@ class AsymmetricAlgorithm
};

#endif // !_SOFTHSM_V2_ASYMMETRICALGORITHM_H

Loading