Skip to content

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
noiser7 wants to merge 15 commits into
base: main
Choose a base branch
from
Open

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
36c5c16
Add OAEP parameter passing
Dec 24, 2025
5bd133f
Add Asymmetric encrypt/decrypt parameter
Dec 24, 2025
73e5fe2
Refactor RSA/ add OAEP encrypt/decrypt parameters
Dec 24, 2025
31b359a
Fix RSA encrypt-decrypt tests
Dec 24, 2025
9edab6c
Fix memory leaks and messages
Dec 25, 2025
9a03665
Add BuildRSAOAEPParam fuction
Dec 25, 2025
eb02aa2
Add OpenSSL 1 support
Dec 26, 2025
bb16236
Remove legacy code
Dec 26, 2025
69d3118
Fix RSA_set_method call
Dec 26, 2025
6c1e1d0
Add PSS salt length setting in VerifyFinal
Dec 26, 2025
530cba7
Add salt length setting in one-stage verify
Dec 26, 2025
65c0423
Add OAEP encrypt-decrypt parameters for Botan
Dec 30, 2025
dd25216
Add asymmetric encrypt-decrypt parameters passing
Dec 30, 2025
2f850fc
Fix sourceData pointer check
Dec 30, 2025
51450ed
Fix maximum length validation for RSA encryption
Dec 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9,221 changes: 4,926 additions & 4,295 deletions src/lib/SoftHSM.cpp
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion src/lib/SoftHSM.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,9 @@ class SoftHSM

CK_RV MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism);
CK_RV MechParamCheckRSAAESKEYWRAP(CK_MECHANISM_PTR pMechanism);

CK_RV BuildRSAOAEPParam(const CK_RSA_PKCS_OAEP_PARAMS* par,
void** parameters,size_t* paramLen,
size_t* hashLen = NULL);
bool isMechanismPermitted(OSObject* key, CK_MECHANISM_TYPE mechanism);
void prepareSupportedMechanisms(std::map<std::string, CK_MECHANISM_TYPE> &t);
bool detectFork(void);
Expand Down
10 changes: 6 additions & 4 deletions src/lib/crypto/AsymmetricAlgorithm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,22 @@ bool AsymmetricAlgorithm::isWrappingMech(AsymMech::Type padding)
}

// Wrap/Unwrap keys
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding)
bool AsymmetricAlgorithm::wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param, size_t paramLen)
{
if (!isWrappingMech(padding))
return false;

return encrypt(publicKey, data, encryptedData, padding);
return encrypt(publicKey, data, encryptedData, padding, param, paramLen);
}

bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding)
bool AsymmetricAlgorithm::unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param, size_t paramLen)
{
if (!isWrappingMech(padding))
return false;

return decrypt(privateKey, encryptedData, data, padding);
return decrypt(privateKey, encryptedData, data, padding, param, paramLen);
}


Expand Down
21 changes: 16 additions & 5 deletions src/lib/crypto/AsymmetricAlgorithm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,14 @@ struct RSA_PKCS_PSS_PARAMS
size_t sLen;
};

struct RSA_PKCS_OAEP_PARAMS
{
HashAlgo::Type hashAlg;
AsymRSAMGF::Type mgf;
void *sourceData;
size_t sourceDataLen;
};

class AsymmetricAlgorithm
{
public:
Expand All @@ -138,14 +146,17 @@ class AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding) = 0;

virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0) = 0;
// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding) = 0;
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0) = 0;

// Wrap/Unwrap keys
bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
bool unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
bool wrapKey(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, size_t paramLen = 0);
bool unwrapKey(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL) = 0;
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanDH.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ bool BotanDH::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool BotanDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("DH does not support encryption");

Expand All @@ -102,7 +103,8 @@ bool BotanDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("DH does not support decryption");

Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanDH.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,12 @@ class BotanDH : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,8 @@ bool BotanDSA::verifyFinal(const ByteString& signature)

// Encryption functions
bool BotanDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("DSA does not support encryption");

Expand All @@ -494,7 +495,8 @@ bool BotanDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /*param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("DSA does not support decryption");

Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@ class BotanDSA : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanECDH.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,8 @@ bool BotanECDH::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool BotanECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("ECDH does not support encryption");

Expand All @@ -103,7 +104,8 @@ bool BotanECDH::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanECDH::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("ECDH does not support decryption");

Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanECDH.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,12 @@ class BotanECDH : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanECDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,8 @@ bool BotanECDSA::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool BotanECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("ECDSA does not support encryption");

Expand All @@ -330,7 +331,8 @@ bool BotanECDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanECDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("ECDSA does not support decryption");

Expand Down
42 changes: 22 additions & 20 deletions src/lib/crypto/BotanECDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,38 +47,40 @@ class BotanECDSA : public AsymmetricAlgorithm
virtual ~BotanECDSA();

// Signing functions
virtual bool sign(PrivateKey* privateKey, const ByteString& dataToSign, ByteString& signature, const AsymMech::Type mechanism, const void* param = NULL, const size_t paramLen = 0);
virtual bool signInit(PrivateKey* privateKey, const AsymMech::Type mechanism, const void* param = NULL, const size_t paramLen = 0);
virtual bool signUpdate(const ByteString& dataToSign);
virtual bool signFinal(ByteString& signature);
virtual bool sign(PrivateKey *privateKey, const ByteString &dataToSign, ByteString &signature, const AsymMech::Type mechanism, const void *param = NULL, const size_t paramLen = 0);
virtual bool signInit(PrivateKey *privateKey, const AsymMech::Type mechanism, const void *param = NULL, const size_t paramLen = 0);
virtual bool signUpdate(const ByteString &dataToSign);
virtual bool signFinal(ByteString &signature);

// Verification functions
virtual bool verify(PublicKey* publicKey, const ByteString& originalData, const ByteString& signature, const AsymMech::Type mechanism, const void* param = NULL, const size_t paramLen = 0);
virtual bool verifyInit(PublicKey* publicKey, const AsymMech::Type mechanism, const void* param = NULL, const size_t paramLen = 0);
virtual bool verifyUpdate(const ByteString& originalData);
virtual bool verifyFinal(const ByteString& signature);
virtual bool verify(PublicKey *publicKey, const ByteString &originalData, const ByteString &signature, const AsymMech::Type mechanism, const void *param = NULL, const size_t paramLen = 0);
virtual bool verifyInit(PublicKey *publicKey, const AsymMech::Type mechanism, const void *param = NULL, const size_t paramLen = 0);
virtual bool verifyUpdate(const ByteString &originalData);
virtual bool verifyFinal(const ByteString &signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey *publicKey, const ByteString &data, ByteString &encryptedData, const AsymMech::Type padding,
const void *param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey *privateKey, const ByteString &encryptedData, ByteString &data, const AsymMech::Type padding,
const void *param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
virtual bool generateKeyPair(AsymmetricKeyPair **ppKeyPair, AsymmetricParameters *parameters, RNG *rng = NULL);
virtual unsigned long getMinKeySize();
virtual unsigned long getMaxKeySize();
virtual bool reconstructKeyPair(AsymmetricKeyPair** ppKeyPair, ByteString& serialisedData);
virtual bool reconstructPublicKey(PublicKey** ppPublicKey, ByteString& serialisedData);
virtual bool reconstructPrivateKey(PrivateKey** ppPrivateKey, ByteString& serialisedData);
virtual bool reconstructParameters(AsymmetricParameters** ppParams, ByteString& serialisedData);
virtual PublicKey* newPublicKey();
virtual PrivateKey* newPrivateKey();
virtual AsymmetricParameters* newParameters();
virtual bool reconstructKeyPair(AsymmetricKeyPair **ppKeyPair, ByteString &serialisedData);
virtual bool reconstructPublicKey(PublicKey **ppPublicKey, ByteString &serialisedData);
virtual bool reconstructPrivateKey(PrivateKey **ppPrivateKey, ByteString &serialisedData);
virtual bool reconstructParameters(AsymmetricParameters **ppParams, ByteString &serialisedData);
virtual PublicKey *newPublicKey();
virtual PrivateKey *newPrivateKey();
virtual AsymmetricParameters *newParameters();

private:
Botan::PK_Signer* signer;
Botan::PK_Verifier* verifier;
Botan::PK_Signer *signer;
Botan::PK_Verifier *verifier;
};

#endif // !_SOFTHSM_V2_BOTANECDSA_H
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanEDDSA.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,8 @@ bool BotanEDDSA::verifyFinal(const ByteString& /*signature*/)

// Encryption functions
bool BotanEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("EDDSA does not support encryption");

Expand All @@ -265,7 +266,8 @@ bool BotanEDDSA::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanEDDSA::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("EDDSA does not support decryption");

Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanEDDSA.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,12 @@ class BotanEDDSA : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanGOST.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,8 @@ bool BotanGOST::verifyFinal(const ByteString& signature)

// Encryption functions
bool BotanGOST::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/)
ByteString& /*encryptedData*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("GOST does not support encryption");

Expand All @@ -335,7 +336,8 @@ bool BotanGOST::encrypt(PublicKey* /*publicKey*/, const ByteString& /*data*/,

// Decryption functions
bool BotanGOST::decrypt(PrivateKey* /*privateKey*/, const ByteString& /*encryptedData*/,
ByteString& /*data*/, const AsymMech::Type /*padding*/)
ByteString& /*data*/, const AsymMech::Type /*padding*/,
const void* /* param = NULL */, const size_t /* paramLen = 0 */)
{
ERROR_MSG("GOST does not support decryption");

Expand Down
6 changes: 4 additions & 2 deletions src/lib/crypto/BotanGOST.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,12 @@ class BotanGOST : public AsymmetricAlgorithm
virtual bool verifyFinal(const ByteString& signature);

// Encryption functions
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding);
virtual bool encrypt(PublicKey* publicKey, const ByteString& data, ByteString& encryptedData, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Decryption functions
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding);
virtual bool decrypt(PrivateKey* privateKey, const ByteString& encryptedData, ByteString& data, const AsymMech::Type padding,
const void* param = NULL, const size_t paramLen = 0);

// Key factory
virtual bool generateKeyPair(AsymmetricKeyPair** ppKeyPair, AsymmetricParameters* parameters, RNG* rng = NULL);
Expand Down
Loading