From 181d4cb5e62f57ddff45050ef86489adf7e09ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Trojnara?= Date: Tue, 18 Aug 2026 14:56:05 +0200 Subject: [PATCH 1/2] Scope RSA PKEY methods with context callbacks Stop registering libp11's RSA EVP_PKEY_METHOD process-wide when a private key is created. Global registration changes method selection for unrelated software and provider-backed RSA keys. Add a per-context callback for returned private EVP_PKEY objects and preserve it when UTIL_CTX recreates its libp11 context. Have the ENGINE use this hook to attach its method dispatch only to keys loaded by that ENGINE, keeping ENGINE integration out of the public libp11 implementation. --- src/eng_back.c | 24 ++++++++++++++++++- src/eng_front.c | 18 ++++---------- src/engine.h | 4 +++- src/libp11-int.h | 11 ++++----- src/libp11.exports | 1 + src/libp11.h | 31 ++++++++++++++++++++++++- src/p11_front.c | 33 ++++++++++++++++++++++++-- src/p11_load.c | 3 --- src/p11_rsa.c | 58 ++++------------------------------------------ src/util.h | 4 +++- src/util_uri.c | 23 +++++++++++++++++- 11 files changed, 128 insertions(+), 82 deletions(-) diff --git a/src/eng_back.c b/src/eng_back.c index feba3cf4..093498c8 100644 --- a/src/eng_back.c +++ b/src/eng_back.c @@ -3,7 +3,7 @@ * Copyright (c) 2002 Juha Yrjölä * Copyright (c) 2002 Olaf Kirch * Copyright (c) 2003 Kevin Stefanik - * Copyright (c) 2016-2025 Michał Trojnara + * Copyright (c) 2016-2026 Michał Trojnara * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -160,6 +160,28 @@ int ENGINE_CTX_finish(ENGINE_CTX *ctx) return 1; } +/* EVP_PKEY_set1_engine() is required for OpenSSL 1.1.x, + * but otherwise setting pkey->engine breaks OpenSSL 1.0.2 */ +#ifdef EVP_F_EVP_PKEY_SET1_ENGINE +static int set_pkey_engine(PKCS11_KEY *key, EVP_PKEY *pkey, void *user_data) +{ + (void)key; + return EVP_PKEY_set1_engine(pkey, user_data) ? 0 : -1; +} +#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */ + +int ENGINE_CTX_set_pkey_callback(ENGINE_CTX *ctx, ENGINE *engine) +{ +#ifdef EVP_F_EVP_PKEY_SET1_ENGINE + return UTIL_CTX_set_pkey_callback(ctx->util_ctx, + PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY, set_pkey_engine, engine); +#else + (void)ctx; + (void)engine; + return 1; +#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */ +} + /******************************************************************************/ /* Engine load public/private key */ /******************************************************************************/ diff --git a/src/eng_front.c b/src/eng_front.c index 39ab9f4d..7d5ad888 100644 --- a/src/eng_front.c +++ b/src/eng_front.c @@ -5,7 +5,7 @@ * Portions Copyright (c) 2003 Kevin Stefanik (kstef@mtppi.org) * Copied/modified by Kevin Stefanik (kstef@mtppi.org) for the OpenSC * project 2003. - * Copyright (c) 2016-2025 Michał Trojnara + * Copyright (c) 2016-2026 Michał Trojnara * * Licensed under the OpenSSL license (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -175,12 +175,13 @@ static EVP_PKEY *load_privkey(ENGINE *engine, const char *s_key_id, UI_METHOD *ui_method, void *ui_data) { ENGINE_CTX *ctx; - EVP_PKEY *pkey; ctx = ENGINE_CTX_get(engine); if (!ctx) return 0; - bind_helper_methods(engine); + if (!bind_helper_methods(engine) || + !ENGINE_CTX_set_pkey_callback(ctx, engine)) + return 0; #if OPENSSL_VERSION_NUMBER >= 0x30000000L /* * A workaround for an OpenSSL bug affecting the handling of foreign @@ -207,16 +208,7 @@ static EVP_PKEY *load_privkey(ENGINE *engine, const char *s_key_id, } } #endif - pkey = ENGINE_CTX_load_privkey(ctx, s_key_id, ui_method, ui_data); -#ifdef EVP_F_EVP_PKEY_SET1_ENGINE - /* EVP_PKEY_set1_engine() is required for OpenSSL 1.1.x, - * but otherwise setting pkey->engine breaks OpenSSL 1.0.2 */ - if (pkey && !EVP_PKEY_set1_engine(pkey, engine)) { - EVP_PKEY_free(pkey); - pkey = NULL; - } -#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */ - return pkey; + return ENGINE_CTX_load_privkey(ctx, s_key_id, ui_method, ui_data); } static int engine_ctrl(ENGINE *engine, int cmd, long i, void *p, void (*f) (void)) diff --git a/src/engine.h b/src/engine.h index 374dcea8..2e5923b4 100644 --- a/src/engine.h +++ b/src/engine.h @@ -2,7 +2,7 @@ * Copyright (c) 2001 Markus Friedl * Copyright (c) 2002 Juha Yrjölä * Copyright (c) 2003 Kevin Stefanik - * Copyright (c) 2016-2025 Michał Trojnara + * Copyright (c) 2016-2026 Michał Trojnara * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -84,6 +84,8 @@ int ENGINE_CTX_init(ENGINE_CTX *ctx); int ENGINE_CTX_finish(ENGINE_CTX *ctx); +int ENGINE_CTX_set_pkey_callback(ENGINE_CTX *ctx, ENGINE *engine); + int ENGINE_CTX_ctrl(ENGINE_CTX *ctx, int cmd, long i, void *p, void (*f)(void)); EVP_PKEY *ENGINE_CTX_load_pubkey(ENGINE_CTX *ctx, const char *s_key_id, diff --git a/src/libp11-int.h b/src/libp11-int.h index 4cf13574..45c60358 100644 --- a/src/libp11-int.h +++ b/src/libp11-int.h @@ -1,7 +1,7 @@ /* libp11, a simple layer on top of PKCS#11 API * Copyright (C) 2005 Olaf Kirch * Copyright (C) 2015-2025 Michał Trojnara - * Copyright © 2025 Mobi - Com Polska Sp. z o.o. + * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -63,6 +63,8 @@ typedef struct pkcs11_keys PKCS11_keys; typedef struct pkcs11_object_ops PKCS11_OBJECT_ops; typedef struct pkcs11_template_st PKCS11_TEMPLATE; +#define PKCS11_PKEY_CALLBACK_COUNT 2 + /* get private implementations of PKCS11 structures */ /* @@ -81,6 +83,8 @@ struct pkcs11_ctx_private { unsigned int forkid; int initialized; void (*vlog_a)(int, const char *, va_list); /* for the logging callback */ + PKCS11_PKEY_CALLBACK pkey_callbacks[PKCS11_PKEY_CALLBACK_COUNT]; + void *pkey_callback_data[PKCS11_PKEY_CALLBACK_COUNT]; }; struct pkcs11_keys { @@ -628,11 +632,6 @@ extern void pkcs11_ed_key_method_free(void); extern void pkcs11_xdh_key_method_free(void); #endif /* !defined(OPENSSL_NO_ECX) && OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */ -#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L -/* Free the global RSA EVP_PKEY_METHOD */ -extern void pkcs11_rsa_key_method_free(void); -# endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */ - #if OPENSSL_VERSION_NUMBER < 0x100020d0L || defined(LIBRESSL_VERSION_NUMBER) /* Get sign_init and sign callbacks from EVP_PKEY_METHOD */ extern void EVP_PKEY_meth_get_sign(EVP_PKEY_METHOD *pmeth, diff --git a/src/libp11.exports b/src/libp11.exports index a0834ed2..b87643f5 100644 --- a/src/libp11.exports +++ b/src/libp11.exports @@ -1,6 +1,7 @@ PKCS11_CTX_init_args PKCS11_CTX_new_ex PKCS11_CTX_new +PKCS11_CTX_set_pkey_callback PKCS11_CTX_load PKCS11_CTX_unload PKCS11_CTX_free diff --git a/src/libp11.h b/src/libp11.h index 08fd9ad4..cbe45c0d 100644 --- a/src/libp11.h +++ b/src/libp11.h @@ -1,6 +1,6 @@ /* libp11, a simple layer on top of PKCS#11 API * Copyright (C) 2005 Olaf Kirch - * Copyright © 2025 Mobi - Com Polska Sp. z o.o. + * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -171,6 +171,18 @@ struct PKCS11_kgen_attrs_st { /** PKCS11 ASCII logging callback */ typedef void (*PKCS11_VLOG_A_CB)(int, const char *, va_list); +/** + * Callback invoked for an EVP_PKEY returned by libp11 + * + * The key arguments are borrowed and must not be freed by the callback. + * The callback may modify the EVP_PKEY and must return 0 on success or -1 + * on error. + */ +typedef int (*PKCS11_PKEY_CALLBACK)(PKCS11_KEY *, EVP_PKEY *, void *); + +/** Callback type for PKCS11_get_private_key() */ +#define PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY 1 + /** * Create a new libp11 context with specified flags * @@ -187,6 +199,23 @@ extern PKCS11_CTX *PKCS11_CTX_new_ex(int flags); */ extern PKCS11_CTX *PKCS11_CTX_new(void); +/** + * Set a callback for EVP_PKEY objects returned by this context + * + * The callback and its user data must remain valid until they are replaced, + * unset, or the context is freed. Callback registration must not be changed + * concurrently with key retrieval. + * + * @param ctx context allocated by PKCS11_CTX_new() + * @param callback_type one of PKCS11_PKEY_CALLBACK_* types + * @param callback callback function, or NULL to unset it + * @param user_data opaque callback data + * @retval 0 success + * @retval -1 unsupported callback type or invalid context + */ +extern int PKCS11_CTX_set_pkey_callback(PKCS11_CTX *ctx, + int callback_type, PKCS11_PKEY_CALLBACK callback, void *user_data); + /** * Specify any private PKCS#11 module initialization args, if necessary * diff --git a/src/p11_front.c b/src/p11_front.c index 6cd6f235..084d69a0 100644 --- a/src/p11_front.c +++ b/src/p11_front.c @@ -1,6 +1,6 @@ /* libp11, a simple layer on top of PKCS#11 API * Copyright (C) 2016-2025 Michał Trojnara - * Copyright © 2025 Mobi - Com Polska Sp. z o.o. + * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -42,6 +42,22 @@ PKCS11_CTX *PKCS11_CTX_new(void) return pkcs11_CTX_new(0); } +int PKCS11_CTX_set_pkey_callback(PKCS11_CTX *pctx, + int callback_type, PKCS11_PKEY_CALLBACK callback, void *user_data) +{ + PKCS11_CTX_private *ctx; + + if (!pctx || callback_type <= 0 || + callback_type >= PKCS11_PKEY_CALLBACK_COUNT) + return -1; + ctx = pctx->_private; + if (check_fork(ctx) < 0) + return -1; + ctx->pkey_callbacks[callback_type] = callback; + ctx->pkey_callback_data[callback_type] = callback ? user_data : NULL; + return 0; +} + void PKCS11_CTX_init_args(PKCS11_CTX *ctx, const char *init_args) { if (check_fork(ctx->_private) < 0) @@ -255,9 +271,22 @@ int PKCS11_get_key_type(PKCS11_KEY *pkey) EVP_PKEY *PKCS11_get_private_key(PKCS11_KEY *pkey) { PKCS11_OBJECT_private *key = pkey->_private; + PKCS11_CTX_private *ctx = key->slot->ctx; + PKCS11_PKEY_CALLBACK callback; + EVP_PKEY *ret; + if (check_object_fork(key) < 0) return NULL; - return pkcs11_get_key(key, CKO_PRIVATE_KEY); + ret = pkcs11_get_key(key, CKO_PRIVATE_KEY); + if (!ret) + return NULL; + callback = ctx->pkey_callbacks[PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY]; + if (callback && callback(pkey, ret, + ctx->pkey_callback_data[PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY])) { + EVP_PKEY_free(ret); + return NULL; + } + return ret; } EVP_PKEY *PKCS11_get_public_key(PKCS11_KEY *pkey) diff --git a/src/p11_load.c b/src/p11_load.c index bd757827..dfea4035 100644 --- a/src/p11_load.c +++ b/src/p11_load.c @@ -39,9 +39,6 @@ static void libp11_global_free(void) #ifndef OPENSSL_NO_RSA pkcs11_rsa_method_free(); -# if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L - pkcs11_rsa_key_method_free(); -# endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */ #endif /* OPENSSL_NO_RSA */ #if !defined(OPENSSL_NO_ECX) && OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L diff --git a/src/p11_rsa.c b/src/p11_rsa.c index 58a80a2d..4809b256 100644 --- a/src/p11_rsa.c +++ b/src/p11_rsa.c @@ -29,10 +29,6 @@ static int rsa_ex_index = 0; static RSA_METHOD *pkcs11_rsa_method = NULL; -#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L -static EVP_PKEY_METHOD *pkey_method_rsa = NULL; -#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */ - static RSA *pkcs11_get1_rsa(PKCS11_OBJECT_private *key) { EVP_PKEY *evp_key = pkcs11_get_key(key, key->object_class); @@ -235,39 +231,6 @@ void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key) RSA_set_ex_data(rsa, rsa_ex_index, key); } -#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L - -/* Global initialize RSA EVP_PKEY_METHOD */ -static int pkcs11_pkey_method_rsa_new(void) -{ - if (pkey_method_rsa) - return 1; /* EVP_PKEY_RSA method already initialized */ - - pkey_method_rsa = pkcs11_pkey_method_rsa(); - if (!pkey_method_rsa) - return 0; - - /* Register the method globally */ - if (!EVP_PKEY_meth_add0(pkey_method_rsa)) { - EVP_PKEY_meth_free(pkey_method_rsa); - pkey_method_rsa = NULL; - return 0; - } - return 1; -} - -void pkcs11_rsa_key_method_free(void) -{ - if (pkey_method_rsa) { - free_pkey_ex_index(); - EVP_PKEY_meth_remove(pkey_method_rsa); - EVP_PKEY_meth_free(pkey_method_rsa); - pkey_method_rsa = NULL; - } -} - -#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */ - /* * Build an EVP_PKEY object */ @@ -285,21 +248,6 @@ static EVP_PKEY *pkcs11_get_evp_key_rsa(PKCS11_OBJECT_private *key) return NULL; } if (key->object_class == CKO_PRIVATE_KEY) { -#if OPENSSL_VERSION_NUMBER >= 0x30000000L -# if OPENSSL_VERSION_NUMBER < 0x40000000L - if ((key->slot->ctx->flags & PKCS11_FLAG_NO_METHODS) == 0) { - /* global initialize RSA EVP_PKEY_METHOD */ - if (!pkcs11_pkey_method_rsa_new()) { - EVP_PKEY_free(pk); - return NULL; - } - alloc_pkey_ex_index(); - pkcs11_set_ex_data_pkey(pk, key); - atexit(pkcs11_rsa_key_method_free); - } -# endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */ -#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */ - /* The RSA object owns the reference stored in its ex_data. */ key = pkcs11_object_ref(key); RSA_set_method(rsa, PKCS11_get_rsa_method()); @@ -318,7 +266,11 @@ static EVP_PKEY *pkcs11_get_evp_key_rsa(PKCS11_OBJECT_private *key) #endif pkcs11_set_ex_data_rsa(rsa, key); - EVP_PKEY_set1_RSA(pk, rsa); /* Also increments the rsa ref count */ + if (!EVP_PKEY_set1_RSA(pk, rsa)) { + RSA_free(rsa); + EVP_PKEY_free(pk); + return NULL; + } RSA_free(rsa); /* Drops our reference to it */ return pk; } diff --git a/src/util.h b/src/util.h index a1379e72..e873409d 100644 --- a/src/util.h +++ b/src/util.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2025 Michał Trojnara + * Copyright (c) 2025-2026 Michał Trojnara * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -58,6 +58,8 @@ void UTIL_CTX_free(UTIL_CTX *ctx); int UTIL_CTX_set_module(UTIL_CTX *ctx, const char *module); int UTIL_CTX_set_init_args(UTIL_CTX *ctx, const char *init_args); int UTIL_CTX_set_ui_method(UTIL_CTX *ctx, UI_METHOD *ui_method, void *ui_data); +int UTIL_CTX_set_pkey_callback(UTIL_CTX *ctx, int callback_type, + PKCS11_PKEY_CALLBACK callback, void *user_data); int UTIL_CTX_enumerate_slots(UTIL_CTX *ctx); void UTIL_CTX_free_libp11(UTIL_CTX *ctx); diff --git a/src/util_uri.c b/src/util_uri.c index 002e6cdb..7390a540 100644 --- a/src/util_uri.c +++ b/src/util_uri.c @@ -3,7 +3,7 @@ * Copyright (c) 2002 Juha Yrjölä * Copyright (c) 2002 Olaf Kirch * Copyright (c) 2003 Kevin Stefanik - * Copyright (c) 2016-2025 Michał Trojnara + * Copyright (c) 2016-2026 Michał Trojnara * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -53,6 +53,9 @@ struct util_ctx_st { char *init_args; UI_METHOD *ui_method; void *ui_data; + int pkey_callback_type; + PKCS11_PKEY_CALLBACK pkey_callback; + void *pkey_callback_data; /* Logging */ int debug_level; /* level of debug output */ @@ -127,6 +130,18 @@ int UTIL_CTX_set_ui_method(UTIL_CTX *ctx, UI_METHOD *ui_method, void *ui_data) return 1; } +int UTIL_CTX_set_pkey_callback(UTIL_CTX *ctx, int callback_type, + PKCS11_PKEY_CALLBACK callback, void *user_data) +{ + ctx->pkey_callback_type = callback_type; + ctx->pkey_callback = callback; + ctx->pkey_callback_data = callback ? user_data : NULL; + if (ctx->pkcs11_ctx && PKCS11_CTX_set_pkey_callback(ctx->pkcs11_ctx, + callback_type, callback, user_data) < 0) + return 0; + return 1; +} + static int util_ctx_enumerate_slots_unlocked(UTIL_CTX *ctx) { /* PKCS11_update_slots() uses C_GetSlotList() via libp11 */ @@ -173,6 +188,12 @@ static int util_ctx_init_libp11(UTIL_CTX *ctx) PKCS11_set_vlog_a_method(ctx->pkcs11_ctx, ctx->vlog); PKCS11_CTX_init_args(ctx->pkcs11_ctx, ctx->init_args); PKCS11_set_ui_method(ctx->pkcs11_ctx, ctx->ui_method, ctx->ui_data); + if (ctx->pkey_callback && PKCS11_CTX_set_pkey_callback(ctx->pkcs11_ctx, + ctx->pkey_callback_type, ctx->pkey_callback, + ctx->pkey_callback_data) < 0) { + UTIL_CTX_free_libp11(ctx); + return -1; + } if (PKCS11_CTX_load(ctx->pkcs11_ctx, ctx->module) < 0) { UTIL_CTX_log(ctx, LOG_ERR, "Unable to load module %s\n", ctx->module); UTIL_CTX_free_libp11(ctx); From 427ac2a638ffbcd7f82b69cf8eb6759cf6cf2d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Trojnara?= Date: Tue, 18 Aug 2026 14:56:39 +0200 Subject: [PATCH 2/2] Test scoped RSA PKEY methods Verify that a key returned directly by PKCS11_get_private_key() signs with PKCS#1 v1.5 through CKM_RSA_PKCS when CKM_RSA_X_509 is disabled. This ensures that removing process-wide PKEY registration retains basic RSA_METHOD dispatch. Verify that loading the PKCS#11 key leaves existing and subsequently generated provider-backed software RSA keys unchanged. Exercise ENGINE signing without passing an ENGINE to EVP_PKEY_CTX_new(), so success depends on the key-loading callback. Run it against both raw-RSA-only and native-PSS-only SoftHSM configurations with explicit digest, MGF1, and salt-length parameters. --- .gitignore | 1 + tests/Makefile.am | 2 + tests/rsa-pss-sign.c | 36 ++++++- tests/rsa-software-key.c | 168 +++++++++++++++++++++++++++++++++ tests/rsa-software-key.softhsm | 148 +++++++++++++++++++++++++++++ 5 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 tests/rsa-software-key.c create mode 100755 tests/rsa-software-key.softhsm diff --git a/.gitignore b/.gitignore index 1fa37af8..28b7d6b5 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ tests/evp-sign tests/fork-change-slot tests/rsa-oaep tests/rsa-pss-sign +tests/rsa-software-key tests/check-privkey tests/dup-key tests/check-privkey-prov diff --git a/tests/Makefile.am b/tests/Makefile.am index 19662201..855aca77 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -21,6 +21,7 @@ check_PROGRAMS = \ rsa-pss-sign-prov \ rsa-oaep \ rsa-oaep-prov \ + rsa-software-key \ check-privkey \ check-privkey-prov \ dup-key \ @@ -46,6 +47,7 @@ dist_check_SCRIPTS = \ rsa-evp-sign.softhsm \ rsa-pss-sign.softhsm \ rsa-oaep.softhsm \ + rsa-software-key.softhsm \ rsa-check-privkey.softhsm \ rsa-cert-store.softhsm \ rsa-keygen.softhsm \ diff --git a/tests/rsa-pss-sign.c b/tests/rsa-pss-sign.c index 37845327..11c553f4 100644 --- a/tests/rsa-pss-sign.c +++ b/tests/rsa-pss-sign.c @@ -40,6 +40,10 @@ #include #include +#ifndef RSA_PSS_SALTLEN_DIGEST +#define RSA_PSS_SALTLEN_DIGEST -1 +#endif + #ifndef OPENSSL_NO_ENGINE static void display_openssl_errors(int l) @@ -183,8 +187,8 @@ int main(int argc, char **argv) EVP_MD_CTX_destroy(md_ctx); - /* Sign the hash */ - pkey_ctx = EVP_PKEY_CTX_new(private_key, e); + /* The key-loading callback selects ENGINE dispatch for this key. */ + pkey_ctx = EVP_PKEY_CTX_new(private_key, NULL); if (pkey_ctx == NULL) { fprintf(stderr, "Could not create context\n"); @@ -210,6 +214,19 @@ int main(int argc, char **argv) exit(1); } + if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, digest_algo) <= 0) { + fprintf(stderr, "Could not set MGF1 digest algorithm\n"); + display_openssl_errors(__LINE__); + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, + RSA_PSS_SALTLEN_DIGEST) <= 0) { + fprintf(stderr, "Could not set RSA-PSS salt length\n"); + display_openssl_errors(__LINE__); + exit(1); + } + sig_len = sizeof(sig); if (EVP_PKEY_sign(pkey_ctx, sig, &sig_len, md, EVP_MD_size(digest_algo)) <= 0) { @@ -222,7 +239,7 @@ int main(int argc, char **argv) printf("Signature created\n"); - pkey_ctx = EVP_PKEY_CTX_new(public_key, e); + pkey_ctx = EVP_PKEY_CTX_new(public_key, NULL); if (pkey_ctx == NULL) { fprintf(stderr, "Could not create context\n"); @@ -248,6 +265,19 @@ int main(int argc, char **argv) exit(1); } + if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, digest_algo) <= 0) { + fprintf(stderr, "Could not set MGF1 digest algorithm\n"); + display_openssl_errors(__LINE__); + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pkey_ctx, + RSA_PSS_SALTLEN_DIGEST) <= 0) { + fprintf(stderr, "Could not set RSA-PSS salt length\n"); + display_openssl_errors(__LINE__); + exit(1); + } + ret = EVP_PKEY_verify(pkey_ctx, sig, sig_len, md, md_len); if (ret < 0) { display_openssl_errors(__LINE__); diff --git a/tests/rsa-software-key.c b/tests/rsa-software-key.c new file mode 100644 index 00000000..2cd731f5 --- /dev/null +++ b/tests/rsa-software-key.c @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2026 OpenSC Project + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include + +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L && \ + !defined(OPENSSL_NO_RSA) + +#include +#include +#include +#include +#include +#include + +static EVP_PKEY *generate_software_key(void) +{ + EVP_PKEY_CTX *ctx; + EVP_PKEY *key = NULL; + + ctx = EVP_PKEY_CTX_new_from_name(NULL, "RSA", NULL); + if (!ctx) + return NULL; + if (EVP_PKEY_keygen_init(ctx) <= 0 || + EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, 2048) <= 0 || + EVP_PKEY_generate(ctx, &key) <= 0) { + EVP_PKEY_free(key); + key = NULL; + } + EVP_PKEY_CTX_free(ctx); + return key; +} + +static int sign_certificate(EVP_PKEY *key) +{ + static const unsigned char common_name[] = "libp11 software RSA test"; + X509_NAME *name; + X509 *cert; + int ret = 0; + + cert = X509_new(); + if (!cert) + return 0; + name = X509_get_subject_name(cert); + if (!name || !X509_set_version(cert, 2) || + !ASN1_INTEGER_set(X509_get_serialNumber(cert), 1) || + !X509_gmtime_adj(X509_getm_notBefore(cert), 0) || + !X509_gmtime_adj(X509_getm_notAfter(cert), 3600) || + !X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + common_name, -1, -1, 0) || + !X509_set_issuer_name(cert, name) || + !X509_set_pubkey(cert, key) || + X509_sign(cert, key, EVP_sha256()) <= 0 || + X509_verify(cert, key) <= 0) + goto cleanup; + ret = 1; + +cleanup: + X509_free(cert); + return ret; +} + +int main(int argc, char **argv) +{ + PKCS11_CTX *ctx = NULL; + PKCS11_SLOT *slots = NULL, *slot; + PKCS11_KEY *keys; + EVP_PKEY *software_key = NULL, *new_software_key = NULL; + EVP_PKEY *token_key = NULL; + unsigned int nslots = 0, nkeys = 0; + int ret = EXIT_FAILURE; + + if (argc != 3) { + fprintf(stderr, "usage: %s [module] [PIN]\n", argv[0]); + return EXIT_FAILURE; + } + + software_key = generate_software_key(); + if (!software_key || !EVP_PKEY_get0_provider(software_key) || + !sign_certificate(software_key)) { + fprintf(stderr, "Initial software RSA operation failed\n"); + goto cleanup; + } + + ctx = PKCS11_CTX_new(); + if (!ctx) { + fprintf(stderr, "Failed to initialize PKCS#11\n"); + goto cleanup; + } + if (PKCS11_CTX_load(ctx, argv[1]) < 0 || + PKCS11_enumerate_slots(ctx, &slots, &nslots) < 0) { + fprintf(stderr, "Failed to initialize PKCS#11\n"); + goto cleanup; + } + slot = PKCS11_find_token(ctx, slots, nslots); + if (!slot || PKCS11_login(slot, 0, argv[2]) < 0 || + PKCS11_enumerate_keys(slot->token, &keys, &nkeys) < 0 || + nkeys == 0) { + fprintf(stderr, "Failed to find a PKCS#11 private key\n"); + goto cleanup; + } + token_key = PKCS11_get_private_key(&keys[0]); + if (!token_key) { + fprintf(stderr, "PKCS11_get_private_key failed\n"); + goto cleanup; + } + /* X509_sign() uses PKCS#1 v1.5 for an unrestricted RSA key. */ + if (!sign_certificate(token_key)) { + fprintf(stderr, "PKCS#1 v1.5 signing with the PKCS#11 key failed\n"); + goto cleanup; + } + + /* Loading a token key must not alter a provider-backed key that was + * created earlier, or change how subsequent software keys are created. */ + new_software_key = generate_software_key(); + if (!EVP_PKEY_get0_provider(software_key) || + !sign_certificate(software_key) || !new_software_key || + !EVP_PKEY_get0_provider(new_software_key) || + !sign_certificate(new_software_key)) { + fprintf(stderr, "PKCS#11 key affected an unrelated software RSA key\n"); + goto cleanup; + } + + ret = EXIT_SUCCESS; + +cleanup: + if (ret != EXIT_SUCCESS) + ERR_print_errors_fp(stderr); + EVP_PKEY_free(new_software_key); + EVP_PKEY_free(token_key); + EVP_PKEY_free(software_key); + if (slots) + PKCS11_release_all_slots(ctx, slots, nslots); + if (ctx) { + PKCS11_CTX_unload(ctx); + PKCS11_CTX_free(ctx); + } + return ret; +} + +#else + +int main(void) +{ + fprintf(stderr, "Skipped: test requires RSA with OpenSSL 3.x\n"); + return 77; +} + +#endif + +/* vim: set noexpandtab: */ diff --git a/tests/rsa-software-key.softhsm b/tests/rsa-software-key.softhsm new file mode 100755 index 00000000..3e4df431 --- /dev/null +++ b/tests/rsa-software-key.softhsm @@ -0,0 +1,148 @@ +#!/bin/bash + +# Copyright (C) 2026 OpenSC Project +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +outdir="output.$$" + +PRIVATE_KEY="pkcs11:token=libp11-0;id=%01%02%03%04;object=server-key-0;type=private;pin-value=1234" +PUBLIC_KEY="pkcs11:token=libp11-0;id=%01%02%03%04;object=server-key-0;type=public;pin-value=1234" + +# Load common test functions +. "${srcdir}/common.sh" + +# Do the token initialization +init_token "rsa" "1" "libp11" "${ID}" "server-key" "privkey" "pubkey" + +# +# Configure SoftHSM2 to expose PKCS#1 v1.5 but not raw RSA. +# +# The direct libp11 test must therefore reach CKM_RSA_PKCS through the +# RSA_METHOD attached to the key returned by PKCS11_get_private_key(). +# +if [[ -n "${SOFTHSM2_CONF}" ]]; then + echo "slots.mechanisms = CKM_RSA_PKCS_KEY_PAIR_GEN,CKM_RSA_PKCS" \ + >>"${SOFTHSM2_CONF}" + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-PKCS + has_pkcs1=$? + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-X-509 + has_raw=$? + if [[ $has_pkcs1 -ne 0 || $has_raw -eq 0 ]]; then + echo "Failed to configure PKCS#1 v1.5 without raw RSA." + exit 1 + fi + echo "PKCS#1 v1.5 enabled and raw RSA disabled." +fi + +# Load OpenSSL settings +. "${srcdir}/openssl-settings.sh" + +# Restore OpenSSL settings +trap cleanup EXIT + +# +# Test direct PKCS#1 signing and isolation between software and PKCS#11 keys. +# +${WRAPPER} ./rsa-software-key "${MODULE}" "${PIN}" +rc=$? + +if [[ $rc -eq 77 ]]; then + echo "Direct libp11 RSA regression test skipped." + rm -rf "$outdir" + exit 77 +elif [[ $rc -ne 0 ]]; then + echo "Direct libp11 RSA regression test failed." + exit 1 +fi + +# +# Configure SoftHSM2 to expose raw RSA but not native RSA-PSS. +# +# This forces libp11/OpenSSL to perform PSS encoding in software and +# use CKM_RSA_X_509 for the actual private-key operation. +# +if [[ -n "${SOFTHSM2_CONF}" ]]; then + grep -v '^slots.mechanisms' "${SOFTHSM2_CONF}" \ + >"${SOFTHSM2_CONF}.tmp" + echo "slots.mechanisms = CKM_RSA_PKCS_KEY_PAIR_GEN,CKM_RSA_X_509" \ + >>"${SOFTHSM2_CONF}.tmp" + mv "${SOFTHSM2_CONF}.tmp" "${SOFTHSM2_CONF}" + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-PKCS-PSS + has_pss=$? + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-X-509 + has_raw=$? + if [[ $has_pss -eq 0 || $has_raw -ne 0 ]]; then + echo "Failed to configure raw RSA without native RSA-PSS." + exit 1 + fi + echo "Raw RSA enabled and native RSA-PSS disabled." +fi + +# +# Test RSA-PSS through the ENGINE interface. +# +# With the SoftHSM2 configuration above, native CKM_RSA_PKCS_PSS is +# unavailable, so this exercises software PSS encoding over +# CKM_RSA_X_509. +# +${WRAPPER} ./rsa-pss-sign "${PIN}" "${outdir}/engines.cnf" \ + "${PRIVATE_KEY}" "${PUBLIC_KEY}" "${MODULE}" +rc=$? + +if [[ $rc -ne 0 && $rc -ne 77 ]]; then + echo "Raw RSA-PSS ENGINE test failed." + exit 1 +fi +echo + +# +# Configure SoftHSM2 for native RSA-PSS. +# +# CKM_RSA_X_509 is deliberately disabled. Therefore, a successful +# RSA-PSS signature cannot use the software-PSS/raw-RSA fallback. +# +if [[ -n "${SOFTHSM2_CONF}" ]]; then + grep -v '^slots.mechanisms' "${SOFTHSM2_CONF}" \ + >"${SOFTHSM2_CONF}.tmp" + echo "slots.mechanisms = CKM_RSA_PKCS_KEY_PAIR_GEN,CKM_RSA_PKCS_PSS" \ + >>"${SOFTHSM2_CONF}.tmp" + mv "${SOFTHSM2_CONF}.tmp" "${SOFTHSM2_CONF}" + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-PKCS-PSS + has_pss=$? + LD_LIBRARY_PATH="${TEMP_LD_LIBRARY_PATH}" has_mechanism RSA-X-509 + has_raw=$? + if [[ $has_pss -ne 0 || $has_raw -eq 0 ]]; then + echo "Failed to configure native RSA-PSS without raw RSA." + exit 1 + fi + echo "Native RSA-PSS enabled and raw RSA disabled." +fi + +# +# Test native RSA-PSS through ENGINE when the ENGINE test is supported. +# +${WRAPPER} ./rsa-pss-sign "${PIN}" "${outdir}/engines.cnf" \ + "${PRIVATE_KEY}" "${PUBLIC_KEY}" "${MODULE}" +rc=$? +if [[ $rc -eq 77 ]]; then + echo "Native RSA-PSS ENGINE test skipped." + exit 77 +elif [[ $rc -ne 0 ]]; then + echo "Native RSA-PSS ENGINE test failed." + exit 1 +fi + +rm -rf "$outdir" +exit 0