diff --git a/crypto/evp/exchange.c b/crypto/evp/exchange.c
index 6423fd9effda1..26d7e1ce95693 100644
--- a/crypto/evp/exchange.c
+++ b/crypto/evp/exchange.c
@@ -264,6 +264,7 @@ int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx)
 
     return ret ? 1 : 0;
  err:
+    evp_pkey_ctx_free_old_ops(ctx);
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
     return 0;
 
diff --git a/crypto/evp/pmeth_fn.c b/crypto/evp/pmeth_fn.c
index 4d8d3e91b1804..8bc59c40b9fc3 100644
--- a/crypto/evp/pmeth_fn.c
+++ b/crypto/evp/pmeth_fn.c
@@ -126,11 +126,8 @@ static int evp_pkey_asym_cipher_init(EVP_PKEY_CTX *ctx, int operation)
         goto err;
     }
 
-    if (ret <= 0) {
-        cipher->freectx(ctx->op.ciph.ciphprovctx);
-        ctx->op.ciph.ciphprovctx = NULL;
+    if (ret <= 0)
         goto err;
-    }
     return 1;
 
  legacy:
@@ -162,8 +159,10 @@ static int evp_pkey_asym_cipher_init(EVP_PKEY_CTX *ctx, int operation)
     }
 
  err:
-    if (ret <= 0)
+    if (ret <= 0) {
+        evp_pkey_ctx_free_old_ops(ctx);
         ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    }
     return ret;
 }
 
diff --git a/crypto/evp/pmeth_gn.c b/crypto/evp/pmeth_gn.c
index a775d2bee7332..fb861d24875c8 100644
--- a/crypto/evp/pmeth_gn.c
+++ b/crypto/evp/pmeth_gn.c
@@ -93,8 +93,10 @@ static int gen_init(EVP_PKEY_CTX *ctx, int operation)
 #endif
 
  end:
-    if (ret <= 0 && ctx != NULL)
+    if (ret <= 0 && ctx != NULL) {
+        evp_pkey_ctx_free_old_ops(ctx);
         ctx->operation = EVP_PKEY_OP_UNDEFINED;
+    }
     return ret;
 
  not_supported:
diff --git a/crypto/evp/signature.c b/crypto/evp/signature.c
index d845ac12dbeb0..b7a7f7960624d 100644
--- a/crypto/evp/signature.c
+++ b/crypto/evp/signature.c
@@ -503,6 +503,7 @@ static int evp_pkey_signature_init(EVP_PKEY_CTX *ctx, int operation)
     return ret;
 
  err:
+    evp_pkey_ctx_free_old_ops(ctx);
     ctx->operation = EVP_PKEY_OP_UNDEFINED;
     return ret;
 }