upstream: make the PKCS#11 RSA code more like the new PKCS#11

ECDSA code: use a single custom RSA_METHOD instead of a method per key

suggested by me, but markus@ did all the work.
ok djm@

OpenBSD-Commit-ID: 8aafcebe923dc742fc5537a995cee549d07e4b2e

1 files changed, 29 insertions, 16 deletions

diff --git a/ssh-pkcs11.c b/ssh-pkcs11.c
index c36d31604..d7b3a65f0 100644
--- a/ssh-pkcs11.c
+++ b/ssh-pkcs11.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssh-pkcs11.c,v 1.32 2019/01/20 23:05:52 djm Exp $ */
+/* $OpenBSD: ssh-pkcs11.c,v 1.33 2019/01/20 23:08:24 djm Exp $ */
 /*
  * Copyright (c) 2010 Markus Friedl.  All rights reserved.
  * Copyright (c) 2014 Pedro Martelletto. All rights reserved.
@@ -334,8 +334,8 @@ pkcs11_rsa_private_encrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
         CK_RV                   rv;
         int                     rval = -1;
 
-        if ((k11 = RSA_get_app_data(rsa)) == NULL) {
-                error("RSA_get_app_data failed for rsa %p", rsa);
+        if ((k11 = RSA_get_ex_data(rsa, 0)) == NULL) {
+                error("RSA_get_ex_data failed for rsa %p", rsa);
                 return (-1);
         }
 
@@ -365,13 +365,35 @@ pkcs11_rsa_private_decrypt(int flen, const u_char *from, u_char *to, RSA *rsa,
         return (-1);
 }
 
+static RSA_METHOD *rsa_method;
+
+static int
+pkcs11_rsa_start_wrapper(void)
+{
+        if (rsa_method != NULL)
+                return (0);
+        rsa_method = RSA_meth_dup(RSA_get_default_method());
+        if (rsa_method == NULL)
+                return (-1);
+        if (!RSA_meth_set1_name(rsa_method, "pkcs11") ||
+            !RSA_meth_set_priv_enc(rsa_method, pkcs11_rsa_private_encrypt) ||
+            !RSA_meth_set_priv_dec(rsa_method, pkcs11_rsa_private_decrypt) ||
+            !RSA_meth_set_finish(rsa_method, pkcs11_rsa_finish)) {
+                error("%s: setup pkcs11 method failed", __func__);
+                return (-1);
+        }
+        return (0);
+}
+
 /* redirect private key operations for rsa key to pkcs11 token */
 static int
 pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
     CK_ATTRIBUTE *keyid_attrib, RSA *rsa)
 {
         struct pkcs11_key       *k11;
-        const RSA_METHOD        *def = RSA_get_default_method();
+
+        if (pkcs11_rsa_start_wrapper() == -1)
+                return (-1);
 
         k11 = xcalloc(1, sizeof(*k11));
         k11->provider = provider;
@@ -383,19 +405,10 @@ pkcs11_rsa_wrap(struct pkcs11_provider *provider, CK_ULONG slotidx,
                 k11->keyid = xmalloc(k11->keyid_len);
                 memcpy(k11->keyid, keyid_attrib->pValue, k11->keyid_len);
         }
-        k11->rsa_method = RSA_meth_dup(def);
-        if (k11->rsa_method == NULL)
-                fatal("%s: RSA_meth_dup failed", __func__);
-        k11->orig_finish = RSA_meth_get_finish(def);
-        if (!RSA_meth_set1_name(k11->rsa_method, "pkcs11") ||
-            !RSA_meth_set_priv_enc(k11->rsa_method,
-            pkcs11_rsa_private_encrypt) ||
-            !RSA_meth_set_priv_dec(k11->rsa_method,
-            pkcs11_rsa_private_decrypt) ||
-            !RSA_meth_set_finish(k11->rsa_method, pkcs11_rsa_finish))
-                fatal("%s: setup pkcs11 method failed", __func__);
+
+        k11->rsa_method = rsa_method;
         RSA_set_method(rsa, k11->rsa_method);
-        RSA_set_app_data(rsa, k11);
+        RSA_set_ex_data(rsa, 0, k11);
         return (0);
 }