- markus@cvs.openbsd.org 2002/03/14 16:56:33

     [auth-rh-rsa.c auth-rsa.c auth.h]
     split auth_rsa() for better readability and privsep; ok provos@

1 files changed, 117 insertions, 85 deletions

diff --git a/auth-rsa.c b/auth-rsa.c
index f7ae03cfd..ff9bf3b64 100644
--- a/auth-rsa.c
+++ b/auth-rsa.c
@@ -14,7 +14,7 @@
  */
 
 #include "includes.h"
-RCSID("$OpenBSD: auth-rsa.c,v 1.50 2001/12/28 14:50:54 markus Exp $");
+RCSID("$OpenBSD: auth-rsa.c,v 1.51 2002/03/14 16:56:33 markus Exp $");
 
 #include <openssl/rsa.h>
 #include <openssl/md5.h>
@@ -52,6 +52,51 @@ extern u_char session_id[16];
  * description of the options.
  */
 
+static BIGNUM *
+auth_rsa_generate_challenge(Key *key)
+{
+        BIGNUM *challenge;
+        BN_CTX *ctx;
+
+        if ((challenge = BN_new()) == NULL)
+                fatal("auth_rsa_generate_challenge: BN_new() failed");
+        /* Generate a random challenge. */
+        BN_rand(challenge, 256, 0, 0);
+        if ((ctx = BN_CTX_new()) == NULL)
+                fatal("auth_rsa_generate_challenge: BN_CTX_new() failed");
+        BN_mod(challenge, challenge, key->rsa->n, ctx);
+        BN_CTX_free(ctx);
+
+        return challenge;
+}
+
+static int
+auth_rsa_verify_response(Key *key, BIGNUM *challenge, u_char response[16])
+{
+        u_char buf[32], mdbuf[16];
+        MD5_CTX md;
+        int len;
+
+        /* The response is MD5 of decrypted challenge plus session id. */
+        len = BN_num_bytes(challenge);
+        if (len <= 0 || len > 32)
+                fatal("auth_rsa_verify_response: bad challenge length %d", len);
+        memset(buf, 0, 32);
+        BN_bn2bin(challenge, buf + 32 - len);
+        MD5_Init(&md);
+        MD5_Update(&md, buf, 32);
+        MD5_Update(&md, session_id, 16);
+        MD5_Final(mdbuf, &md);
+
+        /* Verify that the response is the original challenge. */
+        if (memcmp(response, mdbuf, 16) != 0) {
+                /* Wrong answer. */
+                return (0);
+        }
+        /* Correct answer. */
+        return (1);
+}
+
 /*
  * Performs the RSA authentication challenge-response dialog with the client,
  * and returns true (non-zero) if the client gave the correct answer to
@@ -59,29 +104,19 @@ extern u_char session_id[16];
  */
 
 int
-auth_rsa_challenge_dialog(RSA *pk)
+auth_rsa_challenge_dialog(Key *key)
 {
         BIGNUM *challenge, *encrypted_challenge;
-        BN_CTX *ctx;
-        u_char buf[32], mdbuf[16], response[16];
-        MD5_CTX md;
-        u_int i;
-        int len;
+        u_char response[16];
+        int i, success;
 
         if ((encrypted_challenge = BN_new()) == NULL)
                 fatal("auth_rsa_challenge_dialog: BN_new() failed");
-        if ((challenge = BN_new()) == NULL)
-                fatal("auth_rsa_challenge_dialog: BN_new() failed");
 
-        /* Generate a random challenge. */
-        BN_rand(challenge, 256, 0, 0);
-        if ((ctx = BN_CTX_new()) == NULL)
-                fatal("auth_rsa_challenge_dialog: BN_CTX_new() failed");
-        BN_mod(challenge, challenge, pk->n, ctx);
-        BN_CTX_free(ctx);
+        challenge = auth_rsa_generate_challenge(key);
 
         /* Encrypt the challenge with the public key. */
-        rsa_public_encrypt(encrypted_challenge, challenge, pk);
+        rsa_public_encrypt(encrypted_challenge, challenge, key->rsa);
 
         /* Send the encrypted challenge to the client. */
         packet_start(SSH_SMSG_AUTH_RSA_CHALLENGE);
@@ -96,48 +131,26 @@ auth_rsa_challenge_dialog(RSA *pk)
                 response[i] = packet_get_char();
         packet_check_eom();
 
-        /* The response is MD5 of decrypted challenge plus session id. */
-        len = BN_num_bytes(challenge);
-        if (len <= 0 || len > 32)
-                fatal("auth_rsa_challenge_dialog: bad challenge length %d", len);
-        memset(buf, 0, 32);
-        BN_bn2bin(challenge, buf + 32 - len);
-        MD5_Init(&md);
-        MD5_Update(&md, buf, 32);
-        MD5_Update(&md, session_id, 16);
-        MD5_Final(mdbuf, &md);
+        success = auth_rsa_verify_response(key, challenge, response);
         BN_clear_free(challenge);
-
-        /* Verify that the response is the original challenge. */
-        if (memcmp(response, mdbuf, 16) != 0) {
-                /* Wrong answer. */
-                return 0;
-        }
-        /* Correct answer. */
-        return 1;
+        return (success);
 }
 
 /*
- * Performs the RSA authentication dialog with the client.  This returns
- * 0 if the client could not be authenticated, and 1 if authentication was
- * successful.  This may exit if there is a serious protocol violation.
+ * check if there's user key matching client_n,
+ * return key if login is allowed, NULL otherwise
  */
 
-int
-auth_rsa(struct passwd *pw, BIGNUM *client_n)
+static int
+auth_rsa_key_allowed(struct passwd *pw, BIGNUM *client_n, Key **rkey)
 {
         char line[8192], *file;
-        int authenticated;
+        int allowed;
         u_int bits;
         FILE *f;
         u_long linenum = 0;
         struct stat st;
         Key *key;
-        char *fp;
-
-        /* no user given */
-        if (pw == NULL)
-                return 0;
 
         /* Temporarily use the user's uid. */
         temporarily_use_uid(pw);
@@ -151,29 +164,27 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
                 /* Restore the privileged uid. */
                 restore_uid();
                 xfree(file);
-                return 0;
+                return (NULL);
         }
         /* Open the file containing the authorized keys. */
         f = fopen(file, "r");
         if (!f) {
                 /* Restore the privileged uid. */
                 restore_uid();
-                packet_send_debug("Could not open %.900s for reading.", file);
-                packet_send_debug("If your home is on an NFS volume, it may need to be world-readable.");
                 xfree(file);
-                return 0;
+                return (NULL);
         }
         if (options.strict_modes &&
             secure_filename(f, file, pw, line, sizeof(line)) != 0) {
                 xfree(file);
                 fclose(f);
                 log("Authentication refused: %s", line);
-                packet_send_debug("Authentication refused: %s", line);
                 restore_uid();
-                return 0;
+                return (NULL);
         }
-        /* Flag indicating whether authentication has succeeded. */
-        authenticated = 0;
+
+        /* Flag indicating whether the key is allowed. */
+        allowed = 0;
 
         key = key_new(KEY_RSA1);
 
@@ -238,32 +249,8 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
                 if (!auth_parse_options(pw, options, file, linenum))
                         continue;
 
-                /* Perform the challenge-response dialog for this key. */
-                if (!auth_rsa_challenge_dialog(key->rsa)) {
-                        /* Wrong response. */
-                        verbose("Wrong response to RSA authentication challenge.");
-                        packet_send_debug("Wrong response to RSA authentication challenge.");
-                        /*
-                         * Break out of the loop. Otherwise we might send
-                         * another challenge and break the protocol.
-                         */
-                        break;
-                }
-                /*
-                 * Correct response.  The client has been successfully
-                 * authenticated. Note that we have not yet processed the
-                 * options; this will be reset if the options cause the
-                 * authentication to be rejected.
-                 * Break out of the loop if authentication was successful;
-                 * otherwise continue searching.
-                 */
-                authenticated = 1;
-
-                fp = key_fingerprint(key, SSH_FP_MD5, SSH_FP_HEX);
-                verbose("Found matching %s key: %s",
-                    key_type(key), fp);
-                xfree(fp);
-
+                /* break out, this key is allowed */
+                allowed = 1;
                 break;
         }
 
@@ -274,13 +261,58 @@ auth_rsa(struct passwd *pw, BIGNUM *client_n)
         xfree(file);
         fclose(f);
 
-        key_free(key);
-
-        if (authenticated)
-                packet_send_debug("RSA authentication accepted.");
+        /* return key if allowed */
+        if (allowed && rkey != NULL)
+                *rkey = key;
         else
+                key_free(key);
+        return (allowed);
+}
+
+/*
+ * Performs the RSA authentication dialog with the client.  This returns
+ * 0 if the client could not be authenticated, and 1 if authentication was
+ * successful.  This may exit if there is a serious protocol violation.
+ */
+int
+auth_rsa(struct passwd *pw, BIGNUM *client_n)
+{
+        Key *key;
+        char *fp;
+
+        /* no user given */
+        if (pw == NULL)
+                return 0;
+
+        if (auth_rsa_key_allowed(pw, client_n, &key) == 0) {
                 auth_clear_options();
+                return (0);
+        }
+
+        /* Perform the challenge-response dialog for this key. */
+        if (!auth_rsa_challenge_dialog(key)) {
+                /* Wrong response. */
+                verbose("Wrong response to RSA authentication challenge.");
+                packet_send_debug("Wrong response to RSA authentication challenge.");
+                /*
+                 * Break out of the loop. Otherwise we might send
+                 * another challenge and break the protocol.
+                 */
+                key_free(key);
+                return (0);
+        }
+        /*
+         * Correct response.  The client has been successfully
+         * authenticated. Note that we have not yet processed the
+         * options; this will be reset if the options cause the
+         * authentication to be rejected.
+         */
+        fp = key_fingerprint(key, SSH_FP_MD5, SSH_FP_HEX);
+        verbose("Found matching %s key: %s",
+            key_type(key), fp);
+        xfree(fp);
+        key_free(key);
 
-        /* Return authentication result. */
-        return authenticated;
+        packet_send_debug("RSA authentication accepted.");
+        return (1);
 }