summaryrefslogtreecommitdiff
path: root/rsa.c
diff options
context:
space:
mode:
Diffstat (limited to 'rsa.c')
-rw-r--r--rsa.c113
1 files changed, 75 insertions, 38 deletions
diff --git a/rsa.c b/rsa.c
index d0b5bbf5e..5ecacef90 100644
--- a/rsa.c
+++ b/rsa.c
@@ -1,4 +1,4 @@
1/* $OpenBSD: rsa.c,v 1.31 2014/02/02 03:44:31 djm Exp $ */ 1/* $OpenBSD: rsa.c,v 1.32 2014/06/24 01:13:21 djm Exp $ */
2/* 2/*
3 * Author: Tatu Ylonen <ylo@cs.hut.fi> 3 * Author: Tatu Ylonen <ylo@cs.hut.fi>
4 * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland 4 * Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -67,85 +67,122 @@
67#include <stdarg.h> 67#include <stdarg.h>
68#include <string.h> 68#include <string.h>
69 69
70#include "xmalloc.h"
71#include "rsa.h" 70#include "rsa.h"
72#include "log.h" 71#include "log.h"
72#include "ssherr.h"
73 73
74void 74int
75rsa_public_encrypt(BIGNUM *out, BIGNUM *in, RSA *key) 75rsa_public_encrypt(BIGNUM *out, BIGNUM *in, RSA *key)
76{ 76{
77 u_char *inbuf, *outbuf; 77 u_char *inbuf = NULL, *outbuf = NULL;
78 int len, ilen, olen; 78 int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
79 79
80 if (BN_num_bits(key->e) < 2 || !BN_is_odd(key->e)) 80 if (BN_num_bits(key->e) < 2 || !BN_is_odd(key->e))
81 fatal("rsa_public_encrypt() exponent too small or not odd"); 81 return SSH_ERR_INVALID_ARGUMENT;
82 82
83 olen = BN_num_bytes(key->n); 83 olen = BN_num_bytes(key->n);
84 outbuf = xmalloc(olen); 84 if ((outbuf = malloc(olen)) == NULL) {
85 r = SSH_ERR_ALLOC_FAIL;
86 goto out;
87 }
85 88
86 ilen = BN_num_bytes(in); 89 ilen = BN_num_bytes(in);
87 inbuf = xmalloc(ilen); 90 if ((inbuf = malloc(ilen)) == NULL) {
91 r = SSH_ERR_ALLOC_FAIL;
92 goto out;
93 }
88 BN_bn2bin(in, inbuf); 94 BN_bn2bin(in, inbuf);
89 95
90 if ((len = RSA_public_encrypt(ilen, inbuf, outbuf, key, 96 if ((len = RSA_public_encrypt(ilen, inbuf, outbuf, key,
91 RSA_PKCS1_PADDING)) <= 0) 97 RSA_PKCS1_PADDING)) <= 0) {
92 fatal("rsa_public_encrypt() failed"); 98 r = SSH_ERR_LIBCRYPTO_ERROR;
99 goto out;
100 }
93 101
94 if (BN_bin2bn(outbuf, len, out) == NULL) 102 if (BN_bin2bn(outbuf, len, out) == NULL) {
95 fatal("rsa_public_encrypt: BN_bin2bn failed"); 103 r = SSH_ERR_LIBCRYPTO_ERROR;
104 goto out;
105 }
106 r = 0;
96 107
97 explicit_bzero(outbuf, olen); 108 out:
98 explicit_bzero(inbuf, ilen); 109 if (outbuf != NULL) {
99 free(outbuf); 110 explicit_bzero(outbuf, olen);
100 free(inbuf); 111 free(outbuf);
112 }
113 if (inbuf != NULL) {
114 explicit_bzero(inbuf, ilen);
115 free(inbuf);
116 }
117 return r;
101} 118}
102 119
103int 120int
104rsa_private_decrypt(BIGNUM *out, BIGNUM *in, RSA *key) 121rsa_private_decrypt(BIGNUM *out, BIGNUM *in, RSA *key)
105{ 122{
106 u_char *inbuf, *outbuf; 123 u_char *inbuf = NULL, *outbuf = NULL;
107 int len, ilen, olen; 124 int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
108 125
109 olen = BN_num_bytes(key->n); 126 olen = BN_num_bytes(key->n);
110 outbuf = xmalloc(olen); 127 if ((outbuf = malloc(olen)) == NULL) {
128 r = SSH_ERR_ALLOC_FAIL;
129 goto out;
130 }
111 131
112 ilen = BN_num_bytes(in); 132 ilen = BN_num_bytes(in);
113 inbuf = xmalloc(ilen); 133 if ((inbuf = malloc(ilen)) == NULL) {
134 r = SSH_ERR_ALLOC_FAIL;
135 goto out;
136 }
114 BN_bn2bin(in, inbuf); 137 BN_bn2bin(in, inbuf);
115 138
116 if ((len = RSA_private_decrypt(ilen, inbuf, outbuf, key, 139 if ((len = RSA_private_decrypt(ilen, inbuf, outbuf, key,
117 RSA_PKCS1_PADDING)) <= 0) { 140 RSA_PKCS1_PADDING)) <= 0) {
118 error("rsa_private_decrypt() failed"); 141 r = SSH_ERR_LIBCRYPTO_ERROR;
119 } else { 142 goto out;
120 if (BN_bin2bn(outbuf, len, out) == NULL) 143 } else if (BN_bin2bn(outbuf, len, out) == NULL) {
121 fatal("rsa_private_decrypt: BN_bin2bn failed"); 144 r = SSH_ERR_LIBCRYPTO_ERROR;
145 goto out;
146 }
147 r = 0;
148 out:
149 if (outbuf != NULL) {
150 explicit_bzero(outbuf, olen);
151 free(outbuf);
152 }
153 if (inbuf != NULL) {
154 explicit_bzero(inbuf, ilen);
155 free(inbuf);
122 } 156 }
123 explicit_bzero(outbuf, olen); 157 return r;
124 explicit_bzero(inbuf, ilen);
125 free(outbuf);
126 free(inbuf);
127 return len;
128} 158}
129 159
130/* calculate p-1 and q-1 */ 160/* calculate p-1 and q-1 */
131void 161int
132rsa_generate_additional_parameters(RSA *rsa) 162rsa_generate_additional_parameters(RSA *rsa)
133{ 163{
134 BIGNUM *aux; 164 BIGNUM *aux = NULL;
135 BN_CTX *ctx; 165 BN_CTX *ctx = NULL;
166 int r;
136 167
137 if ((aux = BN_new()) == NULL)
138 fatal("rsa_generate_additional_parameters: BN_new failed");
139 if ((ctx = BN_CTX_new()) == NULL) 168 if ((ctx = BN_CTX_new()) == NULL)
140 fatal("rsa_generate_additional_parameters: BN_CTX_new failed"); 169 return SSH_ERR_ALLOC_FAIL;
170 if ((aux = BN_new()) == NULL) {
171 r = SSH_ERR_ALLOC_FAIL;
172 goto out;
173 }
141 174
142 if ((BN_sub(aux, rsa->q, BN_value_one()) == 0) || 175 if ((BN_sub(aux, rsa->q, BN_value_one()) == 0) ||
143 (BN_mod(rsa->dmq1, rsa->d, aux, ctx) == 0) || 176 (BN_mod(rsa->dmq1, rsa->d, aux, ctx) == 0) ||
144 (BN_sub(aux, rsa->p, BN_value_one()) == 0) || 177 (BN_sub(aux, rsa->p, BN_value_one()) == 0) ||
145 (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0)) 178 (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0)) {
146 fatal("rsa_generate_additional_parameters: BN_sub/mod failed"); 179 r = SSH_ERR_LIBCRYPTO_ERROR;
147 180 goto out;
181 }
182 r = 0;
183 out:
148 BN_clear_free(aux); 184 BN_clear_free(aux);
149 BN_CTX_free(ctx); 185 BN_CTX_free(ctx);
186 return r;
150} 187}
151 188