Merge remote-tracking branch 'origin/master'
[unleashed/lotheac.git] / usr / src / uts / common / rpc / sec_gss / rpcsec_gss_misc.c
blob76e6ea1c3e1b249cab122cec37c434995db78100
1 /*
2 * CDDL HEADER START
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
20 * CDDL HEADER END
23 * Copyright 1996,1997,1999,2002-2003 Sun Microsystems, Inc.
24 * All rights reserved. Use is subject to license terms.
28 * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
30 * $Header:
31 * /afs/gza.com/product/secure/rel-eng/src/1.1/rpc/RCS/auth_gssapi_misc.c,v 1.10
32 * 1994/10/27 12:39:23 jik Exp $
36 * Copyright (c) 2013 by Delphix. All rights reserved.
39 #include <sys/param.h>
40 #include <sys/types.h>
41 #include <sys/stream.h>
42 #include <sys/strsubr.h>
43 #include <sys/cmn_err.h>
44 #include <gssapi/gssapi.h>
45 #include <rpc/rpc.h>
46 #include <rpc/rpcsec_defs.h>
49 * The initial allocation size for dynamic allocation.
51 #define CKU_INITSIZE 2048
54 * The size of additional allocations, if required. It is larger to
55 * reduce the number of actual allocations.
57 #define CKU_ALLOCSIZE 8192
61 * Miscellaneous XDR routines.
63 bool_t
64 __xdr_gss_buf(xdrs, buf)
65 XDR *xdrs;
66 gss_buffer_t buf;
68 uint_t cast_len, bound_len;
71 * We go through this contortion because size_t is a now a ulong,
72 * GSS-API uses ulongs.
75 if (xdrs->x_op != XDR_DECODE) {
76 bound_len = cast_len = (uint_t)buf->length;
77 } else {
78 bound_len = (uint_t)-1;
81 if (xdr_bytes(xdrs, (char **)&buf->value, &cast_len,
82 bound_len) == TRUE) {
83 if (xdrs->x_op == XDR_DECODE)
84 buf->length = cast_len;
86 return (TRUE);
89 return (FALSE);
92 bool_t
93 __xdr_rpc_gss_creds(xdrs, creds)
94 XDR *xdrs;
95 rpc_gss_creds *creds;
97 if (!xdr_u_int(xdrs, (uint_t *)&creds->version) ||
98 !xdr_u_int(xdrs, (uint_t *)&creds->gss_proc) ||
99 !xdr_u_int(xdrs, (uint_t *)&creds->seq_num) ||
100 !xdr_u_int(xdrs, (uint_t *)&creds->service) ||
101 !__xdr_gss_buf(xdrs, &creds->ctx_handle))
102 return (FALSE);
103 return (TRUE);
106 bool_t
107 __xdr_rpc_gss_init_arg(xdrs, init_arg)
108 XDR *xdrs;
109 rpc_gss_init_arg *init_arg;
111 if (!__xdr_gss_buf(xdrs, init_arg))
112 return (FALSE);
113 return (TRUE);
116 bool_t
117 __xdr_rpc_gss_init_res(xdrs, init_res)
118 XDR *xdrs;
119 rpc_gss_init_res *init_res;
121 if (!__xdr_gss_buf(xdrs, &init_res->ctx_handle) ||
122 !xdr_u_int(xdrs, (uint_t *)&init_res->gss_major) ||
123 !xdr_u_int(xdrs, (uint_t *)&init_res->gss_minor) ||
124 !xdr_u_int(xdrs, (uint_t *)&init_res->seq_window) ||
125 !__xdr_gss_buf(xdrs, &init_res->token))
126 return (FALSE);
127 return (TRUE);
131 * Generic routine to wrap data used by client and server sides.
133 bool_t
134 __rpc_gss_wrap_data(service, qop, context, seq_num, out_xdrs,
135 xdr_func, xdr_ptr)
136 OM_uint32 qop;
137 rpc_gss_service_t service;
138 gss_ctx_id_t context;
139 uint_t seq_num;
140 XDR *out_xdrs;
141 bool_t (*xdr_func)();
142 caddr_t xdr_ptr;
144 OM_uint32 major, minor;
145 gss_buffer_desc in_buf, out_buf;
146 XDR temp_xdrs;
147 char *temp_data;
148 bool_t conf_state;
149 bool_t ret = FALSE;
150 int size;
153 * Create a temporary XDR/buffer to hold the data to be wrapped.
154 * We need an extra bit for the sequence number serialized first.
156 size = xdr_sizeof(xdr_func, xdr_ptr) + BYTES_PER_XDR_UNIT;
157 temp_data = kmem_alloc(size, KM_SLEEP);
158 out_buf.length = 0;
160 xdrmem_create(&temp_xdrs, temp_data, size, XDR_ENCODE);
163 * serialize the sequence number into tmp memory
165 if (!xdr_u_int(&temp_xdrs, &seq_num))
166 goto fail;
169 * serialize the arguments into tmp memory
171 if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
172 goto fail;
175 * Data to be wrapped goes in in_buf. If privacy is used,
176 * out_buf will have wrapped data (in_buf will no longer be
177 * needed). If integrity is used, out_buf will have checksum
178 * which will follow the data in in_buf.
180 in_buf.length = xdr_getpos(&temp_xdrs);
181 in_buf.value = (char *)temp_xdrs.x_base;
183 switch (service) {
184 case rpc_gss_svc_privacy:
186 if ((major = kgss_seal(&minor, context, TRUE, qop, &in_buf,
187 &conf_state, &out_buf)) != GSS_S_COMPLETE) {
188 RPCGSS_LOG1(1, "rpc_gss_wrap: kgss_seal failed."
189 "major = %x, minor = %x", major, minor);
190 goto fail;
192 in_buf.length = 0; /* in_buf not needed */
193 if (!conf_state)
194 goto fail;
195 break;
196 case rpc_gss_svc_integrity:
197 if ((major = kgss_sign(&minor, context, qop, &in_buf,
198 &out_buf)) != GSS_S_COMPLETE) {
199 RPCGSS_LOG1(1, "rpc_gss_wrap: kgss_sign failed."
200 "major = %x, minor = %x", major, minor);
201 goto fail;
203 break;
204 default:
205 goto fail;
209 * write out in_buf and out_buf as needed
211 if (in_buf.length != 0) {
212 if (!__xdr_gss_buf(out_xdrs, &in_buf))
213 goto fail;
216 if (!__xdr_gss_buf(out_xdrs, &out_buf))
217 goto fail;
218 ret = TRUE;
219 fail:
220 kmem_free(temp_data, size);
221 if (out_buf.length != 0)
222 (void) gss_release_buffer(&minor, &out_buf);
223 return (ret);
227 * Generic routine to unwrap data used by client and server sides.
229 bool_t
230 __rpc_gss_unwrap_data(service, context, seq_num, qop_check, in_xdrs,
231 xdr_func, xdr_ptr)
232 rpc_gss_service_t service;
233 gss_ctx_id_t context;
234 uint_t seq_num;
235 OM_uint32 qop_check;
236 XDR *in_xdrs;
237 bool_t (*xdr_func)();
238 caddr_t xdr_ptr;
240 gss_buffer_desc in_buf, out_buf;
241 XDR temp_xdrs;
242 uint_t seq_num2;
243 bool_t conf = FALSE;
244 OM_uint32 major = GSS_S_COMPLETE, minor = 0;
245 int qop = 0;
247 in_buf.value = NULL;
248 out_buf.value = NULL;
251 * Pull out wrapped data. For privacy service, this is the
252 * encrypted data. For integrity service, this is the data
253 * followed by a checksum.
255 if (!__xdr_gss_buf(in_xdrs, &in_buf)) {
256 return (FALSE);
259 if (service == rpc_gss_svc_privacy) {
260 major = GSS_S_FAILURE;
261 major = kgss_unseal(&minor, context, &in_buf, &out_buf, &conf,
262 &qop);
263 kmem_free(in_buf.value, in_buf.length);
264 if (major != GSS_S_COMPLETE) {
265 RPCGSS_LOG1(1, "rpc_gss_unwrap: kgss_unseal failed."
266 "major = %x, minor = %x", major, minor);
267 return (FALSE);
270 * Keep the returned token (unencrypted data) in in_buf.
272 in_buf.length = out_buf.length;
273 in_buf.value = out_buf.value;
276 * If privacy was not used, or if QOP is not what we are
277 * expecting, fail.
279 if (!conf || qop != qop_check)
280 goto fail;
282 } else if (service == rpc_gss_svc_integrity) {
283 if (!__xdr_gss_buf(in_xdrs, &out_buf)) {
284 return (FALSE);
286 major = kgss_verify(&minor, context, &in_buf, &out_buf,
287 &qop);
288 kmem_free(out_buf.value, out_buf.length);
289 if (major != GSS_S_COMPLETE) {
290 kmem_free(in_buf.value, in_buf.length);
291 RPCGSS_LOG1(1, "rpc_gss_unwrap: kgss_verify failed."
292 "major = %x, minor = %x", major, minor);
293 return (FALSE);
297 * If QOP is not what we are expecting, fail.
299 if (qop != qop_check)
300 goto fail;
303 xdrmem_create(&temp_xdrs, in_buf.value, in_buf.length, XDR_DECODE);
306 * The data consists of the sequence number followed by the
307 * arguments. Make sure sequence number is what we are
308 * expecting (i.e., the value in the header).
310 if (!xdr_u_int(&temp_xdrs, &seq_num2))
311 goto fail;
312 if (seq_num2 != seq_num)
313 goto fail;
316 * Deserialize the arguments into xdr_ptr, and release in_buf.
318 if (!(*xdr_func)(&temp_xdrs, xdr_ptr)) {
319 goto fail;
322 if (service == rpc_gss_svc_privacy)
323 (void) gss_release_buffer(&minor, &in_buf);
324 else
325 kmem_free(in_buf.value, in_buf.length);
326 XDR_DESTROY(&temp_xdrs);
327 return (TRUE);
328 fail:
329 XDR_DESTROY(&temp_xdrs);
330 if (service == rpc_gss_svc_privacy)
331 (void) gss_release_buffer(&minor, &in_buf);
332 else
333 kmem_free(in_buf.value, in_buf.length);
334 return (FALSE);