Old svc_rdma.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28 /*
29 * Portions of this source code were derived from Berkeley
30 * 4.3 BSD under license from the Regents of the University of
31 * California.
32 */
33
34 #pragma ident "@(#)svc_rdma.c 1.8 05/06/10 SMI"
35
36 /*
37 * Server side of RPC over RDMA in the kernel.
38 */
39
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/user.h>
43 #include <sys/sysmacros.h>
44 #include <sys/proc.h>
45 #include <sys/file.h>
46 #include <sys/errno.h>
47 #include <sys/kmem.h>
48 #include <sys/debug.h>
49 #include <sys/systm.h>
50 #include <sys/cmn_err.h>
51 #include <sys/kstat.h>
52 #include <sys/vtrace.h>
53 #include <sys/debug.h>
54
55 #include <rpc/types.h>
56 #include <rpc/xdr.h>
57 #include <rpc/auth.h>
58 #include <rpc/clnt.h>
59 #include <rpc/rpc_msg.h>
60 #include <rpc/svc.h>
61 #include <rpc/rpc_rdma.h>
62 #include <sys/ddi.h>
63 #include <sys/sunddi.h>
64
65 #include <inet/common.h>
66 #include <inet/ip.h>
67 #include <inet/ip6.h>
68
69 /*
70 * RDMA transport specific data associated with SVCMASTERXPRT
71 */
72 struct rdma_data {
73 SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */
74 struct rdma_svc_data rd_data; /* rdma data */
75 rdma_mod_t *r_mod; /* RDMA module containing ops ptr */
76 };
77
78 /*
79 * Plugin connection specific data stashed away in clone SVCXPRT
80 */
81 struct clone_rdma_data {
82 CONN *conn; /* RDMA connection */
83 rdma_buf_t rpcbuf; /* RPC req/resp buffer */
84 };
85
86 #ifdef DEBUG
87 int rdma_svc_debug = 0;
88 #endif
89
90 #define MAXADDRLEN 128 /* max length for address mask */
91
92 /*
93 * Routines exported through ops vector.
94 */
95 static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *);
96 static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *);
97 static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t);
98 static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t);
99 void svc_rdma_kdestroy(SVCMASTERXPRT *);
100 static int svc_rdma_kdup(struct svc_req *, caddr_t, int,
101 struct dupreq **, bool_t *);
102 static void svc_rdma_kdupdone(struct dupreq *, caddr_t,
103 void (*)(), int, int);
104 static int32_t *svc_rdma_kgetres(SVCXPRT *, int);
105 static void svc_rdma_kfreeres(SVCXPRT *);
106 static void svc_rdma_kclone_destroy(SVCXPRT *);
107 static void svc_rdma_kstart(SVCMASTERXPRT *);
108 void svc_rdma_kstop(SVCMASTERXPRT *);
109
110 /*
111 * Server transport operations vector.
112 */
113 struct svc_ops rdma_svc_ops = {
114 svc_rdma_krecv, /* Get requests */
115 svc_rdma_kgetargs, /* Deserialize arguments */
116 svc_rdma_ksend, /* Send reply */
117 svc_rdma_kfreeargs, /* Free argument data space */
118 svc_rdma_kdestroy, /* Destroy transport handle */
119 svc_rdma_kdup, /* Check entry in dup req cache */
120 svc_rdma_kdupdone, /* Mark entry in dup req cache as done */
121 svc_rdma_kgetres, /* Get pointer to response buffer */
122 svc_rdma_kfreeres, /* Destroy pre-serialized response header */
123 svc_rdma_kclone_destroy, /* Destroy a clone xprt */
124 svc_rdma_kstart /* Tell `ready-to-receive' to rpcmod */
125 };
126
127 /*
128 * Server statistics
129 * NOTE: This structure type is duplicated in the NFS fast path.
130 */
131 struct {
132 kstat_named_t rscalls;
133 kstat_named_t rsbadcalls;
134 kstat_named_t rsnullrecv;
135 kstat_named_t rsbadlen;
136 kstat_named_t rsxdrcall;
137 kstat_named_t rsdupchecks;
138 kstat_named_t rsdupreqs;
139 kstat_named_t rslongrpcs;
140 } rdmarsstat = {
141 { "calls", KSTAT_DATA_UINT64 },
142 { "badcalls", KSTAT_DATA_UINT64 },
143 { "nullrecv", KSTAT_DATA_UINT64 },
144 { "badlen", KSTAT_DATA_UINT64 },
145 { "xdrcall", KSTAT_DATA_UINT64 },
146 { "dupchecks", KSTAT_DATA_UINT64 },
147 { "dupreqs", KSTAT_DATA_UINT64 },
148 { "longrpcs", KSTAT_DATA_UINT64 }
149 };
150
151 kstat_named_t *rdmarsstat_ptr = (kstat_named_t *)&rdmarsstat;
152 uint_t rdmarsstat_ndata = sizeof (rdmarsstat) / sizeof (kstat_named_t);
153
154 #define RSSTAT_INCR(x) rdmarsstat.x.value.ui64++
155
156 /*
157 * Create a transport record.
158 * The transport record, output buffer, and private data structure
159 * are allocated. The output buffer is serialized into using xdrmem.
160 * There is one transport record per user process which implements a
161 * set of services.
162 */
163 /* ARGSUSED */
164 int
165 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id,
166 rdma_xprt_group_t *started_xprts)
167 {
168 int error;
169 SVCMASTERXPRT *xprt;
170 struct rdma_data *rd;
171 rdma_registry_t *rmod;
172 rdma_xprt_record_t *xprt_rec;
173 queue_t *q;
174
175 /*
176 * modload the RDMA plugins is not already done.
177 */
178 if (!rdma_modloaded) {
179 mutex_enter(&rdma_modload_lock);
180 if (!rdma_modloaded) {
181 error = rdma_modload();
182 }
183 mutex_exit(&rdma_modload_lock);
184
185 if (error)
186 return (error);
187 }
188
189 /*
190 * master_xprt_count is the count of master transport handles
191 * that were successfully created and are ready to recieve for
192 * RDMA based access.
193 */
194 error = 0;
195 xprt_rec = NULL;
196 rw_enter(&rdma_lock, RW_READER);
197 if (rdma_mod_head == NULL) {
198 started_xprts->rtg_count = 0;
199 rw_exit(&rdma_lock);
200 if (rdma_dev_available)
201 return (EPROTONOSUPPORT);
202 else
203 return (ENODEV);
204 }
205
206 /*
207 * If we have reached here, then atleast one RDMA plugin has loaded.
208 * Create a master_xprt, make it start listenining on the device,
209 * if an error is generated, record it, we might need to shut
210 * the master_xprt.
211 * SVC_START() calls svc_rdma_kstart which calls plugin binding
212 * routines.
213 */
214 for (rmod = rdma_mod_head; rmod != NULL; rmod = rmod->r_next) {
215
216 /*
217 * One SVCMASTERXPRT per RDMA plugin.
218 */
219 xprt = kmem_zalloc(sizeof (*xprt), KM_SLEEP);
220 xprt->xp_ops = &rdma_svc_ops;
221 xprt->xp_sct = sct;
222 xprt->xp_type = T_RDMA;
223 mutex_init(&xprt->xp_req_lock, NULL, MUTEX_DEFAULT, NULL);
224 mutex_init(&xprt->xp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
225 xprt->xp_req_head = (mblk_t *)0;
226 xprt->xp_req_tail = (mblk_t *)0;
227 xprt->xp_threads = 0;
228 xprt->xp_detached_threads = 0;
229
230 rd = kmem_zalloc(sizeof (*rd), KM_SLEEP);
231 xprt->xp_p2 = (caddr_t)rd;
232 rd->rd_xprt = xprt;
233 rd->r_mod = rmod->r_mod;
234
235 q = &rd->rd_data.q;
236 xprt->xp_wq = q;
237 q->q_ptr = &rd->rd_xprt;
238 xprt->xp_netid = NULL;
239
240 if (netid != NULL) {
241 xprt->xp_netid = kmem_alloc(strlen(netid) + 1,
242 KM_SLEEP);
243 (void) strcpy(xprt->xp_netid, netid);
244 }
245
246 xprt->xp_addrmask.maxlen =
247 xprt->xp_addrmask.len = sizeof (struct sockaddr_in);
248 xprt->xp_addrmask.buf =
249 kmem_zalloc(xprt->xp_addrmask.len, KM_SLEEP);
250 ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_addr.s_addr =
251 (uint32_t)~0;
252 ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_family =
253 (ushort_t)~0;
254
255 /*
256 * Each of the plugins will have their own Service ID
257 * to listener specific mapping, like port number for VI
258 * and service name for IB.
259 */
260 rd->rd_data.svcid = id;
261 error = svc_xprt_register(xprt, id);
262 if (error) {
263 cmn_err(CE_WARN, "svc_rdma_kcreate: svc_xprt_register"
264 "failed");
265 goto cleanup;
266 }
267
268 SVC_START(xprt);
269 if (!rd->rd_data.active) {
270 svc_xprt_unregister(xprt);
271 error = rd->rd_data.err_code;
272 goto cleanup;
273 }
274
275 /*
276 * This is set only when there is atleast one or more
277 * transports successfully created. We insert the pointer
278 * to the created RDMA master xprt into a separately maintained
279 * list. This way we can easily reference it later to cleanup,
280 * when NFS kRPC service pool is going away/unregistered.
281 */
282 started_xprts->rtg_count ++;
283 xprt_rec = kmem_alloc(sizeof (*xprt_rec), KM_SLEEP);
284 xprt_rec->rtr_xprt_ptr = xprt;
285 xprt_rec->rtr_next = started_xprts->rtg_listhead;
286 started_xprts->rtg_listhead = xprt_rec;
287 continue;
288 cleanup:
289 SVC_DESTROY(xprt);
290 if (error == RDMA_FAILED)
291 error = EPROTONOSUPPORT;
292 }
293
294 rw_exit(&rdma_lock);
295
296 /*
297 * Don't return any error even if a single plugin was started
298 * successfully.
299 */
300 if (started_xprts->rtg_count == 0)
301 return (error);
302 return (0);
303 }
304
305 /*
306 * Cleanup routine for freeing up memory allocated by
307 * svc_rdma_kcreate()
308 */
309 void
310 svc_rdma_kdestroy(SVCMASTERXPRT *xprt)
311 {
312 struct rdma_data *rd = (struct rdma_data *)xprt->xp_p2;
313
314
315 mutex_destroy(&xprt->xp_req_lock);
316 mutex_destroy(&xprt->xp_thread_lock);
317 kmem_free(xprt->xp_netid, strlen(xprt->xp_netid) + 1);
318 kmem_free(rd, sizeof (*rd));
319 kmem_free(xprt->xp_addrmask.buf, xprt->xp_addrmask.maxlen);
320 kmem_free(xprt, sizeof (*xprt));
321 }
322
323
324 static void
325 svc_rdma_kstart(SVCMASTERXPRT *xprt)
326 {
327 struct rdma_svc_data *svcdata;
328 rdma_mod_t *rmod;
329
330 svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data;
331 rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod;
332
333 /*
334 * Create a listener for module at this port
335 */
336
337 (*rmod->rdma_ops->rdma_svc_listen)(svcdata);
338 }
339
340 void
341 svc_rdma_kstop(SVCMASTERXPRT *xprt)
342 {
343 struct rdma_svc_data *svcdata;
344 rdma_mod_t *rmod;
345
346 svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data;
347 rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod;
348
349 /*
350 * Call the stop listener routine for each plugin.
351 */
352 (*rmod->rdma_ops->rdma_svc_stop)(svcdata);
353 if (svcdata->active)
354 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC"
355 " listener");
356 }
357
358 /* ARGSUSED */
359 static void
360 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt)
361 {
362 }
363
364 static bool_t
365 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg)
366 {
367 XDR *xdrs;
368 rdma_stat status;
369 struct recv_data *rdp = (struct recv_data *)mp->b_rptr;
370 CONN *conn;
371 struct clone_rdma_data *vd;
372 struct clist *cl;
373 uint_t vers, op, pos;
374 uint32_t xid;
375
376 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
377 RSSTAT_INCR(rscalls);
378 conn = rdp->conn;
379
380 /*
381 * Post a receive descriptor on this
382 * endpoint to ensure all packets are received.
383 */
384 status = rdma_svc_postrecv(conn);
385 if (status != RDMA_SUCCESS) {
386 cmn_err(CE_NOTE,
387 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status);
388 }
389
390 if (rdp->status != 0) {
391 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
392 RDMA_REL_CONN(conn);
393 RSSTAT_INCR(rsbadcalls);
394 freeb(mp);
395 return (FALSE);
396 }
397
398 /*
399 * Decode rpc message
400 */
401 xdrs = &clone_xprt->xp_xdrin;
402 xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE);
403
404 /*
405 * Get the XID
406 */
407 /*
408 * Treat xid as opaque (xid is the first entity
409 * in the rpc rdma message).
410 */
411 xid = *(uint32_t *)rdp->rpcmsg.addr;
412 /* Skip xid and set the xdr position accordingly. */
413 XDR_SETPOS(xdrs, sizeof (uint32_t));
414 if (! xdr_u_int(xdrs, &vers) ||
415 ! xdr_u_int(xdrs, &op)) {
416 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed");
417 XDR_DESTROY(xdrs);
418 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
419 RDMA_REL_CONN(conn);
420 freeb(mp);
421 RSSTAT_INCR(rsbadcalls);
422 return (FALSE);
423 }
424 if (op == RDMA_DONE) {
425 /*
426 * Should not get RDMA_DONE
427 */
428 freeb(mp);
429 XDR_DESTROY(xdrs);
430 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
431 RDMA_REL_CONN(conn);
432 RSSTAT_INCR(rsbadcalls);
433 return (FALSE); /* no response */
434 }
435
436 #ifdef DEBUG
437 if (rdma_svc_debug)
438 printf("svc_rdma_krecv: recv'd call xid %u\n", xid);
439 #endif
440 /*
441 * Now decode the chunk list
442 */
443 cl = NULL;
444 if (! xdr_do_clist(xdrs, &cl)) {
445 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed");
446 }
447
448 /*
449 * A chunk at 0 offset indicates that the RPC call message
450 * is in a chunk. Get the RPC call message chunk.
451 */
452 if (cl != NULL && op == RDMA_NOMSG) {
453 struct clist *cllong; /* Long RPC chunk */
454
455 /* Remove RPC call message chunk from chunklist */
456 cllong = cl;
457 cl = cl->c_next;
458 cllong->c_next = NULL;
459
460 /* Allocate and register memory for the RPC call msg chunk */
461 cllong->c_daddr = (uint64)(uintptr_t)
462 kmem_alloc(cllong->c_len, KM_SLEEP);
463 if (cllong->c_daddr == NULL) {
464 cmn_err(CE_WARN,
465 "svc_rdma_krecv: no memory for rpc call");
466 XDR_DESTROY(xdrs);
467 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
468 RDMA_REL_CONN(conn);
469 freeb(mp);
470 RSSTAT_INCR(rsbadcalls);
471 clist_free(cl);
472 clist_free(cllong);
473 return (FALSE);
474 }
475 status = clist_register(conn, cllong, 0);
476 if (status) {
477 cmn_err(CE_WARN,
478 "svc_rdma_krecv: clist_register failed");
479 kmem_free((void *)(uintptr_t)cllong->c_daddr,
480 cllong->c_len);
481 XDR_DESTROY(xdrs);
482 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
483 RDMA_REL_CONN(conn);
484 freeb(mp);
485 RSSTAT_INCR(rsbadcalls);
486 clist_free(cl);
487 clist_free(cllong);
488 return (FALSE);
489 }
490
491 /*
492 * Now read the RPC call message in
493 */
494 status = RDMA_READ(conn, cllong, WAIT);
495 if (status) {
496 cmn_err(CE_WARN,
497 "svc_rdma_krecv: rdma_read failed %d", status);
498 (void) clist_deregister(conn, cllong, 0);
499 kmem_free((void *)(uintptr_t)cllong->c_daddr,
500 cllong->c_len);
501 XDR_DESTROY(xdrs);
502 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
503 RDMA_REL_CONN(conn);
504 freeb(mp);
505 RSSTAT_INCR(rsbadcalls);
506 clist_free(cl);
507 clist_free(cllong);
508 return (FALSE);
509 }
510 /*
511 * Sync memory for CPU after DMA
512 */
513 status = clist_syncmem(conn, cllong, 0);
514
515 /*
516 * Deregister the chunk
517 */
518 (void) clist_deregister(conn, cllong, 0);
519
520 /*
521 * Setup the XDR for the RPC call message
522 */
523 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr,
524 cllong->c_len, 0, cl, XDR_DECODE, conn);
525 vd->rpcbuf.type = CHUNK_BUFFER;
526 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr;
527 vd->rpcbuf.len = cllong->c_len;
528 vd->rpcbuf.handle.mrc_rmr = 0;
529
530 /*
531 * Free the chunk element with the Long RPC details and
532 * the message received.
533 */
534 clist_free(cllong);
535 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
536 } else {
537 pos = XDR_GETPOS(xdrs);
538
539 /*
540 * Now the RPC call message header
541 */
542 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos,
543 rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn);
544 vd->rpcbuf = rdp->rpcmsg;
545 }
546 if (! xdr_callmsg(xdrs, msg)) {
547 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed");
548 if (cl != NULL)
549 clist_free(cl);
550 XDR_DESTROY(xdrs);
551 rdma_buf_free(conn, &vd->rpcbuf);
552 RDMA_REL_CONN(conn);
553 freeb(mp);
554 RSSTAT_INCR(rsxdrcall);
555 RSSTAT_INCR(rsbadcalls);
556 return (FALSE);
557 }
558
559 /*
560 * Point the remote transport address in the service_transport
561 * handle at the address in the request.
562 */
563 clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf;
564 clone_xprt->xp_rtaddr.len = conn->c_raddr.len;
565 clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len;
566
567 #ifdef DEBUG
568 if (rdma_svc_debug) {
569 struct sockaddr_in *sin4;
570 char print_addr[INET_ADDRSTRLEN];
571
572 sin4 = (struct sockaddr_in *)clone_xprt->xp_rtaddr.buf;
573 bzero(print_addr, INET_ADDRSTRLEN);
574 (void) inet_ntop(AF_INET,
575 &sin4->sin_addr, print_addr, INET_ADDRSTRLEN);
576 cmn_err(CE_NOTE,
577 "svc_rdma_krecv: remote clnt_addr: %s", print_addr);
578 }
579 #endif
580
581 clone_xprt->xp_xid = xid;
582 vd->conn = conn;
583 freeb(mp);
584 return (TRUE);
585 }
586
587 /*
588 * Send rpc reply.
589 */
590 static bool_t
591 svc_rdma_ksend(SVCXPRT *clone_xprt, struct rpc_msg *msg)
592 {
593 struct clone_rdma_data *vd;
594 XDR *xdrs = &(clone_xprt->xp_xdrout), rxdrs;
595 int retval = FALSE;
596 xdrproc_t xdr_results;
597 caddr_t xdr_location;
598 bool_t has_args, reg = FALSE;
599 uint_t len, op;
600 uint_t vers;
601 struct clist *cl = NULL, *cle = NULL;
602 struct clist *sendlist = NULL;
603 int status;
604 int msglen;
605 rdma_buf_t clmsg, longreply, rpcreply;
606
607 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
608
609 /*
610 * If there is a result procedure specified in the reply message,
611 * it will be processed in the xdr_replymsg and SVCAUTH_WRAP.
612 * We need to make sure it won't be processed twice, so we null
613 * it for xdr_replymsg here.
614 */
615 has_args = FALSE;
616 if (msg->rm_reply.rp_stat == MSG_ACCEPTED &&
617 msg->rm_reply.rp_acpt.ar_stat == SUCCESS) {
618 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) {
619 has_args = TRUE;
620 xdr_location = msg->acpted_rply.ar_results.where;
621 msg->acpted_rply.ar_results.proc = xdr_void;
622 msg->acpted_rply.ar_results.where = NULL;
623 }
624 }
625
626 /*
627 * Get the size of the rpc reply message. Need this
628 * to determine if the rpc reply message will fit in
629 * the pre-allocated RDMA buffers. If the rpc reply
630 * message length is greater that the pre-allocated
631 * buffers then, a one time use buffer is allocated
632 * and registered for this rpc reply.
633 */
634 msglen = xdr_sizeof(xdr_replymsg, msg);
635 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor != RPCSEC_GSS) {
636 msglen += xdrrdma_sizeof(xdr_results, xdr_location,
637 rdma_minchunk);
638 if (msglen > RPC_MSG_SZ) {
639
640 /*
641 * Allocate chunk buffer for rpc reply
642 */
643 rpcreply.type = CHUNK_BUFFER;
644 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
645 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
646 cle->c_xdroff = 0;
647 cle->c_len = rpcreply.len = msglen;
648 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
649 cle->c_next = NULL;
650 xdrrdma_create(xdrs, rpcreply.addr, msglen,
651 rdma_minchunk, cle, XDR_ENCODE, NULL);
652 op = RDMA_NOMSG;
653 } else {
654 /*
655 * Get a pre-allocated buffer for rpc reply
656 */
657 rpcreply.type = SEND_BUFFER;
658 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
659 cmn_err(CE_WARN,
660 "svc_rdma_ksend: no free buffers!");
661 return (retval);
662 }
663 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
664 rdma_minchunk, NULL, XDR_ENCODE, NULL);
665 op = RDMA_MSG;
666 }
667
668 /*
669 * Initialize the XDR encode stream.
670 */
671 msg->rm_xid = clone_xprt->xp_xid;
672
673 if (!(xdr_replymsg(xdrs, msg) &&
674 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
675 xdr_results, xdr_location)))) {
676 rdma_buf_free(vd->conn, &rpcreply);
677 if (cle)
678 clist_free(cle);
679 cmn_err(CE_WARN,
680 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
681 "failed");
682 goto out;
683 }
684 len = XDR_GETPOS(xdrs);
685 }
686 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor == RPCSEC_GSS) {
687
688 /*
689 * For RPCSEC_GSS since we cannot accurately presize the
690 * buffer required for encoding, we assume that its going
691 * to be a Long RPC to start with. We also create the
692 * the XDR stream with min_chunk set to 0 which instructs
693 * the XDR layer to not chunk the incoming byte stream.
694 */
695 msglen += 2 * MAX_AUTH_BYTES + 2 * sizeof (struct opaque_auth);
696 msglen += xdr_sizeof(xdr_results, xdr_location);
697
698 /*
699 * Long RPC. Allocate one time use custom buffer.
700 */
701 longreply.type = CHUNK_BUFFER;
702 longreply.addr = kmem_zalloc(msglen, KM_SLEEP);
703 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
704 cle->c_xdroff = 0;
705 cle->c_len = longreply.len = msglen;
706 cle->c_saddr = (uint64)(uintptr_t)longreply.addr;
707 cle->c_next = NULL;
708 xdrrdma_create(xdrs, longreply.addr, msglen, 0, cle,
709 XDR_ENCODE, NULL);
710 op = RDMA_NOMSG;
711 /*
712 * Initialize the XDR encode stream.
713 */
714 msg->rm_xid = clone_xprt->xp_xid;
715
716 if (!(xdr_replymsg(xdrs, msg) &&
717 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
718 xdr_results, xdr_location)))) {
719 if (longreply.addr != xdrs->x_base) {
720 longreply.addr = xdrs->x_base;
721 longreply.len = xdr_getbufsize(xdrs);
722 }
723 rdma_buf_free(vd->conn, &longreply);
724 if (cle)
725 clist_free(cle);
726 cmn_err(CE_WARN,
727 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
728 "failed");
729 goto out;
730 }
731
732 /*
733 * If we had to allocate a new buffer while encoding
734 * then update the addr and len.
735 */
736 if (longreply.addr != xdrs->x_base) {
737 longreply.addr = xdrs->x_base;
738 longreply.len = xdr_getbufsize(xdrs);
739 }
740
741 len = XDR_GETPOS(xdrs);
742
743 /*
744 * If it so happens that the encoded message is after all
745 * not long enough to be a Long RPC then allocate a
746 * SEND_BUFFER and copy the encoded message into it.
747 */
748 if (len > RPC_MSG_SZ) {
749 rpcreply.type = CHUNK_BUFFER;
750 rpcreply.addr = longreply.addr;
751 rpcreply.len = longreply.len;
752 } else {
753 clist_free(cle);
754 XDR_DESTROY(xdrs);
755 /*
756 * Get a pre-allocated buffer for rpc reply
757 */
758 rpcreply.type = SEND_BUFFER;
759 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
760 cmn_err(CE_WARN,
761 "svc_rdma_ksend: no free buffers!");
762 rdma_buf_free(vd->conn, &longreply);
763 return (retval);
764 }
765 bcopy(longreply.addr, rpcreply.addr, len);
766 xdrrdma_create(xdrs, rpcreply.addr, len, 0, NULL,
767 XDR_ENCODE, NULL);
768 rdma_buf_free(vd->conn, &longreply);
769 op = RDMA_MSG;
770 }
771 }
772
773 if (has_args == FALSE) {
774
775 if (msglen > RPC_MSG_SZ) {
776
777 /*
778 * Allocate chunk buffer for rpc reply
779 */
780 rpcreply.type = CHUNK_BUFFER;
781 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
782 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
783 cle->c_xdroff = 0;
784 cle->c_len = rpcreply.len = msglen;
785 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
786 cle->c_next = NULL;
787 xdrrdma_create(xdrs, rpcreply.addr, msglen,
788 rdma_minchunk, cle, XDR_ENCODE, NULL);
789 op = RDMA_NOMSG;
790 } else {
791 /*
792 * Get a pre-allocated buffer for rpc reply
793 */
794 rpcreply.type = SEND_BUFFER;
795 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
796 cmn_err(CE_WARN,
797 "svc_rdma_ksend: no free buffers!");
798 return (retval);
799 }
800 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
801 rdma_minchunk, NULL, XDR_ENCODE, NULL);
802 op = RDMA_MSG;
803 }
804
805 /*
806 * Initialize the XDR encode stream.
807 */
808 msg->rm_xid = clone_xprt->xp_xid;
809
810 if (!xdr_replymsg(xdrs, msg)) {
811 rdma_buf_free(vd->conn, &rpcreply);
812 if (cle)
813 clist_free(cle);
814 cmn_err(CE_WARN,
815 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
816 "failed");
817 goto out;
818 }
819 len = XDR_GETPOS(xdrs);
820 }
821
822 /*
823 * Get clist and a buffer for sending it across
824 */
825 cl = xdrrdma_clist(xdrs);
826 clmsg.type = SEND_BUFFER;
827 if (RDMA_BUF_ALLOC(vd->conn, &clmsg)) {
828 rdma_buf_free(vd->conn, &rpcreply);
829 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!");
830 goto out;
831 }
832
833 /*
834 * Now register the chunks in the list
835 */
836 if (cl != NULL) {
837 status = clist_register(vd->conn, cl, 1);
838 if (status != RDMA_SUCCESS) {
839 rdma_buf_free(vd->conn, &clmsg);
840 cmn_err(CE_WARN,
841 "svc_rdma_ksend: clist register failed");
842 goto out;
843 }
844 reg = TRUE;
845 }
846
847 /*
848 * XDR the XID, vers, and op
849 */
850 /*
851 * Treat xid as opaque (xid is the first entity
852 * in the rpc rdma message).
853 */
854 vers = RPCRDMA_VERS;
855 xdrs = &rxdrs;
856 xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE);
857 (*(uint32_t *)clmsg.addr) = msg->rm_xid;
858 /* Skip xid and set the xdr position accordingly. */
859 XDR_SETPOS(xdrs, sizeof (uint32_t));
860 if (! xdr_u_int(xdrs, &vers) ||
861 ! xdr_u_int(xdrs, &op)) {
862 rdma_buf_free(vd->conn, &rpcreply);
863 rdma_buf_free(vd->conn, &clmsg);
864 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed");
865 goto out;
866 }
867
868 /*
869 * Now XDR the chunk list
870 */
871 (void) xdr_do_clist(xdrs, &cl);
872
873 clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr,
874 NULL, NULL);
875
876 if (op == RDMA_MSG) {
877 clist_add(&sendlist, 0, len, &rpcreply.handle, rpcreply.addr,
878 NULL, NULL);
879 } else {
880 cl->c_len = len;
881 RSSTAT_INCR(rslongrpcs);
882 }
883
884 /*
885 * Send the reply message to the client
886 */
887 if (cl != NULL) {
888 status = clist_syncmem(vd->conn, cl, 1);
889 if (status != RDMA_SUCCESS) {
890 rdma_buf_free(vd->conn, &rpcreply);
891 rdma_buf_free(vd->conn, &clmsg);
892 goto out;
893 }
894 #ifdef DEBUG
895 if (rdma_svc_debug)
896 printf("svc_rdma_ksend: chunk response len %d xid %u\n",
897 cl->c_len, msg->rm_xid);
898 #endif
899 /*
900 * Post a receive buffer because we expect a RDMA_DONE
901 * message.
902 */
903 status = rdma_svc_postrecv(vd->conn);
904
905 /*
906 * Send the RPC reply message and wait for RDMA_DONE
907 */
908 status = RDMA_SEND_RESP(vd->conn, sendlist, msg->rm_xid);
909 if (status != RDMA_SUCCESS) {
910 #ifdef DEBUG
911 if (rdma_svc_debug)
912 cmn_err(CE_NOTE, "svc_rdma_ksend: "
913 "rdma_send_resp failed %d", status);
914 #endif
915 goto out;
916 }
917 #ifdef DEBUG
918 if (rdma_svc_debug)
919 printf("svc_rdma_ksend: got RDMA_DONE xid %u\n", msg->rm_xid);
920 #endif
921 } else {
922 #ifdef DEBUG
923 if (rdma_svc_debug)
924 printf("svc_rdma_ksend: msg response xid %u\n", msg->rm_xid);
925 #endif
926 status = RDMA_SEND(vd->conn, sendlist, msg->rm_xid);
927 if (status != RDMA_SUCCESS) {
928 #ifdef DEBUG
929 if (rdma_svc_debug)
930 cmn_err(CE_NOTE, "svc_rdma_ksend: "
931 "rdma_send failed %d", status);
932 #endif
933 goto out;
934 }
935 }
936
937 retval = TRUE;
938 out:
939 /*
940 * Deregister the chunks
941 */
942 if (cl != NULL) {
943 if (reg)
944 (void) clist_deregister(vd->conn, cl, 1);
945 if (op == RDMA_NOMSG) {
946 /*
947 * Long RPC reply in chunk. Free it up.
948 */
949 rdma_buf_free(vd->conn, &rpcreply);
950 }
951 clist_free(cl);
952 }
953
954 /*
955 * Free up sendlist chunks
956 */
957 if (sendlist != NULL)
958 clist_free(sendlist);
959
960 /*
961 * Destroy private data for xdr rdma
962 */
963 XDR_DESTROY(&(clone_xprt->xp_xdrout));
964
965 /*
966 * This is completely disgusting. If public is set it is
967 * a pointer to a structure whose first field is the address
968 * of the function to free that structure and any related
969 * stuff. (see rrokfree in nfs_xdr.c).
970 */
971 if (xdrs->x_public) {
972 /* LINTED pointer alignment */
973 (**((int (**)())xdrs->x_public))(xdrs->x_public);
974 }
975
976 return (retval);
977 }
978
979 /*
980 * Deserialize arguments.
981 */
982 static bool_t
983 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr)
984 {
985 if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin,
986 xdr_args, args_ptr)) != TRUE)
987 return (FALSE);
988 return (TRUE);
989 }
990
991 static bool_t
992 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args,
993 caddr_t args_ptr)
994 {
995 struct clone_rdma_data *vd;
996 bool_t retval;
997
998 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
999 if (args_ptr) {
1000 XDR *xdrs = &clone_xprt->xp_xdrin;
1001 struct clist *cl;
1002
1003 cl = xdrrdma_clist(xdrs);
1004 if (cl != NULL)
1005 clist_free(cl);
1006
1007 xdrs->x_op = XDR_FREE;
1008 retval = (*xdr_args)(xdrs, args_ptr);
1009 }
1010 XDR_DESTROY(&(clone_xprt->xp_xdrin));
1011 rdma_buf_free(vd->conn, &vd->rpcbuf);
1012 RDMA_REL_CONN(vd->conn);
1013 return (retval);
1014 }
1015
1016 /* ARGSUSED */
1017 static int32_t *
1018 svc_rdma_kgetres(SVCXPRT *clone_xprt, int size)
1019 {
1020 return (NULL);
1021 }
1022
1023 /* ARGSUSED */
1024 static void
1025 svc_rdma_kfreeres(SVCXPRT *clone_xprt)
1026 {
1027 }
1028
1029 /*
1030 * the dup cacheing routines below provide a cache of non-failure
1031 * transaction id's. rpc service routines can use this to detect
1032 * retransmissions and re-send a non-failure response.
1033 */
1034
1035 /*
1036 * MAXDUPREQS is the number of cached items. It should be adjusted
1037 * to the service load so that there is likely to be a response entry
1038 * when the first retransmission comes in.
1039 */
1040 #define MAXDUPREQS 1024
1041
1042 /*
1043 * This should be appropriately scaled to MAXDUPREQS.
1044 */
1045 #define DRHASHSZ 257
1046
1047 #if ((DRHASHSZ & (DRHASHSZ - 1)) == 0)
1048 #define XIDHASH(xid) ((xid) & (DRHASHSZ - 1))
1049 #else
1050 #define XIDHASH(xid) ((xid) % DRHASHSZ)
1051 #endif
1052 #define DRHASH(dr) XIDHASH((dr)->dr_xid)
1053 #define REQTOXID(req) ((req)->rq_xprt->xp_xid)
1054
1055 static int rdmandupreqs = 0;
1056 static int rdmamaxdupreqs = MAXDUPREQS;
1057 static kmutex_t rdmadupreq_lock;
1058 static struct dupreq *rdmadrhashtbl[DRHASHSZ];
1059 static int rdmadrhashstat[DRHASHSZ];
1060
1061 static void unhash(struct dupreq *);
1062
1063 /*
1064 * rdmadrmru points to the head of a circular linked list in lru order.
1065 * rdmadrmru->dr_next == drlru
1066 */
1067 struct dupreq *rdmadrmru;
1068
1069 /*
1070 * svc_rdma_kdup searches the request cache and returns 0 if the
1071 * request is not found in the cache. If it is found, then it
1072 * returns the state of the request (in progress or done) and
1073 * the status or attributes that were part of the original reply.
1074 */
1075 static int
1076 svc_rdma_kdup(struct svc_req *req, caddr_t res, int size, struct dupreq **drpp,
1077 bool_t *dupcachedp)
1078 {
1079 struct dupreq *dr;
1080 uint32_t xid;
1081 uint32_t drhash;
1082 int status;
1083
1084 xid = REQTOXID(req);
1085 mutex_enter(&rdmadupreq_lock);
1086 RSSTAT_INCR(rsdupchecks);
1087 /*
1088 * Check to see whether an entry already exists in the cache.
1089 */
1090 dr = rdmadrhashtbl[XIDHASH(xid)];
1091 while (dr != NULL) {
1092 if (dr->dr_xid == xid &&
1093 dr->dr_proc == req->rq_proc &&
1094 dr->dr_prog == req->rq_prog &&
1095 dr->dr_vers == req->rq_vers &&
1096 dr->dr_addr.len == req->rq_xprt->xp_rtaddr.len &&
1097 bcmp((caddr_t)dr->dr_addr.buf,
1098 (caddr_t)req->rq_xprt->xp_rtaddr.buf,
1099 dr->dr_addr.len) == 0) {
1100 status = dr->dr_status;
1101 if (status == DUP_DONE) {
1102 bcopy(dr->dr_resp.buf, res, size);
1103 if (dupcachedp != NULL)
1104 *dupcachedp = (dr->dr_resfree != NULL);
1105 } else {
1106 dr->dr_status = DUP_INPROGRESS;
1107 *drpp = dr;
1108 }
1109 RSSTAT_INCR(rsdupreqs);
1110 mutex_exit(&rdmadupreq_lock);
1111 return (status);
1112 }
1113 dr = dr->dr_chain;
1114 }
1115
1116 /*
1117 * There wasn't an entry, either allocate a new one or recycle
1118 * an old one.
1119 */
1120 if (rdmandupreqs < rdmamaxdupreqs) {
1121 dr = kmem_alloc(sizeof (*dr), KM_NOSLEEP);
1122 if (dr == NULL) {
1123 mutex_exit(&rdmadupreq_lock);
1124 return (DUP_ERROR);
1125 }
1126 dr->dr_resp.buf = NULL;
1127 dr->dr_resp.maxlen = 0;
1128 dr->dr_addr.buf = NULL;
1129 dr->dr_addr.maxlen = 0;
1130 if (rdmadrmru) {
1131 dr->dr_next = rdmadrmru->dr_next;
1132 rdmadrmru->dr_next = dr;
1133 } else {
1134 dr->dr_next = dr;
1135 }
1136 rdmandupreqs++;
1137 } else {
1138 dr = rdmadrmru->dr_next;
1139 while (dr->dr_status == DUP_INPROGRESS) {
1140 dr = dr->dr_next;
1141 if (dr == rdmadrmru->dr_next) {
1142 cmn_err(CE_WARN, "svc_rdma_kdup no slots free");
1143 mutex_exit(&rdmadupreq_lock);
1144 return (DUP_ERROR);
1145 }
1146 }
1147 unhash(dr);
1148 if (dr->dr_resfree) {
1149 (*dr->dr_resfree)(dr->dr_resp.buf);
1150 }
1151 }
1152 dr->dr_resfree = NULL;
1153 rdmadrmru = dr;
1154
1155 dr->dr_xid = REQTOXID(req);
1156 dr->dr_prog = req->rq_prog;
1157 dr->dr_vers = req->rq_vers;
1158 dr->dr_proc = req->rq_proc;
1159 if (dr->dr_addr.maxlen < req->rq_xprt->xp_rtaddr.len) {
1160 if (dr->dr_addr.buf != NULL)
1161 kmem_free(dr->dr_addr.buf, dr->dr_addr.maxlen);
1162 dr->dr_addr.maxlen = req->rq_xprt->xp_rtaddr.len;
1163 dr->dr_addr.buf = kmem_alloc(dr->dr_addr.maxlen, KM_NOSLEEP);
1164 if (dr->dr_addr.buf == NULL) {
1165 dr->dr_addr.maxlen = 0;
1166 dr->dr_status = DUP_DROP;
1167 mutex_exit(&rdmadupreq_lock);
1168 return (DUP_ERROR);
1169 }
1170 }
1171 dr->dr_addr.len = req->rq_xprt->xp_rtaddr.len;
1172 bcopy(req->rq_xprt->xp_rtaddr.buf, dr->dr_addr.buf, dr->dr_addr.len);
1173 if (dr->dr_resp.maxlen < size) {
1174 if (dr->dr_resp.buf != NULL)
1175 kmem_free(dr->dr_resp.buf, dr->dr_resp.maxlen);
1176 dr->dr_resp.maxlen = (unsigned int)size;
1177 dr->dr_resp.buf = kmem_alloc(size, KM_NOSLEEP);
1178 if (dr->dr_resp.buf == NULL) {
1179 dr->dr_resp.maxlen = 0;
1180 dr->dr_status = DUP_DROP;
1181 mutex_exit(&rdmadupreq_lock);
1182 return (DUP_ERROR);
1183 }
1184 }
1185 dr->dr_status = DUP_INPROGRESS;
1186
1187 drhash = (uint32_t)DRHASH(dr);
1188 dr->dr_chain = rdmadrhashtbl[drhash];
1189 rdmadrhashtbl[drhash] = dr;
1190 rdmadrhashstat[drhash]++;
1191 mutex_exit(&rdmadupreq_lock);
1192 *drpp = dr;
1193 return (DUP_NEW);
1194 }
1195
1196 /*
1197 * svc_rdma_kdupdone marks the request done (DUP_DONE or DUP_DROP)
1198 * and stores the response.
1199 */
1200 static void
1201 svc_rdma_kdupdone(struct dupreq *dr, caddr_t res, void (*dis_resfree)(),
1202 int size, int status)
1203 {
1204 ASSERT(dr->dr_resfree == NULL);
1205 if (status == DUP_DONE) {
1206 bcopy(res, dr->dr_resp.buf, size);
1207 dr->dr_resfree = dis_resfree;
1208 }
1209 dr->dr_status = status;
1210 }
1211
1212 /*
1213 * This routine expects that the mutex, rdmadupreq_lock, is already held.
1214 */
1215 static void
1216 unhash(struct dupreq *dr)
1217 {
1218 struct dupreq *drt;
1219 struct dupreq *drtprev = NULL;
1220 uint32_t drhash;
1221
1222 ASSERT(MUTEX_HELD(&rdmadupreq_lock));
1223
1224 drhash = (uint32_t)DRHASH(dr);
1225 drt = rdmadrhashtbl[drhash];
1226 while (drt != NULL) {
1227 if (drt == dr) {
1228 rdmadrhashstat[drhash]--;
1229 if (drtprev == NULL) {
1230 rdmadrhashtbl[drhash] = drt->dr_chain;
1231 } else {
1232 drtprev->dr_chain = drt->dr_chain;
1233 }
1234 return;
1235 }
1236 drtprev = drt;
1237 drt = drt->dr_chain;
1238 }
1239 }