14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28 /*
29 * Portions of this source code were derived from Berkeley
30 * 4.3 BSD under license from the Regents of the University of
31 * California.
32 */
33
34 #pragma ident "@(#)svc_rdma.c 1.8 05/06/10 SMI"
35
36 /*
37 * Server side of RPC over RDMA in the kernel.
38 */
39
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/user.h>
43 #include <sys/sysmacros.h>
44 #include <sys/proc.h>
45 #include <sys/file.h>
46 #include <sys/errno.h>
47 #include <sys/kmem.h>
48 #include <sys/debug.h>
49 #include <sys/systm.h>
50 #include <sys/cmn_err.h>
51 #include <sys/kstat.h>
52 #include <sys/vtrace.h>
53 #include <sys/debug.h>
54
55 #include <rpc/types.h>
56 #include <rpc/xdr.h>
57 #include <rpc/auth.h>
58 #include <rpc/clnt.h>
59 #include <rpc/rpc_msg.h>
60 #include <rpc/svc.h>
61 #include <rpc/rpc_rdma.h>
62 #include <sys/ddi.h>
63 #include <sys/sunddi.h>
64
65 #include <inet/common.h>
66 #include <inet/ip.h>
67 #include <inet/ip6.h>
68
69 /*
70 * RDMA transport specific data associated with SVCMASTERXPRT
71 */
72 struct rdma_data {
73 SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */
74 struct rdma_svc_data rd_data; /* rdma data */
75 rdma_mod_t *r_mod; /* RDMA module containing ops ptr */
76 };
77
78 /*
79 * Plugin connection specific data stashed away in clone SVCXPRT
80 */
81 struct clone_rdma_data {
82 CONN *conn; /* RDMA connection */
83 rdma_buf_t rpcbuf; /* RPC req/resp buffer */
84 };
85
86 #ifdef DEBUG
87 int rdma_svc_debug = 0;
88 #endif
89
90 #define MAXADDRLEN 128 /* max length for address mask */
91
92 /*
93 * Routines exported through ops vector.
94 */
95 static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *);
96 static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *);
97 static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t);
98 static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t);
99 void svc_rdma_kdestroy(SVCMASTERXPRT *);
100 static int svc_rdma_kdup(struct svc_req *, caddr_t, int,
101 struct dupreq **, bool_t *);
102 static void svc_rdma_kdupdone(struct dupreq *, caddr_t,
103 void (*)(), int, int);
104 static int32_t *svc_rdma_kgetres(SVCXPRT *, int);
105 static void svc_rdma_kfreeres(SVCXPRT *);
106 static void svc_rdma_kclone_destroy(SVCXPRT *);
107 static void svc_rdma_kstart(SVCMASTERXPRT *);
108 void svc_rdma_kstop(SVCMASTERXPRT *);
109
110 /*
111 * Server transport operations vector.
112 */
113 struct svc_ops rdma_svc_ops = {
114 svc_rdma_krecv, /* Get requests */
115 svc_rdma_kgetargs, /* Deserialize arguments */
116 svc_rdma_ksend, /* Send reply */
117 svc_rdma_kfreeargs, /* Free argument data space */
118 svc_rdma_kdestroy, /* Destroy transport handle */
119 svc_rdma_kdup, /* Check entry in dup req cache */
120 svc_rdma_kdupdone, /* Mark entry in dup req cache as done */
121 svc_rdma_kgetres, /* Get pointer to response buffer */
122 svc_rdma_kfreeres, /* Destroy pre-serialized response header */
123 svc_rdma_kclone_destroy, /* Destroy a clone xprt */
124 svc_rdma_kstart /* Tell `ready-to-receive' to rpcmod */
125 };
126
127 /*
128 * Server statistics
129 * NOTE: This structure type is duplicated in the NFS fast path.
130 */
131 struct {
132 kstat_named_t rscalls;
133 kstat_named_t rsbadcalls;
134 kstat_named_t rsnullrecv;
135 kstat_named_t rsbadlen;
136 kstat_named_t rsxdrcall;
137 kstat_named_t rsdupchecks;
138 kstat_named_t rsdupreqs;
139 kstat_named_t rslongrpcs;
140 } rdmarsstat = {
141 { "calls", KSTAT_DATA_UINT64 },
142 { "badcalls", KSTAT_DATA_UINT64 },
143 { "nullrecv", KSTAT_DATA_UINT64 },
144 { "badlen", KSTAT_DATA_UINT64 },
155
156 /*
157 * Create a transport record.
158 * The transport record, output buffer, and private data structure
159 * are allocated. The output buffer is serialized into using xdrmem.
160 * There is one transport record per user process which implements a
161 * set of services.
162 */
163 /* ARGSUSED */
164 int
165 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id,
166 rdma_xprt_group_t *started_xprts)
167 {
168 int error;
169 SVCMASTERXPRT *xprt;
170 struct rdma_data *rd;
171 rdma_registry_t *rmod;
172 rdma_xprt_record_t *xprt_rec;
173 queue_t *q;
174
175 /*
176 * modload the RDMA plugins is not already done.
177 */
178 if (!rdma_modloaded) {
179 mutex_enter(&rdma_modload_lock);
180 if (!rdma_modloaded) {
181 error = rdma_modload();
182 }
183 mutex_exit(&rdma_modload_lock);
184
185 if (error)
186 return (error);
187 }
188
189 /*
190 * master_xprt_count is the count of master transport handles
191 * that were successfully created and are ready to recieve for
192 * RDMA based access.
193 */
194 error = 0;
348
349 /*
350 * Call the stop listener routine for each plugin.
351 */
352 (*rmod->rdma_ops->rdma_svc_stop)(svcdata);
353 if (svcdata->active)
354 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC"
355 " listener");
356 }
357
358 /* ARGSUSED */
359 static void
360 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt)
361 {
362 }
363
364 static bool_t
365 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg)
366 {
367 XDR *xdrs;
368 rdma_stat status;
369 struct recv_data *rdp = (struct recv_data *)mp->b_rptr;
370 CONN *conn;
371 struct clone_rdma_data *vd;
372 struct clist *cl;
373 uint_t vers, op, pos;
374 uint32_t xid;
375
376 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
377 RSSTAT_INCR(rscalls);
378 conn = rdp->conn;
379
380 /*
381 * Post a receive descriptor on this
382 * endpoint to ensure all packets are received.
383 */
384 status = rdma_svc_postrecv(conn);
385 if (status != RDMA_SUCCESS) {
386 cmn_err(CE_NOTE,
387 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status);
388 }
389
390 if (rdp->status != 0) {
391 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
392 RDMA_REL_CONN(conn);
393 RSSTAT_INCR(rsbadcalls);
394 freeb(mp);
395 return (FALSE);
396 }
397
398 /*
399 * Decode rpc message
400 */
401 xdrs = &clone_xprt->xp_xdrin;
402 xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE);
403
404 /*
405 * Get the XID
406 */
407 /*
408 * Treat xid as opaque (xid is the first entity
409 * in the rpc rdma message).
410 */
411 xid = *(uint32_t *)rdp->rpcmsg.addr;
412 /* Skip xid and set the xdr position accordingly. */
413 XDR_SETPOS(xdrs, sizeof (uint32_t));
414 if (! xdr_u_int(xdrs, &vers) ||
415 ! xdr_u_int(xdrs, &op)) {
416 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed");
417 XDR_DESTROY(xdrs);
418 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
419 RDMA_REL_CONN(conn);
420 freeb(mp);
421 RSSTAT_INCR(rsbadcalls);
422 return (FALSE);
423 }
424 if (op == RDMA_DONE) {
425 /*
426 * Should not get RDMA_DONE
427 */
428 freeb(mp);
429 XDR_DESTROY(xdrs);
430 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
431 RDMA_REL_CONN(conn);
432 RSSTAT_INCR(rsbadcalls);
433 return (FALSE); /* no response */
434 }
435
436 #ifdef DEBUG
437 if (rdma_svc_debug)
438 printf("svc_rdma_krecv: recv'd call xid %u\n", xid);
439 #endif
440 /*
441 * Now decode the chunk list
442 */
443 cl = NULL;
444 if (! xdr_do_clist(xdrs, &cl)) {
445 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed");
446 }
447
448 /*
449 * A chunk at 0 offset indicates that the RPC call message
450 * is in a chunk. Get the RPC call message chunk.
451 */
452 if (cl != NULL && op == RDMA_NOMSG) {
453 struct clist *cllong; /* Long RPC chunk */
454
455 /* Remove RPC call message chunk from chunklist */
456 cllong = cl;
457 cl = cl->c_next;
458 cllong->c_next = NULL;
459
460 /* Allocate and register memory for the RPC call msg chunk */
461 cllong->c_daddr = (uint64)(uintptr_t)
462 kmem_alloc(cllong->c_len, KM_SLEEP);
463 if (cllong->c_daddr == NULL) {
464 cmn_err(CE_WARN,
465 "svc_rdma_krecv: no memory for rpc call");
466 XDR_DESTROY(xdrs);
467 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
468 RDMA_REL_CONN(conn);
469 freeb(mp);
470 RSSTAT_INCR(rsbadcalls);
471 clist_free(cl);
472 clist_free(cllong);
473 return (FALSE);
474 }
475 status = clist_register(conn, cllong, 0);
476 if (status) {
477 cmn_err(CE_WARN,
478 "svc_rdma_krecv: clist_register failed");
479 kmem_free((void *)(uintptr_t)cllong->c_daddr,
480 cllong->c_len);
481 XDR_DESTROY(xdrs);
482 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
483 RDMA_REL_CONN(conn);
484 freeb(mp);
485 RSSTAT_INCR(rsbadcalls);
486 clist_free(cl);
487 clist_free(cllong);
488 return (FALSE);
489 }
490
491 /*
492 * Now read the RPC call message in
493 */
494 status = RDMA_READ(conn, cllong, WAIT);
495 if (status) {
496 cmn_err(CE_WARN,
497 "svc_rdma_krecv: rdma_read failed %d", status);
498 (void) clist_deregister(conn, cllong, 0);
499 kmem_free((void *)(uintptr_t)cllong->c_daddr,
500 cllong->c_len);
501 XDR_DESTROY(xdrs);
502 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
503 RDMA_REL_CONN(conn);
504 freeb(mp);
505 RSSTAT_INCR(rsbadcalls);
506 clist_free(cl);
507 clist_free(cllong);
508 return (FALSE);
509 }
510 /*
511 * Sync memory for CPU after DMA
512 */
513 status = clist_syncmem(conn, cllong, 0);
514
515 /*
516 * Deregister the chunk
517 */
518 (void) clist_deregister(conn, cllong, 0);
519
520 /*
521 * Setup the XDR for the RPC call message
522 */
523 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr,
524 cllong->c_len, 0, cl, XDR_DECODE, conn);
525 vd->rpcbuf.type = CHUNK_BUFFER;
526 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr;
527 vd->rpcbuf.len = cllong->c_len;
528 vd->rpcbuf.handle.mrc_rmr = 0;
529
530 /*
531 * Free the chunk element with the Long RPC details and
532 * the message received.
533 */
534 clist_free(cllong);
535 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
536 } else {
537 pos = XDR_GETPOS(xdrs);
538
539 /*
540 * Now the RPC call message header
541 */
542 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos,
543 rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn);
544 vd->rpcbuf = rdp->rpcmsg;
545 }
546 if (! xdr_callmsg(xdrs, msg)) {
547 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed");
548 if (cl != NULL)
549 clist_free(cl);
550 XDR_DESTROY(xdrs);
551 rdma_buf_free(conn, &vd->rpcbuf);
552 RDMA_REL_CONN(conn);
553 freeb(mp);
554 RSSTAT_INCR(rsxdrcall);
555 RSSTAT_INCR(rsbadcalls);
556 return (FALSE);
557 }
558
559 /*
560 * Point the remote transport address in the service_transport
561 * handle at the address in the request.
562 */
563 clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf;
564 clone_xprt->xp_rtaddr.len = conn->c_raddr.len;
565 clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len;
566
567 #ifdef DEBUG
568 if (rdma_svc_debug) {
569 struct sockaddr_in *sin4;
570 char print_addr[INET_ADDRSTRLEN];
571
572 sin4 = (struct sockaddr_in *)clone_xprt->xp_rtaddr.buf;
573 bzero(print_addr, INET_ADDRSTRLEN);
574 (void) inet_ntop(AF_INET,
575 &sin4->sin_addr, print_addr, INET_ADDRSTRLEN);
576 cmn_err(CE_NOTE,
577 "svc_rdma_krecv: remote clnt_addr: %s", print_addr);
578 }
579 #endif
580
581 clone_xprt->xp_xid = xid;
582 vd->conn = conn;
583 freeb(mp);
584 return (TRUE);
585 }
586
587 /*
588 * Send rpc reply.
589 */
590 static bool_t
591 svc_rdma_ksend(SVCXPRT *clone_xprt, struct rpc_msg *msg)
592 {
593 struct clone_rdma_data *vd;
594 XDR *xdrs = &(clone_xprt->xp_xdrout), rxdrs;
595 int retval = FALSE;
596 xdrproc_t xdr_results;
597 caddr_t xdr_location;
598 bool_t has_args, reg = FALSE;
599 uint_t len, op;
600 uint_t vers;
601 struct clist *cl = NULL, *cle = NULL;
602 struct clist *sendlist = NULL;
603 int status;
604 int msglen;
605 rdma_buf_t clmsg, longreply, rpcreply;
606
607 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
608
609 /*
610 * If there is a result procedure specified in the reply message,
611 * it will be processed in the xdr_replymsg and SVCAUTH_WRAP.
612 * We need to make sure it won't be processed twice, so we null
613 * it for xdr_replymsg here.
614 */
615 has_args = FALSE;
616 if (msg->rm_reply.rp_stat == MSG_ACCEPTED &&
617 msg->rm_reply.rp_acpt.ar_stat == SUCCESS) {
618 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) {
619 has_args = TRUE;
620 xdr_location = msg->acpted_rply.ar_results.where;
621 msg->acpted_rply.ar_results.proc = xdr_void;
622 msg->acpted_rply.ar_results.where = NULL;
623 }
624 }
625
626 /*
627 * Get the size of the rpc reply message. Need this
628 * to determine if the rpc reply message will fit in
629 * the pre-allocated RDMA buffers. If the rpc reply
630 * message length is greater that the pre-allocated
631 * buffers then, a one time use buffer is allocated
632 * and registered for this rpc reply.
633 */
634 msglen = xdr_sizeof(xdr_replymsg, msg);
635 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor != RPCSEC_GSS) {
636 msglen += xdrrdma_sizeof(xdr_results, xdr_location,
637 rdma_minchunk);
638 if (msglen > RPC_MSG_SZ) {
639
640 /*
641 * Allocate chunk buffer for rpc reply
642 */
643 rpcreply.type = CHUNK_BUFFER;
644 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
645 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
646 cle->c_xdroff = 0;
647 cle->c_len = rpcreply.len = msglen;
648 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
649 cle->c_next = NULL;
650 xdrrdma_create(xdrs, rpcreply.addr, msglen,
651 rdma_minchunk, cle, XDR_ENCODE, NULL);
652 op = RDMA_NOMSG;
653 } else {
654 /*
655 * Get a pre-allocated buffer for rpc reply
656 */
657 rpcreply.type = SEND_BUFFER;
658 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
659 cmn_err(CE_WARN,
660 "svc_rdma_ksend: no free buffers!");
661 return (retval);
662 }
663 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
664 rdma_minchunk, NULL, XDR_ENCODE, NULL);
665 op = RDMA_MSG;
666 }
667
668 /*
669 * Initialize the XDR encode stream.
670 */
671 msg->rm_xid = clone_xprt->xp_xid;
672
673 if (!(xdr_replymsg(xdrs, msg) &&
674 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
675 xdr_results, xdr_location)))) {
676 rdma_buf_free(vd->conn, &rpcreply);
677 if (cle)
678 clist_free(cle);
679 cmn_err(CE_WARN,
680 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
681 "failed");
682 goto out;
683 }
684 len = XDR_GETPOS(xdrs);
685 }
686 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor == RPCSEC_GSS) {
687
688 /*
689 * For RPCSEC_GSS since we cannot accurately presize the
690 * buffer required for encoding, we assume that its going
691 * to be a Long RPC to start with. We also create the
692 * the XDR stream with min_chunk set to 0 which instructs
693 * the XDR layer to not chunk the incoming byte stream.
694 */
695 msglen += 2 * MAX_AUTH_BYTES + 2 * sizeof (struct opaque_auth);
696 msglen += xdr_sizeof(xdr_results, xdr_location);
697
698 /*
699 * Long RPC. Allocate one time use custom buffer.
700 */
701 longreply.type = CHUNK_BUFFER;
702 longreply.addr = kmem_zalloc(msglen, KM_SLEEP);
703 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
704 cle->c_xdroff = 0;
705 cle->c_len = longreply.len = msglen;
706 cle->c_saddr = (uint64)(uintptr_t)longreply.addr;
707 cle->c_next = NULL;
708 xdrrdma_create(xdrs, longreply.addr, msglen, 0, cle,
709 XDR_ENCODE, NULL);
710 op = RDMA_NOMSG;
711 /*
712 * Initialize the XDR encode stream.
713 */
714 msg->rm_xid = clone_xprt->xp_xid;
715
716 if (!(xdr_replymsg(xdrs, msg) &&
717 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
718 xdr_results, xdr_location)))) {
719 if (longreply.addr != xdrs->x_base) {
720 longreply.addr = xdrs->x_base;
721 longreply.len = xdr_getbufsize(xdrs);
722 }
723 rdma_buf_free(vd->conn, &longreply);
724 if (cle)
725 clist_free(cle);
726 cmn_err(CE_WARN,
727 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
728 "failed");
729 goto out;
730 }
731
732 /*
733 * If we had to allocate a new buffer while encoding
734 * then update the addr and len.
735 */
736 if (longreply.addr != xdrs->x_base) {
737 longreply.addr = xdrs->x_base;
738 longreply.len = xdr_getbufsize(xdrs);
739 }
740
741 len = XDR_GETPOS(xdrs);
742
743 /*
744 * If it so happens that the encoded message is after all
745 * not long enough to be a Long RPC then allocate a
746 * SEND_BUFFER and copy the encoded message into it.
747 */
748 if (len > RPC_MSG_SZ) {
749 rpcreply.type = CHUNK_BUFFER;
750 rpcreply.addr = longreply.addr;
751 rpcreply.len = longreply.len;
752 } else {
753 clist_free(cle);
754 XDR_DESTROY(xdrs);
755 /*
756 * Get a pre-allocated buffer for rpc reply
757 */
758 rpcreply.type = SEND_BUFFER;
759 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
760 cmn_err(CE_WARN,
761 "svc_rdma_ksend: no free buffers!");
762 rdma_buf_free(vd->conn, &longreply);
763 return (retval);
764 }
765 bcopy(longreply.addr, rpcreply.addr, len);
766 xdrrdma_create(xdrs, rpcreply.addr, len, 0, NULL,
767 XDR_ENCODE, NULL);
768 rdma_buf_free(vd->conn, &longreply);
769 op = RDMA_MSG;
770 }
771 }
772
773 if (has_args == FALSE) {
774
775 if (msglen > RPC_MSG_SZ) {
776
777 /*
778 * Allocate chunk buffer for rpc reply
779 */
780 rpcreply.type = CHUNK_BUFFER;
781 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
782 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
783 cle->c_xdroff = 0;
784 cle->c_len = rpcreply.len = msglen;
785 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
786 cle->c_next = NULL;
787 xdrrdma_create(xdrs, rpcreply.addr, msglen,
788 rdma_minchunk, cle, XDR_ENCODE, NULL);
789 op = RDMA_NOMSG;
790 } else {
791 /*
792 * Get a pre-allocated buffer for rpc reply
793 */
794 rpcreply.type = SEND_BUFFER;
795 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
796 cmn_err(CE_WARN,
797 "svc_rdma_ksend: no free buffers!");
798 return (retval);
799 }
800 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
801 rdma_minchunk, NULL, XDR_ENCODE, NULL);
802 op = RDMA_MSG;
803 }
804
805 /*
806 * Initialize the XDR encode stream.
807 */
808 msg->rm_xid = clone_xprt->xp_xid;
809
810 if (!xdr_replymsg(xdrs, msg)) {
811 rdma_buf_free(vd->conn, &rpcreply);
812 if (cle)
813 clist_free(cle);
814 cmn_err(CE_WARN,
815 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
816 "failed");
817 goto out;
818 }
819 len = XDR_GETPOS(xdrs);
820 }
821
822 /*
823 * Get clist and a buffer for sending it across
824 */
825 cl = xdrrdma_clist(xdrs);
826 clmsg.type = SEND_BUFFER;
827 if (RDMA_BUF_ALLOC(vd->conn, &clmsg)) {
828 rdma_buf_free(vd->conn, &rpcreply);
829 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!");
830 goto out;
831 }
832
833 /*
834 * Now register the chunks in the list
835 */
836 if (cl != NULL) {
837 status = clist_register(vd->conn, cl, 1);
838 if (status != RDMA_SUCCESS) {
839 rdma_buf_free(vd->conn, &clmsg);
840 cmn_err(CE_WARN,
841 "svc_rdma_ksend: clist register failed");
842 goto out;
843 }
844 reg = TRUE;
845 }
846
847 /*
848 * XDR the XID, vers, and op
849 */
850 /*
851 * Treat xid as opaque (xid is the first entity
852 * in the rpc rdma message).
853 */
854 vers = RPCRDMA_VERS;
855 xdrs = &rxdrs;
856 xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE);
857 (*(uint32_t *)clmsg.addr) = msg->rm_xid;
858 /* Skip xid and set the xdr position accordingly. */
859 XDR_SETPOS(xdrs, sizeof (uint32_t));
860 if (! xdr_u_int(xdrs, &vers) ||
861 ! xdr_u_int(xdrs, &op)) {
862 rdma_buf_free(vd->conn, &rpcreply);
863 rdma_buf_free(vd->conn, &clmsg);
864 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed");
865 goto out;
866 }
867
868 /*
869 * Now XDR the chunk list
870 */
871 (void) xdr_do_clist(xdrs, &cl);
872
873 clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr,
874 NULL, NULL);
875
876 if (op == RDMA_MSG) {
877 clist_add(&sendlist, 0, len, &rpcreply.handle, rpcreply.addr,
878 NULL, NULL);
879 } else {
880 cl->c_len = len;
881 RSSTAT_INCR(rslongrpcs);
882 }
883
884 /*
885 * Send the reply message to the client
886 */
887 if (cl != NULL) {
888 status = clist_syncmem(vd->conn, cl, 1);
889 if (status != RDMA_SUCCESS) {
890 rdma_buf_free(vd->conn, &rpcreply);
891 rdma_buf_free(vd->conn, &clmsg);
892 goto out;
893 }
894 #ifdef DEBUG
895 if (rdma_svc_debug)
896 printf("svc_rdma_ksend: chunk response len %d xid %u\n",
897 cl->c_len, msg->rm_xid);
898 #endif
899 /*
900 * Post a receive buffer because we expect a RDMA_DONE
901 * message.
902 */
903 status = rdma_svc_postrecv(vd->conn);
904
905 /*
906 * Send the RPC reply message and wait for RDMA_DONE
907 */
908 status = RDMA_SEND_RESP(vd->conn, sendlist, msg->rm_xid);
909 if (status != RDMA_SUCCESS) {
910 #ifdef DEBUG
911 if (rdma_svc_debug)
912 cmn_err(CE_NOTE, "svc_rdma_ksend: "
913 "rdma_send_resp failed %d", status);
914 #endif
915 goto out;
916 }
917 #ifdef DEBUG
918 if (rdma_svc_debug)
919 printf("svc_rdma_ksend: got RDMA_DONE xid %u\n", msg->rm_xid);
920 #endif
921 } else {
922 #ifdef DEBUG
923 if (rdma_svc_debug)
924 printf("svc_rdma_ksend: msg response xid %u\n", msg->rm_xid);
925 #endif
926 status = RDMA_SEND(vd->conn, sendlist, msg->rm_xid);
927 if (status != RDMA_SUCCESS) {
928 #ifdef DEBUG
929 if (rdma_svc_debug)
930 cmn_err(CE_NOTE, "svc_rdma_ksend: "
931 "rdma_send failed %d", status);
932 #endif
933 goto out;
934 }
935 }
936
937 retval = TRUE;
938 out:
939 /*
940 * Deregister the chunks
941 */
942 if (cl != NULL) {
943 if (reg)
944 (void) clist_deregister(vd->conn, cl, 1);
945 if (op == RDMA_NOMSG) {
946 /*
947 * Long RPC reply in chunk. Free it up.
948 */
949 rdma_buf_free(vd->conn, &rpcreply);
950 }
951 clist_free(cl);
952 }
953
954 /*
955 * Free up sendlist chunks
956 */
957 if (sendlist != NULL)
958 clist_free(sendlist);
959
960 /*
961 * Destroy private data for xdr rdma
962 */
963 XDR_DESTROY(&(clone_xprt->xp_xdrout));
964
965 /*
966 * This is completely disgusting. If public is set it is
967 * a pointer to a structure whose first field is the address
968 * of the function to free that structure and any related
969 * stuff. (see rrokfree in nfs_xdr.c).
970 */
971 if (xdrs->x_public) {
972 /* LINTED pointer alignment */
973 (**((int (**)())xdrs->x_public))(xdrs->x_public);
974 }
975
976 return (retval);
977 }
978
979 /*
980 * Deserialize arguments.
981 */
982 static bool_t
983 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr)
984 {
985 if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin,
986 xdr_args, args_ptr)) != TRUE)
987 return (FALSE);
988 return (TRUE);
989 }
990
991 static bool_t
992 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args,
993 caddr_t args_ptr)
994 {
995 struct clone_rdma_data *vd;
1220 uint32_t drhash;
1221
1222 ASSERT(MUTEX_HELD(&rdmadupreq_lock));
1223
1224 drhash = (uint32_t)DRHASH(dr);
1225 drt = rdmadrhashtbl[drhash];
1226 while (drt != NULL) {
1227 if (drt == dr) {
1228 rdmadrhashstat[drhash]--;
1229 if (drtprev == NULL) {
1230 rdmadrhashtbl[drhash] = drt->dr_chain;
1231 } else {
1232 drtprev->dr_chain = drt->dr_chain;
1233 }
1234 return;
1235 }
1236 drtprev = drt;
1237 drt = drt->dr_chain;
1238 }
1239 }
|
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
27 /* All Rights Reserved */
28 /*
29 * Portions of this source code were derived from Berkeley
30 * 4.3 BSD under license from the Regents of the University of
31 * California.
32 */
33
34 /* Copyright (c) 2006, The Ohio State University. All rights reserved.
35 *
36 * Portions of this source code is developed by the team members of
37 * The Ohio State University's Network-Based Computing Laboratory (NBCL),
38 * headed by Professor Dhabaleswar K. (DK) Panda.
39 *
40 * Acknowledgements to contributions from developors:
41 * Ranjit Noronha: noronha@cse.ohio-state.edu
42 * Lei Chai : chail@cse.ohio-state.edu
43 * Weikuan Yu : yuw@cse.ohio-state.edu
44 *
45 */
46
47 #pragma ident "@(#)svc_rdma.c 1.8 05/06/10 SMI"
48
49 /*
50 * Server side of RPC over RDMA in the kernel.
51 */
52
53 #include <sys/param.h>
54 #include <sys/types.h>
55 #include <sys/user.h>
56 #include <sys/sysmacros.h>
57 #include <sys/proc.h>
58 #include <sys/file.h>
59 #include <sys/errno.h>
60 #include <sys/kmem.h>
61 #include <sys/debug.h>
62 #include <sys/systm.h>
63 #include <sys/cmn_err.h>
64 #include <sys/kstat.h>
65 #include <sys/vtrace.h>
66 #include <sys/debug.h>
67
68 #include <rpc/types.h>
69 #include <rpc/xdr.h>
70 #include <rpc/auth.h>
71 #include <rpc/clnt.h>
72 #include <rpc/rpc_msg.h>
73 #include <rpc/svc.h>
74 #include <rpc/rpc_rdma.h>
75 #include <sys/ddi.h>
76 #include <sys/sunddi.h>
77
78 #include <inet/common.h>
79 #include <inet/ip.h>
80 #include <inet/ip6.h>
81
82 #include <nfs/nfs.h>
83
84 #define SVC_RDMA_SUCCESS 0
85 #define SVC_RDMA_FAIL -1
86
87 #define SVC_CREDIT_FACTOR (0.5)
88
89 uint32_t rdma_bufs_granted = RDMA_BUFS_GRANT;
90 extern xdrproc_t x_READ3res;
91
92 /*
93 * RDMA transport specific data associated with SVCMASTERXPRT
94 */
95 struct rdma_data {
96 SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */
97 struct rdma_svc_data rd_data; /* rdma data */
98 rdma_mod_t *r_mod; /* RDMA module containing ops ptr */
99 };
100
101 /*
102 * Plugin connection specific data stashed away in clone SVCXPRT
103 */
104 struct clone_rdma_data {
105 CONN *conn; /* RDMA connection */
106 rdma_buf_t rpcbuf; /* RPC req/resp buffer */
107 struct clist *reply_cl; /* reply chunk buffer info */
108 struct clist *wlist; /* write list clist */
109 };
110
111 #ifdef DEBUG
112 int rdma_svc_debug = 0;
113 #endif
114
115 #define MAXADDRLEN 128 /* max length for address mask */
116
117 /*
118 * Routines exported through ops vector.
119 */
120 static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *);
121 static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *);
122 static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t);
123 static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t);
124 void svc_rdma_kdestroy(SVCMASTERXPRT *);
125 static int svc_rdma_kdup(struct svc_req *, caddr_t, int,
126 struct dupreq **, bool_t *);
127 static void svc_rdma_kdupdone(struct dupreq *, caddr_t,
128 void (*)(), int, int);
129 static int32_t *svc_rdma_kgetres(SVCXPRT *, int);
130 static void svc_rdma_kfreeres(SVCXPRT *);
131 static void svc_rdma_kclone_destroy(SVCXPRT *);
132 static void svc_rdma_kstart(SVCMASTERXPRT *);
133 void svc_rdma_kstop(SVCMASTERXPRT *);
134
135 static int svc_process_wlist(struct clone_rdma_data *, xdrproc_t,
136 caddr_t, int *, unsigned int *);
137
138 static int svc_process_long_reply(SVCXPRT *, CONN *, xdrproc_t,
139 caddr_t, caddr_t vd, XDR **,
140 struct rpc_msg *, bool_t, int *,
141 int *, int *, unsigned int *);
142
143 static int svc_compose_rpcmsg(SVCXPRT *, CONN *, xdrproc_t, caddr_t,
144 rdma_buf_t *, XDR **, struct rpc_msg *,
145 bool_t, int *, unsigned int *);
146 #ifdef DYNAMIC_CREDIT_CONTROL
147 static void svc_consume_credit(CONN *);
148 static void svc_compute_credit(CONN *, uint32_t, int, int, int *);
149 static void svc_update_credit(CONN * , int);
150 static void svc_grant_credit(CONN *, uint32_t *);
151 #endif
152
153 /*
154 * Server transport operations vector.
155 */
156 struct svc_ops rdma_svc_ops = {
157 svc_rdma_krecv, /* Get requests */
158 svc_rdma_kgetargs, /* Deserialize arguments */
159 svc_rdma_ksend, /* Send reply */
160 svc_rdma_kfreeargs, /* Free argument data space */
161 svc_rdma_kdestroy, /* Destroy transport handle */
162 svc_rdma_kdup, /* Check entry in dup req cache */
163 svc_rdma_kdupdone, /* Mark entry in dup req cache as done */
164 svc_rdma_kgetres, /* Get pointer to response buffer */
165 svc_rdma_kfreeres, /* Destroy pre-serialized response header */
166 svc_rdma_kclone_destroy, /* Destroy a clone xprt */
167 svc_rdma_kstart, /* Tell `ready-to-receive' to rpcmod */
168 rdma_get_wchunk_seg
169 };
170
171 /*
172 * Server statistics
173 * NOTE: This structure type is duplicated in the NFS fast path.
174 */
175 struct {
176 kstat_named_t rscalls;
177 kstat_named_t rsbadcalls;
178 kstat_named_t rsnullrecv;
179 kstat_named_t rsbadlen;
180 kstat_named_t rsxdrcall;
181 kstat_named_t rsdupchecks;
182 kstat_named_t rsdupreqs;
183 kstat_named_t rslongrpcs;
184 } rdmarsstat = {
185 { "calls", KSTAT_DATA_UINT64 },
186 { "badcalls", KSTAT_DATA_UINT64 },
187 { "nullrecv", KSTAT_DATA_UINT64 },
188 { "badlen", KSTAT_DATA_UINT64 },
199
200 /*
201 * Create a transport record.
202 * The transport record, output buffer, and private data structure
203 * are allocated. The output buffer is serialized into using xdrmem.
204 * There is one transport record per user process which implements a
205 * set of services.
206 */
207 /* ARGSUSED */
208 int
209 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id,
210 rdma_xprt_group_t *started_xprts)
211 {
212 int error;
213 SVCMASTERXPRT *xprt;
214 struct rdma_data *rd;
215 rdma_registry_t *rmod;
216 rdma_xprt_record_t *xprt_rec;
217 queue_t *q;
218
219 mutex_enter(&rdma_modload_lock);
220 error = rdma_modload();
221 mutex_exit(&rdma_modload_lock);
222
223 /*
224 * modload the RDMA plugins is not already done.
225 */
226 if (!rdma_modloaded) {
227 mutex_enter(&rdma_modload_lock);
228 if (!rdma_modloaded) {
229 error = rdma_modload();
230 }
231 mutex_exit(&rdma_modload_lock);
232
233 if (error)
234 return (error);
235 }
236
237 /*
238 * master_xprt_count is the count of master transport handles
239 * that were successfully created and are ready to recieve for
240 * RDMA based access.
241 */
242 error = 0;
396
397 /*
398 * Call the stop listener routine for each plugin.
399 */
400 (*rmod->rdma_ops->rdma_svc_stop)(svcdata);
401 if (svcdata->active)
402 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC"
403 " listener");
404 }
405
406 /* ARGSUSED */
407 static void
408 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt)
409 {
410 }
411
412 static bool_t
413 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg)
414 {
415 XDR *xdrs;
416 CONN *conn;
417
418 struct recv_data *rdp = (struct recv_data *)mp->b_rptr;
419 struct clone_rdma_data *vd;
420 struct clist *cl = NULL;
421 struct clist *wcl = NULL;
422 struct clist *repcl = NULL;
423 struct clist *cllong = NULL;
424
425 rdma_stat status;
426 rdma_srv_cred_ctrl_t *cc_info;
427
428 uint32_t vers, op, pos, xid;
429 uint32_t rdma_credit;
430 uint32_t wcl_total_length = 0;
431 bool_t wwl= FALSE;
432 int i, numclnts, availbufs, to_be_posted;
433 #ifdef SERVER_REG_CACHE
434 rib_lrc_entry_t *long_reply_buf = NULL;
435 #endif
436 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
437 RSSTAT_INCR(rscalls);
438 conn = rdp->conn;
439
440 #ifdef DYNAMIC_CREDIT_CONTROL
441 RDMA_GET_RESOURCE_INFO(conn, &numclnts, &availbufs);
442 svc_consume_credit(conn);
443 #else
444 status = rdma_svc_postrecv(conn);
445 if (status != RDMA_SUCCESS) {
446 cmn_err(CE_NOTE,
447 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status);
448 goto badrpc_call;
449 }
450 #endif
451
452 xdrs = &clone_xprt->xp_xdrin;
453 xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE);
454 xid = *(uint32_t *)rdp->rpcmsg.addr;
455 XDR_SETPOS(xdrs, sizeof (uint32_t));
456
457 if (! xdr_u_int(xdrs, &vers) ||
458 ! xdr_u_int(xdrs, &rdma_credit) ||
459 ! xdr_u_int(xdrs, &op)) {
460 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed");
461 goto xdr_err;
462 }
463
464 #ifdef DYNAMIC_CREDIT_CONTROL
465 svc_compute_credit(conn, rdma_credit, numclnts, availbufs, &to_be_posted);
466 for(i=0; i<to_be_posted; i++){
467 status = rdma_svc_postrecv(conn);
468 if (status != RDMA_SUCCESS) {
469 cmn_err(CE_NOTE,
470 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status);
471 goto badrpc_call;
472 }
473 }
474 svc_update_credit(conn, to_be_posted);
475 #endif
476
477 if (rdp->status != 0) {
478 cmn_err(CE_NOTE,
479 "svc_rdma_krecv: invalid status %d",
480 rdp->status);
481 goto badrpc_call;
482 }
483
484 if (! xdr_do_clist(xdrs, &cl)) {
485 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed");
486 goto xdr_err;
487 }
488
489 if (!xdr_decode_wlist_new(xdrs, &wcl, &wwl, &wcl_total_length,conn)) {
490 cmn_err(CE_NOTE, "svc recv: xdr_decode_wlist failed");
491 if (cl)
492 clist_free(cl);
493 goto xdr_err;
494 }
495 vd->wlist = wcl;
496
497 (void) xdr_decode_reply_wchunk(xdrs, &repcl, conn);
498 vd->reply_cl = repcl;
499
500 /*
501 * A chunk at 0 offset indicates that the RPC call message
502 * is in a chunk. Get the RPC call message chunk.
503 */
504 if (cl != NULL && op == RDMA_NOMSG) {
505
506 /* Remove RPC call message chunk from chunklist */
507 cllong = cl;
508 cl = cl->c_next;
509 cllong->c_next = NULL;
510
511 /* Allocate and register memory for the RPC call msg chunk */
512 #ifdef SERVER_REG_CACHE
513 long_reply_buf = RDMA_GET_SERVER_CACHE_BUF(conn,cllong->c_len);
514 cllong->long_reply_buf = (uint64)long_reply_buf;
515 cllong->c_daddr = (uint64)(uintptr_t) long_reply_buf->lrc_buf;
516 #else
517 cllong->c_daddr = (uint64)(uintptr_t)
518 kmem_alloc(cllong->c_len, KM_SLEEP);
519 #endif
520 if (cllong->c_daddr == NULL) {
521 cmn_err(CE_WARN, "svc krecv: no memory for rpc call");
522 clist_free(cllong);
523 goto cll_malloc_err;
524 }
525
526 status = clist_register(conn, cllong, 0);
527 if (status) {
528 cmn_err(CE_WARN, "svc krecv: clist_register failed");
529 #ifdef SERVER_REG_CACHE
530 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)cllong->long_reply_buf);
531 #else
532 if(cllong->c_len)
533 kmem_free((void *)(uintptr_t)cllong->c_daddr,
534 cllong->c_len);
535 #endif
536 if(cllong)
537 clist_free(cllong);
538 goto cll_malloc_err;
539 }
540
541 /*
542 * Now read the RPC call message in
543 */
544 status = RDMA_READ(conn, cllong, WAIT);
545 if (status) {
546 cmn_err(CE_WARN, "svc_rdma_krecv: rdma_read failed");
547 (void) clist_deregister(conn, cllong, 0);
548 #ifdef SERVER_REG_CACHE
549 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)cllong->long_reply_buf);
550 #else
551 kmem_free((void *)(uintptr_t)cllong->c_daddr,
552 cllong->c_len);
553 #endif
554 clist_free(cllong);
555 goto cll_malloc_err;
556 }
557
558 status = clist_syncmem(conn, cllong, 0);
559 (void) clist_deregister(conn, cllong, 0);
560
561 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr,
562 cllong->c_len, 0, cl, XDR_DECODE, conn);
563
564 vd->rpcbuf.type = CHUNK_BUFFER;
565 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr;
566 vd->rpcbuf.len = cllong->c_len;
567 vd->rpcbuf.handle.mrc_rmr = 0;
568 #ifdef SERVER_REG_CACHE
569 vd->rpcbuf.long_reply_buf = (rib_lrc_entry_t *)cllong->long_reply_buf;
570 #endif
571 clist_free(cllong);
572 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
573 } else {
574 pos = XDR_GETPOS(xdrs);
575 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos,
576 rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn);
577 vd->rpcbuf = rdp->rpcmsg;
578 }
579
580 if (! xdr_callmsg(xdrs, msg)) {
581 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed");
582 RSSTAT_INCR(rsxdrcall);
583 goto callmsg_err;
584 }
585
586 /*
587 * wlist sent for something besides NFS3 READ, so ignore it.
588 * FTDO: this isn't appropriate for READLINK3, but our client
589 * will never drive writelist for READLINK3, so good enough
590 * for the demo.
591 */
592 if (vd->wlist != NULL &&
593 (msg->rm_call.cb_rpcvers != RPC_MSG_VERSION ||
594 msg->rm_call.cb_prog != NFS3_PROGRAM ||
595 msg->rm_call.cb_vers != NFS_V3 ||
596 msg->rm_call.cb_proc != NFSPROC3_READ)) {
597 #ifdef SERVER_REG_CACHE
598 RDMA_FREE_SERVER_CACHE_BUF(conn,(rib_lrc_entry_t *)wcl->long_reply_buf);
599 #else
600 kmem_free((void *)wcl->c_saddr, wcl_total_length);
601 #endif
602 clist_free(wcl);
603 vd->wlist = NULL;
604 }
605
606 /*
607 * Point the remote transport address in the service_transport
608 * handle at the address in the request.
609 */
610 clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf;
611 clone_xprt->xp_rtaddr.len = conn->c_raddr.len;
612 clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len;
613 clone_xprt->xp_xid = xid;
614 vd->conn = conn;
615
616 freeb(mp);
617 return (TRUE);
618 callmsg_err:
619 rdma_buf_free(conn, &vd->rpcbuf);
620 cll_malloc_err:
621 if (cl)
622 clist_free(cl);
623 if (wcl != NULL) {
624 #ifdef SERVER_REG_CACHE
625 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf);
626 #else
627 kmem_free((void *)wcl->c_saddr, wcl_total_length);
628 #endif
629 clist_free(wcl);
630 }
631 xdr_err:
632 XDR_DESTROY(xdrs);
633 badrpc_call:
634 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
635 RDMA_REL_CONN(conn);
636 freeb(mp);
637 RSSTAT_INCR(rsbadcalls);
638 return (FALSE);
639 }
640
641 #ifdef DYNAMIC_CREDIT_CONTROL
642 static void
643 svc_consume_credit(CONN *conn)
644 {
645 rdma_srv_cred_ctrl_t *cc_info;
646
647 mutex_enter(&conn->c_lock);
648 cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc;
649 cc_info->srv_cc_posted--;
650 mutex_exit(&conn->c_lock);
651 }
652
653 static void
654 svc_compute_credit(CONN *conn, uint32_t rdma_credit, int numclnts,
655 int availbufs, int *to_be_posted)
656 {
657 int average, grant;
658 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc;
659
660 if(numclnts == 0){
661 cmn_err(CE_NOTE, "There is no active client!\n");
662 *to_be_posted = 0;
663 return;
664 }
665
666 average = availbufs/numclnts;
667
668 mutex_enter(&conn->c_lock);
669
670 if(rdma_credit <= cc_info->srv_cc_posted)
671 grant = cc_info->srv_cc_posted;
672 else if(rdma_credit <= average)
673 grant = rdma_credit;
674 else
675 grant = average + (rdma_credit - average) * SVC_CREDIT_FACTOR;
676
677 *to_be_posted = grant - cc_info->srv_cc_posted;
678 if(*to_be_posted < 0)
679 *to_be_posted = 0;
680 if(*to_be_posted > availbufs)
681 *to_be_posted = availbufs/2;
682
683 mutex_exit(&conn->c_lock);
684 }
685
686 static void
687 svc_grant_credit(CONN * conn, uint32_t * rdma_credit)
688 {
689 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc;
690
691 mutex_enter(&conn->c_lock);
692
693 /*
694 * Fill in the granted number of buffers
695 * for credit control.
696 *
697 * XXX Currently ignoring what the client sends.
698 */
699 *rdma_credit = cc_info->srv_cc_buffers_granted;
700 mutex_exit(&conn->c_lock);
701 }
702
703 static void
704 svc_update_credit(CONN * conn, int i)
705 {
706 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc;
707
708 mutex_enter(&conn->c_lock);
709 cc_info->srv_cc_buffers_granted = cc_info->srv_cc_posted + i;
710 cc_info->srv_cc_posted = cc_info->srv_cc_buffers_granted;
711 mutex_exit(&conn->c_lock);
712 }
713 #endif
714
715 static int
716 svc_process_wlist(struct clone_rdma_data *vd, xdrproc_t xdr_results,
717 caddr_t xdr_location, int *num_wsegment,
718 unsigned int *templen)
719 {
720 struct clist *wcl;
721 int data_len, avail_len, num, status;
722 READ3resok *rok;
723
724 rok = &(((READ3res *) xdr_location)->res_u.ok);
725 data_len = num = avail_len = 0;
726
727 wcl = vd->wlist;
728 while (wcl != NULL) {
729 if (wcl->c_dmemhandle.mrc_rmr != 0
730 && xdr_results == x_READ3res) {
731
732 avail_len += wcl->c_len;
733 if (wcl->c_len < rok->count) {
734 data_len += wcl->c_len;
735 } else {
736 /* Can make the rest chunks all 0-len */
737 data_len += rok->count;
738 wcl->c_len = rok->count;
739 }
740 rok->count -= wcl->c_len;
741 num ++;
742 }
743 else {
744 cmn_err(CE_NOTE,
745 "svc_process_wlist: wlist has an error\n");
746 }
747 wcl = wcl->c_next;
748 }
749
750 /*
751 * MUST fail if there are still more data
752 */
753 if (rok->count > 0) {
754 cmn_err(CE_NOTE,
755 "svc_process_wlist: data_len is too short \n");
756 return SVC_RDMA_FAIL;
757 }
758
759 wcl = vd->wlist;
760 rok->count = data_len;
761 rok->wlist_len = data_len;
762 rok->wlist = wcl;
763 *num_wsegment = num;
764 *templen = avail_len;
765
766 /* Register, sync and write over the data */
767 if (data_len > 0) {
768 status = clist_register(vd->conn, wcl, TRUE);
769 if (status != RDMA_SUCCESS) {
770 cmn_err(CE_NOTE,
771 "svc_process_wlist: clist_register "
772 "failed");
773 return SVC_RDMA_FAIL;
774 }
775
776 status = clist_syncmem(vd->conn, wcl, TRUE);
777 if (status != RDMA_SUCCESS) {
778 cmn_err(CE_NOTE,
779 "svc_process_wlist: syncmem failed(%d)",
780 status);
781 return SVC_RDMA_FAIL;
782 }
783
784 status = RDMA_WRITE(vd->conn, wcl, NOWAIT);
785 if (status != RDMA_SUCCESS) {
786 cmn_err(CE_NOTE,
787 "svc_process_wlist: RDMA_WRITE failed(%d)",
788 status);
789 return SVC_RDMA_FAIL;
790 }
791 }
792
793 return SVC_RDMA_SUCCESS;
794 }
795
796 static int
797 svc_process_long_reply(SVCXPRT * clone_xprt, CONN * conn,
798 xdrproc_t xdr_results, caddr_t xdr_location,
799 caddr_t vd, XDR ** xdrs,
800 struct rpc_msg *msg, bool_t has_args, int *msglen,
801 int *freelen, int *num, unsigned int *len)
802 {
803 rdma_buf_t long_rpc = {0};
804 int status;
805 struct clist *ncl = NULL, *wcl = NULL;
806 char *memp = NULL;
807 int avail_len = 0;
808 int count = 0;
809 int data_len = 0;
810 *num = 0;
811 *freelen = 0;
812 /*
813 * If the clone_xprt struct has a reply chunk list,
814 * then we MUST RDMA_WRITE the reply back to the client,
815 * no matter what its size is. This translates to:
816 *
817 * RDMA_WRITE + RDMA_SEND(op = RDMA_NOMSG)
818 *
819 * XXX the rdma write code currently ignores kerberos.
820 */
821
822 (*msglen) += xdrrdma_sizeof(xdr_results, xdr_location, rdma_minchunk);
823
824 wcl = (struct clist *)vd;
825 count = *msglen;
826 while (wcl != NULL) {
827 *freelen += wcl->c_len;
828 if (wcl->c_dmemhandle.mrc_rmr != 0)
829 {
830 avail_len += wcl->c_len;
831 if (wcl->c_len < count) {
832 data_len += wcl->c_len;
833 } else {
834 data_len += count;
835 wcl->c_len = count;
836 }
837 count -= wcl->c_len;
838 *num += 1;
839 }
840 else {
841 cmn_err(CE_NOTE,
842 "svc_process_long_reply: wchunk list has an error\n");
843 }
844 wcl = wcl->c_next;
845 }
846
847 /*
848 * MUST fail if there are still more data
849 */
850 if (count > 0) {
851 cmn_err(CE_NOTE,
852 "svc_process_long_reply: data_len is too short \n");
853 return SVC_RDMA_FAIL;
854 }
855 /*
856 * Setup buffers for long rpc reply
857 */
858
859 /*
860 * We specify 0 for the chunk size since we
861 * don't want a chunk list.
862 */
863 wcl = (struct clist *)vd;
864 xdrrdma_create(*xdrs, (caddr_t)wcl->c_saddr , *msglen, 0,
865 wcl, XDR_ENCODE, NULL);
866
867 msg->rm_xid = clone_xprt->xp_xid;
868
869 if (!(xdr_replymsg(*xdrs, msg) &&
870 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, *xdrs,
871 xdr_results, xdr_location)))) {
872 kmem_free((void *)wcl->c_saddr, *freelen);
873 cmn_err(CE_WARN, "svc_process_long_reply: "
874 "xdr_replymsg/SVCAUTH_WRAP failed "
875 "for long reply\n");
876 return SVC_RDMA_FAIL;
877 }
878 *len = XDR_GETPOS(*xdrs);
879
880 if (clist_register(conn, wcl, TRUE) != RDMA_SUCCESS) {
881 #ifdef SERVER_REG_CACHE
882 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf);
883 #else
884 kmem_free((void *)(wcl->c_saddr), *freelen);
885 #endif
886 cmn_err(CE_NOTE, "svc_process_long_reply: RDMA_WRITE: "
887 "clist reg failed");
888 return SVC_RDMA_FAIL;
889 }
890
891 status = clist_syncmem(conn, wcl, TRUE);
892 if (status) {
893 (void) clist_deregister(conn, wcl, TRUE);
894 #ifdef SERVER_REG_CACHE
895 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf);
896 #else
897 kmem_free((void *)(wcl->c_saddr), *freelen);
898 #endif
899 cmn_err(CE_NOTE,
900 "svc_process_long_reply: sync mem failed %d", status);
901 return SVC_RDMA_FAIL;
902 }
903
904 /*
905 * Note: we must pass WAIT into the rdma write call to
906 * ensure that the call completes before we move on, where
907 * part of 'moving on' is deregistering the memory -- and
908 * if the memory is deregistered before the write completes
909 * we'll have an error.
910 */
911 status = RDMA_WRITE(conn, wcl, NOWAIT);
912 if (status != RDMA_SUCCESS) {
913 (void) clist_deregister(conn, wcl, TRUE);
914 #ifdef SERVER_REG_CACHE
915 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf);
916 #else
917 kmem_free((void *)(wcl->c_saddr), *freelen);
918 #endif
919 cmn_err(CE_NOTE,
920 "svc_process_long_reply: RDMA_WRITE failed %d",
921 status);
922 return SVC_RDMA_FAIL;
923 }
924
925 return SVC_RDMA_SUCCESS;
926 }
927
928 static int
929 svc_compose_rpcmsg(SVCXPRT * clone_xprt, CONN * conn, xdrproc_t xdr_results,
930 caddr_t xdr_location, rdma_buf_t * rpcreply, XDR ** xdrs,
931 struct rpc_msg *msg, bool_t has_args, int *msglen,
932 unsigned int *len)
933 {
934 int auth_flavor = msg->rm_reply.rp_acpt.ar_verf.oa_flavor;
935
936 if (has_args && auth_flavor != RPCSEC_GSS)
937 (*msglen) += xdrrdma_sizeof(xdr_results, xdr_location,
938 rdma_minchunk);
939 else if (has_args && auth_flavor == RPCSEC_GSS) {
940 (*msglen) +=
941 2 * MAX_AUTH_BYTES + 2 * sizeof(struct opaque_auth);
942 (*msglen) += xdr_sizeof(xdr_results, xdr_location);
943 }
944
945 if (*msglen > RPC_MSG_SZ) {
946 cmn_err(CE_NOTE,
947 "svc_compose_rpcmsg: Server needs to send a reply"
948 "larger than RPC_MSG_SZ\n");
949 return SVC_RDMA_FAIL;
950 }
951
952 /*
953 * Get a pre-allocated buffer for rpc reply
954 */
955 rpcreply->type = SEND_BUFFER;
956 if (RDMA_BUF_ALLOC(conn, rpcreply)) {
957 cmn_err(CE_WARN, "svc_compose_rpcmsg: no free buffers!");
958 return SVC_RDMA_FAIL;
959 }
960
961 if (has_args == FALSE || auth_flavor != RPCSEC_GSS) {
962 xdrrdma_create(*xdrs, rpcreply->addr, rpcreply->len,
963 rdma_minchunk, NULL, XDR_ENCODE, NULL);
964 }
965 else {
966 xdrrdma_create(*xdrs, rpcreply->addr, *msglen, 0, NULL,
967 XDR_ENCODE, NULL);
968 }
969
970 msg->rm_xid = clone_xprt->xp_xid;
971
972 if (has_args) {
973 if (!(xdr_replymsg(*xdrs, msg) &&
974 (!has_args
975 || SVCAUTH_WRAP(&clone_xprt->xp_auth, *xdrs,
976 xdr_results, xdr_location)))) {
977 if (auth_flavor == RPCSEC_GSS
978 && rpcreply->addr != (*xdrs)->x_base) {
979 rpcreply->addr = (*xdrs)->x_base;
980 rpcreply->len = xdr_getbufsize(*xdrs);
981 }
982 rdma_buf_free(conn, rpcreply);
983 cmn_err(CE_WARN,
984 "svc_compose_rpcmsg: xdr_replymsg/SVCAUTH_WRAP "
985 "failed");
986 return SVC_RDMA_FAIL;
987 }
988 if (auth_flavor == RPCSEC_GSS
989 && rpcreply->addr != (*xdrs)->x_base) {
990 rpcreply->addr = (*xdrs)->x_base;
991 rpcreply->len = xdr_getbufsize(*xdrs);
992 }
993 }
994 else {
995 if (!xdr_replymsg(*xdrs, msg)) {
996 rdma_buf_free(conn, rpcreply);
997 cmn_err(CE_WARN,
998 "svc_compose_rpcmsg: xdr_replymsg/SVCAUTH_WRAP "
999 "failed");
1000 return SVC_RDMA_FAIL;
1001 }
1002 }
1003
1004 *len = XDR_GETPOS(*xdrs);
1005
1006 if (auth_flavor == RPCSEC_GSS) {
1007 XDR_DESTROY(*xdrs);
1008 xdrrdma_create(*xdrs, rpcreply->addr, *len, 0, NULL,
1009 XDR_ENCODE, NULL);
1010 }
1011
1012 return SVC_RDMA_SUCCESS;
1013 }
1014
1015 /*
1016 * Send rpc reply.
1017 */
1018 static bool_t
1019 svc_rdma_ksend(SVCXPRT * clone_xprt, struct rpc_msg *msg)
1020 {
1021 XDR *xdrs = &(clone_xprt->xp_xdrout);
1022 XDR rxdrs;
1023 CONN *conn = NULL;
1024 rdma_buf_t clmsg = {0}, rpcreply = {0};
1025
1026 struct clone_rdma_data *vd;
1027 struct clist *cl = NULL;
1028 struct clist *sendlist = NULL;
1029 struct clist *wcl = NULL;
1030 struct clist *reply_cl;
1031 xdrproc_t xdr_results;
1032 caddr_t xdr_location;
1033
1034 int retval = FALSE;
1035 int status, msglen, num_wsegment = 0, num_wreply_segments = 0;
1036 uint32_t rdma_credit = 0, templen = 0;
1037 int freelen =0;
1038 bool_t has_args;
1039 uint_t len, op, vers;
1040
1041 vd = (struct clone_rdma_data *) clone_xprt->xp_p2buf;
1042 conn = vd->conn;
1043
1044 /*
1045 * If there is a result procedure specified in the reply message,
1046 * it will be processed in the xdr_replymsg and SVCAUTH_WRAP.
1047 * We need to make sure it won't be processed twice, so we null
1048 * it for xdr_replymsg here.
1049 */
1050 has_args = FALSE;
1051 if (msg->rm_reply.rp_stat == MSG_ACCEPTED &&
1052 msg->rm_reply.rp_acpt.ar_stat == SUCCESS) {
1053 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) {
1054 has_args = TRUE;
1055 xdr_location = msg->acpted_rply.ar_results.where;
1056 msg->acpted_rply.ar_results.proc = xdr_void;
1057 msg->acpted_rply.ar_results.where = NULL;
1058 }
1059 }
1060
1061 /*
1062 * Use RDMA Write to return content requested by wlist.
1063 * Only 1 writechunk in writelist for now,
1064 * but this chunk can contain multiple rdma segments.
1065 */
1066 if (vd->wlist) {
1067 status = svc_process_wlist(vd, xdr_results, xdr_location,
1068 &num_wsegment, &templen);
1069 if (status != SVC_RDMA_SUCCESS) {
1070 goto out;
1071 }
1072 }
1073 #ifdef RPC_RDMA_INLINE
1074 else if (xdr_results == x_READ3res) {
1075 READ3resok *rok;
1076 rok = &(((READ3res *) xdr_location)->res_u.ok);
1077 rok->wlist = NULL;
1078 }
1079 #endif
1080
1081 /*
1082 * Get the size of the rpc reply message.
1083 */
1084 msglen = xdr_sizeof(xdr_replymsg, msg);
1085
1086 /*reply_cl.c_daddr = NULL;*/
1087 reply_cl = vd->reply_cl;
1088
1089 if (vd->reply_cl) {
1090
1091 status = svc_process_long_reply(clone_xprt,
1092 conn, xdr_results,
1093 xdr_location, (caddr_t)vd->reply_cl,
1094 &xdrs, msg, has_args,
1095 &msglen, &freelen, &num_wreply_segments, &len);
1096 if (status == SVC_RDMA_SUCCESS) {
1097 op = RDMA_NOMSG;
1098 cl = NULL;
1099 goto rdma_writed_long_reply_out;
1100 }
1101 else
1102 goto out;
1103 }
1104 status = svc_compose_rpcmsg(clone_xprt, conn, xdr_results,
1105 xdr_location, &rpcreply, &xdrs, msg,
1106 has_args, &msglen, &len);
1107 if (status != SVC_RDMA_SUCCESS)
1108 goto out;
1109
1110 op = RDMA_MSG;
1111
1112 cl = xdrrdma_clist(xdrs);
1113 cl = NULL;
1114 if (cl != NULL) {
1115 cmn_err(CE_NOTE,
1116 "svc_rdma_ksend: Should not provide non-null"
1117 "read chunk list to client\n");
1118 }
1119
1120 rdma_writed_long_reply_out:
1121
1122 clmsg.type = SEND_BUFFER;
1123 if (RDMA_BUF_ALLOC(conn, &clmsg)) {
1124 rdma_buf_free(conn, &rpcreply);
1125 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!");
1126 goto out;
1127 }
1128
1129 #ifdef DYNAMIC_CREDIT_CONTROL
1130 svc_grant_credit(conn, &rdma_credit);
1131 #else
1132 rdma_credit = rdma_bufs_granted;
1133 #endif
1134
1135 vers = RPCRDMA_VERS;
1136 xdrs = &rxdrs;
1137 xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE);
1138 (*(uint32_t *) clmsg.addr) = msg->rm_xid;
1139 /* Skip xid and set the xdr position accordingly. */
1140 XDR_SETPOS(xdrs, sizeof(uint32_t));
1141 if (!xdr_u_int(xdrs, &vers) ||
1142 !xdr_u_int(xdrs, &rdma_credit) || !xdr_u_int(xdrs, &op)) {
1143 rdma_buf_free(conn, &rpcreply);
1144 rdma_buf_free(conn, &clmsg);
1145 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed");
1146 goto out;
1147 }
1148
1149 /*
1150 * Now XDR the read chunk list, actually always NULL
1151 */
1152 (void) xdr_do_clist(xdrs, &cl);
1153
1154 /*
1155 * encode write list -- we already drove RDMA_WRITEs
1156 */
1157 wcl = vd->wlist;
1158 if (!xdr_encode_wlist(xdrs, wcl, num_wsegment)) {
1159 cmn_err(CE_NOTE,
1160 "svc_rdma_ksend: xdr_encode_wlist failed: "
1161 "wcl=%p", (void *) wcl);
1162 rdma_buf_free(conn, &rpcreply);
1163 rdma_buf_free(conn, &clmsg);
1164 goto out;
1165 }
1166
1167 /*
1168 * XDR encode the RDMA_REPLY write chunk
1169 */
1170 (void) xdr_encode_reply_wchunk(xdrs, vd->reply_cl, num_wreply_segments);
1171
1172 clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr,
1173 NULL, NULL);
1174
1175 if (op == RDMA_MSG) {
1176 clist_add(&sendlist, 0, len, &rpcreply.handle,
1177 rpcreply.addr, NULL, NULL);
1178 }
1179
1180 #if defined(ASYNC_SERVER_DEREG)
1181 status = RDMA_SEND_NW(conn, sendlist, msg->rm_xid, (caddr_t)conn,
1182 (caddr_t)vd->wlist,
1183 templen,
1184 (caddr_t)reply_cl,
1185 freelen, num_wsegment, num_wreply_segments );
1186 #else
1187 status = RDMA_SEND(conn, sendlist, msg->rm_xid);
1188 #endif
1189 if (status != RDMA_SUCCESS) {
1190 goto out;
1191 }
1192
1193 retval = TRUE;
1194
1195 out:
1196
1197 /*
1198 * Free up sendlist chunks
1199 */
1200 if (sendlist != NULL)
1201 clist_free(sendlist);
1202
1203 /*
1204 * Destroy private data for xdr rdma
1205 */
1206 if ((clone_xprt->xp_xdrout).x_private)
1207 XDR_DESTROY(&(clone_xprt->xp_xdrout));
1208 if (rxdrs.x_private)
1209 XDR_DESTROY(&rxdrs);
1210
1211 /*
1212 * This is completely disgusting. If public is set it is
1213 * a pointer to a structure whose first field is the address
1214 * of the function to free that structure and any related
1215 * stuff. (see rrokfree in nfs_xdr.c).
1216 */
1217 if (xdrs->x_public) {
1218 /* LINTED pointer alignment */
1219 (**((int (**)()) xdrs->x_public)) (xdrs->x_public);
1220 }
1221
1222 if (vd->wlist != NULL) {
1223 #if defined(ASYNC_SERVER_DEREG)
1224 if(!retval) {
1225 #endif
1226 wcl = vd->wlist;
1227 (void) clist_deregister(vd->conn, wcl, TRUE);
1228 #ifdef SERVER_REG_CACHE
1229 RDMA_FREE_SERVER_CACHE_BUF(vd->conn, (rib_lrc_entry_t *)wcl->long_reply_buf);
1230 #else
1231 if(templen)
1232 kmem_free((void *) (vd->wlist)->c_saddr, templen);
1233 #endif
1234 kmem_free(vd->wlist, num_wsegment * sizeof(struct clist));
1235 #if defined(ASYNC_SERVER_DEREG)
1236 }
1237 #endif
1238 vd->wlist = NULL;
1239 }
1240
1241
1242 if(vd->reply_cl != NULL){
1243 #if defined(ASYNC_SERVER_DEREG)
1244 if(!retval) {
1245 #endif
1246 (void) clist_deregister(conn, reply_cl, TRUE);
1247 #ifdef SERVER_REG_CACHE
1248 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)reply_cl->long_reply_buf);
1249 #else
1250 #ifdef DEBUG
1251 if(rdma_svc_debug > 1)
1252 cmn_err(CE_NOTE, "Freeing up %p of length %d\n",reply_cl->c_saddr,freelen);
1253 #endif
1254 if(freelen)
1255 kmem_free((void *)(reply_cl->c_saddr), freelen);
1256 #endif
1257 kmem_free((void *)vd->reply_cl, num_wreply_segments * sizeof(struct clist));
1258 #if defined(ASYNC_SERVER_DEREG)
1259 }
1260 #endif
1261 vd->reply_cl = NULL;
1262 }
1263 return (retval);
1264 }
1265
1266 /*
1267 * Deserialize arguments.
1268 */
1269 static bool_t
1270 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr)
1271 {
1272 if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin,
1273 xdr_args, args_ptr)) != TRUE)
1274 return (FALSE);
1275 return (TRUE);
1276 }
1277
1278 static bool_t
1279 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args,
1280 caddr_t args_ptr)
1281 {
1282 struct clone_rdma_data *vd;
1507 uint32_t drhash;
1508
1509 ASSERT(MUTEX_HELD(&rdmadupreq_lock));
1510
1511 drhash = (uint32_t)DRHASH(dr);
1512 drt = rdmadrhashtbl[drhash];
1513 while (drt != NULL) {
1514 if (drt == dr) {
1515 rdmadrhashstat[drhash]--;
1516 if (drtprev == NULL) {
1517 rdmadrhashtbl[drhash] = drt->dr_chain;
1518 } else {
1519 drtprev->dr_chain = drt->dr_chain;
1520 }
1521 return;
1522 }
1523 drtprev = drt;
1524 drt = drt->dr_chain;
1525 }
1526 }
1527
1528 bool_t
1529 rdma_get_wchunk_seg(struct svc_req *req, iovec_t *iov)
1530 {
1531 struct clone_rdma_data *rcd;
1532 struct clist *clist;
1533 uint32_t tlen;
1534
1535 if (req->rq_xprt->xp_type != T_RDMA) {
1536 return (FALSE);
1537 }
1538
1539 rcd = (struct clone_rdma_data *)(&req->rq_xprt->xp_p2buf);
1540 if (rcd->wlist == NULL) {
1541 return (FALSE);
1542 }
1543 tlen = 0;
1544 clist = rcd->wlist;
1545 while(clist){
1546 tlen += clist->c_len;
1547 clist = clist->c_next;
1548 }
1549
1550 /*
1551 * set iov to addr+len of first segment of first wchunk of
1552 * wlist sent by client. krecv() already malloc'd a buffer
1553 * large enough, but registration is deferred until we write
1554 * the buffer back to (NFS) client using RDMA_WRITE.
1555 */
1556 iov->iov_base = (caddr_t)rcd->wlist->c_saddr;
1557 iov->iov_len = tlen;
1558
1559 return (TRUE);
1560 }
1561
|