Sdiff svc_rdma.c


14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 /* 29 * Portions of this source code were derived from Berkeley 30 * 4.3 BSD under license from the Regents of the University of 31 * California. 32 */ 33 34 #pragma ident "@(#)svc_rdma.c 1.8 05/06/10 SMI" 35 36 /* 37 * Server side of RPC over RDMA in the kernel. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/types.h> 42 #include <sys/user.h> 43 #include <sys/sysmacros.h> 44 #include <sys/proc.h> 45 #include <sys/file.h> 46 #include <sys/errno.h> 47 #include <sys/kmem.h> 48 #include <sys/debug.h> 49 #include <sys/systm.h> 50 #include <sys/cmn_err.h> 51 #include <sys/kstat.h> 52 #include <sys/vtrace.h> 53 #include <sys/debug.h> 54 55 #include <rpc/types.h> 56 #include <rpc/xdr.h> 57 #include <rpc/auth.h> 58 #include <rpc/clnt.h> 59 #include <rpc/rpc_msg.h> 60 #include <rpc/svc.h> 61 #include <rpc/rpc_rdma.h> 62 #include <sys/ddi.h> 63 #include <sys/sunddi.h> 64 65 #include <inet/common.h> 66 #include <inet/ip.h> 67 #include <inet/ip6.h> 68 69 /* 70 * RDMA transport specific data associated with SVCMASTERXPRT 71 */ 72 struct rdma_data { 73 SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */ 74 struct rdma_svc_data rd_data; /* rdma data */ 75 rdma_mod_t *r_mod; /* RDMA module containing ops ptr */ 76 }; 77 78 /* 79 * Plugin connection specific data stashed away in clone SVCXPRT 80 */ 81 struct clone_rdma_data { 82 CONN *conn; /* RDMA connection */ 83 rdma_buf_t rpcbuf; /* RPC req/resp buffer */ 84 }; 85 86 #ifdef DEBUG 87 int rdma_svc_debug = 0; 88 #endif 89 90 #define MAXADDRLEN 128 /* max length for address mask */ 91 92 /* 93 * Routines exported through ops vector. 94 */ 95 static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *); 96 static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *); 97 static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t); 98 static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t); 99 void svc_rdma_kdestroy(SVCMASTERXPRT *); 100 static int svc_rdma_kdup(struct svc_req *, caddr_t, int, 101 struct dupreq **, bool_t *); 102 static void svc_rdma_kdupdone(struct dupreq *, caddr_t, 103 void (*)(), int, int); 104 static int32_t *svc_rdma_kgetres(SVCXPRT *, int); 105 static void svc_rdma_kfreeres(SVCXPRT *); 106 static void svc_rdma_kclone_destroy(SVCXPRT *); 107 static void svc_rdma_kstart(SVCMASTERXPRT *); 108 void svc_rdma_kstop(SVCMASTERXPRT *); 109 110 /* 111 * Server transport operations vector. 112 */ 113 struct svc_ops rdma_svc_ops = { 114 svc_rdma_krecv, /* Get requests */ 115 svc_rdma_kgetargs, /* Deserialize arguments */ 116 svc_rdma_ksend, /* Send reply */ 117 svc_rdma_kfreeargs, /* Free argument data space */ 118 svc_rdma_kdestroy, /* Destroy transport handle */ 119 svc_rdma_kdup, /* Check entry in dup req cache */ 120 svc_rdma_kdupdone, /* Mark entry in dup req cache as done */ 121 svc_rdma_kgetres, /* Get pointer to response buffer */ 122 svc_rdma_kfreeres, /* Destroy pre-serialized response header */ 123 svc_rdma_kclone_destroy, /* Destroy a clone xprt */ 124 svc_rdma_kstart /* Tell `ready-to-receive' to rpcmod */ 125 }; 126 127 /* 128 * Server statistics 129 * NOTE: This structure type is duplicated in the NFS fast path. 130 */ 131 struct { 132 kstat_named_t rscalls; 133 kstat_named_t rsbadcalls; 134 kstat_named_t rsnullrecv; 135 kstat_named_t rsbadlen; 136 kstat_named_t rsxdrcall; 137 kstat_named_t rsdupchecks; 138 kstat_named_t rsdupreqs; 139 kstat_named_t rslongrpcs; 140 } rdmarsstat = { 141 { "calls", KSTAT_DATA_UINT64 }, 142 { "badcalls", KSTAT_DATA_UINT64 }, 143 { "nullrecv", KSTAT_DATA_UINT64 }, 144 { "badlen", KSTAT_DATA_UINT64 },
155 156 /* 157 * Create a transport record. 158 * The transport record, output buffer, and private data structure 159 * are allocated. The output buffer is serialized into using xdrmem. 160 * There is one transport record per user process which implements a 161 * set of services. 162 */ 163 /* ARGSUSED */ 164 int 165 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id, 166 rdma_xprt_group_t *started_xprts) 167 { 168 int error; 169 SVCMASTERXPRT *xprt; 170 struct rdma_data *rd; 171 rdma_registry_t *rmod; 172 rdma_xprt_record_t *xprt_rec; 173 queue_t *q; 174 175 /* 176 * modload the RDMA plugins is not already done. 177 */ 178 if (!rdma_modloaded) { 179 mutex_enter(&rdma_modload_lock); 180 if (!rdma_modloaded) { 181 error = rdma_modload(); 182 } 183 mutex_exit(&rdma_modload_lock); 184 185 if (error) 186 return (error); 187 } 188 189 /* 190 * master_xprt_count is the count of master transport handles 191 * that were successfully created and are ready to recieve for 192 * RDMA based access. 193 */ 194 error = 0;
348 349 /* 350 * Call the stop listener routine for each plugin. 351 */ 352 (*rmod->rdma_ops->rdma_svc_stop)(svcdata); 353 if (svcdata->active) 354 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC" 355 " listener"); 356 } 357 358 /* ARGSUSED */ 359 static void 360 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt) 361 { 362 } 363 364 static bool_t 365 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg) 366 { 367 XDR *xdrs; 368 rdma_stat status; 369 struct recv_data *rdp = (struct recv_data *)mp->b_rptr; 370 CONN *conn; 371 struct clone_rdma_data *vd; 372 struct clist *cl; 373 uint_t vers, op, pos; 374 uint32_t xid; 375 376 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 377 RSSTAT_INCR(rscalls); 378 conn = rdp->conn; 379 380 /* 381 * Post a receive descriptor on this 382 * endpoint to ensure all packets are received. 383 */ 384 status = rdma_svc_postrecv(conn); 385 if (status != RDMA_SUCCESS) { 386 cmn_err(CE_NOTE, 387 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status); 388 } 389 390 if (rdp->status != 0) { 391 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 392 RDMA_REL_CONN(conn); 393 RSSTAT_INCR(rsbadcalls); 394 freeb(mp); 395 return (FALSE); 396 } 397 398 /* 399 * Decode rpc message 400 */ 401 xdrs = &clone_xprt->xp_xdrin; 402 xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE); 403 404 /* 405 * Get the XID 406 */ 407 /* 408 * Treat xid as opaque (xid is the first entity 409 * in the rpc rdma message). 410 */ 411 xid = *(uint32_t *)rdp->rpcmsg.addr; 412 /* Skip xid and set the xdr position accordingly. */ 413 XDR_SETPOS(xdrs, sizeof (uint32_t)); 414 if (! xdr_u_int(xdrs, &vers) || 415 ! xdr_u_int(xdrs, &op)) { 416 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed"); 417 XDR_DESTROY(xdrs); 418 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 419 RDMA_REL_CONN(conn); 420 freeb(mp); 421 RSSTAT_INCR(rsbadcalls); 422 return (FALSE); 423 } 424 if (op == RDMA_DONE) { 425 /* 426 * Should not get RDMA_DONE 427 */ 428 freeb(mp); 429 XDR_DESTROY(xdrs); 430 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 431 RDMA_REL_CONN(conn); 432 RSSTAT_INCR(rsbadcalls); 433 return (FALSE); /* no response */ 434 } 435 436 #ifdef DEBUG 437 if (rdma_svc_debug) 438 printf("svc_rdma_krecv: recv'd call xid %u\n", xid); 439 #endif 440 /* 441 * Now decode the chunk list 442 */ 443 cl = NULL; 444 if (! xdr_do_clist(xdrs, &cl)) { 445 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed"); 446 } 447 448 /* 449 * A chunk at 0 offset indicates that the RPC call message 450 * is in a chunk. Get the RPC call message chunk. 451 */ 452 if (cl != NULL && op == RDMA_NOMSG) { 453 struct clist *cllong; /* Long RPC chunk */ 454 455 /* Remove RPC call message chunk from chunklist */ 456 cllong = cl; 457 cl = cl->c_next; 458 cllong->c_next = NULL; 459 460 /* Allocate and register memory for the RPC call msg chunk */ 461 cllong->c_daddr = (uint64)(uintptr_t) 462 kmem_alloc(cllong->c_len, KM_SLEEP); 463 if (cllong->c_daddr == NULL) { 464 cmn_err(CE_WARN, 465 "svc_rdma_krecv: no memory for rpc call"); 466 XDR_DESTROY(xdrs); 467 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 468 RDMA_REL_CONN(conn); 469 freeb(mp); 470 RSSTAT_INCR(rsbadcalls); 471 clist_free(cl); 472 clist_free(cllong); 473 return (FALSE); 474 } 475 status = clist_register(conn, cllong, 0); 476 if (status) { 477 cmn_err(CE_WARN, 478 "svc_rdma_krecv: clist_register failed"); 479 kmem_free((void *)(uintptr_t)cllong->c_daddr, 480 cllong->c_len); 481 XDR_DESTROY(xdrs); 482 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 483 RDMA_REL_CONN(conn); 484 freeb(mp); 485 RSSTAT_INCR(rsbadcalls); 486 clist_free(cl); 487 clist_free(cllong); 488 return (FALSE); 489 } 490 491 /* 492 * Now read the RPC call message in 493 */ 494 status = RDMA_READ(conn, cllong, WAIT); 495 if (status) { 496 cmn_err(CE_WARN, 497 "svc_rdma_krecv: rdma_read failed %d", status); 498 (void) clist_deregister(conn, cllong, 0); 499 kmem_free((void *)(uintptr_t)cllong->c_daddr, 500 cllong->c_len); 501 XDR_DESTROY(xdrs); 502 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 503 RDMA_REL_CONN(conn); 504 freeb(mp); 505 RSSTAT_INCR(rsbadcalls); 506 clist_free(cl); 507 clist_free(cllong); 508 return (FALSE); 509 } 510 /* 511 * Sync memory for CPU after DMA 512 */ 513 status = clist_syncmem(conn, cllong, 0); 514 515 /* 516 * Deregister the chunk 517 */ 518 (void) clist_deregister(conn, cllong, 0); 519 520 /* 521 * Setup the XDR for the RPC call message 522 */ 523 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr, 524 cllong->c_len, 0, cl, XDR_DECODE, conn); 525 vd->rpcbuf.type = CHUNK_BUFFER; 526 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr; 527 vd->rpcbuf.len = cllong->c_len; 528 vd->rpcbuf.handle.mrc_rmr = 0; 529 530 /* 531 * Free the chunk element with the Long RPC details and 532 * the message received. 533 */ 534 clist_free(cllong); 535 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 536 } else { 537 pos = XDR_GETPOS(xdrs); 538 539 /* 540 * Now the RPC call message header 541 */ 542 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos, 543 rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn); 544 vd->rpcbuf = rdp->rpcmsg; 545 } 546 if (! xdr_callmsg(xdrs, msg)) { 547 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed"); 548 if (cl != NULL) 549 clist_free(cl); 550 XDR_DESTROY(xdrs); 551 rdma_buf_free(conn, &vd->rpcbuf); 552 RDMA_REL_CONN(conn); 553 freeb(mp); 554 RSSTAT_INCR(rsxdrcall); 555 RSSTAT_INCR(rsbadcalls); 556 return (FALSE); 557 } 558 559 /* 560 * Point the remote transport address in the service_transport 561 * handle at the address in the request. 562 */ 563 clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf; 564 clone_xprt->xp_rtaddr.len = conn->c_raddr.len; 565 clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len; 566 567 #ifdef DEBUG 568 if (rdma_svc_debug) { 569 struct sockaddr_in *sin4; 570 char print_addr[INET_ADDRSTRLEN]; 571 572 sin4 = (struct sockaddr_in *)clone_xprt->xp_rtaddr.buf; 573 bzero(print_addr, INET_ADDRSTRLEN); 574 (void) inet_ntop(AF_INET, 575 &sin4->sin_addr, print_addr, INET_ADDRSTRLEN); 576 cmn_err(CE_NOTE, 577 "svc_rdma_krecv: remote clnt_addr: %s", print_addr); 578 } 579 #endif 580 581 clone_xprt->xp_xid = xid; 582 vd->conn = conn; 583 freeb(mp); 584 return (TRUE); 585 } 586 587 /* 588 * Send rpc reply. 589 */ 590 static bool_t 591 svc_rdma_ksend(SVCXPRT *clone_xprt, struct rpc_msg *msg) 592 { 593 struct clone_rdma_data *vd; 594 XDR *xdrs = &(clone_xprt->xp_xdrout), rxdrs; 595 int retval = FALSE; 596 xdrproc_t xdr_results; 597 caddr_t xdr_location; 598 bool_t has_args, reg = FALSE; 599 uint_t len, op; 600 uint_t vers; 601 struct clist *cl = NULL, *cle = NULL; 602 struct clist *sendlist = NULL; 603 int status; 604 int msglen; 605 rdma_buf_t clmsg, longreply, rpcreply; 606 607 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 608 609 /* 610 * If there is a result procedure specified in the reply message, 611 * it will be processed in the xdr_replymsg and SVCAUTH_WRAP. 612 * We need to make sure it won't be processed twice, so we null 613 * it for xdr_replymsg here. 614 */ 615 has_args = FALSE; 616 if (msg->rm_reply.rp_stat == MSG_ACCEPTED && 617 msg->rm_reply.rp_acpt.ar_stat == SUCCESS) { 618 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) { 619 has_args = TRUE; 620 xdr_location = msg->acpted_rply.ar_results.where; 621 msg->acpted_rply.ar_results.proc = xdr_void; 622 msg->acpted_rply.ar_results.where = NULL; 623 } 624 } 625 626 /* 627 * Get the size of the rpc reply message. Need this 628 * to determine if the rpc reply message will fit in 629 * the pre-allocated RDMA buffers. If the rpc reply 630 * message length is greater that the pre-allocated 631 * buffers then, a one time use buffer is allocated 632 * and registered for this rpc reply. 633 */ 634 msglen = xdr_sizeof(xdr_replymsg, msg); 635 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor != RPCSEC_GSS) { 636 msglen += xdrrdma_sizeof(xdr_results, xdr_location, 637 rdma_minchunk); 638 if (msglen > RPC_MSG_SZ) { 639 640 /* 641 * Allocate chunk buffer for rpc reply 642 */ 643 rpcreply.type = CHUNK_BUFFER; 644 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP); 645 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP); 646 cle->c_xdroff = 0; 647 cle->c_len = rpcreply.len = msglen; 648 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr; 649 cle->c_next = NULL; 650 xdrrdma_create(xdrs, rpcreply.addr, msglen, 651 rdma_minchunk, cle, XDR_ENCODE, NULL); 652 op = RDMA_NOMSG; 653 } else { 654 /* 655 * Get a pre-allocated buffer for rpc reply 656 */ 657 rpcreply.type = SEND_BUFFER; 658 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) { 659 cmn_err(CE_WARN, 660 "svc_rdma_ksend: no free buffers!"); 661 return (retval); 662 } 663 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len, 664 rdma_minchunk, NULL, XDR_ENCODE, NULL); 665 op = RDMA_MSG; 666 } 667 668 /* 669 * Initialize the XDR encode stream. 670 */ 671 msg->rm_xid = clone_xprt->xp_xid; 672 673 if (!(xdr_replymsg(xdrs, msg) && 674 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs, 675 xdr_results, xdr_location)))) { 676 rdma_buf_free(vd->conn, &rpcreply); 677 if (cle) 678 clist_free(cle); 679 cmn_err(CE_WARN, 680 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP " 681 "failed"); 682 goto out; 683 } 684 len = XDR_GETPOS(xdrs); 685 } 686 if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor == RPCSEC_GSS) { 687 688 /* 689 * For RPCSEC_GSS since we cannot accurately presize the 690 * buffer required for encoding, we assume that its going 691 * to be a Long RPC to start with. We also create the 692 * the XDR stream with min_chunk set to 0 which instructs 693 * the XDR layer to not chunk the incoming byte stream. 694 */ 695 msglen += 2 * MAX_AUTH_BYTES + 2 * sizeof (struct opaque_auth); 696 msglen += xdr_sizeof(xdr_results, xdr_location); 697 698 /* 699 * Long RPC. Allocate one time use custom buffer. 700 */ 701 longreply.type = CHUNK_BUFFER; 702 longreply.addr = kmem_zalloc(msglen, KM_SLEEP); 703 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP); 704 cle->c_xdroff = 0; 705 cle->c_len = longreply.len = msglen; 706 cle->c_saddr = (uint64)(uintptr_t)longreply.addr; 707 cle->c_next = NULL; 708 xdrrdma_create(xdrs, longreply.addr, msglen, 0, cle, 709 XDR_ENCODE, NULL); 710 op = RDMA_NOMSG; 711 /* 712 * Initialize the XDR encode stream. 713 */ 714 msg->rm_xid = clone_xprt->xp_xid; 715 716 if (!(xdr_replymsg(xdrs, msg) && 717 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs, 718 xdr_results, xdr_location)))) { 719 if (longreply.addr != xdrs->x_base) { 720 longreply.addr = xdrs->x_base; 721 longreply.len = xdr_getbufsize(xdrs); 722 } 723 rdma_buf_free(vd->conn, &longreply); 724 if (cle) 725 clist_free(cle); 726 cmn_err(CE_WARN, 727 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP " 728 "failed"); 729 goto out; 730 } 731 732 /* 733 * If we had to allocate a new buffer while encoding 734 * then update the addr and len. 735 */ 736 if (longreply.addr != xdrs->x_base) { 737 longreply.addr = xdrs->x_base; 738 longreply.len = xdr_getbufsize(xdrs); 739 } 740 741 len = XDR_GETPOS(xdrs); 742 743 /* 744 * If it so happens that the encoded message is after all 745 * not long enough to be a Long RPC then allocate a 746 * SEND_BUFFER and copy the encoded message into it. 747 */ 748 if (len > RPC_MSG_SZ) { 749 rpcreply.type = CHUNK_BUFFER; 750 rpcreply.addr = longreply.addr; 751 rpcreply.len = longreply.len; 752 } else { 753 clist_free(cle); 754 XDR_DESTROY(xdrs); 755 /* 756 * Get a pre-allocated buffer for rpc reply 757 */ 758 rpcreply.type = SEND_BUFFER; 759 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) { 760 cmn_err(CE_WARN, 761 "svc_rdma_ksend: no free buffers!"); 762 rdma_buf_free(vd->conn, &longreply); 763 return (retval); 764 } 765 bcopy(longreply.addr, rpcreply.addr, len); 766 xdrrdma_create(xdrs, rpcreply.addr, len, 0, NULL, 767 XDR_ENCODE, NULL); 768 rdma_buf_free(vd->conn, &longreply); 769 op = RDMA_MSG; 770 } 771 } 772 773 if (has_args == FALSE) { 774 775 if (msglen > RPC_MSG_SZ) { 776 777 /* 778 * Allocate chunk buffer for rpc reply 779 */ 780 rpcreply.type = CHUNK_BUFFER; 781 rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP); 782 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP); 783 cle->c_xdroff = 0; 784 cle->c_len = rpcreply.len = msglen; 785 cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr; 786 cle->c_next = NULL; 787 xdrrdma_create(xdrs, rpcreply.addr, msglen, 788 rdma_minchunk, cle, XDR_ENCODE, NULL); 789 op = RDMA_NOMSG; 790 } else { 791 /* 792 * Get a pre-allocated buffer for rpc reply 793 */ 794 rpcreply.type = SEND_BUFFER; 795 if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) { 796 cmn_err(CE_WARN, 797 "svc_rdma_ksend: no free buffers!"); 798 return (retval); 799 } 800 xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len, 801 rdma_minchunk, NULL, XDR_ENCODE, NULL); 802 op = RDMA_MSG; 803 } 804 805 /* 806 * Initialize the XDR encode stream. 807 */ 808 msg->rm_xid = clone_xprt->xp_xid; 809 810 if (!xdr_replymsg(xdrs, msg)) { 811 rdma_buf_free(vd->conn, &rpcreply); 812 if (cle) 813 clist_free(cle); 814 cmn_err(CE_WARN, 815 "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP " 816 "failed"); 817 goto out; 818 } 819 len = XDR_GETPOS(xdrs); 820 } 821 822 /* 823 * Get clist and a buffer for sending it across 824 */ 825 cl = xdrrdma_clist(xdrs); 826 clmsg.type = SEND_BUFFER; 827 if (RDMA_BUF_ALLOC(vd->conn, &clmsg)) { 828 rdma_buf_free(vd->conn, &rpcreply); 829 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!"); 830 goto out; 831 } 832 833 /* 834 * Now register the chunks in the list 835 */ 836 if (cl != NULL) { 837 status = clist_register(vd->conn, cl, 1); 838 if (status != RDMA_SUCCESS) { 839 rdma_buf_free(vd->conn, &clmsg); 840 cmn_err(CE_WARN, 841 "svc_rdma_ksend: clist register failed"); 842 goto out; 843 } 844 reg = TRUE; 845 } 846 847 /* 848 * XDR the XID, vers, and op 849 */ 850 /* 851 * Treat xid as opaque (xid is the first entity 852 * in the rpc rdma message). 853 */ 854 vers = RPCRDMA_VERS; 855 xdrs = &rxdrs; 856 xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE); 857 (*(uint32_t *)clmsg.addr) = msg->rm_xid; 858 /* Skip xid and set the xdr position accordingly. */ 859 XDR_SETPOS(xdrs, sizeof (uint32_t)); 860 if (! xdr_u_int(xdrs, &vers) || 861 ! xdr_u_int(xdrs, &op)) { 862 rdma_buf_free(vd->conn, &rpcreply); 863 rdma_buf_free(vd->conn, &clmsg); 864 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed"); 865 goto out; 866 } 867 868 /* 869 * Now XDR the chunk list 870 */ 871 (void) xdr_do_clist(xdrs, &cl); 872 873 clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr, 874 NULL, NULL); 875 876 if (op == RDMA_MSG) { 877 clist_add(&sendlist, 0, len, &rpcreply.handle, rpcreply.addr, 878 NULL, NULL); 879 } else { 880 cl->c_len = len; 881 RSSTAT_INCR(rslongrpcs); 882 } 883 884 /* 885 * Send the reply message to the client 886 */ 887 if (cl != NULL) { 888 status = clist_syncmem(vd->conn, cl, 1); 889 if (status != RDMA_SUCCESS) { 890 rdma_buf_free(vd->conn, &rpcreply); 891 rdma_buf_free(vd->conn, &clmsg); 892 goto out; 893 } 894 #ifdef DEBUG 895 if (rdma_svc_debug) 896 printf("svc_rdma_ksend: chunk response len %d xid %u\n", 897 cl->c_len, msg->rm_xid); 898 #endif 899 /* 900 * Post a receive buffer because we expect a RDMA_DONE 901 * message. 902 */ 903 status = rdma_svc_postrecv(vd->conn); 904 905 /* 906 * Send the RPC reply message and wait for RDMA_DONE 907 */ 908 status = RDMA_SEND_RESP(vd->conn, sendlist, msg->rm_xid); 909 if (status != RDMA_SUCCESS) { 910 #ifdef DEBUG 911 if (rdma_svc_debug) 912 cmn_err(CE_NOTE, "svc_rdma_ksend: " 913 "rdma_send_resp failed %d", status); 914 #endif 915 goto out; 916 } 917 #ifdef DEBUG 918 if (rdma_svc_debug) 919 printf("svc_rdma_ksend: got RDMA_DONE xid %u\n", msg->rm_xid); 920 #endif 921 } else { 922 #ifdef DEBUG 923 if (rdma_svc_debug) 924 printf("svc_rdma_ksend: msg response xid %u\n", msg->rm_xid); 925 #endif 926 status = RDMA_SEND(vd->conn, sendlist, msg->rm_xid); 927 if (status != RDMA_SUCCESS) { 928 #ifdef DEBUG 929 if (rdma_svc_debug) 930 cmn_err(CE_NOTE, "svc_rdma_ksend: " 931 "rdma_send failed %d", status); 932 #endif 933 goto out; 934 } 935 } 936 937 retval = TRUE; 938 out: 939 /* 940 * Deregister the chunks 941 */ 942 if (cl != NULL) { 943 if (reg) 944 (void) clist_deregister(vd->conn, cl, 1); 945 if (op == RDMA_NOMSG) { 946 /* 947 * Long RPC reply in chunk. Free it up. 948 */ 949 rdma_buf_free(vd->conn, &rpcreply); 950 } 951 clist_free(cl); 952 } 953 954 /* 955 * Free up sendlist chunks 956 */ 957 if (sendlist != NULL) 958 clist_free(sendlist); 959 960 /* 961 * Destroy private data for xdr rdma 962 */ 963 XDR_DESTROY(&(clone_xprt->xp_xdrout)); 964 965 /* 966 * This is completely disgusting. If public is set it is 967 * a pointer to a structure whose first field is the address 968 * of the function to free that structure and any related 969 * stuff. (see rrokfree in nfs_xdr.c). 970 */ 971 if (xdrs->x_public) { 972 /* LINTED pointer alignment */ 973 (**((int (**)())xdrs->x_public))(xdrs->x_public); 974 } 975 976 return (retval); 977 } 978 979 /* 980 * Deserialize arguments. 981 */ 982 static bool_t 983 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr) 984 { 985 if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin, 986 xdr_args, args_ptr)) != TRUE) 987 return (FALSE); 988 return (TRUE); 989 } 990 991 static bool_t 992 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, 993 caddr_t args_ptr) 994 { 995 struct clone_rdma_data *vd;
1220 uint32_t drhash; 1221 1222 ASSERT(MUTEX_HELD(&rdmadupreq_lock)); 1223 1224 drhash = (uint32_t)DRHASH(dr); 1225 drt = rdmadrhashtbl[drhash]; 1226 while (drt != NULL) { 1227 if (drt == dr) { 1228 rdmadrhashstat[drhash]--; 1229 if (drtprev == NULL) { 1230 rdmadrhashtbl[drhash] = drt->dr_chain; 1231 } else { 1232 drtprev->dr_chain = drt->dr_chain; 1233 } 1234 return; 1235 } 1236 drtprev = drt; 1237 drt = drt->dr_chain; 1238 } 1239 }


14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 /* 29 * Portions of this source code were derived from Berkeley 30 * 4.3 BSD under license from the Regents of the University of 31 * California. 32 */ 33 34 /* Copyright (c) 2006, The Ohio State University. All rights reserved. 35 * 36 * Portions of this source code is developed by the team members of 37 * The Ohio State University's Network-Based Computing Laboratory (NBCL), 38 * headed by Professor Dhabaleswar K. (DK) Panda. 39 * 40 * Acknowledgements to contributions from developors: 41 * Ranjit Noronha: noronha@cse.ohio-state.edu 42 * Lei Chai : chail@cse.ohio-state.edu 43 * Weikuan Yu : yuw@cse.ohio-state.edu 44 * 45 */ 46 47 #pragma ident "@(#)svc_rdma.c 1.8 05/06/10 SMI" 48 49 /* 50 * Server side of RPC over RDMA in the kernel. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/types.h> 55 #include <sys/user.h> 56 #include <sys/sysmacros.h> 57 #include <sys/proc.h> 58 #include <sys/file.h> 59 #include <sys/errno.h> 60 #include <sys/kmem.h> 61 #include <sys/debug.h> 62 #include <sys/systm.h> 63 #include <sys/cmn_err.h> 64 #include <sys/kstat.h> 65 #include <sys/vtrace.h> 66 #include <sys/debug.h> 67 68 #include <rpc/types.h> 69 #include <rpc/xdr.h> 70 #include <rpc/auth.h> 71 #include <rpc/clnt.h> 72 #include <rpc/rpc_msg.h> 73 #include <rpc/svc.h> 74 #include <rpc/rpc_rdma.h> 75 #include <sys/ddi.h> 76 #include <sys/sunddi.h> 77 78 #include <inet/common.h> 79 #include <inet/ip.h> 80 #include <inet/ip6.h> 81 82 #include <nfs/nfs.h> 83 84 #define SVC_RDMA_SUCCESS 0 85 #define SVC_RDMA_FAIL -1 86 87 #define SVC_CREDIT_FACTOR (0.5) 88 89 uint32_t rdma_bufs_granted = RDMA_BUFS_GRANT; 90 extern xdrproc_t x_READ3res; 91 92 /* 93 * RDMA transport specific data associated with SVCMASTERXPRT 94 */ 95 struct rdma_data { 96 SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */ 97 struct rdma_svc_data rd_data; /* rdma data */ 98 rdma_mod_t *r_mod; /* RDMA module containing ops ptr */ 99 }; 100 101 /* 102 * Plugin connection specific data stashed away in clone SVCXPRT 103 */ 104 struct clone_rdma_data { 105 CONN *conn; /* RDMA connection */ 106 rdma_buf_t rpcbuf; /* RPC req/resp buffer */ 107 struct clist *reply_cl; /* reply chunk buffer info */ 108 struct clist *wlist; /* write list clist */ 109 }; 110 111 #ifdef DEBUG 112 int rdma_svc_debug = 0; 113 #endif 114 115 #define MAXADDRLEN 128 /* max length for address mask */ 116 117 /* 118 * Routines exported through ops vector. 119 */ 120 static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *); 121 static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *); 122 static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t); 123 static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t); 124 void svc_rdma_kdestroy(SVCMASTERXPRT *); 125 static int svc_rdma_kdup(struct svc_req *, caddr_t, int, 126 struct dupreq **, bool_t *); 127 static void svc_rdma_kdupdone(struct dupreq *, caddr_t, 128 void (*)(), int, int); 129 static int32_t *svc_rdma_kgetres(SVCXPRT *, int); 130 static void svc_rdma_kfreeres(SVCXPRT *); 131 static void svc_rdma_kclone_destroy(SVCXPRT *); 132 static void svc_rdma_kstart(SVCMASTERXPRT *); 133 void svc_rdma_kstop(SVCMASTERXPRT *); 134 135 static int svc_process_wlist(struct clone_rdma_data *, xdrproc_t, 136 caddr_t, int *, unsigned int *); 137 138 static int svc_process_long_reply(SVCXPRT *, CONN *, xdrproc_t, 139 caddr_t, caddr_t vd, XDR **, 140 struct rpc_msg *, bool_t, int *, 141 int *, int *, unsigned int *); 142 143 static int svc_compose_rpcmsg(SVCXPRT *, CONN *, xdrproc_t, caddr_t, 144 rdma_buf_t *, XDR **, struct rpc_msg *, 145 bool_t, int *, unsigned int *); 146 #ifdef DYNAMIC_CREDIT_CONTROL 147 static void svc_consume_credit(CONN *); 148 static void svc_compute_credit(CONN *, uint32_t, int, int, int *); 149 static void svc_update_credit(CONN * , int); 150 static void svc_grant_credit(CONN *, uint32_t *); 151 #endif 152 153 /* 154 * Server transport operations vector. 155 */ 156 struct svc_ops rdma_svc_ops = { 157 svc_rdma_krecv, /* Get requests */ 158 svc_rdma_kgetargs, /* Deserialize arguments */ 159 svc_rdma_ksend, /* Send reply */ 160 svc_rdma_kfreeargs, /* Free argument data space */ 161 svc_rdma_kdestroy, /* Destroy transport handle */ 162 svc_rdma_kdup, /* Check entry in dup req cache */ 163 svc_rdma_kdupdone, /* Mark entry in dup req cache as done */ 164 svc_rdma_kgetres, /* Get pointer to response buffer */ 165 svc_rdma_kfreeres, /* Destroy pre-serialized response header */ 166 svc_rdma_kclone_destroy, /* Destroy a clone xprt */ 167 svc_rdma_kstart, /* Tell `ready-to-receive' to rpcmod */ 168 rdma_get_wchunk_seg 169 }; 170 171 /* 172 * Server statistics 173 * NOTE: This structure type is duplicated in the NFS fast path. 174 */ 175 struct { 176 kstat_named_t rscalls; 177 kstat_named_t rsbadcalls; 178 kstat_named_t rsnullrecv; 179 kstat_named_t rsbadlen; 180 kstat_named_t rsxdrcall; 181 kstat_named_t rsdupchecks; 182 kstat_named_t rsdupreqs; 183 kstat_named_t rslongrpcs; 184 } rdmarsstat = { 185 { "calls", KSTAT_DATA_UINT64 }, 186 { "badcalls", KSTAT_DATA_UINT64 }, 187 { "nullrecv", KSTAT_DATA_UINT64 }, 188 { "badlen", KSTAT_DATA_UINT64 },
199 200 /* 201 * Create a transport record. 202 * The transport record, output buffer, and private data structure 203 * are allocated. The output buffer is serialized into using xdrmem. 204 * There is one transport record per user process which implements a 205 * set of services. 206 */ 207 /* ARGSUSED */ 208 int 209 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id, 210 rdma_xprt_group_t *started_xprts) 211 { 212 int error; 213 SVCMASTERXPRT *xprt; 214 struct rdma_data *rd; 215 rdma_registry_t *rmod; 216 rdma_xprt_record_t *xprt_rec; 217 queue_t *q; 218 219 mutex_enter(&rdma_modload_lock); 220 error = rdma_modload(); 221 mutex_exit(&rdma_modload_lock); 222 223 /* 224 * modload the RDMA plugins is not already done. 225 */ 226 if (!rdma_modloaded) { 227 mutex_enter(&rdma_modload_lock); 228 if (!rdma_modloaded) { 229 error = rdma_modload(); 230 } 231 mutex_exit(&rdma_modload_lock); 232 233 if (error) 234 return (error); 235 } 236 237 /* 238 * master_xprt_count is the count of master transport handles 239 * that were successfully created and are ready to recieve for 240 * RDMA based access. 241 */ 242 error = 0;
396 397 /* 398 * Call the stop listener routine for each plugin. 399 */ 400 (*rmod->rdma_ops->rdma_svc_stop)(svcdata); 401 if (svcdata->active) 402 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC" 403 " listener"); 404 } 405 406 /* ARGSUSED */ 407 static void 408 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt) 409 { 410 } 411 412 static bool_t 413 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg) 414 { 415 XDR *xdrs; 416 CONN *conn; 417 418 struct recv_data *rdp = (struct recv_data *)mp->b_rptr; 419 struct clone_rdma_data *vd; 420 struct clist *cl = NULL; 421 struct clist *wcl = NULL; 422 struct clist *repcl = NULL; 423 struct clist *cllong = NULL; 424 425 rdma_stat status; 426 rdma_srv_cred_ctrl_t *cc_info; 427 428 uint32_t vers, op, pos, xid; 429 uint32_t rdma_credit; 430 uint32_t wcl_total_length = 0; 431 bool_t wwl= FALSE; 432 int i, numclnts, availbufs, to_be_posted; 433 #ifdef SERVER_REG_CACHE 434 rib_lrc_entry_t *long_reply_buf = NULL; 435 #endif 436 vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 437 RSSTAT_INCR(rscalls); 438 conn = rdp->conn; 439 440 #ifdef DYNAMIC_CREDIT_CONTROL 441 RDMA_GET_RESOURCE_INFO(conn, &numclnts, &availbufs); 442 svc_consume_credit(conn); 443 #else 444 status = rdma_svc_postrecv(conn); 445 if (status != RDMA_SUCCESS) { 446 cmn_err(CE_NOTE, 447 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status); 448 goto badrpc_call; 449 } 450 #endif 451 452 xdrs = &clone_xprt->xp_xdrin; 453 xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE); 454 xid = *(uint32_t *)rdp->rpcmsg.addr; 455 XDR_SETPOS(xdrs, sizeof (uint32_t)); 456 457 if (! xdr_u_int(xdrs, &vers) || 458 ! xdr_u_int(xdrs, &rdma_credit) || 459 ! xdr_u_int(xdrs, &op)) { 460 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed"); 461 goto xdr_err; 462 } 463 464 #ifdef DYNAMIC_CREDIT_CONTROL 465 svc_compute_credit(conn, rdma_credit, numclnts, availbufs, &to_be_posted); 466 for(i=0; i<to_be_posted; i++){ 467 status = rdma_svc_postrecv(conn); 468 if (status != RDMA_SUCCESS) { 469 cmn_err(CE_NOTE, 470 "svc_rdma_krecv: rdma_svc_postrecv failed %d", status); 471 goto badrpc_call; 472 } 473 } 474 svc_update_credit(conn, to_be_posted); 475 #endif 476 477 if (rdp->status != 0) { 478 cmn_err(CE_NOTE, 479 "svc_rdma_krecv: invalid status %d", 480 rdp->status); 481 goto badrpc_call; 482 } 483 484 if (! xdr_do_clist(xdrs, &cl)) { 485 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed"); 486 goto xdr_err; 487 } 488 489 if (!xdr_decode_wlist_new(xdrs, &wcl, &wwl, &wcl_total_length,conn)) { 490 cmn_err(CE_NOTE, "svc recv: xdr_decode_wlist failed"); 491 if (cl) 492 clist_free(cl); 493 goto xdr_err; 494 } 495 vd->wlist = wcl; 496 497 (void) xdr_decode_reply_wchunk(xdrs, &repcl, conn); 498 vd->reply_cl = repcl; 499 500 /* 501 * A chunk at 0 offset indicates that the RPC call message 502 * is in a chunk. Get the RPC call message chunk. 503 */ 504 if (cl != NULL && op == RDMA_NOMSG) { 505 506 /* Remove RPC call message chunk from chunklist */ 507 cllong = cl; 508 cl = cl->c_next; 509 cllong->c_next = NULL; 510 511 /* Allocate and register memory for the RPC call msg chunk */ 512 #ifdef SERVER_REG_CACHE 513 long_reply_buf = RDMA_GET_SERVER_CACHE_BUF(conn,cllong->c_len); 514 cllong->long_reply_buf = (uint64)long_reply_buf; 515 cllong->c_daddr = (uint64)(uintptr_t) long_reply_buf->lrc_buf; 516 #else 517 cllong->c_daddr = (uint64)(uintptr_t) 518 kmem_alloc(cllong->c_len, KM_SLEEP); 519 #endif 520 if (cllong->c_daddr == NULL) { 521 cmn_err(CE_WARN, "svc krecv: no memory for rpc call"); 522 clist_free(cllong); 523 goto cll_malloc_err; 524 } 525 526 status = clist_register(conn, cllong, 0); 527 if (status) { 528 cmn_err(CE_WARN, "svc krecv: clist_register failed"); 529 #ifdef SERVER_REG_CACHE 530 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)cllong->long_reply_buf); 531 #else 532 if(cllong->c_len) 533 kmem_free((void *)(uintptr_t)cllong->c_daddr, 534 cllong->c_len); 535 #endif 536 if(cllong) 537 clist_free(cllong); 538 goto cll_malloc_err; 539 } 540 541 /* 542 * Now read the RPC call message in 543 */ 544 status = RDMA_READ(conn, cllong, WAIT); 545 if (status) { 546 cmn_err(CE_WARN, "svc_rdma_krecv: rdma_read failed"); 547 (void) clist_deregister(conn, cllong, 0); 548 #ifdef SERVER_REG_CACHE 549 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)cllong->long_reply_buf); 550 #else 551 kmem_free((void *)(uintptr_t)cllong->c_daddr, 552 cllong->c_len); 553 #endif 554 clist_free(cllong); 555 goto cll_malloc_err; 556 } 557 558 status = clist_syncmem(conn, cllong, 0); 559 (void) clist_deregister(conn, cllong, 0); 560 561 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr, 562 cllong->c_len, 0, cl, XDR_DECODE, conn); 563 564 vd->rpcbuf.type = CHUNK_BUFFER; 565 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr; 566 vd->rpcbuf.len = cllong->c_len; 567 vd->rpcbuf.handle.mrc_rmr = 0; 568 #ifdef SERVER_REG_CACHE 569 vd->rpcbuf.long_reply_buf = (rib_lrc_entry_t *)cllong->long_reply_buf; 570 #endif 571 clist_free(cllong); 572 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 573 } else { 574 pos = XDR_GETPOS(xdrs); 575 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos, 576 rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn); 577 vd->rpcbuf = rdp->rpcmsg; 578 } 579 580 if (! xdr_callmsg(xdrs, msg)) { 581 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed"); 582 RSSTAT_INCR(rsxdrcall); 583 goto callmsg_err; 584 } 585 586 /* 587 * wlist sent for something besides NFS3 READ, so ignore it. 588 * FTDO: this isn't appropriate for READLINK3, but our client 589 * will never drive writelist for READLINK3, so good enough 590 * for the demo. 591 */ 592 if (vd->wlist != NULL && 593 (msg->rm_call.cb_rpcvers != RPC_MSG_VERSION || 594 msg->rm_call.cb_prog != NFS3_PROGRAM || 595 msg->rm_call.cb_vers != NFS_V3 || 596 msg->rm_call.cb_proc != NFSPROC3_READ)) { 597 #ifdef SERVER_REG_CACHE 598 RDMA_FREE_SERVER_CACHE_BUF(conn,(rib_lrc_entry_t *)wcl->long_reply_buf); 599 #else 600 kmem_free((void *)wcl->c_saddr, wcl_total_length); 601 #endif 602 clist_free(wcl); 603 vd->wlist = NULL; 604 } 605 606 /* 607 * Point the remote transport address in the service_transport 608 * handle at the address in the request. 609 */ 610 clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf; 611 clone_xprt->xp_rtaddr.len = conn->c_raddr.len; 612 clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len; 613 clone_xprt->xp_xid = xid; 614 vd->conn = conn; 615 616 freeb(mp); 617 return (TRUE); 618 callmsg_err: 619 rdma_buf_free(conn, &vd->rpcbuf); 620 cll_malloc_err: 621 if (cl) 622 clist_free(cl); 623 if (wcl != NULL) { 624 #ifdef SERVER_REG_CACHE 625 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf); 626 #else 627 kmem_free((void *)wcl->c_saddr, wcl_total_length); 628 #endif 629 clist_free(wcl); 630 } 631 xdr_err: 632 XDR_DESTROY(xdrs); 633 badrpc_call: 634 RDMA_BUF_FREE(conn, &rdp->rpcmsg); 635 RDMA_REL_CONN(conn); 636 freeb(mp); 637 RSSTAT_INCR(rsbadcalls); 638 return (FALSE); 639 } 640 641 #ifdef DYNAMIC_CREDIT_CONTROL 642 static void 643 svc_consume_credit(CONN *conn) 644 { 645 rdma_srv_cred_ctrl_t *cc_info; 646 647 mutex_enter(&conn->c_lock); 648 cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc; 649 cc_info->srv_cc_posted--; 650 mutex_exit(&conn->c_lock); 651 } 652 653 static void 654 svc_compute_credit(CONN *conn, uint32_t rdma_credit, int numclnts, 655 int availbufs, int *to_be_posted) 656 { 657 int average, grant; 658 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc; 659 660 if(numclnts == 0){ 661 cmn_err(CE_NOTE, "There is no active client!\n"); 662 *to_be_posted = 0; 663 return; 664 } 665 666 average = availbufs/numclnts; 667 668 mutex_enter(&conn->c_lock); 669 670 if(rdma_credit <= cc_info->srv_cc_posted) 671 grant = cc_info->srv_cc_posted; 672 else if(rdma_credit <= average) 673 grant = rdma_credit; 674 else 675 grant = average + (rdma_credit - average) * SVC_CREDIT_FACTOR; 676 677 *to_be_posted = grant - cc_info->srv_cc_posted; 678 if(*to_be_posted < 0) 679 *to_be_posted = 0; 680 if(*to_be_posted > availbufs) 681 *to_be_posted = availbufs/2; 682 683 mutex_exit(&conn->c_lock); 684 } 685 686 static void 687 svc_grant_credit(CONN * conn, uint32_t * rdma_credit) 688 { 689 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc; 690 691 mutex_enter(&conn->c_lock); 692 693 /* 694 * Fill in the granted number of buffers 695 * for credit control. 696 * 697 * XXX Currently ignoring what the client sends. 698 */ 699 *rdma_credit = cc_info->srv_cc_buffers_granted; 700 mutex_exit(&conn->c_lock); 701 } 702 703 static void 704 svc_update_credit(CONN * conn, int i) 705 { 706 rdma_srv_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_srv_cc; 707 708 mutex_enter(&conn->c_lock); 709 cc_info->srv_cc_buffers_granted = cc_info->srv_cc_posted + i; 710 cc_info->srv_cc_posted = cc_info->srv_cc_buffers_granted; 711 mutex_exit(&conn->c_lock); 712 } 713 #endif 714 715 static int 716 svc_process_wlist(struct clone_rdma_data *vd, xdrproc_t xdr_results, 717 caddr_t xdr_location, int *num_wsegment, 718 unsigned int *templen) 719 { 720 struct clist *wcl; 721 int data_len, avail_len, num, status; 722 READ3resok *rok; 723 724 rok = &(((READ3res *) xdr_location)->res_u.ok); 725 data_len = num = avail_len = 0; 726 727 wcl = vd->wlist; 728 while (wcl != NULL) { 729 if (wcl->c_dmemhandle.mrc_rmr != 0 730 && xdr_results == x_READ3res) { 731 732 avail_len += wcl->c_len; 733 if (wcl->c_len < rok->count) { 734 data_len += wcl->c_len; 735 } else { 736 /* Can make the rest chunks all 0-len */ 737 data_len += rok->count; 738 wcl->c_len = rok->count; 739 } 740 rok->count -= wcl->c_len; 741 num ++; 742 } 743 else { 744 cmn_err(CE_NOTE, 745 "svc_process_wlist: wlist has an error\n"); 746 } 747 wcl = wcl->c_next; 748 } 749 750 /* 751 * MUST fail if there are still more data 752 */ 753 if (rok->count > 0) { 754 cmn_err(CE_NOTE, 755 "svc_process_wlist: data_len is too short \n"); 756 return SVC_RDMA_FAIL; 757 } 758 759 wcl = vd->wlist; 760 rok->count = data_len; 761 rok->wlist_len = data_len; 762 rok->wlist = wcl; 763 *num_wsegment = num; 764 *templen = avail_len; 765 766 /* Register, sync and write over the data */ 767 if (data_len > 0) { 768 status = clist_register(vd->conn, wcl, TRUE); 769 if (status != RDMA_SUCCESS) { 770 cmn_err(CE_NOTE, 771 "svc_process_wlist: clist_register " 772 "failed"); 773 return SVC_RDMA_FAIL; 774 } 775 776 status = clist_syncmem(vd->conn, wcl, TRUE); 777 if (status != RDMA_SUCCESS) { 778 cmn_err(CE_NOTE, 779 "svc_process_wlist: syncmem failed(%d)", 780 status); 781 return SVC_RDMA_FAIL; 782 } 783 784 status = RDMA_WRITE(vd->conn, wcl, NOWAIT); 785 if (status != RDMA_SUCCESS) { 786 cmn_err(CE_NOTE, 787 "svc_process_wlist: RDMA_WRITE failed(%d)", 788 status); 789 return SVC_RDMA_FAIL; 790 } 791 } 792 793 return SVC_RDMA_SUCCESS; 794 } 795 796 static int 797 svc_process_long_reply(SVCXPRT * clone_xprt, CONN * conn, 798 xdrproc_t xdr_results, caddr_t xdr_location, 799 caddr_t vd, XDR ** xdrs, 800 struct rpc_msg *msg, bool_t has_args, int *msglen, 801 int *freelen, int *num, unsigned int *len) 802 { 803 rdma_buf_t long_rpc = {0}; 804 int status; 805 struct clist *ncl = NULL, *wcl = NULL; 806 char *memp = NULL; 807 int avail_len = 0; 808 int count = 0; 809 int data_len = 0; 810 *num = 0; 811 *freelen = 0; 812 /* 813 * If the clone_xprt struct has a reply chunk list, 814 * then we MUST RDMA_WRITE the reply back to the client, 815 * no matter what its size is. This translates to: 816 * 817 * RDMA_WRITE + RDMA_SEND(op = RDMA_NOMSG) 818 * 819 * XXX the rdma write code currently ignores kerberos. 820 */ 821 822 (*msglen) += xdrrdma_sizeof(xdr_results, xdr_location, rdma_minchunk); 823 824 wcl = (struct clist *)vd; 825 count = *msglen; 826 while (wcl != NULL) { 827 *freelen += wcl->c_len; 828 if (wcl->c_dmemhandle.mrc_rmr != 0) 829 { 830 avail_len += wcl->c_len; 831 if (wcl->c_len < count) { 832 data_len += wcl->c_len; 833 } else { 834 data_len += count; 835 wcl->c_len = count; 836 } 837 count -= wcl->c_len; 838 *num += 1; 839 } 840 else { 841 cmn_err(CE_NOTE, 842 "svc_process_long_reply: wchunk list has an error\n"); 843 } 844 wcl = wcl->c_next; 845 } 846 847 /* 848 * MUST fail if there are still more data 849 */ 850 if (count > 0) { 851 cmn_err(CE_NOTE, 852 "svc_process_long_reply: data_len is too short \n"); 853 return SVC_RDMA_FAIL; 854 } 855 /* 856 * Setup buffers for long rpc reply 857 */ 858 859 /* 860 * We specify 0 for the chunk size since we 861 * don't want a chunk list. 862 */ 863 wcl = (struct clist *)vd; 864 xdrrdma_create(*xdrs, (caddr_t)wcl->c_saddr , *msglen, 0, 865 wcl, XDR_ENCODE, NULL); 866 867 msg->rm_xid = clone_xprt->xp_xid; 868 869 if (!(xdr_replymsg(*xdrs, msg) && 870 (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, *xdrs, 871 xdr_results, xdr_location)))) { 872 kmem_free((void *)wcl->c_saddr, *freelen); 873 cmn_err(CE_WARN, "svc_process_long_reply: " 874 "xdr_replymsg/SVCAUTH_WRAP failed " 875 "for long reply\n"); 876 return SVC_RDMA_FAIL; 877 } 878 *len = XDR_GETPOS(*xdrs); 879 880 if (clist_register(conn, wcl, TRUE) != RDMA_SUCCESS) { 881 #ifdef SERVER_REG_CACHE 882 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf); 883 #else 884 kmem_free((void *)(wcl->c_saddr), *freelen); 885 #endif 886 cmn_err(CE_NOTE, "svc_process_long_reply: RDMA_WRITE: " 887 "clist reg failed"); 888 return SVC_RDMA_FAIL; 889 } 890 891 status = clist_syncmem(conn, wcl, TRUE); 892 if (status) { 893 (void) clist_deregister(conn, wcl, TRUE); 894 #ifdef SERVER_REG_CACHE 895 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf); 896 #else 897 kmem_free((void *)(wcl->c_saddr), *freelen); 898 #endif 899 cmn_err(CE_NOTE, 900 "svc_process_long_reply: sync mem failed %d", status); 901 return SVC_RDMA_FAIL; 902 } 903 904 /* 905 * Note: we must pass WAIT into the rdma write call to 906 * ensure that the call completes before we move on, where 907 * part of 'moving on' is deregistering the memory -- and 908 * if the memory is deregistered before the write completes 909 * we'll have an error. 910 */ 911 status = RDMA_WRITE(conn, wcl, NOWAIT); 912 if (status != RDMA_SUCCESS) { 913 (void) clist_deregister(conn, wcl, TRUE); 914 #ifdef SERVER_REG_CACHE 915 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)wcl->long_reply_buf); 916 #else 917 kmem_free((void *)(wcl->c_saddr), *freelen); 918 #endif 919 cmn_err(CE_NOTE, 920 "svc_process_long_reply: RDMA_WRITE failed %d", 921 status); 922 return SVC_RDMA_FAIL; 923 } 924 925 return SVC_RDMA_SUCCESS; 926 } 927 928 static int 929 svc_compose_rpcmsg(SVCXPRT * clone_xprt, CONN * conn, xdrproc_t xdr_results, 930 caddr_t xdr_location, rdma_buf_t * rpcreply, XDR ** xdrs, 931 struct rpc_msg *msg, bool_t has_args, int *msglen, 932 unsigned int *len) 933 { 934 int auth_flavor = msg->rm_reply.rp_acpt.ar_verf.oa_flavor; 935 936 if (has_args && auth_flavor != RPCSEC_GSS) 937 (*msglen) += xdrrdma_sizeof(xdr_results, xdr_location, 938 rdma_minchunk); 939 else if (has_args && auth_flavor == RPCSEC_GSS) { 940 (*msglen) += 941 2 * MAX_AUTH_BYTES + 2 * sizeof(struct opaque_auth); 942 (*msglen) += xdr_sizeof(xdr_results, xdr_location); 943 } 944 945 if (*msglen > RPC_MSG_SZ) { 946 cmn_err(CE_NOTE, 947 "svc_compose_rpcmsg: Server needs to send a reply" 948 "larger than RPC_MSG_SZ\n"); 949 return SVC_RDMA_FAIL; 950 } 951 952 /* 953 * Get a pre-allocated buffer for rpc reply 954 */ 955 rpcreply->type = SEND_BUFFER; 956 if (RDMA_BUF_ALLOC(conn, rpcreply)) { 957 cmn_err(CE_WARN, "svc_compose_rpcmsg: no free buffers!"); 958 return SVC_RDMA_FAIL; 959 } 960 961 if (has_args == FALSE || auth_flavor != RPCSEC_GSS) { 962 xdrrdma_create(*xdrs, rpcreply->addr, rpcreply->len, 963 rdma_minchunk, NULL, XDR_ENCODE, NULL); 964 } 965 else { 966 xdrrdma_create(*xdrs, rpcreply->addr, *msglen, 0, NULL, 967 XDR_ENCODE, NULL); 968 } 969 970 msg->rm_xid = clone_xprt->xp_xid; 971 972 if (has_args) { 973 if (!(xdr_replymsg(*xdrs, msg) && 974 (!has_args 975 || SVCAUTH_WRAP(&clone_xprt->xp_auth, *xdrs, 976 xdr_results, xdr_location)))) { 977 if (auth_flavor == RPCSEC_GSS 978 && rpcreply->addr != (*xdrs)->x_base) { 979 rpcreply->addr = (*xdrs)->x_base; 980 rpcreply->len = xdr_getbufsize(*xdrs); 981 } 982 rdma_buf_free(conn, rpcreply); 983 cmn_err(CE_WARN, 984 "svc_compose_rpcmsg: xdr_replymsg/SVCAUTH_WRAP " 985 "failed"); 986 return SVC_RDMA_FAIL; 987 } 988 if (auth_flavor == RPCSEC_GSS 989 && rpcreply->addr != (*xdrs)->x_base) { 990 rpcreply->addr = (*xdrs)->x_base; 991 rpcreply->len = xdr_getbufsize(*xdrs); 992 } 993 } 994 else { 995 if (!xdr_replymsg(*xdrs, msg)) { 996 rdma_buf_free(conn, rpcreply); 997 cmn_err(CE_WARN, 998 "svc_compose_rpcmsg: xdr_replymsg/SVCAUTH_WRAP " 999 "failed"); 1000 return SVC_RDMA_FAIL; 1001 } 1002 } 1003 1004 *len = XDR_GETPOS(*xdrs); 1005 1006 if (auth_flavor == RPCSEC_GSS) { 1007 XDR_DESTROY(*xdrs); 1008 xdrrdma_create(*xdrs, rpcreply->addr, *len, 0, NULL, 1009 XDR_ENCODE, NULL); 1010 } 1011 1012 return SVC_RDMA_SUCCESS; 1013 } 1014 1015 /* 1016 * Send rpc reply. 1017 */ 1018 static bool_t 1019 svc_rdma_ksend(SVCXPRT * clone_xprt, struct rpc_msg *msg) 1020 { 1021 XDR *xdrs = &(clone_xprt->xp_xdrout); 1022 XDR rxdrs; 1023 CONN *conn = NULL; 1024 rdma_buf_t clmsg = {0}, rpcreply = {0}; 1025 1026 struct clone_rdma_data *vd; 1027 struct clist *cl = NULL; 1028 struct clist *sendlist = NULL; 1029 struct clist *wcl = NULL; 1030 struct clist *reply_cl; 1031 xdrproc_t xdr_results; 1032 caddr_t xdr_location; 1033 1034 int retval = FALSE; 1035 int status, msglen, num_wsegment = 0, num_wreply_segments = 0; 1036 uint32_t rdma_credit = 0, templen = 0; 1037 int freelen =0; 1038 bool_t has_args; 1039 uint_t len, op, vers; 1040 1041 vd = (struct clone_rdma_data *) clone_xprt->xp_p2buf; 1042 conn = vd->conn; 1043 1044 /* 1045 * If there is a result procedure specified in the reply message, 1046 * it will be processed in the xdr_replymsg and SVCAUTH_WRAP. 1047 * We need to make sure it won't be processed twice, so we null 1048 * it for xdr_replymsg here. 1049 */ 1050 has_args = FALSE; 1051 if (msg->rm_reply.rp_stat == MSG_ACCEPTED && 1052 msg->rm_reply.rp_acpt.ar_stat == SUCCESS) { 1053 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) { 1054 has_args = TRUE; 1055 xdr_location = msg->acpted_rply.ar_results.where; 1056 msg->acpted_rply.ar_results.proc = xdr_void; 1057 msg->acpted_rply.ar_results.where = NULL; 1058 } 1059 } 1060 1061 /* 1062 * Use RDMA Write to return content requested by wlist. 1063 * Only 1 writechunk in writelist for now, 1064 * but this chunk can contain multiple rdma segments. 1065 */ 1066 if (vd->wlist) { 1067 status = svc_process_wlist(vd, xdr_results, xdr_location, 1068 &num_wsegment, &templen); 1069 if (status != SVC_RDMA_SUCCESS) { 1070 goto out; 1071 } 1072 } 1073 #ifdef RPC_RDMA_INLINE 1074 else if (xdr_results == x_READ3res) { 1075 READ3resok *rok; 1076 rok = &(((READ3res *) xdr_location)->res_u.ok); 1077 rok->wlist = NULL; 1078 } 1079 #endif 1080 1081 /* 1082 * Get the size of the rpc reply message. 1083 */ 1084 msglen = xdr_sizeof(xdr_replymsg, msg); 1085 1086 /*reply_cl.c_daddr = NULL;*/ 1087 reply_cl = vd->reply_cl; 1088 1089 if (vd->reply_cl) { 1090 1091 status = svc_process_long_reply(clone_xprt, 1092 conn, xdr_results, 1093 xdr_location, (caddr_t)vd->reply_cl, 1094 &xdrs, msg, has_args, 1095 &msglen, &freelen, &num_wreply_segments, &len); 1096 if (status == SVC_RDMA_SUCCESS) { 1097 op = RDMA_NOMSG; 1098 cl = NULL; 1099 goto rdma_writed_long_reply_out; 1100 } 1101 else 1102 goto out; 1103 } 1104 status = svc_compose_rpcmsg(clone_xprt, conn, xdr_results, 1105 xdr_location, &rpcreply, &xdrs, msg, 1106 has_args, &msglen, &len); 1107 if (status != SVC_RDMA_SUCCESS) 1108 goto out; 1109 1110 op = RDMA_MSG; 1111 1112 cl = xdrrdma_clist(xdrs); 1113 cl = NULL; 1114 if (cl != NULL) { 1115 cmn_err(CE_NOTE, 1116 "svc_rdma_ksend: Should not provide non-null" 1117 "read chunk list to client\n"); 1118 } 1119 1120 rdma_writed_long_reply_out: 1121 1122 clmsg.type = SEND_BUFFER; 1123 if (RDMA_BUF_ALLOC(conn, &clmsg)) { 1124 rdma_buf_free(conn, &rpcreply); 1125 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!"); 1126 goto out; 1127 } 1128 1129 #ifdef DYNAMIC_CREDIT_CONTROL 1130 svc_grant_credit(conn, &rdma_credit); 1131 #else 1132 rdma_credit = rdma_bufs_granted; 1133 #endif 1134 1135 vers = RPCRDMA_VERS; 1136 xdrs = &rxdrs; 1137 xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE); 1138 (*(uint32_t *) clmsg.addr) = msg->rm_xid; 1139 /* Skip xid and set the xdr position accordingly. */ 1140 XDR_SETPOS(xdrs, sizeof(uint32_t)); 1141 if (!xdr_u_int(xdrs, &vers) || 1142 !xdr_u_int(xdrs, &rdma_credit) || !xdr_u_int(xdrs, &op)) { 1143 rdma_buf_free(conn, &rpcreply); 1144 rdma_buf_free(conn, &clmsg); 1145 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed"); 1146 goto out; 1147 } 1148 1149 /* 1150 * Now XDR the read chunk list, actually always NULL 1151 */ 1152 (void) xdr_do_clist(xdrs, &cl); 1153 1154 /* 1155 * encode write list -- we already drove RDMA_WRITEs 1156 */ 1157 wcl = vd->wlist; 1158 if (!xdr_encode_wlist(xdrs, wcl, num_wsegment)) { 1159 cmn_err(CE_NOTE, 1160 "svc_rdma_ksend: xdr_encode_wlist failed: " 1161 "wcl=%p", (void *) wcl); 1162 rdma_buf_free(conn, &rpcreply); 1163 rdma_buf_free(conn, &clmsg); 1164 goto out; 1165 } 1166 1167 /* 1168 * XDR encode the RDMA_REPLY write chunk 1169 */ 1170 (void) xdr_encode_reply_wchunk(xdrs, vd->reply_cl, num_wreply_segments); 1171 1172 clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr, 1173 NULL, NULL); 1174 1175 if (op == RDMA_MSG) { 1176 clist_add(&sendlist, 0, len, &rpcreply.handle, 1177 rpcreply.addr, NULL, NULL); 1178 } 1179 1180 #if defined(ASYNC_SERVER_DEREG) 1181 status = RDMA_SEND_NW(conn, sendlist, msg->rm_xid, (caddr_t)conn, 1182 (caddr_t)vd->wlist, 1183 templen, 1184 (caddr_t)reply_cl, 1185 freelen, num_wsegment, num_wreply_segments ); 1186 #else 1187 status = RDMA_SEND(conn, sendlist, msg->rm_xid); 1188 #endif 1189 if (status != RDMA_SUCCESS) { 1190 goto out; 1191 } 1192 1193 retval = TRUE; 1194 1195 out: 1196 1197 /* 1198 * Free up sendlist chunks 1199 */ 1200 if (sendlist != NULL) 1201 clist_free(sendlist); 1202 1203 /* 1204 * Destroy private data for xdr rdma 1205 */ 1206 if ((clone_xprt->xp_xdrout).x_private) 1207 XDR_DESTROY(&(clone_xprt->xp_xdrout)); 1208 if (rxdrs.x_private) 1209 XDR_DESTROY(&rxdrs); 1210 1211 /* 1212 * This is completely disgusting. If public is set it is 1213 * a pointer to a structure whose first field is the address 1214 * of the function to free that structure and any related 1215 * stuff. (see rrokfree in nfs_xdr.c). 1216 */ 1217 if (xdrs->x_public) { 1218 /* LINTED pointer alignment */ 1219 (**((int (**)()) xdrs->x_public)) (xdrs->x_public); 1220 } 1221 1222 if (vd->wlist != NULL) { 1223 #if defined(ASYNC_SERVER_DEREG) 1224 if(!retval) { 1225 #endif 1226 wcl = vd->wlist; 1227 (void) clist_deregister(vd->conn, wcl, TRUE); 1228 #ifdef SERVER_REG_CACHE 1229 RDMA_FREE_SERVER_CACHE_BUF(vd->conn, (rib_lrc_entry_t *)wcl->long_reply_buf); 1230 #else 1231 if(templen) 1232 kmem_free((void *) (vd->wlist)->c_saddr, templen); 1233 #endif 1234 kmem_free(vd->wlist, num_wsegment * sizeof(struct clist)); 1235 #if defined(ASYNC_SERVER_DEREG) 1236 } 1237 #endif 1238 vd->wlist = NULL; 1239 } 1240 1241 1242 if(vd->reply_cl != NULL){ 1243 #if defined(ASYNC_SERVER_DEREG) 1244 if(!retval) { 1245 #endif 1246 (void) clist_deregister(conn, reply_cl, TRUE); 1247 #ifdef SERVER_REG_CACHE 1248 RDMA_FREE_SERVER_CACHE_BUF(conn, (rib_lrc_entry_t *)reply_cl->long_reply_buf); 1249 #else 1250 #ifdef DEBUG 1251 if(rdma_svc_debug > 1) 1252 cmn_err(CE_NOTE, "Freeing up %p of length %d\n",reply_cl->c_saddr,freelen); 1253 #endif 1254 if(freelen) 1255 kmem_free((void *)(reply_cl->c_saddr), freelen); 1256 #endif 1257 kmem_free((void *)vd->reply_cl, num_wreply_segments * sizeof(struct clist)); 1258 #if defined(ASYNC_SERVER_DEREG) 1259 } 1260 #endif 1261 vd->reply_cl = NULL; 1262 } 1263 return (retval); 1264 } 1265 1266 /* 1267 * Deserialize arguments. 1268 */ 1269 static bool_t 1270 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr) 1271 { 1272 if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin, 1273 xdr_args, args_ptr)) != TRUE) 1274 return (FALSE); 1275 return (TRUE); 1276 } 1277 1278 static bool_t 1279 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, 1280 caddr_t args_ptr) 1281 { 1282 struct clone_rdma_data *vd;
1507 uint32_t drhash; 1508 1509 ASSERT(MUTEX_HELD(&rdmadupreq_lock)); 1510 1511 drhash = (uint32_t)DRHASH(dr); 1512 drt = rdmadrhashtbl[drhash]; 1513 while (drt != NULL) { 1514 if (drt == dr) { 1515 rdmadrhashstat[drhash]--; 1516 if (drtprev == NULL) { 1517 rdmadrhashtbl[drhash] = drt->dr_chain; 1518 } else { 1519 drtprev->dr_chain = drt->dr_chain; 1520 } 1521 return; 1522 } 1523 drtprev = drt; 1524 drt = drt->dr_chain; 1525 } 1526 } 1527 1528 bool_t 1529 rdma_get_wchunk_seg(struct svc_req *req, iovec_t *iov) 1530 { 1531 struct clone_rdma_data *rcd; 1532 struct clist *clist; 1533 uint32_t tlen; 1534 1535 if (req->rq_xprt->xp_type != T_RDMA) { 1536 return (FALSE); 1537 } 1538 1539 rcd = (struct clone_rdma_data *)(&req->rq_xprt->xp_p2buf); 1540 if (rcd->wlist == NULL) { 1541 return (FALSE); 1542 } 1543 tlen = 0; 1544 clist = rcd->wlist; 1545 while(clist){ 1546 tlen += clist->c_len; 1547 clist = clist->c_next; 1548 } 1549 1550 /* 1551 * set iov to addr+len of first segment of first wchunk of 1552 * wlist sent by client. krecv() already malloc'd a buffer 1553 * large enough, but registration is deferred until we write 1554 * the buffer back to (NFS) client using RDMA_WRITE. 1555 */ 1556 iov->iov_base = (caddr_t)rcd->wlist->c_saddr; 1557 iov->iov_len = tlen; 1558 1559 return (TRUE); 1560 } 1561