Old svc_rdma.c
  1 /*
  2  * CDDL HEADER START
  3  *
  4  * The contents of this file are subject to the terms of the
  5  * Common Development and Distribution License, Version 1.0 only
  6  * (the "License").  You may not use this file except in compliance
  7  * with the License.
  8  *
  9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 10  * or http://www.opensolaris.org/os/licensing.
 11  * See the License for the specific language governing permissions
 12  * and limitations under the License.
 13  *
 14  * When distributing Covered Code, include this CDDL HEADER in each
 15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 16  * If applicable, add the following below this CDDL HEADER, with the
 17  * fields enclosed by brackets "[]" replaced with your own identifying
 18  * information: Portions Copyright [yyyy] [name of copyright owner]
 19  *
 20  * CDDL HEADER END
 21  */
 22 /*
 23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 24  * Use is subject to license terms.
 25  */
 26 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
 27 /* All Rights Reserved */
 28 /*
 29  * Portions of this source code were derived from Berkeley
 30  * 4.3 BSD under license from the Regents of the University of
 31  * California.
 32  */
 33 
 34 #pragma ident   "@(#)svc_rdma.c 1.8     05/06/10 SMI"
 35 
 36 /*
 37  * Server side of RPC over RDMA in the kernel.
 38  */
 39 
 40 #include <sys/param.h>
 41 #include <sys/types.h>
 42 #include <sys/user.h>
 43 #include <sys/sysmacros.h>
 44 #include <sys/proc.h>
 45 #include <sys/file.h>
 46 #include <sys/errno.h>
 47 #include <sys/kmem.h>
 48 #include <sys/debug.h>
 49 #include <sys/systm.h>
 50 #include <sys/cmn_err.h>
 51 #include <sys/kstat.h>
 52 #include <sys/vtrace.h>
 53 #include <sys/debug.h>
 54 
 55 #include <rpc/types.h>
 56 #include <rpc/xdr.h>
 57 #include <rpc/auth.h>
 58 #include <rpc/clnt.h>
 59 #include <rpc/rpc_msg.h>
 60 #include <rpc/svc.h>
 61 #include <rpc/rpc_rdma.h>
 62 #include <sys/ddi.h>
 63 #include <sys/sunddi.h>
 64 
 65 #include <inet/common.h>
 66 #include <inet/ip.h>
 67 #include <inet/ip6.h>
 68 
 69 /*
 70  * RDMA transport specific data associated with SVCMASTERXPRT
 71  */
 72 struct rdma_data {
 73         SVCMASTERXPRT   *rd_xprt;       /* back ptr to SVCMASTERXPRT */
 74         struct rdma_svc_data rd_data;   /* rdma data */
 75         rdma_mod_t      *r_mod;         /* RDMA module containing ops ptr */
 76 };
 77 
 78 /*
 79  * Plugin connection specific data stashed away in clone SVCXPRT
 80  */
 81 struct clone_rdma_data {
 82         CONN            *conn;          /* RDMA connection */
 83         rdma_buf_t      rpcbuf;         /* RPC req/resp buffer */
 84 };
 85 
 86 #ifdef DEBUG
 87 int rdma_svc_debug = 0;
 88 #endif
 89 
 90 #define MAXADDRLEN      128     /* max length for address mask */
 91 
 92 /*
 93  * Routines exported through ops vector.
 94  */
 95 static bool_t           svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *);
 96 static bool_t           svc_rdma_ksend(SVCXPRT *, struct rpc_msg *);
 97 static bool_t           svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t);
 98 static bool_t           svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t);
 99 void                    svc_rdma_kdestroy(SVCMASTERXPRT *);
100 static int              svc_rdma_kdup(struct svc_req *, caddr_t, int,
101                                 struct dupreq **, bool_t *);
102 static void             svc_rdma_kdupdone(struct dupreq *, caddr_t,
103                                 void (*)(), int, int);
104 static int32_t          *svc_rdma_kgetres(SVCXPRT *, int);
105 static void             svc_rdma_kfreeres(SVCXPRT *);
106 static void             svc_rdma_kclone_destroy(SVCXPRT *);
107 static void             svc_rdma_kstart(SVCMASTERXPRT *);
108 void                    svc_rdma_kstop(SVCMASTERXPRT *);
109 
110 /*
111  * Server transport operations vector.
112  */
113 struct svc_ops rdma_svc_ops = {
114         svc_rdma_krecv,         /* Get requests */
115         svc_rdma_kgetargs,      /* Deserialize arguments */
116         svc_rdma_ksend,         /* Send reply */
117         svc_rdma_kfreeargs,     /* Free argument data space */
118         svc_rdma_kdestroy,      /* Destroy transport handle */
119         svc_rdma_kdup,          /* Check entry in dup req cache */
120         svc_rdma_kdupdone,      /* Mark entry in dup req cache as done */
121         svc_rdma_kgetres,       /* Get pointer to response buffer */
122         svc_rdma_kfreeres,      /* Destroy pre-serialized response header */
123         svc_rdma_kclone_destroy,        /* Destroy a clone xprt */
124         svc_rdma_kstart         /* Tell `ready-to-receive' to rpcmod */
125 };
126 
127 /*
128  * Server statistics
129  * NOTE: This structure type is duplicated in the NFS fast path.
130  */
131 struct {
132         kstat_named_t   rscalls;
133         kstat_named_t   rsbadcalls;
134         kstat_named_t   rsnullrecv;
135         kstat_named_t   rsbadlen;
136         kstat_named_t   rsxdrcall;
137         kstat_named_t   rsdupchecks;
138         kstat_named_t   rsdupreqs;
139         kstat_named_t   rslongrpcs;
140 } rdmarsstat = {
141         { "calls",      KSTAT_DATA_UINT64 },
142         { "badcalls",   KSTAT_DATA_UINT64 },
143         { "nullrecv",   KSTAT_DATA_UINT64 },
144         { "badlen",     KSTAT_DATA_UINT64 },
145         { "xdrcall",    KSTAT_DATA_UINT64 },
146         { "dupchecks",  KSTAT_DATA_UINT64 },
147         { "dupreqs",    KSTAT_DATA_UINT64 },
148         { "longrpcs",   KSTAT_DATA_UINT64 }
149 };
150 
151 kstat_named_t *rdmarsstat_ptr = (kstat_named_t *)&rdmarsstat;
152 uint_t rdmarsstat_ndata = sizeof (rdmarsstat) / sizeof (kstat_named_t);
153 
154 #define RSSTAT_INCR(x)  rdmarsstat.x.value.ui64++
155 
156 /*
157  * Create a transport record.
158  * The transport record, output buffer, and private data structure
159  * are allocated.  The output buffer is serialized into using xdrmem.
160  * There is one transport record per user process which implements a
161  * set of services.
162  */
163 /* ARGSUSED */
164 int
165 svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id,
166         rdma_xprt_group_t *started_xprts)
167 {
168         int error;
169         SVCMASTERXPRT *xprt;
170         struct rdma_data *rd;
171         rdma_registry_t *rmod;
172         rdma_xprt_record_t *xprt_rec;
173         queue_t *q;
174 
175         /*
176          * modload the RDMA plugins is not already done.
177          */
178         if (!rdma_modloaded) {
179                 mutex_enter(&rdma_modload_lock);
180                 if (!rdma_modloaded) {
181                         error = rdma_modload();
182                 }
183                 mutex_exit(&rdma_modload_lock);
184 
185                 if (error)
186                         return (error);
187         }
188 
189         /*
190          * master_xprt_count is the count of master transport handles
191          * that were successfully created and are ready to recieve for
192          * RDMA based access.
193          */
194         error = 0;
195         xprt_rec = NULL;
196         rw_enter(&rdma_lock, RW_READER);
197         if (rdma_mod_head == NULL) {
198                 started_xprts->rtg_count = 0;
199                 rw_exit(&rdma_lock);
200                 if (rdma_dev_available)
201                         return (EPROTONOSUPPORT);
202                 else
203                         return (ENODEV);
204         }
205 
206         /*
207          * If we have reached here, then atleast one RDMA plugin has loaded.
208          * Create a master_xprt, make it start listenining on the device,
209          * if an error is generated, record it, we might need to shut
210          * the master_xprt.
211          * SVC_START() calls svc_rdma_kstart which calls plugin binding
212          * routines.
213          */
214         for (rmod = rdma_mod_head; rmod != NULL; rmod = rmod->r_next) {
215 
216                 /*
217                  * One SVCMASTERXPRT per RDMA plugin.
218                  */
219                 xprt = kmem_zalloc(sizeof (*xprt), KM_SLEEP);
220                 xprt->xp_ops = &rdma_svc_ops;
221                 xprt->xp_sct = sct;
222                 xprt->xp_type = T_RDMA;
223                 mutex_init(&xprt->xp_req_lock, NULL, MUTEX_DEFAULT, NULL);
224                 mutex_init(&xprt->xp_thread_lock, NULL, MUTEX_DEFAULT, NULL);
225                 xprt->xp_req_head = (mblk_t *)0;
226                 xprt->xp_req_tail = (mblk_t *)0;
227                 xprt->xp_threads = 0;
228                 xprt->xp_detached_threads = 0;
229 
230                 rd = kmem_zalloc(sizeof (*rd), KM_SLEEP);
231                 xprt->xp_p2 = (caddr_t)rd;
232                 rd->rd_xprt = xprt;
233                 rd->r_mod = rmod->r_mod;
234 
235                 q = &rd->rd_data.q;
236                 xprt->xp_wq = q;
237                 q->q_ptr = &rd->rd_xprt;
238                 xprt->xp_netid = NULL;
239 
240                 if (netid != NULL) {
241                         xprt->xp_netid = kmem_alloc(strlen(netid) + 1,
242                                                 KM_SLEEP);
243                         (void) strcpy(xprt->xp_netid, netid);
244                 }
245 
246                 xprt->xp_addrmask.maxlen =
247                     xprt->xp_addrmask.len = sizeof (struct sockaddr_in);
248                 xprt->xp_addrmask.buf =
249                     kmem_zalloc(xprt->xp_addrmask.len, KM_SLEEP);
250                 ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_addr.s_addr =
251                     (uint32_t)~0;
252                 ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_family =
253                     (ushort_t)~0;
254 
255                 /*
256                  * Each of the plugins will have their own Service ID
257                  * to listener specific mapping, like port number for VI
258                  * and service name for IB.
259                  */
260                 rd->rd_data.svcid = id;
261                 error = svc_xprt_register(xprt, id);
262                 if (error) {
263                         cmn_err(CE_WARN, "svc_rdma_kcreate: svc_xprt_register"
264                                 "failed");
265                         goto cleanup;
266                 }
267 
268                 SVC_START(xprt);
269                 if (!rd->rd_data.active) {
270                         svc_xprt_unregister(xprt);
271                         error = rd->rd_data.err_code;
272                         goto cleanup;
273                 }
274 
275                 /*
276                  * This is set only when there is atleast one or more
277                  * transports successfully created. We insert the pointer
278                  * to the created RDMA master xprt into a separately maintained
279                  * list. This way we can easily reference it later to cleanup,
280                  * when NFS kRPC service pool is going away/unregistered.
281                  */
282                 started_xprts->rtg_count ++;
283                 xprt_rec = kmem_alloc(sizeof (*xprt_rec), KM_SLEEP);
284                 xprt_rec->rtr_xprt_ptr = xprt;
285                 xprt_rec->rtr_next = started_xprts->rtg_listhead;
286                 started_xprts->rtg_listhead = xprt_rec;
287                 continue;
288 cleanup:
289                 SVC_DESTROY(xprt);
290                 if (error == RDMA_FAILED)
291                         error = EPROTONOSUPPORT;
292         }
293 
294         rw_exit(&rdma_lock);
295 
296         /*
297          * Don't return any error even if a single plugin was started
298          * successfully.
299          */
300         if (started_xprts->rtg_count == 0)
301                 return (error);
302         return (0);
303 }
304 
305 /*
306  * Cleanup routine for freeing up memory allocated by
307  * svc_rdma_kcreate()
308  */
309 void
310 svc_rdma_kdestroy(SVCMASTERXPRT *xprt)
311 {
312         struct rdma_data *rd = (struct rdma_data *)xprt->xp_p2;
313 
314 
315         mutex_destroy(&xprt->xp_req_lock);
316         mutex_destroy(&xprt->xp_thread_lock);
317         kmem_free(xprt->xp_netid, strlen(xprt->xp_netid) + 1);
318         kmem_free(rd, sizeof (*rd));
319         kmem_free(xprt->xp_addrmask.buf, xprt->xp_addrmask.maxlen);
320         kmem_free(xprt, sizeof (*xprt));
321 }
322 
323 
324 static void
325 svc_rdma_kstart(SVCMASTERXPRT *xprt)
326 {
327         struct rdma_svc_data *svcdata;
328         rdma_mod_t *rmod;
329 
330         svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data;
331         rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod;
332 
333         /*
334          * Create a listener for  module at this port
335          */
336 
337         (*rmod->rdma_ops->rdma_svc_listen)(svcdata);
338 }
339 
340 void
341 svc_rdma_kstop(SVCMASTERXPRT *xprt)
342 {
343         struct rdma_svc_data *svcdata;
344         rdma_mod_t *rmod;
345 
346         svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data;
347         rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod;
348 
349         /*
350          * Call the stop listener routine for each plugin.
351          */
352         (*rmod->rdma_ops->rdma_svc_stop)(svcdata);
353         if (svcdata->active)
354                 cmn_err(CE_WARN, "rdma_stop: Failed to shutdown RDMA based kRPC"
355                         "  listener");
356 }
357 
358 /* ARGSUSED */
359 static void
360 svc_rdma_kclone_destroy(SVCXPRT *clone_xprt)
361 {
362 }
363 
364 static bool_t
365 svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg)
366 {
367         XDR *xdrs;
368         rdma_stat status;
369         struct recv_data *rdp = (struct recv_data *)mp->b_rptr;
370         CONN *conn;
371         struct clone_rdma_data *vd;
372         struct clist *cl;
373         uint_t vers, op, pos;
374         uint32_t xid;
375 
376         vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
377         RSSTAT_INCR(rscalls);
378         conn = rdp->conn;
379 
380         /*
381          * Post a receive descriptor on this
382          * endpoint to ensure all packets are received.
383          */
384         status = rdma_svc_postrecv(conn);
385         if (status != RDMA_SUCCESS) {
386                 cmn_err(CE_NOTE,
387                     "svc_rdma_krecv: rdma_svc_postrecv failed %d", status);
388         }
389 
390         if (rdp->status != 0) {
391                 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
392                 RDMA_REL_CONN(conn);
393                 RSSTAT_INCR(rsbadcalls);
394                 freeb(mp);
395                 return (FALSE);
396         }
397 
398         /*
399          * Decode rpc message
400          */
401         xdrs = &clone_xprt->xp_xdrin;
402         xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE);
403 
404         /*
405          * Get the XID
406          */
407         /*
408          * Treat xid as opaque (xid is the first entity
409          * in the rpc rdma message).
410          */
411         xid = *(uint32_t *)rdp->rpcmsg.addr;
412         /* Skip xid and set the xdr position accordingly. */
413         XDR_SETPOS(xdrs, sizeof (uint32_t));
414         if (! xdr_u_int(xdrs, &vers) ||
415             ! xdr_u_int(xdrs, &op)) {
416                 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_u_int failed");
417                 XDR_DESTROY(xdrs);
418                 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
419                 RDMA_REL_CONN(conn);
420                 freeb(mp);
421                 RSSTAT_INCR(rsbadcalls);
422                 return (FALSE);
423         }
424         if (op == RDMA_DONE) {
425                 /*
426                  * Should not get RDMA_DONE
427                  */
428                 freeb(mp);
429                 XDR_DESTROY(xdrs);
430                 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
431                 RDMA_REL_CONN(conn);
432                 RSSTAT_INCR(rsbadcalls);
433                 return (FALSE); /* no response */
434         }
435 
436 #ifdef DEBUG
437         if (rdma_svc_debug)
438                 printf("svc_rdma_krecv: recv'd call xid %u\n", xid);
439 #endif
440         /*
441          * Now decode the chunk list
442          */
443         cl = NULL;
444         if (! xdr_do_clist(xdrs, &cl)) {
445                 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_do_clist failed");
446         }
447 
448         /*
449          * A chunk at 0 offset indicates that the RPC call message
450          * is in a chunk. Get the RPC call message chunk.
451          */
452         if (cl != NULL && op == RDMA_NOMSG) {
453                 struct clist *cllong;   /* Long RPC chunk */
454 
455                 /* Remove RPC call message chunk from chunklist */
456                 cllong = cl;
457                 cl = cl->c_next;
458                 cllong->c_next = NULL;
459 
460                 /* Allocate and register memory for the RPC call msg chunk */
461                 cllong->c_daddr = (uint64)(uintptr_t)
462                     kmem_alloc(cllong->c_len, KM_SLEEP);
463                 if (cllong->c_daddr == NULL) {
464                         cmn_err(CE_WARN,
465                                 "svc_rdma_krecv: no memory for rpc call");
466                         XDR_DESTROY(xdrs);
467                         RDMA_BUF_FREE(conn, &rdp->rpcmsg);
468                         RDMA_REL_CONN(conn);
469                         freeb(mp);
470                         RSSTAT_INCR(rsbadcalls);
471                         clist_free(cl);
472                         clist_free(cllong);
473                         return (FALSE);
474                 }
475                 status = clist_register(conn, cllong, 0);
476                 if (status) {
477                         cmn_err(CE_WARN,
478                                 "svc_rdma_krecv: clist_register failed");
479                         kmem_free((void *)(uintptr_t)cllong->c_daddr,
480                             cllong->c_len);
481                         XDR_DESTROY(xdrs);
482                         RDMA_BUF_FREE(conn, &rdp->rpcmsg);
483                         RDMA_REL_CONN(conn);
484                         freeb(mp);
485                         RSSTAT_INCR(rsbadcalls);
486                         clist_free(cl);
487                         clist_free(cllong);
488                         return (FALSE);
489                 }
490 
491                 /*
492                  * Now read the RPC call message in
493                  */
494                 status = RDMA_READ(conn, cllong, WAIT);
495                 if (status) {
496                         cmn_err(CE_WARN,
497                             "svc_rdma_krecv: rdma_read failed %d", status);
498                         (void) clist_deregister(conn, cllong, 0);
499                         kmem_free((void *)(uintptr_t)cllong->c_daddr,
500                             cllong->c_len);
501                         XDR_DESTROY(xdrs);
502                         RDMA_BUF_FREE(conn, &rdp->rpcmsg);
503                         RDMA_REL_CONN(conn);
504                         freeb(mp);
505                         RSSTAT_INCR(rsbadcalls);
506                         clist_free(cl);
507                         clist_free(cllong);
508                         return (FALSE);
509                 }
510                 /*
511                  * Sync memory for CPU after DMA
512                  */
513                 status = clist_syncmem(conn, cllong, 0);
514 
515                 /*
516                  * Deregister the chunk
517                  */
518                 (void) clist_deregister(conn, cllong, 0);
519 
520                 /*
521                  * Setup the XDR for the RPC call message
522                  */
523                 xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->c_daddr,
524                     cllong->c_len, 0, cl, XDR_DECODE, conn);
525                 vd->rpcbuf.type = CHUNK_BUFFER;
526                 vd->rpcbuf.addr = (caddr_t)(uintptr_t)cllong->c_daddr;
527                 vd->rpcbuf.len = cllong->c_len;
528                 vd->rpcbuf.handle.mrc_rmr = 0;
529 
530                 /*
531                  * Free the chunk element with the Long RPC details and
532                  * the message received.
533                  */
534                 clist_free(cllong);
535                 RDMA_BUF_FREE(conn, &rdp->rpcmsg);
536         } else {
537                 pos = XDR_GETPOS(xdrs);
538 
539                 /*
540                  * Now the RPC call message header
541                  */
542                 xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos,
543                         rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn);
544                 vd->rpcbuf = rdp->rpcmsg;
545         }
546         if (! xdr_callmsg(xdrs, msg)) {
547                 cmn_err(CE_WARN, "svc_rdma_krecv: xdr_callmsg failed");
548                 if (cl != NULL)
549                         clist_free(cl);
550                 XDR_DESTROY(xdrs);
551                 rdma_buf_free(conn, &vd->rpcbuf);
552                 RDMA_REL_CONN(conn);
553                 freeb(mp);
554                 RSSTAT_INCR(rsxdrcall);
555                 RSSTAT_INCR(rsbadcalls);
556                 return (FALSE);
557         }
558 
559         /*
560          * Point the remote transport address in the service_transport
561          * handle at the address in the request.
562          */
563         clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf;
564         clone_xprt->xp_rtaddr.len = conn->c_raddr.len;
565         clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len;
566 
567 #ifdef DEBUG
568         if (rdma_svc_debug) {
569                 struct sockaddr_in *sin4;
570                 char print_addr[INET_ADDRSTRLEN];
571 
572                 sin4 = (struct sockaddr_in *)clone_xprt->xp_rtaddr.buf;
573                 bzero(print_addr, INET_ADDRSTRLEN);
574                 (void) inet_ntop(AF_INET,
575                     &sin4->sin_addr, print_addr, INET_ADDRSTRLEN);
576                 cmn_err(CE_NOTE,
577                     "svc_rdma_krecv: remote clnt_addr: %s", print_addr);
578         }
579 #endif
580 
581         clone_xprt->xp_xid = xid;
582         vd->conn = conn;
583         freeb(mp);
584         return (TRUE);
585 }
586 
587 /*
588  * Send rpc reply.
589  */
590 static bool_t
591 svc_rdma_ksend(SVCXPRT *clone_xprt, struct rpc_msg *msg)
592 {
593         struct clone_rdma_data *vd;
594         XDR *xdrs = &(clone_xprt->xp_xdrout), rxdrs;
595         int retval = FALSE;
596         xdrproc_t xdr_results;
597         caddr_t xdr_location;
598         bool_t has_args, reg = FALSE;
599         uint_t len, op;
600         uint_t vers;
601         struct clist *cl = NULL, *cle = NULL;
602         struct clist *sendlist = NULL;
603         int status;
604         int msglen;
605         rdma_buf_t clmsg, longreply, rpcreply;
606 
607         vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
608 
609         /*
610          * If there is a result procedure specified in the reply message,
611          * it will be processed in the xdr_replymsg and SVCAUTH_WRAP.
612          * We need to make sure it won't be processed twice, so we null
613          * it for xdr_replymsg here.
614          */
615         has_args = FALSE;
616         if (msg->rm_reply.rp_stat == MSG_ACCEPTED &&
617             msg->rm_reply.rp_acpt.ar_stat == SUCCESS) {
618                 if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) {
619                         has_args = TRUE;
620                         xdr_location = msg->acpted_rply.ar_results.where;
621                         msg->acpted_rply.ar_results.proc = xdr_void;
622                         msg->acpted_rply.ar_results.where = NULL;
623                 }
624         }
625 
626         /*
627          * Get the size of the rpc reply message. Need this
628          * to determine if the rpc reply message will fit in
629          * the pre-allocated RDMA buffers. If the rpc reply
630          * message length is greater that the pre-allocated
631          * buffers then, a one time use buffer is allocated
632          * and registered for this rpc reply.
633          */
634         msglen = xdr_sizeof(xdr_replymsg, msg);
635         if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor != RPCSEC_GSS) {
636                 msglen += xdrrdma_sizeof(xdr_results, xdr_location,
637                                 rdma_minchunk);
638                 if (msglen > RPC_MSG_SZ) {
639 
640                         /*
641                          * Allocate chunk buffer for rpc reply
642                          */
643                         rpcreply.type = CHUNK_BUFFER;
644                         rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
645                         cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
646                         cle->c_xdroff = 0;
647                         cle->c_len  = rpcreply.len = msglen;
648                         cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
649                         cle->c_next = NULL;
650                         xdrrdma_create(xdrs, rpcreply.addr, msglen,
651                             rdma_minchunk, cle, XDR_ENCODE, NULL);
652                         op = RDMA_NOMSG;
653                 } else {
654                         /*
655                          * Get a pre-allocated buffer for rpc reply
656                          */
657                         rpcreply.type = SEND_BUFFER;
658                         if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
659                                 cmn_err(CE_WARN,
660                                     "svc_rdma_ksend: no free buffers!");
661                                 return (retval);
662                         }
663                         xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
664                             rdma_minchunk, NULL, XDR_ENCODE, NULL);
665                         op = RDMA_MSG;
666                 }
667 
668                 /*
669                  * Initialize the XDR encode stream.
670                  */
671                 msg->rm_xid = clone_xprt->xp_xid;
672 
673                 if (!(xdr_replymsg(xdrs, msg) &&
674                     (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
675                     xdr_results, xdr_location)))) {
676                         rdma_buf_free(vd->conn, &rpcreply);
677                         if (cle)
678                                 clist_free(cle);
679                         cmn_err(CE_WARN,
680                             "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
681                             "failed");
682                         goto out;
683                 }
684                 len = XDR_GETPOS(xdrs);
685         }
686         if (has_args && msg->rm_reply.rp_acpt.ar_verf.oa_flavor == RPCSEC_GSS) {
687 
688                 /*
689                  * For RPCSEC_GSS since we cannot accurately presize the
690                  * buffer required for encoding, we assume that its going
691                  * to be a Long RPC to start with. We also create the
692                  * the XDR stream with min_chunk set to 0 which instructs
693                  * the XDR layer to not chunk the incoming byte stream.
694                  */
695                 msglen += 2 * MAX_AUTH_BYTES + 2 * sizeof (struct opaque_auth);
696                 msglen += xdr_sizeof(xdr_results, xdr_location);
697 
698                 /*
699                  * Long RPC. Allocate one time use custom buffer.
700                  */
701                 longreply.type = CHUNK_BUFFER;
702                 longreply.addr = kmem_zalloc(msglen, KM_SLEEP);
703                 cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
704                 cle->c_xdroff = 0;
705                 cle->c_len  = longreply.len = msglen;
706                 cle->c_saddr = (uint64)(uintptr_t)longreply.addr;
707                 cle->c_next = NULL;
708                 xdrrdma_create(xdrs, longreply.addr, msglen, 0, cle,
709                     XDR_ENCODE, NULL);
710                 op = RDMA_NOMSG;
711                 /*
712                  * Initialize the XDR encode stream.
713                  */
714                 msg->rm_xid = clone_xprt->xp_xid;
715 
716                 if (!(xdr_replymsg(xdrs, msg) &&
717                     (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, xdrs,
718                     xdr_results, xdr_location)))) {
719                         if (longreply.addr != xdrs->x_base) {
720                                 longreply.addr = xdrs->x_base;
721                                 longreply.len = xdr_getbufsize(xdrs);
722                         }
723                         rdma_buf_free(vd->conn, &longreply);
724                         if (cle)
725                                 clist_free(cle);
726                         cmn_err(CE_WARN,
727                             "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
728                             "failed");
729                         goto out;
730                 }
731 
732                 /*
733                  * If we had to allocate a new buffer while encoding
734                  * then update the addr and len.
735                  */
736                 if (longreply.addr != xdrs->x_base) {
737                         longreply.addr = xdrs->x_base;
738                         longreply.len = xdr_getbufsize(xdrs);
739                 }
740 
741                 len = XDR_GETPOS(xdrs);
742 
743                 /*
744                  * If it so happens that the encoded message is after all
745                  * not long enough to be a Long RPC then allocate a
746                  * SEND_BUFFER and copy the encoded message into it.
747                  */
748                 if (len > RPC_MSG_SZ) {
749                         rpcreply.type = CHUNK_BUFFER;
750                         rpcreply.addr = longreply.addr;
751                         rpcreply.len = longreply.len;
752                 } else {
753                         clist_free(cle);
754                         XDR_DESTROY(xdrs);
755                         /*
756                          * Get a pre-allocated buffer for rpc reply
757                          */
758                         rpcreply.type = SEND_BUFFER;
759                         if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
760                                 cmn_err(CE_WARN,
761                                     "svc_rdma_ksend: no free buffers!");
762                                 rdma_buf_free(vd->conn, &longreply);
763                                 return (retval);
764                         }
765                         bcopy(longreply.addr, rpcreply.addr, len);
766                         xdrrdma_create(xdrs, rpcreply.addr, len, 0, NULL,
767                             XDR_ENCODE, NULL);
768                         rdma_buf_free(vd->conn, &longreply);
769                         op = RDMA_MSG;
770                 }
771         }
772 
773         if (has_args == FALSE) {
774 
775                 if (msglen > RPC_MSG_SZ) {
776 
777                         /*
778                          * Allocate chunk buffer for rpc reply
779                          */
780                         rpcreply.type = CHUNK_BUFFER;
781                         rpcreply.addr = kmem_zalloc(msglen, KM_SLEEP);
782                         cle = kmem_zalloc(sizeof (*cle), KM_SLEEP);
783                         cle->c_xdroff = 0;
784                         cle->c_len  = rpcreply.len = msglen;
785                         cle->c_saddr = (uint64)(uintptr_t)rpcreply.addr;
786                         cle->c_next = NULL;
787                         xdrrdma_create(xdrs, rpcreply.addr, msglen,
788                             rdma_minchunk, cle, XDR_ENCODE, NULL);
789                         op = RDMA_NOMSG;
790                 } else {
791                         /*
792                          * Get a pre-allocated buffer for rpc reply
793                          */
794                         rpcreply.type = SEND_BUFFER;
795                         if (RDMA_BUF_ALLOC(vd->conn, &rpcreply)) {
796                                 cmn_err(CE_WARN,
797                                     "svc_rdma_ksend: no free buffers!");
798                                 return (retval);
799                         }
800                         xdrrdma_create(xdrs, rpcreply.addr, rpcreply.len,
801                             rdma_minchunk, NULL, XDR_ENCODE, NULL);
802                         op = RDMA_MSG;
803                 }
804 
805                 /*
806                  * Initialize the XDR encode stream.
807                  */
808                 msg->rm_xid = clone_xprt->xp_xid;
809 
810                 if (!xdr_replymsg(xdrs, msg)) {
811                         rdma_buf_free(vd->conn, &rpcreply);
812                         if (cle)
813                                 clist_free(cle);
814                         cmn_err(CE_WARN,
815                             "svc_rdma_ksend: xdr_replymsg/SVCAUTH_WRAP "
816                             "failed");
817                         goto out;
818                 }
819                 len = XDR_GETPOS(xdrs);
820         }
821 
822         /*
823          * Get clist and a buffer for sending it across
824          */
825         cl = xdrrdma_clist(xdrs);
826         clmsg.type = SEND_BUFFER;
827         if (RDMA_BUF_ALLOC(vd->conn, &clmsg)) {
828                 rdma_buf_free(vd->conn, &rpcreply);
829                 cmn_err(CE_WARN, "svc_rdma_ksend: no free buffers!!");
830                 goto out;
831         }
832 
833         /*
834          * Now register the chunks in the list
835          */
836         if (cl != NULL) {
837                 status = clist_register(vd->conn, cl, 1);
838                 if (status != RDMA_SUCCESS) {
839                         rdma_buf_free(vd->conn, &clmsg);
840                         cmn_err(CE_WARN,
841                                 "svc_rdma_ksend: clist register failed");
842                         goto out;
843                 }
844                 reg = TRUE;
845         }
846 
847         /*
848          * XDR the XID, vers, and op
849          */
850         /*
851          * Treat xid as opaque (xid is the first entity
852          * in the rpc rdma message).
853          */
854         vers = RPCRDMA_VERS;
855         xdrs = &rxdrs;
856         xdrmem_create(xdrs, clmsg.addr, clmsg.len, XDR_ENCODE);
857         (*(uint32_t *)clmsg.addr) = msg->rm_xid;
858         /* Skip xid and set the xdr position accordingly. */
859         XDR_SETPOS(xdrs, sizeof (uint32_t));
860         if (! xdr_u_int(xdrs, &vers) ||
861             ! xdr_u_int(xdrs, &op)) {
862                 rdma_buf_free(vd->conn, &rpcreply);
863                 rdma_buf_free(vd->conn, &clmsg);
864                 cmn_err(CE_WARN, "svc_rdma_ksend: xdr_u_int failed");
865                 goto out;
866         }
867 
868         /*
869          * Now XDR the chunk list
870          */
871         (void) xdr_do_clist(xdrs, &cl);
872 
873         clist_add(&sendlist, 0, XDR_GETPOS(xdrs), &clmsg.handle, clmsg.addr,
874                 NULL, NULL);
875 
876         if (op == RDMA_MSG) {
877                 clist_add(&sendlist, 0, len, &rpcreply.handle, rpcreply.addr,
878                         NULL, NULL);
879         } else {
880                 cl->c_len = len;
881                 RSSTAT_INCR(rslongrpcs);
882         }
883 
884         /*
885          * Send the reply message to the client
886          */
887         if (cl != NULL) {
888                 status = clist_syncmem(vd->conn, cl, 1);
889                 if (status != RDMA_SUCCESS) {
890                         rdma_buf_free(vd->conn, &rpcreply);
891                         rdma_buf_free(vd->conn, &clmsg);
892                         goto out;
893                 }
894 #ifdef DEBUG
895         if (rdma_svc_debug)
896                 printf("svc_rdma_ksend: chunk response len %d xid %u\n",
897                         cl->c_len, msg->rm_xid);
898 #endif
899                 /*
900                  * Post a receive buffer because we expect a RDMA_DONE
901                  * message.
902                  */
903                 status = rdma_svc_postrecv(vd->conn);
904 
905                 /*
906                  * Send the RPC reply message and wait for RDMA_DONE
907                  */
908                 status = RDMA_SEND_RESP(vd->conn, sendlist, msg->rm_xid);
909                 if (status != RDMA_SUCCESS) {
910 #ifdef DEBUG
911                         if (rdma_svc_debug)
912                                 cmn_err(CE_NOTE, "svc_rdma_ksend: "
913                                         "rdma_send_resp failed %d", status);
914 #endif
915                         goto out;
916                 }
917 #ifdef DEBUG
918         if (rdma_svc_debug)
919                 printf("svc_rdma_ksend: got RDMA_DONE xid %u\n", msg->rm_xid);
920 #endif
921         } else {
922 #ifdef DEBUG
923         if (rdma_svc_debug)
924                 printf("svc_rdma_ksend: msg response xid %u\n", msg->rm_xid);
925 #endif
926                 status = RDMA_SEND(vd->conn, sendlist, msg->rm_xid);
927                 if (status != RDMA_SUCCESS) {
928 #ifdef DEBUG
929                         if (rdma_svc_debug)
930                                 cmn_err(CE_NOTE, "svc_rdma_ksend: "
931                                         "rdma_send failed %d", status);
932 #endif
933                         goto out;
934                 }
935         }
936 
937         retval = TRUE;
938 out:
939         /*
940          * Deregister the chunks
941          */
942         if (cl != NULL) {
943                 if (reg)
944                         (void) clist_deregister(vd->conn, cl, 1);
945                 if (op == RDMA_NOMSG) {
946                         /*
947                          * Long RPC reply in chunk. Free it up.
948                          */
949                         rdma_buf_free(vd->conn, &rpcreply);
950                 }
951                 clist_free(cl);
952         }
953 
954         /*
955          * Free up sendlist chunks
956          */
957         if (sendlist != NULL)
958                 clist_free(sendlist);
959 
960         /*
961          * Destroy private data for xdr rdma
962          */
963         XDR_DESTROY(&(clone_xprt->xp_xdrout));
964 
965         /*
966          * This is completely disgusting.  If public is set it is
967          * a pointer to a structure whose first field is the address
968          * of the function to free that structure and any related
969          * stuff.  (see rrokfree in nfs_xdr.c).
970          */
971         if (xdrs->x_public) {
972                 /* LINTED pointer alignment */
973                 (**((int (**)())xdrs->x_public))(xdrs->x_public);
974         }
975 
976         return (retval);
977 }
978 
979 /*
980  * Deserialize arguments.
981  */
982 static bool_t
983 svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr)
984 {
985         if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin,
986             xdr_args, args_ptr)) != TRUE)
987                 return (FALSE);
988         return (TRUE);
989 }
990 
991 static bool_t
992 svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args,
993     caddr_t args_ptr)
994 {
995         struct clone_rdma_data *vd;
996         bool_t retval;
997 
998         vd = (struct clone_rdma_data *)clone_xprt->xp_p2buf;
999         if (args_ptr) {
1000                 XDR     *xdrs = &clone_xprt->xp_xdrin;
1001                 struct clist *cl;
1002 
1003                 cl = xdrrdma_clist(xdrs);
1004                 if (cl != NULL)
1005                         clist_free(cl);
1006 
1007                 xdrs->x_op = XDR_FREE;
1008                 retval = (*xdr_args)(xdrs, args_ptr);
1009         }
1010         XDR_DESTROY(&(clone_xprt->xp_xdrin));
1011         rdma_buf_free(vd->conn, &vd->rpcbuf);
1012         RDMA_REL_CONN(vd->conn);
1013         return (retval);
1014 }
1015 
1016 /* ARGSUSED */
1017 static int32_t *
1018 svc_rdma_kgetres(SVCXPRT *clone_xprt, int size)
1019 {
1020         return (NULL);
1021 }
1022 
1023 /* ARGSUSED */
1024 static void
1025 svc_rdma_kfreeres(SVCXPRT *clone_xprt)
1026 {
1027 }
1028 
1029 /*
1030  * the dup cacheing routines below provide a cache of non-failure
1031  * transaction id's.  rpc service routines can use this to detect
1032  * retransmissions and re-send a non-failure response.
1033  */
1034 
1035 /*
1036  * MAXDUPREQS is the number of cached items.  It should be adjusted
1037  * to the service load so that there is likely to be a response entry
1038  * when the first retransmission comes in.
1039  */
1040 #define MAXDUPREQS      1024
1041 
1042 /*
1043  * This should be appropriately scaled to MAXDUPREQS.
1044  */
1045 #define DRHASHSZ        257
1046 
1047 #if ((DRHASHSZ & (DRHASHSZ - 1)) == 0)
1048 #define XIDHASH(xid)    ((xid) & (DRHASHSZ - 1))
1049 #else
1050 #define XIDHASH(xid)    ((xid) % DRHASHSZ)
1051 #endif
1052 #define DRHASH(dr)      XIDHASH((dr)->dr_xid)
1053 #define REQTOXID(req)   ((req)->rq_xprt->xp_xid)
1054 
1055 static int      rdmandupreqs = 0;
1056 static int      rdmamaxdupreqs = MAXDUPREQS;
1057 static kmutex_t rdmadupreq_lock;
1058 static struct dupreq *rdmadrhashtbl[DRHASHSZ];
1059 static int      rdmadrhashstat[DRHASHSZ];
1060 
1061 static void unhash(struct dupreq *);
1062 
1063 /*
1064  * rdmadrmru points to the head of a circular linked list in lru order.
1065  * rdmadrmru->dr_next == drlru
1066  */
1067 struct dupreq *rdmadrmru;
1068 
1069 /*
1070  * svc_rdma_kdup searches the request cache and returns 0 if the
1071  * request is not found in the cache.  If it is found, then it
1072  * returns the state of the request (in progress or done) and
1073  * the status or attributes that were part of the original reply.
1074  */
1075 static int
1076 svc_rdma_kdup(struct svc_req *req, caddr_t res, int size, struct dupreq **drpp,
1077         bool_t *dupcachedp)
1078 {
1079         struct dupreq *dr;
1080         uint32_t xid;
1081         uint32_t drhash;
1082         int status;
1083 
1084         xid = REQTOXID(req);
1085         mutex_enter(&rdmadupreq_lock);
1086         RSSTAT_INCR(rsdupchecks);
1087         /*
1088          * Check to see whether an entry already exists in the cache.
1089          */
1090         dr = rdmadrhashtbl[XIDHASH(xid)];
1091         while (dr != NULL) {
1092                 if (dr->dr_xid == xid &&
1093                     dr->dr_proc == req->rq_proc &&
1094                     dr->dr_prog == req->rq_prog &&
1095                     dr->dr_vers == req->rq_vers &&
1096                     dr->dr_addr.len == req->rq_xprt->xp_rtaddr.len &&
1097                     bcmp((caddr_t)dr->dr_addr.buf,
1098                     (caddr_t)req->rq_xprt->xp_rtaddr.buf,
1099                     dr->dr_addr.len) == 0) {
1100                         status = dr->dr_status;
1101                         if (status == DUP_DONE) {
1102                                 bcopy(dr->dr_resp.buf, res, size);
1103                                 if (dupcachedp != NULL)
1104                                         *dupcachedp = (dr->dr_resfree != NULL);
1105                         } else {
1106                                 dr->dr_status = DUP_INPROGRESS;
1107                                 *drpp = dr;
1108                         }
1109                         RSSTAT_INCR(rsdupreqs);
1110                         mutex_exit(&rdmadupreq_lock);
1111                         return (status);
1112                 }
1113                 dr = dr->dr_chain;
1114         }
1115 
1116         /*
1117          * There wasn't an entry, either allocate a new one or recycle
1118          * an old one.
1119          */
1120         if (rdmandupreqs < rdmamaxdupreqs) {
1121                 dr = kmem_alloc(sizeof (*dr), KM_NOSLEEP);
1122                 if (dr == NULL) {
1123                         mutex_exit(&rdmadupreq_lock);
1124                         return (DUP_ERROR);
1125                 }
1126                 dr->dr_resp.buf = NULL;
1127                 dr->dr_resp.maxlen = 0;
1128                 dr->dr_addr.buf = NULL;
1129                 dr->dr_addr.maxlen = 0;
1130                 if (rdmadrmru) {
1131                         dr->dr_next = rdmadrmru->dr_next;
1132                         rdmadrmru->dr_next = dr;
1133                 } else {
1134                         dr->dr_next = dr;
1135                 }
1136                 rdmandupreqs++;
1137         } else {
1138                 dr = rdmadrmru->dr_next;
1139                 while (dr->dr_status == DUP_INPROGRESS) {
1140                         dr = dr->dr_next;
1141                         if (dr == rdmadrmru->dr_next) {
1142                                 cmn_err(CE_WARN, "svc_rdma_kdup no slots free");
1143                                 mutex_exit(&rdmadupreq_lock);
1144                                 return (DUP_ERROR);
1145                         }
1146                 }
1147                 unhash(dr);
1148                 if (dr->dr_resfree) {
1149                         (*dr->dr_resfree)(dr->dr_resp.buf);
1150                 }
1151         }
1152         dr->dr_resfree = NULL;
1153         rdmadrmru = dr;
1154 
1155         dr->dr_xid = REQTOXID(req);
1156         dr->dr_prog = req->rq_prog;
1157         dr->dr_vers = req->rq_vers;
1158         dr->dr_proc = req->rq_proc;
1159         if (dr->dr_addr.maxlen < req->rq_xprt->xp_rtaddr.len) {
1160                 if (dr->dr_addr.buf != NULL)
1161                         kmem_free(dr->dr_addr.buf, dr->dr_addr.maxlen);
1162                 dr->dr_addr.maxlen = req->rq_xprt->xp_rtaddr.len;
1163                 dr->dr_addr.buf = kmem_alloc(dr->dr_addr.maxlen, KM_NOSLEEP);
1164                 if (dr->dr_addr.buf == NULL) {
1165                         dr->dr_addr.maxlen = 0;
1166                         dr->dr_status = DUP_DROP;
1167                         mutex_exit(&rdmadupreq_lock);
1168                         return (DUP_ERROR);
1169                 }
1170         }
1171         dr->dr_addr.len = req->rq_xprt->xp_rtaddr.len;
1172         bcopy(req->rq_xprt->xp_rtaddr.buf, dr->dr_addr.buf, dr->dr_addr.len);
1173         if (dr->dr_resp.maxlen < size) {
1174                 if (dr->dr_resp.buf != NULL)
1175                         kmem_free(dr->dr_resp.buf, dr->dr_resp.maxlen);
1176                 dr->dr_resp.maxlen = (unsigned int)size;
1177                 dr->dr_resp.buf = kmem_alloc(size, KM_NOSLEEP);
1178                 if (dr->dr_resp.buf == NULL) {
1179                         dr->dr_resp.maxlen = 0;
1180                         dr->dr_status = DUP_DROP;
1181                         mutex_exit(&rdmadupreq_lock);
1182                         return (DUP_ERROR);
1183                 }
1184         }
1185         dr->dr_status = DUP_INPROGRESS;
1186 
1187         drhash = (uint32_t)DRHASH(dr);
1188         dr->dr_chain = rdmadrhashtbl[drhash];
1189         rdmadrhashtbl[drhash] = dr;
1190         rdmadrhashstat[drhash]++;
1191         mutex_exit(&rdmadupreq_lock);
1192         *drpp = dr;
1193         return (DUP_NEW);
1194 }
1195 
1196 /*
1197  * svc_rdma_kdupdone marks the request done (DUP_DONE or DUP_DROP)
1198  * and stores the response.
1199  */
1200 static void
1201 svc_rdma_kdupdone(struct dupreq *dr, caddr_t res, void (*dis_resfree)(),
1202         int size, int status)
1203 {
1204         ASSERT(dr->dr_resfree == NULL);
1205         if (status == DUP_DONE) {
1206                 bcopy(res, dr->dr_resp.buf, size);
1207                 dr->dr_resfree = dis_resfree;
1208         }
1209         dr->dr_status = status;
1210 }
1211 
1212 /*
1213  * This routine expects that the mutex, rdmadupreq_lock, is already held.
1214  */
1215 static void
1216 unhash(struct dupreq *dr)
1217 {
1218         struct dupreq *drt;
1219         struct dupreq *drtprev = NULL;
1220         uint32_t drhash;
1221 
1222         ASSERT(MUTEX_HELD(&rdmadupreq_lock));
1223 
1224         drhash = (uint32_t)DRHASH(dr);
1225         drt = rdmadrhashtbl[drhash];
1226         while (drt != NULL) {
1227                 if (drt == dr) {
1228                         rdmadrhashstat[drhash]--;
1229                         if (drtprev == NULL) {
1230                                 rdmadrhashtbl[drhash] = drt->dr_chain;
1231                         } else {
1232                                 drtprev->dr_chain = drt->dr_chain;
1233                         }
1234                         return;
1235                 }
1236                 drtprev = drt;
1237                 drt = drt->dr_chain;
1238         }
1239 }