7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #ifndef _IB_H
28 #define _IB_H
29
30 #pragma ident "@(#)ib.h 1.8 05/06/08 SMI"
31
32 /*
33 * ib.h, rpcib plugin interface.
34 */
35
36 #include <sys/types.h>
37 #include <sys/ddi.h>
38 #include <sys/sunddi.h>
39 #include <sys/conf.h>
40 #include <sys/stat.h>
41 #include <rpc/rpc.h>
42 #include <rpc/rpc_rdma.h>
43 #include <sys/ib/ibtl/ibti.h>
44
45 #ifdef __cplusplus
46 extern "C" {
47 #endif
48
49 #define MAX_BUFS 256 /* max no. of buffers per pool */
50 #define DEF_CQ_SIZE 4096 - 1 /* default CQ size */
51 /*
52 * Tavor returns the next higher power of 2
53 * CQ entries than the requested size.
54 * For instance, if you request (2^12 - 1)
55 * CQ entries, Tavor returns 2^12 entries.
56 * 4K CQ entries suffice. Hence, 4096 - 1.
57 */
58 #define DEF_SQ_SIZE 128 /* default SendQ size */
59 #define DEF_RQ_SIZE 256 /* default RecvQ size */
60 #define DSEG_MAX 2
61 #define RQ_DSEG_MAX 1 /* default RQ data seg */
62 #define IBSRM_HB 0x8000 /* high order bit of pkey */
63 #define NFS_SEC_KEY0 0x6878 /* randomly selected NFS security key */
105 * IB buffer pool management structure
106 */
107
108 /*
109 * Buffer pool info
110 */
111 typedef struct {
112 kmutex_t buflock; /* lock for this structure */
113 caddr_t buf; /* pool address */
114 uint32_t bufhandle; /* rkey for this pool */
115 ulong_t bufsize; /* size of pool */
116 int rsize; /* size of each element */
117 int numelems; /* no. of elements allocated */
118 int buffree; /* no. of free elements */
119 void *buflist[1]; /* free elements in pool */
120 } bufpool_t;
121
122 typedef struct {
123 bufpool_t *bpool;
124 ibt_mr_hdl_t *mr_hdl;
125 ibt_mr_desc_t *mr_desc; /* vaddr, lkey, rkey */
126 } rib_bufpool_t;
127
128 /*
129 * ATS relsted defines and structures.
130 */
131 #define ATS_AR_DATA_LEN 16
132 #define IBD_NAME "ibd"
133 #define N_IBD_INSTANCES 4
134
135 typedef struct rpcib_ats_s {
136 int ras_inst;
137 ib_pkey_t ras_pkey;
138 ib_gid_t ras_port_gid;
139 sa_family_t ras_inet_type;
140 union {
141 struct sockaddr_in ras_sockaddr;
142 struct sockaddr_in6 ras_sockaddr6;
143 } ra_sin;
144 #define ras_sin ra_sin.ras_sockaddr
145 #define ras_sin6 ra_sin.ras_sockaddr6
262 */
263 rib_service_t *service_list;
264 krwlock_t service_list_lock;
265
266 rib_service_t *ats_list; /* Service list for ATS */
267
268 rib_conn_list_t cl_conn_list; /* client conn list */
269 rib_conn_list_t srv_conn_list; /* server conn list */
270
271 rib_cq_t *clnt_scq;
272 rib_cq_t *clnt_rcq;
273 rib_cq_t *svc_scq;
274 rib_cq_t *svc_rcq;
275 kmutex_t cb_lock;
276 kcondvar_t cb_cv;
277
278 rib_bufpool_t *recv_pool; /* recv buf pool */
279 rib_bufpool_t *send_pool; /* send buf pool */
280
281 void *iblock; /* interrupt cookie */
282 };
283
284
285 /*
286 * Structure on wait state of a post send
287 */
288 struct send_wid {
289 uint32_t xid;
290 int cv_sig;
291 kmutex_t sendwait_lock;
292 kcondvar_t wait_cv;
293 uint_t status;
294 rib_qp_t *qp;
295 int nsbufs; /* # of send buffers posted */
296 uint64_t sbufaddr[DSEG_MAX]; /* posted send buffers */
297 };
298
299 /*
300 * Structure on reply descriptor for recv queue.
301 * Different from the above posting of a descriptor.
302 */
303 struct reply {
304 uint32_t xid;
305 uint_t status;
306 uint64_t vaddr_cq; /* buf addr from CQ */
307 uint_t bytes_xfer;
308 kcondvar_t wait_cv;
309 struct reply *next;
310 struct reply *prev;
311 };
312
313 struct svc_recv {
314 rib_qp_t *qp;
315 uint64_t vaddr;
316 uint_t bytes_xfer;
345 kcondvar_t posted_rbufs_cv;
346 kmutex_t posted_rbufs_lock;
347
348 /*
349 * RPC reply
350 */
351 uint_t rep_list_size;
352 struct reply *replylist;
353 kmutex_t replylist_lock;
354
355 /*
356 * server only, RDMA_DONE
357 */
358 struct rdma_done_list *rdlist;
359 kmutex_t rdlist_lock;
360
361 kmutex_t cb_lock;
362 kcondvar_t cb_conn_cv;
363
364 caddr_t q; /* upstream queue */
365 };
366
367 #define ctoqp(conn) ((rib_qp_t *)((conn)->c_private))
368 #define qptoc(rqp) ((CONN *)&((rqp)->rdmaconn))
369
370 /*
371 * Timeout for various calls
372 */
373 #define CONN_WAIT_TIME 40
374 #define SEND_WAIT_TIME 40 /* time for send completion */
375
376 #define REPLY_WAIT_TIME 40 /* time to get reply from remote QP */
377
378 #ifdef __cplusplus
379 }
380 #endif
381
382 #endif /* !_IB_H */
|
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /* Copyright (c) 2006, The Ohio State University. All rights reserved.
28 *
29 * Portions of this source code is developed by the team members of
30 * The Ohio State University's Network-Based Computing Laboratory (NBCL),
31 * headed by Professor Dhabaleswar K. (DK) Panda.
32 *
33 * Acknowledgements to contributions from developors:
34 * Ranjit Noronha: noronha@cse.ohio-state.edu
35 * Lei Chai : chail@cse.ohio-state.edu
36 * Weikuan Yu : yuw@cse.ohio-state.edu
37 *
38 */
39 #ifndef _IB_H
40 #define _IB_H
41
42 #pragma ident "@(#)ib.h 1.8 05/06/08 SMI"
43
44 /*
45 * ib.h, rpcib plugin interface.
46 */
47
48 #include <sys/types.h>
49 #include <sys/ddi.h>
50 #include <sys/sunddi.h>
51 #include <sys/conf.h>
52 #include <sys/stat.h>
53 #include <rpc/rpc.h>
54 #include <rpc/rpc_rdma.h>
55 #include <sys/ib/ibtl/ibti.h>
56 #ifdef SERVER_REG_CACHE
57 #include <sys/avl.h>
58 #endif
59
60 #ifdef __cplusplus
61 extern "C" {
62 #endif
63
64 #define MAX_BUFS 256 /* max no. of buffers per pool */
65 #define DEF_CQ_SIZE 4096 - 1 /* default CQ size */
66 /*
67 * Tavor returns the next higher power of 2
68 * CQ entries than the requested size.
69 * For instance, if you request (2^12 - 1)
70 * CQ entries, Tavor returns 2^12 entries.
71 * 4K CQ entries suffice. Hence, 4096 - 1.
72 */
73 #define DEF_SQ_SIZE 128 /* default SendQ size */
74 #define DEF_RQ_SIZE 256 /* default RecvQ size */
75 #define DSEG_MAX 2
76 #define RQ_DSEG_MAX 1 /* default RQ data seg */
77 #define IBSRM_HB 0x8000 /* high order bit of pkey */
78 #define NFS_SEC_KEY0 0x6878 /* randomly selected NFS security key */
120 * IB buffer pool management structure
121 */
122
123 /*
124 * Buffer pool info
125 */
126 typedef struct {
127 kmutex_t buflock; /* lock for this structure */
128 caddr_t buf; /* pool address */
129 uint32_t bufhandle; /* rkey for this pool */
130 ulong_t bufsize; /* size of pool */
131 int rsize; /* size of each element */
132 int numelems; /* no. of elements allocated */
133 int buffree; /* no. of free elements */
134 void *buflist[1]; /* free elements in pool */
135 } bufpool_t;
136
137 typedef struct {
138 bufpool_t *bpool;
139 ibt_mr_hdl_t *mr_hdl;
140 #ifdef IB_FMR_SUP
141 ibt_ma_hdl_t *ma_hdl;
142 ibt_pmr_desc_t *pmr_desc;
143 #endif
144 ibt_mr_desc_t *mr_desc; /* vaddr, lkey, rkey */
145
146 } rib_bufpool_t;
147
148 /*
149 * ATS relsted defines and structures.
150 */
151 #define ATS_AR_DATA_LEN 16
152 #define IBD_NAME "ibd"
153 #define N_IBD_INSTANCES 4
154
155 typedef struct rpcib_ats_s {
156 int ras_inst;
157 ib_pkey_t ras_pkey;
158 ib_gid_t ras_port_gid;
159 sa_family_t ras_inet_type;
160 union {
161 struct sockaddr_in ras_sockaddr;
162 struct sockaddr_in6 ras_sockaddr6;
163 } ra_sin;
164 #define ras_sin ra_sin.ras_sockaddr
165 #define ras_sin6 ra_sin.ras_sockaddr6
282 */
283 rib_service_t *service_list;
284 krwlock_t service_list_lock;
285
286 rib_service_t *ats_list; /* Service list for ATS */
287
288 rib_conn_list_t cl_conn_list; /* client conn list */
289 rib_conn_list_t srv_conn_list; /* server conn list */
290
291 rib_cq_t *clnt_scq;
292 rib_cq_t *clnt_rcq;
293 rib_cq_t *svc_scq;
294 rib_cq_t *svc_rcq;
295 kmutex_t cb_lock;
296 kcondvar_t cb_cv;
297
298 rib_bufpool_t *recv_pool; /* recv buf pool */
299 rib_bufpool_t *send_pool; /* send buf pool */
300
301 void *iblock; /* interrupt cookie */
302
303 #ifdef IB_FMR_SUP
304 ibt_fmr_pool_hdl_t fmr_pool;
305 #endif /* IB_FMR_SUP */
306
307 #ifdef SERVER_REG_CACHE
308 kmem_cache_t *server_side_cache; /* long reply pool */
309 avl_tree_t avl_tree;
310 kmutex_t avl_lock;
311 krwlock_t avl_rw_lock;
312 volatile bool_t avl_init;
313 #endif
314
315 };
316
317
318 /*
319 * Structure on wait state of a post send
320 */
321 struct send_wid {
322 #if defined (CLNT_INTERRUPT_COAL)
323 struct send_wid *forw;
324 struct send_wid *back;
325 #endif
326 uint32_t xid;
327 int cv_sig;
328 kmutex_t sendwait_lock;
329 kcondvar_t wait_cv;
330 uint_t status;
331 rib_qp_t *qp;
332 int nsbufs; /* # of send buffers posted */
333 uint64_t sbufaddr[DSEG_MAX]; /* posted send buffers */
334 caddr_t c;
335 caddr_t c1;
336 int l1;
337 caddr_t c2;
338 int l2;
339 int wl,rl;
340 };
341
342 /*
343 * Structure on reply descriptor for recv queue.
344 * Different from the above posting of a descriptor.
345 */
346 struct reply {
347 uint32_t xid;
348 uint_t status;
349 uint64_t vaddr_cq; /* buf addr from CQ */
350 uint_t bytes_xfer;
351 kcondvar_t wait_cv;
352 struct reply *next;
353 struct reply *prev;
354 };
355
356 struct svc_recv {
357 rib_qp_t *qp;
358 uint64_t vaddr;
359 uint_t bytes_xfer;
388 kcondvar_t posted_rbufs_cv;
389 kmutex_t posted_rbufs_lock;
390
391 /*
392 * RPC reply
393 */
394 uint_t rep_list_size;
395 struct reply *replylist;
396 kmutex_t replylist_lock;
397
398 /*
399 * server only, RDMA_DONE
400 */
401 struct rdma_done_list *rdlist;
402 kmutex_t rdlist_lock;
403
404 kmutex_t cb_lock;
405 kcondvar_t cb_conn_cv;
406
407 caddr_t q; /* upstream queue */
408 struct send_wid wd;
409 };
410
411 #define ctoqp(conn) ((rib_qp_t *)((conn)->c_private))
412 #define qptoc(rqp) ((CONN *)&((rqp)->rdmaconn))
413
414 /*
415 * Timeout for various calls
416 */
417 #define CONN_WAIT_TIME 40
418 #define SEND_WAIT_TIME 40 /* time for send completion */
419
420 #define REPLY_WAIT_TIME 40 /* time to get reply from remote QP */
421
422 #ifdef __cplusplus
423 }
424 #endif
425
426 #endif /* !_IB_H */
|