Sdiff ib.h


7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _IB_H 28 #define _IB_H 29 30 #pragma ident "@(#)ib.h 1.8 05/06/08 SMI" 31 32 /* 33 * ib.h, rpcib plugin interface. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/ddi.h> 38 #include <sys/sunddi.h> 39 #include <sys/conf.h> 40 #include <sys/stat.h> 41 #include <rpc/rpc.h> 42 #include <rpc/rpc_rdma.h> 43 #include <sys/ib/ibtl/ibti.h> 44 45 #ifdef __cplusplus 46 extern "C" { 47 #endif 48 49 #define MAX_BUFS 256 /* max no. of buffers per pool */ 50 #define DEF_CQ_SIZE 4096 - 1 /* default CQ size */ 51 /* 52 * Tavor returns the next higher power of 2 53 * CQ entries than the requested size. 54 * For instance, if you request (2^12 - 1) 55 * CQ entries, Tavor returns 2^12 entries. 56 * 4K CQ entries suffice. Hence, 4096 - 1. 57 */ 58 #define DEF_SQ_SIZE 128 /* default SendQ size */ 59 #define DEF_RQ_SIZE 256 /* default RecvQ size */ 60 #define DSEG_MAX 2 61 #define RQ_DSEG_MAX 1 /* default RQ data seg */ 62 #define IBSRM_HB 0x8000 /* high order bit of pkey */ 63 #define NFS_SEC_KEY0 0x6878 /* randomly selected NFS security key */
105 * IB buffer pool management structure 106 */ 107 108 /* 109 * Buffer pool info 110 */ 111 typedef struct { 112 kmutex_t buflock; /* lock for this structure */ 113 caddr_t buf; /* pool address */ 114 uint32_t bufhandle; /* rkey for this pool */ 115 ulong_t bufsize; /* size of pool */ 116 int rsize; /* size of each element */ 117 int numelems; /* no. of elements allocated */ 118 int buffree; /* no. of free elements */ 119 void *buflist[1]; /* free elements in pool */ 120 } bufpool_t; 121 122 typedef struct { 123 bufpool_t *bpool; 124 ibt_mr_hdl_t *mr_hdl; 125 ibt_mr_desc_t *mr_desc; /* vaddr, lkey, rkey */ 126 } rib_bufpool_t; 127 128 /* 129 * ATS relsted defines and structures. 130 */ 131 #define ATS_AR_DATA_LEN 16 132 #define IBD_NAME "ibd" 133 #define N_IBD_INSTANCES 4 134 135 typedef struct rpcib_ats_s { 136 int ras_inst; 137 ib_pkey_t ras_pkey; 138 ib_gid_t ras_port_gid; 139 sa_family_t ras_inet_type; 140 union { 141 struct sockaddr_in ras_sockaddr; 142 struct sockaddr_in6 ras_sockaddr6; 143 } ra_sin; 144 #define ras_sin ra_sin.ras_sockaddr 145 #define ras_sin6 ra_sin.ras_sockaddr6
262 */ 263 rib_service_t *service_list; 264 krwlock_t service_list_lock; 265 266 rib_service_t *ats_list; /* Service list for ATS */ 267 268 rib_conn_list_t cl_conn_list; /* client conn list */ 269 rib_conn_list_t srv_conn_list; /* server conn list */ 270 271 rib_cq_t *clnt_scq; 272 rib_cq_t *clnt_rcq; 273 rib_cq_t *svc_scq; 274 rib_cq_t *svc_rcq; 275 kmutex_t cb_lock; 276 kcondvar_t cb_cv; 277 278 rib_bufpool_t *recv_pool; /* recv buf pool */ 279 rib_bufpool_t *send_pool; /* send buf pool */ 280 281 void *iblock; /* interrupt cookie */ 282 }; 283 284 285 /* 286 * Structure on wait state of a post send 287 */ 288 struct send_wid { 289 uint32_t xid; 290 int cv_sig; 291 kmutex_t sendwait_lock; 292 kcondvar_t wait_cv; 293 uint_t status; 294 rib_qp_t *qp; 295 int nsbufs; /* # of send buffers posted */ 296 uint64_t sbufaddr[DSEG_MAX]; /* posted send buffers */ 297 }; 298 299 /* 300 * Structure on reply descriptor for recv queue. 301 * Different from the above posting of a descriptor. 302 */ 303 struct reply { 304 uint32_t xid; 305 uint_t status; 306 uint64_t vaddr_cq; /* buf addr from CQ */ 307 uint_t bytes_xfer; 308 kcondvar_t wait_cv; 309 struct reply *next; 310 struct reply *prev; 311 }; 312 313 struct svc_recv { 314 rib_qp_t *qp; 315 uint64_t vaddr; 316 uint_t bytes_xfer;
345 kcondvar_t posted_rbufs_cv; 346 kmutex_t posted_rbufs_lock; 347 348 /* 349 * RPC reply 350 */ 351 uint_t rep_list_size; 352 struct reply *replylist; 353 kmutex_t replylist_lock; 354 355 /* 356 * server only, RDMA_DONE 357 */ 358 struct rdma_done_list *rdlist; 359 kmutex_t rdlist_lock; 360 361 kmutex_t cb_lock; 362 kcondvar_t cb_conn_cv; 363 364 caddr_t q; /* upstream queue */ 365 }; 366 367 #define ctoqp(conn) ((rib_qp_t *)((conn)->c_private)) 368 #define qptoc(rqp) ((CONN *)&((rqp)->rdmaconn)) 369 370 /* 371 * Timeout for various calls 372 */ 373 #define CONN_WAIT_TIME 40 374 #define SEND_WAIT_TIME 40 /* time for send completion */ 375 376 #define REPLY_WAIT_TIME 40 /* time to get reply from remote QP */ 377 378 #ifdef __cplusplus 379 } 380 #endif 381 382 #endif /* !_IB_H */


7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* Copyright (c) 2006, The Ohio State University. All rights reserved. 28 * 29 * Portions of this source code is developed by the team members of 30 * The Ohio State University's Network-Based Computing Laboratory (NBCL), 31 * headed by Professor Dhabaleswar K. (DK) Panda. 32 * 33 * Acknowledgements to contributions from developors: 34 * Ranjit Noronha: noronha@cse.ohio-state.edu 35 * Lei Chai : chail@cse.ohio-state.edu 36 * Weikuan Yu : yuw@cse.ohio-state.edu 37 * 38 */ 39 #ifndef _IB_H 40 #define _IB_H 41 42 #pragma ident "@(#)ib.h 1.8 05/06/08 SMI" 43 44 /* 45 * ib.h, rpcib plugin interface. 46 */ 47 48 #include <sys/types.h> 49 #include <sys/ddi.h> 50 #include <sys/sunddi.h> 51 #include <sys/conf.h> 52 #include <sys/stat.h> 53 #include <rpc/rpc.h> 54 #include <rpc/rpc_rdma.h> 55 #include <sys/ib/ibtl/ibti.h> 56 #ifdef SERVER_REG_CACHE 57 #include <sys/avl.h> 58 #endif 59 60 #ifdef __cplusplus 61 extern "C" { 62 #endif 63 64 #define MAX_BUFS 256 /* max no. of buffers per pool */ 65 #define DEF_CQ_SIZE 4096 - 1 /* default CQ size */ 66 /* 67 * Tavor returns the next higher power of 2 68 * CQ entries than the requested size. 69 * For instance, if you request (2^12 - 1) 70 * CQ entries, Tavor returns 2^12 entries. 71 * 4K CQ entries suffice. Hence, 4096 - 1. 72 */ 73 #define DEF_SQ_SIZE 128 /* default SendQ size */ 74 #define DEF_RQ_SIZE 256 /* default RecvQ size */ 75 #define DSEG_MAX 2 76 #define RQ_DSEG_MAX 1 /* default RQ data seg */ 77 #define IBSRM_HB 0x8000 /* high order bit of pkey */ 78 #define NFS_SEC_KEY0 0x6878 /* randomly selected NFS security key */
120 * IB buffer pool management structure 121 */ 122 123 /* 124 * Buffer pool info 125 */ 126 typedef struct { 127 kmutex_t buflock; /* lock for this structure */ 128 caddr_t buf; /* pool address */ 129 uint32_t bufhandle; /* rkey for this pool */ 130 ulong_t bufsize; /* size of pool */ 131 int rsize; /* size of each element */ 132 int numelems; /* no. of elements allocated */ 133 int buffree; /* no. of free elements */ 134 void *buflist[1]; /* free elements in pool */ 135 } bufpool_t; 136 137 typedef struct { 138 bufpool_t *bpool; 139 ibt_mr_hdl_t *mr_hdl; 140 #ifdef IB_FMR_SUP 141 ibt_ma_hdl_t *ma_hdl; 142 ibt_pmr_desc_t *pmr_desc; 143 #endif 144 ibt_mr_desc_t *mr_desc; /* vaddr, lkey, rkey */ 145 146 } rib_bufpool_t; 147 148 /* 149 * ATS relsted defines and structures. 150 */ 151 #define ATS_AR_DATA_LEN 16 152 #define IBD_NAME "ibd" 153 #define N_IBD_INSTANCES 4 154 155 typedef struct rpcib_ats_s { 156 int ras_inst; 157 ib_pkey_t ras_pkey; 158 ib_gid_t ras_port_gid; 159 sa_family_t ras_inet_type; 160 union { 161 struct sockaddr_in ras_sockaddr; 162 struct sockaddr_in6 ras_sockaddr6; 163 } ra_sin; 164 #define ras_sin ra_sin.ras_sockaddr 165 #define ras_sin6 ra_sin.ras_sockaddr6
282 */ 283 rib_service_t *service_list; 284 krwlock_t service_list_lock; 285 286 rib_service_t *ats_list; /* Service list for ATS */ 287 288 rib_conn_list_t cl_conn_list; /* client conn list */ 289 rib_conn_list_t srv_conn_list; /* server conn list */ 290 291 rib_cq_t *clnt_scq; 292 rib_cq_t *clnt_rcq; 293 rib_cq_t *svc_scq; 294 rib_cq_t *svc_rcq; 295 kmutex_t cb_lock; 296 kcondvar_t cb_cv; 297 298 rib_bufpool_t *recv_pool; /* recv buf pool */ 299 rib_bufpool_t *send_pool; /* send buf pool */ 300 301 void *iblock; /* interrupt cookie */ 302 303 #ifdef IB_FMR_SUP 304 ibt_fmr_pool_hdl_t fmr_pool; 305 #endif /* IB_FMR_SUP */ 306 307 #ifdef SERVER_REG_CACHE 308 kmem_cache_t *server_side_cache; /* long reply pool */ 309 avl_tree_t avl_tree; 310 kmutex_t avl_lock; 311 krwlock_t avl_rw_lock; 312 volatile bool_t avl_init; 313 #endif 314 315 }; 316 317 318 /* 319 * Structure on wait state of a post send 320 */ 321 struct send_wid { 322 #if defined (CLNT_INTERRUPT_COAL) 323 struct send_wid *forw; 324 struct send_wid *back; 325 #endif 326 uint32_t xid; 327 int cv_sig; 328 kmutex_t sendwait_lock; 329 kcondvar_t wait_cv; 330 uint_t status; 331 rib_qp_t *qp; 332 int nsbufs; /* # of send buffers posted */ 333 uint64_t sbufaddr[DSEG_MAX]; /* posted send buffers */ 334 caddr_t c; 335 caddr_t c1; 336 int l1; 337 caddr_t c2; 338 int l2; 339 int wl,rl; 340 }; 341 342 /* 343 * Structure on reply descriptor for recv queue. 344 * Different from the above posting of a descriptor. 345 */ 346 struct reply { 347 uint32_t xid; 348 uint_t status; 349 uint64_t vaddr_cq; /* buf addr from CQ */ 350 uint_t bytes_xfer; 351 kcondvar_t wait_cv; 352 struct reply *next; 353 struct reply *prev; 354 }; 355 356 struct svc_recv { 357 rib_qp_t *qp; 358 uint64_t vaddr; 359 uint_t bytes_xfer;
388 kcondvar_t posted_rbufs_cv; 389 kmutex_t posted_rbufs_lock; 390 391 /* 392 * RPC reply 393 */ 394 uint_t rep_list_size; 395 struct reply *replylist; 396 kmutex_t replylist_lock; 397 398 /* 399 * server only, RDMA_DONE 400 */ 401 struct rdma_done_list *rdlist; 402 kmutex_t rdlist_lock; 403 404 kmutex_t cb_lock; 405 kcondvar_t cb_conn_cv; 406 407 caddr_t q; /* upstream queue */ 408 struct send_wid wd; 409 }; 410 411 #define ctoqp(conn) ((rib_qp_t *)((conn)->c_private)) 412 #define qptoc(rqp) ((CONN *)&((rqp)->rdmaconn)) 413 414 /* 415 * Timeout for various calls 416 */ 417 #define CONN_WAIT_TIME 40 418 #define SEND_WAIT_TIME 40 /* time for send completion */ 419 420 #define REPLY_WAIT_TIME 40 /* time to get reply from remote QP */ 421 422 #ifdef __cplusplus 423 } 424 #endif 425 426 #endif /* !_IB_H */