xprtrdma: Chain Send to FastReg WRs
With FRWR, the client transport can perform memory registration and post a Send with just a single ib_post_send. This reduces contention between the send_request path and the Send Completion handlers, and reduces the overhead of registering a chunk that has multiple segments. Signed-off-by: Chuck Lever <chuck.lever@oracle.com> Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
This commit is contained in:
Chuck Lever
Anna Schumaker
parent
fb14ae8853
commit
f287762308
|
|
@ -251,6 +251,16 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
|
|||
return ERR_PTR(-EIO);
|
||||
}
|
||||
|
||||
/* Post Send WR containing the RPC Call message.
|
||||
*/
|
||||
static int
|
||||
fmr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
|
||||
{
|
||||
struct ib_send_wr *bad_wr;
|
||||
|
||||
return ib_post_send(ia->ri_id->qp, &req->rl_sendctx->sc_wr, &bad_wr);
|
||||
}
|
||||
|
||||
/* Invalidate all memory regions that were registered for "req".
|
||||
*
|
||||
* Sleeps until it is safe for the host CPU to access the
|
||||
|
|
@ -305,6 +315,7 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
|
|||
|
||||
const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {
|
||||
.ro_map = fmr_op_map,
|
||||
.ro_send = fmr_op_send,
|
||||
.ro_unmap_sync = fmr_op_unmap_sync,
|
||||
.ro_recover_mr = fmr_op_recover_mr,
|
||||
.ro_open = fmr_op_open,
|
||||
|
|
|
|||
|
|
@ -357,8 +357,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
|
|||
struct rpcrdma_mr *mr;
|
||||
struct ib_mr *ibmr;
|
||||
struct ib_reg_wr *reg_wr;
|
||||
struct ib_send_wr *bad_wr;
|
||||
int rc, i, n;
|
||||
int i, n;
|
||||
u8 key;
|
||||
|
||||
mr = NULL;
|
||||
|
|
@ -407,22 +406,12 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
|
|||
ib_update_fast_reg_key(ibmr, ++key);
|
||||
|
||||
reg_wr = &frwr->fr_regwr;
|
||||
reg_wr->wr.next = NULL;
|
||||
reg_wr->wr.opcode = IB_WR_REG_MR;
|
||||
frwr->fr_cqe.done = frwr_wc_fastreg;
|
||||
reg_wr->wr.wr_cqe = &frwr->fr_cqe;
|
||||
reg_wr->wr.num_sge = 0;
|
||||
reg_wr->wr.send_flags = 0;
|
||||
reg_wr->mr = ibmr;
|
||||
reg_wr->key = ibmr->rkey;
|
||||
reg_wr->access = writing ?
|
||||
IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
|
||||
IB_ACCESS_REMOTE_READ;
|
||||
|
||||
rc = ib_post_send(ia->ri_id->qp, ®_wr->wr, &bad_wr);
|
||||
if (rc)
|
||||
goto out_senderr;
|
||||
|
||||
mr->mr_handle = ibmr->rkey;
|
||||
mr->mr_length = ibmr->length;
|
||||
mr->mr_offset = ibmr->iova;
|
||||
|
|
@ -442,11 +431,40 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
|
|||
frwr->fr_mr, n, mr->mr_nents);
|
||||
rpcrdma_mr_defer_recovery(mr);
|
||||
return ERR_PTR(-EIO);
|
||||
}
|
||||
|
||||
out_senderr:
|
||||
pr_err("rpcrdma: FRWR registration ib_post_send returned %i\n", rc);
|
||||
rpcrdma_mr_defer_recovery(mr);
|
||||
return ERR_PTR(-ENOTCONN);
|
||||
/* Post Send WR containing the RPC Call message.
|
||||
*
|
||||
* For FRMR, chain any FastReg WRs to the Send WR. Only a
|
||||
* single ib_post_send call is needed to register memory
|
||||
* and then post the Send WR.
|
||||
*/
|
||||
static int
|
||||
frwr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
|
||||
{
|
||||
struct ib_send_wr *post_wr, *bad_wr;
|
||||
struct rpcrdma_mr *mr;
|
||||
|
||||
post_wr = &req->rl_sendctx->sc_wr;
|
||||
list_for_each_entry(mr, &req->rl_registered, mr_list) {
|
||||
struct rpcrdma_frwr *frwr;
|
||||
|
||||
frwr = &mr->frwr;
|
||||
|
||||
frwr->fr_cqe.done = frwr_wc_fastreg;
|
||||
frwr->fr_regwr.wr.next = post_wr;
|
||||
frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;
|
||||
frwr->fr_regwr.wr.num_sge = 0;
|
||||
frwr->fr_regwr.wr.opcode = IB_WR_REG_MR;
|
||||
frwr->fr_regwr.wr.send_flags = 0;
|
||||
|
||||
post_wr = &frwr->fr_regwr.wr;
|
||||
}
|
||||
|
||||
/* If ib_post_send fails, the next ->send_request for
|
||||
* @req will queue these MWs for recovery.
|
||||
*/
|
||||
return ib_post_send(ia->ri_id->qp, post_wr, &bad_wr);
|
||||
}
|
||||
|
||||
/* Handle a remotely invalidated mr on the @mrs list
|
||||
|
|
@ -561,6 +579,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
|
|||
|
||||
const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {
|
||||
.ro_map = frwr_op_map,
|
||||
.ro_send = frwr_op_send,
|
||||
.ro_reminv = frwr_op_reminv,
|
||||
.ro_unmap_sync = frwr_op_unmap_sync,
|
||||
.ro_recover_mr = frwr_op_recover_mr,
|
||||
|
|
|
|||
|
|
@ -1535,7 +1535,6 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
|
|||
struct rpcrdma_req *req)
|
||||
{
|
||||
struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;
|
||||
struct ib_send_wr *send_wr_fail;
|
||||
int rc;
|
||||
|
||||
if (req->rl_reply) {
|
||||
|
|
@ -1554,7 +1553,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
|
|||
--ep->rep_send_count;
|
||||
}
|
||||
|
||||
rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail);
|
||||
rc = ia->ri_ops->ro_send(ia, req);
|
||||
trace_xprtrdma_post_send(req, rc);
|
||||
if (rc)
|
||||
return -ENOTCONN;
|
||||
|
|
|
|||
|
|
@ -472,6 +472,8 @@ struct rpcrdma_memreg_ops {
|
|||
(*ro_map)(struct rpcrdma_xprt *,
|
||||
struct rpcrdma_mr_seg *, int, bool,
|
||||
struct rpcrdma_mr **);
|
||||
int (*ro_send)(struct rpcrdma_ia *ia,
|
||||
struct rpcrdma_req *req);
|
||||
void (*ro_reminv)(struct rpcrdma_rep *rep,
|
||||
struct list_head *mrs);
|
||||
void (*ro_unmap_sync)(struct rpcrdma_xprt *,
|
||||
|
|
|
|||
Loading…
Reference in New Issue