[lustre-devel] [PATCH 25/39] lnet: o2iblnd: retry qp creation with reduced queue depth
James Simmons
jsimmons at infradead.org
Thu Jan 21 09:16:48 PST 2021
From: Serguei Smirnov <ssmirnov at whamcloud.com>
If negotiated number of frags * queue depth is too large for
successful qp creation, reduce the queue depth in a loop
until qp creation succeeds or the queue depth dips below 2.
Remember the reduced queue depth value to use for later
connections to the same peer.
WC-bug-id: https://jira.whamcloud.com/browse/LU-12901
Lustre-commit: 8a3ef5713cc4ae ("LU-12901 o2iblnd: retry qp creation with reduced queue depth")
Signed-off-by: Serguei Smirnov <ssmirnov at whamcloud.com>
Reviewed-on: https://review.whamcloud.com/40748
Reviewed-by: Amir Shehata <ashehata at whamcloud.com>
Reviewed-by: Cyril Bordage <cbordage at whamcloud.com>
Reviewed-by: Chris Horn <chris.horn at hpe.com>
Reviewed-by: Oleg Drokin <green at whamcloud.com>
Signed-off-by: James Simmons <jsimmons at infradead.org>
---
net/lnet/klnds/o2iblnd/o2iblnd.c | 33 ++++++++++++++++++++++++++-------
net/lnet/klnds/o2iblnd/o2iblnd.h | 2 ++
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/net/lnet/klnds/o2iblnd/o2iblnd.c b/net/lnet/klnds/o2iblnd/o2iblnd.c
index 9c65524..fc515fc 100644
--- a/net/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/net/lnet/klnds/o2iblnd/o2iblnd.c
@@ -336,6 +336,7 @@ int kiblnd_create_peer(struct lnet_ni *ni, struct kib_peer_ni **peerp,
peer_ni->ibp_last_alive = 0;
peer_ni->ibp_max_frags = IBLND_MAX_RDMA_FRAGS;
peer_ni->ibp_queue_depth = ni->ni_net->net_tunables.lct_peer_tx_credits;
+ peer_ni->ibp_queue_depth_mod = 0; /* try to use the default */
atomic_set(&peer_ni->ibp_refcount, 1); /* 1 ref for caller */
INIT_LIST_HEAD(&peer_ni->ibp_list);
@@ -795,13 +796,28 @@ struct kib_conn *kiblnd_create_conn(struct kib_peer_ni *peer_ni,
init_qp_attr.qp_type = IB_QPT_RC;
init_qp_attr.send_cq = cq;
init_qp_attr.recv_cq = cq;
- /* kiblnd_send_wrs() can change the connection's queue depth if
- * the maximum work requests for the device is maxed out
- */
- init_qp_attr.cap.max_send_wr = kiblnd_send_wrs(conn);
- init_qp_attr.cap.max_recv_wr = IBLND_RECV_WRS(conn);
- rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, &init_qp_attr);
+ if (peer_ni->ibp_queue_depth_mod &&
+ peer_ni->ibp_queue_depth_mod < peer_ni->ibp_queue_depth) {
+ conn->ibc_queue_depth = peer_ni->ibp_queue_depth_mod;
+ CDEBUG(D_NET, "Use reduced queue depth %u (from %u)\n",
+ peer_ni->ibp_queue_depth_mod,
+ peer_ni->ibp_queue_depth);
+ }
+
+ do {
+ /* kiblnd_send_wrs() can change the connection's queue depth if
+ * the maximum work requests for the device is maxed out
+ */
+ init_qp_attr.cap.max_send_wr = kiblnd_send_wrs(conn);
+ init_qp_attr.cap.max_recv_wr = IBLND_RECV_WRS(conn);
+ rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd,
+ &init_qp_attr);
+ if (rc != -ENOMEM || conn->ibc_queue_depth < 2)
+ break;
+ conn->ibc_queue_depth--;
+ } while (rc);
+
if (rc) {
CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d, send_sge: %d, recv_sge: %d\n",
rc, init_qp_attr.cap.max_send_wr,
@@ -813,11 +829,14 @@ struct kib_conn *kiblnd_create_conn(struct kib_peer_ni *peer_ni,
conn->ibc_sched = sched;
- if (conn->ibc_queue_depth != peer_ni->ibp_queue_depth)
+ if (!peer_ni->ibp_queue_depth_mod &&
+ conn->ibc_queue_depth != peer_ni->ibp_queue_depth) {
CWARN("peer %s - queue depth reduced from %u to %u to allow for qp creation\n",
libcfs_nid2str(peer_ni->ibp_nid),
peer_ni->ibp_queue_depth,
conn->ibc_queue_depth);
+ peer_ni->ibp_queue_depth_mod = conn->ibc_queue_depth;
+ }
conn->ibc_rxs = kzalloc_cpt(IBLND_RX_MSGS(conn) *
sizeof(*conn->ibc_rxs),
diff --git a/net/lnet/klnds/o2iblnd/o2iblnd.h b/net/lnet/klnds/o2iblnd/o2iblnd.h
index 1fc68e1..424ca07 100644
--- a/net/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/net/lnet/klnds/o2iblnd/o2iblnd.h
@@ -638,6 +638,8 @@ struct kib_peer_ni {
u16 ibp_max_frags;
/* max_peer_credits */
u16 ibp_queue_depth;
+ /* reduced value which allows conn to be created if max fails */
+ u16 ibp_queue_depth_mod;
};
extern struct kib_data kiblnd_data;
--
1.8.3.1
More information about the lustre-devel
mailing list