[lustre-devel] [PATCH 14/32] lnet: Ensure round robin across nets
James Simmons
jsimmons at infradead.org
Wed Aug 3 18:37:59 PDT 2022
From: Chris Horn <chris.horn at hpe.com>
Introduce a global net sequence number and a peer sequence number.
These sequence numbers are used to ensure round robin selection of
local NIs and peer NIs across nets.
Also consolidate the sequence number accounting under
lnet_handle_send(). Previously the sequence number increment for
the final destination peer net/peer NI on a routed send was done
in lnet_handle_find_routed_path().
Some cleanup that is also in this patch:
- Redundant check of null src_nid is removed from
lnet_handle_find_routed_path() (LNET_NID_IS_ANY handles null arg)
- Avoid comparing best_lpn with itself in
lnet_handle_find_routed_path() on the first loop iteration
- In lnet_find_best_ni_on_local_net() check whether we have
a specified lp_disc_net_id outside of the loop to avoid doing
that work on each loop iteration.
Added some debug statements to print information used when selecting
peer net/local net.
HPE-bug-id: LUS-10871
WC-bug-id: https://jira.whamcloud.com/browse/LU-15713
Lustre-commit: 05413b3d84f7d1feb ("LU-15713 lnet: Ensure round robin across nets")
Signed-off-by: Chris Horn <chris.horn at hpe.com>
Reviewed-on: https://review.whamcloud.com/46976
Reviewed-by: Serguei Smirnov <ssmirnov at whamcloud.com>
Reviewed-by: Cyril Bordage <cbordage at whamcloud.com>
Reviewed-by: Oleg Drokin <green at whamcloud.com>
Signed-off-by: James Simmons <jsimmons at infradead.org>
---
include/linux/lnet/lib-types.h | 11 ++++-
net/lnet/lnet/lib-move.c | 96 +++++++++++++++++++++++++++---------------
2 files changed, 72 insertions(+), 35 deletions(-)
diff --git a/include/linux/lnet/lib-types.h b/include/linux/lnet/lib-types.h
index 1827f4e..09b9d8e 100644
--- a/include/linux/lnet/lib-types.h
+++ b/include/linux/lnet/lib-types.h
@@ -765,6 +765,11 @@ struct lnet_peer {
/* cached peer aliveness */
bool lp_alive;
+
+ /* sequence number used to round robin traffic to this peer's
+ * nets/NIs
+ */
+ u32 lp_send_seq;
};
/*
@@ -1205,10 +1210,12 @@ struct lnet {
/* LND instances */
struct list_head ln_nets;
- /* network zombie list */
- struct list_head ln_net_zombie;
+ /* Sequence number used to round robin sends across all nets */
+ u32 ln_net_seq;
/* the loopback NI */
struct lnet_ni *ln_loni;
+ /* network zombie list */
+ struct list_head ln_net_zombie;
/* resend messages list */
struct list_head ln_msg_resend;
/* spin lock to protect the msg resend list */
diff --git a/net/lnet/lnet/lib-move.c b/net/lnet/lnet/lib-move.c
index a514472..6ad0963 100644
--- a/net/lnet/lnet/lib-move.c
+++ b/net/lnet/lnet/lib-move.c
@@ -1658,9 +1658,12 @@ void lnet_usr_translate_stats(struct lnet_ioctl_element_msg_stats *msg_stats,
* local ni and local net so that we pick the next ones
* in Round Robin.
*/
- best_lpni->lpni_peer_net->lpn_seq++;
+ best_lpni->lpni_peer_net->lpn_peer->lp_send_seq++;
+ best_lpni->lpni_peer_net->lpn_seq =
+ best_lpni->lpni_peer_net->lpn_peer->lp_send_seq;
best_lpni->lpni_seq = best_lpni->lpni_peer_net->lpn_seq;
- best_ni->ni_net->net_seq++;
+ the_lnet.ln_net_seq++;
+ best_ni->ni_net->net_seq = the_lnet.ln_net_seq;
best_ni->ni_seq = best_ni->ni_net->net_seq;
CDEBUG(D_NET,
@@ -1743,6 +1746,11 @@ void lnet_usr_translate_stats(struct lnet_ioctl_element_msg_stats *msg_stats,
* lnet_select_pathway() function and is never changed.
* It's safe to use it here.
*/
+ final_dst_lpni->lpni_peer_net->lpn_peer->lp_send_seq++;
+ final_dst_lpni->lpni_peer_net->lpn_seq =
+ final_dst_lpni->lpni_peer_net->lpn_peer->lp_send_seq;
+ final_dst_lpni->lpni_seq =
+ final_dst_lpni->lpni_peer_net->lpn_seq;
msg->msg_hdr.dest_nid = final_dst_lpni->lpni_nid;
} else {
/* if we're not routing set the dest_nid to the best peer
@@ -1968,8 +1976,10 @@ struct lnet_ni *
int best_lpn_healthv = 0;
u32 best_lpn_sel_prio = LNET_MAX_SELECTION_PRIORITY;
- CDEBUG(D_NET, "using src nid %s for route restriction\n",
- src_nid ? libcfs_nidstr(src_nid) : "ANY");
+ CDEBUG(D_NET, "%s route (%s) from local NI %s to destination %s\n",
+ LNET_NID_IS_ANY(&sd->sd_rtr_nid) ? "Lookup" : "Specified",
+ libcfs_nidstr(&sd->sd_rtr_nid), libcfs_nidstr(src_nid),
+ libcfs_nidstr(&sd->sd_dst_nid));
/* If a router nid was specified then we are replying to a GET or
* sending an ACK. In this case we use the gateway associated with the
@@ -1989,8 +1999,7 @@ struct lnet_ni *
}
if (!route_found) {
- if (sd->sd_msg->msg_routing ||
- (src_nid && !LNET_NID_IS_ANY(src_nid))) {
+ if (sd->sd_msg->msg_routing || !LNET_NID_IS_ANY(src_nid)) {
/* If I'm routing this message then I need to find the
* next hop based on the destination NID
*
@@ -2006,6 +2015,8 @@ struct lnet_ni *
libcfs_nidstr(&sd->sd_dst_nid));
return -EHOSTUNREACH;
}
+ CDEBUG(D_NET, "best_rnet %s\n",
+ libcfs_net2str(best_rnet->lrn_net));
} else {
/* we've already looked up the initial lpni using
* dst_nid
@@ -2023,10 +2034,18 @@ struct lnet_ni *
if (!rnet)
continue;
- if (!best_lpn) {
- best_lpn = lpn;
- best_rnet = rnet;
- }
+ if (!best_lpn)
+ goto use_lpn;
+ else
+ CDEBUG(D_NET, "n[%s, %s] h[%d, %d], p[%u, %u], s[%d, %d]\n",
+ libcfs_net2str(lpn->lpn_net_id),
+ libcfs_net2str(best_lpn->lpn_net_id),
+ lpn->lpn_healthv,
+ best_lpn->lpn_healthv,
+ lpn->lpn_sel_priority,
+ best_lpn->lpn_sel_priority,
+ lpn->lpn_seq,
+ best_lpn->lpn_seq);
/* select the preferred peer net */
if (best_lpn_healthv > lpn->lpn_healthv)
@@ -2054,6 +2073,9 @@ struct lnet_ni *
return -EHOSTUNREACH;
}
+ CDEBUG(D_NET, "selected best_lpn %s\n",
+ libcfs_net2str(best_lpn->lpn_net_id));
+
sd->sd_best_lpni = lnet_find_best_lpni(sd->sd_best_ni,
lnet_nid_to_nid4(&sd->sd_dst_nid),
lp,
@@ -2068,12 +2090,6 @@ struct lnet_ni *
* NI's so update the final destination we selected
*/
sd->sd_final_dst_lpni = sd->sd_best_lpni;
-
- /* Increment the sequence number of the remote lpni so
- * we can round robin over the different interfaces of
- * the remote lpni
- */
- sd->sd_best_lpni->lpni_seq++;
}
/* find the best route. Restrict the selection on the net of the
@@ -2139,14 +2155,12 @@ struct lnet_ni *
*gw_lpni = gwni;
*gw_peer = gw;
- /* increment the sequence numbers since now we're sure we're
- * going to use this path
+ /* increment the sequence number since now we're sure we're
+ * going to use this route
*/
if (LNET_NID_IS_ANY(&sd->sd_rtr_nid)) {
LASSERT(best_route && last_route);
best_route->lr_seq = last_route->lr_seq + 1;
- if (best_lpn)
- best_lpn->lpn_seq++;
}
return 0;
@@ -2220,7 +2234,15 @@ struct lnet_ni *
u32 lpn_sel_prio;
u32 best_net_sel_prio = LNET_MAX_SELECTION_PRIORITY;
u32 net_sel_prio;
- bool exit = false;
+
+ /* if this is a discovery message and lp_disc_net_id is
+ * specified then use that net to send the discovery on.
+ */
+ if (discovery && peer->lp_disc_net_id) {
+ best_lpn = lnet_peer_get_net_locked(peer, peer->lp_disc_net_id);
+ if (best_lpn && lnet_get_net_locked(best_lpn->lpn_net_id))
+ goto select_best_ni;
+ }
/* The peer can have multiple interfaces, some of them can be on
* the local network and others on a routed network. We should
@@ -2241,17 +2263,25 @@ struct lnet_ni *
net_healthv = lnet_get_net_healthv_locked(net);
net_sel_prio = net->net_sel_priority;
- /* if this is a discovery message and lp_disc_net_id is
- * specified then use that net to send the discovery on.
- */
- if (peer->lp_disc_net_id == lpn->lpn_net_id &&
- discovery) {
- exit = true;
- goto select_lpn;
- }
-
if (!best_lpn)
goto select_lpn;
+ else
+ CDEBUG(D_NET,
+ "n[%s, %s] ph[%d, %d], pp[%u, %u], nh[%d, %d], np[%u, %u], ps[%u, %u], ns[%u, %u]\n",
+ libcfs_net2str(lpn->lpn_net_id),
+ libcfs_net2str(best_lpn->lpn_net_id),
+ lpn->lpn_healthv,
+ best_lpn_healthv,
+ lpn_sel_prio,
+ best_lpn_sel_prio,
+ net_healthv,
+ best_net_healthv,
+ net_sel_prio,
+ best_net_sel_prio,
+ lpn->lpn_seq,
+ best_lpn->lpn_seq,
+ net->net_seq,
+ best_net->net_seq);
/* always select the lpn with the best health */
if (best_lpn_healthv > lpn->lpn_healthv)
@@ -2291,15 +2321,15 @@ struct lnet_ni *
best_lpn_sel_prio = lpn_sel_prio;
best_lpn = lpn;
best_net = net;
-
- if (exit)
- break;
}
if (best_lpn) {
/* Select the best NI on the same net as best_lpn chosen
* above
*/
+select_best_ni:
+ CDEBUG(D_NET, "selected best_lpn %s\n",
+ libcfs_net2str(best_lpn->lpn_net_id));
best_ni = lnet_find_best_ni_on_spec_net(NULL, peer, best_lpn,
msg, md_cpt);
}
--
1.8.3.1
More information about the lustre-devel
mailing list