[lustre-devel] [PATCH 15/34] lnet: extend zombie handling to nets and nis
NeilBrown
neilb at suse.com
Thu Sep 6 17:49:31 PDT 2018
A zombie lnet_ni is now attached to the lnet_net rather than the
global the_lnet. The zombie lnet_net are attached to the_lnet.
For some reason, we don't drop the refcount on the lnd before shutting
it down now.
This is part of
8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
LU-7734 lnet: Multi-Rail local NI split
Signed-off-by: NeilBrown <neilb at suse.com>
---
.../staging/lustre/include/linux/lnet/lib-types.h | 9 ++-
drivers/staging/lustre/lnet/lnet/api-ni.c | 65 ++++++++++----------
drivers/staging/lustre/lnet/lnet/config.c | 3 +
3 files changed, 42 insertions(+), 35 deletions(-)
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 22957d142cc0..1d372672e2de 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -284,6 +284,9 @@ struct lnet_net {
struct lnet_lnd *net_lnd;
/* list of NIs on this net */
struct list_head net_ni_list;
+
+ /* dying LND instances */
+ struct list_head net_ni_zombie;
};
struct lnet_ni {
@@ -653,11 +656,11 @@ struct lnet {
/* LND instances */
struct list_head ln_nets;
/* NIs bond on specific CPT(s) */
- struct list_head ln_nis_cpt;
- /* dying LND instances */
- struct list_head ln_nis_zombie;
+ struct list_head ln_nis_cpt;
/* the loopback NI */
struct lnet_ni *ln_loni;
+ /* network zombie list */
+ struct list_head ln_net_zombie;
/* remote networks with routes to them */
struct list_head *ln_remote_nets_hash;
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index c3c568e63342..18d111cb826b 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -539,7 +539,6 @@ lnet_prepare(lnet_pid_t requested_pid)
INIT_LIST_HEAD(&the_lnet.ln_test_peers);
INIT_LIST_HEAD(&the_lnet.ln_nets);
INIT_LIST_HEAD(&the_lnet.ln_nis_cpt);
- INIT_LIST_HEAD(&the_lnet.ln_nis_zombie);
INIT_LIST_HEAD(&the_lnet.ln_routers);
INIT_LIST_HEAD(&the_lnet.ln_drop_rules);
INIT_LIST_HEAD(&the_lnet.ln_delay_rules);
@@ -618,7 +617,6 @@ lnet_unprepare(void)
LASSERT(list_empty(&the_lnet.ln_test_peers));
LASSERT(list_empty(&the_lnet.ln_nets));
LASSERT(list_empty(&the_lnet.ln_nis_cpt));
- LASSERT(list_empty(&the_lnet.ln_nis_zombie));
lnet_portals_destroy();
@@ -1095,34 +1093,35 @@ lnet_ni_unlink_locked(struct lnet_ni *ni)
/* move it to zombie list and nobody can find it anymore */
LASSERT(!list_empty(&ni->ni_netlist));
- list_move(&ni->ni_netlist, &the_lnet.ln_nis_zombie);
+ list_move(&ni->ni_netlist, &ni->ni_net->net_ni_zombie);
lnet_ni_decref_locked(ni, 0);
}
static void
-lnet_clear_zombies_nis_locked(void)
+lnet_clear_zombies_nis_locked(struct lnet_net *net)
{
int i;
int islo;
struct lnet_ni *ni;
+ struct list_head *zombie_list = &net->net_ni_zombie;
/*
- * Now wait for the NI's I just nuked to show up on ln_zombie_nis
- * and shut them down in guaranteed thread context
+ * Now wait for the NIs I just nuked to show up on the zombie
+ * list and shut them down in guaranteed thread context
*/
i = 2;
- while (!list_empty(&the_lnet.ln_nis_zombie)) {
+ while (!list_empty(zombie_list)) {
int *ref;
int j;
- ni = list_entry(the_lnet.ln_nis_zombie.next,
+ ni = list_entry(zombie_list->next,
struct lnet_ni, ni_netlist);
list_del_init(&ni->ni_netlist);
cfs_percpt_for_each(ref, j, ni->ni_refs) {
if (!*ref)
continue;
/* still busy, add it back to zombie list */
- list_add(&ni->ni_netlist, &the_lnet.ln_nis_zombie);
+ list_add(&ni->ni_netlist, zombie_list);
break;
}
@@ -1138,18 +1137,13 @@ lnet_clear_zombies_nis_locked(void)
continue;
}
- ni->ni_net->net_lnd->lnd_refcount--;
lnet_net_unlock(LNET_LOCK_EX);
islo = ni->ni_net->net_lnd->lnd_type == LOLND;
LASSERT(!in_interrupt());
- ni->ni_net->net_lnd->lnd_shutdown(ni);
+ net->net_lnd->lnd_shutdown(ni);
- /*
- * can't deref lnd anymore now; it might have unregistered
- * itself...
- */
if (!islo)
CDEBUG(D_LNI, "Removed LNI %s\n",
libcfs_nid2str(ni->ni_nid));
@@ -1162,9 +1156,11 @@ lnet_clear_zombies_nis_locked(void)
}
static void
-lnet_shutdown_lndnis(void)
+lnet_shutdown_lndnet(struct lnet_net *net);
+
+static void
+lnet_shutdown_lndnets(void)
{
- struct lnet_ni *ni;
int i;
struct lnet_net *net;
@@ -1173,30 +1169,35 @@ lnet_shutdown_lndnis(void)
/* All quiet on the API front */
LASSERT(!the_lnet.ln_shutdown);
LASSERT(!the_lnet.ln_refcount);
- LASSERT(list_empty(&the_lnet.ln_nis_zombie));
lnet_net_lock(LNET_LOCK_EX);
the_lnet.ln_shutdown = 1; /* flag shutdown */
- /* Unlink NIs from the global table */
while (!list_empty(&the_lnet.ln_nets)) {
+ /*
+ * move the nets to the zombie list to avoid them being
+ * picked up for new work. LONET is also included in the
+ * Nets that will be moved to the zombie list
+ */
net = list_entry(the_lnet.ln_nets.next,
struct lnet_net, net_list);
- while (!list_empty(&net->net_ni_list)) {
- ni = list_entry(net->net_ni_list.next,
- struct lnet_ni, ni_netlist);
- lnet_ni_unlink_locked(ni);
- }
+ list_move(&net->net_list, &the_lnet.ln_net_zombie);
}
- /* Drop the cached loopback NI. */
+ /* Drop the cached loopback Net. */
if (the_lnet.ln_loni) {
lnet_ni_decref_locked(the_lnet.ln_loni, 0);
the_lnet.ln_loni = NULL;
}
-
lnet_net_unlock(LNET_LOCK_EX);
+ /* iterate through the net zombie list and delete each net */
+ while (!list_empty(&the_lnet.ln_net_zombie)) {
+ net = list_entry(the_lnet.ln_net_zombie.next,
+ struct lnet_net, net_list);
+ lnet_shutdown_lndnet(net);
+ }
+
/*
* Clear lazy portals and drop delayed messages which hold refs
* on their lnet_msg::msg_rxpeer
@@ -1211,8 +1212,6 @@ lnet_shutdown_lndnis(void)
lnet_peer_tables_cleanup(NULL);
lnet_net_lock(LNET_LOCK_EX);
-
- lnet_clear_zombies_nis_locked();
the_lnet.ln_shutdown = 0;
lnet_net_unlock(LNET_LOCK_EX);
}
@@ -1222,6 +1221,7 @@ static void
lnet_shutdown_lndni(struct lnet_ni *ni)
{
int i;
+ struct lnet_net *net = ni->ni_net;
lnet_net_lock(LNET_LOCK_EX);
lnet_ni_unlink_locked(ni);
@@ -1235,7 +1235,7 @@ lnet_shutdown_lndni(struct lnet_ni *ni)
lnet_peer_tables_cleanup(ni);
lnet_net_lock(LNET_LOCK_EX);
- lnet_clear_zombies_nis_locked();
+ lnet_clear_zombies_nis_locked(net);
lnet_net_unlock(LNET_LOCK_EX);
}
@@ -1445,7 +1445,7 @@ lnet_startup_lndnets(struct list_head *netlist)
return ni_count;
failed:
- lnet_shutdown_lndnis();
+ lnet_shutdown_lndnets();
return rc;
}
@@ -1492,6 +1492,7 @@ int lnet_lib_init(void)
the_lnet.ln_refcount = 0;
LNetInvalidateEQHandle(&the_lnet.ln_rc_eqh);
INIT_LIST_HEAD(&the_lnet.ln_lnds);
+ INIT_LIST_HEAD(&the_lnet.ln_net_zombie);
INIT_LIST_HEAD(&the_lnet.ln_rcd_zombie);
INIT_LIST_HEAD(&the_lnet.ln_rcd_deathrow);
@@ -1656,7 +1657,7 @@ LNetNIInit(lnet_pid_t requested_pid)
if (!the_lnet.ln_nis_from_mod_params)
lnet_destroy_routes();
err_shutdown_lndnis:
- lnet_shutdown_lndnis();
+ lnet_shutdown_lndnets();
err_empty_list:
lnet_unprepare();
LASSERT(rc < 0);
@@ -1703,7 +1704,7 @@ LNetNIFini(void)
lnet_acceptor_stop();
lnet_destroy_routes();
- lnet_shutdown_lndnis();
+ lnet_shutdown_lndnets();
lnet_unprepare();
}
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index 380a3fb1caba..2588d67fea1b 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -279,6 +279,8 @@ lnet_net_free(struct lnet_net *net)
struct list_head *tmp, *tmp2;
struct lnet_ni *ni;
+ LASSERT(list_empty(&net->net_ni_zombie));
+
/* delete any nis which have been started. */
list_for_each_safe(tmp, tmp2, &net->net_ni_list) {
ni = list_entry(tmp, struct lnet_ni, ni_netlist);
@@ -312,6 +314,7 @@ lnet_net_alloc(__u32 net_id, struct list_head *net_list)
INIT_LIST_HEAD(&net->net_list);
INIT_LIST_HEAD(&net->net_ni_list);
+ INIT_LIST_HEAD(&net->net_ni_zombie);
net->net_id = net_id;
More information about the lustre-devel
mailing list