[lustre-devel] [PATCH 16/24] lustre: lnet: add discovery thread
James Simmons
jsimmons at infradead.org
Sun Oct 14 15:51:53 PDT 2018
> From: Olaf Weber <olaf at sgi.com>
>
> Add the discovery thread, which will be used to handle peer
> discovery. This change adds the thread and the infrastructure
> that starts and stops it. The thread itself does trivial work.
>
> Peer Discovery gets its own event queue (ln_dc_eqh), a queue
> for peers that are to be discovered (ln_dc_request), a queue
> for peers waiting for an event (ln_dc_working), a wait queue
> head so the thread can sleep (ln_dc_waitq), and start/stop
> state (ln_dc_state).
>
> Peer discovery is started from lnet_select_pathway(), for
> GET and PUT messages not sent to the LNET_RESERVED_PORTAL.
> This criterion means that discovery will not be triggered by
> the messages used in discovery, and neither will an LNet ping
> trigger it.
Reviewed-by: James Simmons <jsimmons at infradead.org>
> WC-bug-id: https://jira.whamcloud.com/browse/LU-9480
> Signed-off-by: Olaf Weber <olaf at sgi.com>
> Signed-off-by: Amir Shehata <amir.shehata at intel.com>
> Reviewed-on: https://review.whamcloud.com/25786
> Reviewed-by: Olaf Weber <olaf.weber at hpe.com>
> Signed-off-by: NeilBrown <neilb at suse.com>
> ---
> .../staging/lustre/include/linux/lnet/lib-lnet.h | 6
> .../staging/lustre/include/linux/lnet/lib-types.h | 71 ++++
> drivers/staging/lustre/lnet/lnet/api-ni.c | 31 ++
> drivers/staging/lustre/lnet/lnet/lib-move.c | 45 ++-
> drivers/staging/lustre/lnet/lnet/peer.c | 325 ++++++++++++++++++++
> 5 files changed, 468 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
> index aad25eb0011b..848d622911a4 100644
> --- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
> +++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
> @@ -438,6 +438,7 @@ bool lnet_is_ni_healthy_locked(struct lnet_ni *ni);
> struct lnet_net *lnet_get_net_locked(u32 net_id);
>
> extern unsigned int lnet_numa_range;
> +extern unsigned int lnet_peer_discovery_disabled;
> extern int portal_rotor;
>
> int lnet_lib_init(void);
> @@ -704,6 +705,9 @@ struct lnet_peer_ni *lnet_nid2peerni_ex(lnet_nid_t nid, int cpt);
> struct lnet_peer_ni *lnet_find_peer_ni_locked(lnet_nid_t nid);
> void lnet_peer_net_added(struct lnet_net *net);
> lnet_nid_t lnet_peer_primary_nid_locked(lnet_nid_t nid);
> +int lnet_discover_peer_locked(struct lnet_peer_ni *lpni, int cpt);
> +int lnet_peer_discovery_start(void);
> +void lnet_peer_discovery_stop(void);
> void lnet_peer_tables_cleanup(struct lnet_net *net);
> void lnet_peer_uninit(void);
> int lnet_peer_tables_create(void);
> @@ -791,4 +795,6 @@ lnet_peer_ni_is_primary(struct lnet_peer_ni *lpni)
> return lpni->lpni_nid == lpni->lpni_peer_net->lpn_peer->lp_primary_nid;
> }
>
> +bool lnet_peer_is_uptodate(struct lnet_peer *lp);
> +
> #endif
> diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
> index 260619e19bde..6394a3af50b7 100644
> --- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
> +++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
> @@ -520,10 +520,61 @@ struct lnet_peer {
>
> /* peer state flags */
> unsigned int lp_state;
> +
> + /* link on discovery-related lists */
> + struct list_head lp_dc_list;
> +
> + /* tasks waiting on discovery of this peer */
> + wait_queue_head_t lp_dc_waitq;
> };
>
> -#define LNET_PEER_MULTI_RAIL BIT(0)
> -#define LNET_PEER_CONFIGURED BIT(1)
> +/*
> + * The status flags in lp_state. Their semantics have chosen so that
> + * lp_state can be zero-initialized.
> + *
> + * A peer is marked MULTI_RAIL in two cases: it was configured using DLC
> + * as multi-rail aware, or the LNET_PING_FEAT_MULTI_RAIL bit was set.
> + *
> + * A peer is marked NO_DISCOVERY if the LNET_PING_FEAT_DISCOVERY bit was
> + * NOT set when the peer was pinged by discovery.
> + */
> +#define LNET_PEER_MULTI_RAIL BIT(0) /* Multi-rail aware */
> +#define LNET_PEER_NO_DISCOVERY BIT(1) /* Peer disabled discovery */
> +/*
> + * A peer is marked CONFIGURED if it was configured by DLC.
> + *
> + * In addition, a peer is marked DISCOVERED if it has fully passed
> + * through Peer Discovery.
> + *
> + * When Peer Discovery is disabled, the discovery thread will mark
> + * peers REDISCOVER to indicate that they should be re-examined if
> + * discovery is (re)enabled on the node.
> + *
> + * A peer that was created as the result of inbound traffic will not
> + * be marked at all.
> + */
> +#define LNET_PEER_CONFIGURED BIT(2) /* Configured via DLC */
> +#define LNET_PEER_DISCOVERED BIT(3) /* Peer was discovered */
> +#define LNET_PEER_REDISCOVER BIT(4) /* Discovery was disabled */
> +/*
> + * A peer is marked DISCOVERING when discovery is in progress.
> + * The other flags below correspond to stages of discovery.
> + */
> +#define LNET_PEER_DISCOVERING BIT(5) /* Discovering */
> +#define LNET_PEER_DATA_PRESENT BIT(6) /* Remote peer data present */
> +#define LNET_PEER_NIDS_UPTODATE BIT(7) /* Remote peer info uptodate */
> +#define LNET_PEER_PING_SENT BIT(8) /* Waiting for REPLY to Ping */
> +#define LNET_PEER_PUSH_SENT BIT(9) /* Waiting for ACK of Push */
> +#define LNET_PEER_PING_FAILED BIT(10) /* Ping send failure */
> +#define LNET_PEER_PUSH_FAILED BIT(11) /* Push send failure */
> +/*
> + * A ping can be forced as a way to fix up state, or as a manual
> + * intervention by an admin.
> + * A push can be forced in circumstances that would normally not
> + * allow for one to happen.
> + */
> +#define LNET_PEER_FORCE_PING BIT(12) /* Forced Ping */
> +#define LNET_PEER_FORCE_PUSH BIT(13) /* Forced Push */
>
> struct lnet_peer_net {
> /* chain on lp_peer_nets */
> @@ -775,6 +826,11 @@ struct lnet_msg_container {
> void **msc_finalizers;
> };
>
> +/* Peer Discovery states */
> +#define LNET_DC_STATE_SHUTDOWN 0 /* not started */
> +#define LNET_DC_STATE_RUNNING 1 /* started up OK */
> +#define LNET_DC_STATE_STOPPING 2 /* telling thread to stop */
> +
> /* Router Checker states */
> enum lnet_rc_state {
> LNET_RC_STATE_SHUTDOWN, /* not started */
> @@ -856,6 +912,17 @@ struct lnet {
> struct lnet_ping_buffer *ln_ping_target;
> atomic_t ln_ping_target_seqno;
>
> + /* discovery event queue handle */
> + struct lnet_handle_eq ln_dc_eqh;
> + /* discovery requests */
> + struct list_head ln_dc_request;
> + /* discovery working list */
> + struct list_head ln_dc_working;
> + /* discovery thread wait queue */
> + wait_queue_head_t ln_dc_waitq;
> + /* discovery startup/shutdown state */
> + int ln_dc_state;
> +
> /* router checker startup/shutdown state */
> enum lnet_rc_state ln_rc_state;
> /* router checker's event queue */
> diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
> index c48bcb8722a0..dccfd5bcc459 100644
> --- a/drivers/staging/lustre/lnet/lnet/api-ni.c
> +++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
> @@ -78,6 +78,13 @@ module_param_call(lnet_interfaces_max, intf_max_set, param_get_int,
> MODULE_PARM_DESC(lnet_interfaces_max,
> "Maximum number of interfaces in a node.");
>
> +unsigned int lnet_peer_discovery_disabled;
> +static int discovery_set(const char *val, const struct kernel_param *kp);
> +module_param_call(lnet_peer_discovery_disabled, discovery_set, param_get_int,
> + &lnet_peer_discovery_disabled, 0644);
> +MODULE_PARM_DESC(lnet_peer_discovery_disabled,
> + "Set to 1 to disable peer discovery on this node.");
> +
> /*
> * This sequence number keeps track of how many times DLC was used to
> * update the local NIs. It is incremented when a NI is added or
> @@ -90,6 +97,23 @@ static atomic_t lnet_dlc_seq_no = ATOMIC_INIT(0);
> static int lnet_ping(struct lnet_process_id id, signed long timeout,
> struct lnet_process_id __user *ids, int n_ids);
>
> +static int
> +discovery_set(const char *val, const struct kernel_param *kp)
> +{
> + int rc;
> + unsigned long value;
> +
> + rc = kstrtoul(val, 0, &value);
> + if (rc) {
> + CERROR("Invalid module parameter value for 'lnet_peer_discovery_disabled'\n");
> + return rc;
> + }
> +
> + *(unsigned int *)kp->arg = !!value;
> +
> + return 0;
> +}
> +
> static int
> intf_max_set(const char *val, const struct kernel_param *kp)
> {
> @@ -1921,6 +1945,10 @@ LNetNIInit(lnet_pid_t requested_pid)
> if (rc)
> goto err_stop_ping;
>
> + rc = lnet_peer_discovery_start();
> + if (rc != 0)
> + goto err_stop_router_checker;
> +
> lnet_fault_init();
> lnet_router_debugfs_init();
>
> @@ -1928,6 +1956,8 @@ LNetNIInit(lnet_pid_t requested_pid)
>
> return 0;
>
> +err_stop_router_checker:
> + lnet_router_checker_stop();
> err_stop_ping:
> lnet_ping_target_fini();
> err_acceptor_stop:
> @@ -1976,6 +2006,7 @@ LNetNIFini(void)
>
> lnet_fault_fini();
> lnet_router_debugfs_fini();
> + lnet_peer_discovery_stop();
> lnet_router_checker_stop();
> lnet_ping_target_fini();
>
> diff --git a/drivers/staging/lustre/lnet/lnet/lib-move.c b/drivers/staging/lustre/lnet/lnet/lib-move.c
> index 4c1eef907dc7..4773180cc7b3 100644
> --- a/drivers/staging/lustre/lnet/lnet/lib-move.c
> +++ b/drivers/staging/lustre/lnet/lnet/lib-move.c
> @@ -1208,6 +1208,27 @@ lnet_get_best_ni(struct lnet_net *local_net, struct lnet_ni *cur_ni,
> return best_ni;
> }
>
> +/*
> + * Traffic to the LNET_RESERVED_PORTAL may not trigger peer discovery,
> + * because such traffic is required to perform discovery. We therefore
> + * exclude all GET and PUT on that portal. We also exclude all ACK and
> + * REPLY traffic, but that is because the portal is not tracked in the
> + * message structure for these message types. We could restrict this
> + * further by also checking for LNET_PROTO_PING_MATCHBITS.
> + */
> +static bool
> +lnet_msg_discovery(struct lnet_msg *msg)
> +{
> + if (msg->msg_type == LNET_MSG_PUT) {
> + if (msg->msg_hdr.msg.put.ptl_index != LNET_RESERVED_PORTAL)
> + return true;
> + } else if (msg->msg_type == LNET_MSG_GET) {
> + if (msg->msg_hdr.msg.get.ptl_index != LNET_RESERVED_PORTAL)
> + return true;
> + }
> + return false;
> +}
> +
> static int
> lnet_select_pathway(lnet_nid_t src_nid, lnet_nid_t dst_nid,
> struct lnet_msg *msg, lnet_nid_t rtr_nid)
> @@ -1220,7 +1241,6 @@ lnet_select_pathway(lnet_nid_t src_nid, lnet_nid_t dst_nid,
> struct lnet_peer *peer;
> struct lnet_peer_net *peer_net;
> struct lnet_net *local_net;
> - __u32 seq;
> int cpt, cpt2, rc;
> bool routing;
> bool routing2;
> @@ -1255,13 +1275,6 @@ lnet_select_pathway(lnet_nid_t src_nid, lnet_nid_t dst_nid,
> routing2 = false;
> local_found = false;
>
> - seq = lnet_get_dlc_seq_locked();
> -
> - if (the_lnet.ln_state != LNET_STATE_RUNNING) {
> - lnet_net_unlock(cpt);
> - return -ESHUTDOWN;
> - }
> -
> /*
> * lnet_nid2peerni_locked() is the path that will find an
> * existing peer_ni, or create one and mark it as having been
> @@ -1272,7 +1285,22 @@ lnet_select_pathway(lnet_nid_t src_nid, lnet_nid_t dst_nid,
> lnet_net_unlock(cpt);
> return PTR_ERR(lpni);
> }
> + /*
> + * Now that we have a peer_ni, check if we want to discover
> + * the peer. Traffic to the LNET_RESERVED_PORTAL should not
> + * trigger discovery.
> + */
> peer = lpni->lpni_peer_net->lpn_peer;
> + if (lnet_msg_discovery(msg) && !lnet_peer_is_uptodate(peer)) {
> + rc = lnet_discover_peer_locked(lpni, cpt);
> + if (rc) {
> + lnet_peer_ni_decref_locked(lpni);
> + lnet_net_unlock(cpt);
> + return rc;
> + }
> + /* The peer may have changed. */
> + peer = lpni->lpni_peer_net->lpn_peer;
> + }
> lnet_peer_ni_decref_locked(lpni);
>
> /* If peer is not healthy then can not send anything to it */
> @@ -1701,6 +1729,7 @@ lnet_select_pathway(lnet_nid_t src_nid, lnet_nid_t dst_nid,
> */
> cpt2 = lnet_cpt_of_nid_locked(best_lpni->lpni_nid, best_ni);
> if (cpt != cpt2) {
> + __u32 seq = lnet_get_dlc_seq_locked();
> lnet_net_unlock(cpt);
> cpt = cpt2;
> lnet_net_lock(cpt);
> diff --git a/drivers/staging/lustre/lnet/lnet/peer.c b/drivers/staging/lustre/lnet/lnet/peer.c
> index d7a0a2f3bdd9..038b58414ce0 100644
> --- a/drivers/staging/lustre/lnet/lnet/peer.c
> +++ b/drivers/staging/lustre/lnet/lnet/peer.c
> @@ -201,6 +201,8 @@ lnet_peer_alloc(lnet_nid_t nid)
>
> INIT_LIST_HEAD(&lp->lp_peer_list);
> INIT_LIST_HEAD(&lp->lp_peer_nets);
> + INIT_LIST_HEAD(&lp->lp_dc_list);
> + init_waitqueue_head(&lp->lp_dc_waitq);
> spin_lock_init(&lp->lp_lock);
> lp->lp_primary_nid = nid;
> lp->lp_cpt = lnet_nid_cpt_hash(nid, LNET_CPT_NUMBER);
> @@ -1457,6 +1459,10 @@ lnet_nid2peerni_ex(lnet_nid_t nid, int cpt)
> return lpni;
> }
>
> +/*
> + * Get a peer_ni for the given nid, create it if necessary. Takes a
> + * hold on the peer_ni.
> + */
> struct lnet_peer_ni *
> lnet_nid2peerni_locked(lnet_nid_t nid, lnet_nid_t pref, int cpt)
> {
> @@ -1510,9 +1516,326 @@ lnet_nid2peerni_locked(lnet_nid_t nid, lnet_nid_t pref, int cpt)
> mutex_unlock(&the_lnet.ln_api_mutex);
> lnet_net_lock(cpt);
>
> + /* Lock has been dropped, check again for shutdown. */
> + if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) {
> + if (!IS_ERR(lpni))
> + lnet_peer_ni_decref_locked(lpni);
> + lpni = ERR_PTR(-ESHUTDOWN);
> + }
> +
> return lpni;
> }
>
> +/*
> + * Peer Discovery
> + */
> +
> +/*
> + * Is a peer uptodate from the point of view of discovery?
> + *
> + * If it is currently being processed, obviously not.
> + * A forced Ping or Push is also handled by the discovery thread.
> + *
> + * Otherwise look at whether the peer needs rediscovering.
> + */
> +bool
> +lnet_peer_is_uptodate(struct lnet_peer *lp)
> +{
> + bool rc;
> +
> + spin_lock(&lp->lp_lock);
> + if (lp->lp_state & (LNET_PEER_DISCOVERING |
> + LNET_PEER_FORCE_PING |
> + LNET_PEER_FORCE_PUSH)) {
> + rc = false;
> + } else if (lp->lp_state & LNET_PEER_REDISCOVER) {
> + if (lnet_peer_discovery_disabled)
> + rc = true;
> + else
> + rc = false;
> + } else if (lp->lp_state & LNET_PEER_DISCOVERED) {
> + if (lp->lp_state & LNET_PEER_NIDS_UPTODATE)
> + rc = true;
> + else
> + rc = false;
> + } else {
> + rc = false;
> + }
> + spin_unlock(&lp->lp_lock);
> +
> + return rc;
> +}
> +
> +/*
> + * Queue a peer for the attention of the discovery thread. Call with
> + * lnet_net_lock/EX held. Returns 0 if the peer was queued, and
> + * -EALREADY if the peer was already queued.
> + */
> +static int lnet_peer_queue_for_discovery(struct lnet_peer *lp)
> +{
> + int rc;
> +
> + spin_lock(&lp->lp_lock);
> + if (!(lp->lp_state & LNET_PEER_DISCOVERING))
> + lp->lp_state |= LNET_PEER_DISCOVERING;
> + spin_unlock(&lp->lp_lock);
> + if (list_empty(&lp->lp_dc_list)) {
> + lnet_peer_addref_locked(lp);
> + list_add_tail(&lp->lp_dc_list, &the_lnet.ln_dc_request);
> + wake_up(&the_lnet.ln_dc_waitq);
> + rc = 0;
> + } else {
> + rc = -EALREADY;
> + }
> +
> + return rc;
> +}
> +
> +/*
> + * Discovery of a peer is complete. Wake all waiters on the peer.
> + * Call with lnet_net_lock/EX held.
> + */
> +static void lnet_peer_discovery_complete(struct lnet_peer *lp)
> +{
> + list_del_init(&lp->lp_dc_list);
> + wake_up_all(&lp->lp_dc_waitq);
> + lnet_peer_decref_locked(lp);
> +}
> +
> +/*
> + * Peer discovery slow path. The ln_api_mutex is held on entry, and
> + * dropped/retaken within this function. An lnet_peer_ni is passed in
> + * because discovery could tear down an lnet_peer.
> + */
> +int
> +lnet_discover_peer_locked(struct lnet_peer_ni *lpni, int cpt)
> +{
> + DEFINE_WAIT(wait);
> + struct lnet_peer *lp;
> + int rc = 0;
> +
> +again:
> + lnet_net_unlock(cpt);
> + lnet_net_lock(LNET_LOCK_EX);
> +
> + /* We're willing to be interrupted. */
> + for (;;) {
> + lp = lpni->lpni_peer_net->lpn_peer;
> + prepare_to_wait(&lp->lp_dc_waitq, &wait, TASK_INTERRUPTIBLE);
> + if (signal_pending(current))
> + break;
> + if (the_lnet.ln_dc_state != LNET_DC_STATE_RUNNING)
> + break;
> + if (lnet_peer_is_uptodate(lp))
> + break;
> + lnet_peer_queue_for_discovery(lp);
> + lnet_peer_addref_locked(lp);
> + lnet_net_unlock(LNET_LOCK_EX);
> + schedule();
> + finish_wait(&lp->lp_dc_waitq, &wait);
> + lnet_net_lock(LNET_LOCK_EX);
> + lnet_peer_decref_locked(lp);
> + /* Do not use lp beyond this point. */
> + }
> + finish_wait(&lp->lp_dc_waitq, &wait);
> +
> + lnet_net_unlock(LNET_LOCK_EX);
> + lnet_net_lock(cpt);
> +
> + if (signal_pending(current))
> + rc = -EINTR;
> + else if (the_lnet.ln_dc_state != LNET_DC_STATE_RUNNING)
> + rc = -ESHUTDOWN;
> + else if (!lnet_peer_is_uptodate(lp))
> + goto again;
> +
> + return rc;
> +}
> +
> +/*
> + * Event handler for the discovery EQ.
> + *
> + * Called with lnet_res_lock(cpt) held. The cpt is the
> + * lnet_cpt_of_cookie() of the md handle cookie.
> + */
> +static void lnet_discovery_event_handler(struct lnet_event *event)
> +{
> + wake_up(&the_lnet.ln_dc_waitq);
> +}
> +
> +/*
> + * Wait for work to be queued or some other change that must be
> + * attended to. Returns non-zero if the discovery thread should shut
> + * down.
> + */
> +static int lnet_peer_discovery_wait_for_work(void)
> +{
> + int cpt;
> + int rc = 0;
> +
> + DEFINE_WAIT(wait);
> +
> + cpt = lnet_net_lock_current();
> + for (;;) {
> + prepare_to_wait(&the_lnet.ln_dc_waitq, &wait,
> + TASK_IDLE);
> + if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING)
> + break;
> + if (!list_empty(&the_lnet.ln_dc_request))
> + break;
> + lnet_net_unlock(cpt);
> + schedule();
> + finish_wait(&the_lnet.ln_dc_waitq, &wait);
> + cpt = lnet_net_lock_current();
> + }
> + finish_wait(&the_lnet.ln_dc_waitq, &wait);
> +
> + if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING)
> + rc = -ESHUTDOWN;
> +
> + lnet_net_unlock(cpt);
> +
> + CDEBUG(D_NET, "woken: %d\n", rc);
> +
> + return rc;
> +}
> +
> +/* The discovery thread. */
> +static int lnet_peer_discovery(void *arg)
> +{
> + struct lnet_peer *lp;
> +
> + CDEBUG(D_NET, "started\n");
> +
> + for (;;) {
> + if (lnet_peer_discovery_wait_for_work())
> + break;
> +
> + lnet_net_lock(LNET_LOCK_EX);
> + if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING)
> + break;
> + while (!list_empty(&the_lnet.ln_dc_request)) {
> + lp = list_first_entry(&the_lnet.ln_dc_request,
> + struct lnet_peer, lp_dc_list);
> + list_move(&lp->lp_dc_list, &the_lnet.ln_dc_working);
> + lnet_net_unlock(LNET_LOCK_EX);
> +
> + /* Just tag and release for now. */
> + spin_lock(&lp->lp_lock);
> + if (lnet_peer_discovery_disabled) {
> + lp->lp_state |= LNET_PEER_REDISCOVER;
> + lp->lp_state &= ~(LNET_PEER_DISCOVERED |
> + LNET_PEER_NIDS_UPTODATE |
> + LNET_PEER_DISCOVERING);
> + } else {
> + lp->lp_state |= (LNET_PEER_DISCOVERED |
> + LNET_PEER_NIDS_UPTODATE);
> + lp->lp_state &= ~(LNET_PEER_REDISCOVER |
> + LNET_PEER_DISCOVERING);
> + }
> + spin_unlock(&lp->lp_lock);
> +
> + lnet_net_lock(LNET_LOCK_EX);
> + if (!(lp->lp_state & LNET_PEER_DISCOVERING))
> + lnet_peer_discovery_complete(lp);
> + if (the_lnet.ln_dc_state == LNET_DC_STATE_STOPPING)
> + break;
> + }
> + lnet_net_unlock(LNET_LOCK_EX);
> + }
> +
> + CDEBUG(D_NET, "stopping\n");
> + /*
> + * Clean up before telling lnet_peer_discovery_stop() that
> + * we're done. Use wake_up() below to somewhat reduce the
> + * size of the thundering herd if there are multiple threads
> + * waiting on discovery of a single peer.
> + */
> + LNetEQFree(the_lnet.ln_dc_eqh);
> + LNetInvalidateEQHandle(&the_lnet.ln_dc_eqh);
> +
> + lnet_net_lock(LNET_LOCK_EX);
> + list_for_each_entry(lp, &the_lnet.ln_dc_request, lp_dc_list) {
> + spin_lock(&lp->lp_lock);
> + lp->lp_state |= LNET_PEER_REDISCOVER;
> + lp->lp_state &= ~(LNET_PEER_DISCOVERED |
> + LNET_PEER_DISCOVERING |
> + LNET_PEER_NIDS_UPTODATE);
> + spin_unlock(&lp->lp_lock);
> + lnet_peer_discovery_complete(lp);
> + }
> + list_for_each_entry(lp, &the_lnet.ln_dc_working, lp_dc_list) {
> + spin_lock(&lp->lp_lock);
> + lp->lp_state |= LNET_PEER_REDISCOVER;
> + lp->lp_state &= ~(LNET_PEER_DISCOVERED |
> + LNET_PEER_DISCOVERING |
> + LNET_PEER_NIDS_UPTODATE);
> + spin_unlock(&lp->lp_lock);
> + lnet_peer_discovery_complete(lp);
> + }
> + lnet_net_unlock(LNET_LOCK_EX);
> +
> + the_lnet.ln_dc_state = LNET_DC_STATE_SHUTDOWN;
> + wake_up(&the_lnet.ln_dc_waitq);
> +
> + CDEBUG(D_NET, "stopped\n");
> +
> + return 0;
> +}
> +
> +/* ln_api_mutex is held on entry. */
> +int lnet_peer_discovery_start(void)
> +{
> + struct task_struct *task;
> + int rc;
> +
> + if (the_lnet.ln_dc_state != LNET_DC_STATE_SHUTDOWN)
> + return -EALREADY;
> +
> + INIT_LIST_HEAD(&the_lnet.ln_dc_request);
> + INIT_LIST_HEAD(&the_lnet.ln_dc_working);
> + init_waitqueue_head(&the_lnet.ln_dc_waitq);
> +
> + rc = LNetEQAlloc(0, lnet_discovery_event_handler, &the_lnet.ln_dc_eqh);
> + if (rc != 0) {
> + CERROR("Can't allocate discovery EQ: %d\n", rc);
> + return rc;
> + }
> +
> + the_lnet.ln_dc_state = LNET_DC_STATE_RUNNING;
> + task = kthread_run(lnet_peer_discovery, NULL, "lnet_discovery");
> + if (IS_ERR(task)) {
> + rc = PTR_ERR(task);
> + CERROR("Can't start peer discovery thread: %d\n", rc);
> +
> + LNetEQFree(the_lnet.ln_dc_eqh);
> + LNetInvalidateEQHandle(&the_lnet.ln_dc_eqh);
> +
> + the_lnet.ln_dc_state = LNET_DC_STATE_SHUTDOWN;
> + }
> +
> + return rc;
> +}
> +
> +/* ln_api_mutex is held on entry. */
> +void lnet_peer_discovery_stop(void)
> +{
> + if (the_lnet.ln_dc_state == LNET_DC_STATE_SHUTDOWN)
> + return;
> +
> + LASSERT(the_lnet.ln_dc_state == LNET_DC_STATE_RUNNING);
> + the_lnet.ln_dc_state = LNET_DC_STATE_STOPPING;
> + wake_up(&the_lnet.ln_dc_waitq);
> +
> + wait_event(the_lnet.ln_dc_waitq,
> + the_lnet.ln_dc_state == LNET_DC_STATE_SHUTDOWN);
> +
> + LASSERT(list_empty(&the_lnet.ln_dc_request));
> + LASSERT(list_empty(&the_lnet.ln_dc_working));
> +}
> +
> +/* Debugging */
> +
> void
> lnet_debug_peer(lnet_nid_t nid)
> {
> @@ -1544,6 +1867,8 @@ lnet_debug_peer(lnet_nid_t nid)
> lnet_net_unlock(cpt);
> }
>
> +/* Gathering information for userspace. */
> +
> int
> lnet_get_peer_ni_info(__u32 peer_index, __u64 *nid,
> char aliveness[LNET_MAX_STR_LEN],
>
>
>
More information about the lustre-devel
mailing list