[lustre-devel] [PATCH 09/34] lnet: add list of cpts to lnet_net.

[Doug Oucharek]

I agree with a comment from James Simmons: __u32 should only be used when the variable is being shared with user space.  We need to start converting all uses of __uXX in LNet to just uXX.  Perhaps that should be a set of future patches once all of MR/DD has landed?

Reviewed-by: Doug Oucharek <dougso at me.com<mailto:dougso at me.com>>

Doug

On Sep 6, 2018, at 5:49 PM, NeilBrown <neilb at suse.com<mailto:neilb at suse.com>> wrote:

struct lnet_net now has a list of cpts, which is the union
of the cpts for each lnet_ni.

This is part of
   8cbb8cd3e771e7f7e0f99cafc19fad32770dc015
      LU-7734 lnet: Multi-Rail local NI split

Signed-off-by: NeilBrown <neilb at suse.com<mailto:neilb at suse.com>>
---
.../staging/lustre/include/linux/lnet/lib-types.h  |    6 +
drivers/staging/lustre/lnet/lnet/config.c          |  164 ++++++++++++++++++++
2 files changed, 170 insertions(+)

diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 2d2c066a11ba..22957d142cc0 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -266,6 +266,12 @@ struct lnet_net {
* lnet/include/lnet/nidstr.h */
__u32 net_id;

+ /* total number of CPTs in the array */
+ __u32 net_ncpts;
+
+ /* cumulative CPTs of all NIs in this net */
+ __u32 *net_cpts;
+
/* network tunables */
struct lnet_ioctl_config_lnd_cmn_tunables net_tunables;

diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index e83bdbec11e3..380a3fb1caba 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -91,11 +91,169 @@ lnet_net_unique(__u32 net, struct list_head *netlist)
return true;
}

+static bool
+in_array(__u32 *array, __u32 size, __u32 value)
+{
+ int i;
+
+ for (i = 0; i < size; i++) {
+ if (array[i] == value)
+ return false;
+ }
+
+ return true;
+}
+
+static int
+lnet_net_append_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+ __u32 *added_cpts = NULL;
+ int i, j = 0, rc = 0;
+
+ /*
+ * no need to go futher since a subset of the NIs already exist on
+ * all CPTs
+ */
+ if (net->net_ncpts == LNET_CPT_NUMBER)
+ return 0;
+
+ if (cpts == NULL) {
+ /* there is an NI which will exist on all CPTs */
+ if (net->net_cpts != NULL)
+ kvfree(net->net_cpts);
+ net->net_cpts = NULL;
+ net->net_ncpts = LNET_CPT_NUMBER;
+ return 0;
+ }
+
+ if (net->net_cpts == NULL) {
+ net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),
+      GFP_KERNEL);
+ if (net->net_cpts == NULL)
+ return -ENOMEM;
+ memcpy(net->net_cpts, cpts, ncpts);
+ return 0;
+ }
+
+ added_cpts = kmalloc_array(LNET_CPT_NUMBER, sizeof(*added_cpts),
+   GFP_KERNEL);
+ if (added_cpts == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < ncpts; i++) {
+ if (!in_array(net->net_cpts, net->net_ncpts, cpts[i])) {
+ added_cpts[j] = cpts[i];
+ j++;
+ }
+ }
+
+ /* append the new cpts if any to the list of cpts in the net */
+ if (j > 0) {
+ __u32 *array = NULL, *loc;
+ __u32 total_entries = j + net->net_ncpts;
+
+ array = kmalloc_array(total_entries, sizeof(*net->net_cpts),
+      GFP_KERNEL);
+ if (array == NULL) {
+ rc = -ENOMEM;
+ goto failed;
+ }
+
+ memcpy(array, net->net_cpts,
+       net->net_ncpts * sizeof(*net->net_cpts));
+ loc = array + net->net_ncpts;
+ memcpy(loc, added_cpts, j * sizeof(*net->net_cpts));
+
+ kfree(net->net_cpts);
+ net->net_ncpts = total_entries;
+ net->net_cpts = array;
+ }
+
+failed:
+ kfree(added_cpts);
+
+ return rc;
+}
+
+static void
+lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net)
+{
+ struct lnet_ni *ni;
+ int rc;
+
+ /*
+ * Operation Assumption:
+ * This function is called after an NI has been removed from
+ * its parent net.
+ *
+ * if we're removing an NI which exists on all CPTs then
+ * we have to check if any of the other NIs on this net also
+ * exists on all CPTs. If none, then we need to build our Net CPT
+ * list based on the remaining NIs.
+ *
+ * If the NI being removed exist on a subset of the CPTs then we
+ * alo rebuild the Net CPT list based on the remaining NIs, which
+ * should resutl in the expected Net CPT list.
+ */
+
+ /*
+ * sometimes this function can be called due to some failure
+ * creating an NI, before any of the cpts are allocated, so check
+ * for that case and don't do anything
+ */
+ if (ncpts == 0)
+ return;
+
+ if (ncpts == LNET_CPT_NUMBER) {
+ /*
+ * first iteration through the NI list in the net to see
+ * if any of the NIs exist on all the CPTs. If one is
+ * found then our job is done.
+ */
+ list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+ if (ni->ni_ncpts == LNET_CPT_NUMBER)
+ return;
+ }
+ }
+
+ /*
+ * Rebuild the Net CPT list again, thereby only including only the
+ * CPTs which the remaining NIs are associated with.
+ */
+ if (net->net_cpts != NULL) {
+ kfree(net->net_cpts);
+ net->net_cpts = NULL;
+ }
+
+ list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
+ rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts,
+  net);
+ if (rc != 0) {
+ CERROR("Out of Memory\n");
+ /*
+ * do our best to keep on going. Delete
+ * the net cpts and set it to NULL. This
+ * way we can keep on going but less
+ * efficiently, since memory accesses might be
+ * accross CPT lines.
+ */
+ if (net->net_cpts != NULL) {
+ kfree(net->net_cpts);
+ net->net_cpts = NULL;
+ net->net_ncpts = LNET_CPT_NUMBER;
+ }
+ return;
+ }
+ }
+}
+
void
lnet_ni_free(struct lnet_ni *ni)
{
int i;

+ lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net);
+
if (ni->ni_refs)
cfs_percpt_free(ni->ni_refs);

@@ -128,6 +286,9 @@ lnet_net_free(struct lnet_net *net)
lnet_ni_free(ni);
}

+ if (net->net_cpts != NULL)
+ kfree(net->net_cpts);
+
kfree(net);
}

@@ -229,6 +390,9 @@ lnet_ni_alloc(struct lnet_net *net, struct cfs_expr_list *el, char *iface)
ni->ni_net_ns = NULL;

ni->ni_last_alive = ktime_get_real_seconds();
+ rc = lnet_net_append_cpts(ni->ni_cpts, ni->ni_ncpts, net);
+ if (rc != 0)
+ goto failed;
list_add_tail(&ni->ni_netlist, &net->net_ni_list);

return ni;



-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.lustre.org/pipermail/lustre-devel-lustre.org/attachments/20180910/741a25c0/attachment-0001.html>