[lustre-devel] [PATCH 10/24] lnet: enhance connect/accept to support large addr
James Simmons
jsimmons at infradead.org
Tue Sep 21 19:19:47 PDT 2021
From: Mr NeilBrown <neilb at suse.de>
This patch introduces a version-2 of the acceptor protocol. This
version uses a 'struct lnet_nid' rather than 'lnet_nid_t'
lnet_connect() now accepts a struct lnet_nid and uses version 2 if
necessary. lnet_accept() accepts either v1 or v2.
WC-bug-id: https://jira.whamcloud.com/browse/LU-10391
Lustre-commit: 9b0738c53c962f426 ("LU-10391 lnet: enhance connect/accept to support large addr")
Signed-off-by: Mr NeilBrown <neilb at suse.de>
Reviewed-on: https://review.whamcloud.com/42105
Reviewed-by: James Simmons <jsimmons at infradead.org>
Reviewed-by: Chris Horn <chris.horn at hpe.com>
Reviewed-by: Serguei Smirnov <ssmirnov at whamcloud.com>
Reviewed-by: Oleg Drokin <green at whamcloud.com>
Signed-off-by: James Simmons <jsimmons at infradead.org>
---
include/linux/lnet/lib-lnet.h | 5 +-
include/uapi/linux/lnet/lnet-idl.h | 11 +++-
net/lnet/klnds/socklnd/socklnd_cb.c | 8 +--
net/lnet/lnet/acceptor.c | 111 +++++++++++++++++++++++-------------
net/lnet/lnet/api-ni.c | 9 +++
5 files changed, 97 insertions(+), 47 deletions(-)
diff --git a/include/linux/lnet/lib-lnet.h b/include/linux/lnet/lib-lnet.h
index 3842976..9e7d0b8 100644
--- a/include/linux/lnet/lib-lnet.h
+++ b/include/linux/lnet/lib-lnet.h
@@ -489,6 +489,7 @@ unsigned int lnet_nid_cpt_hash(struct lnet_nid *nid,
struct lnet_ni *lnet_nid2ni_addref(lnet_nid_t nid);
struct lnet_ni *lnet_net2ni_locked(u32 net, int cpt);
struct lnet_ni *lnet_net2ni_addref(u32 net);
+struct lnet_ni *lnet_nid_to_ni_addref(struct lnet_nid *nid);
struct lnet_net *lnet_get_net_locked(u32 net_id);
extern unsigned int lnet_response_tracking;
@@ -742,9 +743,9 @@ void lnet_copy_kiov2iter(struct iov_iter *to,
void lnet_register_lnd(struct lnet_lnd *lnd);
void lnet_unregister_lnd(struct lnet_lnd *lnd);
-struct socket *lnet_connect(lnet_nid_t peer_nid, int interface,
+struct socket *lnet_connect(struct lnet_nid *peer_nid, int interface,
struct sockaddr *peeraddr, struct net *ns);
-void lnet_connect_console_error(int rc, lnet_nid_t peer_nid,
+void lnet_connect_console_error(int rc, struct lnet_nid *peer_nid,
struct sockaddr *sa);
int lnet_count_acceptor_nets(void);
int lnet_acceptor_timeout(void);
diff --git a/include/uapi/linux/lnet/lnet-idl.h b/include/uapi/linux/lnet/lnet-idl.h
index 3fc0df1..b14723e 100644
--- a/include/uapi/linux/lnet/lnet-idl.h
+++ b/include/uapi/linux/lnet/lnet-idl.h
@@ -191,13 +191,22 @@ struct lnet_magicversion {
/* Acceptor connection request */
struct lnet_acceptor_connreq {
- __u32 acr_magic; /* PTL_ACCEPTOR_PROTO_MAGIC */
+ __u32 acr_magic; /* LNET_PROTO_ACCEPTOR_MAGIC */
__u32 acr_version; /* protocol version */
__u64 acr_nid; /* target NID */
} __attribute__((packed));
#define LNET_PROTO_ACCEPTOR_VERSION 1
+struct lnet_acceptor_connreq_v2 {
+ __u32 acr_magic; /* LNET_PROTO_ACCEPTOR_MAGIC */
+ __u32 acr_version; /* protocol version - 2 */
+ struct lnet_nid acr_nid; /* target NID */
+} __attribute__((packed));
+
+/* For use with 16-byte addresses */
+#define LNET_PROTO_ACCEPTOR_VERSION_16 2
+
struct lnet_counters_common {
__u32 lcc_msgs_alloc;
__u32 lcc_msgs_max;
diff --git a/net/lnet/klnds/socklnd/socklnd_cb.c b/net/lnet/klnds/socklnd/socklnd_cb.c
index a2298275..edc584a 100644
--- a/net/lnet/klnds/socklnd/socklnd_cb.c
+++ b/net/lnet/klnds/socklnd/socklnd_cb.c
@@ -1833,13 +1833,12 @@ void ksocknal_write_callback(struct ksock_conn *conn)
if (ktime_get_seconds() >= deadline) {
rc = -ETIMEDOUT;
- lnet_connect_console_error(rc,
- lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+ lnet_connect_console_error(rc, &peer_ni->ksnp_id.nid,
(struct sockaddr *)&conn_cb->ksnr_addr);
goto failed;
}
- sock = lnet_connect(lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+ sock = lnet_connect(&peer_ni->ksnp_id.nid,
conn_cb->ksnr_myiface,
(struct sockaddr *)&conn_cb->ksnr_addr,
peer_ni->ksnp_ni->ni_net_ns);
@@ -1851,8 +1850,7 @@ void ksocknal_write_callback(struct ksock_conn *conn)
rc = ksocknal_create_conn(peer_ni->ksnp_ni, conn_cb, sock,
type);
if (rc < 0) {
- lnet_connect_console_error(rc,
- lnet_nid_to_nid4(&peer_ni->ksnp_id.nid),
+ lnet_connect_console_error(rc, &peer_ni->ksnp_id.nid,
(struct sockaddr *)&conn_cb->ksnr_addr);
goto failed;
}
diff --git a/net/lnet/lnet/acceptor.c b/net/lnet/lnet/acceptor.c
index 243c34f..2306760 100644
--- a/net/lnet/lnet/acceptor.c
+++ b/net/lnet/lnet/acceptor.c
@@ -85,54 +85,58 @@
EXPORT_SYMBOL(lnet_acceptor_timeout);
void
-lnet_connect_console_error(int rc, lnet_nid_t peer_nid,
+lnet_connect_console_error(int rc, struct lnet_nid *peer_nid,
struct sockaddr *sa)
{
switch (rc) {
/* "normal" errors */
case -ECONNREFUSED:
CNETERR("Connection to %s at host %pISp was refused: check that Lustre is running on that node.\n",
- libcfs_nid2str(peer_nid), sa);
+ libcfs_nidstr(peer_nid), sa);
break;
case -EHOSTUNREACH:
case -ENETUNREACH:
CNETERR("Connection to %s at host %pIS was unreachable: the network or that node may be down, or Lustre may be misconfigured.\n",
- libcfs_nid2str(peer_nid), sa);
+ libcfs_nidstr(peer_nid), sa);
break;
case -ETIMEDOUT:
CNETERR("Connection to %s at host %pISp took too long: that node may be hung or experiencing high load.\n",
- libcfs_nid2str(peer_nid), sa);
+ libcfs_nidstr(peer_nid), sa);
break;
case -ECONNRESET:
LCONSOLE_ERROR_MSG(0x11b,
"Connection to %s at host %pISp was reset: is it running a compatible version of Lustre and is %s one of its NIDs?\n",
- libcfs_nid2str(peer_nid), sa,
- libcfs_nid2str(peer_nid));
+ libcfs_nidstr(peer_nid), sa,
+ libcfs_nidstr(peer_nid));
break;
case -EPROTO:
LCONSOLE_ERROR_MSG(0x11c,
"Protocol error connecting to %s at host %pISp: is it running a compatible version of Lustre?\n",
- libcfs_nid2str(peer_nid), sa);
+ libcfs_nidstr(peer_nid), sa);
break;
case -EADDRINUSE:
LCONSOLE_ERROR_MSG(0x11d,
"No privileged ports available to connect to %s at host %pISp\n",
- libcfs_nid2str(peer_nid), sa);
+ libcfs_nidstr(peer_nid), sa);
break;
default:
LCONSOLE_ERROR_MSG(0x11e,
"Unexpected error %d connecting to %s at host %pISp\n",
- rc, libcfs_nid2str(peer_nid), sa);
+ rc, libcfs_nidstr(peer_nid), sa);
break;
}
}
EXPORT_SYMBOL(lnet_connect_console_error);
struct socket *
-lnet_connect(lnet_nid_t peer_nid, int interface, struct sockaddr *peeraddr,
+lnet_connect(struct lnet_nid *peer_nid, int interface,
+ struct sockaddr *peeraddr,
struct net *ns)
{
- struct lnet_acceptor_connreq cr;
+ struct lnet_acceptor_connreq cr1;
+ struct lnet_acceptor_connreq_v2 cr2;
+ void *cr;
+ int crsize;
struct socket *sock;
int rc;
int port;
@@ -156,20 +160,30 @@ struct socket *
BUILD_BUG_ON(LNET_PROTO_ACCEPTOR_VERSION != 1);
- cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
- cr.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
- cr.acr_nid = peer_nid;
+ if (nid_is_nid4(peer_nid)) {
+ cr1.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
+ cr1.acr_version = LNET_PROTO_ACCEPTOR_VERSION;
+ cr1.acr_nid = lnet_nid_to_nid4(peer_nid);
+ cr = &cr1;
+ crsize = sizeof(cr1);
- if (the_lnet.ln_testprotocompat) {
- /* single-shot proto check */
- if (test_and_clear_bit(2, &the_lnet.ln_testprotocompat))
- cr.acr_version++;
+ if (the_lnet.ln_testprotocompat) {
+ /* single-shot proto check */
+ if (test_and_clear_bit(2, &the_lnet.ln_testprotocompat))
+ cr1.acr_version++;
- if (test_and_clear_bit(3, &the_lnet.ln_testprotocompat))
- cr.acr_magic = LNET_PROTO_MAGIC;
+ if (test_and_clear_bit(3, &the_lnet.ln_testprotocompat))
+ cr1.acr_magic = LNET_PROTO_MAGIC;
+ }
+ } else {
+ cr2.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
+ cr2.acr_version = LNET_PROTO_ACCEPTOR_VERSION_16;
+ cr2.acr_nid = *peer_nid;
+ cr = &cr2;
+ crsize = sizeof(cr2);
}
- rc = lnet_sock_write(sock, &cr, sizeof(cr), accept_timeout);
+ rc = lnet_sock_write(sock, cr, crsize, accept_timeout);
if (rc)
goto failed_sock;
@@ -191,7 +205,10 @@ struct socket *
lnet_accept(struct socket *sock, u32 magic)
{
struct lnet_acceptor_connreq cr;
+ struct lnet_acceptor_connreq_v2 cr2;
+ struct lnet_nid nid;
struct sockaddr_storage peer;
+ int peer_version;
int rc;
int flip;
struct lnet_ni *ni;
@@ -249,14 +266,14 @@ struct socket *
if (flip)
__swab32s(&cr.acr_version);
- if (cr.acr_version != LNET_PROTO_ACCEPTOR_VERSION) {
- /*
- * future version compatibility!
+ switch (cr.acr_version) {
+ default:
+ /* future version compatibility!
* An acceptor-specific protocol rev will first send a version
* query. I send back my current version to tell her I'm
* "old".
*/
- int peer_version = cr.acr_version;
+ peer_version = cr.acr_version;
memset(&cr, 0, sizeof(cr));
cr.acr_magic = LNET_PROTO_ACCEPTOR_MAGIC;
@@ -267,30 +284,47 @@ struct socket *
CERROR("Error sending magic+version in response to version %d from %pIS: %d\n",
peer_version, &peer, rc);
return -EPROTO;
- }
- rc = lnet_sock_read(sock, &cr.acr_nid,
- sizeof(cr) -
- offsetof(struct lnet_acceptor_connreq, acr_nid),
- accept_timeout);
+ case LNET_PROTO_ACCEPTOR_VERSION:
+ rc = lnet_sock_read(sock, &cr.acr_nid,
+ sizeof(cr) -
+ offsetof(struct lnet_acceptor_connreq,
+ acr_nid),
+ accept_timeout);
+ if (rc)
+ break;
+ if (flip)
+ __swab64s(&cr.acr_nid);
+
+ lnet_nid4_to_nid(cr.acr_nid, &nid);
+ break;
+
+ case LNET_PROTO_ACCEPTOR_VERSION_16:
+ rc = lnet_sock_read(sock, &cr2.acr_nid,
+ sizeof(cr2) -
+ offsetof(struct lnet_acceptor_connreq_v2,
+ acr_nid),
+ accept_timeout);
+ if (rc)
+ break;
+ nid = cr2.acr_nid;
+ break;
+ }
if (rc) {
CERROR("Error %d reading connection request from %pIS\n",
rc, &peer);
return -EIO;
}
- if (flip)
- __swab64s(&cr.acr_nid);
-
- ni = lnet_nid2ni_addref(cr.acr_nid);
+ ni = lnet_nid_to_ni_addref(&nid);
if (!ni || /* no matching net */
- lnet_nid_to_nid4(&ni->ni_nid) != cr.acr_nid) {
+ !nid_same(&ni->ni_nid, &nid)) {
/* right NET, wrong NID! */
if (ni)
lnet_ni_decref(ni);
LCONSOLE_ERROR_MSG(0x120,
"Refusing connection from %pIS for %s: No matching NI\n",
- &peer, libcfs_nid2str(cr.acr_nid));
+ &peer, libcfs_nidstr(&nid));
return -EPERM;
}
@@ -299,12 +333,11 @@ struct socket *
lnet_ni_decref(ni);
LCONSOLE_ERROR_MSG(0x121,
"Refusing connection from %pIS for %s: NI doesn not accept IP connections\n",
- &peer, libcfs_nid2str(cr.acr_nid));
+ &peer, libcfs_nidstr(&nid));
return -EPERM;
}
- CDEBUG(D_NET, "Accept %s from %pIS\n",
- libcfs_nid2str(cr.acr_nid), &peer);
+ CDEBUG(D_NET, "Accept %s from %pIS\n", libcfs_nidstr(&nid), &peer);
rc = ni->ni_net->net_lnd->lnd_accept(ni, sock);
diff --git a/net/lnet/lnet/api-ni.c b/net/lnet/lnet/api-ni.c
index 1f053b3..31ccb2c 100644
--- a/net/lnet/lnet/api-ni.c
+++ b/net/lnet/lnet/api-ni.c
@@ -810,6 +810,15 @@ static void lnet_assert_wire_constants(void)
BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq, acr_nid) != 8);
BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq *)0)->acr_nid) != 8);
+ /* Checks for struct lnet_acceptor_connreq_v2 */
+ BUILD_BUG_ON((int)sizeof(struct lnet_acceptor_connreq_v2) != 28);
+ BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_magic) != 0);
+ BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_magic) != 4);
+ BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_version) != 4);
+ BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_version) != 4);
+ BUILD_BUG_ON((int)offsetof(struct lnet_acceptor_connreq_v2, acr_nid) != 8);
+ BUILD_BUG_ON((int)sizeof(((struct lnet_acceptor_connreq_v2 *)0)->acr_nid) != 20);
+
/* Checks for struct lnet_counters_common */
BUILD_BUG_ON((int)sizeof(struct lnet_counters_common) != 60);
BUILD_BUG_ON((int)offsetof(struct lnet_counters_common, lcc_msgs_alloc) != 0);
--
1.8.3.1
More information about the lustre-devel
mailing list