[lustre-devel] [PATCH 07/15] lustre: osc: Support RDMA only pages

[James Simmons]

From: Amir Shehata <ashehata at whamcloud.com>

Some memory architectures and CPU-offload cards with
on-board memory do not map data pages into the CPU
address space. Allow RDMA of data directly into those
pages without accessing contents.

Therefore, made changes to prevent doing checksum on
these type of pages.

WC-bug-id: https://jira.whamcloud.com/browse/LU-14798
Lustre-commit: 29eabeb34c5ba2cffd ("LU-14798 lustre: Support RDMA only pages")
Signed-off-by: Wang Shilong <wshilong at ddn.com>
Signed-off-by: Amir Shehata <ashehata at whamcloud.com>
Lustre-change: https://review.whamcloud.com/37454
Reviewed-by: Andreas Dilger <adilger at whamcloud.com>
Whamcloud-bug-id: EX-773
Reviewed-on: https://review.whamcloud.com/44111
Reviewed-by: Wang Shilong <wangshilong1991 at gmail.com>
Reviewed-by: Patrick Farrell <pfarrell at whamcloud.com>
Signed-off-by: James Simmons <jsimmons at infradead.org>
---
 fs/lustre/include/lustre_osc.h |  6 ++++--
 fs/lustre/osc/osc_io.c         |  2 ++
 fs/lustre/osc/osc_request.c    | 15 +++++++++++----
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/fs/lustre/include/lustre_osc.h b/fs/lustre/include/lustre_osc.h
index 09868ea..cdc9aae 100644
--- a/fs/lustre/include/lustre_osc.h
+++ b/fs/lustre/include/lustre_osc.h
@@ -956,8 +956,10 @@ struct osc_extent {
 				oe_ndelay:1,
 	/* direct IO pages */
 				oe_dio:1,
-	/* this extent consists of RDMA only pages */
-				oe_is_rdma_only;
+	/* this extent consists of pages that are not directly accessible
+	 *  from the CPU
+	 */
+				oe_is_rdma_only:1;
 	/* how many grants allocated for this extent.
 	 *  Grant allocated for this extent. There is no grant allocated
 	 *  for reading extents and sync write extents.
diff --git a/fs/lustre/osc/osc_io.c b/fs/lustre/osc/osc_io.c
index d828ae0..b867985 100644
--- a/fs/lustre/osc/osc_io.c
+++ b/fs/lustre/osc/osc_io.c
@@ -153,6 +153,8 @@ int osc_io_submit(const struct lu_env *env, const struct cl_io_slice *ios,
 	page = cl_page_list_first(qin);
 	if (page->cp_type == CPT_TRANSIENT)
 		brw_flags |= OBD_BRW_NOCACHE;
+	if (lnet_is_rdma_only_page(page->cp_vmpage))
+		brw_flags |= OBD_BRW_RDMA_ONLY;
 
 	/*
 	 * NOTE: here @page is a top-level page. This is done to avoid
diff --git a/fs/lustre/osc/osc_request.c b/fs/lustre/osc/osc_request.c
index 2ac0300..db73fce 100644
--- a/fs/lustre/osc/osc_request.c
+++ b/fs/lustre/osc/osc_request.c
@@ -1402,6 +1402,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 	const char *obd_name = cli->cl_import->imp_obd->obd_name;
 	struct inode *inode = NULL;
 	bool directio = false;
+	bool enable_checksum = true;
 
 	if (pga[0]->pg) {
 		inode = page2inode(pga[0]->pg);
@@ -1545,6 +1546,11 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 		}
 	}
 
+	if (lnet_is_rdma_only_page(pga[0]->pg)) {
+		enable_checksum = false;
+		short_io_size = 0;
+	}
+
 	/* Check if read/write is small enough to be a short io. */
 	if (short_io_size > cli->cl_max_short_io_bytes || niocount > 1 ||
 	    !imp_connect_shortio(cli->cl_import))
@@ -1700,10 +1706,12 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 	if (osc_should_shrink_grant(cli))
 		osc_shrink_grant_local(cli, &body->oa);
 
+	if (!cli->cl_checksum || sptlrpc_flavor_has_bulk(&req->rq_flvr))
+		enable_checksum = false;
+
 	/* size[REQ_REC_OFF] still sizeof (*body) */
 	if (opc == OST_WRITE) {
-		if (cli->cl_checksum &&
-		    !sptlrpc_flavor_has_bulk(&req->rq_flvr)) {
+		if (enable_checksum) {
 			/* store cl_cksum_type in a local variable since
 			 * it can be changed via lprocfs
 			 */
@@ -1743,8 +1751,7 @@ static int osc_brw_prep_request(int cmd, struct client_obd *cli,
 		req_capsule_set_size(pill, &RMF_RCS, RCL_SERVER,
 				     sizeof(u32) * niocount);
 	} else {
-		if (cli->cl_checksum &&
-		    !sptlrpc_flavor_has_bulk(&req->rq_flvr)) {
+		if (enable_checksum) {
 			if ((body->oa.o_valid & OBD_MD_FLFLAGS) == 0)
 				body->oa.o_flags = 0;
 			body->oa.o_flags |= obd_cksum_type_pack(obd_name,
-- 
1.8.3.1