[Lustre-devel] readahead performance improvement
Wallior, Julien
Julien.Wallior at sig.com
Tue Nov 25 16:12:34 UTC 2008
> One of the problems with this change is that the first read of the file
> is at offset 0, but it should not invoke readahead.
In our case, we can do readahead starting at offset 0 because we assume
we'll be reading the whole file.
> The other problem is that this assumes the
> file striping is 1MB, but that isn't always the case.
I'm using the PTLRPC_MAX_BRW_SIZE for 1MB.
> Looking at the existing Lustre ll_readahead code I see:
>
> /* Enlarge the RA window to encompass the full read */
> if (bead != NULL && ras->ras_window_start +
> ras->ras_window_len < bead->lrr_start + bead->lrr_count) {
> ras->ras_window_len = bead->lrr_start + bead->lrr_count -
> ras->ras_window_start;
> }
>
> This is incorrect in my opinion. The readahead window should be grown
> to the end of the first stripe boundary (usually 1MB, but it depends
> on striping), and then continue with aligned stripe sized reads (or
> multiples thereof to make full RPCs). That can be determined by the
> readahead code correctly using the obd_extent_calc() method.
That would probably be a much better fix than mine. When we have a few
reading thread we get only 1MB IOs on the OSS. But when we have a lot
of thread (over 20), we see small IOs hitting the LUNs and that's what I'm
trying to prevent.
> I also see in two places the identical code:
>
> bead.lrr_start = *ppos >> CFS_PAGE_SHIFT;
> bead.lrr_count = (count + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
> ll_ra_read_in(&bead);
>
> It also makes sense that this bead initialization is done in ll_ra_read_in(),
> and it should probably be renamed ll_ra_read_init().
That is probably a good idea. We are not using the path in ll_file_sendfile, so
I didn't change that.
> I'd be interested to see the patches, and I've CC'd lustre-devel for
> further discussion.
The patch is attached.
Julien
IMPORTANT: The information contained in this email and/or its attachments is confidential. If you are not the intended recipient, please notify the sender immediately by reply and immediately delete this message and all its attachments. Any review, use, reproduction, disclosure or dissemination of this message or any attachment by an unintended recipient is strictly prohibited. Neither this message nor any attachment is intended as or should be construed as an offer, solicitation or recommendation to buy or sell any security or other financial instrument. Neither the sender, his or her employer nor any of their respective affiliates makes any warranties as to the completeness or accuracy of any of the information contained herein or that this message or any of its attachments is free of viruses.
-------------- next part --------------
diff -Nur lustre/llite/file.c ../lustre-1.6.4.3-patched/lustre/llite/file.c
--- lustre/llite/file.c 2007-10-02 06:29:13.000000000 -0400
+++ ../lustre-1.6.4.3-patched/lustre/llite/file.c 2008-11-24 14:53:56.938438291 -0500
@@ -1247,6 +1247,33 @@
return rc;
}
+static int init_aligned_ra_window(struct file *file, size_t count, loff_t *ppos,
+ struct ll_sb_info *sbi,
+ struct ll_ra_read *bead)
+{
+ int ra = 0;
+ int ll_aligned_ra_size = PTLRPC_MAX_BRW_SIZE;
+ if ( count >= ll_aligned_ra_size ) {
+ ra = 1;
+ bead->lrr_count = (count + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
+ } else if ( *ppos % ll_aligned_ra_size == 0 ) {
+ /* if IO aligned with ll_aligned_ra_size */
+ ra = 1;
+ if (*ppos == 0)
+ bead->lrr_count = (pgoff_t) ll_aligned_ra_size *
+ sbi->ll_aligned_ra_mf << (20 - CFS_PAGE_SHIFT);
+ else
+ bead->lrr_count = (pgoff_t) ll_aligned_ra_size <<
+ (20 - CFS_PAGE_SHIFT);
+ }
+
+ if (ra == 1) {
+ bead->lrr_start = *ppos >> CFS_PAGE_SHIFT;
+ ll_ra_read_in(file, bead);
+ }
+ return ra;
+}
+
static ssize_t ll_file_read(struct file *file, char *buf, size_t count,
loff_t *ppos)
{
@@ -1377,12 +1404,17 @@
file->f_ra.ra_pages = 0;
#endif
/* initialize read-ahead window once per syscall */
- if (ra == 0) {
+ if (ra == 0 && !sbi->ll_aligned_ra) {
ra = 1;
bead.lrr_start = *ppos >> CFS_PAGE_SHIFT;
- bead.lrr_count = (count + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT;
+ bead.lrr_count = (count + CFS_PAGE_SIZE - 1) >>
+ CFS_PAGE_SHIFT;
ll_ra_read_in(file, &bead);
}
+ if (ra == 0 && sbi->ll_aligned_ra) {
+ ra = init_aligned_ra_window(file, count, ppos, sbi,
+ &bead);
+ }
/* BUG: 5972 */
file_accessed(file);
diff -Nur lustre/llite/llite_internal.h ../lustre-1.6.4.3-patched/lustre/llite/llite_internal.h
--- lustre/llite/llite_internal.h 2007-10-08 10:44:54.000000000 -0400
+++ ../lustre-1.6.4.3-patched/lustre/llite/llite_internal.h 2008-11-24 14:53:56.942438328 -0500
@@ -285,6 +285,10 @@
unsigned long long ll_sa_blocked; /* ls count waiting for
* statahead */
unsigned long long ll_sa_cached; /* ls count got in cache */
+
+ /* aligned read_ahead */
+ unsigned int ll_aligned_ra;
+ unsigned int ll_aligned_ra_mf;
};
#define LL_DEFAULT_MAX_RW_CHUNK (32 * 1024 * 1024)
diff -Nur lustre/llite/llite_lib.c ../lustre-1.6.4.3-patched/lustre/llite/llite_lib.c
--- lustre/llite/llite_lib.c 2007-11-29 05:27:49.000000000 -0500
+++ ../lustre-1.6.4.3-patched/lustre/llite/llite_lib.c 2008-11-24 14:53:56.946438365 -0500
@@ -109,6 +109,10 @@
/* metadata statahead is enabled by default */
sbi->ll_sa_max = LL_STATAHEAD_DEF;
+ /* aligned readahead disabled by default */
+ sbi->ll_aligned_ra = 0;
+ sbi->ll_aligned_ra_mf = 5;
+
RETURN(sbi);
}
diff -Nur lustre/llite/lproc_llite.c ../lustre-1.6.4.3-patched/lustre/llite/lproc_llite.c
--- lustre/llite/lproc_llite.c 2007-10-05 22:45:15.000000000 -0400
+++ ../lustre-1.6.4.3-patched/lustre/llite/lproc_llite.c 2008-11-24 14:53:56.954438440 -0500
@@ -510,6 +510,84 @@
sbi->ll_sa_blocked + sbi->ll_sa_cached);
}
+static int ll_rd_aligned_ra(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{
+ struct super_block *sb = data;
+ struct ll_sb_info *sbi = ll_s2sbi(sb);
+ unsigned int val;
+
+ spin_lock(&sbi->ll_lock);
+ val = sbi->ll_aligned_ra;
+ spin_unlock(&sbi->ll_lock);
+
+ return snprintf(page, count, "%u\n", val);
+}
+
+static int ll_wr_aligned_ra(struct file *file, const char *buffer,
+ unsigned long count, void *data)
+{
+ struct super_block *sb = data;
+ struct ll_sb_info *sbi = ll_s2sbi(sb);
+ int val, rc;
+
+ rc = lprocfs_write_helper(buffer, count, &val);
+ if (rc)
+ return rc;
+
+ if (val) {
+ val = 1;
+ }
+
+ spin_lock(&sbi->ll_lock);
+ sbi->ll_aligned_ra = val;
+ spin_unlock(&sbi->ll_lock);
+
+ return count;
+}
+
+static int ll_rd_aligned_ra_mf(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{
+ struct super_block *sb = data;
+ struct ll_sb_info *sbi = ll_s2sbi(sb);
+ unsigned int val;
+
+ spin_lock(&sbi->ll_lock);
+ val = sbi->ll_aligned_ra_mf;
+ spin_unlock(&sbi->ll_lock);
+
+ return snprintf(page, count, "%u\n", val);
+}
+
+static int ll_wr_aligned_ra_mf(struct file *file, const char *buffer,
+ unsigned long count, void *data)
+{
+ struct super_block *sb = data;
+ struct ll_sb_info *sbi = ll_s2sbi(sb);
+ int val, rc;
+
+ rc = lprocfs_write_helper(buffer, count, &val);
+ if (rc)
+ return rc;
+
+ /* Cap this at the current max readahead window size, the readahead
+ * algorithm does this anyway so it's pointless to set it larger. */
+ if (val < 0 || val > (sbi->ll_ra_info.ra_max_pages >>
+ (20 - CFS_PAGE_SHIFT))) {
+ CERROR("can't set ll_aligned_ra_mf more than "
+ "max_read_ahead_mb: %lu\n",
+ sbi->ll_ra_info.ra_max_pages >> (20 - CFS_PAGE_SHIFT));
+ return -ERANGE;
+ }
+
+ spin_lock(&sbi->ll_lock);
+ sbi->ll_aligned_ra_mf = val;
+ spin_unlock(&sbi->ll_lock);
+
+ return count;
+}
+
static struct lprocfs_vars lprocfs_obd_vars[] = {
{ "uuid", ll_rd_sb_uuid, 0, 0 },
//{ "mntpt_path", ll_rd_path, 0, 0 },
@@ -535,6 +613,8 @@
{ "statahead_count", ll_rd_statahead_count, 0, 0 },
{ "statahead_max", ll_rd_statahead_max, ll_wr_statahead_max, 0 },
{ "statahead_stats", ll_rd_statahead_stats, 0, 0 },
+ { "aligned_read_ahead", ll_rd_aligned_ra, ll_wr_aligned_ra, 0 },
+ { "aligned_read_ahead_mf", ll_rd_aligned_ra_mf, ll_wr_aligned_ra_mf, 0},
{ 0 }
};
diff -Nur lustre/llite/rw.c ../lustre-1.6.4.3-patched/lustre/llite/rw.c
--- lustre/llite/rw.c 2007-10-18 07:19:20.000000000 -0400
+++ ../lustre-1.6.4.3-patched/lustre/llite/rw.c 2008-11-24 14:53:56.962438514 -0500
@@ -1368,7 +1368,8 @@
/* The initial ras_window_len is set to the request size. To avoid
* uselessly reading and discarding pages for random IO the window is
* only increased once per consecutive request received. */
- if (ras->ras_consecutive_requests > 1 && !ras->ras_request_index) {
+ if (!sbi->ll_aligned_ra && ras->ras_consecutive_requests > 1 &&
+ !ras->ras_request_index) {
ras->ras_window_len = min(ras->ras_window_len +
(1024 * 1024 >> CFS_PAGE_SHIFT),
ra->ra_max_pages);
More information about the lustre-devel
mailing list