<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:Helvetica;
panose-1:0 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Menlo;
panose-1:2 11 6 9 3 8 4 2 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0cm;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:#954F72;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0cm;
margin-right:0cm;
margin-bottom:0cm;
margin-left:36.0pt;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0cm;
mso-margin-bottom-alt:auto;
margin-left:0cm;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.p1, li.p1, div.p1
{mso-style-name:p1;
margin:0cm;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:#65798C;}
span.EmailStyle18
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle21
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:612.0pt 792.0pt;
margin:72.0pt 72.0pt 72.0pt 72.0pt;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1049500649;
mso-list-type:hybrid;
mso-list-template-ids:-964400122 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1
{mso-list-id:1108886705;
mso-list-type:hybrid;
mso-list-template-ids:190498986 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l1:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l2
{mso-list-id:1223129501;
mso-list-type:hybrid;
mso-list-template-ids:2009638246 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l2:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l3
{mso-list-id:1990473593;
mso-list-type:hybrid;
mso-list-template-ids:1358955644 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l3:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:\uf0b7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:"Courier New";}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:\uf0a7;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-18.0pt;
font-family:Wingdings;}
ol
{margin-bottom:0cm;}
ul
{margin-bottom:0cm;}
--></style>
</head>
<body lang="EN-US" link="#0563C1" vlink="#954F72">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt">From what I can see I think you just ran the wrong command (</span><span style="font-size:9.0pt;font-family:Menlo;color:black;background:white">lctl list_param -R * </span><span style="font-size:11.0pt">)
</span><span style="font-size:11.0pt">or it doesn\u2019t work as you expected on 2.12.3.<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">But llite params are sure there on a *<b>mounted</b>* Lustre client.
<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">This will give you the parameters you\u2019re looking for and need to modify to have, likely, better read performance:<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt">lctl list_param -R llite | grep max_read_ahead</span><span style="font-size:11.0pt"><o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;color:black"><o:p> </o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-left:36.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">Pinkesh Valdria <pinkesh.valdria@oracle.com><br>
<b>Date: </b>Friday, 13 December 2019 at 17:33<br>
<b>To: </b>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"><o:p> </o:p></span></p>
</div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">This is how I installed lustre clients (only showing packages installed steps).
</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">cat > /etc/yum.repos.d/lustre.repo << EOF</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">[hpddLustreserver]</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">name=CentOS- - Lustre</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/server/</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">gpgcheck=</span><span style="font-size:9.0pt;font-family:Menlo;color:#272AD8">0</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Helvetica"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">[e2fsprogs]</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">name=CentOS- - Ldiskfs</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">baseurl=https://downloads.whamcloud.com/public/e2fsprogs/latest/el7/</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">gpgcheck=</span><span style="font-size:9.0pt;font-family:Menlo;color:#272AD8">0</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Helvetica"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">[hpddLustreclient]</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">name=CentOS- - Lustre</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/client/</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">gpgcheck=</span><span style="font-size:9.0pt;font-family:Menlo;color:#272AD8">0</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">EOF</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">yum install lustre-client -y</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">reboot</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-left:36.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br>
<b>Date: </b>Friday, December 13, 2019 at 2:55 AM<br>
<b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">From what I can see they exist on my 2.12.3 client node:</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">[root@rufus4 ~]# lctl list_param -R llite | grep max_read_ahead</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_mb</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_per_file_mb</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_whole_mb</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt">Regards,</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt;color:black">Diego</span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:36.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">Pinkesh Valdria <pinkesh.valdria@oracle.com><br>
<b>Date: </b>Wednesday, 11 December 2019 at 17:46<br>
<b>To: </b>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">I was not able to find those parameters on my client nodes, OSS or MGS nodes. Here is how I was extracting all parameters .
</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">mkdir -p lctl_list_param_R/</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">cd lctl_list_param_R/</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt;background:white"><span style="font-size:9.0pt;font-family:Menlo;color:black">lctl list_param -R * > lctl_list_param_R</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">[opc@lustre-client-1 lctl_list_param_R]$ less lctl_list_param_R | grep ahead</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsbv-ffff98231c3bc000.statahead_agl</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsbv-ffff98231c3bc000.statahead_max</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsbv-ffff98231c3bc000.statahead_running_max</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsnvme-ffff98232c30e000.statahead_agl</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsnvme-ffff98232c30e000.statahead_max</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsnvme-ffff98232c30e000.statahead_running_max</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">I also tried these commands:
</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt"> </span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt">Not working:
</span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt">On client nodes</span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">lctl get_param llite.lfsbv-*.max_read_ahead_mb</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">error: get_param: param_path 'llite/lfsbv-*/max_read_ahead_mb': No such file or directory</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt"> </span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt">Works
</span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="font-size:11.0pt">On client nodes</span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">lctl get_param llite.*.statahead_agl</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsbv-ffff98231c3bc000.statahead_agl=1</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">llite.lfsnvme-ffff98232c30e000.statahead_agl=1</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-left:72.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br>
<b>Date: </b>Tuesday, December 10, 2019 at 2:06 AM<br>
<b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">With that kind of degradation performance on read I would immediately think on llite\u2019s max_read_ahead parameters on the client. Specifically these 2:</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">max_read_ahead_mb: total amount of MB allocated for read ahead, usually quite low for bandwidth benchmarking purposes and when there\u2019re several files per client</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">max_read_ahead_per_file_mb: the default is quite low for 16MB RPCs (only a few RPCs per file)</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">You probably need to check the effect increasing both of them.</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt">Regards,</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt;color:black">Diego</span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:72.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0cm 0cm 0cm">
<p class="MsoNormal" style="margin-left:108.0pt"><b><span style="color:black">From:
</span></b><span style="color:black">lustre-discuss <lustre-discuss-bounces@lists.lustre.org> on behalf of Pinkesh Valdria <pinkesh.valdria@oracle.com><br>
<b>Date: </b>Tuesday, 10 December 2019 at 09:40<br>
<b>To: </b>"lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>[lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p>
</div>
<div>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt">I was expecting better or same read performance with Large Bulk IO (16MB RPC), but I see degradation in performance. Do I need to tune any other parameter to benefit from Large
Bulk IO? Appreciate if I can get any pointers to troubleshoot further. </span>
<o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><b><span style="font-size:11.0pt">Throughput before
</span></b><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l2 level1 lfo2">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">Read: 2563 MB/s</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l2 level1 lfo2">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">Write: 2585 MB/s</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><b><span style="font-size:11.0pt">Throughput after</span></b><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo4">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">Read: 1527 MB/s. (down by ~1025)</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo4">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">Write: 2859 MB/s</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><b><span style="font-size:11.0pt">Changes I did are:
</span></b><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt">On oss</span><o:p></o:p></p>
<p class="p1" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l1 level1 lfo4">
<![if !supportLists]><span style="font-family:"Calibri",sans-serif"><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span></span><![endif]><i>lctl set_param obdfilter.lfsbv-*.brw_size=16</i><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt">On clients
</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l3 level1 lfo6">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">unmounted and remounted</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l3 level1 lfo6">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:10.0pt">lctl set_param <i>osc.lfsbv-OST*.max_pages_per_rpc</i>=4096 (got auto-updated after re-mount)</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l3 level1 lfo6">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">lctl set_param osc.*.max_rpcs_in_flight=64 (Had to manually increase this to 64, since after re-mount, it was auto-set to 8, but read/write performance was poor)</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l3 level1 lfo6">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:11.0pt">lctl set_param osc.*.max_dirty_mb=2040. (setting the value to 2048 was failing with : Numerical result out of range error. Previously it was set to 2000 when I got good performance.
</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt;color:black">My other settings:
</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l0 level1 lfo8;background:white">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:9.0pt;font-family:Menlo;color:black">lnetctl net add --net tcp1 --if $interface \u2013peer-timeout 180 \u2013peer-credits 128 \u2013credits 1024</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l0 level1 lfo8;background:white">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:9.0pt;font-family:Menlo;color:black">echo "options ksocklnd nscheds=10 sock_timeout=100 credits=2560 peer_credits=63 enable_irq_affinity=0" > /etc/modprobe.d/ksocklnd.conf</span><o:p></o:p></p>
<p class="MsoListParagraph" style="margin-left:144.0pt;text-indent:-18.0pt;mso-list:l0 level1 lfo8;background:white">
<![if !supportLists]><span style="mso-list:Ignore">-<span style="font:7.0pt "Times New Roman"">
</span></span><![endif]><span style="font-size:9.0pt;font-family:Menlo;color:black">lfs setstripe -c 1 -S 1M /mnt/mdt_bv/test1</span><o:p></o:p></p>
<p class="MsoNormal" style="margin-left:108.0pt"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
</body>
</html>