<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40"><head><meta http-equiv=Content-Type content="text/html; charset=utf-8"><meta name=Generator content="Microsoft Word 15 (filtered medium)"><style><!--
/* Font Definitions */
@font-face
{font-family:Helvetica;
panose-1:0 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Consolas;
panose-1:2 11 6 9 2 2 4 3 2 4;}
@font-face
{font-family:Menlo;
panose-1:2 11 6 9 3 8 4 2 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:#0563C1;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:12.0pt;
font-family:"Calibri",sans-serif;}
p.p1, li.p1, div.p1
{mso-style-name:p1;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:#65798C;}
span.EmailStyle21
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:1049500649;
mso-list-type:hybrid;
mso-list-template-ids:-964400122 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l0:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1
{mso-list-id:1108886705;
mso-list-type:hybrid;
mso-list-template-ids:190498986 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l1:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2
{mso-list-id:1223129501;
mso-list-type:hybrid;
mso-list-template-ids:2009638246 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l2:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l3
{mso-list-id:1990473593;
mso-list-type:hybrid;
mso-list-template-ids:1358955644 35951642 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l3:level1
{mso-level-start-at:4;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style></head><body lang=EN-US link="#0563C1" vlink=purple><div class=WordSection1><p class=MsoNormal><span style='font-size:11.0pt'>To close the loop on this topic. <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>The below parameters were not set by default and hence they were not showing up in </span><span style='font-size:9.0pt;font-family:Consolas;color:#24292E;background:white'>lctl list_param commands. I have to set them first. <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>lctl set_param llite.*.max_read_ahead_mb=256<o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>lctl set_param llite.*.max_read_ahead_per_file_mb=256<o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Thanks to the Lustre Community for their help to tune Lustre, I was able to tune Lustre on Oracle Cloud Infrastructure to get good performance on Bare metal nodes with 2x25Gbps network. We have open sourced the deployment of Lustre on Oracle Cloud as well as all the performance tuning done at the Infrastructure level as well as Lustre FS level for everyone to benefit from it. <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><a href="https://github.com/oracle-quickstart/oci-lustre">https://github.com/oracle-quickstart/oci-lustre</a><span style='font-size:11.0pt'><o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Terraform files are in : </span><a href="https://github.com/oracle-quickstart/oci-lustre/tree/master/terraform">https://github.com/oracle-quickstart/oci-lustre/tree/master/terraform</a><span style='font-size:11.0pt'><o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Tuning scripts are in this folder: </span><a href="https://github.com/oracle-quickstart/oci-lustre/tree/master/scripts">https://github.com/oracle-quickstart/oci-lustre/tree/master/scripts</a><span style='font-size:11.0pt'><o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>As next step - I plan to test deployment of Lustre on 100 Gbps RoCEv2 RDMA network (Mellanox CX5). <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Thanks, <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Pinkesh Valdria <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'>Oracle Cloud – Principal Solutions Architect <o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><a href="https://blogs.oracle.com/cloud-infrastructure/lustre-file-system-performance-on-oracle-cloud-infrastructure">https://blogs.oracle.com/cloud-infrastructure/lustre-file-system-performance-on-oracle-cloud-infrastructure</a><o:p></o:p></span></p><p class=MsoNormal><a href="https://blogs.oracle.com/author/pinkesh-valdria">https://blogs.oracle.com/author/pinkesh-valdria</a><span style='font-size:11.0pt'><o:p></o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal><b><span style='color:black'>From: </span></b><span style='color:black'>lustre-discuss <lustre-discuss-bounces@lists.lustre.org> on behalf of Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Date: </b>Friday, December 13, 2019 at 11:14 AM<br><b>To: </b>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)<o:p></o:p></span></p></div><div><p class=MsoNormal><span style='font-size:11.0pt'><o:p> </o:p></span></p></div><p class=MsoNormal><span style='font-size:11.0pt'>I ran the latest command you provided and it does not show the parameter, like you see. I can do screenshare. </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>[opc@lustre-client-1 ~]$ df -h</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>Filesystem Size Used Avail Use% Mounted on</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>/dev/sda3 39G 2.5G 36G 7% /</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>devtmpfs 158G 0 158G 0% /dev</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>tmpfs 158G 0 158G 0% /dev/shm</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>tmpfs 158G 17M 158G 1% /run</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>tmpfs 158G 0 158G 0% /sys/fs/cgroup</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>/dev/sda1 512M 12M 501M 3% /boot/efi</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>10.0.3.6@tcp1:/lfsbv 50T 89M 48T 1% /mnt/mdt_bv</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>10.0.3.6@tcp1:/lfsnvme 185T 8.7M 176T 1% /mnt/mdt_nvme</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>tmpfs 32G 0 32G 0% /run/user/1000</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>[opc@lustre-client-1 ~]$ lctl list_param -R llite | grep max_read_ahead</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>[opc@lustre-client-1 ~]$</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>So I ran this: </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>[opc@lustre-client-1 ~]$ lctl list_param -R llite > llite_parameters.txt</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>There are other parameters under llite. I attached the complete list. </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal><b><span style='color:black'>From: </span></b><span style='color:black'>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br><b>Date: </b>Friday, December 13, 2019 at 8:36 AM<br><b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal><span style='font-size:11.0pt'>From what I can see I think you just ran the wrong command (</span><span style='font-size:9.0pt;font-family:Menlo;color:black;background:white'>lctl list_param -R * </span><span style='font-size:11.0pt'>) or it doesn’t work as you expected on 2.12.3.</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>But llite params are sure there on a *<b>mounted</b>* Lustre client. </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>This will give you the parameters you’re looking for and need to modify to have, likely, better read performance:</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>lctl list_param -R llite | grep max_read_ahead</span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt;color:black'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:.5in'><b><span style='color:black'>From: </span></b><span style='color:black'>Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Date: </b>Friday, 13 December 2019 at 17:33<br><b>To: </b>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>This is how I installed lustre clients (only showing packages installed steps). </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>cat > /etc/yum.repos.d/lustre.repo << EOF</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[hpddLustreserver]</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>name=CentOS- - Lustre</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>baseurl=<a href="https://urldefense.proofpoint.com/v2/url?u=https-3A__downloads.whamcloud.com_public_lustre_latest-2Drelease_el7_server_&d=DwQGaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=4JVRl3NLgV0mYqKJxItBPtSpqgV0NdOryLQavGC9UEQ&s=U9NFtbOC4b2VXBIp1MenZbObM5k4GGL8SftoN3cwJmg&e=">https://downloads.whamcloud.com/public/lustre/latest-release/el7/server/</a></span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>gpgcheck=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>0</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Helvetica;color:black'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[e2fsprogs]</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>name=CentOS- - Ldiskfs</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>baseurl=<a href="https://urldefense.proofpoint.com/v2/url?u=https-3A__downloads.whamcloud.com_public_e2fsprogs_latest_el7_&d=DwQGaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=4JVRl3NLgV0mYqKJxItBPtSpqgV0NdOryLQavGC9UEQ&s=EIdVmiD8kQYOGKmB7pyI8tR4ibOIsFW10lryhGoQd2U&e=">https://downloads.whamcloud.com/public/e2fsprogs/latest/el7/</a></span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>gpgcheck=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>0</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Helvetica;color:black'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[hpddLustreclient]</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>name=CentOS- - Lustre</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>baseurl=<a href="https://urldefense.proofpoint.com/v2/url?u=https-3A__downloads.whamcloud.com_public_lustre_latest-2Drelease_el7_client_&d=DwQGaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=4JVRl3NLgV0mYqKJxItBPtSpqgV0NdOryLQavGC9UEQ&s=UdgVbfvDPd-yy02c-nwyI99COZ1reJNvfcxqqYVzMrM&e=">https://downloads.whamcloud.com/public/lustre/latest-release/el7/client/</a></span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>gpgcheck=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>0</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>EOF</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>yum install lustre-client -y</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>reboot</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:.5in'><b><span style='color:black'>From: </span></b><span style='color:black'>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br><b>Date: </b>Friday, December 13, 2019 at 2:55 AM<br><b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>From what I can see they exist on my 2.12.3 client node:</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>[root@rufus4 ~]# lctl list_param -R llite | grep max_read_ahead</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_mb</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_per_file_mb</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>llite.reprofs-ffff9f7c3b4a8800.max_read_ahead_whole_mb</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'>Regards,</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt;color:black'>Diego</span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:1.0in'><b><span style='color:black'>From: </span></b><span style='color:black'>Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Date: </b>Wednesday, 11 December 2019 at 17:46<br><b>To: </b>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>I was not able to find those parameters on my client nodes, OSS or MGS nodes. Here is how I was extracting all parameters . </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>mkdir -p lctl_list_param_R/</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>cd lctl_list_param_R/</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>lctl list_param -R * > lctl_list_param_R</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>[opc@lustre-client-1 lctl_list_param_R]$ less lctl_list_param_R | grep ahead</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsbv-ffff98231c3bc000.statahead_agl</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsbv-ffff98231c3bc000.statahead_max</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsbv-ffff98231c3bc000.statahead_running_max</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsnvme-ffff98232c30e000.statahead_agl</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsnvme-ffff98232c30e000.statahead_max</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsnvme-ffff98232c30e000.statahead_running_max</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>I also tried these commands: </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'> </span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'>Not working: </span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'>On client nodes</span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>lctl get_param llite.lfsbv-*.max_read_ahead_mb</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>error: get_param: param_path 'llite/lfsbv-*/max_read_ahead_mb': No such file or directory</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'> </span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'>Works </span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><b><span style='font-size:11.0pt'>On client nodes</span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>lctl get_param llite.*.statahead_agl</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsbv-ffff98231c3bc000.statahead_agl=1</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>llite.lfsnvme-ffff98232c30e000.statahead_agl=1</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>[opc@lustre-client-1 lctl_list_param_R]$</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:1.0in'><b><span style='color:black'>From: </span></b><span style='color:black'>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br><b>Date: </b>Tuesday, December 10, 2019 at 2:06 AM<br><b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>With that kind of degradation performance on read I would immediately think on llite’s max_read_ahead parameters on the client. Specifically these 2:</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>max_read_ahead_mb: total amount of MB allocated for read ahead, usually quite low for bandwidth benchmarking purposes and when there’re several files per client</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>max_read_ahead_per_file_mb: the default is quite low for 16MB RPCs (only a few RPCs per file)</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>You probably need to check the effect increasing both of them.</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'>Regards,</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt;color:black'>Diego</span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:1.5in'><b><span style='color:black'>From: </span></b><span style='color:black'>lustre-discuss <lustre-discuss-bounces@lists.lustre.org> on behalf of Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Date: </b>Tuesday, 10 December 2019 at 09:40<br><b>To: </b>"lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>[lustre-discuss] Degraded read performance with Large Bulk IO (16MB RPC)</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p></div><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'>I was expecting better or same read performance with Large Bulk IO (16MB RPC), but I see degradation in performance. Do I need to tune any other parameter to benefit from Large Bulk IO? Appreciate if I can get any pointers to troubleshoot further. </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><b><span style='font-size:11.0pt'>Throughput before </span></b><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l2 level1 lfo2'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>Read: 2563 MB/s</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l2 level1 lfo2'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>Write: 2585 MB/s</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><b><span style='font-size:11.0pt'>Throughput after</span></b><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l1 level1 lfo4'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>Read: 1527 MB/s. (down by ~1025)</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l1 level1 lfo4'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>Write: 2859 MB/s</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><b><span style='font-size:11.0pt'>Changes I did are: </span></b><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'>On oss</span><o:p></o:p></p><p class=p1 style='margin-left:2.0in;text-indent:-.25in;mso-list:l1 level1 lfo4'><![if !supportLists]><span style='font-family:"Calibri",sans-serif'><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span></span><![endif]><i>lctl set_param obdfilter.lfsbv-*.brw_size=16</i><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'>On clients </span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l3 level1 lfo6'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>unmounted and remounted</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l3 level1 lfo6'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:10.0pt'>lctl set_param <i>osc.lfsbv-OST*.max_pages_per_rpc</i>=4096 (got auto-updated after re-mount)</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l3 level1 lfo6'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>lctl set_param osc.*.max_rpcs_in_flight=64 (Had to manually increase this to 64, since after re-mount, it was auto-set to 8, but read/write performance was poor)</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l3 level1 lfo6'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:11.0pt'>lctl set_param osc.*.max_dirty_mb=2040. (setting the value to 2048 was failing with : Numerical result out of range error. Previously it was set to 2000 when I got good performance. </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt;color:black'>My other settings: </span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l0 level1 lfo8;background:white'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:9.0pt;font-family:Menlo;color:black'>lnetctl net add --net tcp1 --if $interface –peer-timeout 180 –peer-credits 128 –credits 1024</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l0 level1 lfo8;background:white'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo "options ksocklnd nscheds=10 sock_timeout=100 credits=2560 peer_credits=63 enable_irq_affinity=0" > /etc/modprobe.d/ksocklnd.conf</span><o:p></o:p></p><p class=MsoListParagraph style='margin-left:2.0in;text-indent:-.25in;mso-list:l0 level1 lfo8;background:white'><![if !supportLists]><span style='mso-list:Ignore'>-<span style='font:7.0pt "Times New Roman"'> </span></span><![endif]><span style='font-size:9.0pt;font-family:Menlo;color:black'>lfs setstripe -c 1 -S 1M /mnt/mdt_bv/test1</span><o:p></o:p></p><p class=MsoNormal style='margin-left:1.5in'><span style='font-size:11.0pt'> </span><o:p></o:p></p><p class=MsoNormal><span style='font-size:11.0pt'>_______________________________________________ lustre-discuss mailing list lustre-discuss@lists.lustre.org https://urldefense.proofpoint.com/v2/url?u=http-3A__lists.lustre.org_listinfo.cgi_lustre-2Ddiscuss-2Dlustre.org&d=DwICAg&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=L9-PfWwM64mRVngdHVwvQCSft2nKU4YiEPCY5x9SDXg&s=dFLf774wg7-h2jkpXEhiPWGgLtbRtb9r9hL-oH-APec&e= <o:p></o:p></span></p></div></body></html>