<html xmlns:v="urn:schemas-microsoft-com:vml" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:x="urn:schemas-microsoft-com:office:excel" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40"><head><meta http-equiv=Content-Type content="text/html; charset=utf-8"><meta name=Generator content="Microsoft Word 15 (filtered medium)"><style><!--
/* Font Definitions */
@font-face
{font-family:Helvetica;
panose-1:0 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:Wingdings;
panose-1:5 0 0 0 0 0 0 0 0 0;}
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Menlo;
panose-1:2 11 6 9 3 8 4 2 2 4;}
@font-face
{font-family:"Malgun Gothic";
panose-1:2 11 5 3 2 0 0 2 0 4;}
@font-face
{font-family:"\@Malgun Gothic";}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
a:visited, span.MsoHyperlinkFollowed
{mso-style-priority:99;
color:purple;
text-decoration:underline;}
p.MsoListParagraph, li.MsoListParagraph, div.MsoListParagraph
{mso-style-priority:34;
margin-top:0in;
margin-right:0in;
margin-bottom:0in;
margin-left:.5in;
margin-bottom:.0001pt;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.msonormal0, li.msonormal0, div.msonormal0
{mso-style-name:msonormal;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.gmail-m-862763174184110697p1, li.gmail-m-862763174184110697p1, div.gmail-m-862763174184110697p1
{mso-style-name:gmail-m_-862763174184110697p1;
mso-margin-top-alt:auto;
margin-right:0in;
mso-margin-bottom-alt:auto;
margin-left:0in;
font-size:11.0pt;
font-family:"Calibri",sans-serif;}
p.p1, li.p1, div.p1
{mso-style-name:p1;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:black;}
span.gmail-m-862763174184110697s1
{mso-style-name:gmail-m_-862763174184110697s1;}
span.EmailStyle22
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.s1
{mso-style-name:s1;
color:#272AD8;}
span.apple-converted-space
{mso-style-name:apple-converted-space;}
span.EmailStyle25
{mso-style-type:personal;
font-family:"Calibri",sans-serif;
color:windowtext;}
span.EmailStyle26
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
p.p2, li.p2, div.p2
{mso-style-name:p2;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:black;}
p.p3, li.p3, div.p3
{mso-style-name:p3;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Helvetica;}
p.p4, li.p4, div.p4
{mso-style-name:p4;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:#1337FF;}
p.p5, li.p5, div.p5
{mso-style-name:p5;
margin:0in;
margin-bottom:.0001pt;
background:white;
font-size:9.0pt;
font-family:Menlo;
color:#65798C;}
span.s2
{mso-style-name:s2;
color:#D12F1B;}
span.s3
{mso-style-name:s3;
color:#65798C;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:94597958;
mso-list-type:hybrid;
mso-list-template-ids:207533352 -656221288 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l0:level1
{mso-level-text:%1;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;}
@list l0:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l0:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l0:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l1
{mso-list-id:1257522563;
mso-list-type:hybrid;
mso-list-template-ids:424861030 2143310908 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l1:level1
{mso-level-start-at:0;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:Calibri;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2
{mso-list-id:1307467471;
mso-list-type:hybrid;
mso-list-template-ids:1347294976 1699761548 67698691 67698693 67698689 67698691 67698693 67698689 67698691 67698693;}
@list l2:level1
{mso-level-start-at:206;
mso-level-number-format:bullet;
mso-level-text:-;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;
mso-fareast-font-family:"Times New Roman";}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:o;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Courier New";}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:Wingdings;}
@list l3
{mso-list-id:1835216796;
mso-list-template-ids:1983274140;}
@list l3:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l4
{mso-list-id:2030569440;
mso-list-type:hybrid;
mso-list-template-ids:-321639244 -1229293790 67698713 67698715 67698703 67698713 67698715 67698703 67698713 67698715;}
@list l4:level1
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;
font-family:"Calibri",sans-serif;}
@list l4:level2
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level3
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l4:level4
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level5
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level6
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
@list l4:level7
{mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level8
{mso-level-number-format:alpha-lower;
mso-level-tab-stop:none;
mso-level-number-position:left;
text-indent:-.25in;}
@list l4:level9
{mso-level-number-format:roman-lower;
mso-level-tab-stop:none;
mso-level-number-position:right;
text-indent:-9.0pt;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style><!--[if gte mso 9]><xml>
<o:shapedefaults v:ext="edit" spidmax="1026" />
</xml><![endif]--><!--[if gte mso 9]><xml>
<o:shapelayout v:ext="edit">
<o:idmap v:ext="edit" data="1" />
</o:shapelayout></xml><![endif]--></head><body lang=EN-US link=blue vlink=purple><div class=WordSection1><p class=MsoNormal>Thanks <a id=OWAAM480DAF200ED8F24B91C4EAF22D630893 href="mailto:diego.moreno@id.ethz.ch"><span style='font-family:"Calibri",sans-serif;text-decoration:none'>@Moreno Diego (ID SIS)</span></a> for a detailed response to my email. It gave me lot of options to further tune my cluster. I have yet to apply those changes, but thought I share the changes I plan to make. Also I have some follow-up questions to ensure the changes I am thinking of applying collectively make sense or not conflict with each other. <o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><ol style='margin-top:0in' start=1 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>On lnet:<o:p></o:p></li></ol><p class=MsoNormal style='margin-left:.5in'>Before<o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>/usr/sbin/lnetctl net add --net tcp1 --if eno2 –peer-timeout 180 <span style='background:yellow;mso-highlight:yellow'>–peer-credits 8</span> –credits 1024 </span></i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:.5in'>After<o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>/usr/sbin/lnetctl net add --net tcp1 --if eno2 –peer-timeout 180 <span style='background:yellow;mso-highlight:yellow'>–peer-credits 128</span> –credits 1024</span></i><span style='font-size:9.0pt;font-family:Menlo;color:black'> </span><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C;background:yellow;mso-highlight:yellow'>-peer_buffer_credits 0</span></i><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p></o:p></span></i></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal><o:p> </o:p></p><ol style='margin-top:0in' start=2 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>Do you have an example of how to set the PCI config to performance? I tried google search, but was unable to find an example. <o:p></o:p></li></ol><p class=MsoListParagraph style='background:white'><span style='color:black'>Currently the RPC size is 4M and related rpcs settings are below</span><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>set_param obdfilter.lfsbv-*.brw_size=4<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>set_param osc.*.max_pages_per_rpc=1024<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>lctl set_param osc.*.max_rpcs_in_flight=256<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>lctl set_param osc.*.max_dirty_mb=2048<o:p></o:p></span></i></p><p class=MsoListParagraph><o:p> </o:p></p><ol style='margin-top:0in' start=3 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>Should I update <i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>brw_size</span></i> to 16M and related settings to higher value for better performance? If yes, does it also require changes to credits and peer_credits value for lnet.conf and ksocklnd.conf file to ensure there is enough credits to send so many RPC requests. Should max_rpcs_in_flights be less than peer_credits value in lnet.conf or they are not related. <o:p></o:p></li></ol><p class=MsoListParagraph><o:p> </o:p></p><p class=MsoNormal style='text-indent:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>set_param obdfilter.lfsbv-*.brw_size=16<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>set_param osc.*.max_pages_per_rpc=4096<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>lctl set_param osc.*.max_rpcs_in_flight=256<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>lctl set_param osc.*.max_dirty_mb=8092<o:p></o:p></span></i></p><p class=MsoListParagraph style='background:white'><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p> </o:p></span></p><p class=MsoListParagraph style='background:white'><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p> </o:p></span></p><p class=MsoNormal style='margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'> </span></i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p></o:p></span></p><p class=MsoNormal><o:p> </o:p></p><ol style='margin-top:0in' start=4 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>On ksocklnd module options: more schedulers (10, 6 by default which was not enough for my server), also changed some of the buffers (tx_buffer_size and rx_buffer_size set to 1073741824) but you need to be very careful on these<o:p></o:p></li></ol><p class=MsoNormal style='margin-left:.5in'><b>Response:</b> I had none before.I plan to add the below, based on various Lustre recommendations in Lustre ppt presentations at Lustre UG meetings. <o:p></o:p></p><p class=p1 style='text-indent:.5in'><o:p> </o:p></p><p class=p1 style='text-indent:.5in'><i><span style='color:#65798C'>echo "options ksocklnd sock_timeout=100 credits=2560 peer_credits=63 enable_irq_affinity=0 concurrent_sends=63 fmr_pool_size=1280 pmr_pool_size=1280 fmr_flush_trigger=1024 nscheds=10 tx_buffer_size=1073741824 rx_buffer_size=1073741824" > /etc/modprobe.d/ksocklnd.conf<o:p></o:p></span></i></p><p class=MsoNormal style='margin-left:.5in'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'><o:p> </o:p></span></i></p><p class=MsoNormal style='margin-left:.5in'><o:p> </o:p></p><ol style='margin-top:0in' start=5 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>Sysctl.conf: increase buffers (tcp_rmem, tcp_wmem, check window_scaling, net.core.max and default, check disabling timestamps if you can afford it)<o:p></o:p></li></ol><p class=MsoNormal style='margin-left:.5in'>Given below are my current settings. My OSS and MDS node have 768 GB memory and 52 physical cores (104 vcpu). And my lustre clients have 320GB memory and 24 physical cores. <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;text-indent:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'><o:p> </o:p></span></p><p class=MsoNormal style='margin-left:.5in;text-indent:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_window_scaling = 1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:.5in'><o:p> </o:p></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_adv_win_scale=2"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_low_latency=1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'><o:p> </o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.wmem_max=16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.rmem_max=16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.wmem_default=16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.rmem_default=16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.optmem_max=16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.netdev_max_backlog=27000"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"kernel.sysrq=1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf<o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"kernel.shmmax=18446744073692774399"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.core.somaxconn=8192"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Helvetica'><o:p> </o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_rmem = 212992 87380 16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_sack = 1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf<o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_timestamps = 1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_window_scaling = 1"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_wmem = 212992 65536 16777216"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"vm.min_free_kbytes = 65536"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Helvetica'><o:p> </o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_no_metrics_save = 0"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_timestamps = 0"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:1.0in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"net.ipv4.tcp_congestion_control = htcp"</span><span style='font-size:9.0pt;font-family:Menlo;color:black'> >> /etc/sysctl.conf</span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'><o:p></o:p></span></p><p class=MsoNormal style='margin-left:.5in'><o:p> </o:p></p><p class=MsoNormal style='margin-left:.5in'><o:p> </o:p></p><ol style='margin-top:0in' start=6 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>I am running lfs 2.12.3 and Lustre 2.12.1 has a fix for single threaded issue with ksocklnd<o:p></o:p></li></ol><p class=MsoNormal style='text-indent:.5in'><a href="http://wiki.lustre.org/Lustre_2.12.1_Changelog">http://wiki.lustre.org/Lustre_2.12.1_Changelog</a> has <a href="https://jira.whamcloud.com/browse/LU-11415"><span style='font-size:10.5pt;font-family:"Arial",sans-serif;color:#663366'>LU-11415</span></a><span style='font-size:10.5pt;font-family:"Arial",sans-serif;color:#222222;background:white'>: ksocklnd performance improvement on 40Gbps ethernet</span><o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal style='margin-left:1.0in'>[opc@lustre-oss-server-nic0-4 ~]$ top <o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 552 root 20 0 0 0 0 S 4.0 0.0 39:37.43 kswapd1<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60869 root 20 0 0 0 0 S 4.0 0.0 81:25.20 socknal_sd01_04<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60870 root 20 0 0 0 0 S 4.0 0.0 81:14.20 socknal_sd01_05<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60865 root 20 0 0 0 0 S 3.6 0.0 81:33.27 socknal_sd01_00<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60866 root 20 0 0 0 0 S 3.6 0.0 81:09.03 socknal_sd01_01<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60867 root 20 0 0 0 0 S 3.6 0.0 81:11.95 socknal_sd01_02<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60868 root 20 0 0 0 0 S 3.6 0.0 81:30.26 socknal_sd01_03<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 551 root 20 0 0 0 0 S 2.6 0.0 39:24.00 kswapd0<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60860 root 20 0 0 0 0 S 2.3 0.0 30:54.35 socknal_sd00_01<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60864 root 20 0 0 0 0 S 2.3 0.0 30:58.20 socknal_sd00_05<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 64426 root 20 0 0 0 0 S 2.3 0.0 7:28.65 ll_ost_io01_102<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60859 root 20 0 0 0 0 S 2.0 0.0 30:56.70 socknal_sd00_00<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60861 root 20 0 0 0 0 S 2.0 0.0 30:54.97 socknal_sd00_02<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60862 root 20 0 0 0 0 S 2.0 0.0 30:56.06 socknal_sd00_03<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 60863 root 20 0 0 0 0 S 2.0 0.0 30:56.32 socknal_sd00_04<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'>64334 root 20 0 0 0 0 D 1.3 0.0 7:19.46 ll_ost_io01_010<o:p></o:p></p><p class=MsoNormal style='margin-left:1.0in'> 64329 root 20 0 0 0 0 S 1.0 0.0 7:46.48 ll_ost_io01_005<o:p></o:p></p><p class=MsoNormal> <o:p></o:p></p><p class=MsoNormal><o:p> </o:p></p><p class=MsoNormal><o:p> </o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal><b><span style='font-size:12.0pt;color:black'>From: </span></b><span style='font-size:12.0pt;color:black'>"Moreno Diego (ID SIS)" <diego.moreno@id.ethz.ch><br><b>Date: </b>Wednesday, December 4, 2019 at 11:12 PM<br><b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com>, Jongwoo Han <jongwoohan@gmail.com><br><b>Cc: </b>"lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Lnet Self Test<o:p></o:p></span></p></div><div><p class=MsoNormal><o:p> </o:p></p></div><p class=MsoNormal>I recently did some work on 40Gb and 100Gb ethernet interfaces and these are a few of the things that helped me during lnet_selftest:<o:p></o:p></p><p class=MsoNormal> <o:p></o:p></p><ol style='margin-top:0in' start=7 type=1><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>On lnet: credits set to higher than the default (e.g: 1024 or more), peer_credits to 128 at least for network testing (it’s just 8 by default which is good for a big cluster maybe not for lnet_selftest with 2 clients),<o:p></o:p></li><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>On ksocklnd module options: more schedulers (10, 6 by default which was not enough for my server), also changed some of the buffers (tx_buffer_size and rx_buffer_size set to 1073741824) but you need to be very careful on these<o:p></o:p></li><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>Sysctl.conf: increase buffers (tcp_rmem, tcp_wmem, check window_scaling, net.core.max and default, check disabling timestamps if you can afford it)<o:p></o:p></li><li class=MsoListParagraph style='margin-left:0in;mso-list:l4 level1 lfo6'>Other: cpupower governor (set to performance at least for testing), BIOS settings (e.g: on my AMD routers it was better to disable HT, disable a few virtualization oriented features and set the PCI config to performance). Basically, be aware that Lustre ethernet’s performance will take CPU resources so better optimize for it<o:p></o:p></li></ol><p class=MsoNormal> <o:p></o:p></p><p class=MsoNormal>Last but not least be aware that Lustre’s ethernet driver (ksocklnd) does not load balance as well as Infiniband’s (ko2iblnd). I already saw sometimes several Lustre peers using the same socklnd thread on the destination but the other socklnd threads might not be active which means that your entire load is on just dependent on one core. For that the best is to try with more clients and check in your node what’s the cpu load per thread with top. 2 clients do not seem enough to me. With the proper configuration you should be perfectly able to saturate a 25Gb link in lnet_selftest.<o:p></o:p></p><p class=MsoNormal> <o:p></o:p></p><p class=MsoNormal>Regards,<o:p></o:p></p><p class=MsoNormal> <o:p></o:p></p><div><p class=MsoNormal><span style='color:black'>Diego</span><o:p></o:p></p></div><p class=MsoNormal> <o:p></o:p></p><p class=MsoNormal> <o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:.5in'><b><span style='font-size:12.0pt;color:black'>From: </span></b><span style='font-size:12.0pt;color:black'>lustre-discuss <lustre-discuss-bounces@lists.lustre.org> on behalf of Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Date: </b>Thursday, 5 December 2019 at 06:14<br><b>To: </b>Jongwoo Han <jongwoohan@gmail.com><br><b>Cc: </b>"lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Lnet Self Test</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p></div><p class=MsoNormal style='margin-left:.5in'>Thanks Jongwoo. <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'>I have the MTU set for 9000 and also ring buffer setting set to max. <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=p1 style='margin-left:.5in'>ip link set dev $primaryNICInterface mtu <span class=s1>9000</span><o:p></o:p></p><p class=p1 style='margin-left:.5in'>ethtool -G $primaryNICInterface rx <span class=s1>2047</span> tx <span class=s1>2047</span> rx-jumbo <span class=s1>8191</span><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'>I read about changing <i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>Interrupt Coalesce, </span></i>but unable to find what values should be changed and also if it really helps or not. <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'># Several packets in a rapid sequence can be coalesced into one interrupt passed up to the CPU, providing more CPU time for application processing.</span></i><o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'>Thanks,<o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'>Pinkesh valdria<o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'>Oracle Cloud<o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='margin-left:.5in'><b><span style='font-size:12.0pt;color:black'>From: </span></b><span style='font-size:12.0pt;color:black'>Jongwoo Han <jongwoohan@gmail.com><br><b>Date: </b>Wednesday, December 4, 2019 at 8:07 PM<br><b>To: </b>Pinkesh Valdria <pinkesh.valdria@oracle.com><br><b>Cc: </b>Andreas Dilger <adilger@whamcloud.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br><b>Subject: </b>Re: [lustre-discuss] Lnet Self Test</span><o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'>Have you tried MTU >= 9000 bytes (AKA jumbo frame) on the 25G ethernet and the switch? <o:p></o:p></p><div><p class=MsoNormal style='margin-left:.5in'>If it is set to 1500 bytes, ethernet + IP + TCP frame headers take quite amount of packet, reducing available bandwidth for data.<o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p></div><div><p class=MsoNormal style='margin-left:.5in'>Jongwoo Han<o:p></o:p></p></div></div><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p><div><div><p class=MsoNormal style='margin-left:.5in'>2019<span style='font-family:"Malgun Gothic",sans-serif'>년</span> 11<span style='font-family:"Malgun Gothic",sans-serif'>월</span> 28<span style='font-family:"Malgun Gothic",sans-serif'>일</span> (<span style='font-family:"Malgun Gothic",sans-serif'>목</span>) <span style='font-family:"Malgun Gothic",sans-serif'>오전</span> 3:44, Pinkesh Valdria <<a href="mailto:pinkesh.valdria@oracle.com">pinkesh.valdria@oracle.com</a>><span style='font-family:"Malgun Gothic",sans-serif'>님이</span> <span style='font-family:"Malgun Gothic",sans-serif'>작성</span>:<o:p></o:p></p></div><blockquote style='border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-top:5.0pt;margin-right:0in;margin-bottom:5.0pt'><div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Thanks Andreas for your response. <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>I ran anotherLnet Self test with 48 concurrent processes, since the nodes have 52 physical cores and I was able to achieve same throughput (<span style='font-size:9.0pt;font-family:Menlo;color:black'>2052.71 MiB/s = 2152 MB/s</span>).<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Is it expected to lose almost 600 MB/s (2750-2150= ) due to overheads on ethernet with Lnet?<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Thanks,<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Pinkesh Valdria<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Oracle Cloud Infrastructure <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><div style='border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in'><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><b><span style='color:black'>From: </span></b><span style='color:black'>Andreas Dilger <<a href="mailto:adilger@whamcloud.com" target="_blank">adilger@whamcloud.com</a>><br><b>Date: </b>Wednesday, November 27, 2019 at 1:25 AM<br><b>To: </b>Pinkesh Valdria <<a href="mailto:pinkesh.valdria@oracle.com" target="_blank">pinkesh.valdria@oracle.com</a>><br><b>Cc: </b>"<a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a>" <<a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a>><br><b>Subject: </b>Re: [lustre-discuss] Lnet Self Test</span><o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p></div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>The first thing to note is that lst reports results in binary units <o:p></o:p></p><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>(MiB/s) while iperf reports results in decimal units (Gbps). If you do the<o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>conversion you get 2055.31 MiB/s = 2155 MB/s.<o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>The other thing to check is the CPU usage. For TCP the CPU usage can<o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>be high. You should try RoCE+o2iblnd instead. <o:p></o:p></p><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><div id="gmail-m_-862763174184110697AppleMailSignature"><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Cheers, Andreas<o:p></o:p></p></div><div><p class=MsoNormal style='mso-margin-top-alt:auto;margin-bottom:12.0pt;margin-left:.5in'><br>On Nov 26, 2019, at 21:26, Pinkesh Valdria <<a href="mailto:pinkesh.valdria@oracle.com" target="_blank">pinkesh.valdria@oracle.com</a>> wrote:<o:p></o:p></p></div><blockquote style='margin-top:5.0pt;margin-bottom:5.0pt'><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Hello All, <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>I created a new Lustre cluster on CentOS7.6 and I am running lnet_selftest_wrapper.sh to measure throughput on the network. The nodes are connected to each other using 25Gbps ethernet, so theoretical max is 25 Gbps * 125 = 3125 MB/s. Using iperf3, I get 22Gbps (2750 MB/s) between the nodes.<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[root@lustre-client-2 ~]# for c in 1 2 4 8 12 16 20 24 ; do echo $c ; ST=lst-output-$(date +%Y-%m-%d-%H:%M:%S) CN=$c SZ=1M TM=30 BRW=write CKSUM=simple LFROM="10.0.3.7@tcp1" LTO="10.0.3.6@tcp1" /root/lnet_selftest_wrapper.sh; done ;<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>When I run lnet_selftest_wrapper.sh (from <a href="https://urldefense.proofpoint.com/v2/url?u=http-3A__wiki.lustre.org_LNET-5FSelftest&d=DwMGaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=dEosA07cQm7WPohubrpzab8agc4uFDGesC-4tI4ylm0&s=-ne2Yke64JRw4BQu9pa0DXwf3tHkDqaUbp7S6Eq_C_Q&e=" target="_blank">Lustre wiki</a>) between 2 nodes, I get a max of 2055.31 MiB/s, Is that expected at the Lnet level? Or can I further tune the network and OS kernel (tuning I applied are below) to get better throughput?<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><b><u>Result Snippet from lnet_selftest_wrapper.sh</u></b><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[LNet Rates of lfrom]<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[R] Avg: 4112 RPC/s Min: 4112 RPC/s Max: 4112 RPC/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[W] Avg: 4112 RPC/s Min: 4112 RPC/s Max: 4112 RPC/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[LNet Bandwidth of lfrom]<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[R] Avg: 0.31 MiB/s Min: 0.31 MiB/s Max: 0.31 MiB/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><span style='color:black;background:yellow'>[W] Avg: 2055.30 MiB/s Min: 2055.30 MiB/s Max: 2055.30 MiB/s</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[LNet Rates of lto]<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[R] Avg: 4136 RPC/s Min: 4136 RPC/s Max: 4136 RPC/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[W] Avg: 4136 RPC/s Min: 4136 RPC/s Max: 4136 RPC/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[LNet Bandwidth of lto]<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><span style='color:black;background:yellow'>[R] Avg: 2055.31 MiB/s Min: 2055.31 MiB/s Max: 2055.31 MiB/s</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>[W] Avg: 0.32 MiB/s Min: 0.32 MiB/s Max: 0.32 MiB/s<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><b><u>Tuning applied: </u></b><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><u>Ethernet NICs: </u><o:p></o:p></p><p class=gmail-m-862763174184110697p1 style='margin-left:.5in'>ip link set dev ens3 mtu <span class=gmail-m-862763174184110697s1>9000</span> <o:p></o:p></p><p class=gmail-m-862763174184110697p1 style='margin-left:.5in'>ethtool -G ens3 rx <span class=gmail-m-862763174184110697s1>2047</span> tx <span class=gmail-m-862763174184110697s1>2047</span> rx-jumbo <span class=gmail-m-862763174184110697s1>8191</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'><u>less /etc/sysctl.conf</u><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.wmem_max=16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.rmem_max=16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.wmem_default=16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.rmem_default=16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.optmem_max=16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.netdev_max_backlog=27000<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>kernel.sysrq=1<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>kernel.shmmax=18446744073692774399<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.core.somaxconn=8192<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_adv_win_scale=2<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_low_latency=1<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_rmem = 212992 87380 16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_sack = 1<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_timestamps = 1<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_window_scaling = 1<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_wmem = 212992 65536 16777216<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>vm.min_free_kbytes = 65536<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_congestion_control = cubic<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_timestamps = 0<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_congestion_control = htcp<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>net.ipv4.tcp_no_metrics_save = 0<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>echo </span><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>"#</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'># tuned configuration</span></i><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><i><span style='font-size:9.0pt;font-family:Menlo;color:#65798C'>#</span></i><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[main]</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>summary=Broadly applicable tuning that provides excellent performance across a variety of common server workloads</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'> </span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[disk]</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>devices=!dm-*, !sda1, !sda2, !sda3</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>readahead=></span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>4096</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Helvetica;color:black'> </span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[cpu]</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>force_latency=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>1</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>governor=performance</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>energy_perf_bias=performance</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>min_perf_pct=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>100</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[vm]</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>transparent_huge_pages=never</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>[sysctl]</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>kernel.sched_min_granularity_ns = </span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>10000000</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>kernel.sched_wakeup_granularity_ns = </span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>15000000</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>vm.dirty_ratio = </span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>30</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>vm.dirty_background_ratio = </span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>10</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>vm.swappiness=</span><span style='font-size:9.0pt;font-family:Menlo;color:#272AD8'>30</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:#D12F1B'>" > lustre-performance/tuned.conf</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Helvetica;color:black'> </span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in;background:white'><span style='font-size:9.0pt;font-family:Menlo;color:black'>tuned-adm profile lustre-performance</span><o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Thanks,<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>Pinkesh Valdria<o:p></o:p></p><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'> <o:p></o:p></p></div></blockquote><blockquote style='margin-top:5.0pt;margin-bottom:5.0pt'><div><p class=MsoNormal style='mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;margin-left:.5in'>_______________________________________________<br>lustre-discuss mailing list<br><a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a><br><a href="https://urldefense.proofpoint.com/v2/url?u=http-3A__lists.lustre.org_listinfo.cgi_lustre-2Ddiscuss-2Dlustre.org&d=DwMGaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=dEosA07cQm7WPohubrpzab8agc4uFDGesC-4tI4ylm0&s=ejwMDqk5D3TzRE5eTzFdEKo9cQ0I6GVqN04wgaJcn0s&e=" target="_blank">http://lists.lustre.org/listinfo.cgi/lustre-discuss-lustre.org</a><o:p></o:p></p></div></blockquote></div></div></div></div><p class=MsoNormal style='margin-left:.5in'>_______________________________________________<br>lustre-discuss mailing list<br><a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a><br><a href="https://urldefense.proofpoint.com/v2/url?u=http-3A__lists.lustre.org_listinfo.cgi_lustre-2Ddiscuss-2Dlustre.org&d=DwMFaQ&c=RoP1YumCXCgaWHvlZYR8PZh8Bv7qIrMUB65eapI_JnE&r=HpfvG0tozSl7HgJJuyxxo2149EjwqpQDE7ytv-4sZuI&m=6atMUkU7ebsLF7ieA6hjGFCUwJjGhXLtGzGLzhmjz1E&s=Xha6x47Y1z2YnkFxI9WFXKuQv-wzpGbnGjd7cIKwt5A&e=" target="_blank">http://lists.lustre.org/listinfo.cgi/lustre-discuss-lustre.org</a><o:p></o:p></p></blockquote></div><p class=MsoNormal style='margin-left:.5in'><br clear=all><o:p></o:p></p><div><p class=MsoNormal style='margin-left:.5in'> <o:p></o:p></p></div><p class=MsoNormal style='margin-left:.5in'>-- <o:p></o:p></p><div><p class=MsoNormal style='margin-left:.5in'><span style='font-family:"Courier New"'>Jongwoo Han</span><o:p></o:p></p><div><p class=MsoNormal style='margin-left:.5in'><span style='font-family:"Courier New"'>+82-505-227-6108</span><o:p></o:p></p></div></div></div></body></html>