<html xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:w="urn:schemas-microsoft-com:office:word" xmlns:m="http://schemas.microsoft.com/office/2004/12/omml" xmlns="http://www.w3.org/TR/REC-html40">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="Generator" content="Microsoft Word 15 (filtered medium)">
<style><!--
/* Font Definitions */
@font-face
{font-family:"Cambria Math";
panose-1:2 4 5 3 5 4 6 3 2 4;}
@font-face
{font-family:Calibri;
panose-1:2 15 5 2 2 2 4 3 2 4;}
@font-face
{font-family:Aptos;
panose-1:2 11 0 4 2 2 2 2 2 4;}
/* Style Definitions */
p.MsoNormal, li.MsoNormal, div.MsoNormal
{margin:0in;
font-size:12.0pt;
font-family:"Aptos",sans-serif;}
a:link, span.MsoHyperlink
{mso-style-priority:99;
color:blue;
text-decoration:underline;}
span.EmailStyle20
{mso-style-type:personal-reply;
font-family:"Calibri",sans-serif;
color:windowtext;}
.MsoChpDefault
{mso-style-type:export-only;
font-size:10.0pt;
mso-ligatures:none;}
@page WordSection1
{size:8.5in 11.0in;
margin:1.0in 1.0in 1.0in 1.0in;}
div.WordSection1
{page:WordSection1;}
/* List Definitions */
@list l0
{mso-list-id:987829718;
mso-list-template-ids:-564336210;}
@list l0:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l0:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1
{mso-list-id:1262759747;
mso-list-template-ids:-784412510;}
@list l1:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l1:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2
{mso-list-id:1388065155;
mso-list-template-ids:1848300844;}
@list l2:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l2:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3
{mso-list-id:2126070938;
mso-list-template-ids:702983090;}
@list l3:level1
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level2
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level3
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:1.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level4
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level5
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:2.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level6
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level7
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:3.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level8
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.0in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
@list l3:level9
{mso-level-number-format:bullet;
mso-level-text:;
mso-level-tab-stop:4.5in;
mso-level-number-position:left;
text-indent:-.25in;
mso-ansi-font-size:10.0pt;
font-family:Symbol;}
ol
{margin-bottom:0in;}
ul
{margin-bottom:0in;}
--></style>
</head>
<body lang="EN-US" link="blue" vlink="purple" style="word-wrap:break-word">
<div class="WordSection1">
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Andreas and WangDi,<br>
<br>
Your comments and suggestions have been very helpful in getting context around this area. Our crash testing validates that we could recover if one MDT had committed the data. Looking forward to your comments Lai and Alex.<br>
<br>
Thanks,<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif">Vandana<o:p></o:p></span></p>
<p class="MsoNormal"><span style="font-size:11.0pt;font-family:"Calibri",sans-serif"><o:p> </o:p></span></p>
<div style="border:none;border-top:solid #B5C4DF 1.0pt;padding:3.0pt 0in 0in 0in">
<p class="MsoNormal"><b><span style="font-family:"Calibri",sans-serif;color:black">From:
</span></b><span style="font-family:"Calibri",sans-serif;color:black">Di Wang <ddiwang@google.com><br>
<b>Date: </b>Monday, July 14, 2025 at 8:29 AM<br>
<b>To: </b>Andreas Dilger <adilger@ddn.com><br>
<b>Cc: </b>"Rungta, Vandana" <vrungta@amazon.com>, "lustre-discuss@lists.lustre.org" <lustre-discuss@lists.lustre.org><br>
<b>Subject: </b>RE: [EXTERNAL] [lustre-discuss] DNE mkdir performance<o:p></o:p></span></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" style="border-collapse:collapse">
<tbody>
<tr style="height:15.25pt">
<td width="1123" valign="top" style="width:842.35pt;border:solid #ED7D31 1.5pt;padding:0in 5.4pt 0in 5.4pt;height:15.25pt">
<p><strong><span style="font-family:"Aptos",sans-serif;color:black;background:#FFFF99">CAUTION</span></strong><span style="color:black;background:#FFFF99">: This email originated from outside of the organization. Do not click links or open attachments unless
you can confirm the sender and know the content is safe.</span><o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">Hello, <o:p></o:p></p>
<div>
<p class="MsoNormal"><br>
IIRC, the reordering actually forces the update log RPC to be serialized. (see LU-7426).
<o:p></o:p></p>
<div>
<p class="MsoNormal">Hmm, this osd_sync was actually introduced by COS for DNE(LU-3538), to ensure synchronization recovery, before cross-MDT recovery is supported. <br>
Though as Andreas mentioned, there should be enough information saved in the DNE recovery log on all MDTs now, we may not need this sync any more. Lai, Alex, could you please comment? Thanks.<br>
<br>
Thanks <o:p></o:p></p>
<div>
<p class="MsoNormal">WangDi<o:p></o:p></p>
</div>
</div>
</div>
</div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">On Sun, Jul 13, 2025 at 12:15<span style="font-family:"Arial",sans-serif"> </span>AM Andreas Dilger <<a href="mailto:adilger@ddn.com" target="_blank">adilger@ddn.com</a>> wrote:<o:p></o:p></p>
</div>
<blockquote style="border:none;border-left:solid #CCCCCC 1.0pt;padding:0in 0in 0in 6.0pt;margin-left:4.8pt;margin-right:0in">
<div>
<p class="MsoNormal">Wang Di write this code originally, and might be able to comment on it better, but my recollection is that it is that the sync is needed to handle the case where there are multiple changes to the DNE recovery llog due to bitmap updates
in the llog header possibly being reordered. <o:p></o:p></p>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Otherwise, there should be enough information saved on at least one of the MDTs to recover the whole operation if *any* of them have saved part of it.<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">There is some work being done by Alex Z. to improve llog performance in patch <a href="https://review.whamcloud.com/c/fs/lustre-release/+/57456:%20LU-18562%20osd:%20improve%20llog%20writes" target="_blank">https://review.whamcloud.com/c/fs/lustre-release/+/57456</a>
("LU-18562 osd: improve llog writes") and <a href="https://review.whamcloud.com/c/fs/lustre-release/+/57261" target="_blank">https://review.whamcloud.com/c/fs/lustre-release/+/57261</a> ("57261: LU-7426 obdclass: indexed llog"), the latter of which should
avoid the requirement for sync of the llog bitmap. <o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Until that point, I don't think it is totally safe to remove the sync for distributed transactions, though a 5.5x speedup is definitely attractive. The best I could suggest is to add a tunable to disable this sync, like "mdd.*.enable_dne_remote_sync"
that defaults to 1, so that it keeps the current behavior, but can be disabled to testing or if performance is more critical than preserving new directories across a full system outage. <o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
</div>
<div>
<p class="MsoNormal">Please file a ticket in Jira to track this change. <o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><o:p> </o:p></p>
<div>
<div>
<p class="MsoNormal">Cheers, Andreas<o:p></o:p></p>
</div>
<div>
<p class="MsoNormal"><br>
<br>
<o:p></o:p></p>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<p class="MsoNormal" style="margin-bottom:12.0pt">On Jul 11, 2025, at 15:06, Rungta, Vandana via lustre-discuss <<a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a>> wrote:<o:p></o:p></p>
</blockquote>
</div>
<blockquote style="margin-top:5.0pt;margin-bottom:5.0pt">
<div>
<div>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Summary</b>:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">When creating a directory (mkdir), lustre does not “sync” by default when there is a single mdt. With multiple mdts where the child directory is a created on a different mdt than
the parent (cross mdt mkdir), lustre does an osd_sync, which we suspect is for atomicity. Our experiments show that if we disable the osd_sync in the cross-mdt case, we don’t lose atomicity and system recovers if any one of the 3 hosts involved is available
(similar to the single mdt case) So, we are wondering if this “osd-sync” is needed in the cross-mdt case, as the call to sync degrades performance.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Issue:</b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">In a Lustre Distributed Namespace Environment (DNE) featuring multiple Metadata Targets (MDTs), the process of creating remote directories is notably slower compared to a single
MDT file system utilizing the osd-zfs backend.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">This performance issue can be consistently replicated using a single client, specifically by creating approximately 1000 child directories with the command
<span style="color:black">lfs mkdir -i 1 </span>. The parent directory is part of MDT-0, while the child directories are created on MDT-1, following a pattern such as /parent/child-0, /parent/child-1, etc.<o:p></o:p></p>
<ul type="disc">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l1 level1 lfo1">
Creating 1000 child directories on Parent MDT (MDT0) takes ~0.9 sec and<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l1 level1 lfo1">
Creating 1000 child directories on remote MDT (parent directory on MDT0, and child directory on MDT1) takes ~12 sec<o:p></o:p></li></ul>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Testing using mdtest with mpirun involving two clients and 50 iterations, directories are generated in a round-robin fashion to utilize both MDTs, as demonstrated by the command
"mpirun -mca routed direct -map-by node -np 16 mdtest -n 625 -i 50 -u -d /lfs/mdtest".<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" style="border-collapse:collapse">
<tbody>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black"> </span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">A</span><o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">B</span><o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">C</span><o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">D</span><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">1</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Operation</b><o:p></o:p></p>
</td>
<td width="407" colspan="3" valign="top" style="width:305.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Directory Operations/Sec</b><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">2</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>With Single MDT</b><o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>With 2 MDTs</b><o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Performance degradation percentage</b><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">3</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Directory creation</b><o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">17260.653<o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">856.898<o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">95.04<o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b> </b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Probable Cause:</b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">The creation of a child directory on the same MDT as the parent does not force a osd_sync.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">The creation of a child directory on a different MDT than the parent triggers an osd_sync of the parent directory.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">The directory creation process first checks and cancels the parent directory lock that was previously acquired during a different operation. If the lock was established as part
of the previous remote directory creation, it was done so in a protected write mode, necessitating a flush of the underlying directory. However, this cancellation process enforces a synchronization of the underlying parent Metadata Target (MDT) device.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">The conditions for enforcing the synchronization path are as follows:<o:p></o:p></p>
<ul type="disc">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo2">
LDLM_CB_CANCELING and BLOCKING_SYNC_ON_CANCEL<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo2">
l_granted_mode is one of (LCK_EX | LCK_PW | LCK_GROUP)<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l0 level1 lfo2">
OBD_CONNECT_MDS_MDS bit set in l_export<o:p></o:p></li></ul>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Corresponding code links<o:p></o:p></p>
<ul type="disc">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l2 level1 lfo3">
Link to check the above conditions at <a href="https://github.com/lustre/lustre-release/blob/b2_15/lustre/target/tgt_handler.c#L1336-L1342" target="_blank">https://github.com/lustre/lustre-release/blob/b2_15/lustre/target/tgt_handler.c#L1336-L1342</a><o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l2 level1 lfo3">
The path that invokes the synchronization is at <a href="https://github.com/lustre/lustre-release/blob/master/lustre/target/tgt_handler.c#L1381-L1394" target="_blank">https://github.com/lustre/lustre-release/blob/master/lustre/target/tgt_handler.c#L1381-L1394</a>,
provided that the locks are not taken with the LDLM_STRIPE option. <o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l2 level1 lfo3">
This entire device synchronization is enforced device sync is at <a href="https://github.com/lustre/lustre-release/blob/b2_15/lustre/target/tgt_handler.c#L1288" target="_blank">https://github.com/lustre/lustre-release/blob/b2_15/lustre/target/tgt_handler.c#L1288</a><o:p></o:p></li></ul>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Experiment:</b><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">I did an experiment where I skipped the osd_sync on directory create, and saw the following results:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Using a single client, specifically by creating approximately 1000 child directories with the command
<span style="color:black">lfs mkdir -i 1 </span>. <o:p></o:p></p>
<ul type="disc">
<li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l3 level1 lfo4">
Creating 1000 child directories on Parent MDT (MDT0) takes ~1.6 sec and<o:p></o:p></li><li class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto;mso-list:l3 level1 lfo4">
Creating 1000 child directories on remote MDT (Child directory on MDT1) takes ~3.8 sec<o:p></o:p></li></ul>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Same test using mdtest with mpirun results:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" style="border-collapse:collapse">
<tbody>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black"> </span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">A</span><o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">B</span><o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">C</span><o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border:solid windowtext 1.0pt;border-left:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">D</span><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">1</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Operation</b><o:p></o:p></p>
</td>
<td width="407" colspan="3" valign="top" style="width:305.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Directory Operations/Sec on DNE filesystem</b><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">2</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Default</b><o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Without osd_sync</b><o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Performance improvement percentage</b><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="29" valign="top" style="width:22.0pt;border:solid windowtext 1.0pt;border-top:none;background:#F0F0F0;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">3</span><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Directory creation</b><o:p></o:p></p>
</td>
<td width="122" valign="top" style="width:91.65pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">856.898<o:p></o:p></p>
</td>
<td width="98" valign="top" style="width:73.3pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;background:#EBFBEA;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="color:black">5659.511</span><o:p></o:p></p>
</td>
<td width="187" valign="top" style="width:140.05pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">560.46<o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">We conducted crash testing with osd_sync disabled, specifically targeting remote directory creation, and observed the following outcomes:<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<table class="MsoNormalTable" border="0" cellspacing="0" cellpadding="0" style="border-collapse:collapse">
<tbody>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Crash</b><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-left:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-left:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</td>
<td width="281" valign="top" style="width:211.1pt;border:solid windowtext 1.0pt;border-left:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Filesystem State</b><o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Client</b><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>MDT0</b><o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>MDT1</b><o:p></o:p></p>
</td>
<td width="281" valign="top" style="width:211.1pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="281" rowspan="6" valign="top" style="width:211.1pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Recovered, healthy and could verify the directory tree<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">No<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
</tr>
<tr>
<td width="88" valign="top" style="width:66.0pt;border:solid windowtext 1.0pt;border-top:none;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="88" valign="top" style="width:66.0pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Yes<o:p></o:p></p>
</td>
<td width="281" valign="top" style="width:211.1pt;border-top:none;border-left:none;border-bottom:solid windowtext 1.0pt;border-right:solid windowtext 1.0pt;padding:0in 5.4pt 0in 5.4pt">
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">Lost some directory entries<o:p></o:p></p>
</td>
</tr>
</tbody>
</table>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto">We are trying to understand the implication of disabling osd_sync. The POSIX spec for mkdir does not explicitly require it to be synchronously durable
<a href="https://pubs.opengroup.org/onlinepubs/9699919799/functions/mkdir.html" target="_blank">
https://pubs.opengroup.org/onlinepubs/9699919799/functions/mkdir.html</a> and pushes the burden to the user to call fsync.<br>
<br>
We do though need mkdir to be atomic and not leave partial directory artifacts on one mdt and not another. This is the part where we would like to understand from you if we are breaking the concurrency behavior here.<o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"> <o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><b>Proposed Change</b><o:p></o:p></p>
<p><span style="color:black">diff --git a/lustre/target/tgt_handler.c b/lustre/target/tgt_handler.c<br>
index 33b9863bdc..80948b5f7a 100644<br>
--- a/lustre/target/tgt_handler.c<br>
+++ b/lustre/target/tgt_handler.c<br>
@@ -1333,12 +1333,17 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,<br>
RETURN(-EINVAL);<br>
}<br>
<br>
+ //<br>
+ //Proposed Change:<br>
+ //Skip the tgt_sync if the corrspoinding operation is across OSDS and inode is being updated under IBITS lock<br>
+ //<br>
if (flag == LDLM_CB_CANCELING &&<br>
(lock->l_granted_mode & (LCK_EX | LCK_PW | LCK_GROUP)) &&<br>
(tgt->lut_sync_lock_cancel == SYNC_LOCK_CANCEL_ALWAYS ||<br>
(tgt->lut_sync_lock_cancel == SYNC_LOCK_CANCEL_BLOCKING &&<br>
ldlm_is_cbpending(lock))) &&<br>
- ((exp_connect_flags(lock->l_export) & OBD_CONNECT_MDS_MDS) ||<br>
+ (((exp_connect_flags(lock->l_export) & OBD_CONNECT_MDS_MDS) &&<br>
+ lock->l_resource->lr_type != LDLM_IBITS) ||<br>
lock->l_resource->lr_type == LDLM_EXTENT)) {<br>
__u64 start = 0;<br>
__u64 end = OBD_OBJECT_EOF;</span><o:p></o:p></p>
<p class="MsoNormal" style="mso-margin-top-alt:auto;mso-margin-bottom-alt:auto"><span style="font-size:11.0pt"> </span><o:p></o:p></p>
</div>
<p class="MsoNormal">_______________________________________________<br>
lustre-discuss mailing list<br>
<a href="mailto:lustre-discuss@lists.lustre.org" target="_blank">lustre-discuss@lists.lustre.org</a><br>
<a href="http://lists.lustre.org/listinfo.cgi/lustre-discuss-lustre.org" target="_blank">http://lists.lustre.org/listinfo.cgi/lustre-discuss-lustre.org</a><o:p></o:p></p>
</div>
</blockquote>
</div>
</div>
</div>
</blockquote>
</div>
</div>
</div>
</body>
</html>