Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3605
ldlm_cb_06    D 0000000000000001     0  3605      1          3606  3604 (L-TLB)
0000010073e01c48 0000000000000046 0000010077e5b6c0 00000100741f0030 
       ffffffff80134b62 0000010073e01ba0 0000010073e01ba0 00000001a01f481a 
       00000100741f0030 00000000000004ca 
Call Trace:<ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80308223>{__down+147} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffff80309cbb>{__down_failed+53} 
       <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
       <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
       <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3606: it was inactive for 18s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3606
ldlm_cb_07    D 0000000000000001     0  3606      1          3607  3605 (L-TLB)
0000010073e03c48 0000000000000046 0000010077e5b6c0 0000010074644800 
       ffffffff80134b62 0000010073e03ba0 0000010073e03ba0 00000001a01f481a 
       0000010074644800 00000000000004aa 
Call Trace:<ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80308223>{__down+147} 
       <ffffffff801331a5>{default_wake_function+0} <1>LustreError: dumping log to /tmp/lustre-log.1191932740.3605
<ffffffff80309cbb>{__down_failed+53} 
       <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
       <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
       <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
LustreError: dumping log to /tmp/lustre-log.1191932740.3606
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 16177: it was inactive for 100s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 16177
ll_ost_io_34  D 0000000000000002     0 16177      1         16178 16176 (L-TLB)
000001001a57f7b8 0000000000000046 000001006bb08000 0000000100000001 
       000001006fd381f0 000001005b085500 0000010062a41c98 0000000060dd9000 
       000001002a013800 00000000000001a8 
Call Trace:<ffffffffa0063fc4>{:jbd:log_wait_commit+202} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffffa043fa43>{:fsfilt_ldiskfs:fsfilt_ldiskfs_commit_wait+143} 
       <ffffffffa04efa68>{:obdfilter:filter_commitrw_write+4241} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
       <4>Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3699: it was inactive for 100s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3699
ll_mdt_26     S 0000010071dc7998     0  3699      1          3700  3698 (L-TLB)
0000010071dc78d8 0000000000000046 000000000549bc5d ffffffff00000073 
       000000000549bc5d 0000000000000000 0000010001021aa0 00000000771e4890 
       000001006d79c030 000000000000089d 
Call Trace:<ffffffff8013f100>{__mod_timer+293} <ffffffff802a97a7>{alloc_skb+92} <ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
       <ffffffffa00d146f>{:e1000:e1000_get_settings+239} <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132384>{move_tasks+200} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
<ffffffff80309a5b>{schedule_timeout+367} 
       <ffffffff8013fb2a>{process_timeout+0} <ffffffffa02aa7cb>{:ptlrpc:ptlrpc_queue_wait+2772} 
       <1>LustreError: dumping log to /tmp/lustre-log.1191932822.16177
<ffffffff801331a5>{default_wake_function+0} <ffffffffa02a63a5>{:ptlrpc:ptlrpc_prep_req_pool+1493} 
       <ffffffffa02a86cc>{:ptlrpc:expired_request+0} <ffffffffa02a8770>{:ptlrpc:interrupted_request+0} 
       <ffffffffa0344eb0>{:lquota:client_quota_ctl+447} <ffffffffa0345469>{:lquota:lov_quota_ctl+1159} 
       <ffffffffa034badb>{:lquota:mds_get_dqblk+1616} <ffffffffa02b05a8>{:ptlrpc:lustre_msg_add_version+67} 
       <ffffffffa02b0dc5>{:ptlrpc:lustre_pack_reply+1928} 
       <ffffffffa0344758>{:lquota:mds_quota_ctl+248} <ffffffffa045a755>{:mds:mds_handle_quotactl+994} 
       <ffffffffa045e19f>{:mds:mds_handle+14732} <ffffffff801315df>{activate_task+124} 
       <ffffffffa02878e8>{:ptlrpc:ldlm_reprocess_queue+316} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80308f39>{thread_return+0} <4>Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 16208: it was inactive for 100s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 16208
ll_ost_io_65  S 0000000000000259     0 16208      1         16209 16207 (L-TLB)
000001002eed75b8 0000000000000046 0000010016e82500 ffffffff801792ac 
       0000000000000016 0000000000000212 000716b1e783829d 000000007b026030 
       0000010062eb6030 000000000000044d 
Call Trace:<ffffffff80308f91>{thread_return+88}<ffffffff801792ac>{__find_get_block+396} <ffffffffa034286a>{:lquota:schedule_dqacq+2775} 
       <ffffffffa03f6515>{:ldiskfs:ldiskfs_ext_find_extent+500} 
        
       <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
ll_ost_io_20  S 0000000000000259     0  3805      1          3806  3804 (L-TLB)
00000100702275b8 0000000000000046 000001003543ed40 ffffffff801792ac 
       0000000000000016 0000000000000212 <ffffffff801a48af>{dqput+136} 
       <ffffffffa0342b49>{:lquota:split_before_schedule_dqacq+248} 
       <ffffffffa034395b>{:lquota:qctxt_adjust_qunit+333} 
       <ffffffffa0440dd4>{:fsfilt_ldiskfs:fsfilt_ldiskfs_map_ext_inode_pages+457} 
       <ffffffffa0346667>{:lquota:filter_quota_acquire+122} 
       <ffffffffa04ee325>{:obdfilter:filter_direct_io+1281} 
       <ffffffffa043f6f2>{:fsfilt_ldiskfs:fsfilt_ldiskfs_brw_start+649} 
       <ffffffffa04ef77d>{:obdfilter:filter_commitrw_write+3494} 
       000716b1e784aa55 000000017b026030 
       0000010070099800 00000000000004d0 
Call Trace:<ffffffff801792ac>{__find_get_block+396} <ffffffffa034286a>{:lquota:schedule_dqacq+2775} 
       <ffffffffa03f6515>{:ldiskfs:ldiskfs_ext_find_extent+500} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffff801a48af>{dqput+136} 
       <ffffffffa0342b49>{:lquota:split_before_schedule_dqacq+248} 
       <ffffffffa034395b>{:lquota:qctxt_adjust_qunit+333} 
       <ffffffffa0440dd4>{:fsfilt_ldiskfs:fsfilt_ldiskfs_map_ext_inode_pages+457} 
       <ffffffffa0346667>{:lquota:filter_quota_acquire+122} 
       <ffffffffa04ee325>{:obdfilter:filter_direct_io+1281} 
       <ffffffffa043f6f2>{:fsfilt_ldiskfs:fsfilt_ldiskfs_brw_start+649} 
       <ffffffffa04ef77d>{:obdfilter:filter_commitrw_write+3494} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80308f91>{thread_return+88} 
       <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} <ffffffff8013f23f>{del_timer+107} 
       <ffffffff8013f2fc>{del_singleshot_timer_sync+9} <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80309a63>{schedule_timeout+375} 
       <ffffffffa04c6db1>{:ost:ost_brw_write+5253} <ffffffff802a97a7>{alloc_skb+92} 
       <ffffffff801331a5>{default_wake_function+0}<ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
        
       <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
       <ffffffff80134b6b>{autoremove_wake_function+9} <ffffffff801331f6>{__wake_up_common+67} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132384>{move_tasks+200} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff802af3b8>{netif_receive_skb+791}<ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0}  <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
<ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3768: it was inactive for 100s
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Skipped 1 previous similar message
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3768
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) Skipped 1 previous similar message
ll_ost_15     D 00000000772215a8     0  3768      1          3769  3767 (L-TLB)
LustreError: 3699:0:(client.c:950:ptlrpc_expire_one_request()) @@@ timeout (sent at 1191932722, 100s ago) req@000001007e122600 x88718429/t0 o19->home-OST0000_UUID@192.168.0.24@tcp:28 lens 240/240 ref 
1 fl Rpc:/0/0 rc 0/-22
Lustre: home-OST0000-osc: Connection to service home-OST0000 via nid 0@lo was lost; in progress operations using this service will wait for recovery to complete.
Lustre: 3767:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Lustre: 3767:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
LustreError: 3767:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001006c86ba00 x88718435/t0 o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl 
Interpret:/0/0 rc -16/0
LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@000001006c86b400 x88718435/t0 o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl 
Rpc:R/0/0 rc 0/-16
000001006fee7908 0000000000000046 00000100010287e0 0000000000000001 
       000000016fee78bc 0000000000000000 0000000000000012 0000000074c4dec8 
       000001006cf0d030 0000000000000473 
Call Trace:<ffffffff80134b6b>{autoremove_wake_function+9} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffffa031253e>{:ksocklnd:ksocknal_queue_tx_locked+527} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f1850>{:ldiskfs:ldiskfs_acquire_dquot+46} 
       <ffffffff801a4efa>{dqget+710} <ffffffff801a695f>{vfs_get_dqblk+75} 
       <ffffffffa0441e21>{:fsfilt_ldiskfs:fsfilt_ldiskfs_quotactl+2032} 
       <ffffffffa0344a0d>{:lquota:filter_quota_ctl+346} <ffffffffa02b0dc5>{:ptlrpc:lustre_pack_reply+1928} 
       <ffffffffa04c8083>{:ost:ost_handle_quotactl+983} <ffffffffa04cbdda>{:ost:ost_handle+13349} 
       <ffffffff802a97a7>{alloc_skb+92} <ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
       <ffffffffa00b5385>{:e1000:e1000_unmap_and_free_tx_resource+213} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
LustreError: dumping log to /tmp/lustre-log.1191932822.3768
LustreError: dumping log to /tmp/lustre-log.1191932822.16208
LustreError: dumping log to /tmp/lustre-log.1191932822.3805
LustreError: dumping log to /tmp/lustre-log.1191932822.3699
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 16196: it was inactive for 100s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 16196
ll_ost_io_53  D 0000000077221621     0 16196      1         16197 16195 (L-TLB)
000001003d94f748 0000000000000046 0000000000000000 ffffffff80308f39 
       000001003d94f768 ffffffff80308f91 ffffffff80366500 000000008024e0f7 
       0000010049ce4030 0000000000000157 
Call Trace:<ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
       <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
       <ffffffff80134b6b>{autoremove_wake_function+9} <ffffffff801331f6>{__wake_up_common+67} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132384>{move_tasks+200} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
LustreError: dumping log to /tmp/lustre-log.1191932822.16196
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3813: it was inactive for 100s
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3813
ll_ost_io_28  D 00000000772216e4     0  3813      1          3814  3812 (L-TLB)
000001006a8a1748 0000000000000046 000001006fd381f0 0000000000000000 
       000716b1e7940d67 000001007c655030 00000100010287e0 0000000000000001 
       000001006b2b6800 0000000000000130 
Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
       <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffff801331f6>{__wake_up_common+67} <ffffffffa01f61d0>{:lnet:lnet_match_blocked_msg+60} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
ll_ost_io_32  <1>LustreError: dumping log to /tmp/lustre-log.1191932822.3813
D 00000000772216ee     0 16175      1         16176 13752 (L-TLB)
000001002874b748 0000000000000046 000001006fd381f0 0000000000000000 
       000716b1e7a67699 000001007c655030 00000100010287e0 0000000000000001 
       000001006e305800 0000000000000127 
Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
       <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
       <ffffffff80134b6b>{autoremove_wake_function+9} ll_ost_io_41  D 0000000077221712     0 16184      1         16185 16183 (L-TLB)
000001005d079748 0000000000000046 0000000000000048 0000000000000001 
       000716b1e7ae9143 000001007c655030 00000100010287e0 0000000100000001 
       000001003b6eb030 000000000000017d 
Call Trace:<ffffffff801331f6>{__wake_up_common+67} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
       <ffffffff80110e1b>{child_rip+0} 
ll_ost_io_44  D 000000007722170e     0 16187      1         16188 16186 (L-TLB)
00000100121db748 0000000000000046 000001006fd381f0 0000000000000246 
       00000100121dbfd8 0000000000000001 0000010077e76e00 0000000000000016 
       000001007093f800 0000000000000128 
Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
       <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
       <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
       <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
       <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
       <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
       <ffffffff80308f39>{thread_return+0} <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
       <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
       <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
       <ffffffff80308f91>{thread_return+88} 
       <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} <ffffffff8013f23f>{del_timer+107} 
       <ffffffff8013f2fc>{del_singleshot_timer_sync+9} <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
       <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
       <ffffffff801331f6>{__wake_up_common+67} <ffffffff8013324c>{__wake_up+54} 
       <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff80309a63>{schedule_timeout+375} 
       <ffffffffa04c6db1>{:ost:ost_brw_write+5253} <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0}<ffffffff801331a5>{default_wake_function+0} 
       <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
       <ffffffffa04ca500>{:ost:ost_handle+6987}  <ffffffff80110e1b>{child_rip+0} 
       
<ffffffff802a97a7>{alloc_skb+92} 
       <ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
       <ffffffff802af3b8>{netif_receive_skb+791} <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
       <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
LustreError: dumping log to /tmp/lustre-log.1191932822.16184
LustreError: dumping log to /tmp/lustre-log.1191932822.16187
LustreError: dumping log to /tmp/lustre-log.1191932822.16175
Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@0000010077e80800 x88718437/t0 o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl 
Interpret:/0/0 rc -16/0
LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@000001007dfd6800 x88718437/t0 o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl 
Rpc:R/0/0 rc 0/-16
Lustre: 3679:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Lustre: 3679:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 0x00000100631c0000/2
LustreError: 3679:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001007dcd6e00 x125395419/t0 o38->f9f8a421-c76c-a905-a4d6-22565b52c6d0@NET_0x2000081ad7644_UUID:-1 lens 
304/200 ref 0 fl Interpret:/0/0 rc -16/0
LustreError: 3489:0:(client.c:950:ptlrpc_expire_one_request()) @@@ timeout (sent at 1191932722, 200s ago) req@000001007dfd6a00 x88718427/t0 o601->@192.168.0.24@tcp:15 lens 144/144 ref 1 fl Rpc:/0/0 
rc 0/-22
Lustre: 3767:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Lustre: 3767:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010077e80400 x88718440/t0 o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl 
Rpc:R/0/0 rc 0/-16
Lustre: 3772:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Lustre: 3772:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 0x0000010060dd9000/9
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3600: it was inactive for 18s
Lustre: 0:0:(watchdog.c:130:lcw_cb()) Skipped 3 previous similar messages
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3600
Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) Skipped 3 previous similar messages
ldlm_cb_01    D 0000000000000001     0  3600      1          3601  3599 (L-TLB)
0000010074be9c48 0000000000000046 0000010077e5b6c0 0000000000000073 
       000001005cb22600 0000000000000000 0000010001029aa0 00000001a01f481a 
       0000010074637800 00000000000003ed 
Call Trace:<ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
       <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
       <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
       <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
       <ffffffff80110e1b>{child_rip+0} 
ldlm_cb_04    D 0000000000000001     0  3603      1          3604  3602 (L-TLB)
0000010073cfbc48 0000000000000046 0000010077e5b6c0 0000000000000073 
       0000010017923800 0000000000000000 0000010001029aa0 00000001a01f481a 
       000001007462f030 00000000000001b4 
Call Trace:<1>LustreError: dumping log to /tmp/lustre-log.1191932940.3600
<ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
       <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
       <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
       <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
       <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
       <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
       <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
       <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
       <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
       
LustreError: dumping log to /tmp/lustre-log.1191932940.3603
Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001006a7da050 x88718443/t0 o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl 
Interpret:/0/0 rc -16/0
LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) Skipped 2 previous similar messages
LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010074af1600 x88718443/t0 o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl 
Rpc:R/0/0 rc 0/-16
Lustre: 3701:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 0x00000100631c0000/2
Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) Skipped 1 previous similar message
[root@lustre-3ware ~]# tail -200 /var/log/messages
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.3768
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16208
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.3805
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.3699
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 16196: it was inactive for 100s
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 16196
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_53  D 0000000077221621     0 16196      1         16197 16195 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001003d94f748 0000000000000046 0000000000000000 ffffffff80308f39 
Oct  9 09:27:02 lustre-3ware kernel:        000001003d94f768 ffffffff80308f91 ffffffff80366500 000000008024e0f7 
Oct  9 09:27:02 lustre-3ware kernel:        0000010049ce4030 0000000000000157 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b6b>{autoremove_wake_function+9} <ffffffff801331f6>{__wake_up_common+67} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132384>{move_tasks+200} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16196
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3813: it was inactive for 100s
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3813
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_28  D 00000000772216e4     0  3813      1          3814  3812 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001006a8a1748 0000000000000046 000001006fd381f0 0000000000000000 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7940d67 000001007c655030 00000100010287e0 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001006b2b6800 0000000000000130 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331f6>{__wake_up_common+67} <ffffffffa01f61d0>{:lnet:lnet_match_blocked_msg+60} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_32  <1>LustreError: dumping log to /tmp/lustre-log.1191932822.3813
Oct  9 09:27:02 lustre-3ware kernel: D 00000000772216ee     0 16175      1         16176 13752 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001002874b748 0000000000000046 000001006fd381f0 0000000000000000 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7a67699 000001007c655030 00000100010287e0 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001006e305800 0000000000000127 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b6b>{autoremove_wake_function+9} ll_ost_io_41  D 0000000077221712     0 16184      1         16185 16183 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001005d079748 0000000000000046 0000000000000048 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7ae9143 000001007c655030 00000100010287e0 0000000100000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001003b6eb030 000000000000017d 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff801331f6>{__wake_up_common+67} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_44  D 000000007722170e     0 16187      1         16188 16186 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 00000100121db748 0000000000000046 000001006fd381f0 0000000000000246 
Oct  9 09:27:02 lustre-3ware kernel:        00000100121dbfd8 0000000000000001 0000010077e76e00 0000000000000016 
Oct  9 09:27:02 lustre-3ware kernel:        000001007093f800 0000000000000128 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} <ffffffff8013f23f>{del_timer+107} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f2fc>{del_singleshot_timer_sync+9} <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331f6>{__wake_up_common+67} <ffffffff8013324c>{__wake_up+54} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04c6db1>{:ost:ost_brw_write+5253} <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0}<ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987}  <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: <ffffffff802a97a7>{alloc_skb+92} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff802af3b8>{netif_receive_skb+791} <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16184
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16187
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16175
Oct  9 09:27:52 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:27:52 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:27:52 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@0000010077e80800 x88718437/t0 
o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:27:52 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@000001007dfd6800 x88718437/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3679:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3679:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x00000100631c0000/2
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3679:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001007dcd6e00 x125395419/t0 
o38->f9f8a421-c76c-a905-a4d6-22565b52c6d0@NET_0x2000081ad7644_UUID:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3489:0:(client.c:950:ptlrpc_expire_one_request()) @@@ timeout (sent at 1191932722, 200s ago) req@000001007dfd6a00 x88718427/t0 
o601->@192.168.0.24@tcp:15 lens 144/144 ref 1 fl Rpc:/0/0 rc 0/-22
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3767:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3767:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010077e80400 x88718440/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3772:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3772:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x0000010060dd9000/9
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3600: it was inactive for 18s
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Skipped 3 previous similar messages
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3600
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) Skipped 3 previous similar messages
Oct  9 09:29:00 lustre-3ware kernel: ldlm_cb_01    D 0000000000000001     0  3600      1          3601  3599 (L-TLB)
Oct  9 09:29:00 lustre-3ware kernel: 0000010074be9c48 0000000000000046 0000010077e5b6c0 0000000000000073 
Oct  9 09:29:00 lustre-3ware kernel:        000001005cb22600 0000000000000000 0000010001029aa0 00000001a01f481a 
Oct  9 09:29:00 lustre-3ware kernel:        0000010074637800 00000000000003ed 
Oct  9 09:29:00 lustre-3ware kernel: Call Trace:<ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:29:00 lustre-3ware kernel: ldlm_cb_04    D 0000000000000001     0  3603      1          3604  3602 (L-TLB)
Oct  9 09:29:00 lustre-3ware kernel: 0000010073cfbc48 0000000000000046 0000010077e5b6c0 0000000000000073 
Oct  9 09:29:00 lustre-3ware kernel:        0000010017923800 0000000000000000 0000010001029aa0 00000001a01f481a 
Oct  9 09:29:00 lustre-3ware kernel:        000001007462f030 00000000000001b4 
Oct  9 09:29:00 lustre-3ware kernel: Call Trace:<1>LustreError: dumping log to /tmp/lustre-log.1191932940.3600
Oct  9 09:29:00 lustre-3ware kernel: <ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:29:00 lustre-3ware kernel:        
Oct  9 09:29:00 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932940.3603
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001006a7da050 x88718443/t0 
o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) Skipped 2 previous similar messages
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010074af1600 x88718443/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x00000100631c0000/2
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) Skipped 1 previous similar message
Oct  9 09:29:53 lustre-3ware sshd(pam_unix)[5261]: session opened for user root by root(uid=0)
[root@lustre-3ware ~]# tail -200 /var/log/messages
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16208
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.3805
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.3699
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 16196: it was inactive for 100s
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 16196
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_53  D 0000000077221621     0 16196      1         16197 16195 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001003d94f748 0000000000000046 0000000000000000 ffffffff80308f39 
Oct  9 09:27:02 lustre-3ware kernel:        000001003d94f768 ffffffff80308f91 ffffffff80366500 000000008024e0f7 
Oct  9 09:27:02 lustre-3ware kernel:        0000010049ce4030 0000000000000157 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b6b>{autoremove_wake_function+9} <ffffffff801331f6>{__wake_up_common+67} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132384>{move_tasks+200} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16196
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3813: it was inactive for 100s
Oct  9 09:27:02 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3813
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_28  D 00000000772216e4     0  3813      1          3814  3812 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001006a8a1748 0000000000000046 000001006fd381f0 0000000000000000 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7940d67 000001007c655030 00000100010287e0 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001006b2b6800 0000000000000130 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331f6>{__wake_up_common+67} <ffffffffa01f61d0>{:lnet:lnet_match_blocked_msg+60} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_32  <1>LustreError: dumping log to /tmp/lustre-log.1191932822.3813
Oct  9 09:27:02 lustre-3ware kernel: D 00000000772216ee     0 16175      1         16176 13752 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001002874b748 0000000000000046 000001006fd381f0 0000000000000000 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7a67699 000001007c655030 00000100010287e0 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001006e305800 0000000000000127 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80131b1a>{try_to_wake_up+876} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b6b>{autoremove_wake_function+9} ll_ost_io_41  D 0000000077221712     0 16184      1         16185 16183 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 000001005d079748 0000000000000046 0000000000000048 0000000000000001 
Oct  9 09:27:02 lustre-3ware kernel:        000716b1e7ae9143 000001007c655030 00000100010287e0 0000000100000001 
Oct  9 09:27:02 lustre-3ware kernel:        000001003b6eb030 000000000000017d 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff801331f6>{__wake_up_common+67} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel: ll_ost_io_44  D 000000007722170e     0 16187      1         16188 16186 (L-TLB)
Oct  9 09:27:02 lustre-3ware kernel: 00000100121db748 0000000000000046 000001006fd381f0 0000000000000246 
Oct  9 09:27:02 lustre-3ware kernel:        00000100121dbfd8 0000000000000001 0000010077e76e00 0000000000000016 
Oct  9 09:27:02 lustre-3ware kernel:        000001007093f800 0000000000000128 
Oct  9 09:27:02 lustre-3ware kernel: Call Trace:<ffffffff80131551>{recalc_task_prio+337} <ffffffffa005e381>{:jbd:start_this_handle+897} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80131b1a>{try_to_wake_up+876} <ffffffff80134b62>{autoremove_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80134b62>{autoremove_wake_function+0} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa005e51e>{:jbd:journal_start+223} <ffffffffa03f174c>{:ldiskfs:ldiskfs_dquot_initialize+27} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04eed9d>{:obdfilter:filter_commitrw_write+966} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f39>{thread_return+0} <ffffffff80308f39>{thread_return+0} <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f5bfe>{:lnet:lnet_send+2251} <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f23f>{del_timer+107} <ffffffff8013f2fc>{del_singleshot_timer_sync+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80308f91>{thread_return+88} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04e9756>{:obdfilter:filter_commitrw+84} <ffffffff8013f23f>{del_timer+107} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f2fc>{del_singleshot_timer_sync+9} <ffffffff80309a63>{schedule_timeout+375} <ffffffffa04c6db1>{:ost:ost_brw_write+5253} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987} <ffffffff80134b6b>{autoremove_wake_function+9} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331f6>{__wake_up_common+67} <ffffffff8013324c>{__wake_up+54} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80309a63>{schedule_timeout+375} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04c6db1>{:ost:ost_brw_write+5253} <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0}<ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04c3513>{:ost:ost_bulk_timeout+0} <ffffffffa02b1471>{:ptlrpc:lustre_msg_get_version+64} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa04ca500>{:ost:ost_handle+6987}  <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: <ffffffff802a97a7>{alloc_skb+92} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa00ba491>{:e1000:e1000_alloc_rx_buffers+641} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff802af3b8>{netif_receive_skb+791} <ffffffffa01f64b5>{:lnet:lnet_match_blocked_msg+801} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff80132452>{move_tasks+406} <ffffffffa0239a51>{:obdclass:class_handle2object+207} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:27:02 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:27:02 lustre-3ware kernel:        
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16184
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16187
Oct  9 09:27:02 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932822.16175
Oct  9 09:27:52 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:27:52 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:27:52 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@0000010077e80800 x88718437/t0 
o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:27:52 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@000001007dfd6800 x88718437/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3679:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3679:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x00000100631c0000/2
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3679:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001007dcd6e00 x125395419/t0 
o38->f9f8a421-c76c-a905-a4d6-22565b52c6d0@NET_0x2000081ad7644_UUID:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3489:0:(client.c:950:ptlrpc_expire_one_request()) @@@ timeout (sent at 1191932722, 200s ago) req@000001007dfd6a00 x88718427/t0 
o601->@192.168.0.24@tcp:15 lens 144/144 ref 1 fl Rpc:/0/0 rc 0/-22
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3767:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3767:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:28:42 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010077e80400 x88718440/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3772:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:28:42 lustre-3ware kernel: Lustre: 3772:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x0000010060dd9000/9
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Watchdog triggered for pid 3600: it was inactive for 18s
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(watchdog.c:130:lcw_cb()) Skipped 3 previous similar messages
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) showing stack for process 3600
Oct  9 09:29:00 lustre-3ware kernel: Lustre: 0:0:(linux-debug.c:166:libcfs_debug_dumpstack()) Skipped 3 previous similar messages
Oct  9 09:29:00 lustre-3ware kernel: ldlm_cb_01    D 0000000000000001     0  3600      1          3601  3599 (L-TLB)
Oct  9 09:29:00 lustre-3ware kernel: 0000010074be9c48 0000000000000046 0000010077e5b6c0 0000000000000073 
Oct  9 09:29:00 lustre-3ware kernel:        000001005cb22600 0000000000000000 0000010001029aa0 00000001a01f481a 
Oct  9 09:29:00 lustre-3ware kernel:        0000010074637800 00000000000003ed 
Oct  9 09:29:00 lustre-3ware kernel: Call Trace:<ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffff80176ae8>{filp_close+103} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80110e23>{child_rip+8} <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:29:00 lustre-3ware kernel: ldlm_cb_04    D 0000000000000001     0  3603      1          3604  3602 (L-TLB)
Oct  9 09:29:00 lustre-3ware kernel: 0000010073cfbc48 0000000000000046 0000010077e5b6c0 0000000000000073 
Oct  9 09:29:00 lustre-3ware kernel:        0000010017923800 0000000000000000 0000010001029aa0 00000001a01f481a 
Oct  9 09:29:00 lustre-3ware kernel:        000001007462f030 00000000000001b4 
Oct  9 09:29:00 lustre-3ware kernel: Call Trace:<1>LustreError: dumping log to /tmp/lustre-log.1191932940.3600
Oct  9 09:29:00 lustre-3ware kernel: <ffffffff80308223>{__down+147} <ffffffff801331a5>{default_wake_function+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff80309cbb>{__down_failed+53} <ffffffffa034cccc>{:lquota:.text.lock.quota_master+15} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02964b6>{:ptlrpc:target_handle_dqacq_callback+953} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa029fec8>{:ptlrpc:ldlm_callback_handler+1486} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b7e57>{:ptlrpc:ptlrpc_server_handle_request+2528} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff8013f100>{__mod_timer+293} <ffffffffa02b9d1b>{:ptlrpc:ptlrpc_main+2018} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffff801331a5>{default_wake_function+0} <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b897a>{:ptlrpc:ptlrpc_retry_rqbds+0} <ffffffff80110e23>{child_rip+8} 
Oct  9 09:29:00 lustre-3ware kernel:        <ffffffffa02b9539>{:ptlrpc:ptlrpc_main+0} <ffffffff80110e1b>{child_rip+0} 
Oct  9 09:29:00 lustre-3ware kernel:        
Oct  9 09:29:00 lustre-3ware kernel: LustreError: dumping log to /tmp/lustre-log.1191932940.3603
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3761:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) @@@ processing error (-16) req@000001006a7da050 x88718443/t0 
o8->home-mdtlov_UUID@192.168.0.24@tcp:-1 lens 304/200 ref 0 fl Interpret:/0/0 rc -16/0
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3761:0:(ldlm_lib.c:1363:target_send_reply_msg()) Skipped 2 previous similar messages
Oct  9 09:29:32 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@0000010074af1600 x88718443/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) home-MDT0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x00000100631c0000/2
Oct  9 09:29:32 lustre-3ware kernel: Lustre: 3701:0:(ldlm_lib.c:709:target_handle_connect()) Skipped 1 previous similar message
Oct  9 09:29:53 lustre-3ware sshd(pam_unix)[5261]: session opened for user root by root(uid=0)
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3762:0:(ldlm_lib.c:497:target_handle_reconnect()) home-OST0000: home-mdtlov_UUID reconnecting
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3762:0:(ldlm_lib.c:497:target_handle_reconnect()) Skipped 1 previous similar message
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3762:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from home-mdtlov_UUID@0@lo to 0x0000010073ab0000/2
Oct  9 09:30:22 lustre-3ware kernel: LustreError: 3490:0:(client.c:574:ptlrpc_check_status()) @@@ type == PTL_RPC_MSG_ERR, err == -16 req@00000100741b3c00 x88718446/t0 
o8->home-OST0000_UUID@192.168.0.24@tcp:28 lens 304/328 ref 1 fl Rpc:R/0/0 rc 0/-16
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3695:0:(ldlm_lib.c:497:target_handle_reconnect()) home-MDT0000: f9f8a421-c76c-a905-a4d6-22565b52c6d0 reconnecting
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3760:0:(ldlm_lib.c:709:target_handle_connect()) home-OST0000: refuse reconnection from f9f8a421-c76c-a905-a4d6-22565b52c6d0@129.173.118.68@tcp to 
0x0000010060dd9000/9
Oct  9 09:30:22 lustre-3ware kernel: Lustre: 3695:0:(ldlm_lib.c:497:target_handle_reconnect()) Skipped 1 previous similar message
[root@lustre-3ware ~]#