00000400:02000000:1:1198000284.340092:0:5462:0:(api-ni.c:1106:lnet_startup_lndnis()) Added LNI 192.168.64.71@o2ib [8/64]
00000020:01200004:2:1198000284.705739:0:5445:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff81041517a400
00000020:01000004:2:1198000284.705748:0:5445:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:2:1198000284.705749:0:5445:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sdb
00000020:01000004:2:1198000284.705751:0:5445:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:2:1198000284.705753:0:5445:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sdb
00000020:01000004:2:1198000284.705757:0:5445:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sdb
00000020:01000004:0:1198000285.205722:0:5445:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000285.221148:0:5445:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000285.221150:0:5445:0:(obd_mount.c:220:ldd_print()) server:  data-OST0000
00000020:01000004:0:1198000285.221152:0:5445:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000285.221153:0:5445:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000285.221154:0:5445:0:(obd_mount.c:223:ldd_print()) index:   0000
00000020:01000004:0:1198000285.221156:0:5445:0:(obd_mount.c:224:ldd_print()) config:  16
00000020:01000004:0:1198000285.221157:0:5445:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000285.221158:0:5445:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000285.221160:0:5445:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000285.221161:0:5445:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000285.221162:0:5445:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000285.224602:0:5445:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sdb errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000285.286677:0:5445:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0000 for fs 'data' on device /dev/sdb
00000020:01000004:0:1198000285.286761:0:5445:0:(obd_mount.c:602:lustre_start_mgc()) Start MGC 'MGC192.168.64.70@o2ib'
00000020:00000080:0:1198000285.286766:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf005
00000020:00000080:0:1198000285.286769:0:5445:0:(obd_config.c:736:class_process_config()) adding mapping from uuid MGC192.168.64.70@o2ib_0 to nid 0x50000c0a84046 (192.168.64.70@o2ib)
00000020:01000004:0:1198000285.286793:0:5445:0:(obd_mount.c:445:lustre_start_simple()) Starting obd MGC192.168.64.70@o2ib (typ=mgc)
00000020:00000080:0:1198000285.286795:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000285.286798:0:5445:0:(obd_config.c:152:class_attach()) attach type mgc name: MGC192.168.64.70@o2ib uuid: ca545541-db09-7bea-14e1-b70fac494af9
00000020:00000080:0:1198000285.320666:0:5445:0:(genops.c:283:class_newdev()) Adding new device MGC192.168.64.70@o2ib (ffff810411444038)
00000020:00000080:0:1198000285.320672:0:5445:0:(obd_config.c:237:class_attach()) OBD: dev 0 attached type mgc with refcount 1
00000020:00000080:0:1198000285.320678:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00010000:00080000:0:1198000285.323871:0:5445:0:(ldlm_lib.c:96:import_set_conn()) imp ffff810410da6800@MGC192.168.64.70@o2ib: add connection MGC192.168.64.70@o2ib_0 at head
00000020:00000080:0:1198000285.323881:0:5445:0:(obd_config.c:320:class_setup()) finished setup of obd MGC192.168.64.70@o2ib (uuid ca545541-db09-7bea-14e1-b70fac494af9)
10000000:01000000:0:1198000285.323888:0:5445:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/1:d0:i0:r0:or0:NEW
00000020:00000080:0:1198000285.323897:0:5445:0:(genops.c:877:class_connect()) connect: client ca545541-db09-7bea-14e1-b70fac494af9, cookie 0x3eb8e9cf2b4bad48
00000100:00080000:0:1198000285.323959:0:5445:0:(import.c:414:ptlrpc_connect_import()) ffff810410da6800 MGS: changing import state from NEW to CONNECTING
00000100:00080000:0:1198000285.323964:0:5445:0:(import.c:291:import_select_connection()) MGC192.168.64.70@o2ib: connect to NID 192.168.64.70@o2ib last attempt 0
00000100:00080000:0:1198000285.323967:0:5445:0:(import.c:372:import_select_connection()) MGC192.168.64.70@o2ib: import ffff810410da6800 using connection MGC192.168.64.70@o2ib_0/192.168.64.70@o2ib
00000100:00080000:0:1198000285.323969:0:5445:0:(import.c:443:ptlrpc_connect_import()) Last connection attempt (1) for MGS
10000000:00080000:0:1198000285.323971:0:5445:0:(mgc_request.c:825:mgc_set_info_async()) MGC192.168.64.70@o2ib: set imp_initial_recov = 0
00000100:00080000:0:1198000285.323973:0:5445:0:(import.c:453:ptlrpc_connect_import()) Last reconnection attempt (1) for MGS
00000100:00080000:0:1198000285.323984:0:5445:0:(pinger.c:280:ptlrpc_pinger_add_import()) adding pingable import ca545541-db09-7bea-14e1-b70fac494af9->MGS
00000020:01000004:0:1198000285.323991:0:5445:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0000
00000020:01000004:0:1198000285.324029:0:5445:0:(obd_mount.c:445:lustre_start_simple()) Starting obd OSS (typ=ost)
00000020:00000080:0:1198000285.324032:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000285.324035:0:5445:0:(obd_config.c:152:class_attach()) attach type ost name: OSS uuid: OSS_uuid
00000800:00000400:3:1198000285.326913:0:3917:0:(o2iblnd_cb.c:2299:kiblnd_passive_connect()) Conn race 192.168.64.70@o2ib
00000400:00000100:3:1198000285.328877:0:5566:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 428 offset 0 length 304: 2
00000400:00000100:3:1198000285.329010:0:5566:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 429 offset 0 length 304: 2
00000400:00000100:3:1198000285.329016:0:5566:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 430 offset 0 length 304: 2
00000400:00000100:3:1198000285.329028:0:5565:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 432 offset 0 length 304: 2
00000400:00000100:3:1198000285.329034:0:5565:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 433 offset 0 length 304: 2
00000400:00000100:0:1198000285.329039:0:5563:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 431 offset 0 length 304: 2
00000400:00000100:3:1198000285.329046:0:5565:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 436 offset 0 length 304: 2
00000400:00000100:3:1198000285.329051:0:5565:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 437 offset 0 length 304: 2
00000400:00000100:3:1198000285.329129:0:5564:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 438 offset 0 length 304: 2
00000400:00000100:3:1198000285.329150:0:5563:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 439 offset 0 length 304: 2
00000400:00000100:3:1198000285.329154:0:5563:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 440 offset 0 length 304: 2
00000400:00000100:3:1198000285.329158:0:5563:0:(lib-move.c:1647:lnet_parse_put()) Dropping PUT from 12345-192.168.64.70@o2ib portal 28 match 441 offset 0 length 304: 2
00000100:00080000:3:1198000285.329185:0:5569:0:(import.c:602:ptlrpc_connect_interpret()) ffff810410da6800 MGS: changing import state from CONNECTING to FULL
10000000:01000000:3:1198000285.329195:0:5569:0:(mgc_request.c:900:mgc_import_event()) import event 0x808005
00000020:00000080:0:1198000285.359462:0:5445:0:(genops.c:283:class_newdev()) Adding new device OSS (ffff81041504c078)
00000020:00000080:0:1198000285.359466:0:5445:0:(obd_config.c:237:class_attach()) OBD: dev 1 attached type ost with refcount 1
00000020:00000080:0:1198000285.359469:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:00000080:0:1198000285.376103:0:5445:0:(obd_config.c:320:class_setup()) finished setup of obd OSS (uuid OSS_uuid)
00000020:01000004:0:1198000285.376109:0:5445:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sdb
00000100:00080000:0:1198000285.376125:0:6021:0:(pinger.c:360:ping_evictor_main()) Starting Ping Evictor
10000000:01000000:0:1198000285.404349:0:5445:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0000
00000020:01000004:0:1198000285.404366:0:5445:0:(obd_mount.c:951:server_register_target()) Registration data-OST0000, fs=data, 192.168.64.71@o2ib, index=0000, flags=0x2
10000000:01000000:0:1198000285.404369:0:5445:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0000 0x2
10000000:01000000:0:1198000285.404386:0:5445:0:(mgc_request.c:732:mgc_target_register()) register data-OST0000
10000000:01000000:0:1198000285.405078:0:5445:0:(mgc_request.c:741:mgc_target_register()) register data-OST0000 got index = 0
00000020:01000004:0:1198000285.405090:0:5445:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff81042ffc54c0 from data-OST0000, vfscount=1
10000000:01000000:0:1198000285.405096:0:5445:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0000 from 0
10000000:01000000:0:1198000285.405097:0:5445:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0000:<NULL>
10000000:01000000:0:1198000285.405101:0:5445:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0000 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000285.405105:0:5445:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0000:<NULL> from 1
10000000:01000000:0:1198000285.405107:0:5445:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0000 (res 0x61746164)
10000000:01000000:0:1198000285.406130:0:5445:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0000 (0)
00000020:01000000:0:1198000285.406148:0:5445:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000285.406151:0:5445:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0000 (0) in 0 of /dev/sdb
00000020:00000080:0:1198000285.406161:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000285.406163:0:5445:0:(obd_config.c:785:class_process_config()) marker 5 (0x1) data-OST0000 add ost
00000020:00000080:0:1198000285.406166:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000285.406168:0:5445:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0000 uuid: data-OST0000_UUID
00000020:00000080:0:1198000285.477973:0:5445:0:(genops.c:283:class_newdev()) Adding new device data-OST0000 (ffff8104076040b8)
00000020:00000080:0:1198000285.477978:0:5445:0:(obd_config.c:237:class_attach()) OBD: dev 2 attached type obdfilter with refcount 1
00000020:00000080:0:1198000285.477987:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000285.478064:0:5445:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff81042ffc54c0 from data-OST0000, refs=2, vfscount=4
00002000:00080000:0:1198000285.481015:0:5445:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 14154078 srv lr: 14154078
00002000:00080000:0:1198000285.481024:0:5445:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 14154078
00002000:00000400:0:1198000285.481028:0:5445:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0000, 2 recoverable clients, last_rcvd 14154078
00002000:00080000:0:1198000285.543206:0:5445:0:(filter.c:983:filter_prep_groups()) data-OST0000: server last_objid group 0: 1092371
00002000:00080000:0:1198000285.562735:0:5445:0:(filter.c:983:filter_prep_groups()) data-OST0000: server last_objid group 1: 0
00002000:00080000:0:1198000285.563091:0:5445:0:(filter.c:983:filter_prep_groups()) data-OST0000: server last_objid group 2: 0
00000040:01000000:0:1198000285.563449:0:5445:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000285.563549:0:6044:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000285.563673:0:5445:0:(filter.c:1775:filter_common_setup()) OST data-OST0000 now serving dev (data-OST0000/07337183-9721-be08-4563-bd138b8f0ad5), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0000/recovery_status.
00000020:00000080:0:1198000285.564790:0:5445:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0000 (uuid data-OST0000_UUID)
00000020:01000000:0:1198000285.564792:0:5445:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000285.564795:0:5445:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000285.564796:0:5445:0:(obd_config.c:785:class_process_config()) marker 5 (0x2) data-OST0000 add ost
00000020:01000000:0:1198000285.564801:0:5445:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0000 gen 1-4 (rc=0)
10000000:01000000:0:1198000285.564814:0:5445:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0000' succeeded (0).
00000020:01000004:0:1198000285.564819:0:5445:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000285.564826:0:5445:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=24
00000020:02000400:0:1198000285.564828:0:5445:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0000 on device /dev/sdb has started
00000020:01200004:3:1198000285.569076:0:6047:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff81042e5e2c00
00000020:01000004:3:1198000285.569083:0:6047:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:3:1198000285.569084:0:6047:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sdc
00000020:01000004:3:1198000285.569086:0:6047:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:3:1198000285.569087:0:6047:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sdc
00000020:01000004:3:1198000285.569093:0:6047:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sdc
00000020:01000004:0:1198000285.911009:0:6047:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000285.937131:0:6047:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000285.937133:0:6047:0:(obd_mount.c:220:ldd_print()) server:  data-OST0001
00000020:01000004:0:1198000285.937134:0:6047:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000285.937136:0:6047:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000285.937138:0:6047:0:(obd_mount.c:223:ldd_print()) index:   0001
00000020:01000004:0:1198000285.937139:0:6047:0:(obd_mount.c:224:ldd_print()) config:  14
00000020:01000004:0:1198000285.937141:0:6047:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000285.937142:0:6047:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000285.937144:0:6047:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000285.937145:0:6047:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000285.937147:0:6047:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000285.940593:0:6047:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sdc errors=remount-ro,extents,mballoc
00010000:00080000:0:1198000285.971980:0:5760:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff8104079c6000, cookie 0x3eb8e9cf2b4bad6b
00010000:00000400:0:1198000285.971984:0:5760:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0000: starting recovery timer
00010000:00080000:0:1198000285.972802:0:5760:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0000: recovery timer will expire in 200 seconds
00010000:02020000:0:1198000285.973036:0:5761:0:(ldlm_lib.c:600:target_handle_connect()) 137-5: UUID 'data-OST0001_UUID' is not available  for connect (no target)
00010000:02020000:3:1198000285.973108:0:5763:0:(ldlm_lib.c:600:target_handle_connect()) 137-5: UUID 'data-OST0002_UUID' is not available  for connect (no target)
00010000:00020000:3:1198000285.973119:0:5763:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-19)  req@ffff8104167b7000 x446/t0 o8-><?>@<?>:0/0 lens 304/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/8/0 rc -19/0
00010000:00080000:3:1198000285.973169:0:5766:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810423ceba00 x447/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:3:1198000285.973169:0:5766:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810423ceba00 x447/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00000400:3:1198000285.973169:0:5766:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0000: 1 recoverable clients remain
00010000:02020000:3:1198000285.973169:0:5762:0:(ldlm_lib.c:600:target_handle_connect()) 137-5: UUID 'data-OST0003_UUID' is not available  for connect (no target)
00010000:00020000:3:1198000285.973169:0:5762:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-19)  req@ffff81041bb52000 x448/t0 o8-><?>@<?>:0/0 lens 304/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/8/0 rc -19/0
00010000:02020000:3:1198000285.973169:0:5764:0:(ldlm_lib.c:600:target_handle_connect()) 137-5: UUID 'data-OST0004_UUID' is not available  for connect (no target)
00010000:00020000:3:1198000285.973169:0:5764:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-19)  req@ffff81040aaaac00 x449/t0 o8-><?>@<?>:0/0 lens 304/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/8/0 rc -19/0
00010000:02020000:3:1198000285.973169:0:5765:0:(ldlm_lib.c:600:target_handle_connect()) 137-5: UUID 'data-OST0005_UUID' is not available  for connect (no target)
00010000:00020000:3:1198000285.973169:0:5765:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-19)  req@ffff81040aaaa800 x450/t0 o8-><?>@<?>:0/0 lens 304/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/8/0 rc -19/0
00010000:00020000:0:1198000285.973684:0:5761:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-19)  req@ffff8104298bc600 x445/t0 o8-><?>@<?>:0/0 lens 304/0 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/8/0 rc -19/0
00000020:01000004:0:1198000286.009951:0:6047:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0001 for fs 'data' on device /dev/sdc
00000020:01000004:0:1198000286.010002:0:6047:0:(obd_mount.c:594:lustre_start_mgc()) MGC192.168.64.70@o2ib: Set MGC reconnect 1
10000000:01000000:0:1198000286.010006:0:6047:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/0:d0:i0:r0:or0:FULL
00000020:01000004:0:1198000286.010009:0:6047:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0001
00000020:01000004:0:1198000286.010010:0:6047:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sdc
10000000:01000000:0:1198000286.010540:0:6047:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0001
00000020:01000004:0:1198000286.010553:0:6047:0:(obd_mount.c:951:server_register_target()) Registration data-OST0001, fs=data, 192.168.64.71@o2ib, index=0001, flags=0x2
10000000:01000000:0:1198000286.010556:0:6047:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0001 0x2
10000000:01000000:0:1198000286.010568:0:6047:0:(mgc_request.c:732:mgc_target_register()) register data-OST0001
10000000:01000000:0:1198000286.010850:0:6047:0:(mgc_request.c:741:mgc_target_register()) register data-OST0001 got index = 1
00000020:01000004:0:1198000286.010862:0:6047:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff810407abcd80 from data-OST0001, vfscount=1
10000000:01000000:0:1198000286.010868:0:6047:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0001 from 0
10000000:01000000:0:1198000286.010870:0:6047:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0001:<NULL>
10000000:01000000:0:1198000286.010875:0:6047:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0001 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000286.010879:0:6047:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0001:<NULL> from 1
10000000:01000000:0:1198000286.010882:0:6047:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0001 (res 0x61746164)
10000000:01000000:0:1198000286.012294:0:6047:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0001 (0)
00000020:01000000:0:1198000286.012311:0:6047:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000286.012314:0:6047:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0001 (1) in 0 of /dev/sdc
00000020:00000080:0:1198000286.012319:0:6047:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000286.012322:0:6047:0:(obd_config.c:785:class_process_config()) marker 8 (0x1) data-OST0001 add ost
00000020:00000080:0:1198000286.012325:0:6047:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000286.012328:0:6047:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0001 uuid: data-OST0001_UUID
00000020:00000080:0:1198000286.012399:0:6047:0:(genops.c:283:class_newdev()) Adding new device data-OST0001 (ffff81040707a0f8)
00000020:00000080:0:1198000286.012402:0:6047:0:(obd_config.c:237:class_attach()) OBD: dev 3 attached type obdfilter with refcount 1
00000020:00000080:0:1198000286.012405:0:6047:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000286.012479:0:6047:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff810407abcd80 from data-OST0001, refs=2, vfscount=4
00002000:00080000:0:1198000286.017650:0:6047:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 13878234 srv lr: 13878234
00002000:00080000:0:1198000286.017661:0:6047:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 13878234
00002000:00000400:0:1198000286.017666:0:6047:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0001, 2 recoverable clients, last_rcvd 13878234
00002000:00080000:0:1198000286.057231:0:6047:0:(filter.c:983:filter_prep_groups()) data-OST0001: server last_objid group 0: 1092501
00002000:00080000:0:1198000286.057579:0:6047:0:(filter.c:983:filter_prep_groups()) data-OST0001: server last_objid group 1: 0
00002000:00080000:0:1198000286.057922:0:6047:0:(filter.c:983:filter_prep_groups()) data-OST0001: server last_objid group 2: 0
00000040:01000000:0:1198000286.058535:0:6047:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000286.058641:0:6065:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000286.058765:0:6047:0:(filter.c:1775:filter_common_setup()) OST data-OST0001 now serving dev (data-OST0001/d7e82116-7f6f-9520-4b23-43950cc2d7ca), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0001/recovery_status.
00000020:00000080:0:1198000286.059350:0:6047:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0001 (uuid data-OST0001_UUID)
00000020:01000000:0:1198000286.059355:0:6047:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000286.059358:0:6047:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000286.059360:0:6047:0:(obd_config.c:785:class_process_config()) marker 8 (0x2) data-OST0001 add ost
00000020:01000000:0:1198000286.059364:0:6047:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0001 gen 1-4 (rc=0)
10000000:01000000:0:1198000286.059377:0:6047:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0001' succeeded (0).
00000020:01000004:0:1198000286.059382:0:6047:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000286.059389:0:6047:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=25
00000020:02000400:0:1198000286.059392:0:6047:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0001 on device /dev/sdc has started
00000020:01200004:0:1198000286.063056:0:6068:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff81040738a800
00000020:01000004:0:1198000286.063062:0:6068:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:0:1198000286.063063:0:6068:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sdd
00000020:01000004:0:1198000286.063065:0:6068:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:0:1198000286.063066:0:6068:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sdd
00000020:01000004:0:1198000286.063072:0:6068:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sdd
00000020:01000004:0:1198000286.455858:0:6068:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000286.467994:0:6068:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000286.467996:0:6068:0:(obd_mount.c:220:ldd_print()) server:  data-OST0002
00000020:01000004:0:1198000286.467998:0:6068:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000286.467999:0:6068:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000286.468000:0:6068:0:(obd_mount.c:223:ldd_print()) index:   0002
00000020:01000004:0:1198000286.468002:0:6068:0:(obd_mount.c:224:ldd_print()) config:  14
00000020:01000004:0:1198000286.468003:0:6068:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000286.468004:0:6068:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000286.468006:0:6068:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000286.468007:0:6068:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000286.468008:0:6068:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000286.472258:0:6068:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sdd errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000286.501187:0:6068:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0002 for fs 'data' on device /dev/sdd
00000020:01000004:0:1198000286.501240:0:6068:0:(obd_mount.c:594:lustre_start_mgc()) MGC192.168.64.70@o2ib: Set MGC reconnect 1
10000000:01000000:0:1198000286.501244:0:6068:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/0:d0:i0:r0:or0:FULL
00000020:01000004:0:1198000286.501249:0:6068:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0002
00000020:01000004:0:1198000286.501251:0:6068:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sdd
10000000:01000000:0:1198000286.516762:0:6068:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0002
00000020:01000004:0:1198000286.516775:0:6068:0:(obd_mount.c:951:server_register_target()) Registration data-OST0002, fs=data, 192.168.64.71@o2ib, index=0002, flags=0x2
10000000:01000000:0:1198000286.516779:0:6068:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0002 0x2
10000000:01000000:0:1198000286.516792:0:6068:0:(mgc_request.c:732:mgc_target_register()) register data-OST0002
10000000:01000000:0:1198000286.518524:0:6068:0:(mgc_request.c:741:mgc_target_register()) register data-OST0002 got index = 2
00000020:01000004:0:1198000286.518537:0:6068:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff810407abddc0 from data-OST0002, vfscount=1
10000000:01000000:0:1198000286.518543:0:6068:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0002 from 0
10000000:01000000:0:1198000286.518544:0:6068:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0002:<NULL>
10000000:01000000:0:1198000286.518549:0:6068:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0002 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000286.518553:0:6068:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0002:<NULL> from 1
10000000:01000000:0:1198000286.518556:0:6068:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0002 (res 0x61746164)
10000000:01000000:0:1198000286.546971:0:6068:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0002 (0)
00000020:01000000:0:1198000286.546992:0:6068:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000286.546995:0:6068:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0002 (2) in 0 of /dev/sdd
00000020:00000080:0:1198000286.547000:0:6068:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000286.547003:0:6068:0:(obd_config.c:785:class_process_config()) marker 11 (0x1) data-OST0002 add ost
00000020:00000080:0:1198000286.547007:0:6068:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000286.547011:0:6068:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0002 uuid: data-OST0002_UUID
00000020:00000080:0:1198000286.547081:0:6068:0:(genops.c:283:class_newdev()) Adding new device data-OST0002 (ffff81040608a138)
00000020:00000080:0:1198000286.547084:0:6068:0:(obd_config.c:237:class_attach()) OBD: dev 4 attached type obdfilter with refcount 1
00000020:00000080:0:1198000286.547087:0:6068:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000286.547161:0:6068:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff810407abddc0 from data-OST0002, refs=2, vfscount=4
00002000:00080000:0:1198000286.550679:0:6068:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 12774721 srv lr: 12774520
00002000:00080000:0:1198000286.550688:0:6068:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 12774721
00002000:00000400:0:1198000286.550692:0:6068:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0002, 2 recoverable clients, last_rcvd 12774721
00002000:00080000:0:1198000286.599811:0:6068:0:(filter.c:983:filter_prep_groups()) data-OST0002: server last_objid group 0: 1092412
00002000:00080000:0:1198000286.600449:0:6068:0:(filter.c:983:filter_prep_groups()) data-OST0002: server last_objid group 1: 0
00002000:00080000:0:1198000286.600547:0:6068:0:(filter.c:983:filter_prep_groups()) data-OST0002: server last_objid group 2: 0
00000040:01000000:0:1198000286.623029:0:6068:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000286.623131:0:6090:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000286.623263:0:6068:0:(filter.c:1775:filter_common_setup()) OST data-OST0002 now serving dev (data-OST0002/42d8e4c9-29c7-8fb9-4a8b-bf0677822c0f), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0002/recovery_status.
00000020:00000080:0:1198000286.624570:0:6068:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0002 (uuid data-OST0002_UUID)
00000020:01000000:0:1198000286.624575:0:6068:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000286.624579:0:6068:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000286.624580:0:6068:0:(obd_config.c:785:class_process_config()) marker 11 (0x2) data-OST0002 add ost
00000020:01000000:0:1198000286.624585:0:6068:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0002 gen 1-4 (rc=0)
10000000:01000000:0:1198000286.624599:0:6068:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0002' succeeded (0).
00000020:01000004:0:1198000286.624603:0:6068:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000286.624610:0:6068:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=26
00000020:02000400:0:1198000286.624613:0:6068:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0002 on device /dev/sdd has started
00000020:01200004:0:1198000286.628188:0:6093:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff8104063ed000
00000020:01000004:0:1198000286.628194:0:6093:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:0:1198000286.628196:0:6093:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sde
00000020:01000004:0:1198000286.628197:0:6093:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:0:1198000286.628199:0:6093:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sde
00000020:01000004:0:1198000286.628205:0:6093:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sde
00000020:01000004:0:1198000287.023559:0:6093:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000287.047139:0:6093:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000287.047142:0:6093:0:(obd_mount.c:220:ldd_print()) server:  data-OST0003
00000020:01000004:0:1198000287.047144:0:6093:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000287.047145:0:6093:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000287.047147:0:6093:0:(obd_mount.c:223:ldd_print()) index:   0003
00000020:01000004:0:1198000287.047148:0:6093:0:(obd_mount.c:224:ldd_print()) config:  14
00000020:01000004:0:1198000287.047150:0:6093:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000287.047151:0:6093:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000287.047153:0:6093:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000287.047154:0:6093:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000287.047156:0:6093:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000287.051705:0:6093:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sde errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000287.082191:0:6093:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0003 for fs 'data' on device /dev/sde
00000020:01000004:0:1198000287.082252:0:6093:0:(obd_mount.c:594:lustre_start_mgc()) MGC192.168.64.70@o2ib: Set MGC reconnect 1
10000000:01000000:0:1198000287.082258:0:6093:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/0:d0:i0:r0:or0:FULL
00000020:01000004:0:1198000287.082262:0:6093:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0003
00000020:01000004:0:1198000287.082264:0:6093:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sde
10000000:01000000:0:1198000287.104757:0:6093:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0003
00000020:01000004:0:1198000287.104771:0:6093:0:(obd_mount.c:951:server_register_target()) Registration data-OST0003, fs=data, 192.168.64.71@o2ib, index=0003, flags=0x2
10000000:01000000:0:1198000287.104775:0:6093:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0003 0x2
10000000:01000000:0:1198000287.104786:0:6093:0:(mgc_request.c:732:mgc_target_register()) register data-OST0003
10000000:01000000:0:1198000287.106729:0:6093:0:(mgc_request.c:741:mgc_target_register()) register data-OST0003 got index = 3
00000020:01000004:0:1198000287.106740:0:6093:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff810407abdbc0 from data-OST0003, vfscount=1
10000000:01000000:0:1198000287.106745:0:6093:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0003 from 0
10000000:01000000:0:1198000287.106747:0:6093:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0003:<NULL>
10000000:01000000:0:1198000287.106750:0:6093:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0003 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000287.106754:0:6093:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0003:<NULL> from 1
10000000:01000000:0:1198000287.106756:0:6093:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0003 (res 0x61746164)
10000000:01000000:0:1198000287.129876:0:6093:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0003 (0)
00000020:01000000:0:1198000287.129902:0:6093:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000287.129905:0:6093:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0003 (3) in 0 of /dev/sde
00000020:00000080:0:1198000287.129909:0:6093:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000287.129911:0:6093:0:(obd_config.c:785:class_process_config()) marker 14 (0x1) data-OST0003 add ost
00000020:00000080:0:1198000287.129914:0:6093:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000287.129917:0:6093:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0003 uuid: data-OST0003_UUID
00000020:00000080:0:1198000287.129974:0:6093:0:(genops.c:283:class_newdev()) Adding new device data-OST0003 (ffff810405406178)
00000020:00000080:0:1198000287.129977:0:6093:0:(obd_config.c:237:class_attach()) OBD: dev 5 attached type obdfilter with refcount 1
00000020:00000080:0:1198000287.129980:0:6093:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000287.130069:0:6093:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff810407abdbc0 from data-OST0003, refs=2, vfscount=4
00002000:00080000:0:1198000287.160033:0:6093:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 12671643 srv lr: 12671532
00002000:00080000:0:1198000287.160044:0:6093:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 12671643
00002000:00000400:0:1198000287.160048:0:6093:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0003, 2 recoverable clients, last_rcvd 12671643
00002000:00080000:0:1198000287.187855:0:6093:0:(filter.c:983:filter_prep_groups()) data-OST0003: server last_objid group 0: 1092476
00002000:00080000:0:1198000287.188154:0:6093:0:(filter.c:983:filter_prep_groups()) data-OST0003: server last_objid group 1: 0
00002000:00080000:0:1198000287.209473:0:6093:0:(filter.c:983:filter_prep_groups()) data-OST0003: server last_objid group 2: 0
00000040:01000000:0:1198000287.233052:0:6093:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000287.243644:0:6235:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000287.243788:0:6093:0:(filter.c:1775:filter_common_setup()) OST data-OST0003 now serving dev (data-OST0003/32f492c0-9f39-b638-4ddd-9330d8da92b5), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0003/recovery_status.
00000020:00000080:0:1198000287.244693:0:6093:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0003 (uuid data-OST0003_UUID)
00000020:01000000:0:1198000287.244700:0:6093:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000287.244704:0:6093:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000287.244706:0:6093:0:(obd_config.c:785:class_process_config()) marker 14 (0x2) data-OST0003 add ost
00000020:01000000:0:1198000287.244712:0:6093:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0003 gen 1-4 (rc=0)
10000000:01000000:0:1198000287.244730:0:6093:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0003' succeeded (0).
00000020:01000004:0:1198000287.244736:0:6093:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000287.244745:0:6093:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=27
00000020:02000400:0:1198000287.244748:0:6093:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0003 on device /dev/sde has started
00000020:01200004:1:1198000287.248329:0:6239:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff810405883c00
00000020:01000004:1:1198000287.248339:0:6239:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:1:1198000287.248340:0:6239:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sdf
00000020:01000004:1:1198000287.248342:0:6239:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:1:1198000287.248343:0:6239:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sdf
00000020:01000004:1:1198000287.248351:0:6239:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sdf
00000020:01000004:0:1198000287.614937:0:6239:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000287.643505:0:6239:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000287.643508:0:6239:0:(obd_mount.c:220:ldd_print()) server:  data-OST0004
00000020:01000004:0:1198000287.643509:0:6239:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000287.643510:0:6239:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000287.643511:0:6239:0:(obd_mount.c:223:ldd_print()) index:   0004
00000020:01000004:0:1198000287.643513:0:6239:0:(obd_mount.c:224:ldd_print()) config:  14
00000020:01000004:0:1198000287.643514:0:6239:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000287.643515:0:6239:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000287.643516:0:6239:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000287.643517:0:6239:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000287.643518:0:6239:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000287.647186:0:6239:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sdf errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000287.680018:0:6239:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0004 for fs 'data' on device /dev/sdf
00000020:01000004:0:1198000287.680079:0:6239:0:(obd_mount.c:594:lustre_start_mgc()) MGC192.168.64.70@o2ib: Set MGC reconnect 1
10000000:01000000:0:1198000287.680083:0:6239:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/0:d0:i0:r0:or0:FULL
00000020:01000004:0:1198000287.680087:0:6239:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0004
00000020:01000004:0:1198000287.680090:0:6239:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sdf
10000000:01000000:0:1198000287.709223:0:6239:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0004
00000020:01000004:0:1198000287.709236:0:6239:0:(obd_mount.c:951:server_register_target()) Registration data-OST0004, fs=data, 192.168.64.71@o2ib, index=0004, flags=0x2
10000000:01000000:0:1198000287.709241:0:6239:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0004 0x2
10000000:01000000:0:1198000287.709254:0:6239:0:(mgc_request.c:732:mgc_target_register()) register data-OST0004
10000000:01000000:0:1198000287.709554:0:6239:0:(mgc_request.c:741:mgc_target_register()) register data-OST0004 got index = 4
00000020:01000004:0:1198000287.709571:0:6239:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff81041e6153c0 from data-OST0004, vfscount=1
10000000:01000000:0:1198000287.709577:0:6239:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0004 from 0
10000000:01000000:0:1198000287.709579:0:6239:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0004:<NULL>
10000000:01000000:0:1198000287.709583:0:6239:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0004 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000287.709588:0:6239:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0004:<NULL> from 1
10000000:01000000:0:1198000287.709591:0:6239:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0004 (res 0x61746164)
10000000:01000000:0:1198000287.831383:0:6239:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0004 (0)
00000020:01000000:0:1198000287.831404:0:6239:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000287.831407:0:6239:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0004 (4) in 0 of /dev/sdf
00000020:00000080:0:1198000287.831412:0:6239:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000287.831414:0:6239:0:(obd_config.c:785:class_process_config()) marker 17 (0x1) data-OST0004 add ost
00000020:00000080:0:1198000287.831417:0:6239:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000287.831420:0:6239:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0004 uuid: data-OST0004_UUID
00000020:00000080:0:1198000287.831478:0:6239:0:(genops.c:283:class_newdev()) Adding new device data-OST0004 (ffff810406ba41b8)
00000020:00000080:0:1198000287.831481:0:6239:0:(obd_config.c:237:class_attach()) OBD: dev 6 attached type obdfilter with refcount 1
00000020:00000080:0:1198000287.831484:0:6239:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000287.831547:0:6239:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff81041e6153c0 from data-OST0004, refs=2, vfscount=4
00002000:00080000:0:1198000287.860504:0:6239:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 14272708 srv lr: 14272708
00002000:00080000:0:1198000287.860514:0:6239:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 14272708
00002000:00000400:0:1198000287.860518:0:6239:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0004, 2 recoverable clients, last_rcvd 14272708
00002000:00080000:0:1198000287.895412:0:6239:0:(filter.c:983:filter_prep_groups()) data-OST0004: server last_objid group 0: 1092016
00002000:00080000:0:1198000287.896266:0:6239:0:(filter.c:983:filter_prep_groups()) data-OST0004: server last_objid group 1: 0
00002000:00080000:0:1198000287.896361:0:6239:0:(filter.c:983:filter_prep_groups()) data-OST0004: server last_objid group 2: 0
00000040:01000000:0:1198000287.910932:0:6239:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000287.911041:0:6378:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000287.911167:0:6239:0:(filter.c:1775:filter_common_setup()) OST data-OST0004 now serving dev (data-OST0004/a8f0c1b4-ff6e-9827-4012-d44b6cc47caa), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0004/recovery_status.
00000020:00000080:0:1198000287.911287:0:6239:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0004 (uuid data-OST0004_UUID)
00000020:01000000:0:1198000287.911290:0:6239:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000287.911292:0:6239:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000287.911293:0:6239:0:(obd_config.c:785:class_process_config()) marker 17 (0x2) data-OST0004 add ost
00000020:01000000:0:1198000287.911297:0:6239:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0004 gen 1-4 (rc=0)
10000000:01000000:0:1198000287.911309:0:6239:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0004' succeeded (0).
00000020:01000004:0:1198000287.911314:0:6239:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000287.911320:0:6239:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=28
00000020:02000400:0:1198000287.911324:0:6239:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0004 on device /dev/sdf has started
00000020:01200004:3:1198000287.915014:0:6381:0:(obd_mount.c:1887:lustre_fill_super()) VFS Op: sb ffff8104057b3c00
00000020:01000004:3:1198000287.915021:0:6381:0:(obd_mount.c:1661:lmd_print())   mount data:
00000020:01000004:3:1198000287.915023:0:6381:0:(obd_mount.c:1664:lmd_print()) device:  /dev/sdg
00000020:01000004:3:1198000287.915024:0:6381:0:(obd_mount.c:1665:lmd_print()) flags:   0
00000020:01000004:3:1198000287.915026:0:6381:0:(obd_mount.c:1920:lustre_fill_super()) Mounting server from /dev/sdg
00000020:01000004:3:1198000287.915032:0:6381:0:(obd_mount.c:1224:server_kernel_mount()) Pre-mount ldiskfs /dev/sdg
00000020:01000004:0:1198000288.289255:0:6381:0:(obd_mount.c:252:ldd_parse()) Have CONFIGS/mountdata, size 12288
00000020:01000004:0:1198000288.311513:0:6381:0:(obd_mount.c:219:ldd_print())   disk data:
00000020:01000004:0:1198000288.311516:0:6381:0:(obd_mount.c:220:ldd_print()) server:  data-OST0005
00000020:01000004:0:1198000288.311517:0:6381:0:(obd_mount.c:221:ldd_print()) uuid:    
00000020:01000004:0:1198000288.311519:0:6381:0:(obd_mount.c:222:ldd_print()) fs:      data
00000020:01000004:0:1198000288.311520:0:6381:0:(obd_mount.c:223:ldd_print()) index:   0005
00000020:01000004:0:1198000288.311522:0:6381:0:(obd_mount.c:224:ldd_print()) config:  14
00000020:01000004:0:1198000288.311523:0:6381:0:(obd_mount.c:225:ldd_print()) flags:   0x2
00000020:01000004:0:1198000288.311525:0:6381:0:(obd_mount.c:226:ldd_print()) diskfs:  ldiskfs
00000020:01000004:0:1198000288.311527:0:6381:0:(obd_mount.c:227:ldd_print()) options: errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000288.311528:0:6381:0:(obd_mount.c:228:ldd_print()) params:   mgsnode=192.168.64.70@o2ib
00000020:01000004:0:1198000288.311530:0:6381:0:(obd_mount.c:229:ldd_print()) comment: 
00000020:01000004:0:1198000288.315521:0:6381:0:(obd_mount.c:1287:server_kernel_mount()) kern_mount: ldiskfs /dev/sdg errors=remount-ro,extents,mballoc
00000020:01000004:0:1198000288.345303:0:6381:0:(obd_mount.c:1550:server_fill_super()) Found service data-OST0005 for fs 'data' on device /dev/sdg
00000020:01000004:0:1198000288.345363:0:6381:0:(obd_mount.c:594:lustre_start_mgc()) MGC192.168.64.70@o2ib: Set MGC reconnect 1
10000000:01000000:0:1198000288.345366:0:6381:0:(mgc_request.c:844:mgc_set_info_async()) InitRecov MGC192.168.64.70@o2ib 1/0:d0:i0:r0:or0:FULL
00000020:01000004:0:1198000288.345370:0:6381:0:(obd_mount.c:1007:server_start_targets()) starting target data-OST0005
00000020:01000004:0:1198000288.345371:0:6381:0:(obd_mount.c:807:server_mgc_set_fs()) Set mgc disk for /dev/sdg
10000000:01000000:0:1198000288.351025:0:6381:0:(mgc_request.c:413:mgc_fs_setup()) MGC using disk labelled=data-OST0005
00000020:01000004:0:1198000288.351037:0:6381:0:(obd_mount.c:951:server_register_target()) Registration data-OST0005, fs=data, 192.168.64.71@o2ib, index=0005, flags=0x2
10000000:01000000:0:1198000288.351042:0:6381:0:(mgc_request.c:857:mgc_set_info_async()) register_target data-OST0005 0x2
10000000:01000000:0:1198000288.351054:0:6381:0:(mgc_request.c:732:mgc_target_register()) register data-OST0005
10000000:01000000:0:1198000288.352155:0:6381:0:(mgc_request.c:741:mgc_target_register()) register data-OST0005 got index = 5
00000020:01000004:0:1198000288.352169:0:6381:0:(obd_mount.c:104:server_register_mount()) reg_mnt ffff810407abcb80 from data-OST0005, vfscount=1
10000000:01000000:0:1198000288.352179:0:6381:0:(mgc_request.c:1226:mgc_process_config()) parse_log data-OST0005 from 0
10000000:01000000:0:1198000288.352181:0:6381:0:(mgc_request.c:181:config_log_add()) adding config log data-OST0005:<NULL>
10000000:01000000:0:1198000288.352187:0:6381:0:(mgc_request.c:67:mgc_name2resid()) log data-OST0005 to resid 0x61746164/0x0 (data)
10000000:01000000:0:1198000288.352192:0:6381:0:(mgc_request.c:1111:mgc_process_log()) Process log data-OST0005:<NULL> from 1
10000000:01000000:0:1198000288.352195:0:6381:0:(mgc_request.c:620:mgc_enqueue()) Enqueue for data-OST0005 (res 0x61746164)
10000000:01000000:0:1198000288.354974:0:6381:0:(mgc_request.c:1075:mgc_copy_llog()) Copied remote log data-OST0005 (0)
00000020:01000000:0:1198000288.354993:0:6381:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x8 mark_flg=0x1
00000020:01000004:0:1198000288.354998:0:6381:0:(obd_mount.c:1689:lustre_check_exclusion()) Check exclusion data-OST0005 (5) in 0 of /dev/sdg
00000020:00000080:0:1198000288.355002:0:6381:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000288.355005:0:6381:0:(obd_config.c:785:class_process_config()) marker 20 (0x1) data-OST0005 add ost
00000020:00000080:0:1198000288.355009:0:6381:0:(obd_config.c:725:class_process_config()) processing cmd: cf001
00000020:00000080:0:1198000288.355012:0:6381:0:(obd_config.c:152:class_attach()) attach type obdfilter name: data-OST0005 uuid: data-OST0005_UUID
00000020:00000080:0:1198000288.355084:0:6381:0:(genops.c:283:class_newdev()) Adding new device data-OST0005 (ffff810405a441f8)
00000020:00000080:0:1198000288.355089:0:6381:0:(obd_config.c:237:class_attach()) OBD: dev 7 attached type obdfilter with refcount 1
00000020:00000080:0:1198000288.355092:0:6381:0:(obd_config.c:725:class_process_config()) processing cmd: cf003
00000020:01000004:0:1198000288.355199:0:6381:0:(obd_mount.c:156:server_get_mount()) get_mnt ffff810407abcb80 from data-OST0005, refs=2, vfscount=4
00002000:00080000:0:1198000288.358460:0:6381:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: 5ca85557-1b99-442d-d494-f1096d3fa4c4 idx: 1 lr: 13558350 srv lr: 13558350
00002000:00080000:0:1198000288.358471:0:6381:0:(filter.c:786:filter_init_server_data()) RCVRNG CLIENT uuid: data-mdtlov_UUID idx: 2 lr: 0 srv lr: 13558350
00002000:00000400:0:1198000288.358474:0:6381:0:(filter.c:833:filter_init_server_data()) RECOVERY: service data-OST0005, 2 recoverable clients, last_rcvd 13558350
00002000:00080000:0:1198000288.414239:0:6381:0:(filter.c:983:filter_prep_groups()) data-OST0005: server last_objid group 0: 1092595
00002000:00080000:0:1198000288.414906:0:6381:0:(filter.c:983:filter_prep_groups()) data-OST0005: server last_objid group 1: 0
00002000:00080000:0:1198000288.415008:0:6381:0:(filter.c:983:filter_prep_groups()) data-OST0005: server last_objid group 2: 0
00000040:01000000:0:1198000288.419590:0:6381:0:(llog_lvfs.c:779:llog_get_cat_list()) cat list: disk size=32, read=32
00000040:00080000:0:1198000288.419716:0:6408:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_01 started
00002000:02000400:0:1198000288.419848:0:6381:0:(filter.c:1775:filter_common_setup()) OST data-OST0005 now serving dev (data-OST0005/871a43f5-8952-b7fa-42f7-04a7d8cc3a0f), but will be in recovery for at least 3:20, or until 2 clients reconnect. During this time new clients will not be allowed to connect. Recovery progress can be monitored by watching /proc/fs/lustre/obdfilter/data-OST0005/recovery_status.
00000020:00000080:0:1198000288.421015:0:6381:0:(obd_config.c:320:class_setup()) finished setup of obd data-OST0005 (uuid data-OST0005_UUID)
00000020:01000000:0:1198000288.421021:0:6381:0:(obd_config.c:967:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2
00000020:00000080:0:1198000288.421025:0:6381:0:(obd_config.c:725:class_process_config()) processing cmd: cf010
00000020:00000080:0:1198000288.421027:0:6381:0:(obd_config.c:785:class_process_config()) marker 20 (0x2) data-OST0005 add ost
00000020:01000000:0:1198000288.421032:0:6381:0:(obd_config.c:1102:class_config_parse_llog()) Processed log data-OST0005 gen 1-4 (rc=0)
10000000:01000000:0:1198000288.421049:0:6381:0:(mgc_request.c:1182:mgc_process_log()) MGC192.168.64.70@o2ib: configuration from log 'data-OST0005' succeeded (0).
00000020:01000004:0:1198000288.421054:0:6381:0:(obd_mount.c:825:server_mgc_clear_fs()) Unassign mgc disk
00000020:01000004:0:1198000288.421063:0:6381:0:(obd_mount.c:1499:server_fill_super_common()) Server sb, dev=29
00000020:02000400:0:1198000288.421066:0:6381:0:(obd_mount.c:1597:server_fill_super()) Server data-OST0005 on device /dev/sdg has started
00010000:00080000:0:1198000296.379966:0:5768:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff810407a32000, cookie 0x3eb8e9cf2b4bad64
00010000:00080000:0:1198000296.379973:0:5768:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0000: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000296.380070:0:5768:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff81040c0b5000, cookie 0x3eb8e9cf2b4bad87
00010000:00000400:0:1198000296.380072:0:5768:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0001: starting recovery timer
00010000:00080000:0:1198000296.380544:0:5768:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0001: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000296.380572:0:5768:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff810406080000, cookie 0x3eb8e9cf2b4bada3
00010000:00000400:0:1198000296.380574:0:5768:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0002: starting recovery timer
00010000:00080000:0:1198000296.381040:0:5768:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0002: recovery timer will expire in 200 seconds
00010000:00080000:3:1198000296.381113:0:5782:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff8104052b4000, cookie 0x3eb8e9cf2b4badbf
00010000:00000400:3:1198000296.381119:0:5782:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0003: starting recovery timer
00010000:00080000:1:1198000296.381144:0:5778:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff810405037000, cookie 0x3eb8e9cf2b4baddb
00010000:00000400:1:1198000296.381149:0:5778:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0004: starting recovery timer
00010000:00080000:1:1198000296.381157:0:5778:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0004: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000296.381198:0:5768:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID '5ca85557-1b99-442d-d494-f1096d3fa4c4' at ffff810404abf000, cookie 0x3eb8e9cf2b4badf7
00010000:00000400:0:1198000296.381201:0:5768:0:(ldlm_lib.c:1122:target_start_recovery_timer()) data-OST0005: starting recovery timer
00010000:00080000:0:1198000296.381207:0:5768:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0005: recovery timer will expire in 200 seconds
00010000:00080000:1:1198000296.381216:0:5778:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810405874600 x35081070/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:0:1198000296.381230:0:5768:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810405b29e00 x35081071/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:3:1198000296.382530:0:5782:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0003: recovery timer will expire in 200 seconds
00010000:00080000:1:1198000296.382637:0:5774:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff8104298bca00 x35081072/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:0:1198000296.382648:0:5769:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810407a35c00 x35081073/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:1:1198000296.382676:0:5775:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff81042f694400 x35081074/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:1:1198000296.382704:0:5776:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff8104167b7000 x35081075/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:1:1198000296.382735:0:5777:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810405b29600 x35081077/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:0:1198000296.382735:0:5770:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff81042b033000 x35081076/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:1:1198000296.382764:0:5779:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810405b29200 x35081078/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:2:1198000296.382767:0:5786:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810423ceba00 x35081079/t0 o101->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 232/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/0/0 rc 0/0
00010000:00080000:0:1198000296.412410:0:5771:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810405b29a00 x35081080/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:0:1198000296.412423:0:5771:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810405b29a00 x35081080/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00000400:0:1198000296.412431:0:5771:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0002: 1 recoverable clients remain
00010000:00080000:0:1198000296.466918:0:5772:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810410f96c50 x35081081/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:0:1198000296.466931:0:5772:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810410f96c50 x35081081/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00000400:0:1198000296.466939:0:5772:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0005: 1 recoverable clients remain
00010000:00080000:1:1198000296.470315:0:5780:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810402dfb800 x35081082/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:1:1198000296.470327:0:5780:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810402dfb800 x35081082/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:1:1198000296.470335:0:5780:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0000: cancel recovery timer
00010000:02000000:1:1198000296.470337:0:5780:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0000: sending delayed replies to recovered clients
00010000:00080000:1:1198000296.470358:0:5780:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810407a35400 x35081082/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:1:1198000296.470369:0:5780:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff81041bb52800 x447/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000291 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:1:1198000296.470377:0:5780:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0000_UUID: recovery complete
00002000:02000400:1:1198000296.471315:0:5781:0:(filter.c:3493:filter_set_info_async()) data-OST0000: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:1:1198000296.471706:0:5784:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 6507
00002000:00080000:2:1198000296.471821:0:5785:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092371 -> diff = -60
00002000:00000400:2:1198000296.471826:0:5785:0:(filter.c:2712:filter_destroy_precreated()) data-OST0000: deleting orphan objects from 1092312 to 1092371
00000040:00080000:1:1198000296.472176:0:6507:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ab75:589c8c72 at index 30 of catalog 0x1378002
00000040:00080000:1:1198000296.472628:0:6507:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137800c:90bd1fa5 at index 31 of catalog 0x1378002
00002000:00080000:1:1198000296.472966:0:6507:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:1:1198000296.472970:0:6507:0:(llog.c:304:llog_process()) recovery from log: 0x137800c:90bd1fa5 stopped
00000040:00080000:1:1198000296.472972:0:6507:0:(llog.c:304:llog_process()) recovery from log: 0x1378002:1237165f stopped
00000040:00080000:1:1198000296.472975:0:6507:0:(recov_thread.c:562:log_process_thread()) send llcd ffff8104079af000:ffff8104091080c0 forcibly after recovery
00000040:00080000:0:1198000296.473076:0:6508:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00010000:00080000:0:1198000296.479513:0:5788:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810402eb8c00 x35081083/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:0:1198000296.479523:0:5788:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810402eb8c00 x35081083/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00000400:0:1198000296.479536:0:5788:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0001: 1 recoverable clients remain
00010000:00080000:0:1198000296.480715:0:5789:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810402eb8a00 x35081084/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:0:1198000296.480722:0:5789:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810402eb8a00 x35081084/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00000400:0:1198000296.480728:0:5789:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0004: 1 recoverable clients remain
00010000:00080000:0:1198000296.482331:0:5790:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810402f0dc00 x35081085/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00000010:00080000:0:1198000296.482338:0:5790:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810402f0dc00 x35081085/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00000400:0:1198000296.482343:0:5790:0:(ldlm_lib.c:1415:target_queue_last_replay_reply()) data-OST0003: 1 recoverable clients remain
00002000:00080000:0:1198000299.026434:0:5785:0:(filter.c:2727:filter_destroy_precreated()) data-OST0000: after destroy: set last_objids[0] = 1092311
00010000:00080000:0:1198000310.955612:0:5792:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff8104079da000, cookie 0x3eb8e9cf2b4bad8e
00010000:00080000:0:1198000310.955618:0:5792:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0001: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000310.956110:0:5793:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff810406081000, cookie 0x3eb8e9cf2b4badaa
00010000:00080000:0:1198000310.956116:0:5793:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0002: recovery timer will expire in 200 seconds
00010000:00080000:2:1198000310.956132:0:5796:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff81042f689600 x457/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:2:1198000310.956142:0:5796:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff81042f689600 x457/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:2:1198000310.956149:0:5796:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0001: cancel recovery timer
00010000:02000000:2:1198000310.956153:0:5796:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0001: sending delayed replies to recovered clients
00010000:00080000:2:1198000310.956174:0:5796:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402dfbc00 x457/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:2:1198000310.956190:0:5796:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402eb8600 x35081083/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:2:1198000310.956200:0:5796:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0001_UUID: recovery complete
00010000:00080000:0:1198000310.956507:0:5794:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff810405526000, cookie 0x3eb8e9cf2b4badc6
00010000:00080000:0:1198000310.956510:0:5794:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0003: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000310.956751:0:5795:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810400d16000 x459/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:0:1198000310.956758:0:5795:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810400d16000 x459/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.956763:0:5795:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0002: cancel recovery timer
00010000:02000000:0:1198000310.956765:0:5795:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0002: sending delayed replies to recovered clients
00010000:00080000:0:1198000310.956777:0:5795:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810400d16400 x459/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000328 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.956787:0:5795:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402d01800 x35081080/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:0:1198000310.956797:0:5795:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0002_UUID: recovery complete
00010000:00080000:0:1198000310.957009:0:5798:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff810405655000, cookie 0x3eb8e9cf2b4bade2
00010000:00080000:0:1198000310.957012:0:5798:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0004: recovery timer will expire in 200 seconds
00010000:00080000:0:1198000310.957125:0:5799:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810400d16800 x461/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:0:1198000310.957131:0:5799:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810400d16800 x461/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.957138:0:5799:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0003: cancel recovery timer
00010000:02000000:0:1198000310.957140:0:5799:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0003: sending delayed replies to recovered clients
00010000:00080000:0:1198000310.957151:0:5799:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402d01800 x461/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.957161:0:5799:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff81042f6af400 x35081085/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:0:1198000310.957176:0:5799:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0003_UUID: recovery complete
00002000:02000400:2:1198000310.957484:0:5800:0:(filter.c:3493:filter_set_info_async()) data-OST0001: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:0:1198000310.957682:0:5802:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 7489
00010000:00080000:0:1198000310.957722:0:5803:0:(ldlm_lib.c:539:target_handle_reconnect()) connect export for UUID 'data-mdtlov_UUID' at ffff810404ae0000, cookie 0x3eb8e9cf2b4badfe
00010000:00080000:0:1198000310.957724:0:5803:0:(ldlm_lib.c:1111:reset_recovery_timer()) data-OST0005: recovery timer will expire in 200 seconds
00010000:00080000:3:1198000310.957755:0:5804:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff81042b041400 x464/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:3:1198000310.957771:0:5804:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff81042b041400 x464/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00002000:00080000:0:1198000310.957777:0:5805:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092501 -> diff = -60
00002000:00000400:0:1198000310.957779:0:5805:0:(filter.c:2712:filter_destroy_precreated()) data-OST0001: deleting orphan objects from 1092442 to 1092501
00010000:00080000:3:1198000310.957785:0:5804:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0004: cancel recovery timer
00010000:02000000:3:1198000310.957788:0:5804:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0004: sending delayed replies to recovered clients
00010000:00080000:3:1198000310.957805:0:5804:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810403379e00 x464/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:3:1198000310.957836:0:5804:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402f0de00 x35081084/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:3:1198000310.957846:0:5804:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0004_UUID: recovery complete
00010000:00080000:0:1198000310.957900:0:5807:0:(ldlm_lib.c:1249:target_queue_recovery_request()) @@@ not queueing  req@ffff810403379200 x467/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/0 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00000010:00080000:0:1198000310.957906:0:5807:0:(ost_handler.c:1660:ost_handle()) @@@ LAST_REPLAY, queuing reply  req@ffff810403379200 x467/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.957910:0:5807:0:(ldlm_lib.c:1092:target_cancel_recovery_timer()) data-OST0005: cancel recovery timer
00010000:02000000:0:1198000310.957911:0:5807:0:(ldlm_lib.c:964:target_finish_recovery()) data-OST0005: sending delayed replies to recovered clients
00010000:00080000:0:1198000310.957923:0:5807:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402dfb800 x467/t0 o400->data-mdtlov_UUID@NET_0x50000c0a84046_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000332 ref 1 fl Interpret:/9/0 rc 0/0
00010000:00080000:0:1198000310.957938:0:5807:0:(ldlm_lib.c:979:target_finish_recovery()) @@@ delayed:  req@ffff810402dfb400 x35081081/t0 o400->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 128/128 e 0 to 0 dl 1198000396 ref 1 fl Interpret:/1/0 rc 0/0
00010000:00080000:0:1198000310.957950:0:5807:0:(ldlm_lib.c:1412:target_queue_last_replay_reply()) data-OST0005_UUID: recovery complete
00000040:00080000:2:1198000310.958386:0:7489:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x1378010:589c8c65 at index 32 of catalog 0x1378003
00000040:00080000:2:1198000310.958821:0:7489:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ab4e:90bd1fa6 at index 33 of catalog 0x1378003
00002000:00080000:2:1198000310.959186:0:7489:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:2:1198000310.959189:0:7489:0:(llog.c:304:llog_process()) recovery from log: 0x137ab4e:90bd1fa6 stopped
00000040:00080000:2:1198000310.959192:0:7489:0:(llog.c:304:llog_process()) recovery from log: 0x1378003:12371661 stopped
00000040:00080000:2:1198000310.959195:0:7489:0:(recov_thread.c:562:log_process_thread()) send llcd ffff8103ffb55000:ffff81040738cdc0 forcibly after recovery
00000040:00080000:0:1198000310.959287:0:7490:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00002000:02000400:2:1198000310.959364:0:5808:0:(filter.c:3493:filter_set_info_async()) data-OST0002: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:3:1198000310.959550:0:5810:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 7491
00002000:00080000:0:1198000310.959643:0:5811:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092412 -> diff = -60
00002000:00000400:0:1198000310.959645:0:5811:0:(filter.c:2712:filter_destroy_precreated()) data-OST0002: deleting orphan objects from 1092353 to 1092412
00000040:00080000:3:1198000310.960040:0:7491:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137800b:589c8c5e at index 32 of catalog 0x1378004
00000040:00080000:3:1198000310.960459:0:7491:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ada5:90bd1fa7 at index 33 of catalog 0x1378004
00002000:02000400:0:1198000310.961361:0:5812:0:(filter.c:3493:filter_set_info_async()) data-OST0003: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:0:1198000310.961518:0:5813:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 7492
00002000:00080000:0:1198000310.961616:0:5814:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092476 -> diff = -59
00002000:00000400:0:1198000310.961619:0:5814:0:(filter.c:2712:filter_destroy_precreated()) data-OST0003: deleting orphan objects from 1092418 to 1092476
00000040:00080000:0:1198000310.961955:0:7492:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ada7:90bd1fb2 at index 32 of catalog 0x1378005
00002000:02000400:0:1198000310.963379:0:5815:0:(filter.c:3493:filter_set_info_async()) data-OST0004: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:0:1198000310.963526:0:5816:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 7493
00002000:00080000:0:1198000310.963613:0:5817:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092016 -> diff = -60
00002000:00000400:0:1198000310.963621:0:5817:0:(filter.c:2712:filter_destroy_precreated()) data-OST0004: deleting orphan objects from 1091957 to 1092016
00000040:00080000:0:1198000310.963986:0:7493:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ad6c:589c8c76 at index 30 of catalog 0x1378006
00000040:00080000:0:1198000310.964371:0:7493:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ada6:90bd1fa8 at index 31 of catalog 0x1378006
00002000:00080000:0:1198000310.964688:0:7493:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:0:1198000310.964690:0:7493:0:(llog.c:304:llog_process()) recovery from log: 0x137ada6:90bd1fa8 stopped
00000040:00080000:0:1198000310.964693:0:7493:0:(llog.c:304:llog_process()) recovery from log: 0x1378006:12371668 stopped
00000040:00080000:0:1198000310.964695:0:7493:0:(recov_thread.c:562:log_process_thread()) send llcd ffff81040009c000:ffff810405656ec0 forcibly after recovery
00000040:00080000:0:1198000310.964745:0:7494:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00002000:02000400:0:1198000310.966412:0:5818:0:(filter.c:3493:filter_set_info_async()) data-OST0005: received MDS connection from 192.168.64.70@o2ib
00000040:00080000:0:1198000310.966568:0:5819:0:(recov_thread.c:595:llog_recovery_generic()) log_process_thread: 7495
00002000:00080000:0:1198000310.966653:0:5820:0:(filter.c:2759:filter_handle_precreate()) filter_last_id() = 1092595 -> diff = -60
00002000:00000400:0:1198000310.966655:0:5820:0:(filter.c:2712:filter_destroy_precreated()) data-OST0005: deleting orphan objects from 1092536 to 1092595
00000040:00080000:0:1198000310.967040:0:7495:0:(llog_cat.c:353:llog_cat_process_cb()) processing log 0x137ada8:90bd1fb8 at index 31 of catalog 0x1378007
00002000:00080000:0:1198000310.967342:0:7495:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:0:1198000310.967344:0:7495:0:(llog.c:304:llog_process()) recovery from log: 0x137ada8:90bd1fb8 stopped
00000040:00080000:0:1198000310.967347:0:7495:0:(llog.c:304:llog_process()) recovery from log: 0x1378007:12371669 stopped
00000040:00080000:0:1198000310.967349:0:7495:0:(recov_thread.c:562:log_process_thread()) send llcd ffff81040009c000:ffff81041de5a3c0 forcibly after recovery
00000040:00080000:0:1198000310.967392:0:7496:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00002000:00080000:1:1198000310.993989:0:7491:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:1:1198000310.993992:0:7491:0:(llog.c:304:llog_process()) recovery from log: 0x137ada5:90bd1fa7 stopped
00000040:00080000:1:1198000310.993995:0:7491:0:(llog.c:304:llog_process()) recovery from log: 0x1378004:12371663 stopped
00000040:00080000:1:1198000310.993999:0:7491:0:(recov_thread.c:562:log_process_thread()) send llcd ffff810406abd000:ffff81040607eec0 forcibly after recovery
00000040:00080000:0:1198000310.994038:0:7497:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00002000:00080000:1:1198000311.004407:0:7492:0:(filter_log.c:242:filter_recov_log_mds_ost_cb()) fetch generation log, send cookie
00000040:00080000:1:1198000311.004409:0:7492:0:(llog.c:304:llog_process()) recovery from log: 0x137ada7:90bd1fb2 stopped
00000040:00080000:1:1198000311.004417:0:7492:0:(llog.c:304:llog_process()) recovery from log: 0x1378005:12371665 stopped
00000040:00080000:1:1198000311.004426:0:7492:0:(recov_thread.c:562:log_process_thread()) send llcd ffff8103ffb55000:ffff810413bce5c0 forcibly after recovery
00000040:00080000:0:1198000311.004458:0:7498:0:(recov_thread.c:238:log_commit_thread()) ll_log_comt_02 started
00002000:00080000:0:1198000311.782720:0:5820:0:(filter.c:2727:filter_destroy_precreated()) data-OST0005: after destroy: set last_objids[0] = 1092535
00002000:00080000:0:1198000311.849640:0:5817:0:(filter.c:2727:filter_destroy_precreated()) data-OST0004: after destroy: set last_objids[0] = 1091956
00002000:00080000:0:1198000312.030921:0:5814:0:(filter.c:2727:filter_destroy_precreated()) data-OST0003: after destroy: set last_objids[0] = 1092417
00002000:00080000:0:1198000312.144457:0:5811:0:(filter.c:2727:filter_destroy_precreated()) data-OST0002: after destroy: set last_objids[0] = 1092352
00002000:00080000:0:1198000312.188421:0:5805:0:(filter.c:2727:filter_destroy_precreated()) data-OST0001: after destroy: set last_objids[0] = 1092441
00002000:00080000:1:1198000330.309877:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272709 committed
00002000:00080000:1:1198000330.329878:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272710 committed
00002000:00080000:1:1198000330.333604:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272711 committed
00002000:00080000:1:1198000330.357713:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272712 committed
00002000:00080000:1:1198000330.361907:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272713 committed
00002000:00080000:1:1198000330.386007:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272714 committed
00002000:00080000:1:1198000330.390234:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272715 committed
00002000:00080000:1:1198000330.414303:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272716 committed
00010000:00000400:3:1198000555.372228:0:5865:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:3:1198000555.372717:0:5865:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:3:1198000555.373410:0:5865:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff810402f0de00 x35081176/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000655 ref 1 fl Interpret:/0/0 rc -16/0
00010000:00000400:3:1198000580.371872:0:5879:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:3:1198000580.372352:0:5879:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:3:1198000580.373039:0:5879:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff810400d16200 x35081183/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000680 ref 1 fl Interpret:/0/0 rc -16/0
00010000:00000400:2:1198000605.372062:0:5887:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:2:1198000605.372555:0:5887:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:2:1198000605.373241:0:5887:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff81042e8d9800 x35081190/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000705 ref 1 fl Interpret:/0/0 rc -16/0
00010000:00000400:0:1198000630.371425:0:5876:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:0:1198000630.371910:0:5876:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:0:1198000630.372594:0:5876:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff81040aaaa000 x35081197/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000730 ref 1 fl Interpret:/0/0 rc -16/0
00010000:00000400:0:1198000655.371865:0:5774:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:0:1198000655.372351:0:5774:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:0:1198000655.373046:0:5774:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff8104298bca00 x35081204/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000755 ref 1 fl Interpret:/0/0 rc -16/0
00010000:00000400:1:1198000680.371402:0:5761:0:(ldlm_lib.c:517:target_handle_reconnect()) data-OST0004: 5ca85557-1b99-442d-d494-f1096d3fa4c4 reconnecting
00010000:00000400:1:1198000680.371881:0:5761:0:(ldlm_lib.c:746:target_handle_connect()) data-OST0004: refuse reconnection from 5ca85557-1b99-442d-d494-f1096d3fa4c4@192.168.64.102@o2ib to 0xffff810405037000; still busy with 9 active RPCs
00010000:00020000:1:1198000680.372567:0:5761:0:(ldlm_lib.c:1460:target_send_reply_msg()) @@@ processing error (-16)  req@ffff81042e5dea00 x35081211/t0 o8->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 304/200 e 0 to 0 dl 1198000780 ref 1 fl Interpret:/0/0 rc -16/0
00002000:00020000:0:1198000692.208584:0:5899:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.209085:0:5899:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00020000:0:1198000692.209647:0:5900:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.210135:0:5900:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00080000:0:1198000692.210667:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272717 committed
00002000:00020000:0:1198000692.210699:0:5898:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00020000:3:1198000692.210814:0:5901:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:3:1198000692.210834:0:5901:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00020000:0:1198000692.212097:0:5898:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00002000:00020000:2:1198000692.212616:0:5899:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00000100:00000400:0:1198000692.212623:0:5898:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081108 took longer than estimated (100+262s); client may timeout.  req@ffff8104100a4850 x35081108/t14272717 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.212654:0:5898:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5898 disabled after 361.7994s
00002000:00020000:0:1198000692.212687:0:5900:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00020000:0:1198000692.212714:0:5902:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.212726:0:5902:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00080000:0:1198000692.212812:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272719 committed
00002000:00080000:0:1198000692.212815:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272718 committed
00002000:00020000:0:1198000692.212845:0:5903:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.212853:0:5903:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00020000:0:1198000692.212865:0:5900:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.212880:0:5900:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081110 took longer than estimated (100+262s); client may timeout.  req@ffff8104167b7000 x35081110/t14272719 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.212891:0:5900:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5900 disabled after 361.7713s
00002000:00020000:0:1198000692.213148:0:5904:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.213158:0:5904:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00020000:0:1198000692.213245:0:5906:0:(filter_io_26.c:710:filter_commitrw_write()) data-OST0004: slow i_mutex 361s
00002000:00020000:0:1198000692.213253:0:5906:0:(filter_io_26.c:723:filter_commitrw_write()) data-OST0004: slow brw_start 361s
00002000:00020000:0:1198000692.215082:0:5902:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00080000:0:1198000692.215101:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272721 committed
00002000:00080000:0:1198000692.215104:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272720 committed
00002000:00020000:0:1198000692.215129:0:5902:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.215143:0:5902:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081112 took longer than estimated (100+262s); client may timeout.  req@ffff81040aaaa600 x35081112/t14272721 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.215154:0:5902:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5902 disabled after 361.7453s
00002000:00020000:0:1198000692.215729:0:5901:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00020000:0:1198000692.215734:0:5901:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.215749:0:5901:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081111 took longer than estimated (100+262s); client may timeout.  req@ffff81042b156a00 x35081111/t14272720 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.215760:0:5901:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5901 disabled after 361.7502s
00002000:00020000:0:1198000692.217011:0:5903:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00020000:0:1198000692.218288:0:5904:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00020000:0:1198000692.219300:0:5906:0:(filter_io_26.c:775:filter_commitrw_write()) data-OST0004: slow direct_io 361s
00002000:00080000:0:1198000692.219409:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272724 committed
00002000:00080000:0:1198000692.219412:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272723 committed
00002000:00080000:0:1198000692.219414:0:6365:0:(obd.h:1075:obd_transno_commit_cb()) data-OST0004: transno 14272722 committed
00002000:00020000:0:1198000692.219422:0:5906:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.219435:0:5906:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081115 took longer than estimated (100+262s); client may timeout.  req@ffff810407a35200 x35081115/t14272724 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.219446:0:5906:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5906 disabled after 361.6973s
00002000:00020000:0:1198000692.219451:0:5904:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.219464:0:5904:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081114 took longer than estimated (100+262s); client may timeout.  req@ffff81040aaaa200 x35081114/t14272723 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.219475:0:5904:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5904 disabled after 361.7213s
00002000:00020000:0:1198000692.219480:0:5903:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:0:1198000692.219492:0:5903:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081113 took longer than estimated (100+262s); client may timeout.  req@ffff81040aaaae00 x35081113/t14272722 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:0:1198000692.219503:0:5903:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5903 disabled after 361.7256s
00002000:00020000:2:1198000692.239359:0:5899:0:(filter_io_26.c:788:filter_commitrw_write()) data-OST0004: slow commitrw commit 361s
00000100:00000400:2:1198000692.239917:0:5899:0:(service.c:1063:ptlrpc_server_handle_request()) @@@ Request x35081109 took longer than estimated (100+262s); client may timeout.  req@ffff8104167b7a00 x35081109/t14272718 o4->5ca85557-1b99-442d-d494-f1096d3fa4c4@NET_0x50000c0a84066_UUID:0/0 lens 384/352 e 0 to 0 dl 1198000430 ref 1 fl Complete:/0/0 rc 0/0
00000400:00000400:2:1198000692.241022:0:5899:0:(watchdog.c:312:lcw_update_time()) Expired watchdog for pid 5899 disabled after 361.8037s
