#!/usr/bin/perl
use strict;
use Sys::Hostname;

# my ($cfg_path) = '/opt/lustre/etc';
my ($cfg_path) = '.';
my ($myhost)   = hostname();

#--------------------------------------------------------------------------
#	Read the Lustre file system config file.
#
sub extract_hosts {
    my ($list) = @_;
    my ($i, @arr, @hosts, $hlist);

    $#hosts = -1;
    for ($i=0; $i < @$list; $i++) {
	@arr = split(',', $list->[$i]);
	$hosts[$#hosts+1] = $arr[0];
    }
    $hlist = join(',', @hosts);
    return "$hlist";
}

sub read_cfg {
    my ($para) = @_;
    my ($FF, $fn, $tag, %fsg, @list);

    $fsg{FSERVER}  = 'NULL';
    $fsg{ZPOOL}    = 'NULL';
    $fsg{MGS}      = 'NULL';
    $fsg{MDT}      = 'NULL';
    $fsg{OST}      = 'NULL';
    $fsg{INACTIVE} = 'NULL';
    $fsg{CLIENT}   = 'NULL';
    $fsg{MGSsrv}   = 'NULL';
    $fsg{MDTsrv}   = 'NULL';
    $fsg{FSNAME}   = $para->{fsname};

    $tag = 'NULL';
    $fn  = "$cfg_path/cfs-$para->{fsname}";
    open($FF, "$fn") || die("read_cfg: cannot open file: $fn\n");
    while (defined($_ = <$FF>)) {
	(/^#/) && next;
	chop();
	s/#.*//;
	s/[ \t]*$//;
	s/^[ \t]*//;
	("$_" eq '') && next;

	if (/^\[.*\]$/) {
	    ("$tag" ne 'NULL') && { $fsg{$tag} = [ @list ] };

	    $#list = -1;
	    ($tag = $_) =~ s/^\[([^\]]+)\]/$1/;
	}
	else {
	    $list[$#list+1] = $_;
	}
    }
    close($FF);
    ("$tag" ne 'NULL') && { $fsg{$tag} = [ @list ] };

    $fsg{MGSsrv} = ("$fsg{MGS}" ne 'NULL') ? extract_hosts($fsg{MGS}) : 'NULL';
    $fsg{MDTsrv} = ("$fsg{MDT}" ne 'NULL') ? extract_hosts($fsg{MDT}) : 'NULL';
    return { %fsg };
}

#--------------------------------------------------------------------------
#	Check whether the host is alive or not.
#
sub check_host {
    my ($host, $wait) = @_;
    my ($r);

    ("$host" eq 'NULL') && return 0;

    while (1) {
	($wait == 1) && printf("Checking $host ....\n");
	$r = system("ping -c 1 -W 1 $host > /dev/null");
	($r == 0) && last;

	if ($wait == 0) {
	    printf("Host not available: $host\n");
	    last;
	}
	else {
	    printf("Waiting $host ....\n");
	    sleep(5);
	}
    }
    return $r;
}

#--------------------------------------------------------------------------
#	Remote excuting commands.
#
sub rcmd {
    my ($rhost, $cmd, $cmdmsg, $dryrun) = @_;
    my ($r);

    ("$rhost"  eq 'NULL') && return 0;
    ("$cmdmsg" eq 'NULL') && { $cmdmsg = $cmd };

    $r = 0;
    printf("%s$rhost: $cmdmsg ....\n", ($dryrun == 0) ? '' : '[DRY] ');
    if ($dryrun == 0) {
	if ("$myhost" eq "$rhost") {
	    $r = system("$cmd");
	} else {
	    $r = check_host($rhost, 0);
	    ($r == 0) && { $r = system("ssh -n $rhost '$cmd'") };
	}
	($r != 0) && printf("rcmd: failed: $rhost: $cmd\n");
    }
    return $r;
}

#--------------------------------------------------------------------------
#
#	Start the Lustre filesystem.
#
#--------------------------------------------------------------------------

sub delay_mount {
    my ($n_time) = @_;
    my ($i, $t_time);

    $t_time = $n_time * 5;
    for ($i=0; $i < $n_time; $i++) {
	printf("Delaying mount [$t_time sec]: %d\n", ($i+1)*5);
	sleep(5);
    }
}

sub fs_ZPool {
    my ($para, $list, $slevel) = @_;
    my ($i, $r, $host, $dev, $cmd1, $cmd2, $dryrun, $zpcmd, $err, %okhost);

    ("$list" eq 'NULL' ||
     ($para->{level} <  0 && $para->{level} !=-$slevel) ||
     ($para->{level} >= 0 && $para->{level} <  $slevel)) && return 0;

    $dryrun = $para->{dryrun};
    $zpcmd  = '/opt/lustre/sbin/zpool';
    $err    = 0;

    for ($i=0; $i < @$list; $i++) {
	($host, $dev) = split(',', $list->[$i]);
	$dev  =~ s/\/.*//;
	(defined($okhost{$host})) && next;

	$cmd1 = "/sbin/modprobe zfs";
	$r    =  rcmd($host, $cmd1, $cmd1, $dryrun);
	$err += $r;
	$okhost{$host} = 1;
	($r != 0) && printf("$host: loading zfs modules failed: $dev\n");
	sleep(1);
    }
    ($err != 0) && return $err;

    if (@$list > 0) {
	printf("Waiting for zfs module initialization ....\n");
	sleep(10);
    }
    for ($i=0; $i < @$list; $i++) {
	($host, $dev) = split(',', $list->[$i]);
	$dev  =~ s/\/.*//;

	$cmd1 = "$zpcmd status $dev > /dev/null 2>&1 ||";
	$cmd2 = "$zpcmd import $dev";
	$r    =  rcmd($host, "$cmd1 $cmd2", $cmd2, $dryrun);
	$err += $r;
	($r != 0) && printf("$host: Import ZPool failed: $dev\n");
	sleep(1);
    }
    ($err != 0) && return $err;

    if (@$list > 0) {
	printf("Waiting for ZPool initialization ....\n");
	sleep(5);
    }
    return 0;
}

sub fs_disableOST {
    my ($para, $fsg, $slevel) = @_;
    my ($i, $j, $r, $fsname, $list, @hosts, $dev);
    my ($lctl, $cmd, $dryrun, $err);

    ("$list" eq 'NULL' ||
     ($para->{level} <  0 && $para->{level} !=-$slevel) ||
     ($para->{level} >= 0 && $para->{level} <  $slevel)) && return 0;

    $dryrun = $para->{dryrun};
    $fsname = $fsg->{FSNAME};
    $list   = $fsg->{INACTIVE};
    @hosts  = split(',', $fsg->{MDTsrv});
    $lctl   = '/opt/lustre/sbin/lctl';
    $err    = 0;

    for ($i=0; $i <= $#hosts; $i++) {
    for ($j=0; $j <  @$list;  $j++) {
	$dev  =  $list->[$j];
	$cmd  = "$lctl set_param osc.$fsname-$dev-osc-MDT0000.max_create_count=0";
	$r    =  rcmd($hosts[$i], $cmd, $cmd, $dryrun);
	$err += $r;
	if ($r != 0) {
	    printf("$hosts[$i]: disabling $dev failed.\n");
	    last;
	}
    }}
    return $err;
}

sub fs_mount {
    my ($para, $list, $slevel, $retry) = @_;
    my ($i, $r, $host, $dev, $dir, $flock, $grant);
    my ($cmd1, $cmd2, $dryrun, $dev2, %errhost, $nok);

    ("$list" eq 'NULL' ||
     ($para->{level} <  0 && $para->{level} !=-$slevel) ||
     ($para->{level} >= 0 && $para->{level} <  $slevel)) && return 0;

    $dryrun = $para->{dryrun};
    $nok    = 0;

    for ($i=0; $i < @$list; $i++) {
	($host, $dev, $dir, $flock, $grant) = split(',', $list->[$i]);
	$errhost{"$host:$dev"} = 1;
    }
    while ($nok < @$list) {
	for ($i=0; $i < @$list; $i++) {
	    ($host, $dev, $dir, $flock, $grant) = split(',', $list->[$i]);
	    ($errhost{"$host:$dev"} == 0) && next;

	    ($dev2 = $dev) =~ s/[^:]+://;
	    $flock = ($flock == 0) ? '' : '-o flock';
	    $cmd1  = "mkdir -p $dir; /bin/mount | grep $dev2 > /dev/null ||";
	    $cmd2  = "/bin/mount -t lustre $flock $dev $dir";
	    $r = rcmd($host, "$cmd1 $cmd2", $cmd2, $dryrun);
	    if ($r != 0) {
		printf("$host: mount $dev on $dir failed.\n");
	    }
	    else {
		$errhost{"$host:$dev"} = 0;
		$nok ++;
	    }
	    if ($grant != 0) {
		$cmd1 = "/opt/lustre/sbin/lctl set_param osc.*.grant_shrink=0";
		$r = rcmd($host, $cmd1, $cmd1, $dryrun);
	    }
	    sleep(1);
	}
	($retry == 0) && last;
    }
    return ($nok == @$list) ? 0 : (@$list-$nok);
}

sub start_fs {
    my ($para, $fsg) = @_;
    my (@host, $r, $h, $nfs);

    foreach $h ( @{$fsg->{FSERVER}} ) {
	@host = split(':', $h);
	check_host($host[2], 1);
    }
    $r = fs_ZPool($para, $fsg->{ZPOOL}, 0);
    ($r != 0) && return;

    $r = fs_mount($para, $fsg->{MGS}, 1, 0);
    ($r != 0) && return;

    $r = fs_mount($para, $fsg->{MDT}, 1, 0);
    ($r != 0) && return;

    $r = fs_disableOST($para, $fsg, 1);
    ($r != 0) && return;

    $r = fs_mount($para, $fsg->{OST}, 2, 0);
    ($r != 0) && return;

    ($para->{level} >= 3) && delay_mount(6);
    $r = fs_mount($para, $fsg->{CLIENT}, 3, 1);
    ($r != 0) && return;
}


#--------------------------------------------------------------------------
#
#	Stop the Lustre file system.
#
#--------------------------------------------------------------------------

sub fs_umount {
    my ($para, $type, $list, $slevel) = @_;
    my ($r, $i, $host, $dev, $dir, $flock, $grant);
    my ($cmd1, $cmd2, $dryrun, $err, %okhost);

    ("$list" eq 'NULL' ||
     ($para->{level} <  0 && $para->{level} !=-$slevel) ||
     ($para->{level} >= 0 && $para->{level} <  $slevel)) && return 0;

    $dryrun = $para->{dryrun};
    $err    = 0;

    for ($i=0; $i < @$list; $i++) {
	($host, $dev, $dir, $flock, $grant) = split(',', $list->[$i]);

	if ($type eq "CLIENT") {
	    (defined($okhost{$host})) && next;
	    $okhost{$host} = 1;
	    $cmd1 = "/opt/lustre/etc/umount.sh $dir";
	    $r    = rcmd($host, $cmd1, $cmd1, $dryrun);
	}
	else {
	    $dev  =! s/[^:]*://;
	    $cmd1 = "if [ ! -z \"\$( /bin/mount | grep $dev )\" ]; then ";
	    $cmd2 = "/opt/lustre/etc/umount.sh $dir";
	    $r    = rcmd($host, "$cmd1 $cmd2; fi", $cmd2, $dryrun);
	}
	$err += $r;
	sleep(3);
    }
    return $err;
}

sub stop_fs {
    my ($para, $fsg) = @_;
    my ($r);

    $r = fs_umount($para, 'CLIENT', $fsg->{CLIENT}, 1);

    $r = fs_umount($para, 'OST', $fsg->{OST}, 2);
    ($r != 0) && return;

    $r = fs_umount($para, 'MDT', $fsg->{MDT}, 3);
    ($r != 0) && return;

    $r = fs_umount($para, 'MGS', $fsg->{MGS}, 3);
}

#--------------------------------------------------------------------------
#	Shutdown the cluster (Emergency)
#
sub get_cfgs {
    my ($para) = @_;
    my ($DIR, @cfgfs, $f, @fsgs);

    opendir($DIR, $cfg_path) ||
	die("!!! get_cfgs: cannot open dir: $cfg_path\n");
    @cfgfs = grep { -f "$cfg_path/$_" && /^cfs-/ } readdir($DIR);
    closedir($DIR);

    $#fsgs = -1;
    for $f ( @cfgfs ) {
	$f =~ s/^cfs-//;
	$para->{fsname} = $f;
	$fsgs[$#fsgs+1] = read_cfg($para);
    }
    return [ @fsgs ];
}

sub shutdown_host {
    my ($para, $type, $list, $delayM) = @_;
    my ($i, $host, $dev, $dir, $flock, $dryrun, %hosts);
    my ($cmd1, $cmd2, $msg1, $msg2);

    ("$list" eq 'NULL') && return;

    $dryrun = $para->{dryrun};
    $delayM = ($delayM == 0) ? 'now' : "+$delayM";

    for ($i=0; $i < @$list; $i++) {
	($host, $dev, $dir, $flock) = split(',', $list->[$i]);
	($hosts{$host} == 1) && next;

	$hosts{$host} = 1;
	$cmd1 = "/etc/init.d/lustre_mnt stop";
	$cmd2 = "/sbin/shutdown -h $delayM > /dev/null 2>&1 < /dev/null";
	$msg1 = "lustre_mnt stop";
	$msg2 = "shutdown";
	if ("$type" eq 'CLIENT') {
	    if ("$host" eq "$myhost") {
		rcmd($host, "$cmd1 &", $msg1, $dryrun);
	    } else {
		rcmd($host, "( $cmd1; $cmd2 ) &", "$msg1; $msg2", $dryrun);
	    }
	}
	else {
	    rcmd($host, "$cmd2 &", "$msg2", $dryrun);
	}
    }
}

sub shutdown_fs {
    my ($para) = @_;
    my ($cmd, $fsgs, $fsg);

    $fsgs = get_cfgs($para);
    shutdown_host($para, 'CLIENT', $fsgs->[0]->{CLIENT}, 0);
    printf("Waiting 30 secs ....\n");
    sleep(30);

    foreach $fsg ( @$fsgs ) {
	fs_umount($para, 'OST', $fsg->{OST}, 0);
	sleep(5);
	fs_umount($para, 'MDT', $fsg->{MDT}, 0);
	sleep(5);
	fs_umount($para, 'MGS', $fsg->{MGS}, 0);
	sleep(5);
    }
    printf("Waiting 30 secs ....\n");
    sleep(30);
    foreach $fsg ( @$fsgs ) {
	shutdown_host($para, 'OST', $fsg->{OST}, 5);
	shutdown_host($para, 'MDT', $fsg->{MDT}, 5);
	shutdown_host($para, 'MGS', $fsg->{MGS}, 5);
    }
    printf("Waiting 360 secs ....\n");
    sleep(360);

    $cmd = "/sbin/shutdown -h now";
    rcmd("$myhost", $cmd, $cmd, $para->{dryrun});
}

#--------------------------------------------------------------------------
#	Parse command line options
#
sub inputs {
    my ($args) = @_;
    my (%para, $i);

    if ($#ARGV == -1) {
	printf("Usage:  $0 [-d] [action] [fs_name] [level]\n");
	printf("option:\n");
	printf("    -d: dry-run (for test)\n");
	printf("action: start:    start Lustre file system.\n");
	printf("        stop:     stop Lustre file system.\n");
	printf("        shutdown: emergency shutdown the cluster.\n");
	printf(" level: for start:\n");
	printf("     0: start ZPOOL\n");
	printf("     1: start ZPOOL/MGS/MDT\n");
	printf("     2: start ZPOOL/MGS/MDT/OST\n");
	printf("     3: start ZPOOL/MGS/MDT/OST/CLIENT (default)\n");
	printf("    -1: start MGS/MDT\n");
	printf("    -2: start OST\n");
	printf("    -3: start CLIENT\n");
	printf(" level: for stop:\n");
	printf("     3: stop  CLIENT/OST/MDT/MGS (default)\n");
	printf("     2: stop  CLIENT/OST\n");
	printf("     1: stop  CLIENT\n");
	printf("    -1: stop  CLIENT\n");
	printf("    -2: stop  OST\n");
	printf("    -3: stop  MDT/MGS\n");
	exit(0);
    }
    $para{action} = 'NULL';
    $para{fsname} = 'NULL';
    $para{level}  =  3;
    $para{dryrun} =  0;

    for ($i=0; $i < @$args; $i++) {
	($args->[$i] =~ /^-/) || last;
	("$args->[$i]" eq '-d') && { $para{dryrun}=1 };
    }
    ($i < @$args) && { $para{action} = $args->[$i++] };
    ($i < @$args) && { $para{fsname} = $args->[$i++] };
    ($i < @$args) && { $para{level}  = $args->[$i++] };

    ("$para{action}" eq 'NULL') && die("!!! Please specify action.\n");
    ("$para{fsname}" eq 'NULL') && ("$para{action}" ne 'shutdown') &&
	die("!!! Please specify filesystem name.\n");

    return { %para };
}

#--------------------------------------------------------------------------
#	Main program.
#
my ($para, $fsg);

$para = inputs(\@ARGV);

if ("$para->{action}" eq 'start') {
    $fsg = read_cfg($para);
    start_fs($para, $fsg);
}
elsif ("$para->{action}" eq 'stop') {
    $fsg = read_cfg($para);
    stop_fs($para, $fsg);
}
elsif ("$para->{action}" eq 'shutdown') {
    shutdown_fs($para);
}
else {
    printf("!!! Unknown action: $para->{action}\n");
    exit(1);
}