#!/usr/bin/perl
# reapp tomcat servers or environments
# Original Author: Patrick.Wolfe@ADP.com

use strict;
use threads;
use Thread::Queue;
use File::Find;

&main_routine(@ARGV);

sub usage
{
	print STDERR "usage: $0 [-c #] [-f] [-u] environment | farm | app-server [...]\n";
	print STDERR "\t-c (concurrency) specifies number of reapps to run concurrently\n";
	print STDERR "\t-f (fast) says to reapp whole farms at one time\n";
	print STDERR "\t-u (unlock) says to ignore any existing lockfile\n";
	print STDERR "environment, farm names and app server names must\n";
	print STDERR "be specified exactly as they are in /env_control/FARM_LAYOUT\n";
	exit(1);
}

sub interrupt_handler
{
	our $lockfile;
	unlink($lockfile) if ($lockfile ne "");
	die("**INTERRUPTED**\n");
}

sub process_request_thread
{
        my $tid = shift(@_);
        my $eof = 0;
	our $request_queue;
	our $result_queue;
	our %app;

	while (! $eof) {
		# get the next env/host to reapp
		my $app = $request_queue->dequeue();
		if ($app eq "EOF") {
			$eof++;
			next;
		}
		my ($env,$farm) = split(/\t/,$app{$app});

		my $logfile = "/tmp/#reapp-$app-$$.log";
		# notify user that this host is being reapped
		$result_queue->enqueue("$app\tstarting\t");

		# open the logfile
		open(LOGFILE, ">$logfile") || die("ERROR(tid $tid): cannot open file $logfile for writing: $!\n");
		my $cmd = "/env_control/en-remote.sh $env host reapp $app";
		my @now = localtime(time);
		printf(LOGFILE "%04d-%02d-%02d %02d:%02d:%02d INFO: running %s\n",
			$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$cmd);
		my $status = "UNKNOWN";
		open(CMD, "$cmd 2>&1 |") || die("ERROR(tid $tid): cannot popen command $cmd for reading: $!\n");
		while (<CMD>) {
			print LOGFILE;
			chomp;
			if (/Polling for tomcat health.*PASS.*/) {
				$status = "PASS";
				last;
			} elsif (/Polling for tomcat health.*FAIL.*/) {
				$status = "FAIL";
				last;
			}
		}
		close(LOGFILE);

		# tell the user the good or bad news
		$result_queue->enqueue("$app\t$status\t$logfile");
	}

	# let parent know we are done
	$result_queue->enqueue("EOF");
}

sub reapp_tomcat
{
	my $successful = 0;
	our $request_queue;
	our $result_queue;
	our $thread_limit;
	our %app;
	our $faststart;

	if ($faststart) {
		$thread_limit = $#_ + 1;
	}

	# spin off threads to handle reapp requests
	my @tidlist = ();
	for (my $ctr = 1; $ctr <= $thread_limit; $ctr++) {
		push(@tidlist,threads->create(\&process_request_thread, $ctr));
	}

	# add all apps to the request queue
	foreach my $app (@_) {
		(my $env, my $farm) = $app{$app};
		$request_queue->enqueue($app);
	}

	# add one EOF message for each thread
	for (my $ctr = 1; $ctr <= $thread_limit; $ctr++) {
		$request_queue->enqueue("EOF");
	}

	# wait for all threads to finish
	for (my $EOF = 1; $EOF <= $thread_limit;) {
		my $result = $result_queue->dequeue();
		#printf STDERR "DEBUG: read result \"$result\"\n";
		if ($result eq "EOF") {
			$EOF++;
		} else	{
			my ($host,$status,$logfile) = split(/\t/,$result);
			if ($status eq "starting") {
				printf ("%-16s : %s\n", $host, $status);
			} elsif ($status eq "PASS") {
				printf ("%-16s : %s\n", $host, $status);
				$successful++;
				unlink($logfile);
			} else	{
				printf ("%-16s : %s (logfile $logfile)\n", $host, $status, $logfile);
			}
		}
	}

	# clean up threads
	foreach my $tid (@tidlist) {
		$tid->join;
	}

	# return number of apps that PASS
	return($successful);
}

sub reapp_farm
{
	my $env = shift(@_);
	my $farm = shift(@_);
	our $faststart;
	our $lockfile;
	our %farmapps;

	my @applist = split(/\t/,$farmapps{"$env\t$farm"});
	shift(@applist);
	# either reapp the whole farm at once
	# or half at a time
	if ($faststart || $#applist == 0) {
		print "reapping $farm farm\n";
		if (&reapp_tomcat(@applist) < 1) {
			unlink($lockfile);
			die("all $farm servers FAILED: reapp $env aborted\n");
		}
	} else	{
		my @first_apps = ();
		my $half = int(($#applist + 1) / 2);
		for (my $ctr = 0; $ctr < $half; $ctr++) {
			push(@first_apps, shift(@applist));
		}
		print "reapping first half of $farm farm\n";
		if (&reapp_tomcat(@first_apps) < 1) {
			unlink($lockfile);
			die("all $farm servers in first group FAILED: reapp $env aborted\n");
		}
		print "reapping rest of $farm farm\n";
		if (&reapp_tomcat(@applist) < 1) {
			unlink($lockfile);
			die("all $farm servers in second group FAILED: reapp $env aborted\n");
		}
	}
}

sub lock_env
{
	my $env = shift(@_);
	our $lockfile = "/tmp/#reapp-$env.lock";
	our $unlock;
	unlink($lockfile) if ($unlock);
	if (-f $lockfile) {
		die("ERROR: another reapp of $env appears to be running already\nto override this, use the '-u' option\n");
	}
	open(LOCKFILE,">$lockfile");
	close(LOCKFILE);
}

######################################################################
# find_old_logs is called by find(), once per directory entry found
# deletes files
######################################################################
sub find_old_logs
{
	our $three_days_ago;
	if (-f $_ && $_ =~ /^#reapp-/ && (stat(_))[9] < $three_days_ago) {
		unlink($File::Find::dir . "/" . $_);
	}
}



sub main_routine
{
	my $LAYOUT = "/env_control/FARM_LAYOUT";
	my %env;
	my %farm;
	our %app;
	my %envfarms;
	our %farmapps;
	my @envs_to_reapp = ();
	my @farms_to_reapp = ();
	my @apps_to_reapp = ();
	my $faststart = 0;
	our $lockfile = "";
	our $thread_limit = 8;
	our $unlock = 0;
	select(STDOUT); $| = 1;

	$SIG{'INT'} = \&interrupt_handler;
	$SIG{'QUIT'} = \&interrupt_handler;

	# process cmdline options
	while ($#_ > -1) {
		last if (substr($_[0],0,1) ne '-');
		my $arg = shift(@_);
		if ($arg eq "-c") {
			my $value = shift(@_);
			if ($value =~ /^(\d+)$/) {
				$thread_limit = $1;
			} else	{
				&usage;
			}
		} elsif ($arg eq "-f") {
			$faststart = 1;
		} elsif ($arg eq "-u") {
			$unlock = 1;
		}
	}

	if ($#_ < 0) {
		&usage;
	}

	# requires superuser privs
	if ($< != 0) {
		print STDERR "reapp requires sudo privs\n";
		exec("sudo", "-u", "root", $0, @_);
		die("ERROR: exec sudo $0 failed: $!\n");
	}

	# read FARM_LAYOUT file into memory
	open(LAYOUT, "<$LAYOUT") || die "ERROR: cannot open file $LAYOUT for reading: $!\n";
	my $linectr = 0;
	while (<LAYOUT>) {
		chomp;
		$linectr++;
		#print STDERR "DEBUG: read \"$_\"\n";
		s/#.*$//; # ignore comments
		next if (/^\s*$/); # ignore blank lines
		my ($env, $app, $farm) = split(' ');
		if (-z $env || -z $app || -z $farm) {
			print STDERR "WARNING: $LAYOUT line $linectr has invalid format\n";
			next;
		}
		$env{$env}++;
		if (! defined $farm{$farm}) {
			$envfarms{$env} .= "\t$farm";
		}
		$farm{$farm} = $env;
		if (defined $app{$app}) {
			die("ERROR: app server $app is listed multiple times in $LAYOUT\n");
		}
		$app{$app} = "$env\t$farm";
		$farmapps{"$env\t$farm"} .= "\t$app";
	}

	# before we do anything, process all command line arguments
	while ($#_ > -1) {
		my $ARG = shift(@_);
		# in norcross, environments have underscores in their names, not dashes
		(my $ARGU = $ARG) =~ s/-/_/g;
		if (defined $env{$ARGU}) { # do environments one at a time
			push(@envs_to_reapp, $ARGU);
			if ($#farms_to_reapp > -1 || $#apps_to_reapp > -1) {
				die "ERROR: do not mix environments, farms and apps in one request\n";
			}

		} elsif (defined $farm{$ARG}) { # do farms one at a time
			push(@farms_to_reapp, $ARG);
			if ($#envs_to_reapp > -1 || $#apps_to_reapp > -1) {
				die "ERROR: do not mix environments, farms and apps in one request\n";
			}

		} elsif (defined $app{$ARG}) {	# build a list of apps and do them all in parallel
			push(@apps_to_reapp, $ARG);
			if ($#envs_to_reapp > -1 || $#farms_to_reapp > -1) {
				die "ERROR: do not mix environments, farms and apps in one request\n";
			}

		} else	{
			print "ERROR: \"$ARG\" was not found in $LAYOUT\n";
			print "\"$ARG\" is not an active environment, app server or farm name\n";
			exit(1);
		}
	}

	my @now = localtime(time);
	printf("%04d-%02d-%02d %02d:%02d:%02d INFO: reapp starting\n",
		$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0]);

	# create request and result queue
	our $request_queue = Thread::Queue->new();
	our $result_queue = Thread::Queue->new();

	# reapp each environment listed on the cmdline, one at a time
	foreach my $env (@envs_to_reapp) {
		&lock_env($env);
		print "reapping $env environment\n";
		my @farmlist = split(/\t/,$envfarms{$env});
		shift(@farmlist);
		# do them one farm at a time
		foreach my $farm (@farmlist) {
			&reapp_farm($env,$farm);
		}
		unlink($lockfile);
	}

	# reapp each farm listed on the cmdline, one at a time
	if ($#farms_to_reapp > -1) {
		foreach my $farm (@farms_to_reapp) {
			my $env = $farm{$farm};
			&lock_env($env);
			&reapp_farm($env,$farm);
			unlink($lockfile);
		}
	}

	# do all tomcat servers listed on the cmdline
	if ($#apps_to_reapp > -1) {
		&reapp_tomcat(@apps_to_reapp);
	}

	## clean up any leftover logfiles
	our $three_days_ago = time - (86400 * 3);
	find(\&find_old_logs, "/tmp");

	@now = localtime(time);
	printf("%04d-%02d-%02d %02d:%02d:%02d INFO: reapp completed\n",
		$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0]);
	exit(0);
}