Linux log puller and mover


[root@logllm01 scripts]# cat LogMover_V2.pl

#!/usr/bin/perl -w

#-------------------------------------------------------------------------------------------------------------------------

# Program               : LogMover.pl

# Called By             : Operator

# Parameters            : N/A

# Authors Name          : Krishnakumar (kdharmarajan@travelclick.net)

# Creation Date         : sep-30-2009

# Version               : 1.0

#-------------------------------------------------------------------------------------------------------------------------



use XML::Simple;

use Data::Dumper;



# Variables

my $program = 'LogMover.pl';

my $version = 1.0;



my $property = '/home/loguser/scripts/LOG_PROPERTY.xml';



# Sub Functions

sub Log($$$);

sub GetTime($);

sub MoveLogFiles($$$$$$);





my $xmltestcases = XMLin($property, forcearray =>1 );

#print Dumper($xmltestcases);



# Create a temporary log file to log and track all the changes

my $time = GetTime(1);

$logFile = "/home/loguser/scripts/logs/$time"."\_LogMover.log";

print "Logging all output to $logFile \n";



# Open the log file

open(LOGFILE,">$logFile") || die (print "Cannot create logfile $logFile \n");



Log("Running verion $version of $program",0,LOGFILE);





#Looping through all the host listed in the xml property file to pull the logs



        #process host groups in sorted order

        foreach (sort {$a<=>$b} keys %{$xmltestcases->{case}})

        {

                my $testnum = $_;

                my @hostList = @{$xmltestcases->{case}->{$testnum}->{host}};

                my @sourcePath = @{$xmltestcases->{case}->{$testnum}->{sourcepath}};

                my $destPath = $xmltestcases->{case}->{$testnum}->{destpath}->[0];

                my $user = $xmltestcases->{case}->{$testnum}->{username}->[0];

                my $retention = $xmltestcases->{case}->{$testnum}->{retention}->[0];



                #process for each host

                 foreach my $host (@hostList)

                {

                        Log("===============================================================",0,LOGFILE);

                        Log("Starting to move the logs from $host to Log Server",0,LOGFILE);

                        chomp($host);

                        #pull the logs from each listed source

                        foreach my $sourceLogPath (@sourcePath)

                        {

                                chomp($sourceLogPath);

                                # Steps to Compress the Log files

                                MoveLogFiles("$host","$sourceLogPath","$destPath","$user","$retention", LOGFILE);

                        }



                        Log("Completed the log rotation job for the $host",0,LOGFILE);

                        Log("===============================================================",0,LOGFILE);

                }

        }







sub Log($$$)

{

    my $text = $_[0];

    my $exitStatus = $_[1];

    my $outputDestination = $_[2];

    my $time = GetTime(0);

    if ($exitStatus == 1)

        {

        $time = GetTime(0);

        printf $outputDestination "[$time] - $text\n";

        printf  "[$time] - $text\n";



        }

    else

        {

        printf $outputDestination "[$time] - $text\n";

        printf "[$time] - $text\n";

        }

}



sub GetTime($)

{

    # Get the locatime

    my ($Second, $Minute, $Hour, $Day, $Month, $Year) = (localtime)[0,1,2,3,4,5];



    # Add padding as required

    $Year = $Year+1900;

    $Month = $Month+1;



    # Format into a nice string

    my $Time = "$Month-$Day-$Year $Hour:$Minute:$Second";



    # Spit it out

    if ($_[0] == 1)

    {

        $Time = "$Month-$Day-$Year\_$Hour\_$Minute\_$Second";

        return $Time;

    }

    return $Time;

}





sub MoveLogFiles($$$$$$)

{

        my $host = $_[0];

        my $sourceLogPath = $_[1];

        my $destLogPath = $_[2];

        my $user = $_[3];

        my $retention =  $_[4];

        my $outputDestination = $_[5];

        my $returnCode;

        #Prepare file list that needs to be moved to Log server

        my @listOfFilesToMove =`ssh -o StrictHostKeyChecking=no -l $user $host -C \" find -L $sourceLogPath -type f \\\( \\\! \\\-iname \\\"\\\*\\\.log\\\" \\\) -mtime +$retention\" 2>&1`;

        #Create the directory with remote host name in log destination if it is not exist

        if ( ! -e "$destLogPath/$host" )

        {

                if (system("mkdir -p $destLogPath/$host"))

                {

                 die Log("Cannot create the $destLogPath/$host",1,LOGFILE);

                }

        }



        foreach $file (@listOfFilesToMove)

        {

                chomp($file);

                my $RSYNC = "rsync -rlzva --rsh=\"ssh -o StrictHostKeyChecking=no -l $user \" $host:\'\"$file\"\' $destLogPath/$host";

#               print "RSYNC $RSYNC ";

                #copy the file to log server using rsync command

                my @RSYNCOUTPUT =`$RSYNC`;

                if ($? == "0")

                {

                        Log("Successfully copied the file $host:$file to $destLogPath/$host",0,$outputDestination);

                        #remove the log files on remote host upon successfull completion of copying log files to log server

                        my @DELETE = `ssh -o StrictHostKeyChecking=no -l $user $host -C "rm -f \'\"$file\"\'" 2>&1`;

                        if ($? == "0")

                                {

                                        Log("Successfully deleted the log file $file from $host",0,$outputDestination);

                                }

                                else

                                {

                                        Log("ERROR:Could not remove the file $file from host $host",0,$outputDestination);

                                        Log("ERROR: Error details @DELETE",0,$outputDestination);

                                }

                }

                else

                {

                        Log("ERROR:Could not copy the file $file to $destLogPath/$host",0,$outputDestination);

                        Log("ERROR: Error details @RSYNCOUTPUT",0,$outputDestination);

                }

        }

}