Sunday, February 3, 2013

Convert tcpflow out put to syslog format for analysis

Convert tcpflow out put to syslog format for analysis


Following is the set of script i used to convert tcpflow out put in to syslog format

#Main script

------------------------------------------------------------------------------------------------------------

#monitor.sh

#!/bin/bash

#
#This script will start tcpflow and construct syslog like log from the out put of tcpflow for analysis
#

RUN_CMD=/usr/local/bin/tcpflow
OUTPUT_DIR=raw_data
LOG_DIR=logs
INTERFACE=eth0

${RUN_CMD} -i ${INTERFACE}  dst port 80 -p -o ${OUTPUT_DIR} > /dev/null 2>&1 &


#Start watch log in background

cd bin;./watch_logs.sh ${OUTPUT_DIR} > /dev/null 2>&1 &

#start raw data delete process

./delete_raw_data.sh > /dev/null 2>&1 &


------------------------------------------------------------------------------------------------------------
#watch_logs.sh
#For this i have use i  apt-get install inotify-tools  to monitor new file creation

#!/bin/sh

RAW_DATA_PATH="../$1"
       while inotifywait --exclude ~*.xml  --format %f   -e create ${RAW_DATA_PATH} | xargs perl create_log.pl ; do
         #if tail -n1 /var/log/messages | grep httpd; then
         #  kdialog --msgbox "Apache needs love!"
         #fi
echo "Log file created" > /dev/null
       done
-----------------------------------------------------------------------------------------------------------
#create_log.pl
#Log path is /tmp/tcpflow.log

#!/usr/bin/perl -w

use POSIX qw/strftime/;

$TCPFLOWFILE_NAME=$ARGV[0];
$TCPFLOWFILE_DIR="../raw_data";
$TCPFLOWFILE_PATH="$TCPFLOWFILE_DIR/$TCPFLOWFILE_NAME";
#Time format Jan  4 12:39:01
$TIME = strftime('%b %d %T',localtime);
$SERVER="titan580";
$PROGRAM="tcpflow";
$LOGFILE="/tmp/cpflow.log";
#-------------------------------------------------
#Get ip address(src and dst)
$TCPFLOWFILE_NAME =~ /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.\d{0,}-(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/;


sub trim_zeros {

$ip_seg = $_[0];
#print "$ip_seg \n";

if ( $ip_seg =~ /0{3}/ ){
$ip_seg = 0;
return $ip_seg;
}else{
$ip_seg =~ s/^0+//;
return $ip_seg;

}

}


#Construct ip address
$srcip1 = trim_zeros $1;
$srcip2 = trim_zeros $2;
$srcip3 = trim_zeros $3;
$srcip4 = trim_zeros $4;
$dstip1 = trim_zeros $5;
$dstip2 = trim_zeros $6;
$dstip3 = trim_zeros $7;
$dstip4 = trim_zeros $8;


$srcip="$srcip1.$srcip2.$srcip3.$srcip4";
$dstip="$dstip1.$dstip2.$dstip3.$dstip4";




#print $TCPFLOWFILE_PATH;

open(TCPFLOWFILE_PATH) or die("Could not open tcpflow  file.");
foreach $line () {

open  LOGFILE_FH, ">>/tmp/tcpflow.log";

    # do line-by-line processing.
if ($line =~ /^\s*$/) {
#open(LOGFILE_FH,'>>LOGFILE') || die("Cannot Open Log File");
print LOGFILE_FH "\n";

}else{
$line =~ s/\n//;
$TCPFLOW="$TIME $SERVER $PROGRAM: $srcip->$dstip:$line";
#print $TCPFLOW;
#open(LOGAPPEND,">>LOGFILE") || die("Cannot Open Log File");
print LOGFILE_FH "$TCPFLOW";

}
close(LOGFILE_FH);

}
close(TCPFLOWFILE_PATH);
#Remove log file
#unlink $TCPFLOWFILE_PATH;


---------------------------------------------------------------------------------------------------
#Log deleting process
#delete_raw_data.sh

#!/bin/bash


#
#This script will  delete created tcp data file which are older than given time
#


EXPIRE_TIME=3 #Time in minutes which log file expire
ROTATION_TIME=3  #Time in minutes wait before next check
FIND=/usr/bin/find
RM=/bin/rm
RAW_DATA_DIR="../raw_data"



while [ 1 -ne 0 ]
do
${FIND} ${RAW_DATA_DIR}  -mmin +${EXPIRE_TIME} -exec ${RM} -rf {} \;

sleep ${ROTATION_TIME}
done



--------------------------------------------------------------------------------------------------


1 comment:

Anonymous said...

Sands Casino - The Star Gold Coast - Singapore Casino
Sands casino in 샌즈카지노 Sydney is the newest resort in the 제왕카지노 iconic Gold Coast which has been open since kadangpintar 1999. Located in the heart of the world of the