Sunday, February 3, 2013

Convert tcpflow out put to syslog format for analysis

Convert tcpflow out put to syslog format for analysis


Following is the set of script i used to convert tcpflow out put in to syslog format

#Main script

------------------------------------------------------------------------------------------------------------

#monitor.sh

#!/bin/bash

#
#This script will start tcpflow and construct syslog like log from the out put of tcpflow for analysis
#

RUN_CMD=/usr/local/bin/tcpflow
OUTPUT_DIR=raw_data
LOG_DIR=logs
INTERFACE=eth0

${RUN_CMD} -i ${INTERFACE}  dst port 80 -p -o ${OUTPUT_DIR} > /dev/null 2>&1 &


#Start watch log in background

cd bin;./watch_logs.sh ${OUTPUT_DIR} > /dev/null 2>&1 &

#start raw data delete process

./delete_raw_data.sh > /dev/null 2>&1 &


------------------------------------------------------------------------------------------------------------
#watch_logs.sh
#For this i have use i  apt-get install inotify-tools  to monitor new file creation

#!/bin/sh

RAW_DATA_PATH="../$1"
       while inotifywait --exclude ~*.xml  --format %f   -e create ${RAW_DATA_PATH} | xargs perl create_log.pl ; do
         #if tail -n1 /var/log/messages | grep httpd; then
         #  kdialog --msgbox "Apache needs love!"
         #fi
echo "Log file created" > /dev/null
       done
-----------------------------------------------------------------------------------------------------------
#create_log.pl
#Log path is /tmp/tcpflow.log

#!/usr/bin/perl -w

use POSIX qw/strftime/;

$TCPFLOWFILE_NAME=$ARGV[0];
$TCPFLOWFILE_DIR="../raw_data";
$TCPFLOWFILE_PATH="$TCPFLOWFILE_DIR/$TCPFLOWFILE_NAME";
#Time format Jan  4 12:39:01
$TIME = strftime('%b %d %T',localtime);
$SERVER="titan580";
$PROGRAM="tcpflow";
$LOGFILE="/tmp/cpflow.log";
#-------------------------------------------------
#Get ip address(src and dst)
$TCPFLOWFILE_NAME =~ /(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.\d{0,}-(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/;


sub trim_zeros {

$ip_seg = $_[0];
#print "$ip_seg \n";

if ( $ip_seg =~ /0{3}/ ){
$ip_seg = 0;
return $ip_seg;
}else{
$ip_seg =~ s/^0+//;
return $ip_seg;

}

}


#Construct ip address
$srcip1 = trim_zeros $1;
$srcip2 = trim_zeros $2;
$srcip3 = trim_zeros $3;
$srcip4 = trim_zeros $4;
$dstip1 = trim_zeros $5;
$dstip2 = trim_zeros $6;
$dstip3 = trim_zeros $7;
$dstip4 = trim_zeros $8;


$srcip="$srcip1.$srcip2.$srcip3.$srcip4";
$dstip="$dstip1.$dstip2.$dstip3.$dstip4";




#print $TCPFLOWFILE_PATH;

open(TCPFLOWFILE_PATH) or die("Could not open tcpflow  file.");
foreach $line () {

open  LOGFILE_FH, ">>/tmp/tcpflow.log";

    # do line-by-line processing.
if ($line =~ /^\s*$/) {
#open(LOGFILE_FH,'>>LOGFILE') || die("Cannot Open Log File");
print LOGFILE_FH "\n";

}else{
$line =~ s/\n//;
$TCPFLOW="$TIME $SERVER $PROGRAM: $srcip->$dstip:$line";
#print $TCPFLOW;
#open(LOGAPPEND,">>LOGFILE") || die("Cannot Open Log File");
print LOGFILE_FH "$TCPFLOW";

}
close(LOGFILE_FH);

}
close(TCPFLOWFILE_PATH);
#Remove log file
#unlink $TCPFLOWFILE_PATH;


---------------------------------------------------------------------------------------------------
#Log deleting process
#delete_raw_data.sh

#!/bin/bash


#
#This script will  delete created tcp data file which are older than given time
#


EXPIRE_TIME=3 #Time in minutes which log file expire
ROTATION_TIME=3  #Time in minutes wait before next check
FIND=/usr/bin/find
RM=/bin/rm
RAW_DATA_DIR="../raw_data"



while [ 1 -ne 0 ]
do
${FIND} ${RAW_DATA_DIR}  -mmin +${EXPIRE_TIME} -exec ${RM} -rf {} \;

sleep ${ROTATION_TIME}
done



--------------------------------------------------------------------------------------------------


Detect malicious PHP shells and quarantine them on Shared Server


Detect malicious PHP shells and quarantine them on Shared Server

Here i have use perl code from  http://ketan.lithiumfox.com/doku.php/phpshell_scanner  to detect malicious php shells and i have written shell script to do further processing after the bad shell is found.

-----------------------------------------------------------------------------------------------------------
# ./execute_findshell.sh

#!/bin/bash

ECHO=/bin/echo
TIME=`/bin/date`
MAIL=/usr/bin/mail
MV=/bin/mv
Quarantine_Dir=/tmp #Directory where quarantine file goes to
WEB_ROOT=/var/www #Web root to watch for changes
LOG=badshell.log
FIND=/usr/bin/find
GREP=/bin/grep
AWK=/usr/bin/awk
LS=/bin/ls
CAT=/bin/cat
SORT=/usr/bin/sort
FILE_INDEX=file_index
TMP_FILE_INDEX=tmp_file_index
CHANGES_IN_INDEX=changes_in_index
MD5_SUM=/usr/bin/md5sum
WC=/usr/bin/wc
DIFF=/usr/bin/diff
TR=/usr/bin/tr

#Execute file checker on shared hosting server on file changes using their checksum values
#*/1 * * * * cd /home/user/scripts; ./execute_findshell.sh > /dev/null 2>&1 

#
#Php shell finder-----
#

check_for_php_shells(){

FILE_NAME=$1
FINDSHELL_OUTPUT=`perl findshell.pl  10 ${FILE_NAME}`

#echo "${FILE_NAME}"

if [ "${FINDSHELL_OUTPUT}" ];then
echo "bad shell found\n"
echo "${FINDSHELL_OUTPUT}" 
${MV} ${FILE_NAME} ${Quarantine_Dir}
${ECHO} "Possible Backdoor Found,and Quarantined. Please Verify...${FINDSHELL_OUTPUT} -  ${TIME} " | ${MAIL} -s 'Possible Back door Found On  Server'  email@address.com
else
echo "bad shell not found \n"
fi


}


#----------------------

#Build file index
NO_OF_FILES=`${FIND} ${WEB_ROOT}  -type f  |${WC} -l`
if [ -f "${FILE_INDEX}" ];then
#if File exist, check for new entrie
#Create tmp file index and compare
${FIND}  ${WEB_ROOT} -type f -exec ${MD5_SUM} {} \; > ${TMP_FILE_INDEX}

${DIFF} ${TMP_FILE_INDEX} ${FILE_INDEX} > ${CHANGES_IN_INDEX}
if [ -s "${CHANGES_IN_INDEX}" ];then
echo "integrity changed \n"
CHANGES_IN_INDEX_ARRAY=(`${CAT} ${CHANGES_IN_INDEX} |${AWK} {'print $3'}| ${GREP} -P '(\/\w+\/.*)' | ${SORT} -u | ${TR} '\n' ' '`)

#Going through changes files
for i in "${CHANGES_IN_INDEX_ARRAY[@]}"
do
check_for_php_shells $i
done
else
echo "no changes found\n"
fi

else
#create file index
${FIND}  ${WEB_ROOT} -type f -exec ${MD5_SUM} {} \; > ${FILE_INDEX}
fi

-----------------------------------------------------------------------------------------------------------
#Perl code
#findshell.pl  

http://ketan.lithiumfox.com/doku.php/phpshell_scanner

------------------------------------------------------------------------------------------------------------

How to use Portsentry with notifications and Iptable firewall rule expiraton

How to use Portsentry with notifications and Iptable  firewall rule expiration


Following is the how i use Portsentry  to block port scan activities.

By default Portsentry detect and block given ip/ports, but it will not expire the Iptable rules after given time,here i have a script written to do that job as well.


#Port Sentry On ubuntu 12.04
apt-get install portsentry


#I use the following config entries for my setup

#######################Start of Config file  entries################################
#######################
# Port Configurations #
#######################
#Under the Port Configurations first set is enabled.[comprehensive set]
TCP_PORTS="1,7,9,11,15,70,79,80,109,110,111,119,138,139,143,512,513,514,515,540,635,1080,1524,2000,2001,4000,4001,5742,6000,6001,6667,12345,12346,20034,27665,30303,32771,32772,32773,32774,31337,40421,40425,49724,54320"
UDP_PORTS="1,7,9,66,67,68,69,111,137,138,161,162,474,513,517,518,635,640,641,666,700,2049,31335,27444,34555,32770,32771,32772,32773,32774,31337,54321"

##################
# Ignore Options #
##################

BLOCK_UDP="1"
BLOCK_TCP="1"



###################
# Dropping Routes:#
###################

KILL_ROUTE="/sbin/iptables -I INPUT -s $TARGET$ -j DROP"



###################
# External Command#
###################
KILL_RUN_CMD_FIRST = "0"
#
#
KILL_RUN_CMD="/etc/portsentry/scripts/notify.sh $TARGET$ $PORT$ $MODE$"

#This script will notify when port scan trigger



#####################
# Scan trigger value#
#####################

SCAN_TRIGGER="2"

#######################End of Config file entries ################################


Below are the scripts used for Portsentry



#----------------------------------Notification script-----------------------------------------------------
##/etc/portsentry/scripts/notify.sh


#!/bin/bash


TARGET=$1
PORT=$2
MODE=$3


echo "Attack reported-${TARGET}- ${PORT} - ${MODE}" | mail -s 'Unauthorized Port scan  Alert'  email@address.com



------------------------------------------------------------------------------------------------------------


-----------Firewall rules expiration script------------------------------------------------------------------
#delete_fw_entries.sh
#This script is running as cronjob every 5 minutes



#!/bin/bash


BLOCKED_LIST_TCP="/var/lib/portsentry/portsentry.blocked.tcp"
BLOCKED_LIST_UDP="/var/lib/portsentry/portsentry.blocked.udp"
ECHO=/bin/echo
TOUCH=/usr/bin/touch
CAT=/bin/cat
AWK=/usr/bin/awk
DATE=/bin/date
EXPR=/usr/bin/expr
FW_EXPIRE_TIME=10 #Iptable rule expiration time in minutes
TR=/usr/bin/tr
CUT=/usr/bin/cut
IPTABLE=/sbin/iptables
SED=/bin/sed
#
#DO NOT CHANGE BELOW THIS LINE--------------------------------------------
#

get_time_diff() {

BLOCKED_TIME=$1

CURRENT_TIME=`${DATE} +%s`
TIME_DIFF_IN_SEC=`${EXPR} ${CURRENT_TIME} - ${BLOCKED_TIME}`
TIME_DIFF_IN_MIN=`${EXPR} ${TIME_DIFF_IN_SEC} / 60`


}



#Get the blocked list entries for processing



BLOCKED_LIST_TCP_ARRAY=(`${CAT} ${BLOCKED_LIST_TCP} | ${TR} -d " " |${TR} '\n' ' '  `)
BLOCKED_LIST_UDP_ARRAY=(`${CAT} ${BLOCKED_LIST_UDP} | ${TR} -d " " | ${TR} '\n' ' ' `)

#TCP

for i in "${BLOCKED_LIST_TCP_ARRAY[@]}"
do
BLOCKED_IP=`${ECHO} $i |${CUT} -d ":" -f4|${CUT} -d "/" -f1`
BLOCKED_TIME=`${ECHO} $i | ${CUT} -d "-" -f1`

get_time_diff ${BLOCKED_TIME}
echo "time$TIME_DIFF_IN_MIN"

echo "ip - ${BLOCKED_IP}"
if [ "${TIME_DIFF_IN_MIN}" -ge ${FW_EXPIRE_TIME} ];then
        echo "need to flush"
${IPTABLE} -D INPUT -s ${BLOCKED_IP} -j DROP
#remove entry from block lidt file
${SED} -i "/${BLOCKED_TIME}/d" ${BLOCKED_LIST_TCP}
else

        echo "wait and see"
fi




done

#UDP

for i in "${BLOCKED_LIST_UDP_ARRAY[@]}"
do    
BLOCKED_IP=`${ECHO} $i |${CUT} -d ":" -f4|${CUT} -d "/" -f1`
BLOCKED_TIME=`${ECHO} $i | ${CUT} -d "-" -f1`

get_time_diff ${BLOCKED_TIME}
echo "time$TIME_DIFF_IN_MIN"

        echo "ip - ${BLOCKED_IP}"
if [ "${TIME_DIFF_IN_MIN}" -ge ${FW_EXPIRE_TIME} ];then
        echo "need to flush"
${IPTABLE} -D INPUT -s ${BLOCKED_IP} -j DROP
#remove entry from block lidt file
                ${SED} -i "/${BLOCKED_TIME}/d" ${BLOCKED_LIST_UDP}


else

        echo "wait and see"
fi


done






------------------------------------------------------------------------------------------------------------


REF:-http://www.symantec.com/connect/articles/portsentry-attack-detection-part-one
REF:-http://www.symantec.com/connect/articles/portsentry-attack-detection-part-two


Wednesday, January 16, 2013

PostgreSQL Synchronous Replication on Ubuntu 12.04

#
#PostgreSQL Synchronous Replication on Ubuntu 12.04
#



1.Download postgres version postgresql-9.2.2.tar.gz from http://www.postgresql.org/ftp/source/v9.2.2/
2.Install dependencies on both servers[master-192.168.0.168,slave-192.168.0.169]

apt-get install postgresql-client-common postgresql-client-9.1 gcc libreadline-dev zlib1g-dev
3.Configure and install postgres on both servers

tar -xvzf postgresql-9.2.2.tar.gz

$./configure --prefix=/opt/postgres
$make
$make install


4.Add user postgres on both master and slave server

5.On Master server initilize database as user "postgres"
NOTE:-On both servers ownership of "/opt/postgres" should be assign to user "postgres"
[drwxr-xr-x 6 postgres root     4096 Jan 15 15:19 /opt/postgres]

$su - postgres
$cd /opt/postgres;
$./bin/initdb -D master
$vim master/postgresql.conf

listen_addresses = '*'
wal_level = hot_standby
max_wal_senders = 10

$vim master/pg_hba.conf

host    all             postgres        192.168.0.169/32        trust

host     replication     postgres        192.168.0.169/32        trust




#Now start the master and see for any errors

$./bin/postgres -D master


6.On slave server

$su - postgres
$cd /opt/postgres
$./bin/initdb -D slave
$vim slave/postgresql.conf
hot_standby = on

$ vim slave/recovery.conf

standby_mode = on
primary_conninfo = 'host=192.168.0.168 application_name=slave1'


#Sync master data with slave

On slave server

$cd /opt/postgres
$rsync   -av --exclude pg_xlog --exclude *.conf  postgres@192.168.0.168:/opt/postgres/master/*  slave/



7.Enabling Synchronous Replaction
#On master

$vim master/postgresql.conf
synchronous_standby_names = 'slave1'


8.Then start master and slave servers



#On master

./bin/postgres  -D slave/
LOG:  database system was shut down in recovery at 2013-01-16 11:27:01 IST
LOG:  entering standby mode
LOG:  WAL file is from different database system
DETAIL:  WAL file database system identifier is 5833920183356866431, pg_control database system identifier is 5833917520477060503.
LOG:  streaming replication successfully connected to primary
LOG:  consistent recovery state reached at 0/16D4DB8
LOG:  database system is ready to accept read only connections



#On slave

./bin/postgres  -D slave/
LOG:  database system was shut down in recovery at 2013-01-16 11:31:19 IST
LOG:  entering standby mode
LOG:  consistent recovery state reached at 0/16D4DB8
LOG:  record with zero length at 0/16D4DB8
LOG:  database system is ready to accept read only connections
LOG:  streaming replication successfully connected to primary
LOG:  redo starts at 0/16D4DB8



9.Verify sync sttus

On Master

$ su - postgres
$ psql -h localhost
postgres=# SELECT usename, application_name, client_addr, client_hostname, sync_state FROM pg_stat_replication;

 usename  | application_name |  client_addr  | client_hostname | sync_state
----------+------------------+---------------+-----------------+------------
 postgres | slave1           | 192.168.0.169 |                 | sync












REF-http://bradmontgomery.net/blog/streaming-replication-in-postgresql-91/