Simple way to automate all your manual process.
###################################################################################################################### #!/bin/bash # Date : 7/20/2018 # Script: Docbase Monitoring # Trigger : Cronjob @every ***** /{path_to sctipt}/universal_dobase_log_monitor.sh # Executable : vi universal_dobase_log_monitor.sh && chmod 755 universal_dobase_log_monitor.sh && crontab -e #-------------------------------------------------------------------------------------------------------------- ######################################################## # VARIABLES #-------------------------------------------------------------- APPNAME=`ls -l /app/documentum/dba/| grep -i "dm_start_*" |grep -v .bak |awk '{print $9}' |cut -c10- | head -1` MAILTO="meme@gmail.com" HOSTNAME=`uname -n` #----------------------------------------------------------------------------------------------------------------------- file1="/tmp/docbase-report" file2="/tmp/docbase_storage" file3="/tmp/docbase_currenterror" file4="/tmp/docbase-report" #--------------------------------------------------------- docbase() { DOCBASE_NAME=( $(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | awk '{print $3}')) cd /app/documentum/dba/log && echo creating docbase arrays echo "**********************************************************************************************" echo "**********************************************************************************************" old_IFS=$IFS IFS=$'\n' DocbaseArray=($(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | cut -d'=' -f2)) IFS=$old_IFS ###################################################################################################### STARTSCRIPT=($(for i in "${DocbaseArray[@]}"; do ls | grep -i $i| grep -i ".log"|grep -v .save| grep -v dm_krb; done)) echo "Testing existing docbases" echo ${STARTSCRIPT[@]} && len=${#STARTSCRIPT[*]} len2=${#DOCBASE_NAME[*]} for (( i=0; i<${len}; i++ )); do for (( i=0; i<${len2}; i++ )); do echo "${STARTSCRIPT[$i]}"; echo "TESTING:" "${STARTSCRIPT[$i]}";sleep 5; tail -n 2 ${STARTSCRIPT[$i]} | grep -E "(Error|Fail|error|fail)"| awk '{out=$30; for(i=1;i<=NF;i++){out=out" "$i}; print out}' | while read line; do echo "$line" >>/tmp/docbase_log_report;sleep 2;done done; done } #--------------------------------------------------------------- ################################################################################# docbase_file_exist() { file1="/tmp/docbase-report" if [ ! -f "$file1" ]; then echo "$0: File '${file1}' not found. !! Creating the file" docbase; sleep 2; echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "File created " else echo "$0: File '${file1}' Exist !! Recycle file Report content" truncate -s 0 "$file1" echo "Generating new report" docbase; sleep 2; echo "~~~~report Recycled~~~Report Created: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ " fi }
#---------------------------------------------------------------------------------- docbase_report_exist_not_empty() { DOCBASE_REPORT="/tmp/docbase_log_report" if [ -f ${DOCBASE_REPORT} ] then if [ -s ${DOCBASE_REPORT} ] then echo "Report exists and not empty" exist_not_empty; #why else echo "File exists but Empty" echo "Creating new report" creating_docbase_log_report; exist_not_empty; fi else echo "File not exists" oldreport; fi
} #--------------------------------------------------- #--------------------------------------------------------- creting_storage() { DOCBASE_NAME=( $(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | awk '{print $3}')) cd /app/documentum/dba/log && echo creating docbase arrays echo "**********************************************************************************************" old_IFS=$IFS IFS=$'\n' DocbaseArray=($(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | cut -d'=' -f2)) IFS=$old_IFS ###################################################################################################### STARTSCRIPT=($(for i in "${DocbaseArray[@]}"; do ls | grep -i $i| grep -i ".log"|grep -v .save| grep -v dm_krb; done)) echo "Testing creating docbase_storage" echo ${STARTSCRIPT[@]} && len=${#STARTSCRIPT[*]} len2=${#DOCBASE_NAME[*]} for (( i=0; i<${len}; i++ )); do for (( i=0; i<${len2}; i++ )); do echo "${STARTSCRIPT[$i]}"; tail -n 5 ${STARTSCRIPT[$i]} | grep -E "(Error|Fail|error|fail)"| head -2 | awk '{out=$13; for(i=10;i<=NF;i++){out=out" "$i}; print out}' | cut -c9-75 >> /tmp/docbase_storage.txt echo "validate file not empty again" done; done } #-----------------------------------
storage_duplicate() { ################ STORAGE DUPLICATES################### DOCBASE_DUPLICATES="`cat /tmp/docbase_storage.txt |sort|uniq --count | awk '{print $1}'| head -1`" if [ $DOCBASE_DUPLICATES > 2 ] 2>/dev/null; then echo "We have duplicate lines: ~~~~~~~Cleaning files: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" truncate -s 0 /tmp/docbase_storage else echo "No duplicate files: ~~~~~~Removing leading spaces: ~~~~~~~~~~~~~" sed -i '/^$/d' /tmp/docbase_storage fi } ############################################################
# create docbase report #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- creating_docbase_log_report() { echo "$0: File '${file3}' Missing : Creating it"; DOCBASE_NAME=( $(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | awk '{print $3}')) cd /app/documentum/dba/log && echo creating docbase arrays echo "**********************************************************************************************" old_IFS=$IFS IFS=$'\n' DocbaseArray=($(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | cut -d'=' -f2)) IFS=$old_IFS ###################################################################################################### STARTSCRIPT=($(for i in "${DocbaseArray[@]}"; do ls | grep -i $i| grep -i ".log"|grep -v .save| grep -v dm_krb; done)) echo "Testing existing docbases" echo ${STARTSCRIPT[@]} && len=${#STARTSCRIPT[*]} len2=${#DOCBASE_NAME[*]} for (( i=0; i<${len}; i++ )); do for (( i=0; i<${len2}; i++ )); do echo "${STARTSCRIPT[$i]}"; tail -n 2 ${STARTSCRIPT[$i]} | grep -E "(Error|Fail|error|fail)"| awk '{out=$30; for(i=1;i<=NF;i++){out=out" "$i}; print out}' | while read line; do echo "$line" >>/tmp/docbase_log_report;sleep 2;done echo "File created "; sleep 1; done; done }
#--------------------------------------------------------------------------------------------------------------------------------# # Error section #--------------------------------------------------------------------------------------- current_error() {
DOCBASE_NAME=( $(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | awk '{print $3}')) cd /app/documentum/dba/log && echo creating docbase arrays echo "**********************************************************************************************" old_IFS=$IFS IFS=$'\n' DocbaseArray=($(cat /app/documentum/dba/config/*/server.ini | grep -i docbase_name | cut -d'=' -f2)) IFS=$old_IFS ###################################################################################################### STARTSCRIPT=($(for i in "${DocbaseArray[@]}"; do ls | grep -i $i| grep -i ".log"|grep -v .save| grep -v dm_krb; done)) echo "Testing creating docbase_storage" echo ${STARTSCRIPT[@]} && len=${#STARTSCRIPT[*]} len2=${#DOCBASE_NAME[*]} for (( i=0; i<${len}; i++ )); do for (( i=0; i<${len2}; i++ )); do echo "${STARTSCRIPT[$i]}"; tail -n 2 DCGDOCP.log | grep -E "(ERROR|Error|error|FAIL|Fail|fail)" |head -2 | awk '{out=$13; for(i=10;i<=NF;i++){out=out" "$i}; print out}' | cut -c9-75 > /tmp/docbase_currenterror; echo "validate file not empty again" exist_not_empty; done; done; }
############################################################################## #-------------------------------------------------------------------------------------- compare_report() { FILE_STORAGE=($(cat /tmp/docbase_storage.txt|sort|uniq)) FILE_ERROR=($(cat /tmp/docbase_currenterror|sort|uniq))
if cmp -s "$FILE_STORAGE" "$FILE_ERROR"; then echo "The files match !!-: SIMILAR REPORT: NO Room for DUPLICATES: Quiting *********" echo "************************Good & Monitoring Done ! " echo "$FILE_ERROR" > /tmp/docbase_storage.txt # keeping it current rm -f /tmp/docbase_currenterror oldreport; #missing exit 1; else echo "The files are different! New Error Found !! Updating Old Report ~~~~~~~~~~~" yes| cp -r /tmp/docbase_storage /tmp/docbase_storage.old #echo "$FILE_ERROR" > /tmp/docbase_storage2.txt echo "$FILE_ERROR" > /tmp/docbase_storage.txt sleep 1; echo "Removing Leading Spaces" sed -i '/^$/d' /tmp/docbase_storage.txt echo "Sending Alert for new Errors" echo "Cleaning Duplicates bloked by reported date" FILE_REPORT=`cat /tmp/docbase_log_report|sort|uniq| cut -d' ' -f2- |uniq` echo "$FILE_REPORT" >/tmp/docbase_clean_report # mail -s " $APPNAME::Prod::$HOSTNAME DOCBASE_LOGS ALERT!" $MAILTO < /tmp/docbase_log_report mail -s " $APPNAME::Prod::$HOSTNAME DOCBASE_LOGS ALERT!" $MAILTO < /tmp/docbase_clean_report echo "creating report backup file" cp -r /tmp/docbase_log_report /tmp/docbase-report.old rm -f /tmp/docbase_log_report rm -f /tmp/docbase_clean_report exit 0 fi } #--------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------- exist_not_empty() { FILE_STORAGE="/tmp/docbase_storage.txt" FILE_ERROR="/tmp/docbase_currenterror" sed -i '/^$/d' $FILE_STORAGE sed -i '/^$/d' $FILE_ERROR
if [ -f ${FILE_STORAGE} ]; then if [ -s ${FILE_STORAGE} ]; then echo "File ${FILE_STORAGE} exists and not empty" if [ -f ${FILE_ERROR} ]; then if [ -s ${FILE_ERROR} ]; then echo "File ${FILE_ERROR} exists and not empty" echo "Comparing files" compare_report; else echo "File ${FILE_ERROR} exists but empty" echo "System has no Error to report: Quiting ********" oldreport; fi else echo "${FILE_ERROR} File: Missing" echo "System has no Error to report: quiting ******" oldreport; fi else echo " ${FILE_STORAGE} Exist but Empty !!!:: Filling Storage" creting_storage; oldreport; fi else echo "${FILE_STORAGE} : Missing!!!:: Creating Storage and mailing current report" creting_storage; echo "Cleaning Duplicates bloked by reported date" FILE_REPORT=`cat /tmp/docbase_log_report|sort|uniq| cut -d' ' -f2- |uniq` echo "$FILE_REPORT" >/tmp/docbase_clean_report mail -s " $APPNAME::Prod::$HOSTNAME DOCBASE_LOGS ALERT!" $MAILTO < /tmp/docbase_clean_report echo "creating report backup file" cp -r /tmp/docbase_log_report /tmp/docbase-report.old rm -f /tmp/docbase_log_report rm -f /tmp/docbase_clean_report fi }
########################################################### # OLD REPORTS #------------------------------------------------------------------------------ oldreport() { oldreport="/tmp/docbase-report.old" if [ -f ${oldreport} ] then if [ -s ${oldreport} ] then echo "Report exists and not empty" echo "VALIDATING TIMEFRAMES" Thirtyminutes="1800" OneHour="3600" Twohours="7200" ThreeHours="14400" #-------------------------------------------------------------- fileage="$(( $(date +"%s") - $(stat -c "%Y" $oldreport) ))" echo $fileage && \ if [ $fileage -gt "$Thirtyminutes" ]; then echo "File was modified in last 30 minutes" elif [ $fileage -gt "$OneHour" ]; then echo "File was modified in the last hour" elif [ $fileage -gt "$Twohours" ]; then echo "File was modified in the last 2 hours" echo "Resending the alert" mail -s " MAJOR ALERT: $APPNAME::Prod::$HOSTNAME DOCBASE_LOG ALERT!" $MAILTO < /tmp/docbase-report.old echo "clean the file" #rm -f /tmp/docbase-report.old elif [ $fileage -gt "$ThreeHours" ]; then echo "File was modified in the last 3 hours" echo "sending the same alert again" else echo "File is not OLD....Quiting...." fi else echo "File exists but Empty" echo "Nothing to report" fi else echo "File MISSING" echo "Try next time" fi
}
#----------------------------------------------------------------------------------------------------------- #--------------------------------------------------------------------------------------------------------------- docbase_monitor() {
set -x #----------------------------------------------------------------------------------------------------------# echo " Testing if the docbase_log file exist" docbase_file_exist; echo "-----------------------------------------------------------------------------" docbase_report_exist_not_empty; # (creates docbase if missing ( creating docbase) # validate and clear docbase duplicates #--------------------------------------------------------------------------------------------------------- storage_duplicate; # clear duplicates #--------------------------------------------------------------------------------------------------------- echo "Creating the current Error report" # Error section !! # creates current error current_error && sed -i '/^$/d' /tmp/docbase_currenterror; #--------------------------------------------------------------------------------------------------------- # create docbase_log_report echo "-----------------------------------------------------------------------------" echo "Create docbase report" creating_docbase_log_report; #--------------------------------------------------------- # DOCBASE FILE NOT EMPTY echo "Validate not empty " docbase_exist_not_empty; #--------------------------------------------------------- echo "-----------------------------------------------------------------------------" echo "Running the main script Completed" echo "Proceeding with monitoring" echo "Done & Good"
}
######################################### # main script ########################################### docbase_monitor