root@t24backup - /root/scripts/flashcopy > cat tsmbackup _before _cob.ksh #!/bin/ksh93 # ######################################################### # # # Script to backup the T24 production # # before COB to TSM server # # # # Written by C Sch # # Last reviewed on March 11th, 2014 # # Last change: Comments added # # # ######################################################### # ###################################################################################################################################### ###################################################################################################################################### # # Variables # ###################################################################################################################################### ###################################################################################################################################### # # This variable is required because of the sudo command. # Sudo is not exporting this variable, but it's needed because an update in the ODM is done. # export ODMDIR=/etc/objrepos # #################################### # Setting log files and timestamp #################################### # LOGTIMESTAMP=`date +“%Y%m%d%H%M”` LOGDIR=/root/scripts/flashcopy/logs LOG=$LOGDIR/tsmbackup_before_cob.$LOGTIMESTAMP.log BACKUP_LOG=$LOGDIR/check_backup_before_cob.$LOGTIMESTAMP.log # #################################### # Setting temporary folder location #################################### # TMPFOLDER=/root/scripts/flashcopy/tmp HDISKS_ALL_SERIAL=$TMPFOLDER/disk_serial.txt # #################################### # Disk configuration #################################### # HDISKS_PRODAS1=/root/scripts/flashcopy/config/before_cob_disks.prodas1 # File that contains the LUN id's of the disks from the primary application server HDISKS_PRODAS2=/root/scripts/flashcopy/config/before_cob_disks.prodas2 # File that contains the LUN id's of the disks from the second application server HDISKS_PRODDS1=/root/scripts/flashcopy/config/before_cob_disks.prodds1 # File that contains the LUN id's of the database data disks from the primary database server HDISKS_PRODDS1_TLOG=/root/scripts/flashcopy/config/before_cob_disks.prodds1.tlog # File that contains the LUN id's of the database transactional log disks from the primary database server HDISKS_GPFST24_BC=/root/scripts/flashcopy/config/before_cob_disks.gpfs_t24 # File that contains the LUN id's of the GPFS disks before COB HDISKS_GPFST24_AC=/root/scripts/flashcopy/config/after_cob_disks.gpfs_t24 # File that contains the LUN id's of the GPFS disks after COB # #################################### # GPFS settings #################################### # GPFS_PATH=/usr/lpp/mmfs/bin # GPFS binary path GPFST24=/gpfs_t24 # GPFS filesystem GPFST24_DEV=gpfs_t24 # Name of the GPFS device T24BACKUPNODE=/gpfs1/t24admin/scripts/flashcopy/config/t24backup.nodes # Name of the GPFS nodes on the T24 backup server GPFSDISKNR=4 # Number of GPFS disks that should be detected GPFSEXPORTFILE=/tmp/gpfs_t24.export # Temporary file to store the GPFS export information # #################################### # Backup archive #################################### # MOUNTPOINT1=/t24_before_cob1 # Mount point for the snapshot of primary application server MOUNTPOINT2=/t24_before_cob2 # Mount point for the snapshot of second application server VOLGRP1=t24bc1vg # Volume group name for the snapshot disks for the primary application server VOLGRP2=t24bc2vg # Volume group name for the snapshot disks for the second application server TARDIR=/t24backups/prodas/precob # Directory for the tar archives PROD1TAR=$TARDIR/before_cob.prodas1.$LOGTIMESTAMP.tar.gz # File name for the primary application server backup –> will be a compress tar archive PROD2TAR=$TARDIR/before_cob.prodas2.$LOGTIMESTAMP.tar.gz # File name for the second application server backup –> will be a compress tar archive GPFSTAR=$TARDIR/before_cob.gpfs_t24.$LOGTIMESTAMP.tar.gz # File name for the GPFS backup –> will be a compress tar archive ENOUGH_SPACE=10 # Used to check if enough space on disk for the daily backup (in GB) ENOUGH_SPACE_MONTHLY=120 # Used to check if enough space on disk for the monthly backup (in GB) ENOUGH_SPACE_YEARLY=150 # Used to check if enough space on disk for the yearly backup (in GB) BACKUP_DEV=t24backups # Local backup device to store all backups DISK_BKP_RET=720 # Retention to keep backups on disk (in minutes) ARCHIVEDBVG=t24db2archivevg # Volume group name for the database archive for the data disks –> used only for MONTHLY and YEARLy only ARCHIVETLOGVG=t24db2tlogvg # Volume group name for the database archive for the transactional log disk –> used only for MONTHLY and YEARLY # #################################### # Backup check #################################### # FILE_SIZE_THRESHOLD_AS=1000000000 # Threshold for decide if the backup on TSM is valid (in bytes) FILE_SIZE_THRESHOLD_GPFS=500000000 # Threshold for decide if the backup on TSM is valid (in bytes) CSVFILE=$TMPFOLDER/mydata.csv # CSV file to generate backup status HTML_FILE=$TMPFOLDER/t24backup_state.html # HTML file with the state of the backups PERL_SCRIPT=/root/scripts/flashcopy/generate_html.pl # Perl script to generate the HTML site with the CSV file DATE=`date “+%d/%m/%Y”` REPORT_RECIPIENTS=“support@mydomain.lu” # Recipients for the daily report. Only used after COB # #################################### # local DB2 config #################################### # DBNAME=T24PRD # Database name to backup DBINST=db2inst # Database instance DB2LOG=$LOGDIR/db2log_before_cob.$DBNAME.$LOGTIMESTAMP.log LASTLOG=$LOGDIR/lastlog_before_cob_archive.$LOGTIMESTAMP.log # #################################### # FlashCopy config #################################### FCM_INQUIRE=/root/scripts/flashcopy/tmp/fcm_inquire.log # Temporary log file to store information about the FlashCopy backups # #################################### # Monthly Backup archive #################################### # MONTHLY_ARCH_TAR_DIR=/t24backups/prodas/archives/monthly/precob # Path to store all monthly backups MONTHLY_PROD1TAR=$MONTHLY_ARCH_TAR_DIR/before_cob.prodas1.$LOGTIMESTAMP.tar.gz # Backup name for monthly backup for the primary application server MONTHLY_PROD2TAR=$MONTHLY_ARCH_TAR_DIR/before_cob.prodas2.$LOGTIMESTAMP.tar.gz # Backup name for monthly backup for the second application server MONTHLY_GPFSTAR=$MONTHLY_ARCH_TAR_DIR/before_cob.gpfs_t24.$LOGTIMESTAMP.tar.gz # Backup name for monthly backup for the gpfs file system # #################################### # Yearly Backup archive #################################### # YEARLY_ARCH_TAR_DIR=/t24backups/prodas/archives/yearly # Path to store all yearly backups YEARLY_PROD1TAR=$YEARLY_ARCH_TAR_DIR/before_cob.prodas1.$LOGTIMESTAMP.tar.gz # Backup name for yearly backup for the primary application server YEARLY_PROD2TAR=$YEARLY_ARCH_TAR_DIR/before_cob.prodas2.$LOGTIMESTAMP.tar.gz # Backup name for yearly backup for the second application server YEARLY_GPFSTAR=$YEARLY_ARCH_TAR_DIR/before_cob.gpfs_t24.$LOGTIMESTAMP.tar.gz # Backup name for yearly backup for the gpfs file system # #################################### # TSM configuation #################################### # DSMOPT=/usr/tivoli/tsm/client/ba/bin64/dsm.opt.prodbackup.daily # TSM option file for the daily backups DSMBIN=/usr/tivoli/tsm/client/ba/bin64 # TSM path to binaries DSMOPTARCHIVE=/usr/tivoli/tsm/client/ba/bin64/dsm.opt.prodbackup.archive # TSM option file for the archive (monthly and yearly) backups MGMT_MONTHLY=T24_PROD_MONTHLY # TSM management class for monthly backups MGMT_YEARLY=T24_PROD_YEARLY # TSM management class for yearly backups MGMT_DAILY=T24_PROD_DAILY # TSM management class for daily backups TSM_SEARCH=$TMPFOLDER/tsm_search.txt # Temporary file with backups queried from TSM DSM_LOG=/var/log/tsm # Path to store the TSM log files # #################################### # PRODDS1 config #################################### # PRODDS1USR=root # User to connect to the database server PRODDS1_HOST=prodds1_nc # Hostname of the database server PRODDS1CON=“ssh $PRODDS1USR@$PRODDS1_HOST” PRODDSVGGRP=prodds11grp # File system group which contains all db2 file systems # #################################### # PRODMQ config #################################### # PRODMQUSR=root # User to connect to the MQ server PRODMQ_HOST=prodmq1_nc # Hostname of the active MQ server PRODMQCON=“ssh $PRODMQUSR@$PRODMQ_HOST” # #################################### # Mail config #################################### # RECIPIENTS=“support@mydomain.lu” # Mail recipients to get problems and errors
MAILLOG=“” # #################################### # Confluence config #################################### # BACKUP_TABLE_NAME=flashcopy_backup_table.csv # CSV file name to store the backup table CONFLU_USER=confluence # User to connect to the documentation server CONLFU_HOST=confluence # Hostname of the documentation server CONFLU_CON=“$CONFLU_USER@$CONLFU_HOST” CONFLU_PATH=“/var/atlassian/application-data/confluence/script/t24prod/backups” # Path on the documentation server to store the backup table # #################################### # Creating trap file #################################### # TRAPFILESPATH=/root/scripts/flashcopy/traps # Folder which contains the trap files TRAPFILEBC=$TRAPFILESPATH/tsmbackup_before_cob.$$ # Define the trap file with the id of current script process touch $TRAPFILEBC # Create the trap file # TRAPFILEAC=`find $TRAPFILESPATH -name “tsmbackup_after_cob*”` # Check if trap files from before COB script are available # OLDTRAPFILEBC=`find $TRAPFILESPATH -name “tsmbackup_before_cob*” | grep -v $TRAPFILEBC` # Check if old trap files from before COB script are available ⇒ means that another script is still running. # ####################################################################################################### # Activate trapping, this allows to remove the pid file even, if the scripts terminated not properly ####################################################################################################### # trap 'my_exit; exit' ERR SIGINT SIGQUIT SIGTERM SIGSTOP my_exit() {
echo "Unexpected exit...Removing the trap file ..." | tee -a $LOG
rm $TRAPFILEBC
} # ###################################################################################################################################### ###################################################################################################################################### # # Functions # ###################################################################################################################################### ###################################################################################################################################### # ############################# # Function wait to continue ############################# # # This function is used to wait until we press any key to continue # function wtc {
echo "\n Press Enter to continue or Ctl-C to stop. \c"
read y
} # ############################################ # Function to print the date and timestamp ############################################ # function date_print {
echo `date +%d/%m/%Y`" "`date +%H:%M:%S"`" - "$1
} # ################################################ # Function to check if log files exists or not ################################################ # function make_logs { logtime=`date +%d/%m/%Y`“ “`date +%H:%M:%S”`
if -e $LOG then
echo "$logtime - Log file exist already! Appending logs...\n\n\n" >> $LOG
else
echo "$logtime - Creating a new log file..." > $LOG
echo "$logtime - New log file $LOG created." >> $LOG
fi } # ############################## # Function to check on errors ############################## # # This function checks the return code of the last executed command, sends a mail in case of error and exit the script. # function error_check {
if [ $RC -ne 0 ]
then
#date_print_red "An error occured! Return code: $RC. Please contact system administrator!!"
date_print "An error occured! Return code: $RC. Please contact system administrator!!" | tee -a $LOG
MAILLOG=$MAILLOG"An error occured! Return code: $RC. Please contact system administrator!!\n"
send_mail;
exit 1
fi
} # ################################## # Function to check on db2 errors ################################## # # This function checks the return code of the db2 backup command, stores it in a file and exit the script in case of errors. # function get_sql {
grep "Backup successful" $LASTLOG > /dev/null
if [ $? != 0 ]
then
# Fetch the db2 SQL or DBI return code
DB2RC=`cat $LASTLOG | awk '/^SQL*/ || /^DBI*/ {print $1}'`
case $DB2RC in
"SQL2062N") date_print "FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!"
date_print "FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!" >> $LOG
MAILLOG=$MAILLOG"FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!\n"
send_mail;
exit 1
;;
*) date_print "An error occured, check log file and contact system administrator!"
date_print "An error occured, check log file and contact system administrator!" >> $LOG
MAILLOG=$MAILLOG"An error occured, check log file and contact system administrator!"
send_mail;
exit 1
;;
esac
else
echo "Backup successfull"
# Fetch the backup timestamp
DBTIMESTAMP=`grep "Backup successful" $LASTLOG | awk -F : '{print $2}' | cut -b2-`
fi
} # ################################### # Function to remove volume groups ################################### # # This function is checking if any filesystem (of the last before COB) are mounted, unmounts, exports the volume group and removes the disks. # function remove_vg {
# Get the volume group name as parameter
VOLGRP=$1
# First check if the file systems are mounted on the T24 backup server
date_print "Checking if file systems are mounted for $VOLGRP..." >> $LOG
for i in `lsvgfs $VOLGRP | sort -r`
do
mount | grep -i "$i " >> $LOG
if [ $? -eq 0 ]
then
date_print "$i is mounted1!" | tee -a $LOG
date_print "Unmounting $i..." | tee -a $LOG
# Kill all processes that use the file system
fuser -kxuc $i >> $LOG 2> /dev/null
# Unmount all the file systems
umount -f $i >> $LOG
RC=$?
error_check;
else
date_print "$i is not mounted!" | tee -a $LOG
fi
done
# Get the hdisks to delete
HDISK_TODEL=`lsvg -p $VOLGRP | awk 'NR>2 {print $1}'`
# Vary off and export the volume group
date_print "Exporting volume group $VOLGRP..." | tee -a $LOG
varyoffvg $VOLGRP
RC=$?
error_check;
exportvg $VOLGRP
RC=$?
error_check;
date_print "Volume group $VOLGRP exported." | tee -a $LOG
# Delete the hdisks
date_print "Deleting disks..." | tee -a $LOG
for i in `echo $HDISK_TODEL`
do
rmdev -dl $i >> $LOG
RC=$?
error_check;
done
date_print "Disks deleted." | tee -a $LOG
} # ################################## # Function to list all disks serial number ################################## # # This function is used to list all disks serial number # function list_serialnumber {
pcmpath query device | egrep "SERIAL|DEVICE" | sed '/DEV/{x;p;x;}' | sed 's/DEV#://g' | sed 's/DEVICE NAME://g' | awk '{print $2}' | sed '/^$/d' | while read word
do
if [ $(echo $word | grep hdisk > /dev/null 2>&1;echo $?) -eq 0 ]
then
hd=$word
else
echo $hd";\c"
echo $word
fi
done > $HDISKS_ALL_SERIAL
} # ################################## # Function to detect new devices ################################## # # This function is used to check for new devices. # function device_detect {
date_print "Starting cfgmgr...." | tee -a $LOG
cfgmgr
pcmpath query device | egrep "SERIAL|DEVICE" | sed '/DEV/{x;p;x;}' | sed 's/DEV#://g' | sed 's/DEVICE NAME://g' | awk '{print $2}' | sed '/^$/d' | while read word
do
if [ $(echo $word | grep hdisk > /dev/null 2>&1;echo $?) -eq 0 ]
then
hd=$word
else
echo $hd";\c"
echo $word
fi
done > $HDISKS_ALL_SERIAL
} # #################################### # Function to create volume groups #################################### # # This function checks if the snapshot disks of prodas1 disks are visible, creates the volume group and mounts the file systems. # function create_vg1 {
# Define an empty variable where the disks will be added if they are found.
HDISKS_1=""
# Loop through the list of the LUN id's of the snapshot and add them to the variable if they were found.
for i in `cat $HDISKS_PRODAS1`
do
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
HDISKS_1=$HDISKS_1" "$CURRENT
done
echo $HDISKS_1
# Check if the disk list is not empty.
if [[ $HDISKS_1 != "" ]]
then
# If the list is not empty, then create the volume group and mount the file systems.
date_print "Creating volume group for PRODAS1..." | tee -a $LOG
recreatevg -L $MOUNTPOINT1 -y $VOLGRP1 $HDISKS_1 2>> $LOG
RC=$?
error_check;
date_print "Mounting PRODAS1 volume..." | tee -a $LOG
mount $MOUNTPOINT1/t24prodas
RC=$?
error_check;
date_print "Changing file systems to no automount for $VOLGRP1..." | tee -a $LOG
for i in `lsvg -l $VOLGRP1 | awk 'NR > 3 {print $7}'`
do
FS=`lsfs | grep "$i " | awk '{print $1}'`
chfs -Ano $FS
RC=$?
error_check;
done
fi
} # #################################### # Function to create volume groups #################################### # # This function checks if the snapshot disks of prodas2 disks are visible, creates the volume group and mounts the file systems. # function create_vg2 {
# Define an empty variable where the disks will be added if they are found.
HDISKS_2=""
# Loop through the list of the LUN id's of the snapshot and add them to the variable if they were found.
for i in `cat $HDISKS_PRODAS2`
do
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
HDISKS_2=$HDISKS_2" "$CURRENT
done
echo $HDISKS_2
# Check if the disk list is not empty.
if [[ $HDISKS_2 != "" ]]
then
# If the list is not empty, then create the volume group and mount the file systems.
date_print "Creating volume group for PRODAS2..." | tee -a $LOG
recreatevg -L $MOUNTPOINT2 -y $VOLGRP2 $HDISKS_2 2>> $LOG
RC=$?
error_check;
date_print "Mounting PRODAS2 volume..." | tee -a $LOG
mount $MOUNTPOINT2/t24prodas
RC=$?
error_check;
date_print "Changing file systems to no automount for $VOLGRP2..." | tee -a $LOG
for i in `lsvg -l $VOLGRP2 | awk 'NR > 3 {print $7}'`
do
FS=`lsfs | grep "$i " | awk '{print $1}'`
chfs -Ano $FS
RC=$?
error_check;
done
fi
} # ###################################################################### # Function to check if backup device is mounted and has enough space ###################################################################### # # This function checks if the backup device is mounted and if enough space is available to create all archives or backup. # The type of backup should be passed as parameter. # function check_backup_dev {
date_print "Checking if $BACKUP_DEV is mounted..." | tee -a $LOG
mount | grep -i "/$BACKUP_DEV" >> $LOG
if [ $? -eq 0 ]
then
date_print "File system is mounted. Proceeding..." | tee -a $LOG
else
date_print "File system not mounted. Mounting it..." | tee -a $LOG
mount $BACKUP_DEV >> $LOG
date_print "File system mounted. Proceeding..." | tee -a $LOG
fi
case $1 in
MONTHLY)
date_print "Checking if enough space is available on $BACKUP_DEV for monthly backups..." | tee -a $LOG
FREE_SPACE=`df -vg | grep -i "/$BACKUP_DEV" | awk '{print $4}'`
if (( FREE_SPACE > ENOUGH_SPACE_MONTHLY ))
then
date_print "$FREE_SPACE GB available. Enough space available. Proceeding..." | tee -a $LOG
else
date_print "$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG"$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!\n"
send_mail;
PRODAS1_BACKUP_STATE="NOK"
PRODAS2_BACKUP_STATE="NOK"
PRODGPFS_BACKUP_STATE="NOK"
DB_BACKUP_STATE="NOK"
echo "$DATE,After COB,$PRODAS1_BACKUP_STATE,$PRODAS2_BACKUP_STATE,$PRODGPFS_BACKUP_STATE,$DB_BACKUP_STATE" >> $CSVFILE
/usr/bin/perl $PERL_SCRIPT > $HTML_FILE
# Sending report to operating --> only after COB
# uuencode $HTML_FILE $HTML_FILE | mail -s "T24 Backup report" $REPORT_RECIPIENTS
exit 1
fi
;;
YEARLY)
date_print "Checking if enough space is available on $BACKUP_DEV for yearly backups..." | tee -a $LOG
FREE_SPACE=`df -vg | grep -i "/$BACKUP_DEV" | awk '{print $4}'`
if (( FREE_SPACE > ENOUGH_SPACE_YEARLY ))
then
date_print "$FREE_SPACE GB available. Enough space available. Proceeding..." | tee -a $LOG
else
date_print "$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG"$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!\n"
send_mail;
exit 1
fi
;;
*)
date_print "Checking if enough space is available on $BACKUP_DEV..." | tee -a $LOG
FREE_SPACE=`df -vg | grep -i "/$BACKUP_DEV" | awk '{print $4}'`
if (( FREE_SPACE > ENOUGH_SPACE ))
then
date_print "$FREE_SPACE GB available. Enough space available. Proceeding..." | tee -a $LOG
else
date_print "$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG"$FREE_SPACE GB available. Not enough space available to create archives. Please contact system administrator!\n"
send_mail;
exit 1
fi
;;
esac
} # ############################## # Function to backup to TSM ############################## # # This function is used to send daily backups to TSM. The files to backup should be passed as parameter. # function tsm_backup {
FILES_TO_BACKUP=""
# Put all arguments in an array
ARGS=("$@")
# Get the number of passed elements
ELEMENTS=$#
# Loop through the list and add the files to a variable
for ((i=0;i<ELEMENTS;i++))
do
FILES_TO_BACKUP=$FILES_TO_BACKUP" "${ARGS[$i]}
done
echo $FILES_TO_BACKUP
# Send the files to TSM
date_print "Starting archiving files to TSM..." | tee -a $LOG
$DSMBIN/dsmc archive -optfile=$DSMOPT -archmc=$MGMT_DAILY -desc="Daily backup before COB $LOGTIMESTAMP" $FILES_TO_BACKUP >> $LOG
RC=$?
error_check;
date_print "Archives backuped to TSM." | tee -a $LOG
} # ########################################## # Function for the monthly backup to TSM ########################################## # # This function is used to send monthly backups to TSM. The files to backup should be passed as parameter. # function tsm_backup_monthly {
FILES_TO_BACKUP=""
# Put all arguments in an array
ARGS=("$@")
# Get the number of passed elements
ELEMENTS=$#
# Loop through the list and add the files to a variable
for ((i=0;i<ELEMENTS;i++))
do
FILES_TO_BACKUP=$FILES_TO_BACKUP" "${ARGS[$i]}
done
echo $FILES_TO_BACKUP
# Send the files to TSM
date_print "Starting monthly archiving files to TSM..." | tee -a $LOG
$DSMBIN/dsmc archive -optfile=$DSMOPTARCHIVE -archmc=$MGMT_MONTHLY -desc="Monthly archive $LOGTIMESTAMP (Precob)" $FILES_TO_BACKUP >> $LOG
RC=$?
error_check;
date_print "Archives backuped to TSM." | tee -a $LOG
} # ########################################## # Function for the yearly backup to TSM ########################################## # # This function is used to send yearly backups to TSM. The files to backup should be passed as parameter. # function tsm_backup_yearly {
FILES_TO_BACKUP=""
# Put all arguments in an array
ARGS=("$@")
# Get the number of passed elements
ELEMENTS=$#
# Loop through the list and add the files to a variable
for ((i=0;i<ELEMENTS;i++))
do
FILES_TO_BACKUP=$FILES_TO_BACKUP" "${ARGS[$i]}
done
echo $FILES_TO_BACKUP
# Send the files to TSM
date_print "Starting yearly archiving files to TSM..." | tee -a $LOG
$DSMBIN/dsmc archive -optfile=$DSMOPTARCHIVE -archmc=$MGMT_YEARLY -desc="Yearly archive $LOGTIMESTAMP (Precob)" $FILES_TO_BACKUP >> $LOG
RC=$?
error_check;
date_print "Archives backuped to TSM." | tee -a $LOG
} # ################################################################## # Function to archive and compress the primary application server ################################################################## # # This function creates the tar archives and compresses at same time for the primary application server. # function create_archive_prodas1 {
cd $MOUNTPOINT1/t24prodas
#ARCH_INPUT="$MOUNTPOINT1/t24prodas/bnk $MOUNTPOINT1/t24prodas/TAFC $MOUNTPOINT1/t24prodas/tcserver"
# Use relativ pathes
ARCH_INPUT="bnk TAFC tcserver"
# Create archives
date_print "Creating tar file for PRODAS1..." | tee -a $LOG
tar cf - $ARCH_INPUT | gzip -c > $PROD1TAR
#tar cf - $ARCH_INPUT | cat > /dev/null #| gzip -c > $PROD1TAR | cat > /dev/null
RC=$?
error_check;
#gzip $PROD1TAR
date_print "Archive created for PRODAS1!" | tee -a $LOG
} # ################################################################# # Function to archive and compress the second application server ################################################################# # # This function creates the tar archives and compresses at same time for the second application server. # function create_archive_prodas2 {
cd $MOUNTPOINT2/t24prodas
#ARCH_INPUT="$MOUNTPOINT2/t24prodas/bnk $MOUNTPOINT2/t24prodas/TAFC $MOUNTPOINT2/t24prodas/tcserver"
# Use relativ pathes
ARCH_INPUT="bnk TAFC tcserver"
# Create archives
date_print "Creating tar file for PRODAS2..." | tee -a $LOG
tar cf - $ARCH_INPUT | gzip -c > $PROD2TAR
# tar cf - $ARCH_INPUT | gzip -c > $PROD2TAR | cat /dev/null
RC=$?
error_check;
date_print "Archive created for PRODAS2!" | tee -a $LOG
} # ################################################################# # Function to archive and compress the GPFS filesystem ################################################################# # # This function creates the tar archives and compresses at same time for the GPFS filesystem. # function create_archive_gpfs {
cd $GPFST24/t24share1
#ARCH_INPUT="$GPFST24/t24share1"
# Use relativ pathes
ARCH_INPUT="bnk.data bnk.interface bnk.arc bnk.dict"
# Create archives
date_print "Creating tar file for GPFS_T24..." | tee -a $LOG
tar cf - $ARCH_INPUT | gzip -c > $GPFSTAR
RC=$?
error_check;
date_print "Archive created for GPFS_T24!" | tee -a $LOG
} # ################################# # Function to delete old backups ################################# # # This function deletes all old backups which passed the retention. The type should be passed as argument. # function delete_old_backups {
case $1 in
MONTHLY)
date_print "Deleting all monthly backups .." | tee -a $LOG
find $MONTHLY_ARCH_TAR_DIR -name "before_cob*" -type f > /tmp/files_to_del
find $MONTHLY_ARCH_TAR_DIR -name "T24PRD*" -type f >> /tmp/files_to_del
RC=$?
error_check;
for i in `cat /tmp/files_to_del`
do
date_print "Deleting file $i" >> $LOG
rm $i
done
rm /tmp/files_to_del
;;
YEARLY)
date_print "Deleting all yearly backups .." | tee -a $LOG
find $YEARLY_ARCH_TAR_DIR -name "before_cob*" -type f > /tmp/files_to_del
find $YEARLY_ARCH_TAR_DIR -name "T24PRD*" -type f >> /tmp/files_to_del
RC=$?
error_check;
for i in `cat /tmp/files_to_del`
do
date_print "Deleting file $i" >> $LOG
rm $i
done
rm /tmp/files_to_del
;;
*)
date_print "Deleting backups older than 1 day ..." | tee -a $LOG
find $TARDIR -name "before_cob*" -type f -mmin +$DISK_BKP_RET > /tmp/files_to_del
RC=$?
error_check;
for i in `cat /tmp/files_to_del`
do
date_print "Deleting file $i" >> $LOG
rm $i
done
rm /tmp/files_to_del
;;
esac
} # ########################################## # Function to send mail in case of error ########################################## # # This function just sends a mail. # function send_mail {
if [ -n "$MAILLOG" ]
then
echo $MAILLOG | mail -s "Error during TSM backup before COB!" $RECIPIENTS
date_print "Sending mail to recipients..." >> $LOG
fi
} # ########################################################### # Function to check if no old or other script is running. ########################################################### # # This function checks if there is no old or other backup script running, otherwise we cannot continue. # function check_running_script {
if [[ $TRAPFILEAC != "" ]]
then
date_print "Before cob script to send backups to TSM is still running. Please restart this script once the other one is finished!" | tee -a $LOG
MAILLOG=$MAILLOG" Before cob script to send backups to TSM is still running. Please restart this script once the other one is finished!\n"
send_mail;
exit 1
else
echo "No AC"
fi
if [[ $OLDTRAPFILEBC != "" ]]
then
date_print "Another before cob script to send backups to TSM is still running. Please restart this script once the other one is finished!" | tee -a $LOG
MAILLOG=$MAILLOG" Another before cob script to send backups to TSM is still running. Please restart this script once the other one is finished!\n"
send_mail;
exit 1
else
echo "No old "
fi
} # ################################ # Function to remove trap file ################################ # # This function removes the trap file in case of clean exit. # function remove_trap {
echo "Removing trap file $TRAPFILEBC ..." | tee -a $LOG
rm $TRAPFILEBC
RC=$?
error_check;
} # ######################################################## # Function to check if GPFS file system is not mounted ######################################################## # # This function checks if the GPFS filesystem is mounted or not. If it is, it will be unmounted. # function check_gpfs_mount {
# Check if the filesystem is mounted.
date_print "Checking if $GPFST24_DEV is mounted..." | tee -a $LOG
$GPFS_PATH/mmlsmount $GPFST24_DEV -L | tee -a $LOG | grep -i "File system $GPFST24_DEV is not mounted."
if [ $? -eq 1 ]
then
# Unmount the filesystem if it was mounted.
date_print "$GPFST24_DEV mounted on any nodes! Unmounting it..." | tee -a $LOG
$GPFS_PATH/mmunmount $GPFST24_DEV -a -f | tee -a $LOG
date_print "$GPFST24_DEV unmounted" | tee -a $LOG
else
date_print "$GPFST24_DEV not mounted. Ready to continue." | tee -a $LOG
fi
} # ################################################### # Function to export the mounted GPFS filesystem ################################################### # # This function checks if there are still GPFS disks. If so, then it will export the filesystem. # function gpfs_export {
# Check if disks are available
$GPFS_PATH/mmlsnsd | grep -i $GPFST24_DEV
if [ $? -eq 0 ]
then
# First check if an old export file exists and remove it.
date_print "$GPFST24_DEV disks still imported." | tee -a $LOG
if [ -f $GPFSEXPORTFILE ]
then
date_print "Export file exists. Removing it..." | tee -a $LOG
rm $GPFSEXPORTFILE
fi
# Then just export the GPFS filesystem.
date_print "Exporting filesystem..." | tee -a $LOG
$GPFS_PATH/mmexportfs $GPFST24_DEV -o $GPFSEXPORTFILE
RC=$?
error_check;
date_print "$GPFST24_DEV successfully exported..." | tee -a $LOG
#rm /tmp/gpfs_t24.export
else
date_print "$GPFST24_DEV filesystem doesn't exist. Ready to continue." | tee -a $LOG
fi
} # ################################################### # Function to import the mounted GPFS filesystem ################################################### # # This function first checks if the disks after COB are visible and removes them. This to avoid to have a mismatch between before and after COB disks, # Then it will loop through the file with the LUN id's and import the right disks. # function gpfs_import {
# First remove the disks from after before COB
for i in `cat $HDISKS_GPFST24_AC`
do
#echo $i
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
echo $CURRENT #>> $LOG
if [[ $CURRENT != "" ]]
then
# Removing disks
rmdev -dl $CURRENT
fi
done
# Check if all GPFS disks were detected
j=0
for i in `cat $HDISKS_GPFST24_BC`
do
#CURRENT1=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT1=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
echo $CURRENT1 #>> $LOG
if [[ $CURRENT1 != "" ]]
then
j=`expr $j + 1`
#echo $j
fi
done
if [[ $j -eq $GPFSDISKNR ]]
then
date_print "All disks were detected. Ready to continue." | tee -a $LOG
else
date_print "Not all GPFS disks were detected. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" Not all GPFS disks were detected. Please contact system administrator!\n"
send_mail;
exit 1
fi
# Import the GPFS file system
date_print "Importing GPFS filesystem..." | tee -a $LOG
$PRODMQCON $GPFS_PATH/mmfsctl $GPFST24_DEV syncFSconfig -n $T24BACKUPNODE 2> $LOG
RC=$?
error_check;
# Put the right NSD servers.
date_print "Changing NSD servers for the imported GPFS disks..." | tee -a $LOG
for i in 1 2 3 4
do
$GPFS_PATH/mmchnsd "nsd1_$i:t24backup_nc" | tee -a $LOG
done
# Mount the filesystems.
date_print "Mounting GPFS filesystem..." | tee -a $LOG
$GPFS_PATH/mmmount $GPFST24_DEV > /dev/null
RC=$?
error_check;
} # #################################################### # Function to read arguments from flashcopy script #################################################### # # The function reads the arguments passed by the FlashCopy script. The snapshot timestamp and the backup type should be passed. # function read_args {
while getopts :t:r: arguments
do
case $arguments in
t) SNAPTIMESTAMP=$OPTARG;;
r) ARCHIVE=$OPTARG ;;
esac
done
} # ################################################################ # Function to copy the daily tar in the right archive folder ################################################################ # # This function is used only for archive backups (monthly or yearly) to copy the daily tar file in the right archive folder. # The target folder should be passed as argument. # function copy_tar {
date_print "Copying the daily archive files to $1..." | tee -a $LOG
cp $PROD1TAR $PROD2TAR $GPFSTAR $1
RC=$?
error_check;
} # ############################################ # Function to import the db2 volume group ############################################ # # The function is used only for the archive backups (monthly or yearly) to import the database snapshost disks. # As a snapshot done by FlashCopy Manager cannot be imported again and backuped with another management class it should be done manually. # function db2_import {
date_print "Starting cfgmgr...." | tee -a $LOG
cfgmgr
HDISKS_DATABASE=""
# Searching for the right data disks
for i in `cat $HDISKS_PRODDS1`
do
echo $i
lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'
if [ $? -eq 0 ]
then
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
HDISKS_DATABASE=$CURRENT"|"$HDISKS_DATABASE
fi
done
# Get all tlog disks
HDISKS_TLOG=""
for i in `cat $HDISKS_PRODDS1_TLOG`
do
echo $i
lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'
if [ $? -eq 0 ]
then
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
HDISKS_TLOG=$CURRENT"|"$HDISKS_TLOG
fi
done
# Removing the last "|" character from string
HDISKS_DATABASE=`print ${HDISKS_DATABASE%?}`
HDISKS_TLOG=`print ${HDISKS_TLOG%?}`
echo $HDISKS_DATABASE
echo $HDISKS_TLOG
lspv | grep -i none | egrep -i -v "$HDISKS_DATABASE|$HDISKS_TLOG" | awk '{print $1}'
# Remove all uneeded disks
date_print "Removing all other unassigned disks..." | tee -a $LOG
for i in `lspv | grep -i none | egrep -i -v "$HDISKS_DATABASE|$HDISKS_TLOG" | awk '{print $1}'`
do
rmdev -dl $i
done
# First remove the disks from snapshot after COB
date_print "Removing GPFS disks from after COB snapshot to avoid problems..." | tee -a $LOG
for i in `cat $HDISKS_GPFST24_AC`
do
#echo $i
#CURRENT=`lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
echo $CURRENT
if [[ $CURRENT != "" ]]
then
# Removing disks
rmdev -dl $CURRENT
fi
done
if [[ $HDISKS_DATABASE != "" ]]
then
# Fetching the first disk to import
FIRST_HDISK_DATABASE=`echo $HDISKS_DATABASE | awk -F "|" '{print $1}' `
echo $FIRST_HDISK_DATABASE
date_print "Importing db2 volume group..." | tee -a $LOG
importvg -y $ARCHIVEDBVG $FIRST_HDISK_DATABASE 2>> $LOG
RC=$?
error_check;
else
date_print "No DB2 disks found to import! Please contact your system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" No DB2 disks found to import! Please contact your system administrator!\n"
send_mail;
exit 1
fi
if [[ $HDISKS_TLOG != "" ]]
then
# Fetching the first disk to import
FIRST_HDISK_TLOG=`echo $HDISKS_TLOG | awk -F "|" '{print $1}' `
echo $FIRST_HDISK_TLOG
date_print "Importing db2 volume group..." | tee -a $LOG
importvg -y $ARCHIVETLOGVG $FIRST_HDISK_TLOG 2>> $LOG
RC=$?
error_check;
else
date_print "No DB2 transactional log disks found to import! Please contact your system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" No DB2 transactional log disks disks found to import! Please contact your system administrator!\n"
send_mail;
exit 1
fi
date_print "Mounting all DB2 file systems..." | tee -a $LOG
mount -t $PRODDSVGGRP
RC=$?
error_check;
} # ######################################################### # Function to create the DB2 database backup to archive ######################################################### # # The function rollforwards the database and backups it offline to disk. # function db2_backup {
# First start the database instance
date_print "Starting the db2 instance..." | tee -a $LOG
su - db2inst -c db2start | tee -a $DB2LOG
# Then roll forward the database to the end
date_print "Rolling the database forward to end of logs..." | tee -a $LOG
su - db2inst -c db2 rollforward db $DBNAME to end of logs and stop noretrieve | tee -a $DB2LOG > $LASTLOG
RC=$?
error_check;
#get_sql;
# And take an offline backup
date_print "Starting the database archive to $1..." | tee -a $LOG
su - db2inst -c db2 backup db $DBNAME to $1 compress | tee -a $DB2LOG > $LASTLOG
RC=$?
error_check;
get_sql;
# Fetch the file name
date_print "Database successfully backuped to $1" | tee -a $LOG
ARCHIVE_PRODDB=`find $1 -name "$DBNAME*$DBTIMESTAMP*"`
echo $ARCHIVE_PRODDB
} # ############################################################ # Function to check if the daily offload is already done ############################################################ # # The function checks if the daily offload is already done before importing and taking the monthly or yearly backup # function db2_check_daily_offload {
# Fetch the snapshot information
date_print "Checking if the daily backup is already offloaded to TSM..." | tee -a $LOG
$PRODDS1CON su - db2inst -c tsm4acs -f inquire_detail -d $DBNAME > $FCM_INQUIRE
# Get only the state
STATE=`cat $FCM_INQUIRE | grep -i $SNAPTIMESTAMP | awk '{print $5}'`
# Get the snaphot timestamp and the TSM timestamp
SNAP_BACKUPID=`cat $FCM_INQUIRE | grep -i $SNAPTIMESTAMP | awk '{print $3}'`
TSM_BACKUPID=`cat $FCM_INQUIRE | grep -i $SNAPTIMESTAMP | awk '{print $4}'`
echo $STATE >> $LOG
} # ################################### # Function to export the db2 disks ################################### # # This function exports the database and exports the volume groups. # function db2_export {
date_print "Stopping the db2 instance ..." | tee -a $LOG
su - $DBINST -c db2stop >> $DB2LOG
date_print "Unmounting the database disks..." | tee -a $LOG
unmount -t $PRODDSVGGRP
RC=$?
error_check;
date_print "Varying off the database volume group..." | tee -a $LOG
varyoffvg $ARCHIVEDBVG
RC=$?
error_check;
date_print "Exporting the database volume group..." | tee -a $LOG
exportvg $ARCHIVEDBVG
RC=$?
error_check;
date_print "Varying off the transactional logs volume group..." | tee -a $LOG
varyoffvg $ARCHIVETLOGVG
RC=$?
error_check;
date_print "Exporting the transactional logs volume group..." | tee -a $LOG
exportvg $ARCHIVETLOGVG
RC=$?
error_check;
} # ############################################################## # Function to update the web site with all timestamps ############################################################## # # The function updates the documentation website. With this backup table backups can be found. # function update_website {
# Check function to check if the offload to TSM is done.
db2_check_daily_offload;
case $STATE in
SUCCESSFUL)
# Check if state is successful. If so then put timestamp, and state in a variable.
date_print "Daily backup successfully offloaded." | tee -a $LOG
date_print "#########################################" | tee -a $BACKUP_LOG
date_print "#\t\t\tBefore COB\t\t#" | tee -a $BACKUP_LOG
date_print "#########################################" | tee -a $BACKUP_LOG
date_print "Backup state:\t $STATE" | tee -a $BACKUP_LOG
date_print "Backup ID:\t $SNAP_BACKUPID"| tee -a $BACKUP_LOG
date_print "TSM ID:\t\t $TSM_BACKUPID" | tee -a $BACKUP_LOG
date_print "#########################################" | tee -a $BACKUP_LOG
TOADD_CONFLU="Before COB ,$STATE ,$SNAP_BACKUPID ,$TSM_BACKUPID ,"
DB_BACKUP_STATE="OK"
;;
IN_PROGRESS)
# If the offload is still in progress then wait for 15 minutes and try again.
date_print "Backup before COB NOT yet offloaded! Waiting for 15 minutes and trying again..." | tee -a $LOG
sleep 900
db2_check_daily_offload;
if [[ $STATE != "SUCCESSFUL" ]]
then
date_print "Backup before COB still NOT successfully offloaded!! Please contact system administrator!"
MAILLOG=$MAILLOG" Backup before COB NOT successfully offloaded!! Please contact sysadmin administrator!\n"
send_mail;
DB_BACKUP_STATE="NOK"
else
date_print "Daily backup successfully offloaded." | tee -a $LOG
date_print "#########################################" | tee -a $BACKUP_LOG
date_print "#\t\tbefore COB\t\t\t#" | tee -a $BACKUP_LOG
date_print "#########################################" | tee -a $BACKUP_LOG
date_print "Backup state:\t $STATE" | tee -a $BACKUP_LOG
date_print "Backup ID:\t $SNAP_BACKUPID"| tee -a $BACKUP_LOG
date_print "TSM ID:\t\t $TSM_BACKUPID" | tee -a $BACKUP_LOG
date_print "#########################################" | tee -a $BACKUP_LOG
TOADD_CONFLU="Before COB ,$STATE ,$SNAP_BACKUPID ,$TSM_BACKUPID ,"
DB_BACKUP_STATE="OK"
fi
;;
*)
date_print "An unknown error occured, please contact system administrator! State: $STATE"
MAILLOG=$MAILLOG" An unknown error occured, please contact system administrator! State: $STATE\n"
send_mail;
;;
esac
date_print "Checking application and gpfs backups on TSM..." | tee -a $LOG
# Define a variable as false to check if the files are backup to TSM.
ERROR=false
# Query TSM for the files and put the list in a file.
$DSMBIN/dsmc query archive -optfile=$DSMOPT "$TARDIR/*" > $TSM_SEARCH
# Then search if archive file for prodas1 could be found.
cat $TSM_SEARCH | grep -i $PROD1TAR
if [ $? -eq 0 ]
then
# If yes then just compare the file size with a minimum size defined. This to avoid that the files has only some bytes and not complete.
date_print "Backup file $PROD1TAR is available on TSM." | tee -a $LOG
PRODAS1_TSM=`grep $PROD1TAR $TSM_SEARCH | awk '{print $5}'`
PRODAS1_TSM_SIZE=`grep $PROD1TAR $TSM_SEARCH | awk '{print $1}' | sed s/,//g`
if [ $PRODAS1_TSM_SIZE -gt $FILE_SIZE_THRESHOLD_AS ]
then
# If the file size is ok, then just put the state as ok
PRODAS1_BACKUP_STATE="OK"
else
# If the size is not ok, then set the state as NOK.
date_print "The $PROD1TAR file size is under the estimate size!" | tee -a $LOG
PRODAS1_BACKUP_STATE="NOK"
fi
else
# Set the state to NOK if the file could not be found.
date_print "The backup file $PROD1TAR is not available on TSM. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" The backup file $PROD1TAR is not available on TSM. Please contact system administrator!\n"
ERROR=true
PRODAS1_BACKUP_STATE="NOK"
fi
# Then search if archive file for prodas2 could be found.
cat $TSM_SEARCH | grep -i $PROD2TAR
if [ $? -eq 0 ]
then
# If yes then just compare the file size with a minimum size defined. This to avoid that the files has only some bytes and not complete.
date_print "Backup file $PROD2TAR is available on TSM." | tee -a $LOG
PRODAS2_TSM=`grep $PROD2TAR $TSM_SEARCH | awk '{print $5}'`
PRODAS2_TSM_SIZE=`grep $PROD2TAR $TSM_SEARCH | awk '{print $1}' | sed s/,//g`
if [ $PRODAS2_TSM_SIZE -gt $FILE_SIZE_THRESHOLD_AS ]
then
# If the file size is ok, then just put the state as ok
PRODAS2_BACKUP_STATE="OK"
else
# If the size is not ok, then set the state as NOK.
date_print "The $PROD2TAR file size is under the estimate size!" | tee -a $LOG
PRODAS2_BACKUP_STATE="NOK"
fi
else
# Set the state to NOK if the file could not be found.
date_print "The backup file $PROD2TAR is not available on TSM. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" The backup file $PROD2TAR is not available on TSM. Please contact system administrator!\n"
ERROR=true
PRODAS2_BACKUP_STATE="NOK"
fi
# Then search if archive file for GPFS could be found.
cat $TSM_SEARCH | grep -i $GPFSTAR
if [ $? -eq 0 ]
then
# If yes then just compare the file size with a minimum size defined. This to avoid that the files has only some bytes and not complete.
date_print "Backup file $GPFSTAR is available on TSM." | tee -a $LOG
GPFSTAR_TSM=`grep $GPFSTAR $TSM_SEARCH | awk '{print $5}'`
GPFSTAR_TSM_SIZE=`grep $GPFSTAR $TSM_SEARCH | awk '{print $1}' | sed s/,//g`
if [ $GPFSTAR_TSM_SIZE -gt $FILE_SIZE_THRESHOLD_GPFS ]
then
# If the file size is ok, then just put the state as ok
PRODGPFS_BACKUP_STATE="OK"
else
# If the size is not ok, then set the state as NOK.
date_print "The $GPFSTAR file size is under the estimate size!" | tee -a $LOG
PRODGPFS_BACKUP_STATE="NOK"
fi
else
# Set the state to NOK if the file could not be found.
date_print "The backup file $GPFSTAR is not available on TSM. Please contact system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" The backup file $GPFSTAR is not available on TSM. Please contact system administrator!\n"
ERROR=true
PRODGPFS_BACKUP_STATE="NOK"
fi
# If the there was an error then generate the HTM file and send a file
if [[ $ERROR != "false" ]]
then
# Generating HTML site for operators to check state
echo "$DATE,Before COB,$PRODAS1_BACKUP_STATE,$PRODAS2_BACKUP_STATE,$PRODGPFS_BACKUP_STATE,$DB_BACKUP_STATE" >> $CSVFILE
/usr/bin/perl $PERL_SCRIPT > $HTML_FILE
send_mail;
fi
# If there was no error then update the backup table on the documentation server.
TOADD_CONFLU=$TOADD_CONFLU"$PRODAS1_TSM ,$PRODAS2_TSM ,$GPFSTAR_TSM"
date_print "Removing old local confluence file..." | tee -a $LOG
rm $TMPFOLDER/$BACKUP_TABLE_NAME
date_print "Getting the file from confluence..." | tee -a $LOG
scp $CONFLU_CON:$CONFLU_PATH/$BACKUP_TABLE_NAME $TMPFOLDER/$BACKUP_TABLE_NAME
date_print "Updating confluence with backups..." | tee -a $LOG
echo $TOADD_CONFLU >> $TMPFOLDER/$BACKUP_TABLE_NAME
scp $TMPFOLDER/$BACKUP_TABLE_NAME $CONFLU_CON:$CONFLU_PATH
# Generating HTML site for operators to check state
echo "$DATE,Before COB,$PRODAS1_BACKUP_STATE,$PRODAS2_BACKUP_STATE,$PRODGPFS_BACKUP_STATE,$DB_BACKUP_STATE" >> $CSVFILE
/usr/bin/perl $PERL_SCRIPT > $HTML_FILE
# Sending report to operating --> only after COB
# uuencode $HTML_FILE $HTML_FILE | mail -s "T24 Backup report" $REPORT_RECIPIENTS
} # # ###################################################################################################################################### ###################################################################################################################################### # # Main prgram # ###################################################################################################################################### ###################################################################################################################################### # make_logs; check_running_script; #wtc; read_args $@; list_serialnumber; device_detect; create_vg1; device_detect; create_vg2; check_gpfs_mount; gpfs_export; gpfs_import; #wtc; delete_old_backups; check_backup_dev; create_archive_prodas1; create_archive_prodas2; create_archive_gpfs; tsm_backup $PROD1TAR $PROD2TAR $GPFSTAR case $ARCHIVE in
MONTHLY) date_print "Monthly backup starting..." | tee -a $LOG
# Delete all old monthly backups
delete_old_backups $ARCHIVE;
# Check if the backup device is ok. Enough space, file system mounted, etc...
check_backup_dev $ARCHIVE;
# Check if the daily offload to TSM was done.
db2_check_daily_offload;
# If the offload to TSM was not yet done, then wait 30 minutes and try again.
if [[ $STATE != "SUCCESSFUL" ]]
then
date_print "Waiting for 30 minutes and trying again..." | tee -a $LOG
sleep 1800
date_print "Check daily offload again..." | tee -a $LOG
db2_check_daily_offload;
# Check the state again. If it's sill not successful then leave the script and send an email.
if [[ $STATE != "SUCCESSFUL" ]]
then
date_print "Daily backup still not offloaded...Please check with system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" Daily backup still not offloaded...Please check with system administrator!\n"
send_mail;
exit 1;
fi
fi
# Copy the daily tar files to the monthly folder to avoid recreating the tar files again.
copy_tar $MONTHLY_ARCH_TAR_DIR;
# Import the db2 snapshot disks
db2_import;
# take an offline backup of DB2
db2_backup $MONTHLY_ARCH_TAR_DIR;
# Export the db2 disks again.
db2_export;
# Finally send all files to TSM.
tsm_backup_monthly $MONTHLY_PROD1TAR $MONTHLY_PROD2TAR $MONTHLY_GPFSTAR $ARCHIVE_PRODDB;
;;
YEARLY) date_print "Yearly backup starting..." | tee -a $LOG
# Delete all old yearly backups
delete_old_backups $ARCHIVE;
# Check if the backup device is ok. Enough space, file system mounted, etc...
check_backup_dev $ARCHIVE;
# Check if the daily offload to TSM was done.
db2_check_daily_offload;
# If the offload to TSM was not yet done, then wait 30 minutes and try again.
if [[ $STATE != "SUCCESSFUL" ]]
then
date_print "Waiting for 30 minutes and trying again..." | tee -a $LOG
sleep 1800
date_print "Check daily offload again..." | tee -a $LOG
db2_check_daily_offload;
# Check the state again. If it's sill not successful then leave the script and send an email.
if [[ $STATE != "SUCCESSFUL" ]]
then
date_print "Daily backup still not offloaded...Please check with system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" Daily backup still not offloaded...Please check with system administrator!\n"
send_mail;
exit 1;
fi
fi
# Copy the daily tar files to the yearly folder to avoid recreating the tar files again.
copy_tar $YEARLY_ARCH_TAR_DIR;
# Import the db2 snapshot disks
db2_import;
# take an offline backup of DB2
db2_backup $YEARLY_ARCH_TAR_DIR;
# Export the db2 disks again.
db2_export;
# Finally send all files to TSM.
tsm_backup_yearly $YEARLY_PROD1TAR $YEARLY_PROD2TAR $YEARLY_GPFSTAR $ARCHIVE_PRODDB;
;;
esac update_website; send_mail; remove_trap; root@t24backup - /root/scripts/flashcopy >
root@prodas1 - /gpfs1/t24admin/scripts/flashcopy > cat backup _before _cob.ksh #!/bin/ksh93 # ######################################################### # # # Script to backup the T24 production # # with FlashCopy # # # # Written by C Sch # # Last reviewed on March 10th, 2014 # # # ######################################################### # ######################################################### ######################################################### # # Variables # ######################################################### ######################################################### # # This variable is required because of the sudo command. # Sudo is not exporting this variable, but it's needed because an update in the ODM is done. # export ODMDIR=/etc/objrepos # ################################### # Setting log files and timestamp ################################### # LOGTIMESTAMP=`date +”%Y%m%d%H%M“` LOGDIR=/gpfs1/t24admin/scripts/flashcopy/logs LOG=$LOGDIR/backup_before_cob.$LOGTIMESTAMP.log HDISKS_ALL_SERIAL=$LOGDIR/disk_serial.txt # List of disk and serial number on T24BACKUP # ####################### # Database config ####################### # DB2PATH=/db2/V9.7/bin/db2 DBUSER=bckuser #PASSWDFILE=/t24prodas/scripts/db2bkp.enc # Encrypted password file for the bckuser #DBPASSWD=`cat $PASSWDFILE | openssl enc -d -base64` # Get the password DBNAME=“T24PRD” # Database name to backup DBINST=db2inst # Database instance DBUSER_SSHKEY=”/home/$DBUSER/.ssh/id_rsa“ # Private SSH key for the bckuser DB2LOG=$LOGDIR/db2log.$DBNAME.backup_before_cob.$LOGTIMESTAMP.log # Datbase log file –> stores the result of all database commands DBHOSTNAME=prodds1_nc # Hostname of the database server LASTLOG=$LOGDIR/lastlog.$LOGTIMESTAMP.log # Log file to store the last return code of a db2 command DB2_SSHCON=“ssh -i $DBUSER_SSHKEY $DBUSER@$DBHOSTNAME” FCMFCGRP=“FCMH5APUBU6PROD” # FlashCopy consistency group –> used to check if another FlashCopy is running # ####################### # FlashCopy config ####################### # SVC=svc-cl-m # SVC controller for the FlashCopies SVC_USER=flashcopy # User on the SVC used to do the FlashCopies SSH_KEY=/gpfs1/t24admin/scripts/flashcopy/ssh_key/flashcopy_private.rsa SVC_CON=“ssh -i ${SSH_KEY} ${SVC_USER}@${SVC}” # ###################### # GPFS settings ###################### # GPFS_PATH=/usr/lpp/mmfs/bin/ # GPFS binary path GPFS_DEV=gpfs_t24 # GPFS device to FlashCopy T24BACKUPNODE=/gpfs1/t24admin/scripts/flashcopy/config/t24backup.nodes # File which contains the GPFS node name for the T24 Backup –> used to import the configuration after snapshot # ##################### # Prodas config ##################### # PRODASUSR=root # User used to connect to the other application server case `hostname` in # Check which is the local and the remote application server prodas1) PRODHOST_REMOTE=prodas2_nc
PRODAS_LOCAL=PRODAS1
PRODAS_REMOTE=PRODAS2
;;
prodas2) PRODHOST_REMOTE=prodas1_nc
PRODAS_LOCAL=PRODAS2
PRODAS_REMOTE=PRODAS1
;;
esac PRODCON=“ssh $PRODASUSR@$PRODHOST_REMOTE” # Connection to the remote application server # #################### # T24BACKUP config #################### # VOLGRP1=t24bc1vg # Volume group name on the T24 backup server for the AS1 VOLGRP2=t24bc2vg # Volume group name on the T24 backup server for the AS2 T24BACKUPUSR=root # User to connect to the T24 Backup server T24BACKUP_HOST=t24backup_nc # T24 backup server hostname T24BACKUPCON=“ssh $T24BACKUPUSR@$T24BACKUP_HOST” HDISKS_GPFST24_BC=/gpfs1/t24admin/scripts/flashcopy/config/before_cob_disks.gpfs_t24 # Contains the LUN id's on the T24 backup server for the disks before COB HDISKS_GPFST24_AC=/gpfs1/t24admin/scripts/flashcopy/config/after_cob_disks.gpfs_t24 # Contains the LUN id's on the T24 backup server for the disks after COB # #################################### # Fetch backup type from parameter #################################### # if $1 == "" then
BACKUPTYPE=DAILY
else
BACKUPTYPE=`echo $1 | tr '[:lower:]' '[:upper:]'`
fi # #################### # Mail config #################### # RECIPIENTS=“support@mydomain.lu” # Mail recipients in case of problems MAILLOG=”“ # ################################################################################ # Check if a trap file for a before COB backup is running on the backup server ################################################################################ # TRAPFILESPATH=/root/scripts/flashcopy/traps # Folder on the T24 backup server which contains the trap files TRAPEFILEBC=`$T24BACKUPCON find $TRAPFILESPATH -name “tsmbackup_before_cob*”` # Searching for other trap files on the T24 backup server # ################### # Disable CTRL+C ################### # trap '' 2 # ######################################################### ######################################################### # # Functions # ######################################################### ######################################################### # ########################################### # Function to print the date and timestamp ########################################### # function date_print {
echo `date +%d/%m/%Y`" "`date +%H:%M:%S"`" - "$1
} # ##################################################### # Function to print with the timestamp in bold white ##################################################### # function date_print_white {
printf "\E[22;1m"`date +%d/%m/%Y`" "`date +%H:%M:%S`" - $1\E[0m\n"
} # ##################################################### # Function to print with the timestamp in bold green ##################################################### # function date_print_green {
printf "\E[32;1m"`date +%d/%m/%Y`" "`date +%H:%M:%S`" - $1\E[0m\n"
} # ##################################################### # Function to print with timestamp in bold red ##################################################### # function date_print_red {
printf "\E[31;1m"`date +%d/%m/%Y`" "`date +%H:%M:%S`" - $1\E[0m\n"
} # ##################################################### # Function to check if log files exists or not ##################################################### # # This function creates the log file. # function make_logs { logtime=`date +%d/%m/%Y`” “`date +%H:%M:%S”`
if -e $LOG then
echo "$logtime - Log file exist already! Appending logs...\n\n\n" >> $LOG
else
echo "$logtime - Creating a new log file..." > $LOG
echo "$logtime - New log file $LOG created." >> $LOG
touch $LASTLOG
touch $DB2LOG
echo "$logtime - Making log files accessible for T24Admins..." >> $LOG
chgrp t24admin $LOG $LASTLOG $DB2LOG
fi } # ##################################################### # Function to get SQLCode from DB2 command ##################################################### # # This function filters on the last SQL or DBI code of the last executed DB2 command. # function get_sql {
grep "Backup successful" $LASTLOG > /dev/null
if [ $? != 0 ]
then
# Filter just the SQL or DBI return code of the last DB2 command
DB2RC=`cat $LASTLOG | awk '/^SQL*/ || /^DBI*/ {print $1}'`
case $DB2RC in
"SQL2062N") date_print_red "FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!"
date_print "FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!" >> $LOG
MAILLOG=$MAILLOG"FlashCopy problem. Return Code: $DB2RC!! An error occurred while accessing media \"libacsdb2.a\". Please contact system administrator!\n"
send_mail;
exit 1
;;
*) date_print_red "An error occured, check log file and contact system administrator!"
date_print "An error occured, check log file and contact system administrator!" >> $LOG
MAILLOG=$MAILLOG"An error occured, check log file and contact system administrator!"
send_mail;
exit 1
;;
esac
else
# Get the timestamp of the DB2 backup
DBTIMESTAMP=`grep "Backup successful" $LASTLOG | awk -F : '{print $2}' | cut -b2-`
echo $DBTIMESTAMP >> $LOG
fi
} # ################################### # Function to backup the database ################################### # # This function backups the database with FlashCopy Manager. The target disks are defined by using the corresponding device class. # function backup_db {
# Load the db2 environment
#. ~t24db2/sqllib/db2profile
date_print_white "Starting database backup with FlashCopy Manager for $DBNAME..."
date_print "Starting database backup with FlashCopy Manager for $DBNAME..." >> $LOG
# Do an SSH connectio on the database server and take a snapshot of the DB
$DB2_SSHCON ". ~$DBINST/.profile ; db2 backup db $DBNAME online use snapshot options DEVICE_CLASS=BACKUP include logs" | tee -a $DB2LOG > $LASTLOG
RC=$?
error_check;
get_sql;
date_print_green "Database backup with FlashCopy Manager for $DBNAME finished successfully."
date_print "Database backup with FlashCopy Manager for $DBNAME finished successfully." >> $LOG
} # ############################### # Function to check on errors ############################### # # This function checks the return code of the commands started before. # function error_check {
if [ RC -ne 0 ]
then
date_print_red "An error occured! Return code: $RC. Please contact system administrator!!"
date_print "An error occured! Return code: $RC. Please contact system administrator!!" | tee -a $LOG
MAILLOG=$MAILLOG"An error occured! Return code: $RC. Please contact system administrator!!\n"
send_mail;
exit 1
fi
} # ######################################################## # Function to Backup the T24 data on application server ######################################################## # # This function puts the disks on both application servers in suspend mode, starts the disk snapshot and resumes the disks. # function backup_t24data {
date_print "Setting the T24 volume groups in drain mode to avoid I/O on $PRODAS_LOCAL" | tee -a $LOG
# Writing buffered files to the hard disk and update the inodes on the local server
sync;sync;sync
# Put all volume groups except rootvg in suspend mode to avoid I/O's on the local server
for i in `lsvg | grep -v -i rootvg`
do
chvg -S $i
RC=$?
error_check;
done
date_print "Setting the T24 volume groups in drain mode to avoid I/O on $PRODAS_REMOTE" | tee -a $LOG
# Writing buffered files to the hard disk and update the inodes on the remote server
$PRODCON "sync;sync;sync"
# Put all volume groups except rootvg in suspend mode to avoid I/O's on the remote server
for i in `$PRODCON lsvg | grep -v -i rootvg`
do
$PRODCON chvg -S $i
RC=$?
error_check;
done
# Then start the FlashCopy
date_print_white "Starting T24 application backup with FlashCopy..."
date_print "Starting T24 application backup with FlashCopy..." >> $LOG
$SVC_CON startfcconsistgrp -prep prodas1 | tee -a $LOG
RC=$?
error_check;
$SVC_CON startfcconsistgrp -prep prodas2 | tee -a $LOG
RC=$?
error_check;
date_print_green "T24 application backup with FlashCopy finished successfully."
date_print "T24 application backup with FlashCopy finished successfully." >> $LOG
# Put all volume groups back in resume mode to allow new I/O's except rootvg on the local server
date_print "Resume volume groups for I/O operations on $PRODAS_LOCAL" | tee -a $LOG
for i in `lsvg | grep -v -i rootvg`
do
chvg -R $i
RC=$?
error_check;
done
date_print_white "Volume groups ready for I/Os on $PRODAS_LOCAL"
date_print "Volume groups ready for I/Os on $PRODAS_LOCAL" >> $LOG
# Put all volume groups back in resume mode to allow new I/O's except rootvg on the remote server
date_print "Resume volume groups for I/O operations on $PRODAS_REMOTE" | tee -a $LOG
for i in `$PRODCON lsvg | grep -v -i rootvg`
do
$PRODCON chvg -R $i
RC=$?
error_check;
done
date_print_white "Volume groups ready for I/Os on $PRODAS_REMOTE"
date_print "Volume groups ready for I/Os on $PRODAS_REMOTE" >> $LOG
} # ##################################### # Function to backup the GPFS disks ##################################### # # This function puts the GPFS disks in suspend mode, takes a disk snapshot and resumes the disks. # function backup_t24share1 {
# Put the GPFS file system in suspend state
date_print "Suspending GPFS for I/O operations..." | tee -a $LOG
$GPFS_PATH/mmfsctl $GPFS_DEV suspend > /dev/null
RC=$?
error_check;
# Start the FlashCopy snapshot
date_print_white "Starting GPFS backup with FlashCopy..."
date_print "Starting GPFS backup with FlashCopy..." >> $LOG
$SVC_CON startfcconsistgrp -prep prodmq_gpfs
RC=$?
error_check;
date_print_green "GPFS backup with FlashCopy finished successfully."
date_print "GPFS backup with FlashCopy finished successfully." >> $LOG
# Synchronize the configuration with the target cluster
date_print "Importing GPFS backup on T24Backup server..."| tee -a $LOG
$GPFS_PATH/mmfsctl $GPFS_DEV syncFSconfig -n $T24BACKUPNODE > /dev/null
RC=$?
error_check;
date_print "GPFS file system successfully imported." | tee -a $LOG
# Resume the file system to allow new I/O's
date_print "Resuming GPFS device for I/O operations..." | tee -a $LOG
$GPFS_PATH/mmfsctl $GPFS_DEV resume > /dev/null
RC=$?
error_check;
date_print_white "GPFS device ready for I/O operations."
date_print "GPFS device ready for I/O operations." >> $LOG
} # ############################################################# # Function to check if reverse flashcopy is running for DB2 ############################################################# # # This function checks the state of the consistency group on the SVC. If the state is different then idle or copied, then another snapshot is still running. # function flash_check_db {
date_print_white "Checking if a FlashCopy is already running for the database..."
date_print "Checking if a FlashCopy is already running for the database..." >> $LOG
BACKUP_DB_RUNNING=0
for i in `$SVC_CON lsfcconsistgrp -delim : | awk 'NR>1'`
do
FCCNAME=`echo $i | awk -F : '{print $2}'`
FCCSTATUS=`echo $i | awk -F : '{print $3}'`
case $FCCNAME in
$FCMFCGRP)
if [ $FCCSTATUS != "idle_or_copied" ]
then
BACKUP_DB_RUNNING=1
fi
;;
esac
done
if [ BACKUP_DB_RUNNING -eq 0 ]
then
#date_print_white "No restore running for the application servers."
date_print "No FlashCopy is running for the database." | tee -a $LOG
else
date_print_red "A FlashCopy is already in progress for the database!"
date_print "A FlashCopy is already in progress for the database!" | tee -a $LOG
MAILLOG=$MAILLOG"A FlashCopy is already in progress for the database!\n"
send_mail;
exit 1
fi
} # ########################################################################## # Function to check if a FlashCopy in running for the applicatin servers ########################################################################## # # This function checks the state of the consistency group on the SVC. If the state is different then idle or copied, then another snapshot is still running. # function flash_check_as {
date_print_white "Checking if a FlashCopy is already running for the application servers..."
date_print "Checking if a FlashCopy is already running for the application servers..." >> $LOG
BACKUP_AS_RUNNING=0
for i in `$SVC_CON lsfcconsistgrp -delim : | awk 'NR>1'`
do
FCCNAME=`echo $i | awk -F : '{print $2}'`
FCCSTATUS=`echo $i | awk -F : '{print $3}'`
case $FCCNAME in
"prodas1")
if [ $FCCSTATUS != "idle_or_copied" ]
then
BACKUP_AS_RUNNING=1
fi
;;
"prodas2")
if [ $FCCSTATUS != "idle_or_copied" ]
then
BACKUP_AS_RUNNING=1
fi
;;
esac
done
if [ BACKUP_AS_RUNNING -eq 0 ]
then
#date_print_white "No restore running for the application servers."
date_print "No FlashCopy is running for the application servers." | tee -a $LOG
else
date_print_red "A FlashCopy is already in progress for the application servers!"
date_print "A FlashCopy is already in progress for the application servers!" | tee -a $LOG
MAILLOG=$MAILLOG"A FlashCopy is already in progress for the application servers!\n"
send_mail;
exit 1
fi
} # ########################################################## # Function to check if FlashCopy is running for the GPFS ########################################################## # # This function checks the state of the consistency group on the SVC. If the state is different then idle or copied, then another snapshot is still running. # function flash_check_gpfs {
date_print_white "Checking if a FlashCopy is already running for the GPFS..."
date_print "Checking if a FlashCopy is already running for the GPFS..." >> $LOG
RESTORE_GPFS_RUNNING=0
for i in `$SVC_CON lsfcconsistgrp -delim : | awk 'NR>1'`
do
FCCNAME=`echo $i | awk -F : '{print $2}'`
FCCSTATUS=`echo $i | awk -F : '{print $3}'`
case $FCCNAME in
"prodmq_gpfs")
if [ $FCCSTATUS != "idle_or_copied" ]
then
RESTORE_GPFS_RUNNING=1
fi
;;
esac
done
if [ $RESTORE_GPFS_RUNNING -eq 0 ]
then
date_print "No backup running for the GPFS." | tee -a $LOG
else
date_print_red "A FlashCopy is already in progress for the GPFS!"
date_print "A FlashCopy is already in progress for the GPFS!" | tee -a $LOG
MAILLOG=$MAILLOG"A FlashCopy is already in progress for the GPFS!\n"
send_mail;
exit 1
fi
} # #################################### # Function to start TSM archiving #################################### # # This function starts a script on the T24 backup server to archive the snapshots on TSM. # The database snapshot timestamp and the archive retention (DAILY, MONTHLY or YEARLY) are required # function tsm_archive {
date_print "Sending backups to TSM..." | tee -a $LOG
$T24BACKUPCON -f /root/scripts/flashcopy/tsmbackup_before_cob.ksh -r $1 -t $2 > /dev/null
} # ################################## # Function to list all disks serial number ################################## # # This function is used to list all disks serial number # function list_serialnumber {
$T24BACKUPCON pcmpath query device | egrep "SERIAL|DEVICE" > $HDISKS_ALL_SERIAL.tmp
cat $HDISKS_ALL_SERIAL.tmp | sed '/DEV/{x;p;x;}' | sed 's/DEV#://g' | sed 's/DEVICE NAME://g' | awk '{print $2}' | sed '/^$/d' | while read word
do
if [ $(echo $word | grep hdisk > /dev/null 2>&1;echo $?) -eq 0 ]
then
hd=$word
else
echo $hd";\c"
echo $word
fi
done > $HDISKS_ALL_SERIAL
} # # ##################################################### # Function to check if volume groups exists already ##################################################### # # This function checks the volume group on T24 backup server if the volume group and file systems exists. # If the volume group exists already, it will called another function to export the volume group, unmount the filesystems and remove the disks. # function check_volgrp {
date_print "Checking if volume group exists and file systems are mounted..." >> $LOG
date_print_white "Checking if volume group exists and file systems are mounted..."
# Checking if volume group exist for PRODAS1
$T24BACKUPCON lsvg | grep -i $VOLGRP1 > /dev/null
if [ $? -eq 0 ]
then
date_print "Volume group $VOLGRP1 exists already on backup server...Removing it..." | tee -a $LOG
remove_vg $VOLGRP1
else
date_print "Volume group $VOLGRP1 does not exist. Ready to continue." | tee -a $LOG
#MAILLOG=$MAILLOG="Volume group $VOLGRP1 does not exist. Ready to continue.\n"
fi
# Checking if volume group exist for PRODAS2
$T24BACKUPCON lsvg | grep -i $VOLGRP2 > /dev/null
if [ $? -eq 0 ]
then
date_print "Volume group $VOLGRP2 exists already...Removing it..." | tee -a $LOG
remove_vg $VOLGRP2
else
date_print "Volume group $VOLGRP2 does not exist. Ready to continue." | tee -a $LOG
#MAILLOG=$MAILLOG"Volume group $VOLGRP2 does not exist. Ready to continue.\n"
fi
} # # #################################### # Function to remove volume groups #################################### # # This function is checking if any filesystem (of the last before COB) are mounted, unmounts, exports the volume group and removes the disks. # function remove_vg {
# Get the volume group name as parameter
VOLGRP=$1
# First check if the file systems are mounted on the T24 backup server
date_print "Checking if file systems are mounted for $VOLGRP..." >> $LOG
for i in `$T24BACKUPCON lsvgfs $VOLGRP | sort -r`
do
$T24BACKUPCON mount | grep -i "$i " >> $LOG
if [ $? -eq 0 ]
then
date_print "$i is mounted1!" | tee -a $LOG
date_print "Unmounting $i..." | tee -a $LOG
# Kill all processes that use the file system
$T24BACKUPCON fuser -kxuc $i >> $LOG > /dev/null
# Unmount all the file systems
$T24BACKUPCON umount -f $i >> $LOG
RC=$?
error_check;
else
date_print "$i is not mounted!" | tee -a $LOG
fi
done
# Get the hdisks to delete
HDISK_TODEL=`$T24BACKUPCON lsvg -p $VOLGRP | awk 'NR>2 {print $1}'`
# Vary off and export the volume group
date_print "Exporting volume group $VOLGRP..." | tee -a $LOG
$T24BACKUPCON varyoffvg $VOLGRP > /dev/null
RC=$?
error_check;
$T24BACKUPCON exportvg $VOLGRP > /dev/null
RC=$?
error_check;
date_print "Volume group $VOLGRP exported on backup server." | tee -a $LOG
# Delete the hdisks
date_print "Deleting disks on backup server..." | tee -a $LOG
for i in `echo $HDISK_TODEL`
do
$T24BACKUPCON rmdev -dl $i >> $LOG > /dev/null
RC=$?
error_check;
done
date_print "Disks deleted on backup server." | tee -a $LOG
} # ############################################################################################### # Function to check if prerequirements are met on T24Backup server before starting flashcopy ############################################################################################### # # This function is checking if all prerequirements are met to start the disk snapshot. The target disks shouldn't be mounted and in use. # function check_prereq_gpfs {
# Checking which disks are visible on the backup server. If the after COB disks are visible, we don't need to unmount the file system because a possible
# backup is running for the after COB.
date_print "Checking if $GPFS_DEV is mounted on T24BACKUP server..." | tee -a $LOG
$T24BACKUPCON $GPFS_PATH/mmlsmount $GPFS_DEV -L | tee -a $LOG | grep -i "File system $GPFS_DEV is not mounted."
if [ $? -eq 1 ]
then
# Define an empty variable to start
MOUNTED_DISKS=""
# Loop through the file which contains all LUN id's for the GPFS file system disks after COB
date_print "Checking which disks are visible for the backup server..." | tee -a $LOG
for i in `cat $HDISKS_GPFST24_AC`
do
# Check each disks if it's visble by the server and if it is put the variable to AC
#CURRENT=`$T24BACKUPCON lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
echo $CURRENT
if [[ $CURRENT != "" ]]
then
MOUNTED_DISKS=AC
fi
done
# Loop through the file which contains all LUN id's for the GPFS file system disks before COB
for i in `cat $HDISKS_GPFST24_BC`
do
# Check each disks if it's visble by the server and if it is put the variable to BC
#CURRENT=`$T24BACKUPCON lscfg -vpl hdisk\*$ | grep -i $i | awk '{print $1}'`
CURRENT=`grep -i $i $HDISKS_ALL_SERIAL | awk -F';' '{print $1}'`
echo $CURRENT
if [[ $CURRENT != "" ]]
then
MOUNTED_DISKS=BC
fi
done
# Check with the variable which disks are mounted. If the before COB disks are mounted then unmount them.
if [[ $MOUNTED_DISKS == "BC" ]]
then
date_print "Before COB snapshot disks mount on backup server..." | tee -a $LOG
date_print "$GPFS_DEV mounted on any nodes! Unmounting it..." | tee -a $LOG
$T24BACKUPCON $GPFS_PATH/mmunmount $GPFS_DEV -a -f | tee -a $LOG
date_print "$GPFS_DEV unmounted" | tee -a $LOG
fi
else
date_print "$GPFS_DEV not mounted. Ready to continue." | tee -a $LOG
fi
echo $MOUNTED_DISKS
} # ############################################################################# # Function do to post poperations for the GPFS file system after FlashCopy ############################################################################# # # This function is mounting the GPFS file system and puts the right NSD servers. # function post_backup_gpfs {
# Loop through the four disks and put the right NSD servers
date_print "Changing NSD servers for the imported GPFS disks..." | tee -a $LOG
for i in 1 2 3 4
do
$T24BACKUPCON $GPFS_PATH/mmchnsd "nsd1_$i:t24backup_nc" | tee -a $LOG
done
# Then mount the file system on the T24 backup server
$T24BACKUPCON $GPFS_PATH/mmmount $GPFS_DEV | tee -a $LOG
RC=$?
error_check;
} # ############################################################################# # Function to send mail in case of error ############################################################################# # # This function is used to send mails in case of errors. # function send_mail {
if [ -n "$MAILLOG" ]
then
echo $MAILLOG | mail -s "Error during TSM backup before COB!" $RECIPIENTS
date_print "Sending mail to recipients..." >> $LOG
fi
} # ######################################################################################## # Function to check if the script to backup on TSM is running on the backup server. ######################################################################################## # # This function is used to check if another script to backup to TSM is running on the T24 backup server. # If the script is running, we cannot continue the FlashCopy because of consistency. # function check_running_archive_script {
date_print "Checking if a before COB backup script is running on the backup server..." | tee -a $LOG
if [[ "$TRAPEFILEBC" != "" ]]
then
date_print "A before cob script to send backups to TSM is still running. Please restart this script once the other one is finished or check with system administrator!" | tee -a $LOG
MAILLOG=$MAILLOG" A before cob script to send backups to TSM is still running. Please restart this script once the other one is finished or check with system administrator!\n"
send_mail;
exit 1
else
date_print "No before COB backup script running on the backup server. Ready to continue." | tee -a $LOG
fi
} # # ############################ ############################ # # Main program # ############################ ############################ # clear make_logs; check_running_archive_script; flash_check_db; flash_check_as; flash_check_gpfs; check_volgrp; check_volgrp; backup_t24data; check_prereq_gpfs; backup_t24share1; #post_backup_gpfs backup_db; tsm_archive $BACKUPTYPE $DBTIMESTAMP send_mail; root@as1 - /gpfs1/t24admin/scripts/flashcopy >
root@prodds1 - /home/db2inst/acs > cat profile
GLOBAL
ACS_DIR /home/db2inst/acs ACSD prodds1 57328 TRACE NO «<
ACSD
ACS_REPOSITORY /home/db2inst/acs/acsrepository # REPOSITORY_LABEL TSM «<
CLIENT
# BACKUPIDPREFIX DB2___ APPLICATION_TYPE DB2 TSM_BACKUP YES USE_FOR BACKUP BACKUP1 TSM_BACKUP NO USE_FOR BACKUP2 # MAX_VERSIONS ADAPTIVE # LVM_FREEZE_THAW AUTO # NEGATIVE_LIST NO_CHECK # TIMEOUT_FLASH 120 # GLOBAL_SYSTEM_IDENTIFIER DEVICE_CLASS BACKUP BACKUP1 BACKUP2 «<
CLONING
DEVICE_CLASS C1 USE_FOR_CLONING T24PRD # FLASH_DIR_LIST # DATABASE_SCHEMA SCHEMA APPLICATION_TYPE DB2 «<
OFFLOAD
BACKUP_METHOD DB2 # OPTIONS # PARALLELISM AUTO NUM_SESSIONS 5 # NUM_BUFFERS AUTO # BUFFER_SIZE AUTO # NUMBER_BACKUPS_IN_PARALLEL 0 # DATABASE_MEMORY 0 «<
DEVICE_CLASS BACKUP
COPYSERVICES_HARDWARE_TYPE SVC COPYSERVICES_PRIMARY_SERVERNAME svc-cl-m COPYSERVICES_USERNAME fcm # CLONE_DATABASE NO # SVC_COPY_RATE 80 # SVC_CLEAN_RATE 50 # SVC_GRAIN_SIZE 256 # COPYSERVICES_COMMPROTOCOL HTTPS # COPYSERVICES_CERTIFICATEFILE NO_CERTIFICATE # COPYSERVICES_SERVERPORT 5989 FLASHCOPY_TYPE INCR # COPYSERVICES_TIMEOUT 6 # RESTORE_FORCE NO # STORAGE_SYSTEM_ID # RECON_INTERVAL 12 # BACKUP_HOST_NAME PREASSIGNED_VOLUMES TARGET_SETS VOLUMES_FILE VOLUMES_FILE /home/db2inst/acs/volumes.fct «<
DEVICE_CLASS BACKUP1
COPYSERVICES_HARDWARE_TYPE SVC COPYSERVICES_PRIMARY_SERVERNAME svc-cl-m COPYSERVICES_USERNAME fcm # CLONE_DATABASE NO # SVC_COPY_RATE 80 # SVC_CLEAN_RATE 50 # SVC_GRAIN_SIZE 256 # COPYSERVICES_COMMPROTOCOL HTTPS # COPYSERVICES_CERTIFICATEFILE NO_CERTIFICATE # COPYSERVICES_SERVERPORT 5989 FLASHCOPY_TYPE INCR # COPYSERVICES_TIMEOUT 6 # RESTORE_FORCE NO # STORAGE_SYSTEM_ID # RECON_INTERVAL 12 # BACKUP_HOST_NAME PREASSIGNED_VOLUMES TARGET_SETS VOLUMES_FILE VOLUMES_FILE /home/db2inst/acs/volumes.fct «<
DEVICE_CLASS C1
COPYSERVICES_HARDWARE_TYPE SVC COPYSERVICES_PRIMARY_SERVERNAME svc-cl-m COPYSERVICES_USERNAME fcm CLONE_DATABASE YES # SVC_COPY_RATE 80 # SVC_CLEAN_RATE 50 # SVC_GRAIN_SIZE 256 # COPYSERVICES_COMMPROTOCOL HTTPS # COPYSERVICES_CERTIFICATEFILE NO_CERTIFICATE # COPYSERVICES_SERVERPORT 5989 # ALLOW_NOCOPY_FLASHCOPY NO FLASHCOPY_TYPE INCR # COPYSERVICES_TIMEOUT 6 # RESTORE_FORCE NO # STORAGE_SYSTEM_ID # RECON_INTERVAL 12 # BACKUP_HOST_NAME PREASSIGNED_VOLUMES TARGET_SETS VOLUMES_FILE VOLUMES_FILE /home/db2inst/acs/volumes_clone.fct «<
DEVICE_CLASS BACKUP2
COPYSERVICES_HARDWARE_TYPE SVC COPYSERVICES_PRIMARY_SERVERNAME svc-cl-m COPYSERVICES_USERNAME fcm # CLONE_DATABASE NO # SVC_COPY_RATE 80 # SVC_CLEAN_RATE 50 # SVC_GRAIN_SIZE 256 # COPYSERVICES_COMMPROTOCOL HTTPS # COPYSERVICES_CERTIFICATEFILE NO_CERTIFICATE # COPYSERVICES_SERVERPORT 5989 FLASHCOPY_TYPE INCR # COPYSERVICES_TIMEOUT 6 # RESTORE_FORCE NO # STORAGE_SYSTEM_ID # RECON_INTERVAL 12 # BACKUP_HOST_NAME PREASSIGNED_VOLUMES TARGET_SETS VOLUMES_FILE VOLUMES_FILE /home/db2inst/acs/volumes.fct «<
/home/db2inst/acs/volumes.fct
TARGET_SET db1
DEVICE_CLASS BACKUP TARGET_VOLUME fc_prodds1_db2_01 prodds1_db2_01 TARGET_VOLUME fc_prodds1_db2_02 prodds1_db2_02 TARGET_VOLUME fc_prodds1_db2_03 prodds1_db2_03 TARGET_VOLUME fc_prodds1_db2_04 prodds1_db2_04 TARGET_VOLUME fc_prodds1_db2_05 prodds1_db2_05 TARGET_VOLUME fc_prodds1_db2_06 prodds1_db2_06 TARGET_VOLUME fc_prodds1_db2_07 prodds1_db2_07 TARGET_VOLUME fc_prodds1_db2_08 prodds1_db2_08 TARGET_VOLUME fc_prodds1_db2_09 prodds1_db2_09 TARGET_VOLUME fc_prodds1_db2_10 prodds1_db2_10 TARGET_VOLUME fc_prodds1_db2_11 prodds1_db2_11 TARGET_VOLUME fc_prodds1_db2_12 prodds1_db2_12 TARGET_VOLUME fc_prodds1_tlog_01 prodds1_tlog_01 TARGET_VOLUME fc_prodds1_tlog_02 prodds1_tlog_02 TARGET_VOLUME fc_prodds1_tlog_03 prodds1_tlog_03 TARGET_VOLUME fc_prodds1_tlog_04 prodds1_tlog_04 «<
»> TARGET_SET db2 DEVICE_CLASS BACKUP1 TARGET_VOLUME fc_2_prodds1_db2_01 prodds1_db2_01 TARGET_VOLUME fc_2_prodds1_db2_02 prodds1_db2_02 TARGET_VOLUME fc_2_prodds1_db2_03 prodds1_db2_03 TARGET_VOLUME fc_2_prodds1_db2_04 prodds1_db2_04 TARGET_VOLUME fc_2_prodds1_db2_05 prodds1_db2_05 TARGET_VOLUME fc_2_prodds1_db2_06 prodds1_db2_06 TARGET_VOLUME fc_2_prodds1_db2_07 prodds1_db2_07 TARGET_VOLUME fc_2_prodds1_db2_08 prodds1_db2_08 TARGET_VOLUME fc_2_prodds1_db2_09 prodds1_db2_09 TARGET_VOLUME fc_2_prodds1_db2_10 prodds1_db2_10 TARGET_VOLUME fc_2_prodds1_db2_11 prodds1_db2_11 TARGET_VOLUME fc_2_prodds1_db2_12 prodds1_db2_12 TARGET_VOLUME fc_2_prodds1_tlog_01 prodds1_tlog_01 TARGET_VOLUME fc_2_prodds1_tlog_02 prodds1_tlog_02 TARGET_VOLUME fc_2_prodds1_tlog_03 prodds1_tlog_03 TARGET_VOLUME fc_2_prodds1_tlog_04 prodds1_tlog_04 «<
»> TARGET_SET db3 DEVICE_CLASS BACKUP2 TARGET_VOLUME fc_3_prodds1_db2_01 prodds1_db2_01 TARGET_VOLUME fc_3_prodds1_db2_02 prodds1_db2_02 TARGET_VOLUME fc_3_prodds1_db2_03 prodds1_db2_03 TARGET_VOLUME fc_3_prodds1_db2_04 prodds1_db2_04 TARGET_VOLUME fc_3_prodds1_db2_05 prodds1_db2_05 TARGET_VOLUME fc_3_prodds1_db2_06 prodds1_db2_06 TARGET_VOLUME fc_3_prodds1_db2_07 prodds1_db2_07 TARGET_VOLUME fc_3_prodds1_db2_08 prodds1_db2_08 TARGET_VOLUME fc_3_prodds1_db2_09 prodds1_db2_09 TARGET_VOLUME fc_3_prodds1_db2_10 prodds1_db2_10 TARGET_VOLUME fc_3_prodds1_db2_11 prodds1_db2_11 TARGET_VOLUME fc_3_prodds1_db2_12 prodds1_db2_12 TARGET_VOLUME fc_3_prodds1_tlog_01 prodds1_tlog_01 TARGET_VOLUME fc_3_prodds1_tlog_02 prodds1_tlog_02 TARGET_VOLUME fc_3_prodds1_tlog_03 prodds1_tlog_03 TARGET_VOLUME fc_3_prodds1_tlog_04 prodds1_tlog_04
