server-retire-linux

server_retire_linux
supporting script is retire_san_removal_linux

code format="bash"
 * 1) !/bin/bash


 * 1)    Author:     Chad Crider        #
 * 2)    Date Written:    2/22/2010        #
 * 3)    Purpose:    Master script to    #
 * 4)            automate or assist in    #
 * 5)            as much of the        #
 * 6)            retirement process as    #
 * 7)            possible.        #
 * 8)    Version:    2.0            #
 * 1)            possible.        #
 * 2)    Version:    2.0            #

export PATH="/usr/sbin:/usr/ccs/bin:/usr/bin:/usr/lib:$PATH:/etc:/usr/platform/$(/bin/uname -m)/sbin:/usr/tools/bin:/usr/bin:~/.bin:/usr/sfw/bin/:/usr/local/ bin"

LOCAL="y" FQHOST=`cat /etc/hosts | grep -v "#" | grep "$(hostname)" | awk '{print $2}'` clear echo "You are currently running this script on `echo $FQHOST | tr '[:lower:]' '[:upper:]'`" echo "" echo -e "Is this the host that's being retired? [y/n] \c " read answer

if [ "$answer" = "n" ]; then LOCAL="n" echo "" echo "*******************************************************" echo "* Please note that certain menu items can ONLY be run *" echo "* Directly on the host being retired.                *" echo "*******************************************************" echo "" echo "Please enter FQDN of host to be retired:" read FQHOST fi export FQHOST SHOST=`echo $FQHOST | tr "." " " | awk '{print $1}'` export SHOST DOMAIN=`echo $FQHOST | tr "." " " | awk '{print $2"."$3"."$4}'`
 * 1) Pull info from Hostedit

URL="" wget -q -O /tmp/$FQHOST $URL

grep "Machine not in host database" /tmp/$FQHOST > /dev/null && { echo "Unable to download the required info from HostEdit, exiting script."; echo "Please check spelling and connectivity and re-run the script."; exit 1; }
 * 1) Make sure we got the info from hostedit by seeing if /tmp/$FQHOST contains the error string

if tail -1 | sed -e :a -e 's/<[^>]*>//g;/</N;//ba' | awk '{print $1}'` != "RHEL" ; then echo "This script can only be run on Linux boxes" exit fi
 * 1) Make sure the script's being run on a Linux box

echo -e "Enter your USERID: \c " read USERID1 export USERID="`echo "${USERID1}" | tr '[:upper:]' '[:lower:]' | cut -c -7`" EMAIL="$USERID@lilly.com" export EMAIL="$EMAIL"
 * 1) Get user's USERID, construct EMAIL, and export it
 * 1) Sanitize the user's input to ensure 7 characters, all lowercase

export FIXSENDMAIL="y" /net/cursor/scripts/retire_san_removal_solaris export FIXSENDMAIL="n"
 * 1) Make sure sendmail is configured properly for the emails to send out

if tr "-" " " | awk '{print $1}'` = "VMware" ; then OS="VM" /inq.linux | grep "/dev" | grep -vi vmware | grep -vi virtual > /tmp/rdmcheck NUMRDMS=`wc -l /tmp/rdmcheck | awk '{print $1}'` if $NUMRDMS -gt 0 ; then MESSAGE="\n************************************************************\nThis host is a VM which has RDMs attached.\nEmail sent to you with the required info for the child CR.\n************************************************************\n" echo -e "Please cut a child CR to GLO/ESX with the following info:\n\nDTN: hit the +1 week button\n\nIn the Short Desc:\n$SHOST is retiring and has $NUMRDMS RDMs.\n\nIn the notes:\n`/net/gutstools.am.lilly.com/tools/SunOS/emc/inq.linux | grep -vi vmware | grep -vi virtual`\n\n`/net/gutstools.am.lilly.com/tools/SunOS/emc/inq.linux -sym_wwn`\n\n`/net/gutstools.am.lilly.com/tools/SunOS/emc/inq.linux -clar_wwn`" | mailx -s "Child CR for $SHOST Retirement" $EMAIL fi fi
 * 1) Notify user if the machine is a VM with RDMs so the child CR can be cut

email_workplan_nosan { echo -ne "Work Plan:\n=========\nRetirement will be done according to  Version 3.0\n\nVerification:\n============\nFrom  Version 3.0:\nVerify all applicable child CR?s are closed.\n\nCommunication:\n=============\nOwner has submitted this request based on the Platfrom Retirement Approval From and is aware of change and the parties have agreed to the change.\n\nJustification:\n=============\nPlatfrom Retirement Approval Form has been signed by the system owner.\n\nTest/Backoff:\n============\nPerfrom a BMR and tape backup restore and re-attach devices to the SAN.\n" | mailx -s "Non-SAN workplan for $SHOST" $EMAIL }
 * 1) Function

retire_notification_cr { if $EUID -ne 0 ; then echo "This step must be run as root" 1>&2 exit 1 fi
 * 1) Function to generate content for UNIX NOTIFICATION CR
 * 1) Check to see if the user is root

set DAYS Sat Sun Mon Tue Wed Thu Fri Sat set MONTHS Dec Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec DAY=$((`date +%e`)) MONTH=$((`date +%m` +1)) YEAR=`date +%Y` NDAY=`date +%u` if [ $MONTH -eq "13" ] then MONTH=1 set `cal $MONTH $YEAR` shift $(($# - 1)) DAY=$1 fi TMONTH=${MONTHS[MONTH]}
 * 1) Calculate the Anticipated retirement date

business_group=`cat /tmp/$FQHOST | sed -e :a -e 's/<[^>]*>//g;/ esac

if $OS = "VM" ; then NUMHBAS="0"; else NUMHBAS=`lspci | grep -i -e fib -e hba | wc -l`; fi MODEL=`grep -A 1 "Model " /tmp/$FQHOST | tail -1 | sed -e :a -e 's/<[^>]*>//g;/</N;//ba'`
 * 1) Get some answers
 * 2) Number of HBAs
 * 1) Server Model

echo " The purpose of this CR is to document Unix server retirement requests and notify the Capacity Planning team when a server is being retired. Please fill in  the following information and assign this CR to Craig Storm.

Current Business Area of server being retired: $SUPPORT

Anticipated retirement date: $MONTH/$DAY/$YEAR

Hostname of server being retired: $FQHOST

OS: `uname` $( $OS = "VM" && echo "VM")

OS Version: `cat /etc/redhat-release`

Server Model: $MODEL

Number of CPUs: `cat /proc/cpuinfo | grep processor | wc -l`

CPU MHz (Run MHz): `cat /proc/cpuinfo | grep "model name" | awk '{print $8}' | uniq`

Amount of Memory: $(($(cat /proc/meminfo | grep MemTotal | awk '{print $2}')/1024))MB

Size and number of internal disks: `fdisk -l | grep Disk | wc -l` Total `fdisk -l | grep Disk | awk '{print $3 $4}' | sort | uniq -c`

Number of HBA ports: $NUMHBAS

Number of Network Interface ports: `ifconfig -a | grep -v ^lo | grep "^.*: " | wc -l`

If this server is clustered and is being redeployed as a standalone server, list the VCS license key:

`if [ -e /etc/llthosts ]; then /opt/VRTS/bin/vxlicrep | grep -B 1 "  Product Name                        = VERITAS Cluster Server" | head -1 | awk '{print $4}' fi`

Has this server already been assigned for redeployment: Yes ____ No ___

If yes, please identify the: Business area: Project name: Customer name:

=
====================== If the server will be sent to salvage, please do the following:

List the Veritas License Keys to be reclaimed:

1. Make sure the server is powered down. 2. Make sure that non-SAN disk drives have been removed and follow the disk drive destruction process to properly dispose of them. 3. Make sure that the server has been tagged for removal. 4. Remove the server from the rack or shelve and take it to the designated salvage area in the data center. 5. Make sure that the server has been physically moved to salvage. "; echo "Press ENTER to return to the menu or CTRL-C to exit" read junk }

retire_esm_cr {
 * 1) Function to Generate content for ESM Server Remove CR

echo " ESM-Server(s) Remove Profile This profile is used to request the removal of all ESM monitoring on any designated Server packages and/or Node Up/Down monitoring.

This request is for ESM Smarts & NetIQ removal and if a server has multiple business area monitoring packages (ie: GUTS, GBIP, MQSERIES, GUTS, or ISG) this request will also remove NetIQ packages.

If this request is only to remove NetIQ packages, please list below:

=
============ Required Information =========================

What Zone is server in - 1

Platform (Windows or UNIX) - UNIX

Physical Server(s) name (FQDN) - `echo $FQHOST | tr '[:lower:]' '[:upper:]'`

Server Physical IP address(s) to be Removed - `nslookup $FQHOST | grep Address | tail -1 | awk '{print $2}'`

Is this server clustered (Yes/No) - ` -e /etc/llthosts && echo "Yes" || echo "No"` For a clustered server, if it is a NT(Windows) or HPUX server, provide Cluster Name(s) and the virtual (logical) IP addresses associated with the clusters -

Remove SMARTS Node up/down monitoring - Yes / No: Yes

ALL NETIQ monitoring will be removed - Yes/ No: Yes

Please list all packages to be removed: ALL "; echo "Press ENTER to return to the menu or CTRL-C to exit" read junk }

retire_dcremove_cr { clear echo -e " Please select the server's disposition: 1) Redeploy 2) Trade-in 3) Salvage 4) Rebuild 5) 22/6 Cage q) Go back to menu
 * 1) Function to Generate content for DCREMOVE CR

Selection: \c " read choice

case "$choice" in 1 ) DISPOSITION=REDEPLOY;; 2 ) DISPOSITION=TRADE-IN;; 3 ) DISPOSITION=SALVAGE;; 4 ) DISPOSITION=REBUILD echo "Enter server name" read SERVER ;; 5 ) DISPOSITION="22/6 CAGE"       SPECIAL="Please place this server in the 22/6 Cage"        ;; q ) display_menu esac

business_group=`cat /tmp/$FQHOST | sed -e :a -e 's/<[^>]*>//g;/ esac

    
 * 1) Fill in contact info

echo "

Please answer as many of the following questions as possible:

Choose one of the following: (Note: UPPER CASE)
 * Reminder* Please indicate Value Cycle in Custom Field One.

$SUPPORT

Thanks.

PLEASE DO NOT COMBINE CHANGES FOR 22/5 AND 141/2 ON SAME REQUEST. (EXAMPLE: DCREMOVE FOR 2 SERVERS AT 22/5 AND 1 SERVER AT 141/2 ON SAME CR)  REQUEST WILL  BE NOT BE ACCEPTED.

Removal Process: The Platform group must power off all devices associated with the removal before facilities can complete this request.

Please complete all line items or the request will be returned to Requestor:

Process the cable removal for the following:

1. Device Name: $FQHOST

2. Which Data Center location (22/5, 141/2) `cat /tmp/$FQHOST | sed -e :a -e 's/<[^>]*>//g;/]*>//g;/]*>//g;/</N;//ba' | grep -A 1 "Floor/Room" | grep -v Floor`

6. Special information important to this request:

7. IP Address(s) to be removed: `nslookup $FQHOST | grep Address | tail -1 | awk '{print $2}'`

8. Final Disposition of Equipment: (choose one): Rebuild (Must enter new server name here)

$DISPOSITION $SERVER

-ALL QUESTIONS BELOW TO BE COMPLETED BY FACILITIES-

1. Please enter specific device location using new Grid pattern (in 22/5, use Column):

2. Cable type Being Removed (Ethernet, Rib, Terminal server, Fiber, Etc.):

3. Total number of cables being removed:

4. Cable Label Information for cables to be removed:

Ethernet     Switch __________ Slot _______  Port _______

Rib             Switch __________  Slot _______  Port _______

Terminal server  Name __________    Port _______

Fiber        Name __________    Port _______

Hub         Name __________    Port _______

Modem Phone Number: ___________

5. Number of power circuits to be removed.

6. Circuit ID(s)

7. 5. Special information important to this request:

- "; echo "Press ENTER to return to the menu or CTRL-C to exit" read junk }

retire_dnschange_cr { echo "
 * 1) Function to Generate content for DNSCHANGE CR

The following information is for a DNS REMOVE /ALIAS/CHANGE. Please fill in all information fields below.

1. Data center location : (22/5, 141/2, N/A) `cat /tmp/$FQHOST | sed -e :a -e 's/<[^>]*>//g;/</N;//ba' | grep -A 1 Location | head -2 | tail -1`

2. DNS request type ( remove, alias,change ) REMOVE

3. Description of request. $FQHOST is a VM and is being retired. Please remove DNS entry.

4. Date and time request can be processed. ASAP

5. Device owner name, phone number, and pager (if available).

6. Device location information. Please give info for existing location below. Will device be changing locations? If yes, please give new location.

EXISTING                                                         NEW

Column no.                     Column no Rack or Cabinet no. Switch name Slot Port

7. Charge No.

8. Additional information "; echo "Press ENTER to return to the menu or CTRL-C to exit" read junk }

display_menu { clear if -e /tmp/done1 ; then DONE1="DONE"; fi if  -e /tmp/done2 ; then DONE2="DONE"; fi if  -e /tmp/done3 ; then DONE3="DONE"; fi if  -e /tmp/done4 ; then DONE4="DONE"; fi if  -e /tmp/done5partial ; then DONE5="PARTIAL"; fi if  -e /tmp/done5 ; then DONE5="DONE"; fi if  -e /tmp/done6 ; then DONE6="DONE"; fi if  -e /tmp/done7 ; then DONE7="DONE"; fi if  -e /tmp/done8 ; then DONE8="DONE"; fi
 * 1) Function to Display the menu
 * 1) Check for done steps and LOCAL status

if $LOCAL = "n" ; then DONE3="DISABLED"; fi if  $LOCAL = "n" ; then DONE5="DISABLED"; fi if  $LOCAL = "n" ; then DONE8="DISABLED"; fi
 * 1) Check for disabled steps

if $OS = "VM" ; then OPTION3="3)$DONE3       Send Email Notification to Craig Storm\n\t(Satisfies section 2.2 step 3)"    OPTION7="7)$DONE7      Generate content for DNSCHANGE CR" else OPTION3="3)$DONE3       Generate content for UNIX RETIREMENT NOTIFICATION CR\n\t*MUST BE RUN AS ROOT FROM SERVER BEING RETIRED\n\t(retrieves info required for section 2.2 step 3)"    OPTION7="7)$DONE7      Generate content for DCREMOVE CR" fi
 * 1) Set Step 3 and 6 displays based on VM or not

grep ldap` && LDAP="Yes" || LDAP="No" if $LDAP = "No" ; then DONE6="DISABLED"; fi
 * 1) Check for LDAP

echo -e " - Retirement for `echo $FQHOST | tr '[:lower:]' '[:upper:]'` to be used in conjunction with  - $MESSAGE 1)$DONE1   Send Retirement Notification email to  (satisfies step 2.1 of <REMOVED>)
 * 1) Display the menu

2)$DONE2     Email yourself the latest sysreport        (retrieves info required section 2.2 step 2)

$OPTION3

4)$DONE4   Generate content for ESM Server Remove CR    (retrieves info required for section 2.2 step 4)

5)$DONE5     Process the EMC and DGC Disk Removals%        *MUST BE RUN AS ROOT FROM SERVER BEING RETIRED    *Automates the steps needed in <REMOVED>        (retreives info required for section 2.2 step 7)

v)   Verify disks removed by GSMO    (Displays inq output for 5 seconds)

6)$DONE6     LDAP found? $LDAP        Remove LDAP Netgroup        (assists in section 2.2 step 11)

NOTE - For step section 2.2 step 11, remove LDAP netgroup if applicable using step 6 above or <REMOVED> or LDIF files on a LDAP Master Server

$OPTION7 (retrieves info required for section 2.2 step 14)

8)$DONE8   Perform graceful 30 seconds shutdown of host    *MUST BE RUN AS ROOT FROM SERVER BEING RETIRED    (satisfies section 2.2 step 12)

Selection [1-8,q]: \c " read choice

case "$choice" in 1 ) # Send Retirement Notification email to <REMOVED>       su $USERID -c 'echo | mailx -s "$SHOST is going down for retirement " <REMOVED>' || { echo "mailx couldn't send email, exiting."; exit 1; }        echo "Email sent"        sleep 1        touch /tmp/done1        display_menu        ;;    2 ) # Email yourself the latest sysreport sed '1,/<OL>/ d' /tmp/$FQHOST | sed -e :a -e 's/<[^>]*>//g;/</ N;//ba' > /tmp/report su $USERID -c 'mailx -s " Latest Sysreport for $SHOST" $EMAIL < /tmp/report' || { echo "mailx couldn't send email, exiting."; exit 1; } echo "Email sent" rm /tmp/report sleep 1 touch /tmp/done2 display_menu ;;   3 ) # Generate content for UNIX RETIREMENT NOTIFICATION CR        if  $LOCAL = "n" ; then                        echo "This step MUST be run from the host being retired, exiting."            sleep 2            display_menu                fi        if  $OS = "VM" ; then            su $USERID -c 'echo | mailx -s "NOTIFICATION - The VM $FQHOST is retiring " craig_storm@lilly.com'        else            retire_notification_cr        fi        touch /tmp/done3        display_menu        ;;    4 ) # Generate content for ESM Server Remove CR        retire_esm_cr touch /tmp/done4 display_menu ;;   5 ) # Process the EMC and DGC Disk Removals            if  $DONE5 = "DISABLED" ; then                    echo "This step has either been disabled or MUST be run from the host being retired, exiting."                    sleep 2            display_menu                fi

echo "This step will perform changes to the system including umounting" echo "filesystems, stopping some key services, and forcefully removing physical volumes." echo "Last chance to turn back before changes are done." echo -e "Proceed? [y/n] \c " read answer if [ "$answer" = "y" ]; then if -x /net/cursor/scripts/retire_san_removal_linux ; then /net/cursor/scripts/retire_san_removal_linux display_menu else echo "/net/cursor/scripts not accessible" sleep 2 display_menu fi               else display_menu fi       ;; v ) # Verify disks removed by running inq       <REMOVED>/inq.linux -clariion | egrep "EMC|DGC"        sleep 5        display_menu        ;;        6 ) # Remove LDAP Netgroup if $DONE6 = "DISABLED" ; then echo "This box doesn't use LDAP. Step disabled." sleep 2 display_menu fi               # Send the needed variables to the supporting script export REMOVENETGROUP="y" export NETGROUP="$SHOST" /net/cursor/scripts/retire_san_removal_solaris export REMOVENETGROUP="n" display_menu ;;

7 ) # Generate content for DCREMOVE CR       if  $OS = "VM" ; then            retire_dnschange_cr        else            retire_dcremove_cr        fi        touch /tmp/done7                display_menu                ;;    8 ) # Perform graceful 30 seconds shutdown of host if $LOCAL = "n" ; then echo "This step MUST be run from the host being retired, exiting." sleep 2 display_menu fi       echo "Shutting Down $SHOST..." echo -e "Are you sure you want to power host down completely? [y/n] \c " read answer if [ "$answer" = "y" ]; then /sbin/poweroff # Monitor ping to know when server is down while ping $FQHOST > /dev/null 2>&1; do sleep 7; echo "$FQHOST is still up"; done; echo "$FQHOST is down...(stopped responding to ping)" else display_menu fi       ;; q ) # Quit script       exit 0        ;;    * ) # For invalid menu choices, re-run the menu echo "Invalid selection" sleep 2 display_menu ;;   esac

}

if $LOCAL = "y" ; then clear echo "Checking for SAN devices..."
 * 1) Check for SAN

# For physical servers if $OS != "VM" ; then <REMOVED>/inq.linux | egrep -e 'EMC|DGC' > /tmp/santemp & PID="$!" i=1 sp="/-\|" echo -n ' ' while [ -d /proc/$PID ] do                           printf "\b${sp:i++%${#sp}:1}" done # Disable option 5 if the server has no SAN attached grep -v Copyright /tmp/santemp > /tmp/sanresults ! -s /tmp/sanresults && DONE5="DISABLED" &&  ! -e /tmp/nosan  && email_workplan_nosan touch /tmp/nosan

# For virtual servers elif $OS="VM" ; then <REMOVED>/inq.linux | grep "/dev" | grep -vi vmware | grep -vi virtual > /tmp/rdmcheck NUMRDMS=`wc -l /tmp/rdmcheck | awk '{print $1}'` if $NUMRDMS -eq 0 ; then ! -e /tmp/nosan && email_workplan_nosan DONE5="DISABLED" touch /tmp/nosan fi   fi fi

display_menu
 * 1) Call initial menu

code