Splunk (syslog)
AIX
Current splunkforwarder version is 9.2
root@aix01 /root> cat /opt/splunkforwarder/etc/splunk.version VERSION=9.0.1 BUILD=82c987350fde PRODUCT=splunk PLATFORM=AIX-powerpc
Create a user splunk and group
check the right limits
root@aix01 /root> mkgroup id=2500 splunk root@aix01 /root> mkuser id=2500 pgrp=splunk groups=staff,splunk fsize=-1 data=2621440 rss=262144 splunk root@aix01 /root> lsuser -f splunk splunk: id=2500 pgrp=staff groups=staff,splunk home=/home/splunk shell=/usr/bin/ksh login=true su=true rlogin=true daemon=true admin=false umask=27 account_locked=false fsize=-1 cpu=-1 data=2139095040 stack=65536 core=0 rss=1073741824 nofiles=8192 core_hard=0
Untar the splunk forwarder package and start install, as splunk user
rootaix01 /opt> chown -R splunk.splunk /opt/splunkforwarder splunk@aix01 /home/splunk> /opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt This appears to be your first time running this version of Splunk. Splunk> CSI: Logfiles. Checking prerequisites... Checking mgmt port [8089]: open Creating: /opt/splunkforwarder/var/lib/splunk Creating: /opt/splunkforwarder/var/run/splunk Creating: /opt/splunkforwarder/var/run/splunk/appserver/i18n Creating: /opt/splunkforwarder/var/run/splunk/appserver/modules/static/css Creating: /opt/splunkforwarder/var/run/splunk/upload Creating: /opt/splunkforwarder/var/run/splunk/search_telemetry Creating: /opt/splunkforwarder/var/spool/splunk Creating: /opt/splunkforwarder/var/spool/dirmoncache Creating: /opt/splunkforwarder/var/lib/splunk/authDb Creating: /opt/splunkforwarder/var/lib/splunk/hashDb New certs have been generated in '/opt/splunkforwarder/etc/auth'. Checking conf files for problems... Invalid key in stanza [webhook] in /opt/splunkforwarder/etc/system/default/alert_actions.conf, line 229: enable_allowlist (value: false). Your indexes and inputs configurations are not internally consistent. For more information, run 'splunk btool check --debug' Done Checking default conf files for edits... Validating installed files against hashes from '/opt/splunkforwarder/splunkforwarder-9.0.1-82c987350fde-AIX-powerpc-manifest' All installed files intact. Done All preliminary checks passed. Starting splunk server daemon (splunkd)... execve: Permission denied while running command /usr/bin/startsrc Splunk boot-start is enabled. please use /usr/bin/startsrc -s splunkd to start splunk
Create ass root a service splunkd
root@aix01 /opt> /opt/splunkforwarder/bin/splunk enable boot-start -user splunk 0513-071 The splunkd Subsystem has been added. SRC subsystem group installed. SRC subsystem group is configured to run at boot. root@aix01 /opt> odmget -q subsysname="splunkd" SRCsubsys SRCsubsys: subsysname = "splunkd" synonym = "" cmdargs = "_internal_exec_splunkd" path = "/opt/splunkforwarder/bin/splunk" uid = 1601 auditid = 0 standin = "/dev/console" standout = "/dev/console" standerr = "/dev/console" action = 1 multi = 0 contact = 2 svrkey = 0 svrmtype = 0 priority = 20 signorm = 2 sigforce = 9 display = 1 waittime = 20 grpname = "splunk" root@aix01 /opt> cat /etc/inittab splunk:2:once:/usr/bin/startsrc -g splunk > /dev/console 2>&1 root@aix01 /opt> lssrc -a | grep -i splunk splunkd splunk inoperative root@aix01 /opt> ps -ef | grep splu splunk 11207102 1 2 16:41:57 - 0:00 splunkd -p 8089 start splunk 11338186 11207102 0 16:41:57 - 0:00 [splunkd pid=11207102] splunkd -p 8089 start [process-runner] root@aix01 /opt> kill 11207102 11338186 root@aix01 /opt> startsrc -s splunkd 0513-059 The splunkd Subsystem has been started. Subsystem PID is 7995758. root@aix01 /opt> ps -ef | grep splu root 6881638 10748408 0 16:47:50 pts/0 0:00 grep splu splunk 7995758 5898518 120 16:47:48 - 0:00 splunkd --nodaemon -p 8089 _internal_exec_splunkd splunk 11469220 7995758 0 16:47:50 - 0:00 [splunkd pid=7995758] splunkd --nodaemon -p 8089 _internal_exec_splunkd [process-runner] root@aix01 /opt> lssrc -a | grep -i splunk splunkd splunk 7995758 active
Debug (if needed)
root@aix01 /root> /opt/splunkforwarder/bin/splunk btool check --debug
Script to Upgrade/install splunk agent
[root@nim01]/root/scripts> cat install_splunk.sh
#!/bin/ksh93
# V1.0 initial version
splunk_version="VERSION=9.2.1"
echo "Install/Upgrade Splunk VERSION=9.2.1"
#-----------------------
create_user()
{
echo "Add user splunk"
mkgroup -a id=1500 splunk
mkuser -a id=1500 pgrp=splunk gecos='splunk' splunk
lsgroup splunk ; lsuser -a id pgrp groups splunk
chuser fsize=-1 data=2621440 rss=262144 splunk
chgrpmem -m + splunk oinstall 2>/dev/null
}
#-----------------------
create_fs()
{
echo "Create /opt/splunkforwarder filesystem"
mv /opt/splunkforwarder /opt/splunkforwarder1
mkdir /opt/splunkforwarder
chmod a+rx /opt/splunkforwarder
mklv -t jfs2 -y splunklv rootvg 1G
crfs -vjfs2 -m /opt/splunkforwarder -d splunklv -Ayes -a log=INLINE
mount /opt/splunkforwarder
chown splunk:splunk /opt/splunkforwarder
mv /opt/splunkforwarder1/* /opt/splunkforwarder
rm -r /opt/splunkforwarder1
startsrc -s splunkd
}
#-----------------------
stop_splunk()
{
echo "Stop process"
stopsrc -g splunk
for i in $(ps -ef | grep -v grep | grep splunk | grep -v install_splunk | awk '{print $2}')
do
kill $i
done
ps -ef | grep splunkd |grep -v grep
}
#-----------------------
backup()
{
mkdir -p /opt/splunkforwarder
mkdir -p /root/old
cd /opt/splunkforwarder/etc/system/; tar cvf /root/old/splunk_local.tar local
rm /etc/rc.d/init.d/splunkforwarder /etc/rc.d/rc2.d/K10splunkforwarder /etc/rc.d/rc2.d/S10splunkforwarder
}
#-----------------------
install_bin()
{
echo "Install Splunk"
mount nim01:/repository1/splunk/aix /mnt
cd /opt
tar xvf /mnt/splunkforwarder-9.2.1-78803f08aabb-AIX-powerpc.tar
umount /mnt
}
#-----------------------
rebuild_outputs()
{
echo "Overwrite: outputs.conf"
cat > /opt/splunkforwarder/etc/system/local/outputs.conf << EOF
[tcpout]
defaultGroup = default-autolb-group
[tcpout:default-autolb-group]
server = splunk-prd.xxx:9997
[tcpout-server://splunk-prd.xxx:9997]
EOF
}
#-----------------------
build_inputs()
{
echo "Overwrite: inputs.conf"
cat > /opt/splunkforwarder/etc/system/local/inputs.conf.tmp << EOF
[default]
host=myhostname
disabled=0
ignoreOlderThan = 30d
EOF
host2=`echo "host="$(hostname -s)`
cat /opt/splunkforwarder/etc/system/local/inputs.conf.tmp | sed "s/host=aixa065/$host2/" > /opt/splunkforwarder/etc/system/local/inputs.conf
rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp
}
#-----------------------
rebuild_inputs()
{
echo "Modify: inputs.conf"
cp /opt/splunkforwarder/etc/system/local/inputs.conf /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
build_inputs
grep -vp '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1 >> /opt/splunkforwarder/etc/system/local/inputs.conf
rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
}
#-----------------------
rebuild_server()
{
echo "Modify: server.conf"
cp /opt/splunkforwarder/etc/system/local/server.conf /opt/splunkforwarder/etc/system/local/server.conf.tmp
cat /opt/splunkforwarder/etc/system/local/server.conf.tmp | sed "s/^serverName\ =\ .*/serverName\ =\ $(hostname -s)/" > /opt/splunkforwarder/etc/system/local/server.conf
rm /opt/splunkforwarder/etc/system/local/server.conf.tmp
}
#-----------------------
change_owner()
{
chown -R splunk:splunk /opt/splunkforwarder
}
#-----------------------
configure()
{
echo "Configure"
su - splunk -c "/opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt"
/opt/splunkforwarder/bin/splunk enable boot-start -user splunk
startsrc -s splunkd
}
#-----------------------
check_status()
{
echo "Check"
lssrc -Ss splunkd
odmget -q subsysname="splunkd" SRCsubsys
startsrc -s splunkd
lssrc -s splunkd
/opt/splunkforwarder/bin/splunk btool check --debug
echo "Process"
ps -ef | grep -v grep | grep splunk | grep -v install_splunk
}
########################
# main
########################
version=$(cat /opt/splunkforwarder/etc/splunk.version | grep VERSION)
create_user
if [[ $(df -g | grep -c '/opt/splunkforwarder') == "0" ]]
then
stop_splunk
create_fs
fi
if [[ "$version" != "$splunk_version" ]]
then
stop_splunk
backup
install_bin
if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
then
rebuild_outputs
fi
if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
then
host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
host2=`echo "host="$(hostname -s)`
if [[ "$host1" != "$host2" ]]
then
rebuild_inputs
fi
else
build_inputs
fi
change_owner
configure
fi
if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
then
rebuild_outputs
stop_splunk
fi
if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
then
host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
host2=`echo "host="$(hostname -s)`
if [[ "$host1" != "$host2" ]]
then
rebuild_inputs
change_owner
stop_splunk
fi
else
build_inputs
change_owner
stop_splunk
fi
if [ -e /opt/splunkforwarder/etc/system/local/server.conf ]
then
host1=$(grep '^serverName' /opt/splunkforwarder/etc/system/local/server.conf | sed 's/\ //g')
host2=`echo "serverName=$(hostname -s)`
if [[ "$host1" != "$host2" ]]
then
rebuild_server
stop_splunk
change_owner
fi
fi
stopsrc -s splunkd; sleep 2; startsrc -s splunkd
check_status