====== Splunk (syslog) ======

===== AIX =====

Current splunkforwarder version is 9.2

<cli prompt='>'>
root@aix01 /root> cat /opt/splunkforwarder/etc/splunk.version
VERSION=9.0.1
BUILD=82c987350fde
PRODUCT=splunk
PLATFORM=AIX-powerpc
</cli>

Create a user splunk and group
FIXME check the right limits
<cli prompt='>'>
root@aix01 /root> mkgroup id=2500 splunk
root@aix01 /root> mkuser id=2500 pgrp=splunk groups=staff,splunk fsize=-1 data=2621440 rss=262144 splunk

root@aix01 /root> lsuser -f splunk
splunk:
        id=2500
        pgrp=staff
        groups=staff,splunk
        home=/home/splunk
        shell=/usr/bin/ksh
        login=true
        su=true
        rlogin=true
        daemon=true
        admin=false
        umask=27
        account_locked=false

        fsize=-1
        cpu=-1
        data=2139095040
        stack=65536
        core=0
        rss=1073741824
        nofiles=8192
        core_hard=0

</cli>

Untar the splunk forwarder package and start install, as **splunk user**
<cli prompt='>'>
rootaix01 /opt> chown -R splunk.splunk /opt/splunkforwarder

splunk@aix01 /home/splunk> /opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt
This appears to be your first time running this version of Splunk.

Splunk> CSI: Logfiles.

Checking prerequisites...
        Checking mgmt port [8089]: open
                Creating: /opt/splunkforwarder/var/lib/splunk
                Creating: /opt/splunkforwarder/var/run/splunk
                Creating: /opt/splunkforwarder/var/run/splunk/appserver/i18n
                Creating: /opt/splunkforwarder/var/run/splunk/appserver/modules/static/css
                Creating: /opt/splunkforwarder/var/run/splunk/upload
                Creating: /opt/splunkforwarder/var/run/splunk/search_telemetry
                Creating: /opt/splunkforwarder/var/spool/splunk
                Creating: /opt/splunkforwarder/var/spool/dirmoncache
               Creating: /opt/splunkforwarder/var/lib/splunk/authDb
                Creating: /opt/splunkforwarder/var/lib/splunk/hashDb
New certs have been generated in '/opt/splunkforwarder/etc/auth'.
        Checking conf files for problems...
                Invalid key in stanza [webhook] in /opt/splunkforwarder/etc/system/default/alert_actions.conf, line 229: enable_allowlist (value: false).
                Your indexes and inputs configurations are not internally consistent. For more information, run 'splunk btool check --debug'
        Done
        Checking default conf files for edits...
        Validating installed files against hashes from '/opt/splunkforwarder/splunkforwarder-9.0.1-82c987350fde-AIX-powerpc-manifest'
        All installed files intact.
        Done
All preliminary checks passed.

Starting splunk server daemon (splunkd)...
execve: Permission denied
  while running command /usr/bin/startsrc
Splunk boot-start is enabled. please use /usr/bin/startsrc -s splunkd to start splunk
</cli>

Create ass root a service **splunkd**
<cli prompt='>'>
root@aix01 /opt> /opt/splunkforwarder/bin/splunk enable boot-start -user splunk
0513-071 The splunkd Subsystem has been added.
SRC subsystem group installed.
SRC subsystem group is configured to run at boot.

root@aix01 /opt> odmget -q subsysname="splunkd" SRCsubsys

SRCsubsys:
        subsysname = "splunkd"
        synonym = ""
        cmdargs = "_internal_exec_splunkd"
        path = "/opt/splunkforwarder/bin/splunk"
        uid = 1601
        auditid = 0
        standin = "/dev/console"
        standout = "/dev/console"
        standerr = "/dev/console"
        action = 1
        multi = 0
        contact = 2
        svrkey = 0
        svrmtype = 0
        priority = 20
        signorm = 2
        sigforce = 9
        display = 1
        waittime = 20
        grpname = "splunk"
        
root@aix01 /opt> cat /etc/inittab
splunk:2:once:/usr/bin/startsrc -g splunk > /dev/console 2>&1

root@aix01 /opt> lssrc -a | grep -i  splunk
 splunkd          splunk                        inoperative

root@aix01 /opt> ps -ef | grep splu
  splunk 11207102        1   2 16:41:57      -  0:00 splunkd -p 8089 start
  splunk 11338186 11207102   0 16:41:57      -  0:00 [splunkd pid=11207102] splunkd -p 8089 start [process-runner]
root@aix01 /opt> kill 11207102 11338186

root@aix01 /opt> startsrc -s splunkd
0513-059 The splunkd Subsystem has been started. Subsystem PID is 7995758.
root@aix01 /opt> ps -ef | grep splu
    root  6881638 10748408   0 16:47:50  pts/0  0:00 grep splu
  splunk  7995758  5898518 120 16:47:48      -  0:00 splunkd --nodaemon -p 8089 _internal_exec_splunkd
  splunk 11469220  7995758   0 16:47:50      -  0:00 [splunkd pid=7995758] splunkd --nodaemon -p 8089 _internal_exec_splunkd [process-runner]
  
root@aix01 /opt> lssrc -a | grep -i  splunk
 splunkd          splunk           7995758      active
</cli>

Debug (if needed)
<cli prompt='>'>
root@aix01 /root> /opt/splunkforwarder/bin/splunk  btool check --debug
</cli>

Script to Upgrade/install splunk agent
<code>
[root@nim01]/root/scripts> cat install_splunk.sh
#!/bin/ksh93
# V1.0 initial version

splunk_version="VERSION=9.2.1"

echo "Install/Upgrade Splunk VERSION=9.2.1"

#-----------------------
create_user()
{
echo "Add user splunk"
mkgroup -a id=1500 splunk
mkuser -a id=1500 pgrp=splunk gecos='splunk' splunk
lsgroup splunk ; lsuser -a id pgrp groups splunk
chuser fsize=-1 data=2621440 rss=262144 splunk

chgrpmem -m + splunk oinstall 2>/dev/null
}

#-----------------------
create_fs()
{
echo "Create /opt/splunkforwarder filesystem"
mv /opt/splunkforwarder /opt/splunkforwarder1
mkdir /opt/splunkforwarder
chmod a+rx /opt/splunkforwarder
mklv -t jfs2 -y splunklv rootvg 1G
crfs -vjfs2 -m /opt/splunkforwarder -d splunklv -Ayes -a log=INLINE
mount /opt/splunkforwarder
chown splunk:splunk /opt/splunkforwarder
mv /opt/splunkforwarder1/* /opt/splunkforwarder
rm -r /opt/splunkforwarder1
startsrc -s splunkd
}

#-----------------------
stop_splunk()
{
echo "Stop process"
stopsrc -g splunk
for i in $(ps -ef | grep -v grep | grep splunk | grep -v install_splunk | awk '{print $2}')
do
kill $i
done
ps -ef | grep splunkd |grep -v grep
}

#-----------------------
backup()
{
mkdir -p /opt/splunkforwarder
mkdir -p /root/old
cd /opt/splunkforwarder/etc/system/; tar cvf /root/old/splunk_local.tar local
rm /etc/rc.d/init.d/splunkforwarder /etc/rc.d/rc2.d/K10splunkforwarder  /etc/rc.d/rc2.d/S10splunkforwarder
}

#-----------------------
install_bin()
{
echo "Install Splunk"
mount nim01:/repository1/splunk/aix /mnt
cd /opt
tar xvf /mnt/splunkforwarder-9.2.1-78803f08aabb-AIX-powerpc.tar
umount /mnt
}

#-----------------------
rebuild_outputs()
{
echo "Overwrite: outputs.conf"
cat > /opt/splunkforwarder/etc/system/local/outputs.conf << EOF
[tcpout]
defaultGroup = default-autolb-group

[tcpout:default-autolb-group]
server = splunk-prd.xxx:9997

[tcpout-server://splunk-prd.xxx:9997]
EOF

}

#-----------------------
build_inputs()
{
echo "Overwrite: inputs.conf"
cat > /opt/splunkforwarder/etc/system/local/inputs.conf.tmp << EOF
[default]
host=myhostname
disabled=0
ignoreOlderThan = 30d

EOF

host2=`echo "host="$(hostname -s)`
cat /opt/splunkforwarder/etc/system/local/inputs.conf.tmp | sed "s/host=aixa065/$host2/" > /opt/splunkforwarder/etc/system/local/inputs.conf
rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp

}

#-----------------------
rebuild_inputs()
{
echo "Modify: inputs.conf"
cp /opt/splunkforwarder/etc/system/local/inputs.conf /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
build_inputs
grep -vp '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1 >> /opt/splunkforwarder/etc/system/local/inputs.conf
rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
}

#-----------------------
rebuild_server()
{
echo "Modify: server.conf"
cp /opt/splunkforwarder/etc/system/local/server.conf /opt/splunkforwarder/etc/system/local/server.conf.tmp
cat /opt/splunkforwarder/etc/system/local/server.conf.tmp | sed "s/^serverName\ =\ .*/serverName\ =\ $(hostname -s)/" > /opt/splunkforwarder/etc/system/local/server.conf
rm /opt/splunkforwarder/etc/system/local/server.conf.tmp
}

#-----------------------
change_owner()
{
chown -R splunk:splunk /opt/splunkforwarder
}

#-----------------------
configure()
{
echo "Configure"

su - splunk -c "/opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt"

/opt/splunkforwarder/bin/splunk enable boot-start -user splunk
startsrc -s splunkd
}

#-----------------------
check_status()
{
echo "Check"
lssrc -Ss splunkd

odmget -q subsysname="splunkd" SRCsubsys

startsrc -s splunkd
lssrc -s splunkd

/opt/splunkforwarder/bin/splunk  btool check --debug

echo "Process"
ps -ef | grep -v grep | grep splunk | grep -v install_splunk
}

########################
# main
########################
version=$(cat /opt/splunkforwarder/etc/splunk.version | grep VERSION)

create_user

if [[ $(df -g | grep -c '/opt/splunkforwarder') == "0" ]]
then
  stop_splunk
  create_fs
fi

if [[ "$version" != "$splunk_version" ]]
then
  stop_splunk
  backup
  install_bin
  if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
  then
    rebuild_outputs
  fi
  if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
  then
    host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
    host2=`echo "host="$(hostname -s)`
    if [[ "$host1" != "$host2" ]]
    then
      rebuild_inputs
    fi
  else
    build_inputs
  fi
  change_owner
  configure
fi

if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
then
  rebuild_outputs
  stop_splunk
fi

if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
then
  host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
  host2=`echo "host="$(hostname -s)`
  if [[ "$host1" != "$host2" ]]
  then
    rebuild_inputs
    change_owner
    stop_splunk
  fi
else
  build_inputs
  change_owner
  stop_splunk
fi

if [ -e /opt/splunkforwarder/etc/system/local/server.conf ]
then
  host1=$(grep '^serverName' /opt/splunkforwarder/etc/system/local/server.conf | sed 's/\ //g')
  host2=`echo "serverName=$(hostname -s)`
  if [[ "$host1" != "$host2" ]]
  then
    rebuild_server
    stop_splunk
    change_owner
  fi
fi

stopsrc -s splunkd; sleep 2; startsrc -s splunkd

check_status
</code>