Differences

This shows you the differences between two versions of the page.

--- monitoring:splunk [2024/03/25 10:22]
manu
+++ monitoring:splunk [2024/07/03 11:13] (current)
manu [AIX]
@@ Line 2: / Line 2: @@
 ===== AIX =====
+Current splunkforwarder version is 9.2
+<cli prompt='>'>
+root@aix01 /root> cat /opt/splunkforwarder/etc/splunk.version
+VERSION=9.0.1
+BUILD=82c987350fde
+PRODUCT=splunk
+PLATFORM=AIX-powerpc
+</cli>
 Create a user splunk and group
+FIXME check the right limits
 <cli prompt='>'>
+root@aix01 /root> mkgroup id=2500 splunk
+root@aix01 /root> mkuser id=2500 pgrp=splunk groups=staff,splunk fsize=-1 data=2621440 rss=262144 splunk
 root@aix01 /root> lsuser -f splunk
 splunk:
@@ Line 31: / Line 45: @@
 </cli>
+Untar the splunk forwarder package and start install, as **splunk user**
+<cli prompt='>'>
+rootaix01 /opt> chown -R splunk.splunk /opt/splunkforwarder
+splunk@aix01 /home/splunk> /opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt
+This appears to be your first time running this version of Splunk.
+Splunk> CSI: Logfiles.
+Checking prerequisites...
+        Checking mgmt port [8089]: open
+                Creating: /opt/splunkforwarder/var/lib/splunk
+                Creating: /opt/splunkforwarder/var/run/splunk
+                Creating: /opt/splunkforwarder/var/run/splunk/appserver/i18n
+                Creating: /opt/splunkforwarder/var/run/splunk/appserver/modules/static/css
+                Creating: /opt/splunkforwarder/var/run/splunk/upload
+                Creating: /opt/splunkforwarder/var/run/splunk/search_telemetry
+                Creating: /opt/splunkforwarder/var/spool/splunk
+                Creating: /opt/splunkforwarder/var/spool/dirmoncache
+               Creating: /opt/splunkforwarder/var/lib/splunk/authDb
+                Creating: /opt/splunkforwarder/var/lib/splunk/hashDb
+New certs have been generated in '/opt/splunkforwarder/etc/auth'.
+        Checking conf files for problems...
+                Invalid key in stanza [webhook] in /opt/splunkforwarder/etc/system/default/alert_actions.conf, line 229: enable_allowlist (value: false).
+                Your indexes and inputs configurations are not internally consistent. For more information, run 'splunk btool check --debug'
+        Done
+        Checking default conf files for edits...
+        Validating installed files against hashes from '/opt/splunkforwarder/splunkforwarder-9.0.1-82c987350fde-AIX-powerpc-manifest'
+        All installed files intact.
+        Done
+All preliminary checks passed.
+Starting splunk server daemon (splunkd)...
+execve: Permission denied
+  while running command /usr/bin/startsrc
+Splunk boot-start is enabled. please use /usr/bin/startsrc -s splunkd to start splunk
+</cli>
+Create ass root a service **splunkd**
+<cli prompt='>'>
+root@aix01 /opt> /opt/splunkforwarder/bin/splunk enable boot-start -user splunk
+-071 The splunkd Subsystem has been added.
+SRC subsystem group installed.
+SRC subsystem group is configured to run at boot.
+root@aix01 /opt> odmget -q subsysname="splunkd" SRCsubsys
+SRCsubsys:
+        subsysname = "splunkd"
+        synonym = ""
+        cmdargs = "_internal_exec_splunkd"
+        path = "/opt/splunkforwarder/bin/splunk"
+        uid = 1601
+        auditid = 0
+        standin = "/dev/console"
+        standout = "/dev/console"
+        standerr = "/dev/console"
+        action = 1
+        multi = 0
+        contact = 2
+        svrkey = 0
+        svrmtype = 0
+        priority = 20
+        signorm = 2
+        sigforce = 9
+        display = 1
+        waittime = 20
+        grpname = "splunk"
+root@aix01 /opt> cat /etc/inittab
+splunk:2:once:/usr/bin/startsrc -g splunk > /dev/console 2>&1
+root@aix01 /opt> lssrc -a | grep -i  splunk
+ splunkd          splunk                        inoperative
+root@aix01 /opt> ps -ef | grep splu
+  splunk 11207102        1   2 16:41:57      -  0:00 splunkd -p 8089 start
+  splunk 11338186 11207102   0 16:41:57      -  0:00 [splunkd pid=11207102] splunkd -p 8089 start [process-runner]
+root@aix01 /opt> kill 11207102 11338186
+root@aix01 /opt> startsrc -s splunkd
+-059 The splunkd Subsystem has been started. Subsystem PID is 7995758.
+root@aix01 /opt> ps -ef | grep splu
+    root  6881638 10748408   0 16:47:50  pts/0  0:00 grep splu
+  splunk  7995758  5898518 120 16:47:48      -  0:00 splunkd --nodaemon -p 8089 _internal_exec_splunkd
+  splunk 11469220  7995758   0 16:47:50      -  0:00 [splunkd pid=7995758] splunkd --nodaemon -p 8089 _internal_exec_splunkd [process-runner]
+root@aix01 /opt> lssrc -a | grep -i  splunk
+ splunkd          splunk           7995758      active
+</cli>
+Debug (if needed)
+<cli prompt='>'>
+root@aix01 /root> /opt/splunkforwarder/bin/splunk  btool check --debug
+</cli>
+Script to Upgrade/install splunk agent
+<code>
+[root@nim01]/root/scripts> cat install_splunk.sh
+#!/bin/ksh93
+# V1.0 initial version
+splunk_version="VERSION=9.2.1"
+echo "Install/Upgrade Splunk VERSION=9.2.1"
+#-----------------------
+create_user()
+{
+echo "Add user splunk"
+mkgroup -a id=1500 splunk
+mkuser -a id=1500 pgrp=splunk gecos='splunk' splunk
+lsgroup splunk ; lsuser -a id pgrp groups splunk
+chuser fsize=-1 data=2621440 rss=262144 splunk
+chgrpmem -m + splunk oinstall 2>/dev/null
+}
+#-----------------------
+create_fs()
+{
+echo "Create /opt/splunkforwarder filesystem"
+mv /opt/splunkforwarder /opt/splunkforwarder1
+mkdir /opt/splunkforwarder
+chmod a+rx /opt/splunkforwarder
+mklv -t jfs2 -y splunklv rootvg 1G
+crfs -vjfs2 -m /opt/splunkforwarder -d splunklv -Ayes -a log=INLINE
+mount /opt/splunkforwarder
+chown splunk:splunk /opt/splunkforwarder
+mv /opt/splunkforwarder1/* /opt/splunkforwarder
+rm -r /opt/splunkforwarder1
+startsrc -s splunkd
+}
+#-----------------------
+stop_splunk()
+{
+echo "Stop process"
+stopsrc -g splunk
+for i in $(ps -ef | grep -v grep | grep splunk | grep -v install_splunk | awk '{print $2}')
+do
+kill $i
+done
+ps -ef | grep splunkd |grep -v grep
+}
+#-----------------------
+backup()
+{
+mkdir -p /opt/splunkforwarder
+mkdir -p /root/old
+cd /opt/splunkforwarder/etc/system/; tar cvf /root/old/splunk_local.tar local
+rm /etc/rc.d/init.d/splunkforwarder /etc/rc.d/rc2.d/K10splunkforwarder  /etc/rc.d/rc2.d/S10splunkforwarder
+}
+#-----------------------
+install_bin()
+{
+echo "Install Splunk"
+mount nim01:/repository1/splunk/aix /mnt
+cd /opt
+tar xvf /mnt/splunkforwarder-9.2.1-78803f08aabb-AIX-powerpc.tar
+umount /mnt
+}
+#-----------------------
+rebuild_outputs()
+{
+echo "Overwrite: outputs.conf"
+cat > /opt/splunkforwarder/etc/system/local/outputs.conf << EOF
+[tcpout]
+defaultGroup = default-autolb-group
+[tcpout:default-autolb-group]
+server = splunk-prd.xxx:9997
+[tcpout-server://splunk-prd.xxx:9997]
+EOF
+}
+#-----------------------
+build_inputs()
+{
+echo "Overwrite: inputs.conf"
+cat > /opt/splunkforwarder/etc/system/local/inputs.conf.tmp << EOF
+[default]
+host=myhostname
+disabled=0
+ignoreOlderThan = 30d
+EOF
+host2=`echo "host="$(hostname -s)`
+cat /opt/splunkforwarder/etc/system/local/inputs.conf.tmp | sed "s/host=aixa065/$host2/" > /opt/splunkforwarder/etc/system/local/inputs.conf
+rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp
+}
+#-----------------------
+rebuild_inputs()
+{
+echo "Modify: inputs.conf"
+cp /opt/splunkforwarder/etc/system/local/inputs.conf /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
+build_inputs
+grep -vp '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1 >> /opt/splunkforwarder/etc/system/local/inputs.conf
+rm /opt/splunkforwarder/etc/system/local/inputs.conf.tmp1
+}
+#-----------------------
+rebuild_server()
+{
+echo "Modify: server.conf"
+cp /opt/splunkforwarder/etc/system/local/server.conf /opt/splunkforwarder/etc/system/local/server.conf.tmp
+cat /opt/splunkforwarder/etc/system/local/server.conf.tmp | sed "s/^serverName\ =\ .*/serverName\ =\ $(hostname -s)/" > /opt/splunkforwarder/etc/system/local/server.conf
+rm /opt/splunkforwarder/etc/system/local/server.conf.tmp
+}
+#-----------------------
+change_owner()
+{
+chown -R splunk:splunk /opt/splunkforwarder
+}
+#-----------------------
+configure()
+{
+echo "Configure"
+su - splunk -c "/opt/splunkforwarder/bin/splunk start --accept-license --answer-yes --auto-ports --no-prompt"
+/opt/splunkforwarder/bin/splunk enable boot-start -user splunk
+startsrc -s splunkd
+}
+#-----------------------
+check_status()
+{
+echo "Check"
+lssrc -Ss splunkd
+odmget -q subsysname="splunkd" SRCsubsys
+startsrc -s splunkd
+lssrc -s splunkd
+/opt/splunkforwarder/bin/splunk  btool check --debug
+echo "Process"
+ps -ef | grep -v grep | grep splunk | grep -v install_splunk
+}
+########################
+# main
+########################
+version=$(cat /opt/splunkforwarder/etc/splunk.version | grep VERSION)
+create_user
+if [[ $(df -g | grep -c '/opt/splunkforwarder') == "0" ]]
+then
+  stop_splunk
+  create_fs
+fi
+if [[ "$version" != "$splunk_version" ]]
+then
+  stop_splunk
+  backup
+  install_bin
+  if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
+  then
+    rebuild_outputs
+  fi
+  if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
+  then
+    host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
+    host2=`echo "host="$(hostname -s)`
+    if [[ "$host1" != "$host2" ]]
+    then
+      rebuild_inputs
+    fi
+  else
+    build_inputs
+  fi
+  change_owner
+  configure
+fi
+if [[ "$(grep -c 'splunk-prd.xxx' /opt/splunkforwarder/etc/system/local/outputs.conf 2>/dev/null)" != "2" ]]
+then
+  rebuild_outputs
+  stop_splunk
+fi
+if [ -e /opt/splunkforwarder/etc/system/local/inputs.conf ]
+then
+  host1=$(grep -p '\[default\]' /opt/splunkforwarder/etc/system/local/inputs.conf | grep '^host=')
+  host2=`echo "host="$(hostname -s)`
+  if [[ "$host1" != "$host2" ]]
+  then
+    rebuild_inputs
+    change_owner
+    stop_splunk
+  fi
+else
+  build_inputs
+  change_owner
+  stop_splunk
+fi
-===== Linux =====
+if [ -e /opt/splunkforwarder/etc/system/local/server.conf ]
+then
+  host1=$(grep '^serverName' /opt/splunkforwarder/etc/system/local/server.conf | sed 's/\ //g')
+  host2=`echo "serverName=$(hostname -s)`
+  if [[ "$host1" != "$host2" ]]
+  then
+    rebuild_server
+    stop_splunk
+    change_owner
+  fi
+fi
+stopsrc -s splunkd; sleep 2; startsrc -s splunkd
+check_status
+</code>

wiki

User Tools

Site Tools

Differences

Page Tools