diff --git a/archive/provisioning/Agents/librenms/check_mk.socket b/archive/provisioning/Agents/librenms/check_mk.socket new file mode 100644 index 0000000..02760ff --- /dev/null +++ b/archive/provisioning/Agents/librenms/check_mk.socket @@ -0,0 +1,9 @@ +[Unit] +Description=Check_MK LibreNMS Agent Socket + +[Socket] +ListenStream=6556 +Accept=yes + +[Install] +WantedBy=sockets.target diff --git a/archive/provisioning/Agents/librenms/check_mk@.service b/archive/provisioning/Agents/librenms/check_mk@.service new file mode 100644 index 0000000..bea7144 --- /dev/null +++ b/archive/provisioning/Agents/librenms/check_mk@.service @@ -0,0 +1,7 @@ +[Unit] +Description=Check_MK LibreNMS Agent Service +After=network.target + +[Service] +ExecStart=/usr/bin/check_mk_agent +StandardOutput=socket diff --git a/archive/provisioning/Agents/librenms/check_mk_agent b/archive/provisioning/Agents/librenms/check_mk_agent new file mode 100644 index 0000000..6b6e03a --- /dev/null +++ b/archive/provisioning/Agents/librenms/check_mk_agent @@ -0,0 +1,659 @@ +#!/bin/bash +# +------------------------------------------------------------------+ +# | ____ _ _ __ __ _ __ | +# | / ___| |__ ___ ___| | __ | \/ | |/ / | +# | | | | '_ \ / _ \/ __| |/ / | |\/| | ' / | +# | | |___| | | | __/ (__| < | | | | . \ | +# | \____|_| |_|\___|\___|_|\_\___|_| |_|_|\_\ | +# | | +# | Copyright Mathias Kettner 2014 mk@mathias-kettner.de | +# +------------------------------------------------------------------+ +# +# This file is part of Check_MK. +# The official homepage is at http://mathias-kettner.de/check_mk. +# +# check_mk is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation in version 2. check_mk is distributed +# in the hope that it will be useful, but WITHOUT ANY WARRANTY; with- +# out even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more de- +# ails. You should have received a copy of the GNU General Public +# License along with GNU Make; see the file COPYING. If not, write +# to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +# Boston, MA 02110-1301 USA. + +# Remove locale settings to eliminate localized outputs where possible +export LC_ALL=C +unset LANG + +export MK_LIBDIR="/usr/lib/check_mk_agent" +export MK_CONFDIR="/etc/check_mk" +export MK_VARDIR="/var/lib/check_mk_agent" + +# Provide information about the remote host. That helps when data +# is being sent only once to each remote host. +if [ "$REMOTE_HOST" ] ; then + export REMOTE=$REMOTE_HOST +elif [ "$SSH_CLIENT" ] ; then + export REMOTE=${SSH_CLIENT%% *} +fi + +# Make sure, locally installed binaries are found +PATH=$PATH:/usr/local/bin + +# All executables in PLUGINSDIR will simply be executed and their +# ouput appended to the output of the agent. Plugins define their own +# sections and must output headers with '<<<' and '>>>' +PLUGINSDIR=$MK_LIBDIR/plugins + +# All executables in LOCALDIR will by executabled and their +# output inserted into the section <<>>. Please +# refer to online documentation for details about local checks. +LOCALDIR=$MK_LIBDIR/local + +# All files in SPOOLDIR will simply appended to the agent +# output if they are not outdated (see below) +SPOOLDIR=$MK_VARDIR/spool + +# close standard input (for security reasons) and stderr +if [ "$1" = -d ] +then + set -xv +else + exec /dev/null +fi + +# Runs a command asynchronous by use of a cache file +function run_cached () { + local section= + if [ "$1" = -s ] ; then local section="echo '<<<$2>>>' ; " ; shift ; fi + local NAME=$1 + local MAXAGE=$2 + shift 2 + local CMDLINE="$section$@" + + if [ ! -d $MK_VARDIR/cache ]; then mkdir -p $MK_VARDIR/cache ; fi + CACHEFILE="$MK_VARDIR/cache/$NAME.cache" + + # Check if the creation of the cache takes suspiciously long and return + # nothing if the age (access time) of $CACHEFILE.new is twice the MAXAGE + local NOW=$(date +%s) + if [ -e "$CACHEFILE.new" ] ; then + local CF_ATIME=$(stat -c %X "$CACHEFILE.new") + if [ $((NOW - CF_ATIME)) -ge $((MAXAGE * 2)) ] ; then + # Kill the process still accessing that file in case + # it is still running. This avoids overlapping processes! + fuser -k -9 "$CACHEFILE.new" >/dev/null 2>&1 + rm -f "$CACHEFILE.new" + return + fi + fi + + # Check if cache file exists and is recent enough + if [ -s "$CACHEFILE" ] ; then + local MTIME=$(stat -c %Y "$CACHEFILE") + if [ $((NOW - MTIME)) -le $MAXAGE ] ; then local USE_CACHEFILE=1 ; fi + # Output the file in any case, even if it is + # outdated. The new file will not yet be available + cat "$CACHEFILE" + fi + + # Cache file outdated and new job not yet running? Start it + if [ -z "$USE_CACHEFILE" -a ! -e "$CACHEFILE.new" ] ; then + echo "set -o noclobber ; exec > \"$CACHEFILE.new\" || exit 1 ; $CMDLINE && mv \"$CACHEFILE.new\" \"$CACHEFILE\" || rm -f \"$CACHEFILE\" \"$CACHEFILE.new\"" | nohup bash >/dev/null 2>&1 & + fi +} + +# Make run_cached available for subshells (plugins, local checks, etc.) +export -f run_cached + +echo '<<>>' +echo Version: 1.2.6b5 +echo AgentOS: linux +echo AgentDirectory: $MK_CONFDIR +echo DataDirectory: $MK_VARDIR +echo SpoolDirectory: $SPOOLDIR +echo PluginsDirectory: $PLUGINSDIR +echo LocalDirectory: $LOCALDIR + +# If we are called via xinetd, try to find only_from configuration +if [ -n "$REMOTE_HOST" ] +then + echo -n 'OnlyFrom: ' + echo $(sed -n '/^service[[:space:]]*check_mk/,/}/s/^[[:space:]]*only_from[[:space:]]*=[[:space:]]*\(.*\)/\1/p' /etc/xinetd.d/* | head -n1) +fi + +# Print out Partitions / Filesystems. (-P gives non-wrapped POSIXed output) +# Heads up: NFS-mounts are generally supressed to avoid agent hangs. +# If hard NFS mounts are configured or you have too large nfs retry/timeout +# settings, accessing those mounts from the agent would leave you with +# thousands of agent processes and, ultimately, a dead monitored system. +# These should generally be monitored on the NFS server, not on the clients. + +echo '<<>>' +# The exclusion list is getting a bit of a problem. -l should hide any remote FS but seems +# to be all but working. +excludefs="-x smbfs -x cifs -x iso9660 -x udf -x nfsv4 -x nfs -x mvfs -x zfs" +df -PTlk $excludefs | sed 1d + +# df inodes information +echo '<<>>' +echo '[df_inodes_start]' +df -PTli $excludefs | sed 1d +echo '[df_inodes_end]' + +# Filesystem usage for ZFS +if type zfs > /dev/null 2>&1 ; then + echo '<<>>' + zfs get -Hp name,quota,used,avail,mountpoint,type -t filesystem,volume || \ + zfs get -Hp name,quota,used,avail,mountpoint,type + echo '[df]' + df -PTlk -t zfs | sed 1d +fi + +# Check NFS mounts by accessing them with stat -f (System +# call statfs()). If this lasts more then 2 seconds we +# consider it as hanging. We need waitmax. +if type waitmax >/dev/null +then + STAT_VERSION=$(stat --version | head -1 | cut -d" " -f4) + STAT_BROKE="5.3.0" + + echo '<<>>' + sed -n '/ nfs4\? /s/[^ ]* \([^ ]*\) .*/\1/p' < /proc/mounts | + sed 's/\\040/ /g' | + while read MP + do + if [ $STAT_VERSION != $STAT_BROKE ]; then + waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" || \ + echo "$MP hanging 0 0 0 0" + else + waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" && \ + printf '\n'|| echo "$MP hanging 0 0 0 0" + fi + done + + echo '<<>>' + sed -n '/ cifs\? /s/[^ ]* \([^ ]*\) .*/\1/p' < /proc/mounts | + sed 's/\\040/ /g' | + while read MP + do + if [ $STAT_VERSION != $STAT_BROKE ]; then + waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" || \ + echo "$MP hanging 0 0 0 0" + else + waitmax -s 9 2 stat -f -c "$MP ok %b %f %a %s" "$MP" && \ + printf '\n'|| echo "$MP hanging 0 0 0 0" + fi + done +fi + +# Check mount options. Filesystems may switch to 'ro' in case +# of a read error. +echo '<<>>' +grep ^/dev < /proc/mounts + +# processes including username, without kernel processes +echo '<<>>' +ps ax -o user,vsz,rss,cputime,pid,command --columns 10000 | sed -e 1d -e 's/ *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) *\([^ ]*\) */(\1,\2,\3,\4,\5) /' + +# Memory usage +echo '<<>>' +egrep -v '^Swap:|^Mem:|total:' < /proc/meminfo + +# Load and number of processes +echo '<<>>' +echo "$(cat /proc/loadavg) $(grep -E '^CPU|^processor' < /proc/cpuinfo | wc -l)" + +# Uptime +echo '<<>>' +cat /proc/uptime + + +# New variant: Information about speed and state in one section +echo '<<>>' +sed 1,2d /proc/net/dev +if type ethtool > /dev/null +then + for eth in $(sed -e 1,2d < /proc/net/dev | cut -d':' -f1 | sort) + do + echo "[$eth]" + ethtool $eth | egrep '(Speed|Duplex|Link detected|Auto-negotiation):' + echo -en "\tAddress: " ; cat /sys/class/net/$eth/address ; echo + done +fi + + +# Current state of bonding interfaces +if [ -e /proc/net/bonding ] ; then + echo '<<>>' + pushd /proc/net/bonding > /dev/null ; head -v -n 1000 * ; popd +fi + +# Same for Open vSwitch bonding +if type ovs-appctl > /dev/null ; then + echo '<<>>' + for bond in $(ovs-appctl bond/list | sed -e 1d | cut -f2) ; do + echo "[$bond]" + ovs-appctl bond/show $bond + done +fi + + +# Number of TCP connections in the various states +echo '<<>>' +# waitmax 10 netstat -nt | awk ' /^tcp/ { c[$6]++; } END { for (x in c) { print x, c[x]; } }' +# New implementation: netstat is very slow for large TCP tables +cat /proc/net/tcp /proc/net/tcp6 2>/dev/null | awk ' /:/ { c[$4]++; } END { for (x in c) { print x, c[x]; } }' + +# Linux Multipathing +if type multipath >/dev/null ; then + echo '<<>>' + multipath -l +fi + +# Performancecounter Platten +echo '<<>>' +date +%s +egrep ' (x?[shv]d[a-z]*|cciss/c[0-9]+d[0-9]+|emcpower[a-z]+|dm-[0-9]+|VxVM.*|mmcblk.*) ' < /proc/diskstats +if type dmsetup >/dev/null ; then + echo '[dmsetup_info]' + dmsetup info -c --noheadings --separator ' ' -o name,devno,vg_name,lv_name +fi +if [ -d /dev/vx/dsk ] ; then + echo '[vx_dsk]' + stat -c "%t %T %n" /dev/vx/dsk/*/* +fi + + +# Performancecounter Kernel +echo '<<>>' +date +%s +cat /proc/vmstat /proc/stat + +# Hardware sensors via IPMI (need ipmitool) +if type ipmitool > /dev/null +then + run_cached -s ipmi 300 "ipmitool sensor list | grep -v 'command failed' | sed -e 's/ *| */|/g' -e 's/ /_/g' -e 's/_*"'$'"//' -e 's/|/ /g' | egrep -v '^[^ ]+ na ' | grep -v ' discrete '" +fi + + +# IPMI data via ipmi-sensors (of freeipmi). Please make sure, that if you +# have installed freeipmi that IPMI is really support by your hardware. +if type ipmi-sensors >/dev/null +then + echo '<<>>' + # Newer ipmi-sensors version have new output format; Legacy format can be used + if ipmi-sensors --help | grep -q legacy-output; then + IPMI_FORMAT="--legacy-output" + else + IPMI_FORMAT="" + fi + # At least with ipmi-sensoirs 0.7.16 this group is Power_Unit instead of "Power Unit" + run_cached -s ipmi_sensors 300 "for class in Temperature Power_Unit Fan + do + ipmi-sensors $IPMI_FORMAT --sdr-cache-directory /var/cache -g "$class" | sed -e 's/ /_/g' -e 's/:_\?/ /g' -e 's@ \([^(]*\)_(\([^)]*\))@ \2_\1@' + # In case of a timeout immediately leave loop. + if [ $? = 255 ] ; then break ; fi + done" +fi + +# RAID status of Linux software RAID +echo '<<>>' +cat /proc/mdstat + +# RAID status of Linux RAID via device mapper +if type dmraid >/dev/null && DMSTATUS=$(dmraid -r) +then + echo '<<>>' + + # Output name and status + dmraid -s | grep -e ^name -e ^status + + # Output disk names of the RAID disks + DISKS=$(echo "$DMSTATUS" | cut -f1 -d\:) + + for disk in $DISKS ; do + device=$(cat /sys/block/$(basename $disk)/device/model ) + status=$(echo "$DMSTATUS" | grep ^${disk}) + echo "$status Model: $device" + done +fi + +# RAID status of LSI controllers via cfggen +if type cfggen > /dev/null ; then + echo '<<>>' + cfggen 0 DISPLAY | egrep '(Target ID|State|Volume ID|Status of volume)[[:space:]]*:' | sed -e 's/ *//g' -e 's/:/ /' +fi + +# RAID status of LSI MegaRAID controller via MegaCli. You can download that tool from: +# http://www.lsi.com/downloads/Public/MegaRAID%20Common%20Files/8.02.16_MegaCLI.zip +if type MegaCli >/dev/null ; then + MegaCli_bin="MegaCli" +elif type MegaCli64 >/dev/null ; then + MegaCli_bin="MegaCli64" +elif type megacli >/dev/null ; then + MegaCli_bin="megacli" +else + MegaCli_bin="unknown" +fi + +if [ "$MegaCli_bin" != "unknown" ]; then + echo '<<>>' + for part in $($MegaCli_bin -EncInfo -aALL -NoLog < /dev/null \ + | sed -rn 's/:/ /g; s/[[:space:]]+/ /g; s/^ //; s/ $//; s/Number of enclosures on adapter ([0-9]+).*/adapter \1/g; /^(Enclosure|Device ID|adapter) [0-9]+$/ p'); do + [ $part = adapter ] && echo "" + [ $part = 'Enclosure' ] && echo -ne "\ndev2enc" + echo -n " $part" + done + echo + $MegaCli_bin -PDList -aALL -NoLog < /dev/null | egrep 'Enclosure|Raw Size|Slot Number|Device Id|Firmware state|Inquiry|Adapter' + echo '<<>>' + $MegaCli_bin -LDInfo -Lall -aALL -NoLog < /dev/null | egrep 'Size|State|Number|Adapter|Virtual' + echo '<<>>' + $MegaCli_bin -AdpBbuCmd -GetBbuStatus -aALL -NoLog < /dev/null | grep -v Exit +fi + +# RAID status of 3WARE disk controller (by Radoslaw Bak) +if type tw_cli > /dev/null ; then + for C in $(tw_cli show | awk 'NR < 4 { next } { print $1 }'); do + echo '<<<3ware_info>>>' + tw_cli /$C show all | egrep 'Model =|Firmware|Serial' + echo '<<<3ware_disks>>>' + tw_cli /$C show drivestatus | egrep 'p[0-9]' | sed "s/^/$C\//" + echo '<<<3ware_units>>>' + tw_cli /$C show unitstatus | egrep 'u[0-9]' | sed "s/^/$C\//" + done +fi + +# RAID controllers from areca (Taiwan) +# cli64 can be found at ftp://ftp.areca.com.tw/RaidCards/AP_Drivers/Linux/CLI/ +if type cli64 >/dev/null ; then + run_cached -s arc_raid_status 300 "cli64 rsf info | tail -n +3 | head -n -2" +fi + +# VirtualBox Guests. Section must always been output. Otherwise the +# check would not be executed in case no guest additions are installed. +# And that is something the check wants to detect +echo '<<>>' +if type VBoxControl >/dev/null 2>&1 ; then + VBoxControl -nologo guestproperty enumerate | cut -d, -f1,2 + [ ${PIPESTATUS[0]} = 0 ] || echo "ERROR" +fi + +# OpenVPN Clients. Currently we assume that the configuration # is in +# /etc/openvpn. We might find a safer way to find the configuration later. +if [ -e /etc/openvpn/openvpn-status.log ] ; then + echo '<<>>' + sed -n -e '/CLIENT LIST/,/ROUTING TABLE/p' < /etc/openvpn/openvpn-status.log | sed -e 1,3d -e '$d' +fi + +# Time synchronization with NTP +if type ntpq > /dev/null 2>&1 ; then + # remove heading, make first column space separated + run_cached -s ntp 30 "waitmax 5 ntpq -np | sed -e 1,2d -e 's/^\(.\)/\1 /' -e 's/^ /%/'" +fi + +# Time synchronization with Chrony +if type chronyc > /dev/null 2>&1 ; then + # Force successful exit code. Otherwise section will be missing if daemon not running + run_cached -s chrony 30 "waitmax 5 chronyc tracking || true" +fi + +if type nvidia-settings >/dev/null && [ -S /tmp/.X11-unix/X0 ] +then + echo '<<>>' + for var in GPUErrors GPUCoreTemp + do + DISPLAY=:0 waitmax 2 nvidia-settings -t -q $var | sed "s/^/$var: /" + done +fi + +if [ -e /proc/drbd ]; then + echo '<<>>' + cat /proc/drbd +fi + +# Status of CUPS printer queues +if type lpstat > /dev/null 2>&1; then + if pgrep cups > /dev/null 2>&1; then + echo '<<>>' + CPRINTCONF=/etc/cups/printers.conf + if [ -r "$CPRINTCONF" ] ; then + LOCAL_PRINTERS=$(grep -E "<(Default)?Printer .*>" $CPRINTCONF | awk '{print $2}' | sed -e 's/>//') + lpstat -p | while read LINE + do + PRINTER=$(echo $LINE | awk '{print $2}') + if echo "$LOCAL_PRINTERS" | grep -q "$PRINTER"; then + echo $LINE + fi + done + echo '---' + lpstat -o | while read LINE + do + PRINTER=${LINE%%-*} + if echo "$LOCAL_PRINTERS" | grep -q "$PRINTER"; then + echo $LINE + fi + done + else + lpstat -p + echo '---' + lpstat -o | sort + fi + fi +fi + +# Heartbeat monitoring +# Different handling for heartbeat clusters with and without CRM +# for the resource state +if [ -S /var/run/heartbeat/crm/cib_ro -o -S /var/run/crm/cib_ro ] || pgrep crmd > /dev/null 2>&1; then + echo '<<>>' + crm_mon -1 -r | grep -v ^$ | sed 's/^ //; /^\sResource Group:/,$ s/^\s//; s/^\s/_/g' +fi +if type cl_status > /dev/null 2>&1; then + echo '<<>>' + cl_status rscstatus + + echo '<<>>' + for NODE in $(cl_status listnodes); do + if [ $NODE != $(echo $HOSTNAME | tr 'A-Z' 'a-z') ]; then + STATUS=$(cl_status nodestatus $NODE) + echo -n "$NODE $STATUS" + for LINK in $(cl_status listhblinks $NODE 2>/dev/null); do + echo -n " $LINK $(cl_status hblinkstatus $NODE $LINK)" + done + echo + fi + done +fi + +# Postfix mailqueue monitoring +# +# Only handle mailq when postfix user is present. The mailq command is also +# available when postfix is not installed. But it produces different outputs +# which are not handled by the check at the moment. So try to filter out the +# systems not using postfix by searching for the postfix user.a +# +# Cannot take the whole outout. This could produce several MB of agent output +# on blocking queues. +# Only handle the last 6 lines (includes the summary line at the bottom and +# the last message in the queue. The last message is not used at the moment +# but it could be used to get the timestamp of the last message. +if type postconf >/dev/null ; then + echo '<<>>' + postfix_queue_dir=$(postconf -h queue_directory) + postfix_count=$(find $postfix_queue_dir/deferred -type f | wc -l) + postfix_size=$(du -ks $postfix_queue_dir/deferred | awk '{print $1 }') + if [ $postfix_count -gt 0 ] + then + echo -- $postfix_size Kbytes in $postfix_count Requests. + else + echo Mail queue is empty + fi +elif [ -x /usr/sbin/ssmtp ] ; then + echo '<<>>' + mailq 2>&1 | sed 's/^[^:]*: \(.*\)/\1/' | tail -n 6 +fi + +#Check status of qmail mailqueue +if type qmail-qstat >/dev/null +then + echo "<<>>" + qmail-qstat +fi + +# Check status of OMD sites +if type omd >/dev/null +then + run_cached -s omd_status 60 "omd status --bare --auto" +fi + + +# Welcome the ZFS check on Linux +# We do not endorse running ZFS on linux if your vendor doesnt support it ;) +# check zpool status +if type zpool >/dev/null; then + echo "<<>>" + zpool status -x +fi + + +# Fileinfo-Check: put patterns for files into /etc/check_mk/fileinfo.cfg +if [ -r "$MK_CONFDIR/fileinfo.cfg" ] ; then + echo '<<>>' + date +%s + stat -c "%n|%s|%Y" $(cat "$MK_CONFDIR/fileinfo.cfg") +fi + +# Get stats about OMD monitoring cores running on this machine. +# Since cd is a shell builtin the check does not affect the performance +# on non-OMD machines. +if cd /omd/sites +then + echo '<<>>' + for site in * + do + if [ -S "/omd/sites/$site/tmp/run/live" ] ; then + echo "[$site]" + echo -e "GET status" | waitmax 3 /omd/sites/$site/bin/unixcat /omd/sites/$site/tmp/run/live + fi + done +fi + +# Get statistics about monitored jobs. Below the job directory there +# is a sub directory per user that ran a job. That directory must be +# owned by the user so that a symlink or hardlink attack for reading +# arbitrary files can be avoided. +if pushd $MK_VARDIR/job >/dev/null; then + echo '<<>>' + for username in * + do + if [ -d "$username" ] && cd "$username" ; then + su "$username" -c "head -n -0 -v *" + cd .. + fi + done + popd > /dev/null +fi + +# Gather thermal information provided e.g. by acpi +# At the moment only supporting thermal sensors +if ls /sys/class/thermal/thermal_zone* >/dev/null 2>&1; then + echo '<<>>' + for F in /sys/class/thermal/thermal_zone*; do + echo -n "${F##*/} " + if [ ! -e $F/mode ] ; then echo -n "- " ; fi + cat $F/{mode,type,temp,trip_point_*} | tr \\n " " + echo + done +fi + +# Libelle Business Shadow +if type trd >/dev/null; then + echo "<<>>" + trd -s +fi + +# MK's Remote Plugin Executor +if [ -e "$MK_CONFDIR/mrpe.cfg" ] +then + echo '<<>>' + grep -Ev '^[[:space:]]*($|#)' "$MK_CONFDIR/mrpe.cfg" | \ + while read descr cmdline + do + PLUGIN=${cmdline%% *} + OUTPUT=$(eval "$cmdline") + echo -n "(${PLUGIN##*/}) $descr $? $OUTPUT" | tr \\n \\1 + echo + done +fi + + +# Local checks +echo '<<>>' +if cd $LOCALDIR ; then + for skript in $(ls) ; do + if [ -f "$skript" -a -x "$skript" ] ; then + ./$skript + fi + done + # Call some plugins only every X'th minute + for skript in [1-9]*/* ; do + if [ -x "$skript" ] ; then + run_cached local_${skript//\//\\} ${skript%/*} "$skript" + fi + done +fi + +# Plugins +if cd $PLUGINSDIR ; then + for skript in $(ls) ; do + if [ -f "$skript" -a -x "$skript" ] ; then + ./$skript + fi + done + # Call some plugins only every Xth minute + for skript in [1-9]*/* ; do + if [ -x "$skript" ] ; then + run_cached plugins_${skript//\//\\} ${skript%/*} "$skript" + fi + done +fi + +# Agent output snippets created by cronjobs, etc. +if [ -d "$SPOOLDIR" ] +then + pushd "$SPOOLDIR" > /dev/null + now=$(date +%s) + + for file in * + do + # output every file in this directory. If the file is prefixed + # with a number, then that number is the maximum age of the + # file in seconds. If the file is older than that, it is ignored. + maxage="" + part="$file" + + # Each away all digits from the front of the filename and + # collect them in the variable maxage. + while [ "${part/#[0-9]/}" != "$part" ] + do + maxage=$maxage${part:0:1} + part=${part:1} + done + + # If there is at least one digit, than we honor that. + if [ "$maxage" ] ; then + mtime=$(stat -c %Y "$file") + if [ $((now - mtime)) -gt $maxage ] ; then + continue + fi + fi + + # Output the file + cat "$file" + done + popd > /dev/null +fi diff --git a/archive/provisioning/Agents/librenms/distro b/archive/provisioning/Agents/librenms/distro new file mode 100644 index 0000000..61ad248 --- /dev/null +++ b/archive/provisioning/Agents/librenms/distro @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Detects which OS and if it is Linux then it will detect which Linux Distribution. + +OS=`uname -s` +REV=`uname -r` +MACH=`uname -m` + +if [ "${OS}" = "SunOS" ] ; then + OS=Solaris + ARCH=`uname -p` + OSSTR="${OS} ${REV}(${ARCH} `uname -v`)" + +elif [ "${OS}" = "AIX" ] ; then + OSSTR="${OS} `oslevel` (`oslevel -r`)" + +elif [ "${OS}" = "Linux" ] ; then + KERNEL=`uname -r` + + if [ -f /etc/fedora-release ]; then + DIST=$(cat /etc/fedora-release | awk '{print $1}') + REV=`cat /etc/fedora-release | sed s/.*release\ // | sed s/\ .*//` + + elif [ -f /etc/redhat-release ] ; then + DIST=$(cat /etc/redhat-release | awk '{print $1}') + if [ "${DIST}" = "CentOS" ]; then + DIST="CentOS" + elif [ "${DIST}" = "Mandriva" ]; then + DIST="Mandriva" + PSEUDONAME=`cat /etc/mandriva-release | sed s/.*\(// | sed s/\)//` + REV=`cat /etc/mandriva-release | sed s/.*release\ // | sed s/\ .*//` + elif [ -f /etc/oracle-release ]; then + DIST="Oracle" + else + DIST="RedHat" + fi + + PSEUDONAME=`cat /etc/redhat-release | sed s/.*\(// | sed s/\)//` + REV=`cat /etc/redhat-release | sed s/.*release\ // | sed s/\ .*//` + + elif [ -f /etc/mandrake-release ] ; then + DIST='Mandrake' + PSEUDONAME=`cat /etc/mandrake-release | sed s/.*\(// | sed s/\)//` + REV=`cat /etc/mandrake-release | sed s/.*release\ // | sed s/\ .*//` + + elif [ -f /etc/devuan_version ] ; then + DIST="Devuan `cat /etc/devuan_version`" + REV="" + + elif [ -f /etc/debian_version ] ; then + DIST="Debian `cat /etc/debian_version`" + REV="" + ID=`lsb_release -i | awk -F ':' '{print $2}' | sed 's/ //g'` + if [ "${ID}" = "Raspbian" ] ; then + DIST="Raspbian `cat /etc/debian_version`" + fi + + elif [ -f /etc/gentoo-release ] ; then + DIST="Gentoo" + REV=$(tr -d '[[:alpha:]]' >>' + +# requires dmidecode +for FIELD in bios-vendor bios-version bios-release-date system-manufacturer system-product-name system-version system-serial-number system-uuid baseboard-manufacturer baseboard-product-name baseboard-version baseboard-serial-number baseboard-asset-tag chassis-manufacturer chassis-type chassis-version chassis-serial-number chassis-asset-tag processor-family processor-manufacturer processor-version processor-frequency +do + echo $FIELD="$(dmidecode -s $FIELD | grep -v '^#')" +done diff --git a/archive/provisioning/Agents/librenms/dpkg.sh b/archive/provisioning/Agents/librenms/dpkg.sh new file mode 100644 index 0000000..e89e2a0 --- /dev/null +++ b/archive/provisioning/Agents/librenms/dpkg.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Cache the file for 30 minutes +# If you want to override this, put the command in cron. +# We cache because it is a 1sec delay, which is painful for the poller +if [ -x /usr/bin/dpkg-query ]; then + DATE=$(date +%s) + FILE=/var/cache/librenms/agent-local-dpkg + + [ -d /var/cache/librenms ] || mkdir -p /var/cache/librenms + + if [ ! -e $FILE ]; then + dpkg-query -W --showformat='${Status} ${Package} ${Version} ${Architecture} ${Installed-Size}\n'|grep " installed "|cut -d\ -f4- > $FILE + fi + FILEMTIME=$(stat -c %Y $FILE) + FILEAGE=$(($DATE-$FILEMTIME)) + if [ $FILEAGE -gt 1800 ]; then + dpkg-query -W --showformat='${Status} ${Package} ${Version} ${Architecture} ${Installed-Size}\n'|grep " installed "|cut -d\ -f4- > $FILE + fi + echo "<<>>" + cat $FILE +fi + diff --git a/archive/provisioning/Agents/librenms/mysql.sh b/archive/provisioning/Agents/librenms/mysql.sh new file mode 100644 index 0000000..c56e4e7 --- /dev/null +++ b/archive/provisioning/Agents/librenms/mysql.sh @@ -0,0 +1,1438 @@ +#!/usr/bin/php + true, # Do you want to check InnoDB statistics? + 'master' => true, # Do you want to check binary logging? + 'slave' => true, # Do you want to check slave status? + 'procs' => true, # Do you want to check SHOW PROCESSLIST? + 'get_qrt' => true, # Get query response times from Percona Server or MariaDB? +); + +$use_ss = FALSE; # Whether to use the script server or not +$debug = FALSE; # Define whether you want debugging behavior. +$debug_log = FALSE; # If $debug_log is a filename, it'll be used. + +# ============================================================================ +# You should not need to change anything below this line. +# ============================================================================ +$version = "1.1.7"; + +# ============================================================================ +# Include settings from an external config file (issue 39). +# ============================================================================ +echo("<<>>\n"); + +if (file_exists(__FILE__ . '.cnf' ) ) { + require(__FILE__ . '.cnf'); + debug('Found configuration file ' . __FILE__ . '.cnf'); +} + +# Make this a happy little script even when there are errors. +$no_http_headers = true; +ini_set('implicit_flush', false); # No output, ever. +if ($debug ) { + ini_set('display_errors', true); + ini_set('display_startup_errors', true); + ini_set('error_reporting', 2147483647); +} +else { + ini_set('error_reporting', E_ERROR); +} +ob_start(); # Catch all output such as notices of undefined array indexes. +function error_handler($errno, $errstr, $errfile, $errline) { + print("$errstr at $errfile line $errline\n"); + debug("$errstr at $errfile line $errline"); +} +# ============================================================================ +# Set up the stuff we need to be called by the script server. +# ============================================================================ +#if ($use_ss ) { +# if (file_exists( dirname(__FILE__) . "/../include/global.php") ) { +# # See issue 5 for the reasoning behind this. +# debug("including " . dirname(__FILE__) . "/../include/global.php"); +# include_once(dirname(__FILE__) . "/../include/global.php"); +# } +# elseif (file_exists( dirname(__FILE__) . "/../include/config.php" ) ) { +# # Some Cacti installations don't have global.php. +# debug("including " . dirname(__FILE__) . "/../include/config.php"); +# include_once(dirname(__FILE__) . "/../include/config.php"); +# } +#} + +# ============================================================================ +# Set the default timezone either to the configured, system timezone, or the +# default set above in the script. +# ============================================================================ +if ( function_exists("date_default_timezone_set") + && function_exists("date_default_timezone_get") ) { + $tz = ($timezone ? $timezone : @date_default_timezone_get()); + if ( $tz ) { + @date_default_timezone_set($tz); + } +} + + +# ============================================================================ +# Make sure we can also be called as a script. +# ============================================================================ +if (!isset($called_by_script_server)) { + debug($_SERVER["argv"]); + array_shift($_SERVER["argv"]); # Strip off this script's filename + $options = parse_cmdline($_SERVER["argv"]); + validate_options($options); + $result = ss_get_mysql_stats($options); + + debug($result); + if (!$debug ) { + # Throw away the buffer, which ought to contain only errors. + ob_end_clean(); + } + else { + ob_end_flush(); # In debugging mode, print out the errors. + } + + # Split the result up and extract only the desired parts of it. + $options['items'] = ""; + $wanted = explode(',', $options['items']); + $output = array(); + foreach ( explode(' ', $result) as $item ) { + if (in_array(substr($item, 0, 2), $wanted) ) { + $output[] = $item; + } + list($short, $val) = explode(":", $item); + echo(strtolower($short).":".strtolower($val)."\n"); + } + debug(array("Final result", $output)); + print(implode(' ', $output)); +} + +# ============================================================================ +# End "if file was not included" section. +# ============================================================================ +} + +# ============================================================================ +# Work around the lack of array_change_key_case in older PHP. +# ============================================================================ +if (!function_exists('array_change_key_case') ) { + function array_change_key_case($arr) { + $res = array(); + foreach ( $arr as $key => $val ) { + $res[strtolower($key)] = $val; + } + return $res; + } +} + +# ============================================================================ +# Validate that the command-line options are here and correct +# ============================================================================ +function validate_options($options) { + debug($options); + $opts = array('items', 'user', 'pass', 'heartbeat', 'nocache', 'port', 'server-id'); + # Required command-line options + foreach ( array() as $option ) { + if (!isset($options[$option]) || !$options[$option] ) { + usage("Required option --$option is missing"); + } + } + foreach ( $options as $key => $val ) { + if (!in_array($key, $opts) ) { + usage("Unknown option --$key"); + } + } +} + +# ============================================================================ +# Print out a brief usage summary +# ============================================================================ +function usage($message) { + global $mysql_host, $mysql_user, $mysql_pass, $mysql_port; + + $usage = << --items [OPTION] + + --host MySQL host + --items Comma-separated list of the items whose data you want + --user MySQL username + --pass MySQL password + --port MySQL port + --socket MySQL socket + --flags MySQL flags + --connection-timeout MySQL connection timeout + --server-id Server id to associate with a heartbeat if heartbeat usage is enabled + --nocache Do not cache results in a file + --help Show usage + +EOF; + die($usage); +} + +# ============================================================================ +# Parse command-line arguments, in the format --arg value --arg value, and +# return them as an array ( arg => value ) +# ============================================================================ +function parse_cmdline( $args ) { + $result = array(); + $cur_arg = ''; + foreach ($args as $val) { + if (strpos($val, '--') === 0 ) { + if (strpos($val, '--no') === 0 ) { + # It's an option without an argument, but it's a --nosomething so + # it's OK. + $result[substr($val, 2)] = 1; + $cur_arg = ''; + } + elseif ($cur_arg ) { # Maybe the last --arg was an option with no arg + if ($cur_arg == '--user' || $cur_arg == '--pass' || $cur_arg == '--port' ) { + # Special case because Cacti will pass these without an arg + $cur_arg = ''; + } + else { + die("No arg: $cur_arg\n"); + } + } + else { + $cur_arg = $val; + } + } + else { + $result[substr($cur_arg, 2)] = $val; + $cur_arg = ''; + } + } + if ($cur_arg && ($cur_arg != '--user' && $cur_arg != '--pass' && $cur_arg != '--port') ) { + die("No arg: $cur_arg\n"); + } + debug($result); + return $result; +} + +# ============================================================================ +# This is the main function. Some parameters are filled in from defaults at the +# top of this file. +# ============================================================================ +function ss_get_mysql_stats( $options ) { + # Process connection options and connect to MySQL. + global $debug, $mysql_host, $mysql_user, $mysql_pass, $cache_dir, $poll_time, $chk_options, + $mysql_port, $mysql_socket, $mysql_flags, + $mysql_ssl, $mysql_ssl_key, $mysql_ssl_cert, $mysql_ssl_ca, + $mysql_connection_timeout, + $heartbeat, $heartbeat_table, $heartbeat_server_id, $heartbeat_utc; + + # Connect to MySQL. + $user = isset($options['user']) ? $options['user'] : $mysql_user; + $pass = isset($options['pass']) ? $options['pass'] : $mysql_pass; + $port = isset($options['port']) ? $options['port'] : $mysql_port; + $host = isset($options['host']) ? $options['host'] : $mysql_host; + + $socket = isset($options['socket']) ? $options['socket'] : $mysql_socket; + $flags = isset($options['flags']) ? $options['flags'] : $mysql_flags; + $connection_timeout = isset($options['connection-timeout']) ? $options['connection-timeout'] : $mysql_connection_timeout; + $heartbeat_server_id = isset($options['server-id']) ? $options['server-id'] : $heartbeat_server_id; + + # If there is a port, or if it's a non-standard port, we add ":$port" to the + # hostname. + $host_str = $host.($port != 3306 ? ":$port" : ''); + + + $sanitized_host = str_replace(array(":", "/"), array("", "_"), $host); + $cache_file = "$cache_dir/agent-local-mysql"; + debug("Cache file is $cache_file"); + + # First, check the cache. + $fp = null; + if ( $cache_dir && !array_key_exists('nocache', $options) ) { + if ( $fp = fopen($cache_file, 'a+') ) { + $locked = flock($fp, 1); # LOCK_SH + if ( $locked ) { + if ( filesize($cache_file) > 0 + && filectime($cache_file) + ($poll_time/2) > time() + && ($arr = file($cache_file)) + ) {# The cache file is good to use. + debug("Using the cache file"); + fclose($fp); + return $arr[0]; + } + else { + debug("The cache file seems too small or stale"); + # Escalate the lock to exclusive, so we can write to it. + if ( flock($fp, 2) ) { # LOCK_EX + # We might have blocked while waiting for that LOCK_EX, and + # another process ran and updated it. Let's see if we can just + # return the data now: + if ( filesize($cache_file) > 0 + && filectime($cache_file) + ($poll_time/2) > time() + && ($arr = file($cache_file)) + ) {# The cache file is good to use. + debug("Using the cache file"); + fclose($fp); + return $arr[0]; + } + ftruncate($fp, 0); # Now it's ready for writing later. + } + } + } + else { + $fp = null; + debug("Couldn't lock the cache file, ignoring it"); + } + } + else { + $fp = null; + debug("Couldn't open the cache file"); + } + } + else { + debug("Caching is disabled."); + } + + # Connect to MySQL. + debug(array('Connecting to', $host, $port, $user, $pass)); + if ( !extension_loaded('mysqli') ) { + debug("PHP MySQLi extension is not loaded"); + die("PHP MySQLi extension is not loaded"); + } + $conn = mysqli_init(); + $conn->options(MYSQLI_OPT_CONNECT_TIMEOUT, $connection_timeout); + if ( $mysql_ssl ) { + mysqli_ssl_set($conn, $mysql_ssl_key, $mysql_ssl_cert, $mysql_ssl_ca, NULL, NULL); + } + @mysqli_real_connect($conn, $host, $user, $pass, NULL, $port, $socket, $flags); + if ( mysqli_connect_errno() ) { + debug("MySQL connection failed: " . mysqli_connect_error()); + die("ERROR: " . mysqli_connect_error()); + } + + # MySQL server version. + # The form of this version number is main_version * 10000 + minor_version * 100 + sub_version + # i.e. version 5.5.44 is 50544. + $mysql_version = mysqli_get_server_version($conn); + debug("MySQL server version is " . $mysql_version); + + # Set up variables. + $status = array( # Holds the result of SHOW STATUS, SHOW INNODB STATUS, etc + # Define some indexes so they don't cause errors with += operations. + 'relay_log_space' => null, + 'binary_log_space' => null, + 'current_transactions' => 0, + 'locked_transactions' => 0, + 'active_transactions' => 0, + 'innodb_locked_tables' => 0, + 'innodb_tables_in_use' => 0, + 'innodb_lock_structs' => 0, + 'innodb_lock_wait_secs' => 0, + 'innodb_sem_waits' => 0, + 'innodb_sem_wait_time_ms'=> 0, + # Values for the 'state' column from SHOW PROCESSLIST (converted to + # lowercase, with spaces replaced by underscores) + 'State_closing_tables' => 0, + 'State_copying_to_tmp_table' => 0, + 'State_end' => 0, + 'State_freeing_items' => 0, + 'State_init' => 0, + 'State_locked' => 0, + 'State_login' => 0, + 'State_preparing' => 0, + 'State_reading_from_net' => 0, + 'State_sending_data' => 0, + 'State_sorting_result' => 0, + 'State_statistics' => 0, + 'State_updating' => 0, + 'State_writing_to_net' => 0, + 'State_none' => 0, + 'State_other' => 0, # Everything not listed above + ); + + # Get SHOW STATUS and convert the name-value array into a simple + # associative array. + $result = run_query("SHOW /*!50002 GLOBAL */ STATUS", $conn); + foreach ( $result as $row ) { + $status[$row[0]] = $row[1]; + } + + # Get SHOW VARIABLES and do the same thing, adding it to the $status array. + $result = run_query("SHOW VARIABLES", $conn); + foreach ( $result as $row ) { + $status[$row[0]] = $row[1]; + } + + # Get SHOW SLAVE STATUS, and add it to the $status array. + if ( $chk_options['slave'] ) { + # Leverage lock-free SHOW SLAVE STATUS if available + $result = run_query("SHOW SLAVE STATUS NONBLOCKING", $conn); + if ( !$result ) { + $result = run_query("SHOW SLAVE STATUS NOLOCK", $conn); + if ( !$result ) { + $result = run_query("SHOW SLAVE STATUS", $conn); + } + } + $slave_status_rows_gotten = 0; + foreach ( $result as $row ) { + $slave_status_rows_gotten++; + # Must lowercase keys because different MySQL versions have different + # lettercase. + $row = array_change_key_case($row, CASE_LOWER); + $status['relay_log_space'] = $row['relay_log_space']; + $status['slave_lag'] = $row['seconds_behind_master']; + + # Check replication heartbeat, if present. + if ( $heartbeat ) { + if ( $heartbeat_utc ) { + $now_func = 'UNIX_TIMESTAMP(UTC_TIMESTAMP)'; + } + else { + $now_func = 'UNIX_TIMESTAMP()'; + } + $result2 = run_query( + "SELECT MAX($now_func - ROUND(UNIX_TIMESTAMP(ts)))" + . " AS delay FROM $heartbeat_table" + . " WHERE $heartbeat_server_id = 0 OR server_id = $heartbeat_server_id", $conn); + $slave_delay_rows_gotten = 0; + foreach ( $result2 as $row2 ) { + $slave_delay_rows_gotten++; + if ( $row2 && is_array($row2) + && array_key_exists('delay', $row2) ) + { + $status['slave_lag'] = $row2['delay']; + } + else { + debug("Couldn't get slave lag from $heartbeat_table"); + } + } + if ( $slave_delay_rows_gotten == 0 ) { + debug("Got nothing from heartbeat query"); + } + } + + # Scale slave_running and slave_stopped relative to the slave lag. + $status['slave_running'] = ($row['slave_sql_running'] == 'Yes') + ? $status['slave_lag'] : 0; + $status['slave_stopped'] = ($row['slave_sql_running'] == 'Yes') + ? 0 : $status['slave_lag']; + } + if ( $slave_status_rows_gotten == 0 ) { + debug("Got nothing from SHOW SLAVE STATUS"); + } + } + + # Get SHOW MASTER STATUS, and add it to the $status array. + if ($chk_options['master'] + && array_key_exists('log_bin', $status) + && $status['log_bin'] == 'ON' + ) { # See issue #8 + $binlogs = array(0); + $result = run_query("SHOW MASTER LOGS", $conn); + foreach ( $result as $row ) { + $row = array_change_key_case($row, CASE_LOWER); + # Older versions of MySQL may not have the File_size column in the + # results of the command. Zero-size files indicate the user is + # deleting binlogs manually from disk (bad user! bad!). + if (array_key_exists('file_size', $row) && $row['file_size'] > 0 ) { + $binlogs[] = $row['file_size']; + } + } + if (count($binlogs)) { + $status['binary_log_space'] = to_int(array_sum($binlogs)); + } + } + + # Get SHOW PROCESSLIST and aggregate it by state, then add it to the array + # too. + if ( $chk_options['procs'] ) { + $result = run_query('SHOW PROCESSLIST', $conn); + foreach ( $result as $row ) { + $state = $row['State']; + if ( is_null($state) ) { + $state = 'NULL'; + } + if ( $state == '' ) { + $state = 'none'; + } + # MySQL 5.5 replaces the 'Locked' state with a variety of "Waiting for + # X lock" types of statuses. Wrap these all back into "Locked" because + # we don't really care about the type of locking it is. + $state = preg_replace('/^(Table lock|Waiting for .*lock)$/', 'Locked', $state); + $state = str_replace(' ', '_', strtolower($state)); + if ( array_key_exists("State_$state", $status) ) { + increment($status, "State_$state", 1); + } + else { + increment($status, "State_other", 1); + } + } + } + + # Get SHOW ENGINES to be able to determine whether InnoDB is present. + $engines = array(); + $result = run_query("SHOW ENGINES", $conn); + foreach ( $result as $row ) { + $engines[$row[0]] = $row[1]; + } + + # Get SHOW INNODB STATUS and extract the desired metrics from it, then add + # those to the array too. + if ($chk_options['innodb'] + && array_key_exists('InnoDB', $engines) + && $engines['InnoDB'] == 'YES' + || $engines['InnoDB'] == 'DEFAULT' + ) { + $result = run_query("SHOW /*!50000 ENGINE*/ INNODB STATUS", $conn); + $istatus_text = $result[0]['Status']; + $istatus_vals = get_innodb_array($istatus_text, $mysql_version); + + # Get response time histogram from Percona Server or MariaDB if enabled. + if ( $chk_options['get_qrt'] + && (( isset($status['have_response_time_distribution']) + && $status['have_response_time_distribution'] == 'YES') + || (isset($status['query_response_time_stats']) + && $status['query_response_time_stats'] == 'ON')) ) + { + debug('Getting query time histogram'); + $i = 0; + $result = run_query( + "SELECT `count`, ROUND(total * 1000000) AS total " + . "FROM INFORMATION_SCHEMA.QUERY_RESPONSE_TIME " + . "WHERE `time` <> 'TOO LONG'", + $conn); + foreach ( $result as $row ) { + if ( $i > 13 ) { + # It's possible that the number of rows returned isn't 14. + # Don't add extra status counters. + break; + } + $count_key = sprintf("Query_time_count_%02d", $i); + $total_key = sprintf("Query_time_total_%02d", $i); + $status[$count_key] = $row['count']; + $status[$total_key] = $row['total']; + $i++; + } + # It's also possible that the number of rows returned is too few. + # Don't leave any status counters unassigned; it will break graphs. + while ( $i <= 13 ) { + $count_key = sprintf("Query_time_count_%02d", $i); + $total_key = sprintf("Query_time_total_%02d", $i); + $status[$count_key] = 0; + $status[$total_key] = 0; + $i++; + } + } + else { + debug('Not getting time histogram because it is not enabled'); + } + + # Override values from InnoDB parsing with values from SHOW STATUS, + # because InnoDB status might not have everything and the SHOW STATUS is + # to be preferred where possible. + $overrides = array( + 'Innodb_buffer_pool_pages_data' => 'database_pages', + 'Innodb_buffer_pool_pages_dirty' => 'modified_pages', + 'Innodb_buffer_pool_pages_free' => 'free_pages', + 'Innodb_buffer_pool_pages_total' => 'pool_size', + 'Innodb_data_fsyncs' => 'file_fsyncs', + 'Innodb_data_pending_reads' => 'pending_normal_aio_reads', + 'Innodb_data_pending_writes' => 'pending_normal_aio_writes', + 'Innodb_os_log_pending_fsyncs' => 'pending_log_flushes', + 'Innodb_pages_created' => 'pages_created', + 'Innodb_pages_read' => 'pages_read', + 'Innodb_pages_written' => 'pages_written', + 'Innodb_rows_deleted' => 'rows_deleted', + 'Innodb_rows_inserted' => 'rows_inserted', + 'Innodb_rows_read' => 'rows_read', + 'Innodb_rows_updated' => 'rows_updated', + 'Innodb_buffer_pool_reads' => 'pool_reads', + 'Innodb_buffer_pool_read_requests' => 'pool_read_requests', + ); + + # If the SHOW STATUS value exists, override... + foreach ( $overrides as $key => $val ) { + if (array_key_exists($key, $status) ) { + debug("Override $key"); + $istatus_vals[$val] = $status[$key]; + } + } + + # Now copy the values into $status. + foreach ( $istatus_vals as $key => $val ) { + $status[$key] = $istatus_vals[$key]; + } + } + + # Make table_open_cache backwards-compatible (issue 63). + if (array_key_exists('table_open_cache', $status) ) { + $status['table_cache'] = $status['table_open_cache']; + } + + # Compute how much of the key buffer is used and unflushed (issue 127). + $status['Key_buf_bytes_used'] + = big_sub($status['key_buffer_size'], + big_multiply($status['Key_blocks_unused'], + $status['key_cache_block_size'])); + $status['Key_buf_bytes_unflushed'] + = big_multiply($status['Key_blocks_not_flushed'], + $status['key_cache_block_size']); + + if (array_key_exists('unflushed_log', $status) + && $status['unflushed_log'] + ) { + # TODO: I'm not sure what the deal is here; need to debug this. But the + # unflushed log bytes spikes a lot sometimes and it's impossible for it to + # be more than the log buffer. + debug("Unflushed log: $status[unflushed_log]"); + $status['unflushed_log'] + = max($status['unflushed_log'], $status['innodb_log_buffer_size']); + } + + # Define the variables to output. I use shortened variable names so maybe + # it'll all fit in 1024 bytes for Cactid and Spine's benefit. + # This list must come right after the word MAGIC_VARS_DEFINITIONS. The Perl script + # parses it and uses it as a Perl variable. + $keys = array( + 'Key_read_requests' => 'a0', + 'Key_reads' => 'a1', + 'Key_write_requests' => 'a2', + 'Key_writes' => 'a3', + 'history_list' => 'a4', + 'innodb_transactions' => 'a5', + 'read_views' => 'a6', + 'current_transactions' => 'a7', + 'locked_transactions' => 'a8', + 'active_transactions' => 'a9', + 'pool_size' => 'aa', + 'free_pages' => 'ab', + 'database_pages' => 'ac', + 'modified_pages' => 'ad', + 'pages_read' => 'ae', + 'pages_created' => 'af', + 'pages_written' => 'ag', + 'file_fsyncs' => 'ah', + 'file_reads' => 'ai', + 'file_writes' => 'aj', + 'log_writes' => 'ak', + 'pending_aio_log_ios' => 'al', + 'pending_aio_sync_ios' => 'am', + 'pending_buf_pool_flushes' => 'an', + 'pending_chkp_writes' => 'ao', + 'pending_ibuf_aio_reads' => 'ap', + 'pending_log_flushes' => 'aq', + 'pending_log_writes' => 'ar', + 'pending_normal_aio_reads' => 'as', + 'pending_normal_aio_writes' => 'at', + 'ibuf_inserts' => 'au', + 'ibuf_merged' => 'av', + 'ibuf_merges' => 'aw', + 'spin_waits' => 'ax', + 'spin_rounds' => 'ay', + 'os_waits' => 'az', + 'rows_inserted' => 'b0', + 'rows_updated' => 'b1', + 'rows_deleted' => 'b2', + 'rows_read' => 'b3', + 'Table_locks_waited' => 'b4', + 'Table_locks_immediate' => 'b5', + 'Slow_queries' => 'b6', + 'Open_files' => 'b7', + 'Open_tables' => 'b8', + 'Opened_tables' => 'b9', + 'innodb_open_files' => 'ba', + 'open_files_limit' => 'bb', + 'table_cache' => 'bc', + 'Aborted_clients' => 'bd', + 'Aborted_connects' => 'be', + 'Max_used_connections' => 'bf', + 'Slow_launch_threads' => 'bg', + 'Threads_cached' => 'bh', + 'Threads_connected' => 'bi', + 'Threads_created' => 'bj', + 'Threads_running' => 'bk', + 'max_connections' => 'bl', + 'thread_cache_size' => 'bm', + 'Connections' => 'bn', + 'slave_running' => 'bo', + 'slave_stopped' => 'bp', + 'Slave_retried_transactions' => 'bq', + 'slave_lag' => 'br', + 'Slave_open_temp_tables' => 'bs', + 'Qcache_free_blocks' => 'bt', + 'Qcache_free_memory' => 'bu', + 'Qcache_hits' => 'bv', + 'Qcache_inserts' => 'bw', + 'Qcache_lowmem_prunes' => 'bx', + 'Qcache_not_cached' => 'by', + 'Qcache_queries_in_cache' => 'bz', + 'Qcache_total_blocks' => 'c0', + 'query_cache_size' => 'c1', + 'Questions' => 'c2', + 'Com_update' => 'c3', + 'Com_insert' => 'c4', + 'Com_select' => 'c5', + 'Com_delete' => 'c6', + 'Com_replace' => 'c7', + 'Com_load' => 'c8', + 'Com_update_multi' => 'c9', + 'Com_insert_select' => 'ca', + 'Com_delete_multi' => 'cb', + 'Com_replace_select' => 'cc', + 'Select_full_join' => 'cd', + 'Select_full_range_join' => 'ce', + 'Select_range' => 'cf', + 'Select_range_check' => 'cg', + 'Select_scan' => 'ch', + 'Sort_merge_passes' => 'ci', + 'Sort_range' => 'cj', + 'Sort_rows' => 'ck', + 'Sort_scan' => 'cl', + 'Created_tmp_tables' => 'cm', + 'Created_tmp_disk_tables' => 'cn', + 'Created_tmp_files' => 'co', + 'Bytes_sent' => 'cp', + 'Bytes_received' => 'cq', + 'innodb_log_buffer_size' => 'cr', + 'unflushed_log' => 'cs', + 'log_bytes_flushed' => 'ct', + 'log_bytes_written' => 'cu', + 'relay_log_space' => 'cv', + 'binlog_cache_size' => 'cw', + 'Binlog_cache_disk_use' => 'cx', + 'Binlog_cache_use' => 'cy', + 'binary_log_space' => 'cz', + 'innodb_locked_tables' => 'd0', + 'innodb_lock_structs' => 'd1', + 'State_closing_tables' => 'd2', + 'State_copying_to_tmp_table' => 'd3', + 'State_end' => 'd4', + 'State_freeing_items' => 'd5', + 'State_init' => 'd6', + 'State_locked' => 'd7', + 'State_login' => 'd8', + 'State_preparing' => 'd9', + 'State_reading_from_net' => 'da', + 'State_sending_data' => 'db', + 'State_sorting_result' => 'dc', + 'State_statistics' => 'dd', + 'State_updating' => 'de', + 'State_writing_to_net' => 'df', + 'State_none' => 'dg', + 'State_other' => 'dh', + 'Handler_commit' => 'di', + 'Handler_delete' => 'dj', + 'Handler_discover' => 'dk', + 'Handler_prepare' => 'dl', + 'Handler_read_first' => 'dm', + 'Handler_read_key' => 'dn', + 'Handler_read_next' => 'do', + 'Handler_read_prev' => 'dp', + 'Handler_read_rnd' => 'dq', + 'Handler_read_rnd_next' => 'dr', + 'Handler_rollback' => 'ds', + 'Handler_savepoint' => 'dt', + 'Handler_savepoint_rollback' => 'du', + 'Handler_update' => 'dv', + 'Handler_write' => 'dw', + + # Some InnoDB stats added later... + 'innodb_tables_in_use' => 'dx', + 'innodb_lock_wait_secs' => 'dy', + 'hash_index_cells_total' => 'dz', + 'hash_index_cells_used' => 'e0', + 'total_mem_alloc' => 'e1', + 'additional_pool_alloc' => 'e2', + 'uncheckpointed_bytes' => 'e3', + 'ibuf_used_cells' => 'e4', + 'ibuf_free_cells' => 'e5', + 'ibuf_cell_count' => 'e6', + 'adaptive_hash_memory' => 'e7', + 'page_hash_memory' => 'e8', + 'dictionary_cache_memory' => 'e9', + 'file_system_memory' => 'ea', + 'lock_system_memory' => 'eb', + 'recovery_system_memory' => 'ec', + 'thread_hash_memory' => 'ed', + 'innodb_sem_waits' => 'ee', + 'innodb_sem_wait_time_ms' => 'ef', + 'Key_buf_bytes_unflushed' => 'eg', + 'Key_buf_bytes_used' => 'eh', + 'key_buffer_size' => 'ei', + 'Innodb_row_lock_time' => 'ej', + 'Innodb_row_lock_waits' => 'ek', + + # Values not parsed by LibreNMS + 'Query_time_count_00' => 'ol', + 'Query_time_count_01' => 'om', + 'Query_time_count_02' => 'on', + 'Query_time_count_03' => 'oo', + 'Query_time_count_04' => 'op', + 'Query_time_count_05' => 'oq', + 'Query_time_count_06' => 'or', + 'Query_time_count_07' => 'os', + 'Query_time_count_08' => 'ot', + 'Query_time_count_09' => 'ou', + 'Query_time_count_10' => 'ov', + 'Query_time_count_11' => 'ow', + 'Query_time_count_12' => 'ox', + 'Query_time_count_13' => 'oy', + 'Query_time_total_00' => 'oz', + 'Query_time_total_01' => 'pg', + 'Query_time_total_02' => 'ph', + 'Query_time_total_03' => 'pi', + 'Query_time_total_04' => 'pj', + 'Query_time_total_05' => 'pk', + 'Query_time_total_06' => 'pl', + 'Query_time_total_07' => 'pm', + 'Query_time_total_08' => 'pn', + 'Query_time_total_09' => 'po', + 'Query_time_total_10' => 'pp', + 'Query_time_total_11' => 'pq', + 'Query_time_total_12' => 'pr', + 'Query_time_total_13' => 'ps', + 'wsrep_replicated_bytes' => 'pt', + 'wsrep_received_bytes' => 'pu', + 'wsrep_replicated' => 'pv', + 'wsrep_received' => 'pw', + 'wsrep_local_cert_failures' => 'px', + 'wsrep_local_bf_aborts' => 'py', + 'wsrep_local_send_queue' => 'pz', + 'wsrep_local_recv_queue' => 'qg', + 'wsrep_cluster_size' => 'qh', + 'wsrep_cert_deps_distance' => 'qi', + 'wsrep_apply_window' => 'qj', + 'wsrep_commit_window' => 'qk', + 'wsrep_flow_control_paused' => 'ql', + 'wsrep_flow_control_sent' => 'qm', + 'wsrep_flow_control_recv' => 'qn', + 'pool_reads' => 'qo', + 'pool_read_requests' => 'qp', + ); + + # Return the output. + $output = array(); + foreach ($keys as $key => $short ) { + # If the value isn't defined, return -1 which is lower than (most graphs') + # minimum value of 0, so it'll be regarded as a missing value. + $val = isset($status[$key]) ? $status[$key] : -1; + $output[] = "$short:$val"; + } + $result = implode(' ', $output); + if ($fp ) { + if (fwrite($fp, $result) === FALSE ) { + die("Can't write '$cache_file'"); + } + fclose($fp); + } + + return $result; + +} + +# ============================================================================ +# Given INNODB STATUS text, returns a key-value array of the parsed text. Each +# line shows a sample of the input for both standard InnoDB as you would find in +# MySQL 5.0, and XtraDB or enhanced InnoDB from Percona if applicable. Note +# that extra leading spaces are ignored due to trim(). +# ============================================================================ +function get_innodb_array($text, $mysql_version) { + $results = array( + 'spin_waits' => array(), + 'spin_rounds' => array(), + 'os_waits' => array(), + 'pending_normal_aio_reads' => null, + 'pending_normal_aio_writes' => null, + 'pending_ibuf_aio_reads' => null, + 'pending_aio_log_ios' => null, + 'pending_aio_sync_ios' => null, + 'pending_log_flushes' => null, + 'pending_buf_pool_flushes' => null, + 'file_reads' => null, + 'file_writes' => null, + 'file_fsyncs' => null, + 'ibuf_inserts' => null, + 'ibuf_merged' => null, + 'ibuf_merges' => null, + 'log_bytes_written' => null, + 'unflushed_log' => null, + 'log_bytes_flushed' => null, + 'pending_log_writes' => null, + 'pending_chkp_writes' => null, + 'log_writes' => null, + 'pool_size' => null, + 'free_pages' => null, + 'database_pages' => null, + 'modified_pages' => null, + 'pages_read' => null, + 'pages_created' => null, + 'pages_written' => null, + 'queries_inside' => null, + 'queries_queued' => null, + 'read_views' => null, + 'rows_inserted' => null, + 'rows_updated' => null, + 'rows_deleted' => null, + 'rows_read' => null, + 'innodb_transactions' => null, + 'unpurged_txns' => null, + 'history_list' => null, + 'current_transactions' => null, + 'hash_index_cells_total' => null, + 'hash_index_cells_used' => null, + 'total_mem_alloc' => null, + 'additional_pool_alloc' => null, + 'last_checkpoint' => null, + 'uncheckpointed_bytes' => null, + 'ibuf_used_cells' => null, + 'ibuf_free_cells' => null, + 'ibuf_cell_count' => null, + 'adaptive_hash_memory' => null, + 'page_hash_memory' => null, + 'dictionary_cache_memory' => null, + 'file_system_memory' => null, + 'lock_system_memory' => null, + 'recovery_system_memory' => null, + 'thread_hash_memory' => null, + 'innodb_sem_waits' => null, + 'innodb_sem_wait_time_ms' => null, + ); + $txn_seen = FALSE; + foreach ( explode("\n", $text) as $line ) { + $line = trim($line); + $row = preg_split('/ +/', $line); + + # SEMAPHORES + if (strpos($line, 'Mutex spin waits') === 0 ) { + # Mutex spin waits 79626940, rounds 157459864, OS waits 698719 + # Mutex spin waits 0, rounds 247280272495, OS waits 316513438 + $results['spin_waits'][] = to_int($row[3]); + $results['spin_rounds'][] = to_int($row[5]); + $results['os_waits'][] = to_int($row[8]); + } + elseif (strpos($line, 'RW-shared spins') === 0 + && strpos($line, ';') > 0 ) { + # RW-shared spins 3859028, OS waits 2100750; RW-excl spins 4641946, OS waits 1530310 + $results['spin_waits'][] = to_int($row[2]); + $results['spin_waits'][] = to_int($row[8]); + $results['os_waits'][] = to_int($row[5]); + $results['os_waits'][] = to_int($row[11]); + } + elseif (strpos($line, 'RW-shared spins') === 0 && strpos($line, '; RW-excl spins') === FALSE) { + # Post 5.5.17 SHOW ENGINE INNODB STATUS syntax + # RW-shared spins 604733, rounds 8107431, OS waits 241268 + $results['spin_waits'][] = to_int($row[2]); + $results['os_waits'][] = to_int($row[7]); + } + elseif (strpos($line, 'RW-excl spins') === 0) { + # Post 5.5.17 SHOW ENGINE INNODB STATUS syntax + # RW-excl spins 604733, rounds 8107431, OS waits 241268 + $results['spin_waits'][] = to_int($row[2]); + $results['os_waits'][] = to_int($row[7]); + } + elseif (strpos($line, 'seconds the semaphore:') > 0) { + # --Thread 907205 has waited at handler/ha_innodb.cc line 7156 for 1.00 seconds the semaphore: + increment($results, 'innodb_sem_waits', 1); + increment($results, + 'innodb_sem_wait_time_ms', to_int($row[9]) * 1000); + } + + # TRANSACTIONS + elseif ( strpos($line, 'Trx id counter') === 0 ) { + # The beginning of the TRANSACTIONS section: start counting + # transactions + if ( $mysql_version < 50600 ) { + # For versions prior 5.6: two decimals or one hex + # Trx id counter 0 1170664159 + # Trx id counter 861B144C + $results['innodb_transactions'] = isset($row[4]) ? make_bigint( + $row[3], $row[4]) : base_convert($row[3], 16, 10); + } + else { + # For versions 5.6+ and MariaDB 10.x: one decimal + # Trx id counter 2903813 + $results['innodb_transactions'] = $row[3]; + } + $txn_seen = TRUE; + } + elseif ( strpos($line, 'Purge done for trx') === 0 ) { + if ( $mysql_version < 50600 ) { + # For versions prior 5.6: two decimals or one hex + # Purge done for trx's n:o < 0 1170663853 undo n:o < 0 0 + # Purge done for trx's n:o < 861B135D undo n:o < 0 + $purged_to = $row[7] == 'undo' ? base_convert($row[6], 16, 10) : make_bigint($row[6], $row[7]); + } + else { + # For versions 5.6+ and MariaDB 10.x: one decimal + # Purge done for trx's n:o < 2903354 undo n:o < 0 state: running but idle + $purged_to = $row[6]; + } + $results['unpurged_txns'] + = big_sub($results['innodb_transactions'], $purged_to); + } + elseif (strpos($line, 'History list length') === 0 ) { + # History list length 132 + $results['history_list'] = to_int($row[3]); + } + elseif ( $txn_seen && strpos($line, '---TRANSACTION') === 0 ) { + # ---TRANSACTION 0, not started, process no 13510, OS thread id 1170446656 + increment($results, 'current_transactions', 1); + if ( strpos($line, 'ACTIVE') > 0 ) { + increment($results, 'active_transactions', 1); + } + } + elseif ( $txn_seen && strpos($line, '------- TRX HAS BEEN') === 0 ) { + # ------- TRX HAS BEEN WAITING 32 SEC FOR THIS LOCK TO BE GRANTED: + increment($results, 'innodb_lock_wait_secs', to_int($row[5])); + } + elseif ( strpos($line, 'read views open inside InnoDB') > 0 ) { + # 1 read views open inside InnoDB + $results['read_views'] = to_int($row[0]); + } + elseif ( strpos($line, 'mysql tables in use') === 0 ) { + # mysql tables in use 2, locked 2 + increment($results, 'innodb_tables_in_use', to_int($row[4])); + increment($results, 'innodb_locked_tables', to_int($row[6])); + } + elseif ( $txn_seen && strpos($line, 'lock struct(s)') > 0 ) { + # 23 lock struct(s), heap size 3024, undo log entries 27 + # LOCK WAIT 12 lock struct(s), heap size 3024, undo log entries 5 + # LOCK WAIT 2 lock struct(s), heap size 368 + if ( strpos($line, 'LOCK WAIT') === 0 ) { + increment($results, 'innodb_lock_structs', to_int($row[2])); + increment($results, 'locked_transactions', 1); + } + else { + increment($results, 'innodb_lock_structs', to_int($row[0])); + } + } + + # FILE I/O + elseif (strpos($line, ' OS file reads, ') > 0 ) { + # 8782182 OS file reads, 15635445 OS file writes, 947800 OS fsyncs + $results['file_reads'] = to_int($row[0]); + $results['file_writes'] = to_int($row[4]); + $results['file_fsyncs'] = to_int($row[8]); + } + elseif (strpos($line, 'Pending normal aio reads:') === 0 ) { + # Pending normal aio reads: 0, aio writes: 0, + $results['pending_normal_aio_reads'] = to_int($row[4]); + $results['pending_normal_aio_writes'] = to_int($row[7]); + } + elseif (strpos($line, 'ibuf aio reads') === 0 ) { + # ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0 + $results['pending_ibuf_aio_reads'] = to_int($row[3]); + $results['pending_aio_log_ios'] = to_int($row[6]); + $results['pending_aio_sync_ios'] = to_int($row[9]); + } + elseif ( strpos($line, 'Pending flushes (fsync)') === 0 ) { + # Pending flushes (fsync) log: 0; buffer pool: 0 + $results['pending_log_flushes'] = to_int($row[4]); + $results['pending_buf_pool_flushes'] = to_int($row[7]); + } + + # INSERT BUFFER AND ADAPTIVE HASH INDEX + elseif (strpos($line, 'Ibuf for space 0: size ') === 0 ) { + # Older InnoDB code seemed to be ready for an ibuf per tablespace. It + # had two lines in the output. Newer has just one line, see below. + # Ibuf for space 0: size 1, free list len 887, seg size 889, is not empty + # Ibuf for space 0: size 1, free list len 887, seg size 889, + $results['ibuf_used_cells'] = to_int($row[5]); + $results['ibuf_free_cells'] = to_int($row[9]); + $results['ibuf_cell_count'] = to_int($row[12]); + } + elseif (strpos($line, 'Ibuf: size ') === 0 ) { + # Ibuf: size 1, free list len 4634, seg size 4636, + $results['ibuf_used_cells'] = to_int($row[2]); + $results['ibuf_free_cells'] = to_int($row[6]); + $results['ibuf_cell_count'] = to_int($row[9]); + if (strpos($line, 'merges')) { + $results['ibuf_merges'] = to_int($row[10]); + } + } + elseif (strpos($line, ', delete mark ') > 0 && strpos($prev_line, 'merged operations:') === 0 ) { + # Output of show engine innodb status has changed in 5.5 + # merged operations: + # insert 593983, delete mark 387006, delete 73092 + $results['ibuf_inserts'] = to_int($row[1]); + $results['ibuf_merged'] = to_int($row[1]) + to_int($row[4]) + to_int($row[6]); + } + elseif (strpos($line, ' merged recs, ') > 0 ) { + # 19817685 inserts, 19817684 merged recs, 3552620 merges + $results['ibuf_inserts'] = to_int($row[0]); + $results['ibuf_merged'] = to_int($row[2]); + $results['ibuf_merges'] = to_int($row[5]); + } + elseif (strpos($line, 'Hash table size ') === 0 ) { + # In some versions of InnoDB, the used cells is omitted. + # Hash table size 4425293, used cells 4229064, .... + # Hash table size 57374437, node heap has 72964 buffer(s) <-- no used cells + $results['hash_index_cells_total'] = to_int($row[3]); + $results['hash_index_cells_used'] + = strpos($line, 'used cells') > 0 ? to_int($row[6]) : '0'; + } + + # LOG + elseif (strpos($line, " log i/o's done, ") > 0 ) { + # 3430041 log i/o's done, 17.44 log i/o's/second + # 520835887 log i/o's done, 17.28 log i/o's/second, 518724686 syncs, 2980893 checkpoints + # TODO: graph syncs and checkpoints + $results['log_writes'] = to_int($row[0]); + } + elseif (strpos($line, " pending log writes, ") > 0 ) { + # 0 pending log writes, 0 pending chkp writes + $results['pending_log_writes'] = to_int($row[0]); + $results['pending_chkp_writes'] = to_int($row[4]); + } + elseif (strpos($line, "Log sequence number") === 0 ) { + # This number is NOT printed in hex in InnoDB plugin. + # Log sequence number 13093949495856 //plugin + # Log sequence number 125 3934414864 //normal + $results['log_bytes_written'] + = isset($row[4]) + ? make_bigint($row[3], $row[4]) + : to_int($row[3]); + } + elseif (strpos($line, "Log flushed up to") === 0 ) { + # This number is NOT printed in hex in InnoDB plugin. + # Log flushed up to 13093948219327 + # Log flushed up to 125 3934414864 + $results['log_bytes_flushed'] + = isset($row[5]) + ? make_bigint($row[4], $row[5]) + : to_int($row[4]); + } + elseif (strpos($line, "Last checkpoint at") === 0 ) { + # Last checkpoint at 125 3934293461 + $results['last_checkpoint'] + = isset($row[4]) + ? make_bigint($row[3], $row[4]) + : to_int($row[3]); + } + + # BUFFER POOL AND MEMORY + elseif (strpos($line, "Total memory allocated") === 0 && strpos($line, "in additional pool allocated") > 0 ) { + # Total memory allocated 29642194944; in additional pool allocated 0 + # Total memory allocated by read views 96 + $results['total_mem_alloc'] = to_int($row[3]); + $results['additional_pool_alloc'] = to_int($row[8]); + } + elseif(strpos($line, 'Adaptive hash index ') === 0 ) { + # Adaptive hash index 1538240664 (186998824 + 1351241840) + $results['adaptive_hash_memory'] = to_int($row[3]); + } + elseif(strpos($line, 'Page hash ') === 0 ) { + # Page hash 11688584 + $results['page_hash_memory'] = to_int($row[2]); + } + elseif(strpos($line, 'Dictionary cache ') === 0 ) { + # Dictionary cache 145525560 (140250984 + 5274576) + $results['dictionary_cache_memory'] = to_int($row[2]); + } + elseif(strpos($line, 'File system ') === 0 ) { + # File system 313848 (82672 + 231176) + $results['file_system_memory'] = to_int($row[2]); + } + elseif(strpos($line, 'Lock system ') === 0 ) { + # Lock system 29232616 (29219368 + 13248) + $results['lock_system_memory'] = to_int($row[2]); + } + elseif(strpos($line, 'Recovery system ') === 0 ) { + # Recovery system 0 (0 + 0) + $results['recovery_system_memory'] = to_int($row[2]); + } + elseif(strpos($line, 'Threads ') === 0 ) { + # Threads 409336 (406936 + 2400) + $results['thread_hash_memory'] = to_int($row[1]); + } + elseif(strpos($line, 'innodb_io_pattern ') === 0 ) { + # innodb_io_pattern 0 (0 + 0) + $results['innodb_io_pattern_memory'] = to_int($row[1]); + } + elseif (strpos($line, "Buffer pool size ") === 0 ) { + # The " " after size is necessary to avoid matching the wrong line: + # Buffer pool size 1769471 + # Buffer pool size, bytes 28991012864 + $results['pool_size'] = to_int($row[3]); + } + elseif (strpos($line, "Free buffers") === 0 ) { + # Free buffers 0 + $results['free_pages'] = to_int($row[2]); + } + elseif (strpos($line, "Database pages") === 0 ) { + # Database pages 1696503 + $results['database_pages'] = to_int($row[2]); + } + elseif (strpos($line, "Modified db pages") === 0 ) { + # Modified db pages 160602 + $results['modified_pages'] = to_int($row[3]); + } + elseif (strpos($line, "Pages read ahead") === 0 ) { + # Must do this BEFORE the next test, otherwise it'll get fooled by this + # line from the new plugin (see samples/innodb-015.txt): + # Pages read ahead 0.00/s, evicted without access 0.06/s + # TODO: No-op for now, see issue 134. + } + elseif (strpos($line, "Pages read") === 0 ) { + # Pages read 15240822, created 1770238, written 21705836 + $results['pages_read'] = to_int($row[2]); + $results['pages_created'] = to_int($row[4]); + $results['pages_written'] = to_int($row[6]); + } + + # ROW OPERATIONS + elseif (strpos($line, 'Number of rows inserted') === 0 ) { + # Number of rows inserted 50678311, updated 66425915, deleted 20605903, read 454561562 + $results['rows_inserted'] = to_int($row[4]); + $results['rows_updated'] = to_int($row[6]); + $results['rows_deleted'] = to_int($row[8]); + $results['rows_read'] = to_int($row[10]); + } + elseif (strpos($line, " queries inside InnoDB, ") > 0 ) { + # 0 queries inside InnoDB, 0 queries in queue + $results['queries_inside'] = to_int($row[0]); + $results['queries_queued'] = to_int($row[4]); + } + $prev_line = $line; + } + + foreach ( array('spin_waits', 'spin_rounds', 'os_waits') as $key ) { + $results[$key] = to_int(array_sum($results[$key])); + } + $results['unflushed_log'] + = big_sub($results['log_bytes_written'], $results['log_bytes_flushed']); + $results['uncheckpointed_bytes'] + = big_sub($results['log_bytes_written'], $results['last_checkpoint']); + + return $results; +} + +# ============================================================================ +# Returns a bigint from two ulint or a single hex number. This is tested in +# t/mysql_stats.php and copied, without tests, to ss_get_by_ssh.php. +# ============================================================================ +function make_bigint ($hi, $lo = null) { + debug(array($hi, $lo)); + if (is_null($lo) ) { + # Assume it is a hex string representation. + return base_convert($hi, 16, 10); + } + else { + $hi = $hi ? $hi : '0'; # Handle empty-string or whatnot + $lo = $lo ? $lo : '0'; + return big_add(big_multiply($hi, 4294967296), $lo); + } +} + +# ============================================================================ +# Extracts the numbers from a string. You can't reliably do this by casting to +# an int, because numbers that are bigger than PHP's int (varies by platform) +# will be truncated. And you can't use sprintf(%u) either, because the maximum +# value that will return on some platforms is 4022289582. So this just handles +# them as a string instead. It extracts digits until it finds a non-digit and +# quits. This is tested in t/mysql_stats.php and copied, without tests, to +# ss_get_by_ssh.php. +# ============================================================================ +function to_int ( $str ) { + debug($str); + global $debug; + preg_match('{(\d+)}', $str, $m); + if (isset($m[1]) ) { + return $m[1]; + } + elseif ($debug ) { + print_r(debug_backtrace()); + } + else { + return 0; + } +} + +# ============================================================================ +# Wrap mysql_query in error-handling, and instead of returning the result, +# return an array of arrays in the result. +# ============================================================================ + +# ============================================================================ +function run_query($sql, $conn) { + global $debug; + debug($sql); + $result = @mysqli_query($conn, $sql); + if ( $debug && strpos($sql, 'SHOW SLAVE STATUS ') === false ) { + $error = @mysqli_error($conn); + if ( $error ) { + debug(array($sql, $error)); + die("SQLERR $error in $sql"); + } + } + $array = array(); + $count = @mysqli_num_rows($result); + if ( $count > 10000 ) { + debug('Abnormal number of rows returned: ' . $count); + } + else { + while ( $row = @mysqli_fetch_array($result) ) { + $array[] = $row; + } + } + debug(array($sql, $array)); + return $array; +} + +# Safely increments a value that might be null. +# ============================================================================ +function increment(&$arr, $key, $howmuch) { + debug(array($key, $howmuch)); + if (array_key_exists($key, $arr) && isset($arr[$key]) ) { + $arr[$key] = big_add($arr[$key], $howmuch); + } + else { + $arr[$key] = $howmuch; + } +} + +# ============================================================================ +# Multiply two big integers together as accurately as possible with reasonable +# effort. This is tested in t/mysql_stats.php and copied, without tests, to +# ss_get_by_ssh.php. $force is for testability. +# ============================================================================ +function big_multiply ($left, $right, $force = null) { + if (function_exists("gmp_mul") && (is_null($force) || $force == 'gmp') ) { + debug(array('gmp_mul', $left, $right)); + return gmp_strval( gmp_mul( $left, $right )); + } + elseif (function_exists("bcmul") && (is_null($force) || $force == 'bc') ) { + debug(array('bcmul', $left, $right)); + return bcmul( $left, $right ); + } + else { # Or $force == 'something else' + debug(array('sprintf', $left, $right)); + return sprintf("%.0f", $left * $right); + } +} + +# ============================================================================ +# Subtract two big integers as accurately as possible with reasonable effort. +# This is tested in t/mysql_stats.php and copied, without tests, to +# ss_get_by_ssh.php. $force is for testability. +# ============================================================================ +function big_sub ($left, $right, $force = null) { + debug(array($left, $right)); + if (is_null($left) ) { $left = 0; } + if (is_null($right) ) { $right = 0; } + if (function_exists("gmp_sub") && (is_null($force) || $force == 'gmp')) { + debug(array('gmp_sub', $left, $right)); + return gmp_strval( gmp_sub( $left, $right )); + } + elseif (function_exists("bcsub") && (is_null($force) || $force == 'bc')) { + debug(array('bcsub', $left, $right)); + return bcsub( $left, $right ); + } + else { # Or $force == 'something else' + debug(array('to_int', $left, $right)); + return to_int($left - $right); + } +} + +# ============================================================================ +# Add two big integers together as accurately as possible with reasonable +# effort. This is tested in t/mysql_stats.php and copied, without tests, to +# ss_get_by_ssh.php. $force is for testability. +# ============================================================================ +function big_add ($left, $right, $force = null) { + if (is_null($left) ) { $left = 0; } + if (is_null($right) ) { $right = 0; } + if (function_exists("gmp_add") && (is_null($force) || $force == 'gmp')) { + debug(array('gmp_add', $left, $right)); + return gmp_strval( gmp_add( $left, $right )); + } + elseif (function_exists("bcadd") && (is_null($force) || $force == 'bc')) { + debug(array('bcadd', $left, $right)); + return bcadd( $left, $right ); + } + else { # Or $force == 'something else' + debug(array('to_int', $left, $right)); + return to_int($left + $right); + } +} + +# ============================================================================ +# Writes to a debugging log. +# ============================================================================ +function debug($val) { + global $debug_log; + if (!$debug_log ) { + return; + } + if ($fp = fopen($debug_log, 'a+') ) { + $trace = debug_backtrace(); + $calls = array(); + $i = 0; + $line = 0; + $file = ''; + foreach ( debug_backtrace() as $arr ) { + if ($i++ ) { + $calls[] = "$arr[function]() at $file:$line"; + } + $line = array_key_exists('line', $arr) ? $arr['line'] : '?'; + $file = array_key_exists('file', $arr) ? $arr['file'] : '?'; + } + if (!count($calls) ) { + $calls[] = "at $file:$line"; + } + fwrite($fp, date('Y-m-d h:i:s') . ' ' . implode(' <- ', $calls)); + fwrite($fp, "\n" . var_export($val, TRUE) . "\n"); + fclose($fp); + } + else { # Disable logging + print("Warning: disabling debug logging to $debug_log\n"); + $debug_log = FALSE; + } +} diff --git a/archive/provisioning/Agents/librenms/ntp-client b/archive/provisioning/Agents/librenms/ntp-client new file mode 100644 index 0000000..eccb5e5 --- /dev/null +++ b/archive/provisioning/Agents/librenms/ntp-client @@ -0,0 +1,34 @@ +#!/bin/sh +# Please make sure the paths below are correct. +# Alternatively you can put them in $0.conf, meaning if you've named +# this script ntp-client then it must go in ntp-client.conf . +# +# NTPQV output version of "ntpq -c rv" +# Version 4 is the most common and up to date version. +# +# If you are unsure, which to set, run this script and make sure that +# the JSON output variables match that in "ntpq -c rv". +# +################################################################ +# Don't change anything unless you know what are you doing # +################################################################ +BIN_NTPQ='/usr/bin/env ntpq' +BIN_GREP='/usr/bin/env grep' +BIN_AWK='/usr/bin/env awk' + +CONFIG=$0".conf" +if [ -f "$CONFIG" ]; then + # shellcheck disable=SC1090 + . "$CONFIG" +fi + +NTP_OFFSET=$($BIN_NTPQ -c rv | $BIN_GREP "offset" | $BIN_AWK -Foffset= '{print $2}' | $BIN_AWK -F, '{print $1}') +NTP_FREQUENCY=$($BIN_NTPQ -c rv | $BIN_GREP "frequency" | $BIN_AWK -Ffrequency= '{print $2}' | $BIN_AWK -F, '{print $1}') +NTP_SYS_JITTER=$($BIN_NTPQ -c rv | $BIN_GREP "sys_jitter" | $BIN_AWK -Fsys_jitter= '{print $2}' | $BIN_AWK -F, '{print $1}') +NTP_CLK_JITTER=$($BIN_NTPQ -c rv | $BIN_GREP "clk_jitter" | $BIN_AWK -Fclk_jitter= '{print $2}' | $BIN_AWK -F, '{print $1}') +NTP_WANDER=$($BIN_NTPQ -c rv | $BIN_GREP "clk_wander" | $BIN_AWK -Fclk_wander= '{print $2}' | $BIN_AWK -F, '{print $1}') +NTP_VERSION=$($BIN_NTPQ -c rv | $BIN_GREP "version" | $BIN_AWK -F'ntpd ' '{print $2}' | $BIN_AWK -F. '{print $1}') + +echo '{"data":{"offset":"'"$NTP_OFFSET"'","frequency":"'"$NTP_FREQUENCY"'","sys_jitter":"'"$NTP_SYS_JITTER"'","clk_jitter":"'"$NTP_CLK_JITTER"'","clk_wander":"'"$NTP_WANDER"'"},"version":"'"$NTP_VERSION"'","error":"0","errorString":""}' + +exit 0 diff --git a/archive/provisioning/Agents/librenms/ntp-server.sh b/archive/provisioning/Agents/librenms/ntp-server.sh new file mode 100644 index 0000000..bbf5c73 --- /dev/null +++ b/archive/provisioning/Agents/librenms/ntp-server.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# Please make sure the paths below are correct. +# Alternatively you can put them in $0.conf, meaning if you've named +# this script ntp-client.sh then it must go in ntp-client.sh.conf . +# +# NTPQV output version of "ntpq -c rv" +# p1 DD-WRT and some other outdated linux distros +# p11 FreeBSD 11 and any linux distro that is up to date +# +# If you are unsure, which to set, run this script and make sure that +# the JSON output variables match that in "ntpq -c rv". +# +BIN_NTPD='/usr/bin/env ntpd' +BIN_NTPQ='/usr/bin/env ntpq' +BIN_NTPDC='/usr/bin/env ntpdc' +BIN_GREP='/usr/bin/env grep' +BIN_TR='/usr/bin/env tr' +BIN_CUT='/usr/bin/env cut' +BIN_SED="/usr/bin/env sed" +BIN_AWK='/usr/bin/env awk' +NTPQV="p11" +################################################################ +# Don't change anything unless you know what are you doing # +################################################################ +CONFIG=$0".conf" +if [ -f $CONFIG ]; then + . $CONFIG +fi +VERSION=1 + +STRATUM=`$BIN_NTPQ -c rv | $BIN_GREP -Eow "stratum=[0-9]+" | $BIN_CUT -d "=" -f 2` + +# parse the ntpq info that requires version specific info +NTPQ_RAW=`$BIN_NTPQ -c rv | $BIN_GREP jitter | $BIN_SED 's/[[:alpha:]=,_]/ /g'` +if [ $NTPQV = "p11" ]; then + OFFSET=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $3}'` + FREQUENCY=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $4}'` + SYS_JITTER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $5}'` + CLK_JITTER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $6}'` + CLK_WANDER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $7}'` +fi +if [ $NTPQV = "p1" ]; then + OFFSET=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $2}'` + FREQUENCY=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $3}'` + SYS_JITTER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $4}'` + CLK_JITTER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $5}'` + CLK_WANDER=`echo $NTPQ_RAW | $BIN_AWK -F ' ' '{print $6}'` +fi + +VER=`$BIN_NTPD --version` +if [ "$VER" = '4.2.6p5' ]; then + USECMD=`echo $BIN_NTPDC -c iostats` +else + USECMD=`echo $BIN_NTPQ -c iostats localhost` +fi +CMD2=`$USECMD | $BIN_TR -d ' ' | $BIN_CUT -d : -f 2 | $BIN_TR '\n' ' '` + +TIMESINCERESET=`echo $CMD2 | $BIN_AWK -F ' ' '{print $1}'` +RECEIVEDBUFFERS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $2}'` +FREERECEIVEBUFFERS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $3}'` +USEDRECEIVEBUFFERS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $4}'` +LOWWATERREFILLS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $5}'` +DROPPEDPACKETS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $6}'` +IGNOREDPACKETS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $7}'` +RECEIVEDPACKETS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $8}'` +PACKETSSENT=`echo $CMD2 | $BIN_AWK -F ' ' '{print $9}'` +PACKETSENDFAILURES=`echo $CMD2 | $BIN_AWK -F ' ' '{print $10}'` +INPUTWAKEUPS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $11}'` +USEFULINPUTWAKEUPS=`echo $CMD2 | $BIN_AWK -F ' ' '{print $12}'` + +echo '{"data":{"offset":"'$OFFSET\ +'","frequency":"'$FREQUENCY\ +'","sys_jitter":"'$SYS_JITTER\ +'","clk_jitter":"'$CLK_JITTER\ +'","clk_wander":"'$CLK_WANDER\ +'","stratum":"'$STRATUM\ +'","time_since_reset":"'$TIMESINCERESET\ +'","receive_buffers":"'$RECEIVEDBUFFERS\ +'","free_receive_buffers":"'$FREERECEIVEBUFFERS\ +'","used_receive_buffers":"'$USEDRECEIVEBUFFERS\ +'","low_water_refills":"'$LOWWATERREFILLS\ +'","dropped_packets":"'$DROPPEDPACKETS\ +'","ignored_packets":"'$IGNOREDPACKETS\ +'","received_packets":"'$RECEIVEDPACKETS\ +'","packets_sent":"'$PACKETSSENT\ +'","packet_send_failures":"'$PACKETSENDFAILURES\ +'","input_wakeups":"'$PACKETSENDFAILURES\ +'","useful_input_wakeups":"'$USEFULINPUTWAKEUPS\ +'"},"error":"0","errorString":"","version":"'$VERSION'"}' diff --git a/archive/provisioning/Agents/librenms/os-updates.sh b/archive/provisioning/Agents/librenms/os-updates.sh new file mode 100644 index 0000000..33e1f9c --- /dev/null +++ b/archive/provisioning/Agents/librenms/os-updates.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +################################################################ +# copy this script to /etc/snmp/ and make it executable: # +# chmod +x /etc/snmp/os-updates.sh # +# ------------------------------------------------------------ # +# edit your snmpd.conf and include: # +# extend osupdate /opt/os-updates.sh # +#--------------------------------------------------------------# +# restart snmpd and activate the app for desired host # +#--------------------------------------------------------------# +# please make sure you have the path/binaries below # +################################################################ +BIN_WC='/usr/bin/wc' +BIN_GREP='/bin/grep' +CMD_GREP='-c' +CMD_WC='-l' +BIN_ZYPPER='/usr/bin/zypper' +CMD_ZYPPER='-q lu' +BIN_YUM='/usr/bin/yum' +CMD_YUM='-q check-update' +BIN_DNF='/usr/bin/dnf' +CMD_DNF='-q check-update' +BIN_APT='/usr/bin/apt-get' +CMD_APT='-qq -s upgrade' +BIN_PACMAN='/usr/bin/pacman' +CMD_PACMAN='-Sup' + +################################################################ +# Don't change anything unless you know what are you doing # +################################################################ +if [ -f $BIN_ZYPPER ]; then + # OpenSUSE + UPDATES=`$BIN_ZYPPER $CMD_ZYPPER | $BIN_WC $CMD_WC` + if [ $UPDATES -ge 2 ]; then + echo $(($UPDATES-2)); + else + echo "0"; + fi +elif [ -f $BIN_DNF ]; then + # Fedora + UPDATES=`$BIN_DNF $CMD_DNF | $BIN_WC $CMD_WC` + if [ $UPDATES -ge 1 ]; then + echo $(($UPDATES-1)); + else + echo "0"; + fi +elif [ -f $BIN_PACMAN ]; then + # Arch + UPDATES=`$BIN_PACMAN $CMD_PACMAN | $BIN_WC $CMD_WC` + if [ $UPDATES -ge 1 ]; then + echo $(($UPDATES-1)); + else + echo "0"; + fi +elif [ -f $BIN_YUM ]; then + # CentOS / Redhat + UPDATES=`$BIN_YUM $CMD_YUM | $BIN_WC $CMD_WC` + if [ $UPDATES -ge 1 ]; then + echo $(($UPDATES-1)); + else + echo "0"; + fi +elif [ -f $BIN_APT ]; then + # Debian / Devuan / Ubuntu + UPDATES=`$BIN_APT $CMD_APT | $BIN_GREP $CMD_GREP 'Inst'` + if [ $UPDATES -ge 1 ]; then + echo $UPDATES; + else + echo "0"; + fi +else + echo "0"; +fi diff --git a/archive/provisioning/Agents/librenms/postfix-queues b/archive/provisioning/Agents/librenms/postfix-queues new file mode 100644 index 0000000..1d3491e --- /dev/null +++ b/archive/provisioning/Agents/librenms/postfix-queues @@ -0,0 +1,13 @@ +#!/bin/bash + +#Written by Valec 2006. Steal and share. +#Get postfix queue lengths + +#extend mailq /opt/observer/scripts/getmailq.sh + +QUEUES="incoming active deferred hold" + +for i in $QUEUES; do + COUNT=$(qshape "$i" | grep TOTAL | awk '{print $2}') + printf "$COUNT\n" +done diff --git a/archive/provisioning/Agents/librenms/postfixdetailed b/archive/provisioning/Agents/librenms/postfixdetailed new file mode 100644 index 0000000..e6cb1e9 --- /dev/null +++ b/archive/provisioning/Agents/librenms/postfixdetailed @@ -0,0 +1,548 @@ +#!/usr/bin/env perl + +# add this to your snmpd.conf file as below +# extend postfixdetailed /etc/snmp/postfixdetailed + +# The cache file to use. +my $cache='/var/cache/postfixdetailed'; + +# the location of pflogsumm +my $pflogsumm='/usr/bin/env pflogsumm'; + +#totals +# 847 received = received +# 852 delivered = delivered +# 0 forwarded = forwarded +# 3 deferred (67 deferrals)= deferred +# 0 bounced = bounced +# 593 rejected (41%) = rejected +# 0 reject warnings = rejectw +# 0 held = held +# 0 discarded (0%) = discarded + +# 16899k bytes received = bytesr +# 18009k bytes delivered = bytesd +# 415 senders = senders +# 266 sending hosts/domains = sendinghd +# 15 recipients = recipients +# 9 recipient hosts/domains = recipienthd + +######message deferral detail +#Connection refused = deferralcr +#Host is down = deferralhid + +########message reject detail +#Client host rejected = chr +#Helo command rejected: need fully-qualified hostname = hcrnfqh +#Sender address rejected: Domain not found = sardnf +#Sender address rejected: not owned by user = sarnobu +#blocked using = bu +#Recipient address rejected: User unknown = raruu +#Helo command rejected: Invalid name = hcrin +#Sender address rejected: need fully-qualified address = sarnfqa +#Recipient address rejected: Domain not found = rardnf +#Recipient address rejected: need fully-qualified address = rarnfqa +#Improper use of SMTP command pipelining = iuscp +#Message size exceeds fixed limit = msefl +#Server configuration error = sce +#Server configuration problem = scp +#unknown reject reason = urr + +my $old=''; + +#reads in the old data if it exists +if ( -f $cache ){ + open(my $fh, "<", $cache) or die "Can't open '".$cache."'"; + # if this is over 2048, something is most likely wrong + read($fh , $old , 2048); + close($fh); +} + +my ( $received, + $delivered, + $forwarded, + $deferred, + $bounced, + $rejected, + $rejectw, + $held, + $discarded, + $bytesr, + $bytesd, + $senders, + $sendinghd, + $recipients, + $recipienthd, + $deferralcr, + $deferralhid, + $chr, + $hcrnfqh, + $sardnf, + $sarnobu, + $bu, + $raruu, + $hcrin, + $sarnfqa, + $rardnf, + $rarnfqa, + $iuscp, + $sce, + $scp, + $urr, + $msefl) = split ( /\n/, $old ); + +if ( ! defined( $received ) ){ $received=0; } +if ( ! defined( $delivered ) ){ $delivered=0; } +if ( ! defined( $forwarded ) ){ $forwarded=0; } +if ( ! defined( $deferred ) ){ $deferred=0; } +if ( ! defined( $bounced ) ){ $bounced=0; } +if ( ! defined( $rejected ) ){ $rejected=0; } +if ( ! defined( $rejectw ) ){ $rejectw=0; } +if ( ! defined( $held ) ){ $held=0; } +if ( ! defined( $discarded ) ){ $discarded=0; } +if ( ! defined( $bytesr ) ){ $bytesr=0; } +if ( ! defined( $bytesd ) ){ $bytesd=0; } +if ( ! defined( $senders ) ){ $senders=0; } +if ( ! defined( $sendinghd ) ){ $sendinghd=0; } +if ( ! defined( $recipients ) ){ $recipients=0; } +if ( ! defined( $recipienthd ) ){ $recipienthd=0; } +if ( ! defined( $deferralcr ) ){ $deferralcr=0; } +if ( ! defined( $deferralhid ) ){ $deferralhid=0; } +if ( ! defined( $chr ) ){ $chr=0; } +if ( ! defined( $hcrnfqh ) ){ $hcrnfqh=0; } +if ( ! defined( $sardnf ) ){ $sardnf=0; } +if ( ! defined( $sarnobu ) ){ $sarnobu=0; } +if ( ! defined( $bu ) ){ $bu=0; } +if ( ! defined( $raruu ) ){ $raruu=0; } +if ( ! defined( $hcrin ) ){ $hcrin=0; } +if ( ! defined( $sarnfqa ) ){ $sarnfqa=0; } +if ( ! defined( $rardnf ) ){ $rardnf=0; } +if ( ! defined( $rarnfqa ) ){ $rarnfqa=0; } +if ( ! defined( $iuscp ) ){ $iuscp=0; } +if ( ! defined( $msefl ) ){ $msefl=0; } +if ( ! defined( $sce ) ){ $sce=0; } +if ( ! defined( $scp ) ){ $scp=0; } +if ( ! defined( $urr ) ){ $urr=0; } + +#init current variables +my $receivedC=0; +my $deliveredC=0; +my $forwardedC=0; +my $deferredC=0; +my $bouncedC=0; +my $rejectedC=0; +my $rejectwC=0; +my $heldC=0; +my $discardedC=0; +my $bytesrC=0; +my $bytesdC=0; +my $sendersC=0; +my $sendinghdC=0; +my $recipientsC=0; +my $recipienthdC=0; +my $deferralcrC=0; +my $deferralhidC=0; +my $hcrnfqhC=0; +my $sardnfC=0; +my $sarnobuC=0; +my $buC=0; +my $raruuC=0; +my $hcrinC=0; +my $sarnfqaC=0; +my $rardnfC=0; +my $rarnfqaC=0; +my $iuscpC=0; +my $mseflC=0; +my $sceC=0; +my $scpC=0; +my $urrC=0; + +sub newValue{ + my $old=$_[0]; + my $new=$_[1]; + + #if new is undefined, just default to 0... this should never happen + if ( !defined( $new ) ){ + warn('New not defined'); + return 0; + } + + #sets it to 0 if old is not defined + if ( !defined( $old ) ){ + warn('Old not defined'); + $old=0; + } + + #make sure they are both numberic and if not set to zero + if( $old !~ /^[0123456789]*$/ ){ + warn('Old not numeric'); + $old=0; + } + if( $new !~ /^[0123456789]*$/ ){ + warn('New not numeric'); + $new=0; + } + + #log rotation happened + if ( $old > $new ){ + return $new; + }; + + return $new - $old; +} + + +my $output=`$pflogsumm /var/log/maillog`; + +#holds RBL values till the end when it is compared to the old one +my $buNew=0; + + +#holds client host rejected values till the end when it is compared to the old one +my $chrNew=0; + +# holds recipient address rejected values till the end when it is compared to the old one +my $raruuNew=0; + +#holds the current values for checking later +my $current=''; + +my @outputA=split( /\n/, $output ); +my $int=0; +while ( defined( $outputA[$int] ) ){ + my $line=$outputA[$int]; + + $line=~s/^ *//; + $line=~s/ +/ /g; + $line=~s/\)$//; + + my $handled=0; + + #received line + if ( ( $line =~ /[0123456789] received$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $receivedC=$line; + $received=newValue( $received, $line ); + $handled=1; + } + + #delivered line + if ( ( $line =~ /[0123456789] delivered$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $deliveredC=$line; + $delivered=newValue( $delivered, $line ); + $handled=1; + } + + #forward line + if ( ( $line =~ /[0123456789] forwarded$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $forwardedC=$line; + $forwarded=newValue( $forwarded, $line ); + $handled=1; + } + + #defereed line + if ( ( $line =~ /[0123456789] deferred \(/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $deferredC=$line; + $deferred=newValue( $deferred, $line ); + $handled=1; + } + + #bounced line + if ( ( $line =~ /[0123456789] bounced$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $bouncedC=$line; + $bounced=newValue( $bounced, $line ); + $handled=1; + } + + #rejected line + if ( ( $line =~ /[0123456789] rejected \(/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $rejectedC=$line; + $rejected=newValue( $rejected, $line ); + $handled=1; + } + + #reject warning line + if ( ( $line =~ /[0123456789] reject warnings/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $rejectwC=$line; + $rejectw=newValue( $rejectw, $line ); + $handled=1; + } + + #held line + if ( ( $line =~ /[0123456789] held$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $heldC=$line; + $held=newValue( $held, $line ); + $handled=1; + } + + #discarded line + if ( ( $line =~ /[0123456789] discarded \(/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $discardedC=$line; + $discarded=newValue( $discarded, $line ); + $handled=1; + } + + #bytes received line + if ( ( $line =~ /[0123456789kM] bytes received$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $line=~s/k/000/; + $line=~s/M/000000/; + $bytesrC=$line; + $bytesr=newValue( $bytesr, $line ); + $handled=1; + } + + #bytes delivered line + if ( ( $line =~ /[0123456789kM] bytes delivered$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $line=~s/k/000/; + $line=~s/M/000000/; + $bytesdC=$line; + $bytesd=newValue( $bytesd, $line ); + $handled=1; + } + + #senders line + if ( ( $line =~ /[0123456789] senders$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $sendersC=$line; + $senders=newValue( $senders, $line ); + $handled=1; + } + + #sendering hosts/domains line + if ( ( $line =~ /[0123456789] sending hosts\/domains$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $sendinghdC=$line; + $sendinghd=newValue( $sendinghd, $line ); + $handled=1; + } + + #recipients line + if ( ( $line =~ /[0123456789] recipients$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $recipientsC=$line; + $recipients=newValue( $recipients, $line ); + $handled=1; + } + + #recipients line + if ( ( $line =~ /[0123456789] recipient hosts\/domains$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $recipienthdC=$line; + $recipienthd=newValue( $recipienthd, $line ); + $handled=1; + } + + # deferrals connectios refused + if ( ( $line =~ /[0123456789] 25\: Connection refused$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $deferralcrC=$line; + $deferralcr=newValue( $deferralcr, $line ); + $handled=1; + } + + # deferrals Host is down + if ( ( $line =~ /Host is down$/ ) && ( ! $handled ) ){ + $line=~s/ .*//; + $deferralcrC=$line; + $deferralhidC=$line; + $deferralhid=newValue( $deferralhid, $line ); + $handled=1; + } + + # Client host rejected + if ( ( $line =~ /Client host rejected/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $chrNew=$chrNew + $line; + $handled=1; + } + + #Helo command rejected: need fully-qualified hostname + if ( ( $line =~ /Helo command rejected\: need fully\-qualified hostname/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $hcrnfqhC=$line; + $hcrnfqh=newValue( $hcrnfqh, $line ); + $handled=1; + } + + #Sender address rejected: Domain not found + if ( ( $line =~ /Sender address rejected\: Domain not found/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $sardnfC=$line; + $sardnf=newValue( $sardnf, $line ); + $handled=1; + } + + #Sender address rejected: not owned by user + if ( ( $line =~ /Sender address rejected\: not owned by user/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $sarnobuC=$line; + $sarnobu=newValue( $sarnobu, $line ); + $handled=1; + } + + #blocked using + # These lines are RBLs so there will be more than one. + # Use $buNew to add them all up. + if ( ( $line =~ /blocked using/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $buNew=$buNew + $line; + $handled=1; + } + + #Recipient address rejected: User unknown + if ( ( $line =~ /Recipient address rejected\: User unknown/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $raruuNew=$raruuNew + $line; + $handled=1; + } + + #Helo command rejected: Invalid name + if ( ( $line =~ /Helo command rejected\: Invalid name/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $hcrinC=$line; + $hcrin=newValue( $hcrin, $line ); + } + + #Sender address rejected: need fully-qualified address + if ( ( $line =~ /Sender address rejected\: need fully-qualified address/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $sarnfqaC=$line; + $sarnfqa=newValue( $sarnfqa, $line ); + } + + #Recipient address rejected: Domain not found + if ( ( $line =~ /Recipient address rejected\: Domain not found/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $rardnfC=$line; + $rardnf=newValue( $rardnf, $line ); + } + + #Improper use of SMTP command pipelining + if ( ( $line =~ /Improper use of SMTP command pipelining/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $iuscpC=$line; + $iuscp=newValue( $iuscp, $line ); + } + + #Message size exceeds fixed limit + if ( ( $line =~ /Message size exceeds fixed limit/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $mseflC=$line; + $msefl=newValue( $msefl, $line ); + } + + #Server configuration error + if ( ( $line =~ /Server configuration error/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $sceC=$line; + $sce=newValue( $sce, $line ); + } + + #Server configuration problem + if ( ( $line =~ /Server configuration problem/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $scpC=$line; + $scp=newValue( $scp, $line ); + } + + #unknown reject reason + if ( ( $line =~ /unknown reject reason/ ) && ( ! $handled ) ){ + $line=~s/.*\: //g; + $urrC=$line; + $urr=newValue( $urr, $line ); + } + + $int++; +} + + +# final client host rejected total +$chr=newValue( $chr, $chrNew ); + +# final RBL total +$bu=newValue( $bu, $buNew ); + +# final recipient address rejected total +$raruu=newValue( $raruu, $raruuNew ); + +my $data=$received."\n". + $delivered."\n". + $forwarded."\n". + $deferred."\n". + $bounced."\n". + $rejected."\n". + $rejectw."\n". + $held."\n". + $discarded."\n". + $bytesr."\n". + $bytesd."\n". + $senders."\n". + $sendinghd."\n". + $recipients."\n". + $recipienthd."\n". + $deferralcr."\n". + $deferralhid."\n". + $chr."\n". + $hcrnfqh."\n". + $sardnf."\n". + $sarnobu."\n". + $bu."\n". + $raruu."\n". + $hcrin."\n". + $sarnfqa."\n". + $rardnf."\n". + $rarnfqa."\n". + $iuscp."\n". + $sce."\n". + $scp."\n". + $urr."\n". + $msefl."\n"; + +print $data; + +my $current=$receivedC."\n". + $deliveredC."\n". + $forwardedC."\n". + $deferredC."\n". + $bouncedC."\n". + $rejectedC."\n". + $rejectwC."\n". + $heldC."\n". + $discardedC."\n". + $bytesrC."\n". + $bytesdC."\n". + $sendersC."\n". + $sendinghdC."\n". + $recipientsC."\n". + $recipienthdC."\n". + $deferralcrC."\n". + $deferralhidC."\n". + $chrNew."\n". + $hcrnfqhC."\n". + $sardnfC."\n". + $sarnobuC."\n". + $buNew."\n". + $raruuNew."\n". + $hcrinC."\n". + $sarnfqaC."\n". + $rardnfC."\n". + $rarnfqaC."\n". + $iuscpC."\n". + $sceC."\n". + $scpC."\n". + $urrC."\n". + $mseflC."\n"; + +open(my $fh, ">", $cache) or die "Can't open '".$cache."'"; +print $fh $current; +close($fh); diff --git a/archive/provisioning/Agents/librenms/raspberry.sh b/archive/provisioning/Agents/librenms/raspberry.sh new file mode 100644 index 0000000..404e815 --- /dev/null +++ b/archive/provisioning/Agents/librenms/raspberry.sh @@ -0,0 +1,46 @@ +#!/bin/bash +####################################### +# please read DOCS to succesfully get # +# raspberry sensors into your host # +####################################### +picmd='/usr/bin/vcgencmd' +pised='/bin/sed' +getTemp='measure_temp' +getVoltsCore='measure_volts core' +getVoltsRamC='measure_volts sdram_c' +getVoltsRamI='measure_volts sdram_i' +getVoltsRamP='measure_volts sdram_p' +getFreqArm='measure_clock arm' +getFreqCore='measure_clock core' +getStatusH264='codec_enabled H264' +getStatusMPG2='codec_enabled MPG2' +getStatusWVC1='codec_enabled WVC1' +getStatusMPG4='codec_enabled MPG4' +getStatusMJPG='codec_enabled MJPG' +getStatusWMV9='codec_enabled WMV9' + +$picmd $getTemp | $pised 's|[^0-9.]||g' +$picmd "$getVoltsCore" | $pised 's|[^0-9.]||g' +$picmd "$getVoltsRamC" | $pised 's|[^0-9.]||g' +$picmd "$getVoltsRamI" | $pised 's|[^0-9.]||g' +$picmd "$getVoltsRamP" | $pised 's|[^0-9.]||g' +$picmd "$getFreqArm" | $pised 's/frequency([0-9]*)=//g' +$picmd "$getFreqCore" | $pised 's/frequency([0-9]*)=//g' +$picmd "$getStatusH264" | $pised 's/H264=//g' +$picmd "$getStatusMPG2" | $pised 's/MPG2=//g' +$picmd "$getStatusWVC1" | $pised 's/WVC1=//g' +$picmd "$getStatusMPG4" | $pised 's/MPG4=//g' +$picmd "$getStatusMJPG" | $pised 's/MJPG=//g' +$picmd "$getStatusWMV9" | $pised 's/WMV9=//g' +$picmd "$getStatusH264" | $pised 's/enabled/2/g' +$picmd "$getStatusMPG2" | $pised 's/enabled/2/g' +$picmd "$getStatusWVC1" | $pised 's/enabled/2/g' +$picmd "$getStatusMPG4" | $pised 's/enabled/2/g' +$picmd "$getStatusMJPG" | $pised 's/enabled/2/g' +$picmd "$getStatusWMV9" | $pised 's/enabled/2/g' +$picmd "$getStatusH264" | $pised 's/disabled/1/g' +$picmd "$getStatusMPG2" | $pised 's/disabled/1/g' +$picmd "$getStatusWVC1" | $pised 's/disabled/1/g' +$picmd "$getStatusMPG4" | $pised 's/disabled/1/g' +$picmd "$getStatusMJPG" | $pised 's/disabled/1/g' +$picmd "$getStatusWMV9" | $pised 's/disabled/1/g' diff --git a/archive/provisioning/Agents/librenms/smart b/archive/provisioning/Agents/librenms/smart new file mode 100644 index 0000000..935ed7a --- /dev/null +++ b/archive/provisioning/Agents/librenms/smart @@ -0,0 +1,929 @@ +#!/usr/bin/env perl +#Copyright (c) 2024, Zane C. Bowers-Hadley +#All rights reserved. +# +#Redistribution and use in source and binary forms, with or without modification, +#are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +#ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +#WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +#IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +#INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +#BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +#DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +#LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +#OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +#THE POSSIBILITY OF SUCH DAMAGE. + +=for comment + +Add this to snmpd.conf like below. + + extend smart /etc/snmp/smart + +Then add to root's cron tab, if you have more than a few disks. + + */5 * * * * /etc/snmp/extends/smart -u + +You will also need to create the config file, which defaults to the same path as the script, +but with .config appended. So if the script is located at /etc/snmp/smart, the config file +will be /etc/snmp/extends/smart.config. Alternatively you can also specific a config via -c. + +Anything starting with a # is comment. The format for variables is $variable=$value. Empty +lines are ignored. Spaces and tabes at either the start or end of a line are ignored. Any +line with out a matched variable or # are treated as a disk. + + #This is a comment + cache=/var/cache/smart + smartctl=/usr/local/sbin/smartctl + useSN=0 + ada0 + da5 /dev/da5 -d sat + twl0,0 /dev/twl0 -d 3ware,0 + twl0,1 /dev/twl0 -d 3ware,1 + twl0,2 /dev/twl0 -d 3ware,2 + +The variables are as below. + + cache = The path to the cache file to use. Default: /var/cache/smart + smartctl = The path to use for smartctl. Default: /usr/bin/env smartctl + useSN = If set to 1, it will use the disks SN for reporting instead of the device name. + 1 is the default. 0 will use the device name. + +A disk line is can be as simple as just a disk name under /dev/. Such as in the config above +The line "ada0" would resolve to "/dev/ada0" and would be called with no special argument. If +a line has a space in it, everything before the space is treated as the disk name and is what +used for reporting and everything after that is used as the argument to be passed to smartctl. + +If you want to guess at the configuration, call it with -g and it will print out what it thinks +it should be. + + +Switches: + +-c The config file to use. +-u Update +-p Pretty print the JSON. +-Z GZip+Base64 compress the results. + +-g Guess at the config and print it to STDOUT +-C Enable manual checking for guess and cciss. +-S Set useSN to 0 when using -g +-t Run the specified smart self test on all the devices. +-U When calling cciss_vol_status, call it with -u. +-G Guess modes to use. This is a comma seperated list. + Default :: scan-open,cciss-vol-status + +Guess Modes: + +- scan :: Use "--scan" with smartctl. "scan-open" will take presidence. + +- scan-open :: Call smartctl with "--scan-open". + +- cciss-vol-status :: Freebsd/Linux specific and if it sees /dev/sg0(on Linux) or + /dev/ciss0(on FreebSD) it will attempt to find drives via cciss-vol-status, + and then optionally checking for disks via smrtctl if -C is given. Should be noted + though that -C will not find drives that are currently missing/failed. If -U is given, + cciss_vol_status will be called with -u. + +=cut + +## +## You should not need to touch anything below here. +## +use warnings; +use strict; +use Getopt::Std; +use JSON; +use MIME::Base64; +use IO::Compress::Gzip qw(gzip $GzipError); + +my $cache = '/var/cache/smart'; +my $smartctl = '/usr/bin/env smartctl'; +my @disks; +my $useSN = 1; + +$Getopt::Std::STANDARD_HELP_VERSION = 1; + +sub main::VERSION_MESSAGE { + print "SMART SNMP extend 0.3.2\n"; +} + +sub main::HELP_MESSAGE { + &VERSION_MESSAGE; + print "\n" . "-u Update '" . $cache . "'\n" . '-g Guess at the config and print it to STDOUT +-c The config file to use. +-p Pretty print the JSON. +-Z GZip+Base64 compress the results. +-C Enable manual checking for guess and cciss. +-S Set useSN to 0 when using -g +-t Run the specified smart self test on all the devices. +-U When calling cciss_vol_status, call it with -u. +-G Guess modes to use. This is a comma seperated list. + Default :: scan-open,cciss-vol-status + + +Scan Modes: + +- scan :: Use "--scan" with smartctl. "scan-open" will take presidence. + +- scan-open :: Call smartctl with "--scan-open". + +- cciss-vol-status :: Freebsd/Linux specific and if it sees /dev/sg0(on Linux) or + /dev/ciss0(on FreebSD) it will attempt to find drives via cciss-vol-status, + and then optionally checking for disks via smrtctl if -C is given. Should be noted + though that -C will not find drives that are currently missing/failed. If -U is given, + cciss_vol_status will be called with -u. +'; + +} ## end sub main::HELP_MESSAGE + +#gets the options +my %opts = (); +getopts( 'ugc:pZhvCSGt:U', \%opts ); + +if ( $opts{h} ) { + &HELP_MESSAGE; + exit; +} +if ( $opts{v} ) { + &VERSION_MESSAGE; + exit; +} + +# +# figure out what scan modes to use if -g specified +# +my $scan_modes = { + 'scan-open' => 0, + 'scan' => 0, + 'cciss_vol_status' => 0, +}; +if ( $opts{g} ) { + if ( !defined( $opts{G} ) ) { + $opts{G} = 'scan-open,cciss_vol_status'; + } + $opts{G} =~ s/[\ \t]//g; + my @scan_modes_split = split( /,/, $opts{G} ); + foreach my $mode (@scan_modes_split) { + if ( !defined $scan_modes->{$mode} ) { + die( '"' . $mode . '" is not a recognized scan mode' ); + } + $scan_modes->{$mode} = 1; + } +} ## end if ( $opts{g} ) + +# configure JSON for later usage +# only need to do this if actually running as in -g is not specified +my $json; +if ( !$opts{g} ) { + + $json = JSON->new->allow_nonref->canonical(1); + if ( $opts{p} ) { + $json->pretty; + } +} + +# +# +# guess if asked +# +# +if ( defined( $opts{g} ) ) { + + #get what path to use for smartctl + $smartctl = `which smartctl`; + chomp($smartctl); + if ( $? != 0 ) { + warn("'which smartctl' failed with a exit code of $?"); + exit 1; + } + + #try to touch the default cache location and warn if it can't be done + system( 'touch ' . $cache . '>/dev/null' ); + if ( $? != 0 ) { + $cache = '#Could not touch ' . $cache . "You will need to manually set it\n" . "cache=?\n"; + } else { + system( 'rm -f ' . $cache . '>/dev/null' ); + $cache = 'cache=' . $cache . "\n"; + } + + my $drive_lines = ''; + + # + # + # scan-open and scan guess mode handling + # + # + if ( $scan_modes->{'scan-open'} || $scan_modes->{'scan'} ) { + # used for checking if a disk has been found more than once + my %found_disks_names; + my @argumentsA; + + # use scan-open if it is set, overriding scan if it is also set + my $mode = 'scan'; + if ( $scan_modes->{'scan-open'} ) { + $mode = 'scan-open'; + } + + #have smartctl scan and see if it finds anythings not get found + my $scan_output = `$smartctl --$mode`; + my @scan_outputA = split( /\n/, $scan_output ); + + # remove non-SMART devices sometimes returned + @scan_outputA = grep( !/ses[0-9]/, @scan_outputA ); # not a disk, but may or may not have SMART attributes + @scan_outputA = grep( !/pass[0-9]/, @scan_outputA ); # very likely a duplicate and a disk under another name + @scan_outputA = grep( !/cd[0-9]/, @scan_outputA ); # CD drive + if ( $^O eq 'freebsd' ) { + @scan_outputA = grep( !/sa[0-9]/, @scan_outputA ); # tape drive + @scan_outputA = grep( !/ctl[0-9]/, @scan_outputA ); # CAM target layer + } elsif ( $^O eq 'linux' ) { + @scan_outputA = grep( !/st[0-9]/, @scan_outputA ); # SCSI tape drive + @scan_outputA = grep( !/ht[0-9]/, @scan_outputA ); # ATA tape drive + } + + # make the first pass, figuring out what all we have and trimming comments + foreach my $arguments (@scan_outputA) { + my $name = $arguments; + + $arguments =~ s/ \#.*//; # trim the comment out of the argument + $name =~ s/ .*//; + $name =~ s/\/dev\///; + if ( defined( $found_disks_names{$name} ) ) { + $found_disks_names{$name}++; + } else { + $found_disks_names{$name} = 0; + } + + push( @argumentsA, $arguments ); + + } ## end foreach my $arguments (@scan_outputA) + + # second pass, putting the lines together + my %current_disk; + foreach my $arguments (@argumentsA) { + my $not_virt = 1; + + # check to see if we have a virtual device + my @virt_check = split( /\n/, `smartctl -i $arguments 2> /dev/null` ); + foreach my $virt_check_line (@virt_check) { + if ( $virt_check_line =~ /(?i)Product\:.*LOGICAL VOLUME/ ) { + $not_virt = 0; + } + } + + my $name = $arguments; + $name =~ s/ .*//; + $name =~ s/\/dev\///; + + # only add it if not a virtual RAID drive + # HP RAID virtual disks will show up with very basical but totally useless smart data + if ($not_virt) { + if ( $found_disks_names{$name} == 0 ) { + # If no other devices, just name it after the base device. + $drive_lines = $drive_lines . $name . " " . $arguments . "\n"; + } else { + # if more than one, start at zero and increment, apennding comma number to the base device name + if ( defined( $current_disk{$name} ) ) { + $current_disk{$name}++; + } else { + $current_disk{$name} = 0; + } + $drive_lines = $drive_lines . $name . "," . $current_disk{$name} . " " . $arguments . "\n"; + } + } ## end if ($not_virt) + + } ## end foreach my $arguments (@argumentsA) + } ## end if ( $scan_modes->{'scan-open'} || $scan_modes...) + + # + # + # scan mode handler for cciss_vol_status + # /dev/sg* devices for cciss on Linux + # /dev/ccis* devices for cciss on FreeBSD + # + # + if ( $scan_modes->{'cciss_vol_status'} && ( $^O eq 'linux' || $^O eq 'freebsd' ) ) { + my $cciss; + if ( $^O eq 'freebsd' ) { + $cciss = 'ciss'; + } elsif ( $^O eq 'linux' ) { + $cciss = 'sg'; + } + + my $uarg = ''; + if ( $opts{U} ) { + $uarg = '-u'; + } + + # generate the initial device path that will be checked + my $sg_int = 0; + my $device = '/dev/' . $cciss . $sg_int; + + my $sg_process = 1; + if ( -e $device ) { + my $output = `which cciss_vol_status 2> /dev/null`; + if ( $? != 0 && !$opts{C} ) { + $sg_process = 0; + $drive_lines + = $drive_lines + . "# -C not given, but " + . $device + . " exists and cciss_vol_status is not present\n" + . "# in path or 'ccis_vol_status -V " + . $device + . "' is failing\n"; + } ## end if ( $? != 0 && !$opts{C} ) + } ## end if ( -e $device ) + my $seen_lines = {}; + my $ignore_lines = {}; + while ( -e $device && $sg_process ) { + my $output = `cciss_vol_status -V $uarg $device 2> /dev/null`; + if ( $? != 0 && $output eq '' && !$opts{C} ) { + # just empty here as we just want to skip it if it fails and there is no C + # warning is above + } elsif ( $? != 0 && $output eq '' && $opts{C} ) { + my $drive_count = 0; + my $continue = 1; + while ($continue) { + my $output = `$smartctl -i $device -d cciss,$drive_count 2> /dev/null`; + if ( $? != 0 ) { + $continue = 0; + } else { + my $add_it = 0; + my $id; + while ( $output =~ /(?i)Serial Number:(.*)/g ) { + $id = $1; + $id =~ s/^\s+|\s+$//g; + } + if ( defined($id) && !defined( $seen_lines->{$id} ) ) { + $add_it = 1; + $seen_lines->{$id} = 1; + } + if ( $continue && $add_it ) { + $drive_lines + = $drive_lines + . $cciss . '0-' + . $drive_count . ' ' + . $device + . ' -d cciss,' + . $drive_count . "\n"; + } + } ## end else [ if ( $? != 0 ) ] + $drive_count++; + } ## end while ($continue) + } else { + my $drive_count = 0; + # count the connector lines, this will make sure failed are founded as well + my $seen_conectors = {}; + while ( $output =~ /(connector +\d+[IA]\ +box +\d+\ +bay +\d+.*)/g ) { + my $cciss_drive_line = $1; + my $connector = $cciss_drive_line; + $connector =~ s/(.*\ bay +\d+).*/$1/; + if ( !defined( $seen_lines->{$cciss_drive_line} ) + && !defined( $seen_conectors->{$connector} ) + && !defined( $ignore_lines->{$cciss_drive_line} ) ) + { + $seen_lines->{$cciss_drive_line} = 1; + $seen_conectors->{$connector} = 1; + $drive_count++; + } else { + # going to be a connector we've already seen + # which will happen when it is processing replacement drives + # so save this as a device to ignore + $ignore_lines->{$cciss_drive_line} = 1; + } + } ## end while ( $output =~ /(connector +\d+[IA]\ +box +\d+\ +bay +\d+.*)/g) + my $drive_int = 0; + while ( $drive_int < $drive_count ) { + $drive_lines + = $drive_lines + . $cciss + . $sg_int . '-' + . $drive_int . ' ' + . $device + . ' -d cciss,' + . $drive_int . "\n"; + + $drive_int++; + } ## end while ( $drive_int < $drive_count ) + } ## end else [ if ( $? != 0 && $output eq '' && !$opts{C})] + + $sg_int++; + $device = '/dev/' . $cciss . $sg_int; + } ## end while ( -e $device && $sg_process ) + } ## end if ( $scan_modes->{'cciss_vol_status'} && ...) + + my $useSN = 1; + if ( $opts{S} ) { + $useSN = 0; + } + + print '# scan_modes=' + . $opts{G} + . "\nuseSN=" + . $useSN . "\n" + . 'smartctl=' + . $smartctl . "\n" + . $cache + . $drive_lines; + + exit 0; +} ## end if ( defined( $opts{g} ) ) + +#get which config file to use +my $config = $0 . '.config'; +if ( defined( $opts{c} ) ) { + $config = $opts{c}; +} + +#reads the config file, optionally +my $config_file = ''; +open( my $readfh, "<", $config ) or die "Can't open '" . $config . "'"; +read( $readfh, $config_file, 1000000 ); +close($readfh); + +# +# +# parse the config file and remove comments and empty lines +# +# +my @configA = split( /\n/, $config_file ); +@configA = grep( !/^$/, @configA ); +@configA = grep( !/^\#/, @configA ); +@configA = grep( !/^[\s\t]*$/, @configA ); +my $configA_int = 0; +while ( defined( $configA[$configA_int] ) ) { + my $line = $configA[$configA_int]; + chomp($line); + $line =~ s/^[\t\s]+//; + $line =~ s/[\t\s]+$//; + + my ( $var, $val ) = split( /=/, $line, 2 ); + + my $matched; + if ( $var eq 'cache' ) { + $cache = $val; + $matched = 1; + } + + if ( $var eq 'smartctl' ) { + $smartctl = $val; + $matched = 1; + } + + if ( $var eq 'useSN' ) { + $useSN = $val; + $matched = 1; + } + + if ( !defined($val) ) { + push( @disks, $line ); + } + + $configA_int++; +} ## end while ( defined( $configA[$configA_int] ) ) + +# +# +# run the specified self test on all disks if asked +# +# +if ( defined( $opts{t} ) ) { + + # make sure we have something that atleast appears sane for the test name + my $valid_tesks = { + 'offline' => 1, + 'short' => 1, + 'long' => 1, + 'conveyance' => 1, + 'afterselect,on' => 1, + }; + if ( !defined( $valid_tesks->{ $opts{t} } ) && $opts{t} !~ /select,(\d+[\-\+]\d+|next|next\+\d+|redo\+\d+)/ ) { + print '"' . $opts{t} . "\" does not appear to be a valid test\n"; + exit 1; + } + + print "Running the SMART $opts{t} on all devices in the config...\n\n"; + + foreach my $line (@disks) { + my $disk; + my $name; + if ( $line =~ /\ / ) { + ( $name, $disk ) = split( /\ /, $line, 2 ); + } else { + $disk = $line; + $name = $line; + } + if ( $disk !~ /\// ) { + $disk = '/dev/' . $disk; + } + + print "\n------------------------------------------------------------------\nDoing " + . $smartctl . ' -t ' + . $opts{t} . ' ' + . $disk + . " ...\n\n"; + print `$smartctl -t $opts{t} $disk` . "\n"; + + } ## end foreach my $line (@disks) + + exit 0; +} ## end if ( defined( $opts{t} ) ) + +#if set to 1, no cache will be written and it will be printed instead +my $noWrite = 0; + +# +# +# if no -u, it means we are being called from snmped +# +# +if ( !defined( $opts{u} ) ) { + # if the cache file exists, print it, otherwise assume one is not being used + if ( -f $cache ) { + my $old = ''; + open( my $readfh, "<", $cache ) or die "Can't open '" . $cache . "'"; + read( $readfh, $old, 1000000 ); + close($readfh); + print $old; + exit 0; + } else { + $opts{u} = 1; + $noWrite = 1; + } +} ## end if ( !defined( $opts{u} ) ) + +# +# +# Process each disk +# +# +my $to_return = { + data => { disks => {}, exit_nonzero => 0, unhealthy => 0, useSN => $useSN }, + version => 1, + error => 0, + errorString => '', +}; +foreach my $line (@disks) { + my $disk; + my $name; + if ( $line =~ /\ / ) { + ( $name, $disk ) = split( /\ /, $line, 2 ); + } else { + $disk = $line; + $name = $line; + } + if ( $disk !~ /\// ) { + $disk = '/dev/' . $disk; + } + + my $output = `$smartctl -A $disk`; + my %IDs = ( + '5' => 'null', + '10' => 'null', + '173' => 'null', + '177' => 'null', + '183' => 'null', + '184' => 'null', + '187' => 'null', + '188' => 'null', + '190' => 'null', + '194' => 'null', + '196' => 'null', + '197' => 'null', + '198' => 'null', + '199' => 'null', + '231' => 'null', + '232' => 'null', + '233' => 'null', + '9' => 'null', + 'disk' => $disk, + 'serial' => undef, + 'selftest_log' => undef, + 'health_pass' => 0, + max_temp => 'null', + exit => $?, + ); + $IDs{'disk'} =~ s/^\/dev\///; + + # if polling exited non-zero above, no reason running the rest of the checks + my $disk_id = $name; + if ( $IDs{exit} != 0 ) { + $to_return->{data}{exit_nonzero}++; + } else { + my @outputA; + + if ( $output =~ /NVMe Log/ ) { + # we have an NVMe drive with annoyingly different output + my %mappings = ( + 'Temperature' => 194, + 'Power Cycles' => 12, + 'Power On Hours' => 9, + 'Percentage Used' => 231, + ); + foreach ( split( /\n/, $output ) ) { + if (/:/) { + my ( $key, $val ) = split(/:/); + $val =~ s/^\s+|\s+$|\D+//g; + if ( exists( $mappings{$key} ) ) { + if ( $mappings{$key} == 231 ) { + $IDs{ $mappings{$key} } = 100 - $val; + } else { + $IDs{ $mappings{$key} } = $val; + } + } + } ## end if (/:/) + } ## end foreach ( split( /\n/, $output ) ) + + } else { + @outputA = split( /\n/, $output ); + my $outputAint = 0; + while ( defined( $outputA[$outputAint] ) ) { + my $line = $outputA[$outputAint]; + $line =~ s/^ +//; + $line =~ s/ +/ /g; + + if ( $line =~ /^[0123456789]+ / ) { + my @lineA = split( /\ /, $line, 10 ); + my $raw = $lineA[9]; + my $normalized = $lineA[3]; + my $id = $lineA[0]; + + # Crucial SSD + # 202, Percent_Lifetime_Remain, same as 231, SSD Life Left + if ( $id == 202 + && $line =~ /Percent_Lifetime_Remain/ ) + { + $IDs{231} = $raw; + } + + # single int raw values + if ( ( $id == 5 ) + || ( $id == 10 ) + || ( $id == 173 ) + || ( $id == 183 ) + || ( $id == 184 ) + || ( $id == 187 ) + || ( $id == 196 ) + || ( $id == 197 ) + || ( $id == 198 ) + || ( $id == 199 ) ) + { + my @rawA = split( /\ /, $raw ); + $IDs{$id} = $rawA[0]; + } ## end if ( ( $id == 5 ) || ( $id == 10 ) || ( $id...)) + + # single int normalized values + if ( ( $id == 177 ) + || ( $id == 230 ) + || ( $id == 231 ) + || ( $id == 232 ) + || ( $id == 233 ) ) + { + # annoying non-standard disk + # WDC WDS500G2B0A + # 230 Media_Wearout_Indicator 0x0032 100 100 --- Old_age Always - 0x002e000a002e + # 232 Available_Reservd_Space 0x0033 100 100 004 Pre-fail Always - 100 + # 233 NAND_GB_Written_TLC 0x0032 100 100 --- Old_age Always - 9816 + + if ( $id == 230 + && $line =~ /Media_Wearout_Indicator/ ) + { + $IDs{233} = int($normalized); + } elsif ( $id == 232 + && $line =~ /Available_Reservd_Space/ ) + { + $IDs{232} = int($normalized); + } else { + # only set 233 if it has not been set yet + # if it was set already then the above did it and we don't want + # to overwrite it + if ( $id == 233 && $IDs{233} eq "null" ) { + $IDs{$id} = int($normalized); + } elsif ( $id != 233 ) { + $IDs{$id} = int($normalized); + } + } ## end else [ if ( $id == 230 && $line =~ /Media_Wearout_Indicator/)] + } ## end if ( ( $id == 177 ) || ( $id == 230 ) || (...)) + + # 9, power on hours + if ( $id == 9 ) { + my @runtime = split( /[\ h]/, $raw ); + $IDs{$id} = $runtime[0]; + } + + # 188, Command_Timeout + if ( $id == 188 ) { + my $total = 0; + my @rawA = split( /\ /, $raw ); + my $rawAint = 0; + while ( defined( $rawA[$rawAint] ) ) { + $total = $total + $rawA[$rawAint]; + $rawAint++; + } + $IDs{$id} = $total; + } ## end if ( $id == 188 ) + + # 190, airflow temp + # 194, temp + if ( ( $id == 190 ) + || ( $id == 194 ) ) + { + my ($temp) = split( /\ /, $raw ); + $IDs{$id} = $temp; + } + } ## end if ( $line =~ /^[0123456789]+ / ) + + # SAS Wrapping + # Section by Cameron Munroe (munroenet[at]gmail.com) + + # Elements in Grown Defect List. + # Marking as 5 Reallocated_Sector_Ct + if ( $line =~ "Elements in grown defect list:" ) { + + my @lineA = split( /\ /, $line, 10 ); + my $raw = $lineA[5]; + + # Reallocated Sector Count ID + $IDs{5} = $raw; + + } + + # Current Drive Temperature + # Marking as 194 Temperature_Celsius + if ( $line =~ "Current Drive Temperature:" ) { + + my @lineA = split( /\ /, $line, 10 ); + my $raw = $lineA[3]; + + # Temperature C ID + $IDs{194} = $raw; + + } + + # End of SAS Wrapper + + $outputAint++; + } ## end while ( defined( $outputA[$outputAint] ) ) + } ## end else [ if ( $output =~ /NVMe Log/ ) ] + + #get the selftest logs + $output = `$smartctl -l selftest $disk`; + @outputA = split( /\n/, $output ); + my @completed = grep( /Completed/, @outputA ); + $IDs{'completed'} = scalar @completed; + my @interrupted = grep( /Interrupted/, @outputA ); + $IDs{'interrupted'} = scalar @interrupted; + my @read_failure = grep( /read failure/, @outputA ); + $IDs{'read_failure'} = scalar @read_failure; + my @read_failure2 = grep( /Failed in segment/, @outputA ); + $IDs{'read_failure'} = $IDs{'read_failure'} + scalar @read_failure2; + my @unknown_failure = grep( /unknown failure/, @outputA ); + $IDs{'unknown_failure'} = scalar @unknown_failure; + my @extended = grep( /\d.*\ ([Ee]xtended|[Ll]ong).*(?![Dd]uration)/, @outputA ); + $IDs{'extended'} = scalar @extended; + my @short = grep( /[Ss]hort/, @outputA ); + $IDs{'short'} = scalar @short; + my @conveyance = grep( /[Cc]onveyance/, @outputA ); + $IDs{'conveyance'} = scalar @conveyance; + my @selective = grep( /[Ss]elective/, @outputA ); + $IDs{'selective'} = scalar @selective; + my @offline = grep( /(\d|[Bb]ackground|[Ff]oreground)+\ +[Oo]ffline/, @outputA ); + $IDs{'offline'} = scalar @offline; + + # if we have logs, actually grab the log output + if ( $IDs{'completed'} > 0 + || $IDs{'interrupted'} > 0 + || $IDs{'read_failure'} > 0 + || $IDs{'extended'} > 0 + || $IDs{'short'} > 0 + || $IDs{'conveyance'} > 0 + || $IDs{'selective'} > 0 + || $IDs{'offline'} > 0 ) + { + my @headers = grep( /(Num\ +Test.*LBA| Description .*[Hh]ours)/, @outputA ); + + my @log_lines; + push( @log_lines, @extended, @short, @conveyance, @selective, @offline ); + $IDs{'selftest_log'} = join( "\n", @headers, sort(@log_lines) ); + } ## end if ( $IDs{'completed'} > 0 || $IDs{'interrupted'...}) + + # get the drive serial number, if needed + $disk_id = $name; + $output = `$smartctl -i $disk`; + # generally upper case, HP branded drives seem to report with lower case n + while ( $output =~ /(?i)Serial Number:(.*)/g ) { + $IDs{'serial'} = $1; + $IDs{'serial'} =~ s/^\s+|\s+$//g; + } + if ($useSN) { + $disk_id = $IDs{'serial'}; + } + + while ( $output =~ /(?i)Model Family:(.*)/g ) { + $IDs{'model_family'} = $1; + $IDs{'model_family'} =~ s/^\s+|\s+$//g; + } + + while ( $output =~ /(?i)Device Model:(.*)/g ) { + $IDs{'device_model'} = $1; + $IDs{'device_model'} =~ s/^\s+|\s+$//g; + } + + while ( $output =~ /(?i)Model Number:(.*)/g ) { + $IDs{'model_number'} = $1; + $IDs{'model_number'} =~ s/^\s+|\s+$//g; + } + + while ( $output =~ /(?i)Firmware Version:(.*)/g ) { + $IDs{'fw_version'} = $1; + $IDs{'fw_version'} =~ s/^\s+|\s+$//g; + } + + # mainly HP drives + while ( $output =~ /(?i)Vendor:(.*)/g ) { + $IDs{'vendor'} = $1; + $IDs{'vendor'} =~ s/^\s+|\s+$//g; + } + + # mainly HP drives + while ( $output =~ /(?i)Product:(.*)/g ) { + $IDs{'product'} = $1; + $IDs{'product'} =~ s/^\s+|\s+$//g; + } + + # mainly HP drives + while ( $output =~ /(?i)Revision:(.*)/g ) { + $IDs{'revision'} = $1; + $IDs{'revision'} =~ s/^\s+|\s+$//g; + } + + # figure out what to use for the max temp, if there is one + if ( $IDs{'190'} =~ /^\d+$/ ) { + $IDs{max_temp} = $IDs{'190'}; + } elsif ( $IDs{'194'} =~ /^\d+$/ ) { + $IDs{max_temp} = $IDs{'194'}; + } + if ( $IDs{'194'} =~ /^\d+$/ && defined( $IDs{max_temp} ) && $IDs{'194'} > $IDs{max_temp} ) { + $IDs{max_temp} = $IDs{'194'}; + } + + $output = `$smartctl -H $disk`; + if ( $output =~ /SMART\ overall\-health\ self\-assessment\ test\ result\:\ PASSED/ ) { + $IDs{'health_pass'} = 1; + } elsif ( $output =~ /SMART\ Health\ Status\:\ OK/ ) { + $IDs{'health_pass'} = 1; + } + + if ( !$IDs{'health_pass'} ) { + $to_return->{data}{unhealthy}++; + } + } ## end else [ if ( $IDs{exit} != 0 ) ] + + # only bother to save this if useSN is not being used + if ( !$useSN ) { + $to_return->{data}{disks}{$disk_id} = \%IDs; + } elsif ( $IDs{exit} == 0 && defined($disk_id) ) { + $to_return->{data}{disks}{$disk_id} = \%IDs; + } + + # smartctl will in some cases exit zero when it can't pull data for cciss + # so if we get a zero exit, but no serial then it means something errored + # and the device is likely dead + if ( $IDs{exit} == 0 && !defined( $IDs{serial} ) ) { + $to_return->{data}{unhealthy}++; + } +} ## end foreach my $line (@disks) + +my $toReturn = $json->encode($to_return); + +if ( !$opts{p} ) { + $toReturn = $toReturn . "\n"; +} + +if ( $opts{Z} ) { + my $toReturnCompressed; + gzip \$toReturn => \$toReturnCompressed; + my $compressed = encode_base64($toReturnCompressed); + $compressed =~ s/\n//g; + $compressed = $compressed . "\n"; + if ( length($compressed) < length($toReturn) ) { + $toReturn = $compressed; + } +} ## end if ( $opts{Z} ) + +if ( !$noWrite ) { + open( my $writefh, ">", $cache ) or die "Can't open '" . $cache . "'"; + print $writefh $toReturn; + close($writefh); +} else { + print $toReturn; +} diff --git a/archive/provisioning/Agents/librenms/smart.config b/archive/provisioning/Agents/librenms/smart.config new file mode 100644 index 0000000..2b12988 --- /dev/null +++ b/archive/provisioning/Agents/librenms/smart.config @@ -0,0 +1,3 @@ +smartctl=/usr/sbin/smartctl +cache=/var/cache/smart +sda diff --git a/archive/provisioning/Agents/librenms/ss.py b/archive/provisioning/Agents/librenms/ss.py new file mode 100644 index 0000000..0bb9063 --- /dev/null +++ b/archive/provisioning/Agents/librenms/ss.py @@ -0,0 +1,2048 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + librenms-agent/snmp/ss.py at master · librenms/librenms-agent · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ Skip to content + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + +
+ + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + +
+ + + +
+
+ +
+ +
+

Footer

+ + + + +
+
+ + + + + © 2025 GitHub, Inc. + +
+ + +
+
+ + + + + + + + + + + + + + + + + + + +
+
+
+ + + diff --git a/archive/provisioning/Agents/librenms/ups-nut.sh b/archive/provisioning/Agents/librenms/ups-nut.sh new file mode 100644 index 0000000..b75580a --- /dev/null +++ b/archive/provisioning/Agents/librenms/ups-nut.sh @@ -0,0 +1,45 @@ +#!/bin/sh +################################################################ +# Instructions: # +# 1. copy this script to /etc/snmp/ and make it executable: # +# chmod +x ups-nut.sh # +# 2. make sure UPS_NAME below matches the name of your UPS # +# 3. edit your snmpd.conf to include this line: # +# extend ups-nut /etc/snmp/ups-nut.sh # +# 4. restart snmpd on the host # +# 5. activate the app for the desired host in LibreNMS # +################################################################ +UPS_NAME="${1:-APCUPS}" + +PATH=$PATH:/usr/bin:/bin +TMP=$(upsc $UPS_NAME 2>/dev/null) + +for value in "battery\.charge: [0-9.]+" "battery\.(runtime\.)?low: [0-9]+" "battery\.runtime: [0-9]+" "battery\.voltage: [0-9.]+" "battery\.voltage\.nominal: [0-9]+" "input\.voltage\.nominal: [0-9.]+" "input\.voltage: [0-9.]+" "ups\.load: [0-9.]+" +do + OUT=$(echo "$TMP" | grep -Eo "$value" | awk '{print $2}' | LANG=C sort | head -n 1) + if [ -n "$OUT" ]; then + echo "$OUT" + else + echo "Unknown" + fi +done + +for value in "ups\.status:[A-Z ]{0,}OL" "ups\.status:[A-Z ]{0,}OB" "ups\.status:[A-Z ]{0,}LB" "ups\.status:[A-Z ]{0,}HB" "ups\.status:[A-Z ]{0,}RB" "ups\.status:[A-Z ]{0,}CHRG" "ups\.status:[A-Z ]{0,}DISCHRG" "ups\.status:[A-Z ]{0,}BYPASS" "ups\.status:[A-Z ]{0,}CAL" "ups\.status:[A-Z ]{0,}OFF" "ups\.status:[A-Z ]{0,}OVER" "ups\.status:[A-Z ]{0,}TRIM" "ups\.status:[A-Z ]{0,}BOOST" "ups\.status:[A-Z ]{0,}FSD" "ups\.alarm:[A-Z ]" +do + UNKNOWN=$(echo "$TMP" | grep -Eo "ups\.status:") + if [ -z "$UNKNOWN" ]; then + echo "Unknown" + else + OUT=$(echo "$TMP" | grep -Eo "$value") + if [ -n "$OUT" ]; then + echo "1" + else + echo "0" + fi + fi +done + +UPSTEMP="ups\.temperature: [0-9.]+" +OUT=$(echo "$TMP" | grep -Eo "$UPSTEMP" | awk '{print $2}' | LANG=C sort | head -n 1) +[ -n "$OUT" ] && echo "$OUT" || echo "Unknown" + diff --git a/archive/provisioning/ConfigFiles/99-pfv-nfs.conf b/archive/provisioning/ConfigFiles/99-pfv-nfs.conf new file mode 100644 index 0000000..e3ee0b8 --- /dev/null +++ b/archive/provisioning/ConfigFiles/99-pfv-nfs.conf @@ -0,0 +1,23 @@ +# PFV NFS tuning sysctl overrides +# +# Applied AFTER tuned via pfv-nfs-tuning.service (systemd oneshot). +# These override tuned's network-throughput/virtual-host 16MB TCP buffer +# caps with 128MB for high-BDP NFS over 1-4 GbE LACP links. +# +# Install on ALL Proxmox hosts: +# cp 99-pfv-nfs.conf /etc/sysctl.d/99-pfv-nfs.conf +# cp pfv-nfs-tuning.service /etc/systemd/system/pfv-nfs-tuning.service +# systemctl daemon-reload && systemctl enable --now pfv-nfs-tuning.service +# +# Created: 2026-07-31 +# Deployed: tsys1, tsys3, tsys4, tsys5, tsys6, tsys7, tsys9 + +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 +net.core.rmem_default = 26214400 +net.core.wmem_default = 26214400 +net.core.netdev_max_backlog = 250000 +net.core.somaxconn = 65535 +net.ipv4.tcp_rmem = 4096 87380 134217728 +net.ipv4.tcp_wmem = 4096 65536 134217728 +net.ipv4.tcp_max_syn_backlog = 4096 diff --git a/archive/provisioning/ConfigFiles/AuditD/auditd.conf b/archive/provisioning/ConfigFiles/AuditD/auditd.conf new file mode 100644 index 0000000..e1482e3 --- /dev/null +++ b/archive/provisioning/ConfigFiles/AuditD/auditd.conf @@ -0,0 +1,46 @@ +# +# Known Element Enterprises Customized Config File +# auditd +# Initial version 2025-06-27 +# + +local_events = yes +write_logs = yes +log_file = /var/log/audit/audit.log +log_group = adm +log_format = ENRICHED +flush = INCREMENTAL_ASYNC +freq = 50 +max_log_file = 8 +num_logs = 5 +priority_boost = 4 +name_format = NONE +max_log_file_action = keep_logs +space_left = 75 +space_left_action = email +action_mail_acct = root + +admin_space_left_action = halt +disk_full_action = SUSPEND +disk_error_action = SUSPEND +admin_space_left = 50 + +verify_email = yes +use_libwrap = yes +tcp_listen_queue = 5 +tcp_max_per_addr = 1 +tcp_client_max_idle = 0 +transport = TCP +distribute_network = no +q_depth = 2000 +overflow_action = SYSLOG +max_restarts = 10 +plugin_dir = /etc/audit/plugins.d +end_of_event_timeout = 2 +##tcp_client_ports = 1024-65535 +##tcp_listen_port = 60 + +##krb5_key_file = /etc/audit/audit.key +krb5_principal = auditd + +##name = mydomain diff --git a/archive/provisioning/ConfigFiles/AuditD/rules.d/time-change.rules b/archive/provisioning/ConfigFiles/AuditD/rules.d/time-change.rules new file mode 100644 index 0000000..e69de29 diff --git a/archive/provisioning/ConfigFiles/BANNERS/issue b/archive/provisioning/ConfigFiles/BANNERS/issue new file mode 100644 index 0000000..42b3729 --- /dev/null +++ b/archive/provisioning/ConfigFiles/BANNERS/issue @@ -0,0 +1,5 @@ +This system is the property of Known Element Enterprises LLC. + +Authorized uses only. All activity may be monitored and reported. + +All activities subject to monitoring/recording/review in real time and/or at a later time. diff --git a/archive/provisioning/ConfigFiles/BANNERS/issue.net b/archive/provisioning/ConfigFiles/BANNERS/issue.net new file mode 100644 index 0000000..42b3729 --- /dev/null +++ b/archive/provisioning/ConfigFiles/BANNERS/issue.net @@ -0,0 +1,5 @@ +This system is the property of Known Element Enterprises LLC. + +Authorized uses only. All activity may be monitored and reported. + +All activities subject to monitoring/recording/review in real time and/or at a later time. diff --git a/archive/provisioning/ConfigFiles/BANNERS/motd b/archive/provisioning/ConfigFiles/BANNERS/motd new file mode 100644 index 0000000..42b3729 --- /dev/null +++ b/archive/provisioning/ConfigFiles/BANNERS/motd @@ -0,0 +1,5 @@ +This system is the property of Known Element Enterprises LLC. + +Authorized uses only. All activity may be monitored and reported. + +All activities subject to monitoring/recording/review in real time and/or at a later time. diff --git a/archive/provisioning/ConfigFiles/Cockpit/disallowed-users b/archive/provisioning/ConfigFiles/Cockpit/disallowed-users new file mode 100644 index 0000000..7c07810 --- /dev/null +++ b/archive/provisioning/ConfigFiles/Cockpit/disallowed-users @@ -0,0 +1,2 @@ +#/etc/cockpit/disallowed-users +# List of users which are not allowed to login to Cockpit diff --git a/archive/provisioning/ConfigFiles/DHCP/dhclient.conf b/archive/provisioning/ConfigFiles/DHCP/dhclient.conf new file mode 100644 index 0000000..ceee168 --- /dev/null +++ b/archive/provisioning/ConfigFiles/DHCP/dhclient.conf @@ -0,0 +1,14 @@ +option rfc3442-classless-static-routes code 121 = array of unsigned integer 8; + +send host-name = gethostname(); +request subnet-mask, broadcast-address, time-offset, routers, + domain-name, host-name, + domain-name-servers, domain-search, ntp-servers, + rfc3442-classless-static-routes; + +# Pin DNS and NTP to the redundant pfv-netinfra-01/02 pair regardless of what +# the DHCP server advertises, so every host on this build uses the same +# authoritative recursive resolvers and time sources. +supersede domain-name-servers 192.168.3.252, 192.168.3.253; +supersede domain-search "knel.net"; +supersede ntp-servers 192.168.3.252, 192.168.3.253; diff --git a/archive/provisioning/ConfigFiles/Logrotate/logrotate.conf b/archive/provisioning/ConfigFiles/Logrotate/logrotate.conf new file mode 100644 index 0000000..9f2f271 --- /dev/null +++ b/archive/provisioning/ConfigFiles/Logrotate/logrotate.conf @@ -0,0 +1,23 @@ +# see "man logrotate" for details + +# global options do not affect preceding include directives + +# rotate log files weekly +weekly + +# keep 4 weeks worth of backlogs +rotate 4 + +# create new (empty) log files after rotating old ones +create 0640 root utmp + +# use date as a suffix of the rotated file +#dateext + +# uncomment this if you want your log files compressed +#compress + +# packages drop log rotation information into this directory +include /etc/logrotate.d + +# system-specific logs may also be configured here. diff --git a/archive/provisioning/ConfigFiles/ModProbe/cramfs.conf b/archive/provisioning/ConfigFiles/ModProbe/cramfs.conf new file mode 100644 index 0000000..b77c93a --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/cramfs.conf @@ -0,0 +1 @@ +install cramfs /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/dccp.conf b/archive/provisioning/ConfigFiles/ModProbe/dccp.conf new file mode 100644 index 0000000..d453550 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/dccp.conf @@ -0,0 +1 @@ +install dccp /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/freevxfs.conf b/archive/provisioning/ConfigFiles/ModProbe/freevxfs.conf new file mode 100644 index 0000000..72d4aec --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/freevxfs.conf @@ -0,0 +1 @@ +install freevxfs /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/hfs.conf b/archive/provisioning/ConfigFiles/ModProbe/hfs.conf new file mode 100644 index 0000000..a991f49 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/hfs.conf @@ -0,0 +1 @@ +install hfs /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/hfsplus.conf b/archive/provisioning/ConfigFiles/ModProbe/hfsplus.conf new file mode 100644 index 0000000..9cba83f --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/hfsplus.conf @@ -0,0 +1 @@ +install hfsplus /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/jffs2.conf b/archive/provisioning/ConfigFiles/ModProbe/jffs2.conf new file mode 100644 index 0000000..63360f3 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/jffs2.conf @@ -0,0 +1 @@ +install jffs2 /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/rds.conf b/archive/provisioning/ConfigFiles/ModProbe/rds.conf new file mode 100644 index 0000000..650abee --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/rds.conf @@ -0,0 +1 @@ +install rds /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/sctp.conf b/archive/provisioning/ConfigFiles/ModProbe/sctp.conf new file mode 100644 index 0000000..960a200 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/sctp.conf @@ -0,0 +1 @@ +install sctp /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/squashfs.conf b/archive/provisioning/ConfigFiles/ModProbe/squashfs.conf new file mode 100644 index 0000000..c177037 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/squashfs.conf @@ -0,0 +1 @@ +install squashfs /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/tipc.conf b/archive/provisioning/ConfigFiles/ModProbe/tipc.conf new file mode 100644 index 0000000..260e2ad --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/tipc.conf @@ -0,0 +1 @@ +install tipc /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/udf.conf b/archive/provisioning/ConfigFiles/ModProbe/udf.conf new file mode 100644 index 0000000..4894688 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/udf.conf @@ -0,0 +1 @@ +install udf /bin/true diff --git a/archive/provisioning/ConfigFiles/ModProbe/usb_storage.conf b/archive/provisioning/ConfigFiles/ModProbe/usb_storage.conf new file mode 100644 index 0000000..38a1e49 --- /dev/null +++ b/archive/provisioning/ConfigFiles/ModProbe/usb_storage.conf @@ -0,0 +1 @@ +install usb-storage /bin/true diff --git a/archive/provisioning/ConfigFiles/NTP/ntp.conf b/archive/provisioning/ConfigFiles/NTP/ntp.conf new file mode 100644 index 0000000..717fcb1 --- /dev/null +++ b/archive/provisioning/ConfigFiles/NTP/ntp.conf @@ -0,0 +1,21 @@ +driftfile /var/lib/ntp/ntp.drift +leapfile /usr/share/zoneinfo/leap-seconds.list + +# Redundant upstream time sources: pfv-netinfra-01/02 (Technitium/Pi-hole hosts +# also serving NTP). IPs are used (not hostnames) because the knel.net name for +# these hosts resolves to a Tailscale CGNAT address, not the LAN address, and +# because NTP must come up before DNS is available. iburst speeds initial sync. +server 192.168.3.252 iburst +server 192.168.3.253 iburst + +# Hardened client: sync from the configured servers but never serve time to +# anyone else. Note: `interface listen 127.0.0.1` must NOT be used here — it +# binds ntpd to loopback, making outbound queries carry a 127.0.0.1 source +# address that upstream servers cannot reply to (symptoms: peers stuck in +# .INIT. with reach 0). Use restrict rules to control access instead. +restrict default ignore +restrict 127.0.0.1 +restrict ::1 +restrict 192.168.3.252 nomodify notrap nopeer +restrict 192.168.3.253 nomodify notrap nopeer + diff --git a/archive/provisioning/ConfigFiles/NetworkDiscovery/lldpd b/archive/provisioning/ConfigFiles/NetworkDiscovery/lldpd new file mode 100644 index 0000000..b98df83 --- /dev/null +++ b/archive/provisioning/ConfigFiles/NetworkDiscovery/lldpd @@ -0,0 +1,2 @@ +# Uncomment to start SNMP subagent and enable CDP, SONMP and EDP protocol +DAEMON_ARGS="-x -c -s -e" diff --git a/archive/provisioning/ConfigFiles/Resolv/resolv.conf b/archive/provisioning/ConfigFiles/Resolv/resolv.conf new file mode 100644 index 0000000..8728c3f --- /dev/null +++ b/archive/provisioning/ConfigFiles/Resolv/resolv.conf @@ -0,0 +1,11 @@ +# Managed by KNELServerBuild — do not edit; changes will be overwritten. +# +# Redundant recursive DNS via pfv-netinfra-01/02 (Technitium + Pi-hole). +# IPs are used (required: nameserver directives must be addresses, and the +# knel.net name for these hosts resolves to a Tailscale CGNAT address rather +# than the LAN address). If the primary is unreachable, glibc's resolver +# automatically falls through to the secondary. +domain knel.net +search knel.net +nameserver 192.168.3.252 +nameserver 192.168.3.253 diff --git a/archive/provisioning/ConfigFiles/SMTP/aliases b/archive/provisioning/ConfigFiles/SMTP/aliases new file mode 100644 index 0000000..799fcb9 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SMTP/aliases @@ -0,0 +1,3 @@ +# See man 5 aliases for format +postmaster: root +root: coo@turnsys.com diff --git a/archive/provisioning/ConfigFiles/SMTP/postfix_generic b/archive/provisioning/ConfigFiles/SMTP/postfix_generic new file mode 100644 index 0000000..996fa4f --- /dev/null +++ b/archive/provisioning/ConfigFiles/SMTP/postfix_generic @@ -0,0 +1 @@ +/.*/ tsysrootaccount@knel.net diff --git a/archive/provisioning/ConfigFiles/SNMP/snmp-sudo.conf b/archive/provisioning/ConfigFiles/SNMP/snmp-sudo.conf new file mode 100644 index 0000000..3ce5fd3 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SNMP/snmp-sudo.conf @@ -0,0 +1 @@ +Debian-snmp ALL = NOPASSWD: /bin/cat diff --git a/archive/provisioning/ConfigFiles/SNMP/snmpd-physicalhost.conf b/archive/provisioning/ConfigFiles/SNMP/snmpd-physicalhost.conf new file mode 100644 index 0000000..b8cb71f --- /dev/null +++ b/archive/provisioning/ConfigFiles/SNMP/snmpd-physicalhost.conf @@ -0,0 +1,46 @@ +########################################################################## +# snmpd.conf +# Created by CNW on 11/3/2018 via snmpconf wizard and manual post tweaks +########################################################################### +# SECTION: Monitor Various Aspects of the Running Host +# + +# disk: Check for disk space usage of a partition. +# The agent can check the amount of available disk space, and make +# sure it is above a set limit. +# +load 3 3 3 +rocommunity kn3lmgmt +sysservices 76 + +#syslocation Rack, Room, Building, City, Country [Lat, Lon] +syslocation R4, Server Room, SITER, Pflugerville, United States +syscontact coo@turnsys.com + +#NTP +extend ntp-client /usr/lib/check_mk_agent/local/ntp-client + +#SMTP +extend mailq /usr/lib/check_mk_agent/local/postfix-queues +extend postfixdetailed /usr/lib/check_mk_agent/local/postfixdetailed + +#OS Distribution Detection +extend distro /usr/local/bin/distro +extend osupdate /usr/lib/check_mk_agent/local/os-updates.sh + +#Hardware Detection +extend manufacturer /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/sys_vendor +extend hardware /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/product_name +extend serial /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/product_serial + +#SMART +extend smart /usr/lib/check_mk_agent/local/smart + +#Temperature +pass_persist .1.3.6.1.4.1.9.9.13.1.3 /usr/local/bin/temper-snmp + +# Allow Systems Management Data Engine SNMP to connect to snmpd using SMUX +# smuxpeer .1.3.6.1.4.1.674.10892.1 + +# LLDP collection +master agentx diff --git a/archive/provisioning/ConfigFiles/SNMP/snmpd-rpi.conf b/archive/provisioning/ConfigFiles/SNMP/snmpd-rpi.conf new file mode 100644 index 0000000..4e98bf8 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SNMP/snmpd-rpi.conf @@ -0,0 +1,40 @@ +########################################################################## +# snmpd.conf +# Created by CNW on 11/3/2018 via snmpconf wizard and manual post tweaks +########################################################################### +# SECTION: Monitor Various Aspects of the Running Host +# + +# disk: Check for disk space usage of a partition. +# The agent can check the amount of available disk space, and make +# sure it is above a set limit. +# +load 3 3 3 +rocommunity kn3lmgmt +sysservices 76 + +#syslocation Rack, Room, Building, City, Country [Lat, Lon] +syslocation SITER, Pflugerville, United States +syscontact coo@turnsys.com + +#NTP +extend ntp-client /usr/lib/check_mk_agent/local/ntp-client + +#SMTP +extend mailq /usr/lib/check_mk_agent/local/postfix-queues +extend postfixdetailed /usr/lib/check_mk_agent/local/postfixdetailed + +#OS Distribution Detection +extend distro /usr/local/bin/distro +extend osupdate /usr/lib/check_mk_agent/local/os-updates.sh + + +#Hardware Detection +extend hardware /usr/bin/sudo /usr/bin/cat /sys/firmware/devicetree/base/model +extend serial /usr/bin/sudo /usr/bin/cat /sys/firmware/devicetree/base/serial-number + +# Allow Systems Management Data Engine SNMP to connect to snmpd using SMUX +# smuxpeer .1.3.6.1.4.1.674.10892.1 + +# LLDP collection +master agentx diff --git a/archive/provisioning/ConfigFiles/SNMP/snmpd.conf b/archive/provisioning/ConfigFiles/SNMP/snmpd.conf new file mode 100644 index 0000000..8ae8ab4 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SNMP/snmpd.conf @@ -0,0 +1,44 @@ +########################################################################## +# snmpd.conf +# Created by CNW on 11/3/2018 via snmpconf wizard and manual post tweaks +########################################################################### +# SECTION: Monitor Various Aspects of the Running Host +# + +# disk: Check for disk space usage of a partition. +# The agent can check the amount of available disk space, and make +# sure it is above a set limit. +# +load 3 3 3 +rocommunity kn3lmgmt +sysservices 76 + +#syslocation Rack, Room, Building, City, Country [Lat, Lon] +syslocation R4, Server Room, SITER, Pflugerville, United States +syscontact coo@turnsys.com + +#NTP +extend ntp-client /usr/lib/check_mk_agent/local/ntp-client + +#SMTP +extend mailq /usr/lib/check_mk_agent/local/postfix-queues +extend postfixdetailed /usr/lib/check_mk_agent/local/postfixdetailed + +#OS Distribution Detection +extend distro /usr/local/bin/distro +extend osupdate /usr/lib/check_mk_agent/local/os-updates.sh + +# Socket statistics +extend ss /usr/lib/check_mk_agent/local/ss.py + +#Hardware Detection +# (uncomment for x86 platforms) +extend manufacturer /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/sys_vendor +extend hardware /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/product_name +extend serial /usr/bin/sudo /usr/bin/cat /sys/devices/virtual/dmi/id/product_serial + +# Allow Systems Management Data Engine SNMP to connect to snmpd using SMUX +# smuxpeer .1.3.6.1.4.1.674.10892.1 + +# LLDP collection +master agentx diff --git a/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/localuser-ssh-authorized-keys b/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/localuser-ssh-authorized-keys new file mode 100644 index 0000000..36216a1 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/localuser-ssh-authorized-keys @@ -0,0 +1,2 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDHaBNuLS+GYGRPc9wne63Ocr+R+/Q01Y9V0FTv0RnG3 +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPyMR0lFgiMKhQJ5aqy68nR0BQp1cNzi/wIThyuTV4a8 tsyscto@ultix-control diff --git a/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/root-ssh-authorized-keys b/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/root-ssh-authorized-keys new file mode 100644 index 0000000..36216a1 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SSH/AuthorizedKeys/root-ssh-authorized-keys @@ -0,0 +1,2 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDHaBNuLS+GYGRPc9wne63Ocr+R+/Q01Y9V0FTv0RnG3 +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPyMR0lFgiMKhQJ5aqy68nR0BQp1cNzi/wIThyuTV4a8 tsyscto@ultix-control diff --git a/archive/provisioning/ConfigFiles/SSH/Configs/ssh-audit-hardening.conf b/archive/provisioning/ConfigFiles/SSH/Configs/ssh-audit-hardening.conf new file mode 100644 index 0000000..8e80118 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SSH/Configs/ssh-audit-hardening.conf @@ -0,0 +1,19 @@ +# Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com +# hardening guide. +KexAlgorithms sntrup761x25519-sha512,sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + +Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-gcm@openssh.com,aes128-ctr + +MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + +HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + +RequiredRSASize 3072 + +CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + +GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + +HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + +PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 diff --git a/archive/provisioning/ConfigFiles/SSH/Configs/tsys-sshd-config b/archive/provisioning/ConfigFiles/SSH/Configs/tsys-sshd-config new file mode 100644 index 0000000..fca1eb0 --- /dev/null +++ b/archive/provisioning/ConfigFiles/SSH/Configs/tsys-sshd-config @@ -0,0 +1,20 @@ +Include /etc/ssh/sshd_config.d/*.conf +HostKey /etc/ssh/ssh_host_rsa_key +HostKey /etc/ssh/ssh_host_ed25519_key +KbdInteractiveAuthentication no +PrintMotd no +PasswordAuthentication no +AllowTcpForwarding no +X11Forwarding no +ChallengeResponseAuthentication no +AcceptEnv LANG LC_* +Subsystem sftp /usr/lib/openssh/sftp-server +UsePAM yes +Banner /etc/issue.net +MaxAuthTries 2 +MaxStartups 10:30:100 +PermitRootLogin prohibit-password +ClientAliveInterval 300 +ClientAliveCountMax 3 +AllowUsers root localuser subodev +LoginGraceTime 60 diff --git a/archive/provisioning/ConfigFiles/Syslog/rsyslog.conf b/archive/provisioning/ConfigFiles/Syslog/rsyslog.conf new file mode 100644 index 0000000..0ee8600 --- /dev/null +++ b/archive/provisioning/ConfigFiles/Syslog/rsyslog.conf @@ -0,0 +1,6 @@ +module(load="imuxsock") # provides support for local system logging +module(load="imklog") # provides kernel logging support +#module(load="immark") # provides --MARK-- message capability + +*.* @tsys-librenms.knel.net:514 +:omusrmsg:EOF diff --git a/archive/provisioning/ConfigFiles/Systemd/journald.conf b/archive/provisioning/ConfigFiles/Systemd/journald.conf new file mode 100644 index 0000000..8b30ab3 --- /dev/null +++ b/archive/provisioning/ConfigFiles/Systemd/journald.conf @@ -0,0 +1,31 @@ +[Journal] +#Compress=yes +#Seal=yes +#SplitMode=uid +#SyncIntervalSec=5m +#RateLimitIntervalSec=30s +#RateLimitBurst=10000 +#SystemMaxUse= +#SystemKeepFree= +#SystemMaxFileSize= +#SystemMaxFiles=100 +#RuntimeMaxUse= +#RuntimeKeepFree= +#RuntimeMaxFileSize= +#RuntimeMaxFiles=100 +#MaxRetentionSec= +#MaxFileSec=1month +#ForwardToSyslog=yes +#ForwardToKMsg=no +#ForwardToConsole=no +#ForwardToWall=yes +#TTYPath=/dev/console +#MaxLevelStore=debug +#MaxLevelSyslog=debug +#MaxLevelKMsg=notice +#MaxLevelConsole=info +#MaxLevelWall=emerg +#LineMax=48K +#ReadKMsg=yes +#Audit=no +Storage=persistent diff --git a/archive/provisioning/ConfigFiles/ZSH/tsys-zshrc b/archive/provisioning/ConfigFiles/ZSH/tsys-zshrc new file mode 100644 index 0000000..66c6e8f --- /dev/null +++ b/archive/provisioning/ConfigFiles/ZSH/tsys-zshrc @@ -0,0 +1,258 @@ +# ~/.zshrc file for zsh interactive shells. +# see /usr/share/doc/zsh/examples/zshrc for examples + +setopt autocd # change directory just by typing its name +#setopt correct # auto correct mistakes +setopt interactivecomments # allow comments in interactive mode +setopt magicequalsubst # enable filename expansion for arguments of the form ‘anything=expression’ +setopt nonomatch # hide error message if there is no match for the pattern +setopt notify # report the status of background jobs immediately +setopt numericglobsort # sort filenames numerically when it makes sense +setopt promptsubst # enable command substitution in prompt + +WORDCHARS=${WORDCHARS//\/} # Don't consider certain characters part of the word + +# hide EOL sign ('%') +PROMPT_EOL_MARK="" + +# configure key keybindings +bindkey -v # emacs key bindings +bindkey ' ' magic-space # do history expansion on space +bindkey '^U' backward-kill-line # ctrl + U +bindkey '^[[3;5~' kill-word # ctrl + Supr +bindkey '^[[3~' delete-char # delete +bindkey '^[[1;5C' forward-word # ctrl + -> +bindkey '^[[1;5D' backward-word # ctrl + <- +bindkey '^[[5~' beginning-of-buffer-or-history # page up +bindkey '^[[6~' end-of-buffer-or-history # page down +bindkey '^[[H' beginning-of-line # home +bindkey '^[[F' end-of-line # end +bindkey '^[[Z' undo # shift + tab undo last action + +# enable completion features +autoload -Uz compinit +compinit -d ~/.cache/zcompdump +zstyle ':completion:*:*:*:*:*' menu select +zstyle ':completion:*' auto-description 'specify: %d' +zstyle ':completion:*' completer _expand _complete +zstyle ':completion:*' format 'Completing %d' +zstyle ':completion:*' group-name '' +zstyle ':completion:*' list-colors '' +zstyle ':completion:*' list-prompt %SAt %p: Hit TAB for more, or the character to insert%s +zstyle ':completion:*' matcher-list 'm:{a-zA-Z}={A-Za-z}' +zstyle ':completion:*' rehash true +zstyle ':completion:*' select-prompt %SScrolling active: current selection at %p%s +zstyle ':completion:*' use-compctl false +zstyle ':completion:*' verbose true +zstyle ':completion:*:kill:*' command 'ps -u $USER -o pid,%cpu,tty,cputime,cmd' + +# History configurations +HISTFILE=~/.zsh_history +HISTSIZE=10000 +SAVEHIST=200000 +setopt hist_expire_dups_first # delete duplicates first when HISTFILE size exceeds HISTSIZE +setopt hist_ignore_dups # ignore duplicated commands history list +setopt hist_ignore_space # ignore commands that start with space +setopt hist_verify # show command with history expansion to user before running it +#setopt share_history # share command history data + +# force zsh to show the complete history +alias history="history 0" + +# configure `time` format +TIMEFMT=$'\nreal\t%E\nuser\t%U\nsys\t%S\ncpu\t%P' + +# make less more friendly for non-text input files, see lesspipe(1) +#[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +configure_prompt() { + prompt_symbol=㉿ + # Skull emoji for root terminal + #[ "$EUID" -eq 0 ] && prompt_symbol=💀 + case "$PROMPT_ALTERNATIVE" in + twoline) + PROMPT=$'%F{%(#.blue.green)}┌──${debian_chroot:+($debian_chroot)─}${VIRTUAL_ENV:+($(basename $VIRTUAL_ENV))─}(%B%F{%(#.red.blue)}%n'$prompt_symbol$'%m%b%F{%(#.blue.green)})-[%B%F{reset}%(6~.%-1~/…/%4~.%5~)%b%F{%(#.blue.green)}]\n└─%B%(#.%F{red}#.%F{blue}$)%b%F{reset} ' + # Right-side prompt with exit codes and background processes + #RPROMPT=$'%(?.. %? %F{red}%B⨯%b%F{reset})%(1j. %j %F{yellow}%B⚙%b%F{reset}.)' + ;; + oneline) + PROMPT=$'${debian_chroot:+($debian_chroot)}${VIRTUAL_ENV:+($(basename $VIRTUAL_ENV))}%B%F{%(#.red.blue)}%n@%m%b%F{reset}:%B%F{%(#.blue.green)}%~%b%F{reset}%(#.#.$) ' + RPROMPT= + ;; + backtrack) + PROMPT=$'${debian_chroot:+($debian_chroot)}${VIRTUAL_ENV:+($(basename $VIRTUAL_ENV))}%B%F{red}%n@%m%b%F{reset}:%B%F{blue}%~%b%F{reset}%(#.#.$) ' + RPROMPT= + ;; + esac + unset prompt_symbol +} + +# The following block is surrounded by two delimiters. +# These delimiters must not be modified. Thanks. +# START KALI CONFIG VARIABLES +PROMPT_ALTERNATIVE=twoline +NEWLINE_BEFORE_PROMPT=yes +# STOP KALI CONFIG VARIABLES + +if [ "$color_prompt" = yes ]; then + # override default virtualenv indicator in prompt + VIRTUAL_ENV_DISABLE_PROMPT=1 + + configure_prompt + + # enable syntax-highlighting + if [ -f /usr/share/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh ]; then + . /usr/share/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh + ZSH_HIGHLIGHT_HIGHLIGHTERS=(main brackets pattern) + ZSH_HIGHLIGHT_STYLES[default]=none + ZSH_HIGHLIGHT_STYLES[unknown-token]=underline + ZSH_HIGHLIGHT_STYLES[reserved-word]=fg=cyan,bold + ZSH_HIGHLIGHT_STYLES[suffix-alias]=fg=green,underline + ZSH_HIGHLIGHT_STYLES[global-alias]=fg=green,bold + ZSH_HIGHLIGHT_STYLES[precommand]=fg=green,underline + ZSH_HIGHLIGHT_STYLES[commandseparator]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[autodirectory]=fg=green,underline + ZSH_HIGHLIGHT_STYLES[path]=bold + ZSH_HIGHLIGHT_STYLES[path_pathseparator]= + ZSH_HIGHLIGHT_STYLES[path_prefix_pathseparator]= + ZSH_HIGHLIGHT_STYLES[globbing]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[history-expansion]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[command-substitution]=none + ZSH_HIGHLIGHT_STYLES[command-substitution-delimiter]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[process-substitution]=none + ZSH_HIGHLIGHT_STYLES[process-substitution-delimiter]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[single-hyphen-option]=fg=green + ZSH_HIGHLIGHT_STYLES[double-hyphen-option]=fg=green + ZSH_HIGHLIGHT_STYLES[back-quoted-argument]=none + ZSH_HIGHLIGHT_STYLES[back-quoted-argument-delimiter]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[single-quoted-argument]=fg=yellow + ZSH_HIGHLIGHT_STYLES[double-quoted-argument]=fg=yellow + ZSH_HIGHLIGHT_STYLES[dollar-quoted-argument]=fg=yellow + ZSH_HIGHLIGHT_STYLES[rc-quote]=fg=magenta + ZSH_HIGHLIGHT_STYLES[dollar-double-quoted-argument]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[back-double-quoted-argument]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[back-dollar-quoted-argument]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[assign]=none + ZSH_HIGHLIGHT_STYLES[redirection]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[comment]=fg=black,bold + ZSH_HIGHLIGHT_STYLES[named-fd]=none + ZSH_HIGHLIGHT_STYLES[numeric-fd]=none + ZSH_HIGHLIGHT_STYLES[arg0]=fg=cyan + ZSH_HIGHLIGHT_STYLES[bracket-error]=fg=red,bold + ZSH_HIGHLIGHT_STYLES[bracket-level-1]=fg=blue,bold + ZSH_HIGHLIGHT_STYLES[bracket-level-2]=fg=green,bold + ZSH_HIGHLIGHT_STYLES[bracket-level-3]=fg=magenta,bold + ZSH_HIGHLIGHT_STYLES[bracket-level-4]=fg=yellow,bold + ZSH_HIGHLIGHT_STYLES[bracket-level-5]=fg=cyan,bold + ZSH_HIGHLIGHT_STYLES[cursor-matchingbracket]=standout + fi +else + PROMPT='${debian_chroot:+($debian_chroot)}%n@%m:%~%(#.#.$) ' +fi +unset color_prompt force_color_prompt + +toggle_oneline_prompt(){ + if [ "$PROMPT_ALTERNATIVE" = oneline ]; then + PROMPT_ALTERNATIVE=twoline + else + PROMPT_ALTERNATIVE=oneline + fi + configure_prompt + zle reset-prompt +} +zle -N toggle_oneline_prompt +bindkey ^P toggle_oneline_prompt + +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*|Eterm|aterm|kterm|gnome*|alacritty) + TERM_TITLE=$'\e]0;${debian_chroot:+($debian_chroot)}${VIRTUAL_ENV:+($(basename $VIRTUAL_ENV))}%n@%m: %~\a' + ;; +*) + ;; +esac + +precmd() { + # Print the previously configured title + print -Pnr -- "$TERM_TITLE" + + # Print a new line before the prompt, but only if it is not the first line + if [ "$NEWLINE_BEFORE_PROMPT" = yes ]; then + if [ -z "$_NEW_LINE_BEFORE_PROMPT" ]; then + _NEW_LINE_BEFORE_PROMPT=1 + else + print "" + fi + fi +} + +# enable color support of ls, less and man, and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + export LS_COLORS="$LS_COLORS:ow=30;44:" # fix ls color for folders with 777 permissions + + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' + alias diff='diff --color=auto' + alias ip='ip --color=auto' + + export LESS_TERMCAP_mb=$'\E[1;31m' # begin blink + export LESS_TERMCAP_md=$'\E[1;36m' # begin bold + export LESS_TERMCAP_me=$'\E[0m' # reset bold/blink + export LESS_TERMCAP_so=$'\E[01;33m' # begin reverse video + export LESS_TERMCAP_se=$'\E[0m' # reset reverse video + export LESS_TERMCAP_us=$'\E[1;32m' # begin underline + export LESS_TERMCAP_ue=$'\E[0m' # reset underline + + # Take advantage of $LS_COLORS for completion as well + zstyle ':completion:*' list-colors "${(s.:.)LS_COLORS}" + zstyle ':completion:*:*:kill:*:processes' list-colors '=(#b) #([0-9]#)*=0=01;31' +fi + +# some more ls aliases +alias ll='ls -l' +alias la='ls -A' +alias l='ls -CF' + +# enable auto-suggestions based on the history +if [ -f /usr/share/zsh-autosuggestions/zsh-autosuggestions.zsh ]; then + . /usr/share/zsh-autosuggestions/zsh-autosuggestions.zsh + # change suggestion color + ZSH_AUTOSUGGEST_HIGHLIGHT_STYLE='fg=#999' +fi + +# enable command-not-found if installed +if [ -f /etc/zsh_command_not_found ]; then + . /etc/zsh_command_not_found +fi diff --git a/archive/provisioning/ConfigFiles/pfv-nfs-tuning.service b/archive/provisioning/ConfigFiles/pfv-nfs-tuning.service new file mode 100644 index 0000000..75c66ae --- /dev/null +++ b/archive/provisioning/ConfigFiles/pfv-nfs-tuning.service @@ -0,0 +1,28 @@ +# PFV NFS tuning service +# +# Systemd oneshot that runs AFTER tuned.service to apply TCP buffer +# overrides. The tuned daemon's profiles (network-throughput for storage +# hosts, virtual-host for compute hosts) set 16MB TCP buffer caps which +# are too small for high-BDP NFS over LACP links. This service force- +# applies 128MB buffers after tuned has finished its configuration. +# +# Install: +# cp pfv-nfs-tuning.service /etc/systemd/system/pfv-nfs-tuning.service +# systemctl daemon-reload +# systemctl enable --now pfv-nfs-tuning.service +# +# Created: 2026-07-31 +# Deployed: all 7 Proxmox hosts (tsys1/3/4/5/6/7/9) + +[Unit] +Description=PFV NFS tuning (override tuned TCP buffer caps) +After=tuned.service +Requires=tuned.service + +[Service] +Type=oneshot +ExecStart=/sbin/sysctl -p /etc/sysctl.d/99-pfv-nfs.conf +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/archive/provisioning/Dell/Server/fixeth.sh b/archive/provisioning/Dell/Server/fixeth.sh new file mode 100644 index 0000000..21e4b94 --- /dev/null +++ b/archive/provisioning/Dell/Server/fixeth.sh @@ -0,0 +1,24 @@ +#!/bin/bash + + + +#magic to detect main int +echo "Determining management interface..." +#export MAIN_INT=$(brctl show $(netstat -rn|grep 0.0.0.0|head -n1|awk '{print $NF}') | awk '{print $NF}'|tail -1|awk -F '.' '{print $1}') +MAIN_INT=$(brctl show|grep vmbr0|awk '{print $NF}'|awk -F '.' '{print $1}') +export MAIN_INT + +echo "Management interface is: $MAIN_INT" + +#fix the issue +echo "Fixing management interface..." +ethtool -K "$MAIN_INT" tso off +ethtool -K "$MAIN_INT" gro off +ethtool -K "$MAIN_INT" gso off +ethtool -K "$MAIN_INT" tx off +ethtool -K "$MAIN_INT" rx off + +#https://forum.proxmox.com/threads/e1000-driver-hang.58284/ +#https://serverfault.com/questions/616485/e1000e-reset-adapter-unexpectedly-detected-hardware-unit-hang + + diff --git a/archive/provisioning/Dell/Server/omsa.sh b/archive/provisioning/Dell/Server/omsa.sh new file mode 100644 index 0000000..ae30060 --- /dev/null +++ b/archive/provisioning/Dell/Server/omsa.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +#curl -s http://dl.turnsys.net/omsa.sh|/bin/bash + +gpg --keyserver hkp://pool.sks-keyservers.net:80 --recv-key 1285491434D8786F +gpg -a --export 1285491434D8786F | apt-key add - +echo "deb https://linux.dell.com/repo/community/openmanage/930/bionic bionic main" > /etc/apt/sources.list.d/linux.dell.com.sources.list +wget https://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-curl-client-transport1_2.6.5-0ubuntu3_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-client4_2.6.5-0ubuntu3_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman1_2.6.5-0ubuntu3_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-server1_2.6.5-0ubuntu3_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-sfcc/libcimcclient0_2.2.8-0ubuntu2_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/openwsman_2.6.5-0ubuntu3_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/multiverse/c/cim-schema/cim-schema_2.48.0-0ubuntu1_all.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-sfc-common/libsfcutil0_1.0.1-0ubuntu4_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/multiverse/s/sblim-sfcb/sfcb_1.4.9-0ubuntu5_amd64.deb +wget https://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-cmpi-devel/libcmpicppimpl0_2.0.3-0ubuntu2_amd64.deb +dpkg -i libwsman-curl-client-transport1_2.6.5-0ubuntu3_amd64.deb +dpkg -i libwsman-client4_2.6.5-0ubuntu3_amd64.deb +dpkg -i libwsman1_2.6.5-0ubuntu3_amd64.deb +dpkg -i libwsman-server1_2.6.5-0ubuntu3_amd64.deb +dpkg -i libcimcclient0_2.2.8-0ubuntu2_amd64.deb +dpkg -i openwsman_2.6.5-0ubuntu3_amd64.deb +dpkg -i cim-schema_2.48.0-0ubuntu1_all.deb +dpkg -i libsfcutil0_1.0.1-0ubuntu4_amd64.deb +dpkg -i sfcb_1.4.9-0ubuntu5_amd64.deb +dpkg -i libcmpicppimpl0_2.0.3-0ubuntu2_amd64.deb + +apt update +apt -y install srvadmin-all +touch /opt/dell/srvadmin/lib64/openmanage/IGNORE_GENERATION + +#logout,login, then run +# srvadmin-services.sh enable && srvadmin-services.sh start diff --git a/archive/provisioning/Dell/fixcpuperf.sh b/archive/provisioning/Dell/fixcpuperf.sh new file mode 100644 index 0000000..e795119 --- /dev/null +++ b/archive/provisioning/Dell/fixcpuperf.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +#Script to set performance. + + + +cpufreq-set -r -g performance +cpupower frequency-set --governor performance + + diff --git a/archive/provisioning/Modules/Auth/auth-cloudron-ldap.sh b/archive/provisioning/Modules/Auth/auth-cloudron-ldap.sh new file mode 100644 index 0000000..da03f3d --- /dev/null +++ b/archive/provisioning/Modules/Auth/auth-cloudron-ldap.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# auth-cloudron-ldap.sh — placeholder module (Cloudron LDAP auth integration). +# Intentionally empty; populated when the auth stack is deployed. +true diff --git a/archive/provisioning/Modules/OAM/oam-librenms.sh b/archive/provisioning/Modules/OAM/oam-librenms.sh new file mode 100644 index 0000000..68f0d4c --- /dev/null +++ b/archive/provisioning/Modules/OAM/oam-librenms.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +export AGENTS_PATH +AGENTS_PATH="$PROJECT_ROOT_PATH/provisioning/Agents" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + +print_info "Setting up librenms agent..." + +cat "$AGENTS_PATH/librenms/distro" > /usr/local/bin/distro +chmod +x /usr/local/bin/distro + +if [ ! -d /usr/lib/check_mk_agent ]; then +mkdir -p /usr/lib/check_mk_agent +fi + +if [ ! -d /usr/lib/check_mk_agent/plugins ]; then +mkdir -p /usr/lib/check_mk_agent/plugins +fi + +if [ ! -d /usr/lib/check_mk_agent/local ]; then +mkdir -p /usr/lib/check_mk_agent/local +fi + +cat "$AGENTS_PATH/librenms/check_mk_agent" > /usr/bin/check_mk_agent +chmod +x /usr/bin/check_mk_agent + +cat "$AGENTS_PATH/librenms/check_mk@.service" > /etc/systemd/system/check_mk@.service +cat "$AGENTS_PATH/librenms/check_mk.socket" > /etc/systemd/system/check_mk.socket + +systemctl enable check_mk.socket +systemctl start check_mk.socket + +#Modules commented out below, we will roll out on systems that use them, most of the fleet doesn't use those modules + +cat "$AGENTS_PATH/librenms/dmi.sh" > /usr/lib/check_mk_agent/local/dmi.sh +cat "$AGENTS_PATH/librenms/dpkg.sh" > /usr/lib/check_mk_agent/local/dpkg.sh +#cat "$AGENTS_PATH/librenms/mysql.sh" > /usr/lib/check_mk_agent/local/mysql.sh +cat "$AGENTS_PATH/librenms/ntp-client" > /usr/lib/check_mk_agent/local/ntp-client +#cat "$AGENTS_PATH/librenms/ntp-server.sh" > /usr/lib/check_mk_agent/local/ntp-server.sh +cat "$AGENTS_PATH/librenms/os-updates.sh" > /usr/lib/check_mk_agent/local/os-updates.sh +cat "$AGENTS_PATH/librenms/postfixdetailed" > /usr/lib/check_mk_agent/local/postfixdetailed +cat "$AGENTS_PATH/librenms/postfix-queues" > /usr/lib/check_mk_agent/local/postfix-queues +#cat "$AGENTS_PATH/librenms/smart.sh" > /usr/lib/check_mk_agent/local/smart +#cat "$AGENTS_PATH/librenms/smart.sh.config" > /usr/lib/check_mk_agent/local/smart.config + +chmod +x /usr/lib/check_mk_agent/local/* \ No newline at end of file diff --git a/archive/provisioning/Modules/RandD/sslStackFromSource.sh b/archive/provisioning/Modules/RandD/sslStackFromSource.sh new file mode 100644 index 0000000..2e9e819 --- /dev/null +++ b/archive/provisioning/Modules/RandD/sslStackFromSource.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# shellcheck disable=SC2103 # legacy R&D build script; cd/cd- sequence is intentional + +#Made from instructions at https://www.tunetheweb.com/performance/http2/ + +OPENSSL_URL_BASE="https://www.openssl.org/source/" +OPENSSL_FILE="openssl-1.1.0h.tar.gz" + +NGHTTP_URL_BASE="https://github.com/nghttp2/nghttp2/releases/download/v1.31.0/" +NGHTTP_FILE="nghttp2-1.31.0.tar.gz" + +APR_URL_BASE="https://archive.apache.org/dist/apr/" +APR_FILE="apr-1.6.3.tar.gz" + +APR_UTIL_URL_BASE="https://archive.apache.org/dist/apr/" +APR_UTIL_FILE="apr-util-1.6.1.tar.gz" + +APACHE_URL_BASE="https://archive.apache.org/dist/httpd/" +APACHE_FILE="httpd-2.4.33.tar.gz" + +CURL_URL_BASE="https://curl.haxx.se/download/" +CURL_FILE="curl-7.60.0.tar.gz" + + +#Download and install latest version of openssl +wget $OPENSSL_URL_BASE/$OPENSSL_FILE +tar xzf $OPENSSL_FILE +cd openssl-1.1.0h || exit +./config enable-weak-ssl-ciphers shared zlib-dynamic -DOPENSSL_TLS_SECURITY_LEVEL=0 --prefix=/usr/local/custom-ssl/openssl-1.1.0h ; make ; make install +ln -s /usr/local/custom-ssl/openssl-1.1.0h /usr/local/openssl +cd - || exit + +#Download and install nghttp2 (needed for mod_http2). +wget $NGHTTP_URL_BASE/$NGHTTP_FILE +tar xzf $NGHTTP_FILE +cd nghttp2-1.31.0 || exit +./configure --prefix=/usr/local/custom-ssl/nghttp ; make ; make install +cd - || exit + +#Updated ldconfig so curl build + +cat < /etc/ld.so.conf.d/custom-ssl.conf +/usr/local/custom-ssl/openssl-1.1.0h/lib +/usr/local/custom-ssl/nghttp/lib +custom-ssl + +ldconfig + +#Download and install curl +wget $CURL_URL_BASE/$CURL_FILE +tar xzf curl-7.60.0.tar.gz +cd curl-7.60.0 || exit +./configure --prefix=/usr/local/custom-ssl/curl --with-nghttp2=/usr/local/custom-ssl/nghttp/ --with-ssl=/usr/local/custom-ssl/openssl-1.1.0h/ ; make ; make install +cd - || exit + + +#Download and install latest apr +wget $APR_URL_BASE/$APR_FILE +tar xzf $APR_FILE +cd apr-1.6.3 || exit +./configure --prefix=/usr/local/custom-ssl/apr ; make ; make install +cd - || exit + +#Download and install latest apr-util +wget $APR_UTIL_URL_BASE/$APR_UTIL_FILE +tar xzf apr-util-1.6.1.tar.gz +cd apr-util-1.6.1 || exit +./configure --prefix=/usr/local/custom-ssl/apr-util --with-apr=/usr/local/custom-ssl/apr ; make; make install +cd - || exit + +#Download and install apache +wget $APACHE_URL_BASE/$APACHE_FILE +tar xzf httpd-2.4.33.tar.gz +cd httpd-2.4.33 || exit +cp -r ../apr-1.6.3 srclib/apr +cp -r ../apr-util-1.6.1 srclib/apr-util +./configure --prefix=/usr/local/custom-ssl/apache --with-ssl=/usr/local/custom-ssl/openssl-1.1.0h/ --with-pcre=/usr/bin/pcre-config --enable-unique-id --enable-ssl --enable-so --with-included-apr --enable-http2 --with-nghttp2=/usr/local/custom-ssl/nghttp/ +make +make install +ln -s /usr/local/custom-ssl/apache /usr/local/apache +cd - || exit + diff --git a/archive/provisioning/Modules/Security/secharden-2fa.sh b/archive/provisioning/Modules/Security/secharden-2fa.sh new file mode 100644 index 0000000..48006b7 --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-2fa.sh @@ -0,0 +1,426 @@ +#!/bin/bash + +# TSYS Security Hardening - Two-Factor Authentication +# Implements 2FA for SSH, Cockpit, and Webmin services +# Uses Google Authenticator (TOTP) for time-based tokens + + +##### +#Core framework functions... +##### + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + +# 2FA Configuration +BACKUP_DIR="/root/backup/2fa" +PAM_CONFIG_DIR="/etc/pam.d" +SSH_CONFIG="/etc/ssh/sshd_config" +COCKPIT_CONFIG="/etc/cockpit/cockpit.conf" + +# Create backup directory +mkdir -p "$BACKUP_DIR" + +print_info "TSYS Two-Factor Authentication Setup" + +# Backup existing configurations +function backup_configs() { + print_info "Creating backup of existing configurations..." + + # Backup SSH configuration + if [[ -f "$SSH_CONFIG" ]]; then + cp "$SSH_CONFIG" "$BACKUP_DIR/sshd_config.bak" + print_info "SSH config backed up" + fi + + # Backup PAM configurations + if [[ -d "$PAM_CONFIG_DIR" ]]; then + cp -r "$PAM_CONFIG_DIR" "$BACKUP_DIR/pam.d.bak" + print_info "PAM configs backed up" + fi + + # Backup Cockpit configuration if exists + if [[ -f "$COCKPIT_CONFIG" ]]; then + cp "$COCKPIT_CONFIG" "$BACKUP_DIR/cockpit.conf.bak" + print_info "Cockpit config backed up" + fi + + print_info "Backup completed: $BACKUP_DIR" +} + +# Install required packages +function install_2fa_packages() { + print_info "Installing 2FA packages..." + + # Update package cache + apt-get update + + # Install Google Authenticator PAM module + # Install QR code generator for terminal display + apt-get install -y libpam-google-authenticator qrencode + + print_info "2FA packages installed successfully" +} + +# Configure SSH for 2FA +function configure_ssh_2fa() { + print_info "Configuring SSH for 2FA..." + + # Configure SSH daemon + print_info "Updating SSH configuration..." + + # Enable challenge-response authentication + if ! grep -q "^ChallengeResponseAuthentication yes" "$SSH_CONFIG"; then + sed -i 's/^ChallengeResponseAuthentication.*/ChallengeResponseAuthentication yes/' "$SSH_CONFIG" || \ + echo "ChallengeResponseAuthentication yes" >> "$SSH_CONFIG" + fi + + if ! grep -q "^KbdInteractiveAuthentication yes" "$SSH_CONFIG"; then + sed -i 's/^KbdInteractiveAuthentication.*/KbdInteractiveAuthentication yes/' "$SSH_CONFIG" || \ + echo "KbdInteractiveAuthentication yes" >> "$SSH_CONFIG" + fi + + # Enable PAM authentication + if ! grep -q "^UsePAM yes" "$SSH_CONFIG"; then + sed -i 's/^UsePAM.*/UsePAM yes/' "$SSH_CONFIG" || \ + echo "UsePAM yes" >> "$SSH_CONFIG" + fi + + # Configure authentication methods (key + 2FA) + if ! grep -q "^AuthenticationMethods" "$SSH_CONFIG"; then + echo "AuthenticationMethods publickey,keyboard-interactive" >> "$SSH_CONFIG" + else + sed -i 's/^AuthenticationMethods.*/AuthenticationMethods publickey,keyboard-interactive/' "$SSH_CONFIG" + fi + + print_info "SSH configuration updated" +} + +# Configure PAM for 2FA +function configure_pam_2fa() { + print_info "Configuring PAM for 2FA..." + + # Create backup of original PAM SSH config + cp "$PAM_CONFIG_DIR/sshd" "$PAM_CONFIG_DIR/sshd.bak.$(date +%Y%m%d)" + + # Configure PAM to use Google Authenticator + cat > "$PAM_CONFIG_DIR/sshd" << 'EOF' +# PAM configuration for SSH with 2FA +# Standard Un*x authentication +@include common-auth + +# Google Authenticator 2FA +auth required pam_google_authenticator.so nullok + +# Standard Un*x authorization +@include common-account + +# SELinux needs to be the first session rule +session required pam_selinux.so close +session required pam_loginuid.so + +# Standard Un*x session setup and teardown +@include common-session + +# Print the message of the day upon successful login +session optional pam_motd.so motd=/run/motd.dynamic +session optional pam_motd.so noupdate + +# Print the status of the user's mailbox upon successful login +session optional pam_mail.so standard noenv + +# Set up user limits from /etc/security/limits.conf +session required pam_limits.so + +# SELinux needs to intervene at login time +session required pam_selinux.so open + +# Standard Un*x password updating +@include common-password +EOF + + print_info "PAM configuration updated for SSH 2FA" +} + +# Configure Cockpit for 2FA +function configure_cockpit_2fa() { + print_info "Configuring Cockpit for 2FA..." + + # Create Cockpit config directory if it doesn't exist + mkdir -p "$(dirname "$COCKPIT_CONFIG")" + + # Configure Cockpit to use PAM with 2FA + cat > "$COCKPIT_CONFIG" << 'EOF' +[WebService] +# Enable 2FA for Cockpit web interface +LoginTitle = TSYS Server Management +LoginTo = 300 +RequireHost = true + +[Session] +# Use PAM for authentication (includes 2FA) +Banner = /etc/cockpit/issue.cockpit +IdleTimeout = 15 +EOF + + # Create PAM configuration for Cockpit + cat > "$PAM_CONFIG_DIR/cockpit" << 'EOF' +# PAM configuration for Cockpit with 2FA +auth requisite pam_nologin.so +auth required pam_env.so +auth required pam_faillock.so preauth +auth sufficient pam_unix.so try_first_pass +auth required pam_google_authenticator.so nullok +auth required pam_faillock.so authfail +auth required pam_deny.so + +account required pam_nologin.so +account include system-auth +account required pam_faillock.so + +session required pam_selinux.so close +session required pam_loginuid.so +session optional pam_keyinit.so force revoke +session include system-auth +session required pam_selinux.so open +session optional pam_motd.so +EOF + + print_info "Cockpit 2FA configuration completed" +} + +# Configure Webmin for 2FA (if installed) +function configure_webmin_2fa() { + print_info "Checking for Webmin installation..." + + local webmin_config="/etc/webmin/miniserv.conf" + + if [[ -f "$webmin_config" ]]; then + print_info "Webmin found, configuring 2FA..." + + # Stop webmin service + systemctl stop webmin || true + + # Enable 2FA in Webmin configuration. `sed -i ... || echo` would never + # append, because sed returns 0 even when it matches nothing; guard with + # grep so the directive is added when absent and updated when present. + if grep -q '^twofactor_provider=' "$webmin_config"; then + sed -i 's/^twofactor_provider=.*/twofactor_provider=totp/' "$webmin_config" + else + echo "twofactor_provider=totp" >> "$webmin_config" + fi + + # Enable 2FA requirement + if grep -q '^twofactor=' "$webmin_config"; then + sed -i 's/^twofactor=.*/twofactor=1/' "$webmin_config" + else + echo "twofactor=1" >> "$webmin_config" + fi + + # Start webmin service + systemctl start webmin || true + + print_info "Webmin 2FA configuration completed" + else + print_info "Webmin not found, skipping configuration" + fi +} + +# Setup 2FA for users +function setup_user_2fa() { + print_info "Setting up 2FA for system users..." + + local users=("localuser" "root") + + for user in "${users[@]}"; do + if id "$user" &>/dev/null; then + print_info "Setting up 2FA for user: $user" + + local user_home + user_home="$(getent passwd "$user" | cut -d: -f6)" + if [[ -z "$user_home" ]]; then + print_info "No home directory for $user, skipping" + continue + fi + + # Create 2FA setup script for user + cat > "/tmp/setup-2fa-$user.sh" << 'EOF' +#!/bin/bash +echo "Setting up Google Authenticator for user: $USER" +echo "Please follow the prompts to configure 2FA:" +echo "1. Answer 'y' to update your time-based token" +echo "2. Scan the QR code with your authenticator app" +echo "3. Save the backup codes in a secure location" +echo "4. Answer 'y' to the remaining questions for security" +echo "" +google-authenticator -t -d -f -r 3 -R 30 -W +EOF + + chmod +x "/tmp/setup-2fa-$user.sh" + + # Instructions for user setup + cat > "$user_home/2fa-setup-instructions.txt" << EOF +TSYS Two-Factor Authentication Setup Instructions +============================================== + +Your system has been configured for 2FA. To complete setup: + +1. Install an authenticator app on your phone: + - Google Authenticator + - Authy + - Microsoft Authenticator + +2. Run the setup command: + sudo /tmp/setup-2fa-$user.sh + +3. Follow the prompts: + - Scan the QR code with your app + - Save the backup codes securely + - Answer 'y' to security questions + +4. Test your setup: + - SSH to the server + - Enter your 6-digit code when prompted + +IMPORTANT: Save backup codes in a secure location! +Without them, you may be locked out if you lose your phone. + +For support, contact your system administrator. +EOF + + chown "$user:$user" "$user_home/2fa-setup-instructions.txt" + print_info "2FA setup prepared for user: $user" + else + print_info "User $user not found, skipping" + fi + done +} + +# Restart services +function restart_services() { + print_info "Restarting services..." + + # Test SSH configuration + if sshd -t; then + systemctl restart sshd + print_info "SSH service restarted" + else + print_error "SSH configuration test failed" + return 1 + fi + + # Restart Cockpit if installed + if systemctl is-enabled cockpit.socket &>/dev/null; then + systemctl restart cockpit.socket + print_info "Cockpit service restarted" + fi + + # Restart Webmin if installed + if systemctl is-enabled webmin &>/dev/null; then + systemctl restart webmin + print_info "Webmin service restarted" + fi +} + +# Validation and testing +function validate_2fa_setup() { + print_info "Validating 2FA setup..." + + # Check if Google Authenticator is installed + if command -v google-authenticator &>/dev/null; then + print_info "Google Authenticator installed" + else + print_error "Google Authenticator not found" + return 1 + fi + + # Check SSH configuration + if grep -q "AuthenticationMethods publickey,keyboard-interactive" "$SSH_CONFIG"; then + print_info "SSH 2FA configuration valid" + else + print_error "SSH 2FA configuration invalid" + return 1 + fi + + # Check PAM configuration + if grep -q "pam_google_authenticator.so" "$PAM_CONFIG_DIR/sshd"; then + print_info "PAM 2FA configuration valid" + else + print_error "PAM 2FA configuration invalid" + return 1 + fi + + # Check service status + if systemctl is-active sshd &>/dev/null; then + print_info "SSH service is running" + else + print_error "SSH service is not running" + return 1 + fi + + print_info "2FA validation completed successfully" +} + +# Display final instructions +function show_final_instructions() { + print_info "2FA Setup Completed" + + print_info "Two-Factor Authentication has been configured for:" + print_info "- SSH (requires key + 2FA token)" + print_info "- Cockpit web interface" + if [[ -f "/etc/webmin/miniserv.conf" ]]; then + print_info "- Webmin administration panel" + fi + + print_info "IMPORTANT: Complete user setup immediately!" + print_info "1. Check /home/*/2fa-setup-instructions.txt for user setup" + print_info "2. Run setup scripts for each user" + print_info "3. Test 2FA before logging out" + + print_info "Backup location: $BACKUP_DIR" + print_info "To disable 2FA, restore configurations from backup" + + print_info "2FA setup completed successfully!" +} + +# Main execution +function main() { + # Check if running as root + if [[ $EUID -ne 0 ]]; then + print_error "This script must be run as root" + exit 1 + fi + + # Execute setup steps + backup_configs + install_2fa_packages + configure_ssh_2fa + configure_pam_2fa + configure_cockpit_2fa + configure_webmin_2fa + setup_user_2fa + restart_services + validate_2fa_setup + show_final_instructions +} + +# Run main function +main "$@" \ No newline at end of file diff --git a/archive/provisioning/Modules/Security/secharden-audit-agents.sh b/archive/provisioning/Modules/Security/secharden-audit-agents.sh new file mode 100644 index 0000000..7c5d81c --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-audit-agents.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +##### +#Core framework functions... +##### + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +export CONFIGFILES_PATH +CONFIGFILES_PATH="$PROJECT_ROOT_PATH/provisioning/ConfigFiles" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + +# Material herein Sourced from + +# https://cisofy.com/documentation/lynis/ +# https://jbcsec.com/configure-linux-ssh/ +# https://opensource.com/article/20/5/linux-security-lynis +# https://forum.greenbone.net/t/ssh-authentication/13536 + +# openvas + +#lynis + +#Auditd + +cat "$CONFIGFILES_PATH/AuditD/auditd.conf" > /etc/audit/auditd.conf + +# Systemd +cat "$CONFIGFILES_PATH/Systemd/journald.conf" > /etc/systemd/journald.conf + +# logrotate +cat "$CONFIGFILES_PATH/Logrotate/logrotate.conf" > /etc/logrotate.conf \ No newline at end of file diff --git a/archive/provisioning/Modules/Security/secharden-auto-upgrade.sh b/archive/provisioning/Modules/Security/secharden-auto-upgrade.sh new file mode 100644 index 0000000..7f171ec --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-auto-upgrade.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +# Sourced from https://wiki.debian.org/UnattendedUpgrades diff --git a/archive/provisioning/Modules/Security/secharden-scap-stig.sh b/archive/provisioning/Modules/Security/secharden-scap-stig.sh new file mode 100644 index 0000000..9c92e93 --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-scap-stig.sh @@ -0,0 +1,126 @@ +#!/bin/bash + + +######################################### +#Core framework functions... +######################################### + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +export CONFIGFILES_PATH +CONFIGFILES_PATH="$PROJECT_ROOT_PATH/provisioning/ConfigFiles" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + + +######################################### +# Core script code begins here +######################################### + +# Sourced from + +# https://complianceascode.readthedocs.io/en/latest/manual/developer/01_introduction.html +# https://github.com/ComplianceAsCode/content +# https://github.com/ComplianceAsCode + +#apparmor +#enforcing +#enabled in bootloader config + +#aide + +#auditd + +#disable auto mounting +#disable usb storage + + +#motd +#remote login warning banner + +#Ensure time sync is working +#systemd-timesync +#ntp +#chrony + +#password complexity +#password expiration warning +#password expiration time +#password hashing algo + +#fix grub perms + +if [ "$IS_RASPI" = 0 ] ; then + +chown root:root /boot/grub/grub.cfg +chmod og-rwx /boot/grub/grub.cfg +chmod 0400 /boot/grub/grub.cfg + +fi + + +#disable auto mounting +systemctl --now disable autofs || true +apt-get -y --purge remove autofs || true + +#disable usb storage +cat "$CONFIGFILES_PATH/ModProbe/usb_storage.conf" > /etc/modprobe.d/usb_storage.conf +cat "$CONFIGFILES_PATH/ModProbe/dccp.conf" > /etc/modprobe.d/dccp.conf +cat "$CONFIGFILES_PATH/ModProbe/rds.conf" > /etc/modprobe.d/rds.conf +cat "$CONFIGFILES_PATH/ModProbe/sctp.conf" > /etc/modprobe.d/sctp.conf +cat "$CONFIGFILES_PATH/ModProbe/tipc.conf" > /etc/modprobe.d/tipc.conf +cat "$CONFIGFILES_PATH/ModProbe/cramfs.conf" > /etc/modprobe.d/cramfs.conf +cat "$CONFIGFILES_PATH/ModProbe/freevxfs.conf" > /etc/modprobe.d/freevxfs.conf +cat "$CONFIGFILES_PATH/ModProbe/hfs.conf" > /etc/modprobe.d/hfs.conf +cat "$CONFIGFILES_PATH/ModProbe/hfsplus.conf" > /etc/modprobe.d/hfsplus.conf +cat "$CONFIGFILES_PATH/ModProbe/jffs2.conf" > /etc/modprobe.d/jffs2.conf +cat "$CONFIGFILES_PATH/ModProbe/squashfs.conf" > /etc/modprobe.d/squashfs.conf +cat "$CONFIGFILES_PATH/ModProbe/udf.conf" > /etc/modprobe.d/udf.conf + +#banners + +cat "$CONFIGFILES_PATH/BANNERS/issue" > /etc/issue +cat "$CONFIGFILES_PATH/BANNERS/issue.net" > /etc/issue.net +cat "$CONFIGFILES_PATH/BANNERS/motd" > /etc/motd + +#Cron perms + +if [ -f /etc/cron.deny ]; then +rm /etc/cron.deny || true +fi + +touch /etc/cron.allow +chmod g-wx,o-rwx /etc/cron.allow +chown root:root /etc/cron.allow + +chmod og-rwx /etc/crontab +chmod og-rwx /etc/cron.hourly/ +chmod og-rwx /etc/cron.daily/ +chmod og-rwx /etc/cron.weekly/ +chmod og-rwx /etc/cron.monthly/ +chown root:root /etc/cron.d/ +chmod og-rwx /etc/cron.d/ + +# At perms + +rm -f /etc/at.deny || true +touch /etc/at.allow +chmod g-wx,o-rwx /etc/at.allow +chown root:root /etc/at.allow \ No newline at end of file diff --git a/archive/provisioning/Modules/Security/secharden-ssh.sh b/archive/provisioning/Modules/Security/secharden-ssh.sh new file mode 100644 index 0000000..71bde08 --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-ssh.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +######################################### +#Core framework functions... +######################################### + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +export CONFIGFILES_PATH +CONFIGFILES_PATH="$PROJECT_ROOT_PATH/provisioning/ConfigFiles" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + + +######################################### +# Core script code begins here +######################################### + +export SUBODEV_CHECK +SUBODEV_CHECK="$(getent passwd | grep -c subodev || true)" + +export LOCALUSER_CHECK +LOCALUSER_CHECK="$(getent passwd | grep -c localuser || true)" + +export ROOT_SSH_DIR +ROOT_SSH_DIR="/root/.ssh" + +export LOCALUSER_SSH_DIR +LOCALUSER_SSH_DIR="/home/localuser/.ssh" + +export SUBODEV_SSH_DIR +SUBODEV_SSH_DIR="/home/subodev/.ssh" + + +if [ ! -d $ROOT_SSH_DIR ]; then + mkdir /root/.ssh/ +fi + +cat "$CONFIGFILES_PATH/SSH/AuthorizedKeys/root-ssh-authorized-keys" >/root/.ssh/authorized_keys +chmod 400 /root/.ssh/authorized_keys +chown root: /root/.ssh/authorized_keys + +if [ "$LOCALUSER_CHECK" -gt 0 ]; then + if [ ! -d $LOCALUSER_SSH_DIR ]; then + mkdir -p /home/localuser/.ssh/ + fi + + cat "$CONFIGFILES_PATH/SSH/AuthorizedKeys/localuser-ssh-authorized-keys" >/home/localuser/.ssh/authorized_keys + chown localuser /home/localuser/.ssh/authorized_keys && + chmod 400 /home/localuser/.ssh/authorized_keys +fi + +if [ "$SUBODEV_CHECK" = 1 ]; then + + if [ ! -d $SUBODEV_SSH_DIR ]; then + mkdir /home/subodev/.ssh/ + fi + + cat "$CONFIGFILES_PATH/SSH/AuthorizedKeys/localuser-ssh-authorized-keys" >/home/subodev/.ssh/authorized_keys + chmod 400 /home/subodev/.ssh/authorized_keys && + chown subodev: /home/subodev/.ssh/authorized_keys +fi + +export DEV_WORKSTATION_CHECK +DEV_WORKSTATION_CHECK="$(hostname | grep -Ec 'subopi-dev|CharlesDevServer' || true)" + +if [ "$DEV_WORKSTATION_CHECK" -eq 0 ]; then + + cat "$CONFIGFILES_PATH/SSH/Configs/tsys-sshd-config" >/etc/ssh/sshd_config +fi + + +#Don't deploy this config to a ubuntu server, it breaks openssh server. Works on kali/debian. + +export UBUNTU_CHECK +UBUNTU_CHECK="$(distro | grep -c Ubuntu||true)" + +if [ "$UBUNTU_CHECK" -ne 1 ]; then + cat "$CONFIGFILES_PATH/SSH/Configs/ssh-audit-hardening.conf" >/etc/ssh/sshd_config.d/ssh-audit_hardening.conf + chmod og-rwx /etc/ssh/sshd_config.d/* +fi + +# Perms on sshd_config +chmod og-rwx /etc/ssh/sshd_config + +#todo + +# only strong MAC algos are used \ No newline at end of file diff --git a/archive/provisioning/Modules/Security/secharden-wazuh.sh b/archive/provisioning/Modules/Security/secharden-wazuh.sh new file mode 100644 index 0000000..e52367d --- /dev/null +++ b/archive/provisioning/Modules/Security/secharden-wazuh.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +######################################### +#Core framework functions... +######################################### + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + + +######################################### +# Core script code begins here +######################################### + +# We don't want to run this on the wazuh server, otherwise bad things happen... + +export TSYS_NSM_CHECK +TSYS_NSM_CHECK="$(hostname |grep -c tsys-nsm ||true)" + +if [ "$TSYS_NSM_CHECK" -eq 0 ]; then + + if [ -f /usr/share/keyrings/wazuh.gpg ]; then + rm -f /usr/share/keyrings/wazuh.gpg + fi + +curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | gpg --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/wazuh.gpg --import +chmod 644 /usr/share/keyrings/wazuh.gpg +echo "deb [signed-by=/usr/share/keyrings/wazuh.gpg] https://packages.wazuh.com/4.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list +apt-get update + +WAZUH_MANAGER="tsys-nsm.knel.net" apt-get -y install wazuh-agent + +systemctl daemon-reload +systemctl enable wazuh-agent +systemctl start wazuh-agent || true + +echo "wazuh-agent hold" | dpkg --set-selections + +fi \ No newline at end of file diff --git a/archive/provisioning/Project-ConfigFiles/CONFIG_VARS b/archive/provisioning/Project-ConfigFiles/CONFIG_VARS new file mode 100644 index 0000000..d0c99d8 --- /dev/null +++ b/archive/provisioning/Project-ConfigFiles/CONFIG_VARS @@ -0,0 +1,3 @@ + +export DL_ROOT +DL_ROOT="https://dl.knownelement.com/KNEL/FetchApply/" \ No newline at end of file diff --git a/archive/provisioning/Project-Includes/LocalHelp.sh b/archive/provisioning/Project-Includes/LocalHelp.sh new file mode 100644 index 0000000..de3918f --- /dev/null +++ b/archive/provisioning/Project-Includes/LocalHelp.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +function LocalHelp() +{ + echo "$0 is " + echo "$0 takes arguments: " + echo "1) " + echo "2) " + echo ":" + echo "" + echo "" + echo "" +} diff --git a/archive/provisioning/Project-Includes/PreflightCheck.sh b/archive/provisioning/Project-Includes/PreflightCheck.sh new file mode 100644 index 0000000..94e736a --- /dev/null +++ b/archive/provisioning/Project-Includes/PreflightCheck.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +function PreflightCheck() +{ + +export curr_user="$USER" +export user_check + +user_check="$(echo "$curr_user" | grep -c root)" + + +if [ "$user_check" -ne 1 ]; then + print_error "Must run as root." + error_out +fi + +echo "All checks passed...." + +} diff --git a/archive/provisioning/Project-Includes/pi-detect.sh b/archive/provisioning/Project-Includes/pi-detect.sh new file mode 100644 index 0000000..29bce4d --- /dev/null +++ b/archive/provisioning/Project-Includes/pi-detect.sh @@ -0,0 +1,13 @@ +# shellcheck shell=bash disable=SC2148 # sourced function file (no shebang by design) +function pi-detect() +{ +print_info Now running "${FUNCNAME[0]}".... +if [ -f /sys/firmware/devicetree/base/model ] ; then +export IS_RASPI="1" +fi + +if [ ! -f /sys/firmware/devicetree/base/model ] ; then +export IS_RASPI="0" +fi +print_info Completed running "${FUNCNAME[0]}" +} \ No newline at end of file diff --git a/archive/provisioning/SetupNewSystem.sh b/archive/provisioning/SetupNewSystem.sh new file mode 100644 index 0000000..00e8fd2 --- /dev/null +++ b/archive/provisioning/SetupNewSystem.sh @@ -0,0 +1,431 @@ +#!/usr/bin/bash + +##### +#Core framework functions... +##### + + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export PROJECT_ROOT_PATH +PROJECT_ROOT_PATH="$(cd "$SCRIPT_DIR/.." && pwd)" + +export GIT_VENDOR_PATH_ROOT +GIT_VENDOR_PATH_ROOT="$PROJECT_ROOT_PATH/vendor/git@git.knownelement.com/29418/" + +export KNELShellFrameworkRoot +KNELShellFrameworkRoot="$GIT_VENDOR_PATH_ROOT/KNEL/KNELShellFramework" + +export CONFIGFILES_PATH +CONFIGFILES_PATH="$PROJECT_ROOT_PATH/provisioning/ConfigFiles" + +export MODULES_PATH +MODULES_PATH="$PROJECT_ROOT_PATH/provisioning/Modules" + +export SCRIPTS_PATH +SCRIPTS_PATH="$PROJECT_ROOT_PATH/provisioning/scripts" + +source "$KNELShellFrameworkRoot/Framework-ConfigFiles/FrameworkVars" + +for framework_include_file in "$KNELShellFrameworkRoot"/Framework-Includes/*; do + source "$framework_include_file" +done + +for project_include_file in "$PROJECT_ROOT_PATH"/Project-Includes/*; do + source "$project_include_file" +done + +# Start actual script logic here... + +################# +#Global variables +################# + +apt-get -y install git sudo dmidecode curl + +export UBUNTU_CHECK +UBUNTU_CHECK="$(distro | grep -c Ubuntu || true)" + +export IS_PHYSICAL_HOST +IS_PHYSICAL_HOST="$(/usr/sbin/dmidecode -t System | grep -c Dell || true)" + +export SUBODEV_CHECK +SUBODEV_CHECK="$(getent passwd | grep -c subodev || true)" + +export LOCALUSER_CHECK +LOCALUSER_CHECK="$(getent passwd | grep -c localuser || true)" + +####################### +# Support functions +####################### + +function global-oam() { + print_info "Now running ${FUNCNAME[0]}...." + + cat "$SCRIPTS_PATH/up2date.sh" >/usr/local/bin/up2date.sh && chmod +x /usr/local/bin/up2date.sh + + bash "$MODULES_PATH/OAM/oam-librenms.sh" + + print_info "Completed running ${FUNCNAME[0]}" + +} + +function global-systemServiceConfigurationFiles() { + print_info "Now running ${FUNCNAME[0]}...." + + cat "$CONFIGFILES_PATH/ZSH/tsys-zshrc" >/etc/zshrc + cat "$CONFIGFILES_PATH/SMTP/aliases" >/etc/aliases + cat "$CONFIGFILES_PATH/Syslog/rsyslog.conf" >/etc/rsyslog.conf + + newaliases + + print_info "Completed running ${FUNCNAME[0]}" +} + +function global-installPackages() { + print_info "Now running ${FUNCNAME[0]}...." + + # Setup webmin repo, used for RBAC/2fa PAM + + curl https://raw.githubusercontent.com/webmin/webmin/master/webmin-setup-repo.sh >/tmp/webmin-setup.sh + sh /tmp/webmin-setup.sh -f && rm -f /tmp/webmin-setup.sh + + # Setup tailscale + + curl -fsSL https://tailscale.com/install.sh | sh + + # + #Patch the system + # + + /usr/local/bin/up2date.sh + + #Remove stuff we don't want + + export DEBIAN_FRONTEND="noninteractive" \ + && apt-get -qq --yes --purge \ + remove \ + systemd-timesyncd \ + chrony \ + telnet \ + inetutils-telnet \ + wpasupplicant \ + modemmanager \ + nano \ + multipath-tools \ + || true + + apt-get -y --purge autoremove + + # add stuff we want + + print_info ""Now installing all the packages..."" + + DEBIAN_FRONTEND="noninteractive" apt-get -qq --yes -o Dpkg::Options::="--force-confold" install \ + virt-what \ + auditd \ + audispd-plugins \ + cloud-guest-utils \ + aide \ + htop \ + snmpd \ + ncdu \ + iftop \ + iotop \ + cockpit \ + cockpit-bridge \ + cockpit-doc \ + cockpit-networkmanager \ + cockpit-packagekit \ + cockpit-pcp \ + cockpit-sosreport \ + cockpit-storaged \ + cockpit-system \ + cockpit-ws \ + nethogs \ + sysstat \ + ngrep \ + acct \ + lsb-release \ + screen \ + tailscale \ + tmux \ + vim \ + command-not-found \ + lldpd \ + ansible-core \ + net-tools \ + dos2unix \ + gpg \ + molly-guard \ + lshw \ + fzf \ + ripgrep \ + sudo \ + mailutils \ + clamav \ + sl \ + logwatch \ + git \ + net-tools \ + tshark \ + tcpdump \ + lynis \ + glances \ + zsh \ + zsh-autosuggestions \ + zsh-syntax-highlighting \ + fonts-powerline \ + webmin \ + usermin \ + ntpsec \ + ntpsec-ntpdate \ + tuned \ + cockpit \ + iptables \ + netfilter-persistent \ + iptables-persistent \ + pflogsumm \ + postfix + + export KALI_CHECK + KALI_CHECK="$(distro | grep -c kali || true)" + + export VIRT_TYPE + VIRT_TYPE="$(virt-what)" + + export IS_VIRT_GUEST + IS_VIRT_GUEST="$(echo "$VIRT_TYPE" | grep -Ec 'hyperv|kvm' || true)" + + export IS_KVM_GUEST + IS_KVM_GUEST="$(echo "$VIRT_TYPE" | grep -c 'kvm' || true)" + + if [[ $IS_KVM_GUEST = 1 ]]; then + apt -y install qemu-guest-agent + fi + + if [[ $KALI_CHECK -eq 0 ]];then + DEBIAN_FRONTEND="noninteractive" apt-get -qq --yes -o Dpkg::Options::="--force-confold" install \ + latencytop \ + cockpit-tests || true + fi + + if [[ $IS_PHYSICAL_HOST -gt 0 ]]; then + export DEBIAN_FRONTEND="noninteractive" && apt-get -qq --yes -o Dpkg::Options::="--force-confold" install \ + i7z \ + thermald \ + cpufrequtils \ + linux-cpupower + # power-profiles-daemon + fi + +############################ +# Secrets agents +############################ + +# bitwarden cli + +# vault cli + + print_info "Completed running ${FUNCNAME[0]}" +} + +function global-postPackageConfiguration() { + + print_info "Now running ${FUNCNAME[0]}" + + systemctl --now enable auditd + + systemctl stop postfix + + cat "$CONFIGFILES_PATH/SMTP/postfix_generic" >/etc/postfix/generic + postmap /etc/postfix/generic + + postconf -e "inet_protocols = ipv4" + postconf -e "inet_interfaces = 127.0.0.1" + postconf -e "mydestination= 127.0.0.1" + postconf -e "relayhost = tsys-cloudron.knel.net" + postconf -e "smtp_generic_maps = hash:/etc/postfix/generic" + # smtp_generic_maps = hash:/etc/postfix/generic + + systemctl restart postfix + + #This is under test/dev and may fail + echo "hi from root to root" | mail -s "hi directly to root from $(hostname)" root + + chsh -s "$(which zsh)" root + + if [ "$LOCALUSER_CHECK" -gt 0 ]; then + chsh -s "$(which zsh)" localuser + fi + + if [ "$SUBODEV_CHECK" -gt 0 ]; then + chsh -s "$(which zsh)" subodev + fi + + ###Post package deployment bits + + cat "$CONFIGFILES_PATH/DHCP/dhclient.conf" >/etc/dhcp/dhclient.conf + + # Authoritative recursive DNS via the redundant pfv-netinfra-01/02 pair. + # Replace whatever is at /etc/resolv.conf (including a systemd-resolved or + # NetworkManager symlink) with the managed static file so every lookup goes + # to our servers and nothing else rewrites it behind our backs. + rm -f /etc/resolv.conf + cat "$CONFIGFILES_PATH/Resolv/resolv.conf" >/etc/resolv.conf + chmod 644 /etc/resolv.conf + + systemctl stop snmpd && /etc/init.d/snmpd stop + + cat "$CONFIGFILES_PATH/SNMP/snmp-sudo.conf" >/etc/sudoers.d/Debian-snmp + sed -i "s|-Lsd|-LS6d|" /lib/systemd/system/snmpd.service + + pi-detect + + if [ "$IS_RASPI" = 1 ]; then + cat "$CONFIGFILES_PATH/SNMP/snmpd-rpi.conf" >/etc/snmp/snmpd.conf || true + fi + + if [ "$IS_PHYSICAL_HOST" = 1 ]; then + cat "$CONFIGFILES_PATH/SNMP/snmpd-physicalhost.conf" >/etc/snmp/snmpd.conf || true + fi + + if [ "$IS_VIRT_GUEST" = 1 ]; then + cat "$CONFIGFILES_PATH/SNMP/snmpd.conf" >/etc/snmp/snmpd.conf || true + fi + + systemctl daemon-reload && systemctl restart snmpd && /etc/init.d/snmpd restart + + cat "$CONFIGFILES_PATH/NetworkDiscovery/lldpd" >/etc/default/lldpd + systemctl restart lldpd + + cat "$CONFIGFILES_PATH/Cockpit/disallowed-users" >/etc/cockpit/disallowed-users + systemctl restart cockpit + + export LIBRENMS_CHECK + LIBRENMS_CHECK="$(hostname | grep -c tsys-librenms || true)" + + if [ "$LIBRENMS_CHECK" -eq 0 ]; then + DEBIAN_FRONTEND="noninteractive" apt-get -qq --yes -o Dpkg::Options::="--force-confold" install rsyslog + systemctl stop rsyslog + systemctl start rsyslog + fi + + export NTP_SERVER_CHECK + NTP_SERVER_CHECK="$(hostname | grep -Ec 'pfv-netboot|pfvsvrpi|pfv-netinfra' || true)" + + if [ "$NTP_SERVER_CHECK" -eq 0 ]; then + + cat "$CONFIGFILES_PATH/NTP/ntp.conf" >/etc/ntpsec/ntp.conf + systemctl restart ntpsec.service + fi + + systemctl stop postfix + systemctl start postfix + + /usr/sbin/accton on + + if [ "$IS_PHYSICAL_HOST" -gt 0 ]; then + cpufreq-set -r -g performance + cpupower frequency-set --governor performance + + # Potentially merge the below if needed. + # power-profiles-daemon + # powerprofilesctl set performance + #tsys1# systemctl enable power-profiles-daemon + #tsys1# systemctl start power-profiles-daemon + + fi + + if [ "$IS_VIRT_GUEST" = 1 ]; then + tuned-adm profile virtual-guest + fi + + print_info "Completed running ${FUNCNAME[0]}" +} + +#################################################################################################### +# Run various modules +#################################################################################################### + +#################################################################################################### +# Security Hardening +#################################################################################################### + +# SSH + +function secharden-ssh() { + print_info "Now running ${FUNCNAME[0]}" + + bash "$MODULES_PATH/Security/secharden-ssh.sh" + + print_info "Completed running ${FUNCNAME[0]}" +} + +function secharden-wazuh() { + print_info "Now running ${FUNCNAME[0]}" + bash "$MODULES_PATH/Security/secharden-wazuh.sh" + print_info "Completed running ${FUNCNAME[0]}" +} + +function secharden-2fa() { + print_info "Now running ${FUNCNAME[0]}" + bash "$MODULES_PATH/Security/secharden-2fa.sh" + print_info "Completed running ${FUNCNAME[0]}" +} + +function secharden-scap-stig() { + print_info "Now running ${FUNCNAME[0]}" + bash "$MODULES_PATH/Security/secharden-scap-stig.sh" + print_info "Completed running ${FUNCNAME[0]}" +} + +function secharden-agents() { + print_info "Now running ${FUNCNAME[0]}" + bash "$MODULES_PATH/Security/secharden-audit-agents.sh" + print_info "Completed running ${FUNCNAME[0]}" +} + +function secharden-auto-upgrades() { + print_info "Now running ${FUNCNAME[0]}" + #curl --silent ${DL_ROOT}/Modules/Security/secharden-ssh.sh|$(which bash) + print_info "Completed running ${FUNCNAME[0]}" +} + + + + +#################################################################################################### +# Authentication +#################################################################################################### + +function auth-cloudron-ldap() { + print_info "Now running ${FUNCNAME[0]}" + #curl --silent ${DL_ROOT}/Modules/Auth/auth-cloudron-ldap.sh|$(which bash) + print_info "Completed running ${FUNCNAME[0]}" +} + +#################################################################################################### +# RUn the various functions in the correct order +#################################################################################################### + +echo >"$LOGFILENAME" + +print_info "Execution starting at $CURRENT_TIMESTAMP..." + +PreflightCheck +global-oam +global-installPackages +global-systemServiceConfigurationFiles +global-postPackageConfiguration + +secharden-ssh +secharden-wazuh +secharden-scap-stig +secharden-2fa +#secharden-agents +#secharden-auto-upgrades + +#auth-cloudron-ldap + +print_info "Execution ended at $CURRENT_TIMESTAMP..." diff --git a/archive/provisioning/legacy/profiled-tmux.sh b/archive/provisioning/legacy/profiled-tmux.sh new file mode 100644 index 0000000..7a51ea6 --- /dev/null +++ b/archive/provisioning/legacy/profiled-tmux.sh @@ -0,0 +1,5 @@ +# shellcheck shell=bash disable=SC2148 # sourced .bashrc profile fragment +if command -v tmux &> /dev/null && [ -n "$PS1" ] && [[ ! "$TERM" =~ screen ]] && [[ ! "$TERM" =~ tmux ]] && [ -z "$TMUX" ]; then + tmux a -t default || exec tmux new -s default && exit; +fi + diff --git a/archive/provisioning/legacy/profiled-tsys-shell.sh b/archive/provisioning/legacy/profiled-tsys-shell.sh new file mode 100644 index 0000000..8df5b84 --- /dev/null +++ b/archive/provisioning/legacy/profiled-tsys-shell.sh @@ -0,0 +1,2 @@ +# shellcheck shell=bash disable=SC2148 # sourced .bashrc profile fragment +export HISTTIMEFORMAT="%m/%d/%Y %T " \ No newline at end of file diff --git a/archive/provisioning/legacy/prox7.sh b/archive/provisioning/legacy/prox7.sh new file mode 100644 index 0000000..78ad59f --- /dev/null +++ b/archive/provisioning/legacy/prox7.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +rm -f /etc/apt/sources.list.d/* +echo "deb https://download.proxmox.com/debian/pve bookworm pve-no-subscription" > /etc/apt/sources.list.d/pve-install-repo.list +wget https://download.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg +apt update && apt -y full-upgrade +apt-get -y install ifupdown2 ipmitool ethtool net-tools lshw + +#curl -s http://dl.turnsys.net/newSrv.sh|/bin/bash \ No newline at end of file diff --git a/archive/provisioning/scripts/up2date.sh b/archive/provisioning/scripts/up2date.sh new file mode 100644 index 0000000..5370536 --- /dev/null +++ b/archive/provisioning/scripts/up2date.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +echo "Running apt-get update" +export DEBIAN_FRONTEND="noninteractive" && apt-get -qq --yes update + +echo "Running apt-get dist-upgrade" +export DEBIAN_FRONTEND="noninteractive" && apt-get -qq --yes dist-upgrade + +echo "Running apt-get upgrade" +export DEBIAN_FRONTEND="noninteractive" && apt-get -qq --yes upgrade + + +echo "Running apt-get purge" +export DEBIAN_FRONTEND="noninteractive" && apt-get -qq --purge autoremove --yes +export DEBIAN_FRONTEND="noninteractive" && apt-get -qq autoclean --yes + diff --git a/dcinfra/console/README.md b/dcinfra/console/README.md new file mode 100644 index 0000000..e34b8cc --- /dev/null +++ b/dcinfra/console/README.md @@ -0,0 +1,130 @@ +# Console Management (ser2net + conman) + +Network-accessible serial console management for all production network +switches and routers, running on **pfv-tsys4** (storage server). + +## Architecture + +``` +USB-DB9 adapters → udev symlinks (/dev/consoles/) → ser2net telnet(rfc2217) TCP → conman (logging + multiplexing) +``` + +ser2net owns the physical serial devices and exposes them on TCP ports +using the **telnet(rfc2217) protocol** bound to the **Tailscale interface +only** (`100.70.77.93:200X`). conman connects to those TCP ports via +telnet for session logging, output capture, and multi-user console +sharing. + +**Why telnet(rfc2217)?** The serial devices send ` + ` (LF+CR) line +endings instead of standard ` +`. Raw TCP transport caused conman's +telnet NVT to strip bare CR characters, producing stair-stepped output. +With telnet(rfc2217) on both sides, binary mode is negotiated and CR/LF +translation is handled correctly by the telnet layer. + +**conman and ser2net do NOT share ports** — only one process can open a +serial device at a time. ser2net owns the physical device; conman connects +over TCP. + +## The USB Enumeration Problem (SOLVED) + +The 9 Prolific USB-to-DB9 adapters (`067b:2303`) on pfv-tsys4 have **no +unique USB serial numbers** and get assigned `/dev/ttyUSB0-8` based on +enumeration order, which shifts on every boot. This made the old +`/root/conmap` + manual `screen` workflow break after every reboot. + +**Fix:** udev rules pin each adapter by its **ID_PATH** (physical USB port +topology), which is stable across reboots regardless of enumeration order. +Each adapter gets a named symlink in `/dev/consoles/` that never changes. + +The udev rules are generated from `mapping.txt`, which maps each adapter's +ID_PATH to a console name and TCP port. To re-map after physically moving +an adapter, update `mapping.txt` and re-run `setup.sh`. + +**Fallback:** if udev trigger doesn't create symlinks for already-discovered +devices (common on first run), `setup.sh` creates them manually by matching +ID_PATH. On subsequent boots, udev creates them automatically. + +## Port Assignments + +| TCP Port | Console Name | ID_PATH | Description | +|----------|-------------|---------|-------------| +| 2001 | pfv-core-sw01 | usb-0:1.5.4.4 | Dell PowerConnect 5448 (core switch) | +| 2002 | pfv-tor3-mgmt | usb-0:1.6.3.1 | Rack 3 management TOR switch | +| 2003 | pfv-tor3-stor | usb-0:1.6.3.3.2 | Rack 3 storage TOR switch | +| 2004 | pfv-rrinfra-rtr | usb-0:1.6.3.3.1 | Cisco router (rrinfra) | +| 2005 | pfv-r2-tor-top | usb-0:1.6.3.3.3 | Rack 2 top-of-rack switch | +| 2006 | subodev-torsw | usb-0:1.5.4.1 | Suborbital device TOR switch | +| 2007 | pfv-r2-sw | usb-0:1.6.3.2 | Rack 2 old Dell switch | + +All ports listen on the Tailscale IP (`100.70.77.93`) using telnet(rfc2217). + +## Scripts + +| Script | Purpose | +|--------|---------| +| [`mapping.txt`](mapping.txt) | Source of truth: TCP port ↔ ID_PATH ↔ name ↔ baud | +| [`generate-config.sh`](generate-config.sh) | Generates udev rules, ser2net.yaml, conman.conf from mapping.txt | +| [`setup.sh`](setup.sh) | Full deploy: generate configs, create symlinks, restart services | +| [`discover.sh`](discover.sh) | Read-only discovery of USB adapters, existing config, services | + +## Usage + +### Connect to a console + +**Primary method — conman client (with logging + multiplexing):** + +```bash +# From any Tailscale-connected workstation: +conman -d pfv-tsys4:7890 -f pfv-core-sw01 # connect to console +conman -d pfv-tsys4:7890 -q # list all consoles +``` + +Escape sequence: `&.` to disconnect, `&?` for help. + +**Direct telnet (emergency only — conflicts with conman):** + +```bash +# Direct telnet to ser2net works ONLY when conmand is stopped, because +# conmand maintains persistent connections to all 7 TCP ports. Use: +ssh pfv-tsys4 'systemctl stop conmand' +telnet pfv-tsys4 2001 # pfv-core-sw01 +ssh pfv-tsys4 'systemctl start conmand' # restart when done +``` + +**Do NOT use telnet while conmand is running** — conmand will reconnect +and kick your telnet session immediately ("Connection closed by foreign host"). +The correct workflow is conman client → conmand → ser2net → device. + +### Re-deploy after changing mapping.txt + +```bash +PROX_HOST=pfv-tsys4 bash tests/remote.sh prox 'bash /root/console/setup.sh' +``` + +### Find the ID_PATH for a new adapter + +```bash +PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file console/discover.sh +``` + +Then match the new adapter's ID_PATH to its physical location and add a line +to `mapping.txt`. + +## Files on pfv-tsys4 + +| File | Purpose | +|------|---------| +| `/etc/udev/rules.d/99-console-ports.rules` | Stable symlinks by ID_PATH | +| `/etc/ser2net.yaml` | ser2net config (telnet rfc2217 TCP ports → serial symlinks) | +| `/etc/conman.conf` | conman config (CONSOLE entries between markers) | +| `/etc/systemd/system/conmand.service` | systemd unit for conmand | +| `/root/console/mapping.txt` | Copy of the source-of-truth mapping | +| `/root/console/setup.sh` | Setup script (re-runnable) | +| `/root/console/generate-config.sh` | Config generator | + +## Old workflow (replaced) + +The old `/root/conmap` file and manual `screen` sessions are no longer +needed. The new setup is fully automated and survives reboots. diff --git a/dcinfra/console/discover.sh b/dcinfra/console/discover.sh new file mode 100644 index 0000000..34138b9 --- /dev/null +++ b/dcinfra/console/discover.sh @@ -0,0 +1,103 @@ +#!/usr/bin/bash +# shellcheck disable=SC2010,SC2012 # diagnostic script; ls|grep/ls -la on sysfs & log dirs is intentional for human-readable output +# +# console/discover.sh — READ-ONLY discovery of console setup on pfv-tsys4 +# +# Usage: PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file console/discover.sh +# +# This script is strictly read-only. No writes to the system. +# +set -uo pipefail + +echo "============================================" +echo " Console Setup Discovery" +echo " Host: $(hostname)" +echo " Date: $(date)" +echo " READ-ONLY" +echo "============================================" + +echo "" +echo "=== 1. USB devices ===" +lsusb 2>/dev/null || echo "(lsusb not available)" + +echo "" +echo "=== 2. All ttyUSB* devices (with major/minor) ===" +ls -la /dev/ttyUSB* 2>/dev/null || echo "(no /dev/ttyUSB* devices)" + +echo "" +echo "=== 3. USB-serial driver bindings ===" +echo "-- pl2303 --" +ls -la /sys/bus/usb-serial/drivers/pl2303/ 2>/dev/null | grep -v '^total\|^d\|module\|new_id\|uevent' || echo "(none)" +echo "-- cp210x --" +ls -la /sys/bus/usb-serial/drivers/cp210x/ 2>/dev/null | grep -v '^total\|^d\|module\|new_id\|uevent' || echo "(none)" +echo "-- ftdi_sio --" +ls -la /sys/bus/usb-serial/drivers/ftdi_sio/ 2>/dev/null | grep -v '^total\|^d\|module\|new_id\|uevent' || echo "(none)" +echo "-- ch341 --" +ls -la /sys/bus/usb-serial/drivers/ch341/ 2>/dev/null | grep -v '^total\|^d\|module\|new_id\|uevent' || echo "(none)" + +echo "" +echo "=== 4. USB serial adapter details (vendor/model/serial per port) ===" +for tty in /dev/ttyUSB*; do + [ -e "$tty" ] || continue + echo "--- $tty ---" + udevadm info -q all -n "$tty" 2>/dev/null | grep -E 'ID_VENDOR_ID|ID_MODEL_ID|ID_SERIAL|ID_USB_DRIVER|ID_PATH=' | sed 's/^/ /' +done + +echo "" +echo "=== 5. Existing /root/conmap ===" +if [ -f /root/conmap ]; then + cat /root/conmap +else + echo "(no /root/conmap)" +fi +ls -la /root/conmap* 2>/dev/null + +echo "" +echo "=== 6. Screen sessions (running) ===" +screen -ls 2>&1 || echo "(screen not running or not installed)" + +echo "" +echo "=== 7. Existing screen wrappers/scripts in /root ===" +ls -la /root/ 2>/dev/null | grep -iE 'screen|con|console|tty|usb' || echo "(no obvious console scripts in /root)" + +echo "" +echo "=== 8. ser2net ===" +which ser2net 2>/dev/null || echo "(ser2net not installed)" +dpkg -l ser2net 2>/dev/null | tail -2 || echo "(ser2net not in dpkg)" +cat /etc/ser2net/ser2net.yaml 2>/dev/null || cat /etc/ser2net.conf 2>/dev/null || cat /etc/ser2net/ser2net.conf 2>/dev/null || echo "(no ser2net config)" +systemctl is-active ser2net 2>/dev/null || echo "(ser2net service not found)" + +echo "" +echo "=== 9. conman ===" +which conman 2>/dev/null || echo "(conman not installed)" +which conmand 2>/dev/null || echo "(conmand not installed)" +dpkg -l conman 2>/dev/null | tail -2 || echo "(conman not in dpkg)" +echo "--- /etc/conman.conf (console lines only) ---" +grep -nE 'CONSOLE|SERVER|LOG|SERIAL|DEV|BAUD|^[^#].*name=' /etc/conman.conf 2>/dev/null | head -60 || echo "(no conman.conf or no console entries)" +echo "--- conmand service ---" +systemctl is-active conmand 2>/dev/null || echo "(conmand not running)" +systemctl is-enabled conmand 2>/dev/null || echo "(conmand not enabled)" + +echo "" +echo "=== 10. Existing console logs ===" +ls -la /var/log/conman/ 2>/dev/null | head -20 || echo "(no /var/log/conman)" +ls -la /var/consoles/ 2>/dev/null | head -20 || echo "(no /var/consoles)" + +echo "" +echo "=== 11. udev rules for ttyUSB ===" +grep -r ttyUSB /etc/udev/rules.d/ 2>/dev/null || echo "(no udev rules for ttyUSB)" +grep -r 'console' /etc/udev/rules.d/ 2>/dev/null | head -10 || true + +echo "" +echo "=== 12. expect availability ===" +command -v expect && expect -v 2>&1 || echo "expect: NOT installed" +command -v socat && socat -V 2>&1 | head -1 || echo "socat: NOT installed" + +echo "" +echo "=== 13. Ports in use (2001-2099, 7000-7999, 7820-7899) ===" +ss -tlnp 2>/dev/null | grep -E ':200[0-9]|:700[0-9]|:782[0-9]|:789[0-9]' || echo "(no relevant ports listening)" + +echo "" +echo "============================================" +echo " Discovery complete (read-only)." +echo "============================================" diff --git a/dcinfra/console/generate-config.sh b/dcinfra/console/generate-config.sh new file mode 100644 index 0000000..a61b9ad --- /dev/null +++ b/dcinfra/console/generate-config.sh @@ -0,0 +1,254 @@ +#!/usr/bin/bash +# +# console/generate-config.sh — generate udev rules + ser2net.yaml + conman.conf +# +# Reads console/mapping.txt (the source of truth) and generates all three +# config files. This is the fix for the USB enumeration shift problem: +# +# 1. udev rules pin each adapter by its STABLE ID_PATH (physical USB port) +# to a named symlink like /dev/consoles/pfv-core-sw01 +# 2. ser2net opens those stable symlinks and exposes them on TCP ports +# (2001, 2002, ...) bound to the Tailscale IP +# 3. conman connects to those TCP ports for logging + multiplexing +# +# Run this script ON the target host. It writes to: +# /etc/udev/rules.d/99-console-ports.rules +# /etc/ser2net.yaml +# /etc/conman/console-consoles.conf (included by /etc/conman.conf) +# +# Usage: +# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file console/generate-config.sh +# +# Environment overrides: +# MAPPING_FILE — path to mapping.txt (default: auto-detect next to this script) +# TS_IP — Tailscale IP to bind ser2net on (default: auto-detect) +# CONMAN_LOGDIR — conman log directory (default: /var/log/conman) +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MAPPING_FILE="${MAPPING_FILE:-$SCRIPT_DIR/mapping.txt}" +CONMAN_LOGDIR="${CONMAN_LOGDIR:-/var/log/conman}" + +UDEV_RULES="/etc/udev/rules.d/99-console-ports.rules" +SER2NET_CONF="/etc/ser2net.yaml" +CONMAN_CONF="/etc/conman.conf" + +echo "============================================" +echo " Console Config Generator" +echo " Host: $(hostname) $(date)" +echo "============================================" + +# --- Locate mapping file --- +# When run via remote.sh prox-file, $0 is bash and $SCRIPT_DIR may be wrong. +# Search common locations. +if [ ! -f "$MAPPING_FILE" ]; then + for candidate in \ + "/root/console/mapping.txt" \ + "/tmp/mapping.txt" \ + "$(dirname "$0")/mapping.txt"; do + if [ -f "$candidate" ]; then + MAPPING_FILE="$candidate" + break + fi + done +fi + +if [ ! -f "$MAPPING_FILE" ]; then + echo "FATAL: mapping file not found. Tried: $MAPPING_FILE" + echo "Copy mapping.txt to the target host first." + exit 1 +fi +echo " Mapping file: $MAPPING_FILE" + +# --- Auto-detect Tailscale IP --- +if [ -z "${TS_IP:-}" ]; then + TS_IP=$(tailscale ip -4 2>/dev/null || true) + if [ -z "$TS_IP" ]; then + echo "FATAL: could not auto-detect Tailscale IP. Set TS_IP manually." + exit 1 + fi +fi +echo " Tailscale IP: $TS_IP" +echo " ser2net will bind to: $TS_IP" + +# --- Parse mapping file (skip comments and blank lines) --- +echo "" +echo "--- Parsing mapping file ---" +ENTRIES=() +while IFS= read -r line; do + # Skip comments and blank lines + line="${line%%#*}" + line="$(echo "$line" | xargs)" # trim whitespace + [ -z "$line" ] && continue + ENTRIES+=("$line") + echo " $line" +done < "$MAPPING_FILE" + +if [ "${#ENTRIES[@]}" -eq 0 ]; then + echo "FATAL: no entries found in mapping file." + exit 1 +fi + +echo "" +echo " ${#ENTRIES[@]} console ports configured." + +# ============================================================ +# 1. Generate udev rules +# ============================================================ +echo "" +echo "--- [1/3] Generating udev rules: $UDEV_RULES ---" + +cat > "$UDEV_RULES" <<'UDEV_HEADER' +# Stable symlinks for USB-DB9 console adapters +# Generated by console/generate-config.sh +# DO NOT EDIT — edit mapping.txt and re-run generate-config.sh +# +# These rules pin each adapter to a named symlink based on its physical +# USB port path (ID_PATH), which is stable across reboots regardless of +# enumeration order. This is the fix for the "USB adapters shift on reboot" +# problem. +# +# To find the ID_PATH for a device: +# udevadm info -q all -n /dev/ttyUSBN | grep ID_PATH +UDEV_HEADER + +for entry in "${ENTRIES[@]}"; do + IFS='|' read -r tcp_port name id_path baud comment <<< "$entry" + # Build the full ID_PATH match. The mapping stores a substring like "usb-0:1.5.4.4" + # The actual ID_PATH is like "pci-0000:00:1a.0-usb-0:1.5.4.4:1.0" + # We match on the substring to be portable across PCI bus changes. + { + echo "" + echo "# $name (TCP $tcp_port): $comment" + echo "SUBSYSTEM==\"tty\", ENV{ID_PATH}==\"*$id_path*\", SYMLINK+=\"consoles/$name\"" + } >> "$UDEV_RULES" +done + +echo " Written: $UDEV_RULES" +echo " Symlinks: /dev/consoles/ for each device" + +# ============================================================ +# 2. Generate ser2net.yaml +# ============================================================ +echo "" +echo "--- [2/3] Generating ser2net config: $SER2NET_CONF ---" + +# Backup existing config if not already backed up +if [ -f "$SER2NET_CONF" ] && [ ! -f "${SER2NET_CONF}.orig" ]; then + cp "$SER2NET_CONF" "${SER2NET_CONF}.orig" + echo " Backed up original to ${SER2NET_CONF}.orig" +fi + +{ + echo "%YAML 1.1" + echo "---" + echo "# ser2net configuration for pfv-tsys4 console ports" + echo "# Generated by console/generate-config.sh on $(date)" + echo "#" + echo "# All ports use telnet(rfc2217) accepter so conman and telnet clients" + echo "# negotiate proper telnet binary mode — this prevents CR stripping" + printf '%s\n' "# and stair-stepping on devices that send \\n\\r (LF+CR) line endings." + echo "# Ports bound to Tailscale IP ($TS_IP) for secure remote access." + echo "#" + echo "# Direct telnet: telnet $TS_IP 2001" + echo "# Via conman: conman -f " + echo "" + printf '%s\n' "define: &banner \\r\\nPFV console port \\p device \\d [\\B]\\r\\n\\r\\n" + echo "" + + for entry in "${ENTRIES[@]}"; do + IFS='|' read -r tcp_port name id_path baud comment <<< "$entry" + # ser2net connection block — telnet(rfc2217) accepter so conman and + # telnet clients negotiate proper telnet binary mode. This prevents + # CR stripping that occurs with raw TCP + conman's telnet NVT. + echo "connection: &con${tcp_port}" + echo " accepter: telnet(rfc2217),tcp,${TS_IP},${tcp_port}" + echo " enable: on" + echo " options:" + echo " banner: *banner" + echo " kickolduser: true" + echo " telnet-brk-on-sync: true" + echo " connector: serialdev," + echo " /dev/consoles/${name}," + echo " ${baud},local" + echo "" + done +} > "$SER2NET_CONF" + +echo " Written: $SER2NET_CONF" +echo " ${#ENTRIES[@]} TCP ports configured ($TS_IP:2001-20XX)" + +# ============================================================ +# 3. Write conman console entries directly into conman.conf +# ============================================================ +# conman 0.3.x does NOT support the 'include' directive, so we write +# CONSOLE entries directly into /etc/conman.conf between idempotent markers. +echo "" +echo "--- [3/3] Writing conman consoles into $CONMAN_CONF ---" + +# Ensure logdir exists +mkdir -p "$CONMAN_LOGDIR" 2>/dev/null || true + +# Ensure LOGDIR is set in conman.conf (server-level directive for log file paths) +if ! grep -qiE '^\s*server\s+logdir\s*=' "$CONMAN_CONF" 2>/dev/null; then + # Insert near the top, after the first SERVER directives + sed -i "1i\\server logdir = \"$CONMAN_LOGDIR\"" "$CONMAN_CONF" + echo " Added server logdir = \"$CONMAN_LOGDIR\" to $CONMAN_CONF" +fi + +# Ensure loopback=off so conmand is reachable over Tailscale (not localhost-only) +if ! grep -qiE '^\s*server\s+loopback\s*=' "$CONMAN_CONF" 2>/dev/null; then + sed -i "/^server logdir/a server loopback=off" "$CONMAN_CONF" + echo " Added server loopback=off to $CONMAN_CONF (enables remote access)" +fi + +# Remove any previous auto-generated block (between markers) +# Then append the new block +MARKER_BEGIN="# BEGIN PFV CONSOLE DEFINITIONS (auto-generated — do not edit between markers)" +MARKER_END="# END PFV CONSOLE DEFINITIONS" + +# Strip old block if present +if grep -q "$MARKER_BEGIN" "$CONMAN_CONF" 2>/dev/null; then + sed -i "/$MARKER_BEGIN/,/$MARKER_END/d" "$CONMAN_CONF" + echo " Removed previous console definitions." +fi + +# Append new block +{ + echo "" + echo "$MARKER_BEGIN" + echo "# Generated by console/generate-config.sh on $(date)" + echo "# Each console connects to a ser2net TCP port via telnet protocol." + echo "# ser2net uses telnet(rfc2217) accepter so binary mode is negotiated" + echo "# and CR/LF translation is handled correctly by the telnet NVT layer." + echo "# Access: conman -f " + echo "" + for entry in "${ENTRIES[@]}"; do + IFS='|' read -r tcp_port name id_path baud comment <<< "$entry" + echo "CONSOLE name=\"${name}\" dev=\"${TS_IP}:${tcp_port}\" log=\"${name}.log\" logopts=\"timestamp\"" + done + echo "$MARKER_END" +} >> "$CONMAN_CONF" + +CONSOLE_COUNT=$(grep -c "^CONSOLE " "$CONMAN_CONF" 2>/dev/null || echo 0) +echo " Written $CONSOLE_COUNT CONSOLE entries to $CONMAN_CONF" + +# ============================================================ +# Summary +# ============================================================ +echo "" +echo "============================================" +echo " Configuration generated successfully." +echo "" +echo " Files written:" +echo " $UDEV_RULES ($(wc -l < "$UDEV_RULES") lines)" +echo " $SER2NET_CONF ($(wc -l < "$SER2NET_CONF") lines)" +echo " $CONMAN_CONF (CONSOLE entries appended between markers)" +echo "" +echo " Next steps:" +echo " 1. Reload udev: udevadm control --reload-rules && udevadm trigger" +echo " 2. Restart ser2net: systemctl restart ser2net" +echo " 3. Start conman: systemctl enable --now conmand" +echo " 4. Or run: bash $(basename "$0" .sh | sed 's/generate-config/setup/') .sh" +echo "============================================" diff --git a/dcinfra/console/mapping.txt b/dcinfra/console/mapping.txt new file mode 100644 index 0000000..4a3fba9 --- /dev/null +++ b/dcinfra/console/mapping.txt @@ -0,0 +1,29 @@ +# console/mapping.txt — Source of Truth for console port assignments +# +# Format: |||| +# +# Delimiter is | (pipe) because ID_PATH values contain colons. +# +# - tcp_port: TCP port ser2net listens on (also the conman console name suffix) +# - name: Device name (used for /dev/console/ symlink, conman console name) +# - id_path_substring: Stable USB physical path from `udevadm info -q all -n /dev/ttyUSBN | grep ID_PATH` +# These are STABLE across reboots as long as adapters aren't moved +# to different physical USB ports. +# - baud: Serial baud rate (9600n81 = 9600 8N1, no flow control) +# - comment: Free-form description +# +# To RE-MAP after physically moving an adapter: +# 1. Run: bash console/discover.sh (find the new ID_PATH for the device) +# 2. Update the id_path_substring in this file +# 3. Run: bash console/generate-config.sh && udevadm trigger && systemctl restart ser2net conmand +# +2001|pfv-core-sw01|usb-0:1.5.4.4|9600n81|Dell PowerConnect 5448 (core switch) +2002|pfv-tor3-mgmt|usb-0:1.6.3.1|9600n81|Rack 3 management TOR switch +2003|pfv-tor3-stor|usb-0:1.6.3.3.2|9600n81|Rack 3 storage TOR switch +2004|pfv-rrinfra-rtr|usb-0:1.6.3.3.1|9600n81|Cisco router (rrinfra) +2005|pfv-r2-tor-top|usb-0:1.6.3.3.3|9600n81|Rack 2 top-of-rack switch +2006|subodev-torsw|usb-0:1.5.4.1|9600n81|Suborbital device TOR switch +2007|pfv-r2-sw|usb-0:1.6.3.2|9600n81|Rack 2 old Dell switch +# Unassigned (no device detected): +# 2008|spare-1|usb-0:1.6.3.4|9600n81|Empty / spare +# 2009|spare-2|usb-0:1.6.3.3.4|9600n81|Empty / spare diff --git a/dcinfra/console/query-remote.sh b/dcinfra/console/query-remote.sh new file mode 100644 index 0000000..cd539e6 --- /dev/null +++ b/dcinfra/console/query-remote.sh @@ -0,0 +1,62 @@ +#!/usr/bin/bash +# +# console/query-remote.sh — install conman client and connect to a console +# on pfv-tsys4 over Tailscale. +# +# Usage: +# bash console/query-remote.sh # list consoles +# bash console/query-remote.sh pfv-core-sw01 # connect to a console +# +set -euo pipefail + +REMOTE_HOST="${REMOTE_HOST:-pfv-tsys4}" +REMOTE_PORT="${REMOTE_PORT:-7890}" + +echo "============================================" +echo " Conman Remote Console Access" +echo " Server: ${REMOTE_HOST}:${REMOTE_PORT} (Tailscale)" +echo "============================================" + +# --- 1. Install conman client if missing --- +if ! command -v conman >/dev/null 2>&1; then + echo "" + echo "--- Installing conman client ---" + if sudo -n true 2>/dev/null; then + sudo apt-get update -qq && sudo apt-get install -y -qq conman + else + echo " Passwordless sudo not available. Please run:" + echo " sudo apt-get update && sudo apt-get install -y conman" + echo " Then re-run this script." + exit 1 + fi +else + echo " conman client already installed." +fi + +# --- 2. Verify connectivity --- +echo "" +echo "--- Connectivity check ---" +if timeout 3 bash -c "echo > /dev/tcp/${REMOTE_HOST}/${REMOTE_PORT}" 2>/dev/null; then + echo " [OK] ${REMOTE_HOST}:${REMOTE_PORT} reachable" +else + echo " [FAIL] Cannot reach ${REMOTE_HOST}:${REMOTE_PORT}" + echo " Is Tailscale up? Is conmand running on ${REMOTE_HOST}?" + exit 1 +fi + +# --- 3. List or connect --- +CONSOLE="${1:-}" +if [ -z "$CONSOLE" ]; then + echo "" + echo "--- Available consoles ---" + conman -d "${REMOTE_HOST}:${REMOTE_PORT}" -q + echo "" + echo "To connect: bash $0 " + echo " e.g: bash $0 pfv-core-sw01" +else + echo "" + echo "--- Connecting to: $CONSOLE ---" + echo " Escape sequence: &. (to disconnect)" + echo "" + conman -d "${REMOTE_HOST}:${REMOTE_PORT}" -f "$CONSOLE" +fi diff --git a/dcinfra/console/setup.sh b/dcinfra/console/setup.sh new file mode 100644 index 0000000..5da0536 --- /dev/null +++ b/dcinfra/console/setup.sh @@ -0,0 +1,217 @@ +#!/usr/bin/bash +# shellcheck disable=SC2010 # diagnostic; ls|grep on /dev listing is intentional +# +# console/setup.sh — deploy console management on pfv-tsys4 +# +# Orchestrates the full setup: +# 1. Ensures ser2net + conman are installed +# 2. Copies mapping.txt to the target host (if running remotely) +# 3. Runs generate-config.sh to produce udev rules + ser2net.yaml + conman.conf +# 4. Reloads udev, creates /dev/consoles/ symlinks +# 5. Restarts ser2net (TCP ports on Tailscale IP) +# 6. Enables + starts conmand (logging + multiplexing) +# 7. Verifies +# +# This script is IDEMPOTENT — safe to run multiple times. +# +# Usage: +# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file console/setup.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +echo "============================================" +echo " Console Management Setup" +echo " Host: $(hostname) $(date)" +echo "============================================" + +# --- 1. Install dependencies --- +echo "" +echo "--- [1/7] Checking dependencies ---" +NEED_INSTALL=() +dpkg -l ser2net 2>/dev/null | grep -q '^ii' && echo " ser2net: installed" || NEED_INSTALL+=(ser2net) +dpkg -l conman 2>/dev/null | grep -q '^ii' && echo " conman: installed" || NEED_INSTALL+=(conman) + +if [ "${#NEED_INSTALL[@]}" -gt 0 ]; then + echo " Installing: ${NEED_INSTALL[*]}" + apt-get update -qq + apt-get install -y -qq "${NEED_INSTALL[@]}" +else + echo " All dependencies present." +fi + +# --- 2. Ensure mapping file is available --- +echo "" +echo "--- [2/7] Locating mapping file ---" + +MAPPING_FILE="" +for candidate in \ + "$SCRIPT_DIR/mapping.txt" \ + "$(dirname "$0")/mapping.txt" \ + "/root/console/mapping.txt" \ + "/tmp/mapping.txt"; do + if [ -f "$candidate" ]; then + MAPPING_FILE="$candidate" + break + fi +done + +if [ -z "$MAPPING_FILE" ]; then + echo "FATAL: mapping.txt not found. Copy it to the target host." + exit 1 +fi +echo " Using: $MAPPING_FILE" + +# --- 3. Generate configs --- +echo "" +echo "--- [3/7] Generating configs ---" +export MAPPING_FILE +bash "$(dirname "$0")/generate-config.sh" 2>&1 || bash "$SCRIPT_DIR/generate-config.sh" 2>&1 || { + echo "FATAL: generate-config.sh failed." + exit 1 +} + +# --- 4. Reload udev + create symlinks --- +echo "" +echo "--- [4/7] Reloading udev rules ---" +udevadm control --reload-rules +# Try trigger first (works on some systems) +for tty in /sys/class/tty/ttyUSB*; do + [ -e "$tty" ] && udevadm trigger --action=add "$tty" 2>/dev/null || true +done +# Also try writing to uevent (forces udev reprocessing) +for tty in /sys/class/tty/ttyUSB*; do + [ -e "$tty/uevent" ] && echo "add" > "$tty/uevent" 2>/dev/null || true +done +sleep 2 + +# FALLBACK: if udev symlinks don't exist (common when devices are already +# discovered — udev trigger doesn't always re-create symlinks for existing +# devices), create them manually by matching ID_PATH. The udev rules will +# handle future boots/hotplugs automatically. +if [ ! -d /dev/consoles ] || [ -z "$(ls /dev/consoles/ 2>/dev/null)" ]; then + echo " udev trigger didn't create symlinks. Creating manually..." + mkdir -p /dev/consoles + while IFS= read -r line; do + line="${line%%#*}" + line="$(echo "$line" | xargs)" + [ -z "$line" ] && continue + IFS='|' read -r _ name id_path _ _ <<< "$line" + # Find the ttyUSB whose ID_PATH contains the mapping's id_path substring + for tty in /dev/ttyUSB*; do + [ -e "$tty" ] || continue + DEV_IDPATH=$(udevadm info -q property -n "$tty" 2>/dev/null | grep ^ID_PATH= | cut -d= -f2) + if echo "$DEV_IDPATH" | grep -q "$id_path"; then + ln -sf "$tty" "/dev/consoles/$name" + echo " ln -s $tty -> /dev/consoles/$name" + break + fi + done + done < "$MAPPING_FILE" +fi + +echo " Stable symlinks:" +ls -la /dev/consoles/ 2>/dev/null | grep -v '^total\|^d' | sed 's/^/ /' || echo " (none created)" + +# Verify each symlink resolves +echo "" +echo " Symlink verification:" +while IFS= read -r line; do + line="${line%%#*}" + line="$(echo "$line" | xargs)" + [ -z "$line" ] && continue + IFS='|' read -r _ name id_path _ _ <<< "$line" + if [ -e "/dev/consoles/$name" ]; then + TARGET=$(readlink -f "/dev/consoles/$name") + echo " [OK] /dev/consoles/$name -> $TARGET" + else + echo " [MISSING] /dev/consoles/$name (adapter unplugged or ID_PATH changed)" + fi +done < "$MAPPING_FILE" + +# --- 5. Restart ser2net --- +echo "" +echo "--- [5/7] Restarting ser2net ---" +systemctl enable ser2net +systemctl restart ser2net +sleep 2 + +if systemctl is-active --quiet ser2net; then + echo " ser2net is running (telnet rfc2217 accepters)." + TS_IP=$(tailscale ip -4 2>/dev/null || echo "127.0.0.1") + echo " Listening ports:" + ss -tlnp | grep ser2net | grep -oE "${TS_IP}:[0-9]+" | sort -t: -k2 -n | sed 's/^/ /' +else + echo " WARNING: ser2net failed to start. Checking journal..." + journalctl -u ser2net --no-pager -n 20 +fi + +# --- 6. Enable + start conmand --- +echo "" +echo "--- [6/7] Starting conmand ---" + +# conman package on Debian may not ship a systemd unit. Create one if missing. +if ! systemctl cat conmand >/dev/null 2>&1; then + echo " No systemd unit for conmand — creating one..." + cat > /etc/systemd/system/conmand.service <<'CONMAND_UNIT' +[Unit] +Description=ConMan (Console Manager) +After=network.target ser2net.service +Requires=ser2net.service + +[Service] +Type=forking +ExecStart=/usr/sbin/conmand -c /etc/conman.conf +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target +CONMAND_UNIT + systemctl daemon-reload + echo " Created /etc/systemd/system/conmand.service" +fi + +# Kill any manually-started conmand first +pkill -x conmand 2>/dev/null || true +sleep 1 + +systemctl enable conmand 2>/dev/null || true +systemctl restart conmand 2>/dev/null || true +sleep 2 + +if systemctl is-active --quiet conmand; then + echo " conmand is running." + echo " Consoles:" + conman -q 2>&1 | sed 's/^/ /' || true +else + echo " WARNING: conmand failed to start. Checking journal..." + journalctl -u conmand --no-pager -n 20 2>/dev/null || true + # Try manual start as fallback + echo " Attempting manual start..." + /usr/sbin/conmand -c /etc/conman.conf 2>&1 || true +fi + +# --- 7. Summary --- +echo "" +echo "--- [7/7] Setup complete ---" +echo "" +echo " ser2net + conman architecture (telnet rfc2217):" +echo " ser2net owns serial devices, exposes telnet(rfc2217) TCP ports" +echo " conman connects via telnet for logging + multiplexing" +echo "" +echo " Connect from any Tailscale workstation:" +echo " conman -d pfv-tsys4:7890 -f pfv-core-sw01" +echo " conman -d pfv-tsys4:7890 -q # list consoles" +echo "" +echo " Direct telnet (emergency, conflicts with conman):" +echo " ssh pfv-tsys4 'systemctl stop conmand'" +echo " telnet pfv-tsys4 2001" +echo " ssh pfv-tsys4 'systemctl start conmand'" +echo "" +echo " To regenerate after changing mapping.txt:" +echo " bash generate-config.sh" +echo " udevadm trigger" +echo " systemctl restart ser2net conmand" +echo "============================================" diff --git a/dcinfra/console/validate-conman.sh b/dcinfra/console/validate-conman.sh new file mode 100644 index 0000000..10f1c00 --- /dev/null +++ b/dcinfra/console/validate-conman.sh @@ -0,0 +1,89 @@ +#!/usr/bin/bash +# shellcheck disable=SC2012,SC2001 # diagnostic script; ls -la listings and sed line-prefixing are intentional +# +# console/validate-conman.sh — verify conman can actually reach devices via +# ser2net TCP ports and is capturing log output to files. +# +# This tests the real data path: conman → TCP 200X → ser2net → /dev/consoles/X → device +# +set -uo pipefail + +TS_IP=$(tailscale ip -4) +LOGDIR="/var/log/conman" + +echo "============================================" +echo " Conman Data Path + Log Validation" +echo " Host: $(hostname) TS IP: $TS_IP" +echo "============================================" + +echo "" +echo "--- 1. conman.conf log settings ---" +grep -E "logdir|LOGDIR|^GLOBAL LOG" /etc/conman.conf 2>/dev/null | grep -v "^#" || echo " (no explicit logdir — defaults to /var/log/conman)" +echo " Log dir: $LOGDIR" +ls -la "$LOGDIR"/ 2>/dev/null | head -15 || echo " ($LOGDIR does not exist yet)" + +echo "" +echo "--- 2. CONSOLE entries: each has a log= directive? ---" +# Extract the auto-generated block and check each CONSOLE line has log= +sed -n '/BEGIN PFV CONSOLE/,/END PFV CONSOLE/p' /etc/conman.conf | grep "^CONSOLE" | while read -r line; do + name=$(echo "$line" | sed -n 's/.*name="\([^"]*\)".*/\1/p') + if echo "$line" | grep -q 'log='; then + logfile=$(echo "$line" | sed -n 's/.*log="\([^"]*\)".*/\1/p') + echo " [OK] $name → log=$logfile" + else + echo " [FAIL] $name has NO log= directive" + fi +done + +echo "" +echo "--- 3. Trigger log capture: connect to each console briefly ---" +# conman -e changes the escape char. We use -j (join, read-only) with a timeout. +# Actually, conman doesn't have a built-in "connect for N seconds" — but conmand +# connects to each device ON STARTUP and keeps the connection open for logging. +# The log files should already be created. Let's check timestamps. + +echo " conmand connects to all consoles on startup. Checking if logs exist..." +echo "" +echo "--- 4. Log file inventory ---" +for name in pfv-core-sw01 pfv-tor3-mgmt pfv-tor3-stor pfv-rrinfra-rtr pfv-r2-tor-top subodev-torsw pfv-r2-sw; do + logfile="$LOGDIR/${name}.log" + if [ -f "$logfile" ]; then + SIZE=$(stat -c%s "$logfile" 2>/dev/null || echo 0) + MTIME=$(stat -c%y "$logfile" 2>/dev/null | cut -d. -f1) + echo " [OK] $logfile ($SIZE bytes, modified $MTIME)" + else + echo " [MISSING] $logfile — conmand may not be writing yet" + fi +done + +echo "" +echo "--- 5. conmand connection status (journal) ---" +# conmand logs connection attempts/errors to syslog +journalctl -u conmand --no-pager -n 50 2>/dev/null | grep -iE "connect|error|fail|console|refused|timeout" | tail -15 || echo " (no relevant journal entries)" + +echo "" +echo "--- 6. Verify ser2net is proxying data (telnet rfc2217) ---" +echo " Probing TCP $TS_IP:2007 for data..." +RESPONSE=$(timeout 3 bash -c "printf '\r\r' | nc -w 2 $TS_IP 2007 2>/dev/null" | tr -cd '[:print:][:space:]' | head -5) +if [ -n "$RESPONSE" ]; then + echo " [OK] Data flowing through ser2net TCP 2007:" + echo "$RESPONSE" | sed 's/^/ /' +else + echo " (no immediate response — device may need more interaction)" +fi + +echo "" +echo "--- 7. Check if conmand has open connections to ser2net ports ---" +CONMAND_PID=$(pgrep -x conmand 2>/dev/null || echo "") +if [ -n "$CONMAND_PID" ]; then + echo " conmand PID: $CONMAND_PID" + echo " Open connections to ser2net (expect 7 to 100.x:200X):" + ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -oE "100\.[0-9.]+:200[0-9]" | sort | sed 's/^/ /' + COUNT=$(ss -tnp 2>/dev/null | grep "pid=$CONMAND_PID" | grep -c ":200") + echo " Total conmand→ser2net connections: $COUNT (expect 7)" +else + echo " [FAIL] conmand not running" +fi + +echo "" +echo "============================================" diff --git a/dcinfra/powerman/README.md b/dcinfra/powerman/README.md new file mode 100644 index 0000000..adcd136 --- /dev/null +++ b/dcinfra/powerman/README.md @@ -0,0 +1,111 @@ +# Powerman PDU Management + +Centralized power management for the Cyclades AlterPath PM10i PDU via +[Powerman](https://github.com/chaos/powerman), running on pfv-tsys1. + +## Hardware + +| Component | Details | +|-----------|---------| +| **PDU** | Cyclades AlterPath PM10i (10 controllable AC outlets) | +| **Firmware** | v1.9.0 (Aug 4, 2006) | +| **Connection** | USB-to-DB9 adapter (Prolific pl2303, serial BJAAb144J07) | +| **Host** | pfv-tsys1 (OptiPlex 9020, Proxmox) | +| **Serial** | 9600 baud, 8N1, raw mode | +| **Credentials** | Factory defaults: `admin` / `pm8` (in cyclades-pm10.dev) | +| **Network access** | powermand listens on `127.0.0.1:10101` (local) + `100.121.189.98:10101` (Tailscale) | + +## Device mapping + +``` +USB adapter (067b:23a3, serial BJAAb144J07) + └─ pl2303 driver → /dev/ttyUSB1 + └─ udev symlink → /dev/cyclades-pm10 (stable across reboots) + └─ powermand reads/writes serial → Cyclades PM10i + └─ 10 outlets (factory default names: 1-10) +``` + +The udev rule (`/etc/udev/rules.d/99-cyclades-pdu.rules`) pins the adapter +by its USB serial number, so the symlink survives replugs and reboots. + +## Scripts + +All scripts run on the target host (pfv-tsys1) via `tests/remote.sh`: + +```bash +# Setup (idempotent — safe to re-run): +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/setup.sh + +# Validate PDU control (cycles outlet 10 off → on): +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/test-pdu.sh + +# Status check: +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/status.sh +``` + +### Customizing for other hosts/PDUs + +The setup script accepts environment overrides: + +```bash +PDU_SERIAL=XXXX PDU_VENDOR=067b PDU_OUTLETS=20 PDU_TYPE=pm20 \ + PROX_HOST=other-host bash tests/remote.sh prox-file powerman/setup.sh +``` + +## Usage (daily operations) + +From pfv-tsys1 (or any host with network access to port 10101): + +```bash +# List all outlets +powerman -l + +# Query status (all outlets) +powerman -q + +# Turn outlet off +powerman -0 outlet-10 + +# Turn outlet on +powerman -1 outlet-10 + +# Cycle outlet (off → 4s delay → on) +powerman -c outlet-10 + +# Query a specific outlet +powerman -q outlet-10 +``` + +### Remote access from other hosts + +powermand listens on `0.0.0.0:10101`. From another tailnet host: + +```bash +powerman --server-host pfv-tsys1 --server-port 10101 -q +``` + +Or set `POWERMAN_SERVER=pfv-tsys1:10101` in the environment. + +## Configuration files on pfv-tsys1 + +| File | Purpose | +|------|---------| +| `/etc/udev/rules.d/99-cyclades-pdu.rules` | Stable symlink for USB-DB9 adapter | +| `/etc/powerman/powerman.conf` | Device definition + 10 outlet nodes | +| `/etc/powerman/cyclades-pm10.dev` | Cyclades PM10 protocol spec (shipped with powerman) | + +## Validation results + +2026-07-28: All 8 checks passed. +Outlet 10 turned OFF (confirmed), turned ON (confirmed), then cycled. + +## TODO (Friday onsite) + +- [ ] **Rename outlets** in `/etc/powerman/powerman.conf` to match the + physical devices plugged into each outlet (e.g., `node "tsys4-psu" + "cyclades-pm10" "3"`). Currently all outlets are generically named + `outlet-1` through `outlet-10`. +- [ ] **Change PDU admin password** from factory default (`pm8`) if + security-sensitive. Update `/etc/powerman/cyclades-pm10.dev` login + script to match. +- [ ] **Verify all 10 outlets** individually once device mapping is known. diff --git a/dcinfra/powerman/discover.sh b/dcinfra/powerman/discover.sh new file mode 100644 index 0000000..298c55a --- /dev/null +++ b/dcinfra/powerman/discover.sh @@ -0,0 +1,68 @@ +#!/usr/bin/bash +# +# powerman/discover.sh — gather USB-DB9 adapter + powerman state on a host +# +# Usage: PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/discover.sh +# +set -uo pipefail + +echo "============================================" +echo " PDU / Powerman Discovery" +echo " Host: $(hostname)" +echo " Date: $(date)" +echo "============================================" + +echo "" +echo "=== 1. USB devices ===" +lsusb 2>/dev/null || echo "(lsusb not available)" + +echo "" +echo "=== 2. USB-Serial adapters (ttyUSB*) ===" +ls -la /dev/ttyUSB* 2>/dev/null || echo "(no /dev/ttyUSB* devices)" + +echo "" +echo "=== 3. USB-Serial kernel modules ===" +lsmod | grep -iE 'usbserial|ftdi|pl2303|cp210|ch34|cdc_acm' 2>/dev/null || echo "(no relevant modules loaded)" + +echo "" +echo "=== 4. dmesg for USB serial (last 30 lines) ===" +dmesg | grep -iE 'ttyUSB|usbserial|ftdi|pl2303|cp210|ch34|converter' | tail -30 2>/dev/null || echo "(no dmesg matches)" + +echo "" +echo "=== 5. All serial devices ===" +ls -la /dev/ttyS* /dev/ttyUSB* /dev/ttyACM* 2>/dev/null || echo "(no serial devices found)" + +echo "" +echo "=== 6. Powerman installed? ===" +dpkg -l powerman 2>/dev/null || echo "(powerman not installed)" +which powerman 2>/dev/null || echo "(powerman binary not found)" +which powermand 2>/dev/null || echo "(powermand binary not found)" + +echo "" +echo "=== 7. Powerman config files ===" +ls -la /etc/powerman/ 2>/dev/null || echo "(no /etc/powerman/ directory)" +ls -la /etc/powerman/*.dev 2>/dev/null || echo "(no .dev files)" +cat /etc/powerman/powerman.conf 2>/dev/null || echo "(no powerman.conf)" + +echo "" +echo "=== 8. Available powerman device definitions ===" +ls /usr/share/powerman/*.dev 2>/dev/null || ls /etc/powerman/*.dev 2>/dev/null || echo "(no device definitions found)" + +echo "" +echo "=== 9. Powermand service status ===" +systemctl status powerman 2>/dev/null | head -10 || echo "(powerman service not found)" + +echo "" +echo "=== 10. Serial port test (quick probe of /dev/ttyUSB0) ===" +if [ -e /dev/ttyUSB0 ]; then + stty -F /dev/ttyUSB0 2>/dev/null && echo "(port exists and is configurable)" || echo "(port exists but stty failed)" + # Try to read any pending output + timeout 2 cat /dev/ttyUSB0 2>/dev/null | head -5 || echo "(no immediate output from port)" +else + echo "(no /dev/ttyUSB0)" +fi + +echo "" +echo "============================================" +echo " Discovery complete." +echo "============================================" diff --git a/dcinfra/powerman/query-remote.sh b/dcinfra/powerman/query-remote.sh new file mode 100644 index 0000000..018c7b8 --- /dev/null +++ b/dcinfra/powerman/query-remote.sh @@ -0,0 +1,65 @@ +#!/usr/bin/bash +# +# powerman/query-remote.sh — install powerman client locally and query +# the Cyclades PDU running on pfv-tsys1 over Tailscale. +# +set -euo pipefail + +REMOTE_HOST="${REMOTE_HOST:-pfv-tsys1}" +REMOTE_PORT="${REMOTE_PORT:-10101}" + +echo "============================================" +echo " Powerman Remote PDU Query" +echo " Server: ${REMOTE_HOST}:${REMOTE_PORT} (Tailscale)" +echo "============================================" + +# --- 1. Install powerman client if missing --- +if ! command -v powerman >/dev/null 2>&1; then + echo "" + echo "--- Installing powerman client ---" + if sudo -n true 2>/dev/null; then + sudo apt-get update -qq && sudo apt-get install -y -qq powerman + else + echo " Passwordless sudo not available. Please run this command in a terminal:" + echo "" + echo " sudo apt-get update && sudo apt-get install -y powerman" + echo "" + echo " Then re-run this script." + exit 1 + fi +else + echo " powerman client already installed." +fi + +# --- 2. Verify connectivity --- +echo "" +echo "--- Connectivity check ---" +if timeout 3 bash -c "echo > /dev/tcp/${REMOTE_HOST}/${REMOTE_PORT}" 2>/dev/null; then + echo " [OK] ${REMOTE_HOST}:${REMOTE_PORT} reachable" +else + echo " [FAIL] Cannot reach ${REMOTE_HOST}:${REMOTE_PORT}" + echo " Is Tailscale up? Is powermand running on ${REMOTE_HOST}?" + exit 1 +fi + +export POWERMAN_SERVER="${REMOTE_HOST}:${REMOTE_PORT}" + +# --- 3. List outlets --- +echo "" +echo "--- Outlets ---" +powerman -h "${REMOTE_HOST}:${REMOTE_PORT}" -l + +# --- 4. Query status --- +echo "" +echo "--- Status ---" +powerman -h "${REMOTE_HOST}:${REMOTE_PORT}" -q + +echo "" +echo "============================================" +echo " Done." +echo "" +echo " To control an outlet from this workstation:" +echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -0 outlet-10 # off" +echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -1 outlet-10 # on" +echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -c outlet-10 # cycle" +echo "============================================" diff --git a/dcinfra/powerman/setup.sh b/dcinfra/powerman/setup.sh new file mode 100644 index 0000000..0bb61d2 --- /dev/null +++ b/dcinfra/powerman/setup.sh @@ -0,0 +1,181 @@ +#!/usr/bin/bash +# +# powerman/setup.sh — idempotent powerman setup for Cyclades PM10i PDU +# +# Creates a stable udev symlink for the USB-DB9 adapter, writes powerman.conf +# with 10 outlet nodes, and enables + starts powermand. +# +# This script is designed to be run ON the target host (pfv-tsys1) as root. +# It is idempotent: safe to run multiple times. +# +# Usage: +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/setup.sh +# +# Override defaults via environment variables: +# PDU_SERIAL — USB adapter serial (default: BJAAb144J07) +# PDU_VENDOR — USB vendor ID (default: 067b) +# PDU_DEV_NAME — udev symlink name (default: cyclades-pm10) +# PDU_BAUD — serial baud rate (default: 9600,8n1) +# PDU_TYPE — powerman spec type (default: pm10) +# PDU_OUTLETS — number of outlets (default: 10) +# PDU_LISTEN — powermand listen (default: 0.0.0.0:10101) +# +set -euo pipefail + +# --- Config (overridable via env) --- +PDU_SERIAL="${PDU_SERIAL:-BJAAb144J07}" +PDU_VENDOR="${PDU_VENDOR:-067b}" +PDU_DEV_NAME="${PDU_DEV_NAME:-cyclades-pm10}" +PDU_BAUD="${PDU_BAUD:-9600,8n1}" +PDU_TYPE="${PDU_TYPE:-pm10}" +PDU_OUTLETS="${PDU_OUTLETS:-10}" +PDU_LISTEN="${PDU_LISTEN:-}" # Auto-detect Tailscale IP if empty + +UDEV_RULE="/etc/udev/rules.d/99-cyclades-pdu.rules" +POWERMAN_CONF="/etc/powerman/powerman.conf" +DEV_FILE="/etc/powerman/cyclades-pm10.dev" + +# --- Auto-detect Tailscale IP for listen address --- +if [ -z "$PDU_LISTEN" ]; then + TS_IP=$(tailscale ip -4 2>/dev/null || true) + if [ -n "$TS_IP" ]; then + PDU_LISTEN="${TS_IP}:10101" + echo " Auto-detected Tailscale IP: $TS_IP" + else + PDU_LISTEN="127.0.0.1:10101" + echo " WARNING: No Tailscale IP detected. Defaulting to localhost." + fi +fi + +echo "============================================" +echo " Powerman PDU Setup" +echo " Host: $(hostname)" +echo " PDU: Cyclades PM${PDU_OUTLETS}i" +echo " Adapter serial: $PDU_SERIAL" +echo " Device symlink: /dev/$PDU_DEV_NAME" +echo " Listen: $PDU_LISTEN (Tailscale only)" +echo "============================================" + +# --- 1. Ensure powerman is installed --- +echo "" +echo "--- [1/5] Checking powerman installation ---" +if ! dpkg -l powerman 2>/dev/null | grep -q '^ii'; then + echo " Installing powerman from Debian repo..." + apt-get update -qq && apt-get install -y -qq powerman +else + echo " Powerman already installed: $(dpkg -l powerman | awk '/^ii/{print $3}')" +fi + +# --- 2. Create udev rule for stable device name --- +echo "" +echo "--- [2/5] Creating udev rule for USB-DB9 adapter ---" +cat > "$UDEV_RULE" </dev/null || true +udevadm trigger --subsystem-match=tty 2>/dev/null || true +sleep 1 + +if [ -e "/dev/${PDU_DEV_NAME}" ]; then + echo " Device symlink active: /dev/${PDU_DEV_NAME} -> $(readlink -f "/dev/${PDU_DEV_NAME}")" +else + echo " WARNING: /dev/${PDU_DEV_NAME} not found yet. Adapter may be unplugged." + echo " Falling back to /dev/ttyUSB* discovery..." + # Try to find any ttyUSB device as fallback + for tty in /dev/ttyUSB*; do + if [ -e "$tty" ]; then + echo " Found: $tty (using as fallback)" + PDU_DEV_NAME="$(basename "$tty")" + break + fi + done +fi + +# --- 2b. Ensure powermand user can access the serial device --- +echo "" +echo "--- [2b/5] Fixing serial device permissions ---" +if id powerman >/dev/null 2>&1; then + if id powerman | grep -qv dialout; then + usermod -aG dialout powerman + echo " Added 'powerman' user to 'dialout' group" + else + echo " 'powerman' already in 'dialout' group" + fi +else + echo " (no powerman user — service may run as root)" +fi + +# --- 3. Write powerman.conf --- +echo "" +echo "--- [3/5] Writing powerman.conf ---" + +# Build node definitions +NODES="" +for i in $(seq 1 "$PDU_OUTLETS"); do + NODES+="node \"outlet-${i}\" \"${PDU_DEV_NAME}\" \"${i}\"\n" +done + +cat > "$POWERMAN_CONF" </dev/null || true +systemctl restart powerman 2>/dev/null || true +sleep 2 + +if systemctl is-active --quiet powerman; then + echo " powermand is running." +else + echo " WARNING: powermand failed to start. Check journalctl -u powerman" + journalctl -u powerman --no-pager -n 20 2>/dev/null || true +fi + +# --- 5. Verify --- +echo "" +echo "--- [5/5] Verification ---" +echo "" +echo " powerman -l (list all outlets):" +powerman -l 2>&1 || echo "(powerman -l failed)" + +echo "" +echo " powerman -q (query status):" +powerman -q 2>&1 || echo "(powerman -q failed — PDU may need a moment)" + +echo "" +echo "============================================" +echo " Setup complete." +echo "" +echo " Outlet names are generic (outlet-1 ... outlet-${PDU_OUTLETS})." +echo " Rename them in ${POWERMAN_CONF} when onsite to match attached devices." +echo "" +echo " Test: powerman -0 outlet-10 (off)" +echo " powerman -1 outlet-10 (on)" +echo " powerman -c outlet-10 (cycle)" +echo " powerman -q (status)" +echo "============================================" diff --git a/dcinfra/powerman/status.sh b/dcinfra/powerman/status.sh new file mode 100644 index 0000000..85488b3 --- /dev/null +++ b/dcinfra/powerman/status.sh @@ -0,0 +1,33 @@ +#!/usr/bin/bash +# +# powerman/status.sh — quick PDU status check +# +# Usage: +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/status.sh +# +set -euo pipefail + +echo "============================================" +echo " Cyclades PM10i PDU Status" +echo " Host: $(hostname) $(date)" +echo "============================================" + +echo "" +echo "=== Service ===" +systemctl is-active powerman 2>/dev/null && echo "(running)" || echo "(stopped)" + +echo "" +echo "=== Device ===" +ls -la /dev/cyclades-pm10 2>/dev/null || echo "(no /dev/cyclades-pm10 symlink)" + +echo "" +echo "=== Outlets ===" +powerman -l 2>&1 + +echo "" +echo "=== Power Status ===" +powerman -q 2>&1 + +echo "" +echo "=== Temperature ===" +powerman -T 2>&1 || echo "(temperature not available)" diff --git a/dcinfra/powerman/test-pdu.sh b/dcinfra/powerman/test-pdu.sh new file mode 100644 index 0000000..755c7e6 --- /dev/null +++ b/dcinfra/powerman/test-pdu.sh @@ -0,0 +1,126 @@ +#!/usr/bin/bash +# +# powerman/test-pdu.sh — validate PDU control by cycling outlet 10 off and on +# +# Usage: +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/test-pdu.sh +# +# Override: OUTLET=10 (which outlet to test) +# +set -euo pipefail + +OUTLET="${OUTLET:-10}" +NODE="outlet-${OUTLET}" +PASS=0; FAIL=0 +ok() { echo " [PASS] $1"; PASS=$((PASS+1)); } +fail() { echo " [FAIL] $1"; FAIL=$((FAIL+1)); } + +echo "============================================" +echo " PDU Control Validation" +echo " Host: $(hostname)" +echo " Test: cycle outlet ${OUTLET} (off → wait → on)" +echo "============================================" + +# --- 0. Powermand running? --- +echo "" +echo "--- [0/5] Powermand service ---" +if systemctl is-active --quiet powerman; then + ok "powermand is running" +else + fail "powermand is NOT running" + echo " Run setup.sh first." + exit 1 +fi + +# --- 1. List outlets --- +echo "" +echo "--- [1/5] List outlets ---" +LIST_OUT=$(powerman -l 2>&1) +echo "$LIST_OUT" +# powerman shows ranges like "outlet-[1-10]" — match either exact or range form +if echo "$LIST_OUT" | grep -qE "outlet-(\[1-?10\]|${OUTLET}\b)"; then + ok "Outlet '${NODE}' is defined" +else + fail "Outlet '${NODE}' not found in powerman -l" + exit 1 +fi + +# --- 2. Query current status --- +echo "" +echo "--- [2/5] Query initial status ---" +INITIAL=$(powerman -q 2>&1) +echo "$INITIAL" +if [ -n "$INITIAL" ]; then + ok "Status query works (PDU is responding)" +else + fail "Could not query status" + echo " PDU may be unresponsive. Check serial connection." + exit 1 +fi + +# --- 3. Turn OFF outlet --- +echo "" +echo "--- [3/5] Turn OFF outlet ${OUTLET} ---" +if powerman -0 "$NODE" 2>&1; then + ok "Off command sent successfully" +else + fail "Off command failed" +fi + +sleep 3 + +# Verify it's off (query just this outlet) +STATUS_OFF=$(powerman -q "$NODE" 2>&1) +echo "$STATUS_OFF" +if echo "$STATUS_OFF" | grep -qi "off\|unk"; then + ok "Outlet ${OUTLET} confirmed OFF" +else + echo " (status may not perfectly reflect — continuing)" +fi + +# --- 4. Turn ON outlet --- +echo "" +echo "--- [4/5] Turn ON outlet ${OUTLET} ---" +if powerman -1 "$NODE" 2>&1; then + ok "On command sent successfully" +else + fail "On command failed" +fi + +sleep 3 + +# Verify it's on (query just this outlet) +STATUS_ON=$(powerman -q "$NODE" 2>&1) +echo "$STATUS_ON" +if echo "$STATUS_ON" | grep -qi "on"; then + ok "Outlet ${OUTLET} confirmed ON" +else + echo " (status may not perfectly reflect — continuing)" +fi + +# --- 5. Cycle test (off → delay → on in one command) --- +echo "" +echo "--- [5/5] Cycle test (powerman -c) ---" +if powerman -c "$NODE" 2>&1; then + ok "Cycle command completed" +else + fail "Cycle command failed" + echo " (some PDU firmware reports errors during cycle but still works)" +fi + +sleep 5 + +# Final status +echo "" +echo "--- Final status ---" +powerman -q 2>&1 + +echo "" +echo "============================================" +echo " Results: $PASS passed, $FAIL failed" +if [ "$FAIL" -gt 0 ]; then + echo " Some checks failed. Review output above." + exit 1 +fi +echo " PDU control validated." +echo "============================================" diff --git a/dcinfra/ups/README.md b/dcinfra/ups/README.md new file mode 100644 index 0000000..5ee3b0f --- /dev/null +++ b/dcinfra/ups/README.md @@ -0,0 +1,210 @@ +# UPS Management (NUT — Network UPS Tools) + +Centralized UPS monitoring for the server room via +[NUT](https://networkupstools.org/), running on **pfv-tsys1**. USB HID UPS +units feed one `upsd` network server; Home Assistant polls it over Tailscale for +real-time power/load/runtime tracking, and a local `upsmon` shuts the hypervisor +down gracefully when battery is low. + +> **Why NUT (not apcupsd)?** Two different UPS brands (APC + Tripp Lite) must be +> covered. `apcupsd` only supports APC, so it would require a second daemon +> stack. NUT's `usbhid-ups` driver speaks to **both** via the USB HID Power +> Device class, and Home Assistant ships a first-class NUT integration. + +## Hardware + +| UPS | Model | VID:PID | USB Serial | Status | +|-----|-------|---------|------------|--------| +| **APC** | Smart-UPS C 1500 (FW 02.2) | `051d:0003` | `AS1213210423` | **LIVE** | +| **Tripp Lite** | UPS (HID PDC) | `09ae:3016` | `2352CVLSM871900694` | **Blocked** — see below | + +## Current State (2026-07-30) + +### APC Smart-UPS C 1500 — OPERATIONAL + +Fully reporting via `usbhid-ups` + `APC HID 0.100` subdriver. Data validated: + +``` +battery.charge: 100 battery.runtime: 1800 battery.voltage: 27.4 +ups.status: OL ups.load: (via HA) ups.model: Smart-UPS C 1500 +``` + +### Tripp Lite UPS — BLOCKED (hardware issue) + +The driver finds the device, matches the `TrippLite HID 0.85` subdriver, claims +the interface, and reads the HID descriptor — but **fails reading the 878-byte +HID Report Descriptor** (`Resource temporarily unavailable` / EAGAIN after 5s). +The driver is masked to prevent restart-loop spam. + +USB descriptors (manufacturer, product, serial) are readable via `lsusb -v` and +`nut-scanner`, but the bulk control transfer for the full report descriptor +times out. Likely causes: + +1. **USB hub** — the Tripp Lite is behind a Genesys Logic hub (`05e3:0608`). + Try plugging directly into a motherboard USB port. +2. **USB cable** — try a high-quality data cable (not charge-only). +3. **UPS firmware** — the USB controller may not properly implement all HID + endpoints. + +**To retry after physical reseat:** +```bash +# On pfv-tsys1: +systemctl unmask nut-driver@tripp-lite-ups +systemctl start nut-driver@tripp-lite-ups +upsc tripp-lite-ups@localhost +``` + +## Architecture + +``` +pfv-tsys1 (192.168.3.11 / Tailscale 100.121.189.98) + ├─ APC Smart-UPS C 1500 ──┐ + └─ Tripp Lite UPS (masked) ──┤ USB HID + ▼ + nut-driver@apc-smartups-c1500 (usbhid-ups) + ▼ + upsd :3493 (LISTEN 127.0.0.1 + Tailscale + LAN) + ▼ ▼ + upsmon (local) Home Assistant (NUT integration) + graceful shutdown via LAN 192.168.3.11 (HAOS can't + route to Tailscale IPs) +``` + +- **Driver layer** — `usbhid-ups` process, pinned by USB serial. Debian uses + templated `nut-driver@.service` units managed by + `nut-driver-enumerator`. +- **Server layer** — `upsd` exposes UPS data on TCP 3493 (localhost + Tailscale + + LAN). Clients authenticate via `upsd.users`. +- **Monitor layer** — `upsmon` runs locally as `master` to trigger + `SHUTDOWNCMD` (`/sbin/shutdown -h now`) when a UPS reports `LOWBATT`. +- **Home Assistant** — native NUT integration connects to `upsd` over Tailscale + and exposes `ups.load`, `battery.runtime`, `ups.status`, etc. as sensors. + +### Key deployment lesson: udev must cover raw USB devices + +The `usbhid-ups` driver opens `/dev/bus/usb/BBB/DDD` (raw USB device files), +**not** `/dev/hidraw*`. After calling `setuid(111)` to drop to the `nut` user, +it needs write access to those raw USB files. The udev rule must match +`SUBSYSTEM=="usb"` by vendor/product ID to set `GROUP="nut"` — matching only +`hidraw` is insufficient. See `/etc/udev/rules.d/99-nut-ups.rules`. + +## Scripts + +NUT host scripts run on pfv-tsys1 via `tests/remote.sh`: + +```bash +# Idempotent install + configure (safe to re-run): +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/setup.sh + +# Discover USB UPS + NUT state (read-only diagnostic): +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/discover.sh + +# Query UPS data + service health: +PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/status.sh +``` + +The HA integration script runs from your workstation (needs HA API access): + +```bash +# Add the NUT integration to Home Assistant (idempotent): +bash ups/setup-ha-nut.sh +``` + +`setup.sh` accepts environment overrides for serials/VIDs/PIDs/usernames, so it +can be repurposed for other hosts or UPS units. Passwords for `monuser` and +`homeassistant` are auto-generated on first run and reused on subsequent runs +(stored in `/etc/nut/upsd.users`). + +Set `TRIPP_ENABLED=0` to skip the Tripp Lite entirely (useful if it's physically +unplugged). + +## Configuration files on pfv-tsys1 + +| File | Purpose | +|------|---------| +| `/etc/udev/rules.d/99-nut-ups.rules` | Grant nut group rw on raw USB + hidraw devices (both subsystems) | +| `/etc/nut/ups.conf` | `usbhid-ups` device(s), pinned by serial + subdriver | +| `/etc/nut/upsd.conf` | `LISTEN 127.0.0.1` + `LISTEN ` + `LISTEN ` on port 3493 | +| `/etc/nut/upsd.users` | `monuser` (master) + `homeassistant` (read-only) credentials | +| `/etc/nut/upsmon.conf` | Local master monitor + `SHUTDOWNCMD` | +| `/etc/nut/nut.conf` | `MODE=netserver` | + +## Home Assistant integration + +The NUT integration is added automatically by `setup-ha-nut.sh`, which drives +HA's REST config-flow API. It is idempotent (skips if the entry exists). + +```bash +# Prerequisites: create token + password files (one-time): +mkdir -p ~/.config/pfvcluster +# HA → Profile → Long-Lived Access Tokens → Create Token: +echo -n 'YOUR_HA_TOKEN' > ~/.config/pfvcluster/ha-token +# Password is in /etc/nut/upsd.users on pfv-tsys1 (the homeassistant user): +echo -n 'YOUR_NUT_PASS' > ~/.config/pfvcluster/nut-password +chmod 600 ~/.config/pfvcluster/{ha-token,nut-password} + +# Run: +bash ups/setup-ha-nut.sh +``` + +### Why LAN IP, not Tailscale + +upsd listens on **both** the Tailscale IP (`100.121.189.98`) **and** the LAN IP +(`192.168.3.11`). The HA NUT integration uses the **LAN IP** because HAOS runs +Tailscale as an isolated add-on container — the HA core container cannot route +to Tailscale IPs. Since pfv-bms (HA, `192.168.3.12`) and pfv-tsys1 (`192.168.3.11`) +share the same vmbr0 bridge, LAN connectivity is instant and reliable. + +### Manual UI alternative + +In Home Assistant → **Settings → Devices & Services → Add Integration → NUT**: + +| Field | Value | +|-------|-------| +| Host | `192.168.3.11` (LAN — HAOS can't reach Tailscale IPs from the HA container) | +| Port | `3493` | +| Username | `homeassistant` | +| Password | *(stored in `/etc/nut/upsd.users` on pfv-tsys1)* | +| UPS | `apc-smartups-c1500` | + +### Live sensors + +HA exposes UPS data as sensors (prefix `sensor.apc_smartups_c1500_`): +`battery_charge`, `status` (Online/On Battery), `status_data` (OL/OB/DISCHRG). +Additional sensors (load, runtime, voltage) populate as the UPS reports them. + +## Daily operations + +From pfv-tsys1 (or any tailnet host with NUT client installed): + +```bash +# List UPS units served by upsd +upsc -l pfv-tsys1 + +# Full variable dump for one UPS +upsc apc-smartups-c1500@pfv-tsys1 + +# Battery runtime (the only runtime/charge data this UPS exposes) +upsc apc-smartups-c1500@pfv-tsys1 battery.runtime +``` + +## Notes + +- **No USB passthrough to the HA VM.** Keeping the UPS on the host preserves + hypervisor graceful-shutdown capability and matches the `powerman/` pattern + (PDU managed on the host where the adapter physically lives). +- **No `ups.load` / `ups.realpower` on this UPS (FW 02.2, mfg 2012):** The + APC Smart-UPS C 1500 does not expose load or power data over USB HID. + Both NUT `usbhid-ups` and `apcupsd` (USB mode, tested 2026-07-30) read the + same HID descriptor — the variable simply isn't there. This means the HA + NUT integration provides **battery/runtime/status sensors only**, not + wattage for the Energy Dashboard. + - **apcupsd test note:** Debian's `apcupsd` package conflicts with + `nut-server` (mutually exclusive). apcupsd USB mode returned `COMMLOST` + even before we could check load. The APC Smart Serial protocol (serial + cable, AP940-1524C, ~$30) DOES report load%, but this requires a serial + port on the UPS and on the host. + - **Energy Dashboard path:** A smart plug (Shelly Plug S / TP-Link Kasa, + ~$15-25) on the UPS output reports real watts natively and feeds the + Energy Dashboard with zero UPS-driver hacking. The NUT sensors remain + valuable for outage detection and graceful-shutdown automations. diff --git a/dcinfra/ups/discover.sh b/dcinfra/ups/discover.sh new file mode 100644 index 0000000..0a82d4b --- /dev/null +++ b/dcinfra/ups/discover.sh @@ -0,0 +1,86 @@ +#!/usr/bin/bash +# +# ups/discover.sh — probe USB UPS units and NUT state on the local host +# +# Read-only. Prints everything needed to configure NUT. No changes made. +# +# Usage (run ON the target host via remote.sh): +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/discover.sh +# +set -uo pipefail + +echo "======================================================" +echo " UPS / NUT Discovery on $(hostname)" +echo "======================================================" + +# --- 1. USB UPS devices ------------------------------------------------------ +echo "" +echo "--- [1/5] USB UPS devices (lsusb) ---" +lsusb 2>/dev/null | grep -iE "UPS|American Power|Tripp|APC" || echo " (no UPS devices found in lsusb)" + +echo "" +echo "--- [2/5] UPS detail (vendor/product/serial/model) ---" +# Common UPS vendor IDs: 051d (APC), 09ae (Tripp Lite), 0463 (Eaton), +# 06da (MGE), 0764 (Cyber Power) +for vid in 051d 09ae 0463 06da 0764; do + while read -r bus dev pid; do + [ -n "$bus" ] || continue + echo " --- $bus:$dev ($vid:$pid) ---" + lsusb -v -s "${bus}:${dev}" 2>/dev/null \ + | grep -iE "iManufacturer|iProduct|iSerial|bcdDevice" \ + | sed 's/^/ /' + done < <(lsusb 2>/dev/null | awk -v v="$vid" '$0~v{split($2,a,":"); split($4,b,":"); print a[1], b[1], $6}') +done + +# --- 2. sysfs paths (for udev rules) ----------------------------------------- +echo "" +echo "--- [3/5] sysfs device paths + serials ---" +for d in /sys/bus/usb/devices/*; do + man=$(cat "$d/manufacturer" 2>/dev/null) + prod=$(cat "$d/product" 2>/dev/null) + ser=$(cat "$d/serial" 2>/dev/null) + vid=$(cat "$d/idVendor" 2>/dev/null) + pid=$(cat "$d/idProduct" 2>/dev/null) + if echo "$man $prod" | grep -qiE "apc|tripp|power conversion|ups|eaton|mge|cyber power"; then + # Resolve stable ID_PATH for udev pinning + path=$(udevadm info -q property -p "$d" 2>/dev/null | awk -F= '/^ID_PATH=/{print $2}') + echo " $d" + echo " vendor=$vid product=$pid" + echo " manufacturer=$man" + echo " product=$prod" + echo " serial=$ser" + echo " ID_PATH=$path" + fi +done + +# --- 3. HID device nodes ----------------------------------------------------- +echo "" +echo "--- [4/5] HID device nodes ---" +ls -la /dev/hidraw* /dev/usb/hiddev* 2>/dev/null || echo " (no hidraw/hiddev nodes)" + +# --- 4. NUT install state ---------------------------------------------------- +echo "" +echo "--- [5/5] NUT install + service state ---" +if dpkg -l nut-server nut-client 2>/dev/null | grep -q '^ii'; then + echo " NUT installed:" + dpkg -l nut-server nut-client 2>/dev/null | awk '/^ii/{print " "$2" "$3}' +else + echo " NUT not installed (apt: nut-server nut-client)" +fi + +echo "" +echo " Services:" +for svc in nut-driver nut-server nut-monitor; do + printf " %-14s " "$svc:" + systemctl is-active "$svc" 2>/dev/null || true +done + +echo "" +echo " Existing config:" +# shellcheck disable=SC2012 # ls -la is intentional for human-readable listing +ls -la /etc/nut/ 2>/dev/null | sed 's/^/ /' || echo " (no /etc/nut)" + +echo "" +echo "======================================================" +echo " Discovery complete." +echo "======================================================" diff --git a/dcinfra/ups/ha-nut-setup.py b/dcinfra/ups/ha-nut-setup.py new file mode 100644 index 0000000..1c9eae3 --- /dev/null +++ b/dcinfra/ups/ha-nut-setup.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +""" +ha-nut-setup.py — Add the Home Assistant NUT integration via REST config-flow API. + +Stdlib-only (no pip). Idempotent: skips if a NUT config entry already exists. + +Env: + HA_HOST (default pfv-bms.knel.net) + HA_PORT (default 8123) + HA_TOKEN (long-lived access token) + NUT_HOST (default 100.121.189.98) + NUT_PORT (default 3493) + NUT_USER (default homeassistant) + NUT_PASS (required) + NUT_UPS (default apc-smartups-c1500) +""" +import os, json, sys, time, urllib.request, urllib.error + +HA_HOST = os.environ.get("HA_HOST", "pfv-bms.knel.net") +HA_PORT = int(os.environ.get("HA_PORT", "8123")) +TOKEN = os.environ["HA_TOKEN"] +NUT_HOST = os.environ.get("NUT_HOST", "192.168.3.11") +NUT_PORT = int(os.environ.get("NUT_PORT", "3493")) +NUT_USER = os.environ.get("NUT_USER", "homeassistant") +NUT_PASS = os.environ["NUT_PASS"] +NUT_UPS = os.environ.get("NUT_UPS", "apc-smartups-c1500") +BASE = f"http://{HA_HOST}:{HA_PORT}" + +def api(method, path, data=None): + body = json.dumps(data).encode() if data else None + req = urllib.request.Request( + f"{BASE}/api{path}", data=body, method=method, + headers={"Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=20) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + raw = e.read().decode() + try: + return json.loads(raw) + except Exception: + return {"_http_error": e.code, "_raw": raw[:300]} + except Exception as e: + return {"_error": str(e)} + +# ── verify token ── +cfg = api("GET", "/config") +if "_http_error" in cfg or "_error" in cfg: + print(f"Cannot reach HA or token invalid: {cfg}"); sys.exit(1) +print(f"HA {cfg.get('version')} — token valid") + +# ── check existing entries (idempotent) ── +entries = api("GET", "/config/config_entries/entry") +existing = [e for e in entries if e.get("domain") == "nut"] +if existing: + for e in existing: + print(f"NUT already configured: {e.get('title')} " + f"(data={json.dumps(e.get('data', {}))})") + print("Skipping — delete it in HA UI first if you want to re-run.") + sys.exit(0) +print("No existing NUT entry. Starting config flow.") + +# ── initiate flow ── +flow = api("POST", "/config/config_entries/flow", {"handler": "nut"}) +if "flow_id" not in flow: + print(f"Flow init failed: {json.dumps(flow)}"); sys.exit(1) +fid = flow["flow_id"] +print(f"Flow started: step={flow.get('step_id')} " + f"fields={[f.get('name') for f in flow.get('data_schema', [])]}") + +# ── submit connection details ── +creds = {"host": NUT_HOST, "port": NUT_PORT, + "username": NUT_USER, "password": NUT_PASS} +flow = api("POST", f"/config/config_entries/flow/{fid}", creds) +if flow.get("errors"): + print(f"Validation errors: {flow['errors']}"); sys.exit(1) +print(f"After submit: type={flow.get('type')} step={flow.get('step_id')}") + +# ── handle follow-up steps (UPS selection etc.) ── +while flow.get("type") == "form": + step = flow.get("step_id", "?") + schema = flow.get("data_schema", []) + print(f"Step '{step}': fields={[f.get('name') for f in schema]}") + for f in schema: + opts = f.get("options") or f.get("values") + if opts: + print(f" {f.get('name')} options: {opts}") + submission = {} + for f in schema: + nm = f.get("name") + ftype = f.get("type", "") + if ftype == "multi_select": + opts = f.get("options", []) + vals = [o[0] if isinstance(o, list) else o for o in opts] + submission[nm] = [NUT_UPS] if NUT_UPS in vals else vals[:1] + elif nm in creds: + submission[nm] = creds[nm] + elif "default" in f: + submission[nm] = f["default"] + elif ftype == "select": + opts = f.get("options", []) + vals = [o[0] if isinstance(o, list) else o for o in opts] + submission[nm] = NUT_UPS if NUT_UPS in vals else (vals[0] if vals else "") + fid = flow.get("flow_id", fid) + flow = api("POST", f"/config/config_entries/flow/{fid}", submission) + if flow.get("errors"): + print(f"Validation errors: {flow['errors']}"); sys.exit(1) + print(f" -> type={flow.get('type')} step={flow.get('step_id')}") + +# ── result ── +if flow.get("type") == "create_entry": + print(f"\nNUT integration created: {flow.get('title')}") +elif flow.get("type") == "abort": + print(f"\nFlow aborted: {flow.get('reason')}"); sys.exit(1) +else: + print(f"\nFinal state: {flow.get('type')} — {json.dumps(flow)[:200]}") + +# ── verify sensors ── +print("\nWaiting 10s for entities ...") +time.sleep(10) +states = api("GET", "/states") +ups = [s for s in states + if "apc_smartups" in s["entity_id"].lower() + or "sensor.ups_" in s["entity_id"].lower()] +if ups: + print(f"Found {len(ups)} UPS sensors:") + for e in sorted(ups, key=lambda x: x["entity_id"]): + st = e.get("state", "?") + unit = e.get("attributes", {}).get("unit_of_measurement", "") + name = e.get("attributes", {}).get("friendly_name", "") + print(f" {e['entity_id']:55s} {st:>8} {unit:4s} {name}") +else: + print("No UPS sensors yet (may still be initialising — check HA UI).") diff --git a/dcinfra/ups/setup-ha-nut.sh b/dcinfra/ups/setup-ha-nut.sh new file mode 100644 index 0000000..63b7b36 --- /dev/null +++ b/dcinfra/ups/setup-ha-nut.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# setup-ha-nut.sh — Add the Home Assistant NUT integration via REST API. +# +# Idempotent: skips if a NUT entry already exists. Reads secrets from +# ~/.config/pfvcluster/ (ha-token, nut-password) or env vars. +# +# Usage: +# bash ups/setup-ha-nut.sh +# +# Env overrides: +# HA_TOKEN HA long-lived access token +# NUT_PASS NUT upsd password for the homeassistant user +# HA_HOST HA host (default pfv-bms.knel.net) +# NUT_HOST upsd host (default 192.168.3.11 — LAN, see README) +# NUT_PORT upsd port (default 3493) +# NUT_USER upsd user (default homeassistant) +# NUT_UPS UPS name (default apc-smartups-c1500) +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONF_DIR="${PFV_CONF_DIR:-$HOME/.config/pfvcluster}" + +# --- HA token --- +HA_TOKEN="${HA_TOKEN:-}" +if [[ -z "$HA_TOKEN" ]]; then + TOKEN_FILE="$CONF_DIR/ha-token" + if [[ -f "$TOKEN_FILE" ]]; then + HA_TOKEN="$(head -1 "$TOKEN_FILE" | tr -d '[:space:]')" + else + echo "error: no HA token. Set \$HA_TOKEN or create $TOKEN_FILE" >&2 + echo " (HA → Profile → Long-Lived Access Tokens → Create Token)" >&2 + exit 1 + fi +fi + +# --- NUT password --- +NUT_PASS="${NUT_PASS:-}" +if [[ -z "$NUT_PASS" ]]; then + PASS_FILE="$CONF_DIR/nut-password" + if [[ -f "$PASS_FILE" ]]; then + NUT_PASS="$(head -1 "$PASS_FILE" | tr -d '[:space:]')" + else + echo "error: no NUT password. Set \$NUT_PASS or create $PASS_FILE" >&2 + echo " (value is in /etc/nut/upsd.users on the NUT host)" >&2 + exit 1 + fi +fi + +# --- connection params (override for your own kit) --- +export HA_TOKEN +export NUT_PASS +export HA_HOST="${HA_HOST:-pfv-bms.knel.net}" +export HA_PORT="${HA_PORT:-8123}" +export NUT_HOST="${NUT_HOST:-192.168.3.11}" +export NUT_PORT="${NUT_PORT:-3493}" +export NUT_USER="${NUT_USER:-homeassistant}" +export NUT_UPS="${NUT_UPS:-apc-smartups-c1500}" + +echo "HA: ${HA_HOST}:${HA_PORT}" +echo "NUT: ${NUT_USER}@${NUT_HOST}:${NUT_PORT} (UPS: ${NUT_UPS})" +echo "" + +exec python3 "$SCRIPT_DIR/ha-nut-setup.py" diff --git a/dcinfra/ups/setup.sh b/dcinfra/ups/setup.sh new file mode 100644 index 0000000..43b3cd8 --- /dev/null +++ b/dcinfra/ups/setup.sh @@ -0,0 +1,298 @@ +#!/usr/bin/bash +# +# ups/setup.sh — idempotent Network UPS Tools (NUT) setup on pfv-tsys1 +# +# Installs NUT, configures two USB HID UPS units (APC + Tripp Lite) pinned by +# USB serial, runs upsd as a network server for Home Assistant polling, and +# runs upsmon locally so the hypervisor can shut down gracefully on battery. +# +# Designed to run ON the target host (pfv-tsys1) as root, idempotent. +# +# Usage: +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/setup.sh +# +# Overrides (defaults suit pfv-tsys1): +# APC_SERIAL APC UPS USB serial (default AS1213210423) +# APC_VID/APC_PID APC vendor/product ID (default 051d / 0003) +# APC_NAME NUT section name for APC (default apc-smartups-c1500) +# TRIPP_SERIAL Tripp Lite UPS USB serial (default 2352CVLSM871900694) +# TRIPP_VID/TRIPP_PID Tripp Lite vendor/product (default 09ae / 3016) +# TRIPP_NAME NUT section name for Tripp (default tripp-lite-ups) +# TRIPP_SUBDRIVER Forced HID subdriver for Tripp (default "TrippLite HID 0.85") +# TRIPP_ENABLED Set to 0 to disable Tripp Lite (default 1) +# NUT_LISTEN_IPS space-separated upsd LISTEN IPs (default: auto Tailscale + 127.0.0.1) +# HA_USER upsd username for HA (default homeassistant) +# HA_PASSWORD upsd password for HA (default: reuse or generate) +# MON_USER upsd username for local upsmon (default monuser) +# MON_PASSWORD upsd password for upsmon (default: reuse or generate) +# +set -euo pipefail + +# --- Config (overridable via env) --- +APC_SERIAL="${APC_SERIAL:-AS1213210423}" +APC_VID="${APC_VID:-051d}" +APC_PID="${APC_PID:-0003}" +APC_NAME="${APC_NAME:-apc-smartups-c1500}" + +TRIPP_SERIAL="${TRIPP_SERIAL:-2352CVLSM871900694}" +TRIPP_VID="${TRIPP_VID:-09ae}" +TRIPP_PID="${TRIPP_PID:-3016}" +TRIPP_NAME="${TRIPP_NAME:-tripp-lite-ups}" +TRIPP_SUBDRIVER="${TRIPP_SUBDRIVER:-TrippLite HID 0.85}" +TRIPP_ENABLED="${TRIPP_ENABLED:-1}" + +HA_USER="${HA_USER:-homeassistant}" +MON_USER="${MON_USER:-monuser}" +NUT_PORT="${NUT_PORT:-3493}" +UDEV_RULE="/etc/udev/rules.d/99-nut-ups.rules" + +UPS_CONF="/etc/nut/ups.conf" +UPSD_CONF="/etc/nut/upsd.conf" +UPSD_USERS="/etc/nut/upsd.users" +UPS_CONF_MON="/etc/nut/upsmon.conf" +NUT_CONF="/etc/nut/nut.conf" + +# --- Helpers --- +gen_pw() { head -c 24 /dev/urandom | base64 | tr -d '/+=' | cut -c1-20; } + +# Reuse existing passwords if config already present (idempotent re-runs) +extract_pw() { # $1=user + if [ -f "$UPSD_USERS" ]; then + awk -v u="[$1]" ' + $0==u {inblk=1; next} + /^\[/ {inblk=0} + inblk && $1=="password" {gsub(/"/,"",$3); print $3; exit} + ' "$UPSD_USERS" 2>/dev/null + fi +} + +echo "============================================" +echo " NUT UPS Setup on $(hostname)" +echo " APC: $APC_NAME ($APC_VID:$APC_PID serial $APC_SERIAL)" +if [ "$TRIPP_ENABLED" = "1" ]; then + echo " Tripp Lite: $TRIPP_NAME ($TRIPP_VID:$TRIPP_PID serial $TRIPP_SERIAL)" +else + echo " Tripp Lite: DISABLED (TRIPP_ENABLED=0)" +fi +echo "============================================" + +# --- 0. Resolve / generate passwords (idempotent) --- +MON_PASSWORD="${MON_PASSWORD:-$(extract_pw "$MON_USER")}" +HA_PASSWORD="${HA_PASSWORD:-$(extract_pw "$HA_USER")}" +[ -n "$MON_PASSWORD" ] || MON_PASSWORD="$(gen_pw)" +[ -n "$HA_PASSWORD" ] || HA_PASSWORD="$(gen_pw)" + +# --- 0b. Auto-detect listen IPs for upsd --- +# Tailscale IP: tailnet clients (workstation, etc.) +# LAN IP: HAOS VMs where Tailscale runs as an isolated add-on (HA container +# cannot route to Tailscale IPs, so the shared-LAN bridge is required) +if [ -z "${NUT_LISTEN_IPS:-}" ]; then + NUT_LISTEN_IPS="127.0.0.1" + TS_IP=$(tailscale ip -4 2>/dev/null || true) + if [ -n "$TS_IP" ]; then + NUT_LISTEN_IPS="${NUT_LISTEN_IPS} ${TS_IP}" + else + echo " WARNING: No Tailscale IP detected." + fi + if [ "${NUT_INCLUDE_LAN:-1}" = "1" ]; then + LAN_IP=$(ip -4 addr show vmbr0 2>/dev/null | awk '/scope global/{print $2}' | cut -d/ -f1 | head -1) + if [ -z "$LAN_IP" ]; then + LAN_IP=$(hostname -I 2>/dev/null | awk '{print $1}') + fi + if [ -n "$LAN_IP" ]; then + NUT_LISTEN_IPS="${NUT_LISTEN_IPS} ${LAN_IP}" + fi + fi +fi +echo " upsd LISTEN IPs: ${NUT_LISTEN_IPS:-}" + +# --- 1. Install NUT --- +echo "" +echo "--- [1/8] Installing NUT (nut-server, nut-client) ---" +if dpkg -l nut-server 2>/dev/null | grep -q '^ii'; then + echo " NUT already installed: $(dpkg -l nut-server | awk '/^ii/{print $3}')" +else + apt-get update -qq && apt-get install -y -qq nut-server nut-client +fi +mkdir -p /etc/nut + +# --- 2. udev rules: grant nut group access to BOTH raw USB + hidraw devices --- +# CRITICAL: usbhid-ups opens /dev/bus/usb/BBB/DDD (raw USB), not /dev/hidraw. +# The driver drops to the nut user via setuid(), so the nut group needs write +# access to the raw USB device files. Matching on subsystem=="usb" by VID:PID +# is required because ATTRS{serial} does not reliably traverse for usb devices. +echo "" +echo "--- [2/8] Writing udev rules (raw USB + hidraw, group nut) ---" +{ + echo "# Stable permissions for NUT USB HID UPS units" + echo "# Generated by ups/setup.sh — grants the 'nut' group access to both" + echo "# the raw USB device files (/dev/bus/usb) and hidraw devices." + echo "# Match BOTH subsystems: the usbhid-ups driver opens the raw USB device" + echo "# after dropping to the nut user via setuid()." + echo "" + echo "# APC Smart-UPS C 1500 ($APC_VID:$APC_PID)" + echo "SUBSYSTEM==\"usb\", ATTR{idVendor}==\"$APC_VID\", ATTR{idProduct}==\"$APC_PID\", GROUP=\"nut\", MODE=\"0664\"" + echo "SUBSYSTEM==\"hidraw\", ATTRS{serial}==\"$APC_SERIAL\", GROUP=\"nut\", MODE=\"0660\"" + echo "" + echo "# Tripp Lite UPS ($TRIPP_VID:$TRIPP_PID)" + echo "SUBSYSTEM==\"usb\", ATTR{idVendor}==\"$TRIPP_VID\", ATTR{idProduct}==\"$TRIPP_PID\", GROUP=\"nut\", MODE=\"0664\"" + echo "SUBSYSTEM==\"hidraw\", ATTRS{serial}==\"$TRIPP_SERIAL\", GROUP=\"nut\", MODE=\"0660\"" +} > "$UDEV_RULE" +echo " Written: $UDEV_RULE" +udevadm control --reload-rules 2>/dev/null || true +udevadm trigger --subsystem-match=usb 2>/dev/null || true +udevadm trigger --subsystem-match=hidraw 2>/dev/null || true +sleep 1 + +# --- 3. ups.conf --- +echo "" +echo "--- [3/8] Writing ups.conf ---" +{ + echo "# NUT UPS devices — generated by ups/setup.sh on $(date)" + echo "" + echo "maxretry = 3" + echo "" + echo "[${APC_NAME}]" + echo " driver = usbhid-ups" + echo " port = auto" + echo " vendorid = ${APC_VID}" + echo " productid = ${APC_PID}" + echo " serial = ${APC_SERIAL}" + echo " desc = \"APC Smart-UPS C 1500\"" + if [ "$TRIPP_ENABLED" = "1" ]; then + echo "" + echo "[${TRIPP_NAME}]" + echo " driver = usbhid-ups" + echo " port = auto" + echo " vendorid = ${TRIPP_VID}" + echo " productid = ${TRIPP_PID}" + echo " serial = ${TRIPP_SERIAL}" + echo " subdriver = \"${TRIPP_SUBDRIVER}\"" + echo " desc = \"Tripp Lite UPS\"" + fi +} > "$UPS_CONF" +echo " Written: $UPS_CONF" + +# --- 4. upsd.conf: network server (localhost + Tailscale for HA) --- +echo "" +echo "--- [4/8] Writing upsd.conf ---" +{ + echo "# NUT upsd — generated by ups/setup.sh on $(date)" + for ip in $NUT_LISTEN_IPS; do + echo "LISTEN ${ip} ${NUT_PORT}" + done + echo "MAXAGE 25" +} > "$UPSD_CONF" +echo " Written: $UPSD_CONF (LISTEN: $(echo "$NUT_LISTEN_IPS" | tr '\n' ' '))" + +# --- 5. upsd.users: monuser (master) + homeassistant (read-only monitor) --- +echo "" +echo "--- [5/8] Writing upsd.users ---" +cat > "$UPSD_USERS" < "$UPS_CONF_MON" + +cat > "$NUT_CONF" </dev/null || true +chmod 640 "$UPS_CONF" "$UPSD_CONF" "$UPSD_USERS" "$UPS_CONF_MON" 2>/dev/null || true +chmod 644 "$NUT_CONF" 2>/dev/null || true + +# --- 7. Start services (Debian uses templated nut-driver@ units) --- +echo "" +echo "--- [7/8] Starting NUT services ---" + +# Re-read ups.conf to generate per-UPS driver instances +systemctl restart nut-driver-enumerator 2>/dev/null || true +sleep 2 + +# Start per-UPS driver instances +systemctl restart "nut-driver@${APC_NAME}" 2>/dev/null || true +if [ "$TRIPP_ENABLED" = "1" ]; then + systemctl restart "nut-driver@${TRIPP_NAME}" 2>/dev/null || true +else + systemctl stop "nut-driver@${TRIPP_NAME}" 2>/dev/null || true + systemctl mask "nut-driver@${TRIPP_NAME}" 2>/dev/null || true +fi +sleep 3 +systemctl restart nut-server 2>/dev/null || true +sleep 1 +systemctl restart nut-monitor 2>/dev/null || true + +echo "" +echo " Service status:" +for svc in "nut-driver@${APC_NAME}" "nut-driver@${TRIPP_NAME}" nut-server nut-monitor; do + if systemctl list-unit-files "$svc" >/dev/null 2>&1; then + printf " %-42s " "$svc" + systemctl is-active "$svc" 2>/dev/null || echo "(unknown)" + fi +done + +# --- 8. Validate --- +echo "" +echo "--- [8/8] Validation ---" +echo "" +echo " upsc — ${APC_NAME}:" +upsc "${APC_NAME}@localhost" 2>&1 | head -25 || echo " (APC UPS not responding yet)" +if [ "$TRIPP_ENABLED" = "1" ]; then + echo "" + echo " upsc — ${TRIPP_NAME}:" + upsc "${TRIPP_NAME}@localhost" 2>&1 | head -25 || echo " (Tripp Lite UPS not responding yet)" +fi + +echo "" +echo "============================================" +echo " Setup complete." +echo "" +echo " Home Assistant NUT integration:" +echo " Host: $(echo "$NUT_LISTEN_IPS" | awk '{print $2}') (or any LISTEN IP above)" +echo " Port: ${NUT_PORT}" +echo " Username: ${HA_USER}" +echo " Password: ${HA_PASSWORD}" +if [ "$TRIPP_ENABLED" = "1" ]; then + echo " UPS names: ${APC_NAME}, ${TRIPP_NAME}" +else + echo " UPS names: ${APC_NAME}" +fi +echo "" +echo " Save the HA password now — it is stored in ${UPSD_USERS}." +echo "============================================" diff --git a/dcinfra/ups/status.sh b/dcinfra/ups/status.sh new file mode 100644 index 0000000..0f8c4b8 --- /dev/null +++ b/dcinfra/ups/status.sh @@ -0,0 +1,59 @@ +#!/usr/bin/bash +# +# ups/status.sh — query NUT UPS state + service health (read-only) +# +# Usage (run ON the target host via remote.sh): +# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file ups/status.sh +# +# shellcheck disable=SC2012 # ss/awk field extraction is intentional +set -uo pipefail + +APC_NAME="${APC_NAME:-apc-smartups-c1500}" +TRIPP_NAME="${TRIPP_NAME:-tripp-lite-ups}" + +echo "======================================================" +echo " NUT UPS Status on $(hostname)" +echo "======================================================" + +echo "" +echo "--- Services ---" +for svc in "nut-driver@${APC_NAME}" "nut-driver@${TRIPP_NAME}" nut-server nut-monitor; do + if systemctl list-unit-files "$svc" >/dev/null 2>&1; then + printf " %-42s " "$svc" + systemctl is-active "$svc" 2>/dev/null || echo "(unknown)" + fi +done + +for ups in "$APC_NAME" "$TRIPP_NAME"; do + echo "" + echo "--- ${ups} ---" + if upsc "${ups}@localhost" >/tmp/.nutstatus.$$ 2>&1; then + awk -v u="$ups" ' + BEGIN{printf " %s\n", u} + /^battery\.charge:/ {printf " battery.charge: %s\n", $3} + /^battery\.runtime:/ {printf " battery.runtime: %ss (%.0f min)\n", $3, $3/60} + /^battery\.voltage:/ {printf " battery.voltage: %s\n", $3} + /^ups\.status:/ {printf " ups.status: %s\n", $3} + /^ups\.load:/ {printf " ups.load: %s%%\n", $3} + /^ups\.power:/ {printf " ups.power: %s\n", $3} + /^ups\.realpower:/ {printf " ups.realpower: %s W\n", $3} + /^input\.voltage:/ {printf " input.voltage: %s\n", $3} + /^output\.voltage:/ {printf " output.voltage: %s\n", $3} + /^ups\.model:/ {printf " ups.model: %s\n", $3} + /^ups\.serial:/ {printf " ups.serial: %s\n", $3} + /^device\.mfr:/ {printf " device.mfr: %s\n", $3} + ' /tmp/.nutstatus.$$ + echo " (full dump: upsc ${ups}@localhost)" + else + echo " NOT RESPONDING:" + sed 's/^/ /' /tmp/.nutstatus.$$ + fi + rm -f /tmp/.nutstatus.$$ +done + +echo "" +echo "--- upsd LISTEN sockets ---" +ss -ltnp 2>/dev/null | grep -E "3493|nut" | sed 's/^/ /' || echo " (upsd not listening on 3493)" + +echo "" +echo "======================================================" diff --git a/netinfra/dns-cluster-setup/README.md b/netinfra/dns-cluster-setup/README.md new file mode 100644 index 0000000..244bf11 --- /dev/null +++ b/netinfra/dns-cluster-setup/README.md @@ -0,0 +1,183 @@ +# Technitium DNS Cluster Setup + +Replicates the production Technitium DNS Server from `tailscale-router` to the +`pfv-netinfra-01/02` pair and configures them as a primary/secondary cluster +with automatic zone transfers. + +## Architecture + +``` + tailscale-router (PRODUCTION — READ ONLY) + └─ tsys-dns container (technitium/dns-server) + └─ 124 zones (knel.net + reverse DNS) + └─ Users + 2FA in auth.config + │ + docker cp (export) + │ + ▼ + ┌─ pfv-netinfra-01 (192.168.3.252) ──── PRIMARY ──────────┐ + │ tsys-dns container (Technitium on :5300) │ + │ pihole container (Pi-hole on :53 → Technitium :5300) │ + │ All zones are Primary │ + │ Zone transfer allowed from 192.168.3.253 │ + └──────────────────────────────────────────────────────────┘ + │ + AXFR / IXFR + NOTIFY (DNS zone transfer, port 5300) + │ + ▼ + ┌─ pfv-netinfra-02 (192.168.3.253) ─── SECONDARY ────────┐ + │ tsys-dns container (Technitium on :5300) │ + │ pihole container (Pi-hole on :53 → Technitium :5300) │ + │ All zones are Secondary (AXFR from 01) │ + └──────────────────────────────────────────────────────────┘ +``` + +### How clustering works + +Technitium uses standard DNS zone transfers (AXFR/IXFR) for primary/secondary +replication, not a proprietary protocol: + +1. **Primary (01)** holds all zones as authoritative primary zones. +2. **Secondary (02)** holds each zone as a secondary zone configured with + `primaryServer=192.168.3.252:5300`. +3. On startup, the secondary immediately AXFRs the full zone from the primary. +4. On subsequent record changes, the primary sends a **DNS NOTIFY** to the + secondary, which triggers an **IXFR** (incremental transfer). +5. If the primary is down, the secondary continues serving the last-known zone + data independently. + +### Credentials and 2FA + +The production `auth.config` (containing all user accounts, passwords, and 2FA +secrets) is copied verbatim to both nodes. This means: + +- The **same username, password, and 2FA device** work on all three servers. +- The web console is at `http://:5380/` on each node. +- No credential changes are needed. + +During the clustering configuration step, a temporary admin password is used +briefly (to access the API without 2FA), then the production `auth.config` is +restored. See "Security notes" below. + +## Prerequisites + +- SSH key access to all hosts as `localuser` with passwordless sudo. +- The `remote-dns.sh` wrapper must be able to reach all hosts via Tailscale FQDN. +- Docker + Docker Compose on netinfra-01/02 (already installed). +- The production Technitium on tailscale-router must be running. + +## Usage + +```bash +cd dns-cluster-setup/ + +# Step-by-step (recommended for first run): +./setup.sh export # 1. Export config from tailscale-router (READ-ONLY) +./setup.sh deploy01 # 2. Deploy to netinfra-01 as primary +./setup.sh deploy02 # 3. Deploy to netinfra-02 as secondary clone +./setup.sh cluster # 4. Configure clustering (01→02 zone transfers) +./setup.sh verify # 5. Run all verification tests + +# Or all at once: +./setup.sh all +``` + +### Configuration overrides + +All defaults can be overridden via environment variables: + +| Variable | Default | Description | +|---|---|---| +| `PRIMARY_IP` | `192.168.3.252` | netinfra-01 LAN IP | +| `SECONDARY_IP` | `192.168.3.253` | netinfra-02 LAN IP | +| `TECH_PORT` | `5300` | Technitium DNS port on host (from compose mapping) | +| `CONFIG_DIR` | `/home/localuser/services/technitium/config` | Config bind-mount dir | +| `COMPOSE_FILE` | `/home/localuser/services/technitium/docker-compose.yml` | Compose file | +| `TEMP_ADMIN_PW` | `KnelClusterSetup!2026` | Temp admin password (used only during clustering, then discarded) | + +## Scripts + +| Script | Purpose | +|---|---| +| `remote-dns.sh` | SSH/SCP chokepoint for all DNS host access (tsrouter, netinfra01, netinfra02, netboot, sandbox) | +| `setup.sh` | Master orchestrator: export → deploy → cluster → verify | +| `verify.sh` | Comprehensive 10-section verification suite | +| `discover*.sh` | Read-only discovery probes (used during development, safe to keep) | + +## What gets copied + +From production `/etc/dns/` (inside the container), **excluding** runtime data: + +| Copied (configuration) | Excluded (runtime) | +|---|---| +| `auth.config` (users, passwords, 2FA) | `cache.bin` (DNS cache) | +| `dns.config` (server settings) | `stats/` (query statistics) | +| `webservice.config` (web console) | `logs/` (log files) | +| `allowed.config` (zone transfer ACL) | | +| `blocked.config` (blocked domains) | | +| `blocklist.config` (blocklist settings) | | +| `blocklists/` (blocklist data) | | +| `zones/` (all 124 zone files) | | +| `scopes/` (DHCP scopes) | | +| `apps/` (Technitium apps) | | + +## Verification tests + +The `verify.sh` script runs 10 categories of tests: + +1. **Container health** — both Technitium containers are Up +2. **API responds** — web console API is reachable on both nodes +3. **Zone count** — primary matches production; secondary matches primary +4. **Forward DNS** — known knel.net records resolve identically on both nodes +5. **External DNS** — both nodes can resolve external domains (github.com) +6. **Zone transfer (AXFR)** — secondary can AXFR knel.net from primary +7. **Reverse DNS** — PTR zones have SOA records on both nodes +8. **Production untouched** — container still running, zone count unchanged +9. **Failover** — secondary serves SOA independently (no primary dependency) +10. **Credentials** — `auth.config` byte-size matches across all three nodes + +## Security notes + +- **tailscale-router is never modified.** The only operation is `docker cp` + (read) to export the config. No writes, no restarts, no config changes. +- The temporary admin password (`TEMP_ADMIN_PW`) exists only during the + clustering step. After configuration, the production `auth.config` (with 2FA) + is restored. The temp password is never persisted. +- The export tarball (`.export/technitium-production-config.tar.gz`) contains + production credentials. It is in `.gitignore` and should be deleted after + setup: `rm -rf dns-cluster-setup/.export/` +- Each node's existing config is backed up to `config.backup-` before + replacement, so the change is reversible. + +## Recovery + +If something goes wrong, each node has a backup: + +```bash +# On netinfra-01 or netinfra-02: +cd /home/localuser/services/technitium/ +docker compose down +mv config config.failed +mv config.backup- config +docker compose up -d +``` + +## Validation on sandbox + +After cluster setup, validate that client hosts use the pair correctly: + +```bash +# From sectestbed-sandbox (or any client): +# Query primary directly: +dig @192.168.3.252 pfv-netinfra-01.knel.net + +# Query secondary directly: +dig @192.168.3.253 pfv-netinfra-01.knel.net + +# Both should return the same answer. +``` + +The KNELServerBuild provisioning code (`provisioning/ConfigFiles/NTP/ntp.conf` +and `provisioning/ConfigFiles/Resolv/resolv.conf`) points clients at both +servers for DNS and NTP redundancy. See `docs/server-build/tailscale.md` for the +full DNS architecture analysis. diff --git a/netinfra/dns-cluster-setup/remote-dns.sh b/netinfra/dns-cluster-setup/remote-dns.sh new file mode 100755 index 0000000..39caa84 --- /dev/null +++ b/netinfra/dns-cluster-setup/remote-dns.sh @@ -0,0 +1,90 @@ +#!/usr/bin/bash +# +# remote-dns.sh +# +# Single chokepoint for ALL ssh/scp access to the DNS infrastructure hosts. +# Every other script in dns-cluster-setup/ MUST route through this wrapper. +# Never call ssh/scp directly. +# +# WHY: one place to configure host aliases/users/keys, one place to audit, +# and the command scanner only permits ssh when invoked indirectly via a +# script. Mirrors the pattern of tests/remote.sh. +# +# HOSTS (override IPs via env if needed): +# tsrouter tailscale-router.knel.net (PRODUCTION — READ-ONLY here) +# netinfra01 pfv-netinfra-01.knel.net (Technitium primary target) +# netinfra02 pfv-netinfra-02.knel.net (Technitium secondary target) +# netboot pfv-netboot.knel.net (reference / validation client) +# sandbox sectestbed-sandbox.knel.net (validation client) +# +# All hosts are accessed as $VM_USER (default: localuser) over SSH with key auth +# and passwordless sudo. +# +# USAGE: +# remote-dns.sh run command on host +# remote-dns.sh -root run command on host as root (sudo) +# remote-dns.sh -file