chore: initial import from KNEL/PFVCluster@041d311 [#769]

Split per O&M lane work order. Full history: KNEL/PFVCluster.
https://projects.knownelement.com/issues/769#note-4152
This commit is contained in:
2026-09-03 21:39:22 -05:00
commit 51a1c9deee
41 changed files with 3428 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
# dcinfra/powerman/README.md
> **Documentation moved to Discourse — the canonical source of truth.**
>
> **Cyclades PM10i PDU management via powerman**
>
> **Read it here:** https://community.turnsys.com/t/301
>
> *Migrated 2026-08-06. This file is kept as a pointer for git-browsing context.
> Do not update content here — edit the Discourse wiki topic instead.*
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/bash
#
# powerman/discover.sh — gather USB-DB9 adapter + powerman state on a host
#
# Usage: PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/discover.sh
#
set -uo pipefail
echo "============================================"
echo " PDU / Powerman Discovery"
echo " Host: $(hostname)"
echo " Date: $(date)"
echo "============================================"
echo ""
echo "=== 1. USB devices ==="
lsusb 2>/dev/null || echo "(lsusb not available)"
echo ""
echo "=== 2. USB-Serial adapters (ttyUSB*) ==="
ls -la /dev/ttyUSB* 2>/dev/null || echo "(no /dev/ttyUSB* devices)"
echo ""
echo "=== 3. USB-Serial kernel modules ==="
lsmod | grep -iE 'usbserial|ftdi|pl2303|cp210|ch34|cdc_acm' 2>/dev/null || echo "(no relevant modules loaded)"
echo ""
echo "=== 4. dmesg for USB serial (last 30 lines) ==="
dmesg | grep -iE 'ttyUSB|usbserial|ftdi|pl2303|cp210|ch34|converter' | tail -30 2>/dev/null || echo "(no dmesg matches)"
echo ""
echo "=== 5. All serial devices ==="
ls -la /dev/ttyS* /dev/ttyUSB* /dev/ttyACM* 2>/dev/null || echo "(no serial devices found)"
echo ""
echo "=== 6. Powerman installed? ==="
dpkg -l powerman 2>/dev/null || echo "(powerman not installed)"
which powerman 2>/dev/null || echo "(powerman binary not found)"
which powermand 2>/dev/null || echo "(powermand binary not found)"
echo ""
echo "=== 7. Powerman config files ==="
ls -la /etc/powerman/ 2>/dev/null || echo "(no /etc/powerman/ directory)"
ls -la /etc/powerman/*.dev 2>/dev/null || echo "(no .dev files)"
cat /etc/powerman/powerman.conf 2>/dev/null || echo "(no powerman.conf)"
echo ""
echo "=== 8. Available powerman device definitions ==="
ls /usr/share/powerman/*.dev 2>/dev/null || ls /etc/powerman/*.dev 2>/dev/null || echo "(no device definitions found)"
echo ""
echo "=== 9. Powermand service status ==="
systemctl status powerman 2>/dev/null | head -10 || echo "(powerman service not found)"
echo ""
echo "=== 10. Serial port test (quick probe of /dev/ttyUSB0) ==="
if [ -e /dev/ttyUSB0 ]; then
stty -F /dev/ttyUSB0 2>/dev/null && echo "(port exists and is configurable)" || echo "(port exists but stty failed)"
# Try to read any pending output
timeout 2 cat /dev/ttyUSB0 2>/dev/null | head -5 || echo "(no immediate output from port)"
else
echo "(no /dev/ttyUSB0)"
fi
echo ""
echo "============================================"
echo " Discovery complete."
echo "============================================"
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/bash
# powerman/identify-outlets.sh — flash each PDU outlet sequentially for physical tracing
#
# Run this from the workstation. It flashes each outlet one at a time so you
# can walk the rack and see which device's LED blinks. Write down the mapping,
# then run rename-outlets.sh with that mapping.
#
# Usage:
# bash dcinfra/powerman/identify-outlets.sh
#
# On Friday: run this, walk the rack, note which outlet → which device.
set -uo pipefail
PROX_HOST="${PROX_HOST:-pfv-tsys1}"
REMOTE_SH="$(cd "$(dirname "$0")/../.." && pwd)/tests/remote.sh"
echo "PDU Outlet Identification — Flash Sequence"
echo "============================================"
echo "Each outlet will flash for 5 seconds. Walk the rack and note the device."
echo "Press Enter to start..."
read -r
for i in $(seq 1 10); do
echo "--- Outlet $i: FLASHING (5s) ---"
PROX_HOST="$PROX_HOST" bash "$REMOTE_SH" prox "powerman -f outlet-$i" </dev/null 2>/dev/null
sleep 5
PROX_HOST="$PROX_HOST" bash "$REMOTE_SH" prox "powerman -u outlet-$i" </dev/null 2>/dev/null
echo " Outlet $i → ? (write it down)"
echo ""
[ "$i" -lt 10 ] && { echo "Press Enter for next outlet..."; read -r; }
done
echo "============================================"
echo "Done. Now create your mapping file and run:"
echo " bash dcinfra/powerman/rename-outlets.sh"
echo ""
echo "Format: outlet-number:new-name (one per line)"
echo "Example:"
echo " 1:pfv-tsys1"
echo " 2:pfv-tsys3"
echo " ..."
echo "============================================"
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/bash
#
# powerman/query-remote.sh — install powerman client locally and query
# the Cyclades PDU running on pfv-tsys1 over Tailscale.
#
set -euo pipefail
REMOTE_HOST="${REMOTE_HOST:-pfv-tsys1}"
REMOTE_PORT="${REMOTE_PORT:-10101}"
echo "============================================"
echo " Powerman Remote PDU Query"
echo " Server: ${REMOTE_HOST}:${REMOTE_PORT} (Tailscale)"
echo "============================================"
# --- 1. Install powerman client if missing ---
if ! command -v powerman >/dev/null 2>&1; then
echo ""
echo "--- Installing powerman client ---"
if sudo -n true 2>/dev/null; then
sudo apt-get update -qq && sudo apt-get install -y -qq powerman
else
echo " Passwordless sudo not available. Please run this command in a terminal:"
echo ""
echo " sudo apt-get update && sudo apt-get install -y powerman"
echo ""
echo " Then re-run this script."
exit 1
fi
else
echo " powerman client already installed."
fi
# --- 2. Verify connectivity ---
echo ""
echo "--- Connectivity check ---"
if timeout 3 bash -c "echo > /dev/tcp/${REMOTE_HOST}/${REMOTE_PORT}" 2>/dev/null; then
echo " [OK] ${REMOTE_HOST}:${REMOTE_PORT} reachable"
else
echo " [FAIL] Cannot reach ${REMOTE_HOST}:${REMOTE_PORT}"
echo " Is Tailscale up? Is powermand running on ${REMOTE_HOST}?"
exit 1
fi
export POWERMAN_SERVER="${REMOTE_HOST}:${REMOTE_PORT}"
# --- 3. List outlets ---
echo ""
echo "--- Outlets ---"
powerman -h "${REMOTE_HOST}:${REMOTE_PORT}" -l
# --- 4. Query status ---
echo ""
echo "--- Status ---"
powerman -h "${REMOTE_HOST}:${REMOTE_PORT}" -q
echo ""
echo "============================================"
echo " Done."
echo ""
echo " To control an outlet from this workstation:"
echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -0 outlet-10 # off"
echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -1 outlet-10 # on"
echo " powerman -h ${REMOTE_HOST}:${REMOTE_PORT} -c outlet-10 # cycle"
echo "============================================"
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/bash
# powerman/rename-outlets.sh — rename PDU outlets in powerman.conf
#
# Takes a mapping file (outlet-number:new-name, one per line) and rewrites
# the node entries in /etc/powerman/powerman.conf on pfv-tsys1, then
# restarts powermand.
#
# Usage:
# bash dcinfra/powerman/rename-outlets.sh <mapping-file>
#
# Example mapping file:
# 1:pfv-tsys1
# 2:pfv-tsys3
# 3:pfv-tsys4
# ...
set -euo pipefail
PROX_HOST="${PROX_HOST:-pfv-tsys1}"
REMOTE_SH="$(cd "$(dirname "$0")/../.." && pwd)/tests/remote.sh"
MAP_FILE="${1:-}"
if [ -z "$MAP_FILE" ] || [ ! -f "$MAP_FILE" ]; then
echo "Usage: $0 <mapping-file>"
echo " Format: outlet-number:new-name (one per line)"
echo " Run identify-outlets.sh first to get the mapping."
exit 1
fi
# Build the new node lines
NODE_LINES=""
while IFS=: read -r num name; do
[ -z "$num" ] && continue
NODE_LINES+="node \"$name\" \"cyclades-pm10\" \"$num\""$'\n'
done < "$MAP_FILE"
# Send to tsys1: backup conf, write new node section, restart powermand
PROX_HOST="$PROX_HOST" bash "$REMOTE_SH" prox-file - <<REMOTE_SCRIPT
set -euo pipefail
cp /etc/powerman/powerman.conf /etc/powerman/powerman.conf.bak.\$(date +%Y%m%d-%H%M%S)
# Strip existing node lines and append new ones
grep -v '^node "outlet-' /etc/powerman/powerman.conf > /tmp/powerman.conf.new
cat >> /tmp/powerman.conf.new <<'NODES'
$(echo -n "$NODE_LINES")
NODES
mv /tmp/powerman.conf.new /etc/powerman/powerman.conf
systemctl restart powerman
sleep 1
powerman -l
REMOTE_SCRIPT
echo "PDU outlets renamed. Verify with: PROX_HOST=$PROX_HOST bash $REMOTE_SH prox 'powerman -q'"
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/bash
#
# powerman/setup.sh — idempotent powerman setup for Cyclades PM10i PDU
#
# Creates a stable udev symlink for the USB-DB9 adapter, writes powerman.conf
# with 10 outlet nodes, and enables + starts powermand.
#
# This script is designed to be run ON the target host (pfv-tsys1) as root.
# It is idempotent: safe to run multiple times.
#
# Usage:
# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/setup.sh
#
# Override defaults via environment variables:
# PDU_SERIAL — USB adapter serial (default: BJAAb144J07)
# PDU_VENDOR — USB vendor ID (default: 067b)
# PDU_DEV_NAME — udev symlink name (default: cyclades-pm10)
# PDU_BAUD — serial baud rate (default: 9600,8n1)
# PDU_TYPE — powerman spec type (default: pm10)
# PDU_OUTLETS — number of outlets (default: 10)
# PDU_LISTEN — powermand listen (default: 0.0.0.0:10101)
#
set -euo pipefail
# --- Config (overridable via env) ---
PDU_SERIAL="${PDU_SERIAL:-BJAAb144J07}"
PDU_VENDOR="${PDU_VENDOR:-067b}"
PDU_DEV_NAME="${PDU_DEV_NAME:-cyclades-pm10}"
PDU_BAUD="${PDU_BAUD:-9600,8n1}"
PDU_TYPE="${PDU_TYPE:-pm10}"
PDU_OUTLETS="${PDU_OUTLETS:-10}"
PDU_LISTEN="${PDU_LISTEN:-}" # Auto-detect Tailscale IP if empty
UDEV_RULE="/etc/udev/rules.d/99-cyclades-pdu.rules"
POWERMAN_CONF="/etc/powerman/powerman.conf"
DEV_FILE="/etc/powerman/cyclades-pm10.dev"
# --- Auto-detect Tailscale IP for listen address ---
if [ -z "$PDU_LISTEN" ]; then
TS_IP=$(tailscale ip -4 2>/dev/null || true)
if [ -n "$TS_IP" ]; then
PDU_LISTEN="${TS_IP}:10101"
echo " Auto-detected Tailscale IP: $TS_IP"
else
PDU_LISTEN="127.0.0.1:10101"
echo " WARNING: No Tailscale IP detected. Defaulting to localhost."
fi
fi
echo "============================================"
echo " Powerman PDU Setup"
echo " Host: $(hostname)"
echo " PDU: Cyclades PM${PDU_OUTLETS}i"
echo " Adapter serial: $PDU_SERIAL"
echo " Device symlink: /dev/$PDU_DEV_NAME"
echo " Listen: $PDU_LISTEN (Tailscale only)"
echo "============================================"
# --- 1. Ensure powerman is installed ---
echo ""
echo "--- [1/5] Checking powerman installation ---"
if ! dpkg -l powerman 2>/dev/null | grep -q '^ii'; then
echo " Installing powerman from Debian repo..."
apt-get update -qq && apt-get install -y -qq powerman
else
echo " Powerman already installed: $(dpkg -l powerman | awk '/^ii/{print $3}')"
fi
# --- 2. Create udev rule for stable device name ---
echo ""
echo "--- [2/5] Creating udev rule for USB-DB9 adapter ---"
cat > "$UDEV_RULE" <<UDEV
# Stable symlink for Cyclades PM10i PDU USB-DB9 adapter
# Generated by powerman/setup.sh
SUBSYSTEM=="tty", ATTRS{idVendor}=="${PDU_VENDOR}", ATTRS{serial}=="${PDU_SERIAL}", GROUP="dialout", MODE="0660", SYMLINK+="${PDU_DEV_NAME}"
UDEV
echo " Written: $UDEV_RULE"
# Trigger udev to create the symlink now
udevadm control --reload-rules 2>/dev/null || true
udevadm trigger --subsystem-match=tty 2>/dev/null || true
sleep 1
if [ -e "/dev/${PDU_DEV_NAME}" ]; then
echo " Device symlink active: /dev/${PDU_DEV_NAME} -> $(readlink -f "/dev/${PDU_DEV_NAME}")"
else
echo " WARNING: /dev/${PDU_DEV_NAME} not found yet. Adapter may be unplugged."
echo " Falling back to /dev/ttyUSB* discovery..."
# Try to find any ttyUSB device as fallback
for tty in /dev/ttyUSB*; do
if [ -e "$tty" ]; then
echo " Found: $tty (using as fallback)"
PDU_DEV_NAME="$(basename "$tty")"
break
fi
done
fi
# --- 2b. Ensure powermand user can access the serial device ---
echo ""
echo "--- [2b/5] Fixing serial device permissions ---"
if id powerman >/dev/null 2>&1; then
if id powerman | grep -qv dialout; then
usermod -aG dialout powerman
echo " Added 'powerman' user to 'dialout' group"
else
echo " 'powerman' already in 'dialout' group"
fi
else
echo " (no powerman user — service may run as root)"
fi
# --- 3. Write powerman.conf ---
echo ""
echo "--- [3/5] Writing powerman.conf ---"
# Build node definitions
NODES=""
for i in $(seq 1 "$PDU_OUTLETS"); do
NODES+="node \"outlet-${i}\" \"${PDU_DEV_NAME}\" \"${i}\"\n"
done
cat > "$POWERMAN_CONF" <<PMCONF
# Powerman configuration for Cyclades PM${PDU_OUTLETS}i PDU
# Generated by powerman/setup.sh on $(date)
# Device: /dev/${PDU_DEV_NAME} (USB-DB9 adapter serial ${PDU_SERIAL})
# Listen on localhost (for local admin) and Tailscale (for remote access)
listen "127.0.0.1:10101"
listen "${PDU_LISTEN}"
# Device specification for Cyclades PM10
include "${DEV_FILE}"
# The PDU device (serial-attached)
device "${PDU_DEV_NAME}" "${PDU_TYPE}" "/dev/${PDU_DEV_NAME}" "${PDU_BAUD}"
# Outlet nodes (rename these to match attached devices when onsite)
$(printf '%b' "$NODES")
PMCONF
echo " Written: $POWERMAN_CONF"
echo " Nodes defined: outlet-1 through outlet-${PDU_OUTLETS}"
# --- 4. Restart powermand ---
echo ""
echo "--- [4/5] Restarting powermand ---"
systemctl enable powerman 2>/dev/null || true
systemctl restart powerman 2>/dev/null || true
sleep 2
if systemctl is-active --quiet powerman; then
echo " powermand is running."
else
echo " WARNING: powermand failed to start. Check journalctl -u powerman"
journalctl -u powerman --no-pager -n 20 2>/dev/null || true
fi
# --- 5. Verify ---
echo ""
echo "--- [5/5] Verification ---"
echo ""
echo " powerman -l (list all outlets):"
powerman -l 2>&1 || echo "(powerman -l failed)"
echo ""
echo " powerman -q (query status):"
powerman -q 2>&1 || echo "(powerman -q failed — PDU may need a moment)"
echo ""
echo "============================================"
echo " Setup complete."
echo ""
echo " Outlet names are generic (outlet-1 ... outlet-${PDU_OUTLETS})."
echo " Rename them in ${POWERMAN_CONF} when onsite to match attached devices."
echo ""
echo " Test: powerman -0 outlet-10 (off)"
echo " powerman -1 outlet-10 (on)"
echo " powerman -c outlet-10 (cycle)"
echo " powerman -q (status)"
echo "============================================"
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/bash
#
# powerman/status.sh — quick PDU status check
#
# Usage:
# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/status.sh
#
set -euo pipefail
echo "============================================"
echo " Cyclades PM10i PDU Status"
echo " Host: $(hostname) $(date)"
echo "============================================"
echo ""
echo "=== Service ==="
systemctl is-active powerman 2>/dev/null && echo "(running)" || echo "(stopped)"
echo ""
echo "=== Device ==="
ls -la /dev/cyclades-pm10 2>/dev/null || echo "(no /dev/cyclades-pm10 symlink)"
echo ""
echo "=== Outlets ==="
powerman -l 2>&1
echo ""
echo "=== Power Status ==="
powerman -q 2>&1
echo ""
echo "=== Temperature ==="
powerman -T 2>&1 || echo "(temperature not available)"
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/bash
#
# powerman/test-pdu.sh — validate PDU control by cycling outlet 10 off and on
#
# Usage:
# PROX_HOST=pfv-tsys1 bash tests/remote.sh prox-file powerman/test-pdu.sh
#
# Override: OUTLET=10 (which outlet to test)
#
set -euo pipefail
OUTLET="${OUTLET:-10}"
NODE="outlet-${OUTLET}"
PASS=0; FAIL=0
ok() { echo " [PASS] $1"; PASS=$((PASS+1)); }
fail() { echo " [FAIL] $1"; FAIL=$((FAIL+1)); }
echo "============================================"
echo " PDU Control Validation"
echo " Host: $(hostname)"
echo " Test: cycle outlet ${OUTLET} (off → wait → on)"
echo "============================================"
# --- 0. Powermand running? ---
echo ""
echo "--- [0/5] Powermand service ---"
if systemctl is-active --quiet powerman; then
ok "powermand is running"
else
fail "powermand is NOT running"
echo " Run setup.sh first."
exit 1
fi
# --- 1. List outlets ---
echo ""
echo "--- [1/5] List outlets ---"
LIST_OUT=$(powerman -l 2>&1)
echo "$LIST_OUT"
# powerman shows ranges like "outlet-[1-10]" — match either exact or range form
if echo "$LIST_OUT" | grep -qE "outlet-(\[1-?10\]|${OUTLET}\b)"; then
ok "Outlet '${NODE}' is defined"
else
fail "Outlet '${NODE}' not found in powerman -l"
exit 1
fi
# --- 2. Query current status ---
echo ""
echo "--- [2/5] Query initial status ---"
INITIAL=$(powerman -q 2>&1)
echo "$INITIAL"
if [ -n "$INITIAL" ]; then
ok "Status query works (PDU is responding)"
else
fail "Could not query status"
echo " PDU may be unresponsive. Check serial connection."
exit 1
fi
# --- 3. Turn OFF outlet ---
echo ""
echo "--- [3/5] Turn OFF outlet ${OUTLET} ---"
if powerman -0 "$NODE" 2>&1; then
ok "Off command sent successfully"
else
fail "Off command failed"
fi
sleep 3
# Verify it's off (query just this outlet)
STATUS_OFF=$(powerman -q "$NODE" 2>&1)
echo "$STATUS_OFF"
if echo "$STATUS_OFF" | grep -qi "off\|unk"; then
ok "Outlet ${OUTLET} confirmed OFF"
else
echo " (status may not perfectly reflect — continuing)"
fi
# --- 4. Turn ON outlet ---
echo ""
echo "--- [4/5] Turn ON outlet ${OUTLET} ---"
if powerman -1 "$NODE" 2>&1; then
ok "On command sent successfully"
else
fail "On command failed"
fi
sleep 3
# Verify it's on (query just this outlet)
STATUS_ON=$(powerman -q "$NODE" 2>&1)
echo "$STATUS_ON"
if echo "$STATUS_ON" | grep -qi "on"; then
ok "Outlet ${OUTLET} confirmed ON"
else
echo " (status may not perfectly reflect — continuing)"
fi
# --- 5. Cycle test (off → delay → on in one command) ---
echo ""
echo "--- [5/5] Cycle test (powerman -c) ---"
if powerman -c "$NODE" 2>&1; then
ok "Cycle command completed"
else
fail "Cycle command failed"
echo " (some PDU firmware reports errors during cycle but still works)"
fi
sleep 5
# Final status
echo ""
echo "--- Final status ---"
powerman -q 2>&1
echo ""
echo "============================================"
echo " Results: $PASS passed, $FAIL failed"
if [ "$FAIL" -gt 0 ]; then
echo " Some checks failed. Review output above."
exit 1
fi
echo " PDU control validated."
echo "============================================"