diff --git a/console/generate-config.sh b/console/generate-config.sh index 9d2ba78..a61b9ad 100644 --- a/console/generate-config.sh +++ b/console/generate-config.sh @@ -121,7 +121,7 @@ for entry in "${ENTRIES[@]}"; do { echo "" echo "# $name (TCP $tcp_port): $comment" - echo "SUBSYSTEM==\"tty\", ENV{ID_PATH}==\"*$id_path*\", SYMLINK+=\"console/$name\"" + echo "SUBSYSTEM==\"tty\", ENV{ID_PATH}==\"*$id_path*\", SYMLINK+=\"consoles/$name\"" } >> "$UDEV_RULES" done diff --git a/perf/scripts/conman-console.py b/perf/scripts/conman-console.py new file mode 100644 index 0000000..9516fff --- /dev/null +++ b/perf/scripts/conman-console.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +conman-console.py — Drive a serial console via a conman server, read-only. + +Connects to a conmand server (RFC 127-style multiplexer) over the network, +opens a named console, sends the commands from a .cmds file, captures all +output, and disconnects. Requires no expect/tcl — pure stdlib PTY. + +This replaces the old sw-capture-remote.sh workflow that killed the serial +device holder (conflicting with conman/ser2net). Instead, it talks to conman +over TCP, which multiplexes safely with other sessions. + +All endpoints are configurable via environment variables so this works on +any network with a conman server: + + CONMAN_SERVER conman server host:port (default: via CONSOLE_HOST) + CONSOLE console name to open (required) + CMDS_FILE file of commands to send (required) + TIMEOUT overall timeout in seconds (default: 45) + CMD_DELAY seconds between commands (default: 3) + WAKE_DELAY seconds after connect (default: 2) + +Usage: + CONMAN_SERVER=console-host:7890 \\ + python3 conman-console.py --console pfv-core-sw01 --cmds switches/pfv-core-sw01.cmds + +Lines starting with '!' or '#' in the cmds file are comments (skipped). +Blank lines are skipped. The conman escape sequence (&.) is sent automatically +to disconnect. A password prompt aborts immediately (we never send creds). + +Exit codes: + 0 clean run + 1 usage / setup error + 2 could not connect to conman server + 3 timeout (partial output still printed) + 4 password prompt encountered (aborted) +""" +import argparse +import os +import pty +import re +import select +import sys +import time + +PWD_RE = re.compile(rb"[Pp]assword:\s*$") +MORE_RE = re.compile(rb"--\s*[Mm]ore\s*--|[Mm]ore:\s*") + + +def main(): + ap = argparse.ArgumentParser( + description="Drive a conman console session read-only via PTY") + ap.add_argument("--console", required=True, + help="console name (e.g. pfv-core-sw01)") + ap.add_argument("--cmds", required=True, + help="command file (one command per line; !/# = comment)") + ap.add_argument("--server", + default=os.environ.get("CONMAN_SERVER", ""), + help="conman server host:port (env: CONMAN_SERVER)") + ap.add_argument("--timeout", type=int, + default=int(os.environ.get("TIMEOUT", "45")), + help="overall timeout seconds (env: TIMEOUT)") + ap.add_argument("--cmd-delay", type=float, + default=float(os.environ.get("CMD_DELAY", "3")), + help="seconds between commands (env: CMD_DELAY)") + ap.add_argument("--wake-delay", type=float, + default=float(os.environ.get("WAKE_DELAY", "2")), + help="seconds after connect before first command (env: WAKE_DELAY)") + args = ap.parse_args() + + if not args.server: + sys.stderr.write("ERROR: --server or CONMAN_SERVER env required\n") + return 1 + + with open(args.cmds) as f: + cmds = [l.strip() for l in f + if l.strip() and not l.strip().startswith(("!", "#"))] + + pid, fd = pty.fork() + if pid == 0: + try: + os.execvp("conman", + ["conman", "-d", args.server, "-f", args.console]) + except OSError as e: + sys.stderr.write(f"ERROR: cannot exec conman: {e}\n") + os._exit(2) + os._exit(2) + + output = b"" + cmd_queue = list(cmds) + sent_disconnect = False + start = time.time() + last_action = 0.0 + phase = "connect" + + while time.time() - start < args.timeout: + ready, _, _ = select.select([fd], [], [], 0.5) + if ready: + try: + data = os.read(fd, 8192) + except OSError: + break + if not data: + break + output += data + + if PWD_RE.search(output.split(b"\n")[-1] if output else b""): + sys.stderr.write("[ABORT] password prompt detected — " + "never sending credentials\n") + os.write(fd, b"&.\n") + break + + # Handle pagination: send space to continue + if MORE_RE.search(output[-200:] if output else b""): + os.write(fd, b" ") + time.sleep(0.5) + + elapsed = time.time() - start + gap = elapsed - last_action + + if phase == "connect" and gap >= args.wake_delay: + os.write(fd, b"\n") + phase = "send" + last_action = elapsed + elif phase == "send" and gap >= args.cmd_delay: + if cmd_queue: + cmd = cmd_queue.pop(0) + os.write(fd, (cmd + "\n").encode()) + last_action = elapsed + else: + phase = "drain" + last_action = elapsed + elif phase == "drain" and gap >= args.cmd_delay: + os.write(fd, b"&.\n") + sent_disconnect = True + phase = "done" + last_action = elapsed + elif phase == "done" and gap >= 2: + break + + if not sent_disconnect: + try: + os.write(fd, b"&.\n") + except OSError: + pass + + try: + os.close(fd) + except OSError: + pass + try: + os.waitpid(pid, 0) + except ChildProcessError: + pass + + sys.stdout.buffer.write(output) + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/perf/scripts/probe-network.sh b/perf/scripts/probe-network.sh new file mode 100644 index 0000000..f9c2ba8 --- /dev/null +++ b/perf/scripts/probe-network.sh @@ -0,0 +1,82 @@ +#!/bin/bash +############################################################################### +# probe-network.sh +# +# READ-ONLY network + NFS ground-truth probe. Writes only stdout. +# Run on any Proxmox host to inventory its NICs, bonds, LLDP neighbors, +# NFS client mounts (including nconnect), ethtool link state, and error +# counters. No hardcoded values — fully portable. +# +# Usage (via tests/remote.sh): +# PROX_HOST=pfv-tsys6 bash tests/remote.sh prox-file perf/scripts/probe-network.sh +############################################################################### +set -u +echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) =====" +echo +echo "##### ip -br link #####" +ip -br link 2>&1 +echo +echo "##### ip -br addr #####" +ip -br addr 2>&1 +echo +echo "##### /etc/network/interfaces #####" +cat /etc/network/interfaces 2>&1 +echo +echo "##### bond0 state (if present) #####" +if [ -r /proc/net/bonding/bond0 ]; then + cat /proc/net/bonding/bond0 2>&1 +else + echo "(no bond0)" +fi +echo +echo "##### ethtool per physical NIC #####" +for nic in /sys/class/net/*; do + nic=$(basename "$nic") + case "$nic" in lo|bond*|br*|venet*|veth*|docker*|tap*|vnet*|fw*) continue;; esac + echo "--- ethtool $nic ---" + ethtool "$nic" 2>&1 | grep -iE 'Speed|Duplex|Port|Link|Supported link modes|Advertising|Auto-neg|Settings' || echo "(ethtool failed for $nic)" +done +echo +echo "##### lldpcli (if installed) #####" +if command -v lldpcli >/dev/null 2>&1; then + echo "--- lldpcli show neighbors ---" + lldpcli show neighbors 2>&1 + echo + echo "--- lldpcli show interfaces ---" + lldpcli show interfaces 2>&1 + echo + echo "--- lldpcli show chassis ---" + lldpcli show chassis 2>&1 +else + echo "(lldpcli not installed)" +fi +echo +echo "##### lldpd / lldpad service #####" +systemctl is-active lldpd 2>&1 || true +systemctl is-enabled lldpd 2>&1 || true +echo +echo "##### NFS mounts (mount | grep nfs) #####" +mount | grep -i nfs 2>&1 || echo "(no nfs mounts)" +echo +echo "##### mount nconnect detail (nfsstat -m) #####" +nfsstat -m 2>&1 +echo +echo "##### storage.cfg NFS stanzas (options) #####" +grep -A3 '^nfs:' /etc/pve/storage.cfg 2>&1 +echo +echo "##### ip route #####" +ip route 2>&1 +echo +echo "##### ethtool -S bond slaves (key counters) #####" +if [ -r /proc/net/bonding/bond0 ]; then + # shellcheck disable=SC2013 # intentional: extract NIC names from bonding info + for nic in $(grep -oE 'eth[0-9]+|en[psx][a-z0-9]+' /proc/net/bonding/bond0 2>/dev/null | sort -u); do + echo "--- ethtool -S $nic (errors) ---" + ethtool -S "$nic" 2>/dev/null | grep -iE 'error|drop|discard|crc|pause|miss' || echo "(no error counters)" + done +fi +echo +echo "##### ip neigh (ARP table, reachable/stale) #####" +ip neigh show 2>&1 | grep -vE ' FAILED|INCOMPLETE' | sort -t. -k4 -n +echo +echo "===== END $(hostname -s) =====" diff --git a/perf/scripts/probe-storage.sh b/perf/scripts/probe-storage.sh new file mode 100644 index 0000000..423381b --- /dev/null +++ b/perf/scripts/probe-storage.sh @@ -0,0 +1,75 @@ +#!/bin/bash +############################################################################### +# probe-storage.sh +# +# READ-ONLY storage + disk ground-truth probe. Writes only stdout. +# Run on any Proxmox host (or any Linux NFS server) to inventory its physical +# disks, mounts, exports, SMART health, and Proxmox storage config. +# +# Portable: no hardcoded values. Uses only standard CLI tools + smartmontools. +# +# Usage (via tests/remote.sh): +# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file perf/scripts/probe-storage.sh +# +# Or directly on a host: +# bash probe-storage.sh > storage-audit.txt +############################################################################### +set -u +echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) =====" +echo +echo "##### lsblk (tree, with model/serial/size/type) #####" +lsblk -o NAME,MAJ:MIN,SIZE,TYPE,MOUNTPOINT,MODEL,SERIAL,STATE,ROTA,TRAN,REV 2>&1 +echo +echo "##### block devices by-id #####" +for dev in /dev/disk/by-id/*; do + [ -L "$dev" ] || continue + case "$(basename "$dev")" in *part[0-9]*) continue;; esac + ls -l "$dev" +done 2>&1 +echo +echo "##### nvme list (if any) #####" +command -v nvme >/dev/null 2>&1 && nvme list 2>&1 || echo "(no nvme-cli or no nvme devices)" +echo +echo "##### blkid #####" +blkid 2>&1 +echo +echo "##### mounted filesystems #####" +findmnt -o TARGET,SOURCE,FSTYPE,OPTIONS 2>&1 +echo +echo "##### /etc/fstab #####" +cat /etc/fstab 2>&1 +echo +echo "##### /etc/exports (+ exports.d) #####" +cat /etc/exports 2>&1 +for f in /etc/exports.d/*.exports; do [ -f "$f" ] && echo "--- $f ---" && cat "$f"; done 2>&1 +echo +echo "##### df -h (all mounts) #####" +df -h 2>&1 +echo +echo "##### smartctl -a per block device #####" +command -v smartctl >/dev/null 2>&1 || echo "(smartctl not installed)" +for d in /dev/sd? /dev/nvme?n1; do + [ -b "$d" ] || continue + echo "----- smartctl -a $d -----" + smartctl -a "$d" 2>&1 | grep -iE 'Device Model|Model Number|Serial|Firmware|User Capacity|Rotation Rate|Form Factor|SATA Version|NVMe|SMART overall|Reallocated|Pending|Uncorrect|Power On|Temperature|Media and Data Integrity' || true +done +echo +echo "##### /etc/pve/storage.cfg #####" +cat /etc/pve/storage.cfg 2>&1 +echo +echo "##### pvesm status #####" +pvesm status 2>&1 +echo +echo "##### pvesm list per store #####" +for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3>0 {print $1}'); do + echo "--- pvesm list $s ---" + pvesm list "$s" 2>&1 | head -40 +done +echo +echo "##### zpool status (if any) #####" +command -v zpool >/dev/null 2>&1 && zpool status 2>&1 || echo "(no zfs)" +echo +echo "##### lvm: pvs/vgs/lvs #####" +command -v pvs >/dev/null 2>&1 && { pvs 2>&1; echo; vgs 2>&1; echo; lvs 2>&1; } || echo "(no lvm tools)" +echo +echo "===== END $(hostname -s) =====" diff --git a/perf/scripts/snmp-switch-audit.py b/perf/scripts/snmp-switch-audit.py new file mode 100644 index 0000000..1075f48 --- /dev/null +++ b/perf/scripts/snmp-switch-audit.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +""" +snmp-switch-audit.py — READ-ONLY switch inventory via SNMP. + +Gathers interface status/speed/errors, LLDP neighbor topology, port-channel +(LAG) membership, and VLAN membership from any SNMPv2c-capable switch. +Designed for Dell/Radlan (Neyland) and standard IF/LLDP/Q-BRIDGE MIB switches, +but works on any SNMP-manageable device. + +All parameters configurable via env vars or CLI flags so this works on any +network: + + SNMP_COMMUNITY SNMPv2c community string (env, default: public) + SWITCH_IPS space-separated switch IPs (env, or pass as args) + OUTPUT_DIR where to write per-switch (env, default: returned-logs/snmp) + +Usage: + SNMP_COMMUNITY=kn3lmgmt SWITCH_IPS="192.168.0.9 192.168.0.12" \\ + python3 snmp-switch-audit.py + + # or pass IPs as positional args: + SNMP_COMMUNITY=kn3lmgmt python3 snmp-switch-audit.py 192.168.0.9 192.168.0.12 + +Requires: pysnmp (pip install pysnmp) or net-snmp utils (snmpwalk) on PATH. +Outputs: per-switch JSON + human-readable text in OUTPUT_DIR. + +Read-only: sends only SNMP GET/GETNEXT/GETBULK. Never SETs anything. +""" +import argparse +import json +import os +import re +import subprocess +import sys + +COMMUNITY = os.environ.get("SNMP_COMMUNITY", "public") +OUTPUT_DIR = os.environ.get("OUTPUT_DIR", + os.path.join(os.path.dirname(__file__), "..", + "..", "returned-logs", "snmp")) + +# OID constants +OID_SYSDESCR = "1.3.6.1.2.1.1.1.0" +OID_SYSNAME = "1.3.6.1.2.1.1.5.0" +OID_IF_NAME = "1.3.6.1.2.1.31.1.1.1.1" +OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5" +OID_IF_OPER = "1.3.6.1.2.1.2.2.1.8" +OID_IF_INERR = "1.3.6.1.2.1.2.2.1.14" +OID_IF_OUTERR = "1.3.6.1.2.1.2.2.1.20" +OID_IF_INOCT = "1.3.6.1.2.1.31.1.1.1.6" +OID_IF_OUTOCT = "1.3.6.1.2.1.31.1.1.1.10" +OID_LACP_LAG = "1.2.840.10006.300.43.1.1.1.1" +OID_LLDP_REM_PORT = "1.0.8802.1.1.2.1.4.1.1.7" +OID_LLDP_REM_SYSNAME = "1.0.8802.1.1.2.1.4.1.1.9" +OID_LLDP_REM_CHASSIS = "1.0.8802.1.1.2.1.4.1.1.6" +OID_LLDP_REM_LOCALPORT = "1.0.8802.1.1.2.1.4.1.1.3" +OID_QBRIDGE_VLAN = "1.3.6.1.2.1.17.7.1.4.3.1.1" + + +def snmpget(ip, oid): + """Single SNMP GET, returns string value or None.""" + try: + r = subprocess.run( + ["snmpget", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid], + capture_output=True, text=True, timeout=10) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip().strip('"') + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + return None + + +def snmpwalk(ip, oid): + """SNMP BULKWALK, returns dict of ifIndex -> value.""" + try: + r = subprocess.run( + ["snmpbulkwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid], + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + r = subprocess.run( + ["snmpwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid], + capture_output=True, text=True, timeout=30) + except (subprocess.TimeoutExpired, FileNotFoundError): + return {} + result = {} + for i, line in enumerate(r.stdout.strip().split("\n"), 1): + line = line.strip().strip('"') + if line: + result[i] = line + return result + + +def walk_indexed(ip, oid): + """SNMP walk preserving OID index. Returns dict: index_str -> value.""" + try: + r = subprocess.run( + ["snmpbulkwalk", "-v2c", "-c", COMMUNITY, ip, oid], + capture_output=True, text=True, timeout=30) + if r.returncode != 0: + r = subprocess.run( + ["snmpwalk", "-v2c", "-c", COMMUNITY, ip, oid], + capture_output=True, text=True, timeout=30) + except (subprocess.TimeoutExpired, FileNotFoundError): + return {} + result = {} + for line in r.stdout.strip().split("\n"): + m = re.search(r'(\d+)\s*=\s*(.+)', line) + if not m: + m = re.search(r'\.(\d+)\s*=\s*(.+)', line) + if m: + idx = m.group(1).split(".")[-1] + val = m.group(2).strip() + val = re.sub(r'^(INTEGER: |STRING: |Hex-STRING: |Gauge32: |Counter32: |Counter64: )', '', val) + result[idx] = val + return result + + +def audit_switch(ip): + """Gather all data for one switch.""" + data = {"ip": ip} + data["sysDescr"] = snmpget(ip, OID_SYSDESCR) + data["sysName"] = snmpget(ip, OID_SYSNAME) + if not data["sysDescr"]: + return data + + names = snmpwalk(ip, OID_IF_NAME) + speeds = snmpwalk(ip, OID_IF_SPEED) + oper = snmpwalk(ip, OID_IF_OPER) + inerr = snmpwalk(ip, OID_IF_INERR) + outerr = snmpwalk(ip, OID_IF_OUTERR) + + interfaces = [] + for idx in sorted(names.keys()): + if idx not in names: + continue + speed_raw = speeds.get(idx, "0") + try: + speed_mbps = int(re.sub(r'\D', '', str(speed_raw))) // 1000000 + except (ValueError, TypeError): + speed_mbps = 0 + is_up = str(oper.get(idx, "0")).strip() == "1" + interfaces.append({ + "ifIndex": idx, + "name": names[idx], + "speedMbps": speed_mbps, + "up": is_up, + "inErrors": inerr.get(idx, "0"), + "outErrors": outerr.get(idx, "0"), + }) + data["interfaces"] = interfaces + + # LLDP neighbors + rem_ports = walk_indexed(ip, OID_LLDP_REM_PORT) + rem_sysnames = walk_indexed(ip, OID_LLDP_REM_SYSNAME) + rem_chassis = walk_indexed(ip, OID_LLDP_REM_CHASSIS) + rem_local = walk_indexed(ip, OID_LLDP_REM_LOCALPORT) + lldp = [] + for idx in rem_ports: + lldp.append({ + "localPort": rem_local.get(idx, "?"), + "remotePort": rem_ports[idx], + "remoteSysName": rem_sysnames.get(idx, ""), + "remoteChassis": rem_chassis.get(idx, ""), + }) + data["lldpNeighbors"] = lldp + + # LACP LAG table + lag_data = walk_indexed(ip, OID_LACP_LAG) + data["lagTable"] = lag_data + + # VLAN membership + vlan_data = walk_indexed(ip, OID_QBRIDGE_VLAN) + data["vlans"] = vlan_data + + return data + + +def print_switch(data): + """Human-readable summary.""" + print(f"\n{'='*60}") + print(f" {data.get('sysName', data['ip'])} ({data['ip']})") + print(f" {data.get('sysDescr', '?')}") + print(f"{'='*60}") + + print(f"\n Active ports (UP only):") + print(f" {'Port':<12} {'Speed':>10} {'InErrors':>10} {'OutErrors':>10}") + print(f" {'-'*12} {'-'*10} {'-'*10} {'-'*10}") + for iface in data.get("interfaces", []): + if iface["up"]: + print(f" {iface['name']:<12} {iface['speedMbps']:>8}Mb " + f"{iface['inErrors']:>10} {iface['outErrors']:>10}") + + err_ports = [i for i in data.get("interfaces", []) + if i["up"] and (int(i["inErrors"] or 0) > 0 + or int(i["outErrors"] or 0) > 0)] + if err_ports: + print(f"\n *** PORTS WITH ERRORS ***") + for p in err_ports: + print(f" {p['name']}: inErr={p['inErrors']} outErr={p['outErrors']}") + + if data.get("lldpNeighbors"): + print(f"\n LLDP neighbors:") + for n in data["lldpNeighbors"]: + sysname = n.get("remoteSysName", "") or "(unknown)" + print(f" local={n['localPort']:<6} remote={n['remotePort']:<20} {sysname}") + + if data.get("lagTable"): + print(f"\n LACP/LAG table entries: {len(data['lagTable'])}") + + +def main(): + ap = argparse.ArgumentParser( + description="READ-ONLY SNMP switch audit (portable, config-driven)") + ap.add_argument("switches", nargs="*", + help="switch IPs (env: SWITCH_IPS)") + ap.add_argument("--community", default=COMMUNITY, + help=f"SNMPv2c community (env: SNMP_COMMUNITY, default: {COMMUNITY})") + ap.add_argument("--output", default=OUTPUT_DIR, + help=f"output dir (env: OUTPUT_DIR)") + args = ap.parse_args() + + global COMMUNITY + COMMUNITY = args.community + + ips = args.switches + if not ips: + env_ips = os.environ.get("SWITCH_IPS", "") + ips = env_ips.split() + + if not ips: + ap.error("no switch IPs provided (pass as args or set SWITCH_IPS)") + + os.makedirs(args.output, exist_ok=True) + all_data = [] + + for ip in ips: + data = audit_switch(ip.strip()) + all_data.append(data) + print_switch(data) + outpath = os.path.join(args.output, f"switch-{ip}.json") + with open(outpath, "w") as f: + json.dump(data, f, indent=2) + print(f"\n -> {outpath}") + + combined = os.path.join(args.output, "switches-all.json") + with open(combined, "w") as f: + json.dump(all_data, f, indent=2) + print(f"\n Combined: {combined}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/perf/scripts/sw-capture-remote.sh b/perf/scripts/sw-capture-remote.sh deleted file mode 100755 index a05e3c8..0000000 --- a/perf/scripts/sw-capture-remote.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -# sw-capture-remote.sh - orchestrate a serial capture from this workstation. -# -# Flow: -# 1. De-conflict: abort if any local ssh to pfv-tsys4 is in flight -# (other agent could be there). -# 2. Free the serial port: kill whatever holds /dev/ttyUSBx -# (typically a screen session). Targeted, not blanket. -# 3. scp driver + .cmds to pfv-tsys4. -# 4. Run driver over ssh, capture stderr to console. -# 5. scp the resulting log back to returned-logs/. -# -# Usage: -# sw-capture-remote.sh [device] -# -# e.g. pfv-core-sw01 (must have switches/.cmds) -# [device] /dev/ttyUSBx on pfv-tsys4. Defaults per switch map below. -# -# Currently scoped to pfv-core-sw01 only (per user direction). The other -# two switches are deferred; their defaults are placeholders. -set -u - -SWITCH=${1:-} -DEVICE=${2:-} - -if [ -z "$SWITCH" ]; then - echo "Usage: $0 [device]" >&2 - echo " e.g. $0 pfv-core-sw01 /dev/ttyUSB2" >&2 - exit 2 -fi - -# Switch -> default device map (ttyUSB2 = core-sw01 confirmed by user). -case "$SWITCH" in - pfv-core-sw01) - [ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB2 ;; - pfv-r3-tor-mgmt) - [ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB0 # TENTATIVE - unconfirmed - if [ "${2:-}" = "" ]; then - echo "NOTE: pfv-r3-tor-mgmt device is tentative (/dev/ttyUSB0)." >&2 - echo " Pass the device explicitly if different." >&2 - fi ;; - pfv-r3-tor-stor) - [ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB1 # TENTATIVE - unconfirmed - if [ "${2:-}" = "" ]; then - echo "NOTE: pfv-r3-tor-stor device is tentative (/dev/ttyUSB1)." >&2 - echo " Pass the device explicitly if different." >&2 - fi ;; - *) - echo "unknown switch: $SWITCH" >&2; exit 2 ;; -esac - -BAUD=9600 -HOST=root@pfv-tsys4 -HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -LOCAL_DRIVER=$HERE/scripts/sw-capture.py -LOCAL_CMDS=$HERE/switches/$SWITCH.cmds -LOCAL_LOG=$HERE/returned-logs/$SWITCH.log -REMOTE_DRIVER=/root/sw-capture.py -REMOTE_CMDS=/root/$SWITCH.cmds -REMOTE_LOG=/root/$SWITCH.log - -[ -f "$LOCAL_DRIVER" ] || { echo "missing $LOCAL_DRIVER" >&2; exit 2; } -[ -f "$LOCAL_CMDS" ] || { echo "missing $LOCAL_CMDS" >&2; exit 2; } - -ts() { date +%H:%M:%S; } - -echo "[$(ts)] switch=$SWITCH device=$DEVICE baud=$BAUD host=$HOST" - -# 1. De-conflict: any local ssh to pfv-tsys4 in flight? -echo "[$(ts)] checking for in-flight ssh to pfv-tsys4..." -# shellcheck disable=SC2009 # intentional: need full ps columns filtered by process args -if ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys4|scp.*pfv-tsys4' | grep -v grep >/tmp/.swcap.ps 2>&1; then - cat /tmp/.swcap.ps - echo "[$(ts)] ABORT: another ssh/scp to pfv-tsys4 is running (other agent?)." >&2 - exit 1 -fi -echo "[$(ts)] clear." -rm -f /tmp/.swcap.ps - -# 2. Free the serial port: kill whatever holds $DEVICE. -echo "[$(ts)] freeing $DEVICE on $HOST (targeted; other screen sessions untouched)..." -ssh -o BatchMode=yes -o ConnectTimeout=8 "$HOST" \ - "fuser -v $DEVICE 2>&1 | tee /dev/stderr; \ - fuser -k -TERM $DEVICE 2>/dev/null; sleep 1; \ - if fuser $DEVICE 2>/dev/null; then \ - echo 'still held after SIGTERM, escalating to SIGKILL'; \ - fuser -k -KILL $DEVICE 2>/dev/null; sleep 1; \ - fi; \ - fuser $DEVICE 2>/dev/null && echo 'STILL HELD' || echo 'FREE'" - -# Re-check; abort if still held. -HELD=$(ssh -o BatchMode=yes "$HOST" "fuser $DEVICE 2>/dev/null && echo HELD || echo FREE") -if [ "$HELD" = "HELD" ]; then - echo "[$(ts)] ABORT: $DEVICE still held on $HOST." >&2 - exit 1 -fi - -# 3. Copy driver + cmds. -echo "[$(ts)] copying driver + cmds to $HOST..." -scp -q "$LOCAL_DRIVER" "$HOST:$REMOTE_DRIVER" -scp -q "$LOCAL_CMDS" "$HOST:$REMOTE_CMDS" - -# 4. Run the capture on pfv-tsys4. Stream stderr (progress) to console. -echo "[$(ts)] running capture..." -ssh -o BatchMode=yes -o ServerAliveInterval=10 "$HOST" \ - "python3 $REMOTE_DRIVER \ - --device $DEVICE --baud $BAUD \ - --cmds $REMOTE_CMDS --log $REMOTE_LOG" -RC=$? -echo "[$(ts)] capture exit code: $RC" - -# 5. Pull log back. -echo "[$(ts)] pulling log back to $LOCAL_LOG..." -mkdir -p "$(dirname "$LOCAL_LOG")" -scp -q "$HOST:$REMOTE_LOG" "$LOCAL_LOG" -if [ -f "$LOCAL_LOG" ]; then - SZ=$(wc -c < "$LOCAL_LOG") - echo "[$(ts)] OK: $LOCAL_LOG ($SZ bytes)" - echo "----- head -----" - head -30 "$LOCAL_LOG" - echo "----- tail -----" - tail -10 "$LOCAL_LOG" -else - echo "[$(ts)] ERROR: log not pulled back." >&2 - exit 1 -fi - -exit $RC diff --git a/perf/scripts/sw-capture.py b/perf/scripts/sw-capture.py deleted file mode 100644 index c9a3668..0000000 --- a/perf/scripts/sw-capture.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -""" -sw-capture.py - drive a Dell switch over a serial console and log all output. - -Read-only. Sends ONLY the commands listed in the supplied .cmds file -(comment lines starting with '!' and blank lines are skipped). Handles -`--More--` pagination by sending a space. Aborts cleanly on any password -prompt (we never supply credentials). - -Pure stdlib (termios + select). No pyserial/expect required. - -Exit codes: - 0 clean run, every command saw a prompt again - 2 could not synchronize with a prompt during wake - 3 one or more commands timed out (log still written) - 4 password prompt encountered (aborted) - -Usage: - sw-capture.py --device /dev/ttyUSB2 --baud 9600 \\ - --cmds pfv-core-sw01.cmds --log /root/pfv-core-sw01.log -""" -import argparse -import os -import re -import select -import sys -import termios -import time - -PROMPT_RE = re.compile(rb'[>#]\s*$') # ends in # or > + spaces -MORE_RE = re.compile(rb'--\s*More\s*--') # pagination prompt -PWD_RE = re.compile(rb'[Pp]assword:\s*$') # enable / login password - -BAUDS = { - '9600': termios.B9600, - '19200': termios.B19200, - '38400': termios.B38400, - '57600': termios.B57600, - '115200': termios.B115200, -} - - -def log(msg, level='INFO'): - sys.stderr.write(f'[{level}] {msg}\n') - sys.stderr.flush() - - -def open_port(device, baud): - """Open the serial device raw at the requested baud, 8N1, no flow ctrl.""" - fd = os.open(device, os.O_RDWR | os.O_NOCTTY) - try: - attrs = termios.tcgetattr(fd) - except termios.error: - log(f'{device} is not a termios-capable device', 'WARN') - return fd - - # raw input - attrs[0] &= ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK | - termios.ISTRIP | termios.INLCR | termios.IGNCR | - termios.ICRNL | termios.IXON) - # raw output - attrs[1] &= ~termios.OPOST - # 8N1, enable receiver, ignore modem control lines - attrs[2] &= ~(termios.CSIZE | termios.PARENB | termios.CSTOPB) - attrs[2] |= termios.CS8 | termios.CREAD | termios.CLOCAL - # raw local - attrs[3] &= ~(termios.ECHO | termios.ECHONL | termios.ICANON | - termios.ISIG | termios.IEXTEN) - # non-blocking-ish reads (select is the primary gate) - attrs[6][termios.VMIN] = 0 - attrs[6][termios.VTIME] = 1 - - b = BAUDS.get(str(baud)) - if b is None: - raise SystemExit(f'unsupported baud: {baud}') - # Set ispeed/ospeed directly on the attribute list. (Equivalent to - # termios.cfsetispeed/cfsetospeed, which are missing on some Python - # builds — e.g. the one on pfv-tsys4.) - attrs[4] = b # ispeed - attrs[5] = b # ospeed - termios.tcsetattr(fd, termios.TCSANOW, attrs) - return fd - - -def read_chunk(fd, timeout): - """Read whatever arrives within `timeout`. Extends briefly on activity.""" - buf = b'' - deadline = time.time() + timeout - while True: - remaining = deadline - time.time() - if remaining <= 0: - return buf - r, _, _ = select.select([fd], [], [], min(0.5, remaining)) - if not r: - if buf: - return buf - continue - try: - chunk = os.read(fd, 4096) - except OSError: - return buf - if not chunk: - return buf - buf += chunk - # keep collecting as long as bytes are flowing - deadline = time.time() + 0.3 - - -def drain(fd, timeout=1.0): - total = 0 - while True: - b = read_chunk(fd, timeout=timeout) - if not b: - return total - total += len(b) - - -def send(fd, s): - if isinstance(s, str): - s = s.encode() - os.write(fd, s) - - -def wait_for(fd, regex, timeout, on_more=None, on_pwd=None): - """Read until `regex` matches the tail of the buffer, or timeout.""" - buf = b'' - deadline = time.time() + timeout - while time.time() < deadline: - remaining = deadline - time.time() - chunk = read_chunk(fd, timeout=min(1.0, remaining)) - if chunk: - buf += chunk - tail64 = buf[-64:] - tail32 = buf[-32:] - tail128 = buf[-128:] - if on_more and MORE_RE.search(tail64): - on_more(fd) - continue - if on_pwd and PWD_RE.search(tail32): - on_pwd(buf) - return buf, 'pwd' - if regex.search(tail128): - return buf, 'ok' - return buf, 'timeout' - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument('--device', required=True) - ap.add_argument('--baud', type=int, default=9600) - ap.add_argument('--cmds', required=True) - ap.add_argument('--log', required=True) - ap.add_argument('--per-cmd-timeout', type=float, default=45.0) - ap.add_argument('--wake-timeout', type=float, default=15.0) - ap.add_argument('--session-max', type=float, default=600.0) - args = ap.parse_args() - - cmds = [] - with open(args.cmds) as f: - for raw in f: - s = raw.rstrip('\n').strip() - if not s or s.startswith('!'): - continue - cmds.append(s) - log(f'parsed {len(cmds)} commands from {args.cmds}') - - logf = open(args.log, 'wb', buffering=0) - - def w(b): - if isinstance(b, str): - b = b.encode() - logf.write(b) - - w(f'==== sw-capture {time.strftime("%Y-%m-%d %H:%M:%S")} ====\n') - w(f'device={args.device} baud={args.baud} cmds={args.cmds} ' - f'n={len(cmds)} per_cmd_timeout={args.per_cmd_timeout}\n\n') - - fd = open_port(args.device, args.baud) - log(f'opened {args.device} @ {args.baud} 8N1 raw') - - session_start = time.time() - abort = False - - def on_more(fd_): - log('--More-- -> space') - send(fd_, b' ') - - def on_pwd(buf): - nonlocal abort - abort = True - log('password prompt detected (enable or login) - aborting; ' - 'no credentials supplied', 'ERROR') - w(buf) - w(b'\n[PASSWORD PROMPT - ABORTED]\n') - - # WAKE: nudge with Ctrl-C + Enter, look for any prompt - drain(fd, 0.5) - synced = False - wake_deadline = time.time() + args.wake_timeout - attempt = 0 - while time.time() < wake_deadline: - attempt += 1 - send(fd, b'\x03') - time.sleep(0.2) - send(fd, b'\r') - buf, status = wait_for(fd, PROMPT_RE, timeout=3.0, - on_more=on_more, on_pwd=on_pwd) - w(buf) - if status == 'pwd': - logf.close(); os.close(fd); sys.exit(4) - if status == 'ok': - synced = True - log(f'prompt synced after {attempt} attempt(s)') - break - if not synced: - w(b'\n[NO PROMPT - ABORT]\n') - log('no prompt detected during wake window', 'ERROR') - logf.close(); os.close(fd); sys.exit(2) - - # RUN commands verbatim from the .cmds list - failures = 0 - for idx, cmd in enumerate(cmds, 1): - if time.time() - session_start > args.session_max: - log('session_max exceeded - stopping early', 'ERROR') - w(b'\n[SESSION_MAX - STOP]\n') - break - if abort: - break - log(f'[{idx}/{len(cmds)}] {cmd}') - send(fd, cmd + '\r') - buf, status = wait_for(fd, PROMPT_RE, - timeout=args.per_cmd_timeout, - on_more=on_more, on_pwd=on_pwd) - w(buf) - if status == 'pwd': - failures += 1 - break - if status == 'timeout': - log(f'timeout after: {cmd}', 'WARN') - failures += 1 - # try to resync: Ctrl-C + drain - send(fd, b'\x03') - time.sleep(0.3) - drain(fd, 0.5) - - w(f'\n==== end {time.strftime("%Y-%m-%d %H:%M:%S")} ' - f'failures={failures} ====\n') - logf.close() - os.close(fd) - log(f'done -> {args.log} failures={failures}') - sys.exit(0 if failures == 0 else 3) - - -if __name__ == '__main__': - main() diff --git a/perf/scripts/sw-conman-probe.sh b/perf/scripts/sw-conman-probe.sh deleted file mode 100644 index 7540634..0000000 --- a/perf/scripts/sw-conman-probe.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -# Probe conman state + expect availability on pfv-tsys4. -# Read-only. Decides whether we drive via conman+expect or expect-only. -set -u - -# De-conflict: any ssh to pfv-tsys4 right now? -echo "===== LOCAL ssh activity =====" -# shellcheck disable=SC2009 # intentional: need full ps columns filtered by process args -ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys' | grep -v grep || echo "(none to pfv-tsys4)" - -echo -echo "===== pfv-tsys4: conman + expect state =====" -ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE' -echo "--- conmand service ---" -systemctl is-active conmand 2>&1 || true -systemctl is-enabled conmand 2>&1 || true -systemctl status conmand --no-pager 2>&1 | head -15 || true - -echo -echo "--- conman binary ---" -command -v conman && conman --version 2>&1 | head -2 || echo "conman: MISSING" -command -v conmand && echo "conmand present" || echo "conmand: MISSING" - -echo -echo "--- /etc/conman.conf: ttyUSB2 entries ---" -grep -nE "ttyUSB2|core-sw|CONSOLE|LOG|SERIAL|BAUD" /etc/conman.conf 2>/dev/null | head -40 || echo "(no matches / no file)" - -echo -echo "--- conman log dir ---" -ls -la /var/log/conman/ 2>&1 | head -20 || echo "(no /var/log/conman)" -ls -la /var/consoles/ 2>&1 | head -20 || echo "(no /var/consoles)" - -echo -echo "--- expect availability ---" -command -v expect && expect -v 2>&1 || echo "expect: NOT installed" -echo "apt-cache policy expect:" -apt-cache policy expect 2>/dev/null | head -10 || echo "(apt-cache failed)" - -echo -echo "--- other useful drivers ---" -for t in tclsh socat cu tip; do - command -v "$t" 2>/dev/null && echo " $t: present" || true -done - -echo -echo "--- apt network reachability (quick) ---" -timeout 5 bash -c 'echo > /dev/tcp/deb.debian.org/80' 2>&1 && echo "apt network: OK" || echo "apt network: UNREACHABLE" - -echo -echo "--- disk space for log ---" -df -h /root 2>&1 | tail -2 - -echo -echo "--- screen sessions (still 3?) ---" -screen -ls 2>&1 || true -REMOTE diff --git a/perf/scripts/sw-probe.sh b/perf/scripts/sw-probe.sh deleted file mode 100644 index a2b3fc0..0000000 --- a/perf/scripts/sw-probe.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Probe pfv-tsys4 for what's available to drive the serial console. -# Also snapshots local ssh/scp activity so we can de-conflict with the -# other agent running in this directory. -set -u - -echo "===== LOCAL ssh/scp activity (other-agent de-confliction) =====" -# shellcheck disable=SC2009 # intentional: need full ps columns (etime,args) filtered by process args -ps -eo pid,ppid,etime,user,args | grep -E 'ssh|scp' | grep -v grep || echo "(none)" - -echo -echo "===== Ping pfv-tsys4 =====" -ping -c1 -W2 pfv-tsys4 >/dev/null 2>&1 && echo "ping OK" || echo "ping FAIL" - -echo -echo "===== Probe pfv-tsys4 over ssh =====" -ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE' -echo "--- host ---" -hostname; uname -a -echo "--- tools ---" -for t in python3 python expect screen minicom picocom stty fuser lsof; do - p=$(command -v "$t" 2>/dev/null) && echo "$t -> $p" || echo "$t -> MISSING" -done -echo "--- pyserial ---" -python3 -c "import serial; print('pyserial', serial.__version__)" 2>&1 -echo "--- device node ---" -ls -l /dev/ttyUSB2 2>&1 -stat -c '%n owner=%U:%G mode=%a' /dev/ttyUSB2 2>&1 || true -echo "--- who holds /dev/ttyUSB2 ---" -fuser -v /dev/ttyUSB2 2>&1 || echo "(fuser: none or n/a)" -lsof /dev/ttyUSB2 2>&1 | head -20 || true -echo "--- screen sessions on this host ---" -screen -ls 2>&1 || echo "(no screen / not installed)" -echo "--- current tty settings (only readable if not held exclusively) ---" -stty -F /dev/ttyUSB2 2>&1 || echo "(held exclusively - expected if screen is up)" -echo "--- baud hints in config/history ---" -grep -riE "ttyUSB2|115200|9600|baud" /etc/ ~/.screenrc ~/.bash_history 2>/dev/null | head -20 || true -echo "--- recent console-related processes ---" -ps -eo pid,etime,user,args | grep -E 'screen|minicom|picocom|ttyUSB' | grep -v grep || echo "(none)" -REMOTE diff --git a/switches/pfv-core-sw01.cmds b/switches/pfv-core-sw01.cmds index 0c2db73..5db89b2 100644 --- a/switches/pfv-core-sw01.cmds +++ b/switches/pfv-core-sw01.cmds @@ -1,93 +1,11 @@ -!############################################################################### -!# pfv-core-sw01 -!# -!# Dell PowerConnect 5448 (48-port 1GbE copper, 4× combo SFP / 10G via stacking) -!# OS: DNOS 4.x / PowerConnect CLI -!# -!# This file is a flat list of READ-ONLY show commands. -!# Paste them in order from enable mode. Capture the whole session to -!# returned-logs/pfv-core-sw01.log -!# -!# To capture from a terminal: -!# 1. SSH/telnet in -!# 2. enable -!# 3. terminal datadump (disables --More-- pause) -!# 4. paste everything below -!############################################################################### - -enable +! pfv-core-sw01 — Dell PowerConnect 5448 (core switch, rack 5) +! Read-only show commands for LAG/port-channel/VLAN audit terminal datadump - -! --- Identify & versions --------------------------------------------------- show version -show system -show inventory -show bootvar -show running-config -show stack -show cpu utilization -show memory -show logging - -! --- Interfaces ------------------------------------------------------------ show interfaces status -show interfaces configuration -show interfaces description -show interfaces counters -show interfaces counters errors -show interfaces counters detail -show interfaces counters cable 1-52 - -! --- Port-channels / LAG (4-port LACP to pfv-r3-tor-stor) ----------------- show interfaces port-channel -show lacp 1 -show lacp 2 -show lacp 3 -show lacp 4 -show lacp 5 -show lacp 6 - -! --- VLANs ----------------------------------------------------------------- +show port-channel summary +show lacp show vlan -show vlan detail -show interfaces vlan 1 -show interfaces vlan 1000 -show vlan info - -! --- Spanning tree --------------------------------------------------------- -show spanning-tree -show spanning-tree active -show spanning-tree summary -show spanning-tree detail - -! --- IGMP / multicast (matters for storage/iSCSI-NFS sometimes) ----------- -show ip igmp snooping -show ip igmp snooping vlan 1 -show ip igmp snooping vlan 1000 - -! --- QoS / CoS ------------------------------------------------------------- -show qos -show qos interface -show class-of-service - -! --- Storm control / loop protection -------------------------------------- -show storm-control -show ports jumbo-frame - -! --- ARP / MAC / FDB ------------------------------------------------------- -show mac address-table -show arp -show ip route - -! --- Environmental --------------------------------------------------------- -show environment -show fan -show power -show hardware - -! --- Flow control ---------------------------------------------------------- -show flowcontrol -show interfaces flowcontrol - -! --- Save the capture, then exit ------------------------------------------ -exit +show running-config interface port-channel 1 +show running-config interface port-channel 3 diff --git a/switches/pfv-r3-tor-mgmt.cmds b/switches/pfv-r3-tor-mgmt.cmds deleted file mode 100644 index 2e8644d..0000000 --- a/switches/pfv-r3-tor-mgmt.cmds +++ /dev/null @@ -1,90 +0,0 @@ -!############################################################################### -!# pfv-r3-tor-mgmt -!# -!# Rack 3 management TOR (VLAN1 only for this switch). -!# Dell switch - OS variant unknown. Commands below cover the common -!# Dell CLI flavors (DNOS 4/6/9, OS6/OS9, N-Series). Some may not be -!# recognized on every OS; that is fine - skip errors and keep going. -!# -!# Paste from enable mode after `terminal datadump` (or `terminal length 0` -!# on DNOS 6/9). Capture session to returned-logs/pfv-r3-tor-mgmt.log -!############################################################################### - -enable -terminal datadump -terminal length 0 - -! --- Identify --------------------------------------------------------------- -show version -show system -show inventory -show running-config -show bootvar -show cpu -show cpu utilization -show memory -show processes -show logging - -! --- Interfaces ------------------------------------------------------------- -show interface status -show interface description -show interface configuration -show interface counters -show interface counters errors -show interface counters detail -show interfaces status -show interfaces description -show interfaces counters -show interfaces counters errors - -! --- Port channels / LAG ---------------------------------------------------- -show port-channel summary -show port-channel detail -show interfaces port-channel -show lacp -show lacp neighbor -show lag - -! --- VLANs ------------------------------------------------------------------ -show vlan -show vlan brief -show vlan detail - -! --- Spanning tree ---------------------------------------------------------- -show spanning-tree -show spanning-tree summary -show spanning-tree active - -! --- Multicast / snooping --------------------------------------------------- -show ip igmp snooping -show ip igmp snooping vlan 1 - -! --- QoS -------------------------------------------------------------------- -show qos -show qos interface - -! --- Jumbo / storm control -------------------------------------------------- -show ports jumbo-frame -show storm-control - -! --- FDB / ARP / routes ----------------------------------------------------- -show mac address-table -show mac-address-table -show arp -show ip route - -! --- Environment / hardware ------------------------------------------------- -show environment -show fan -show power supplies -show power -show hardware -show transceiver -show inventory - -! --- Flow control ----------------------------------------------------------- -show flowcontrol -show interfaces flowcontrol - -exit diff --git a/switches/pfv-r3-tor-stor-LACP-FIX.cmds b/switches/pfv-r3-tor-stor-LACP-FIX.cmds deleted file mode 100644 index 4e1a5ac..0000000 --- a/switches/pfv-r3-tor-stor-LACP-FIX.cmds +++ /dev/null @@ -1,88 +0,0 @@ -!############################################################################### -!# pfv-r3-tor-stor — TARGETED LACP hash verification -!# -!# Purpose: confirm whether `port-channel load-balance layer-2-3-4` actually -!# took effect, and gather the LAG state needed to decide whether the LAG -!# must be bounced. -!# -!# Context: tsys6<->tsys7 is capped at exactly single-link speed (~943 Mbps) -!# with 90K-136K retransmits on multi-stream TCP, but a clean 928 Mbps / 0 -!# retransmits on single-stream. That = offered load above 1 Gbps is being -!# dropped at a single egress port. We need to know if the switch hash is -!# really layer-2-3-4, and whether tsys6 + tsys7 ports are correctly -!# bundled into LACP groups. -!# -!# Paste from enable mode. Capture to: -!# returned-logs/pfv-r3-tor-stor-lacp-verify.log -!############################################################################### - -enable -terminal datadump -terminal length 0 - -! --- 1. THE KEY COMMAND: what hash is actually active? ----------------------- -! Expected: layer-2-3-4. If it still shows source-mac / dst-mac / -! src-dst-mac (layer 2 only), the change did NOT take effect. -show port-channel load-balance -show port-channel protocol - -! --- 2. Which LAGs exist and which physical ports are members ---------------- -! Look for the port-channels carrying tsys6 and tsys7. Confirm each has -! TWO member ports both in "bundled" / "Forwarding" state. -show port-channel summary -show port-channel detail - -! --- 3. LACP actor/partner state per member port ---------------------------- -! Every member port should show: Actor State Flag = AcLg (Active, Aggr), -! Partner Operational, and a non-zero Partner System ID. -show lacp -show lacp neighbor - -! --- 4. Per-port counters during/after an iperf run ------------------------- -! THIS is the switch-side equivalent of the host RX-counter test. -! Run an 8-stream iperf tsys7->tsys6 first, then immediately capture -! these so the byte counters reflect the test. The TWO tsys6-facing -! ports should show SIMILAR "RX bytes" (switch ingress from tsys6's -! bond is not the test direction; look at TX-out bytes toward tsys6, -! i.e. the ports' output counters). Replace Te1/0/x with the real port -! IDs shown by "show port-channel summary" above: -show interfaces counters -show interfaces counters errors - -! --- 5. Running config line that set the hash (proof it was committed) ------ -show running-config | include port-channel - -! --- 6. Switch identity (to look up correct hash syntax for this model) ----- -show version -show system - -exit - -!############################################################################### -!# DECISION TREE after reading the output: -!# -!# A) show port-channel load-balance reports layer-2-3-4 AND per-port -!# counters show traffic split across BOTH tsys6-facing ports: -!# -> Hash is fine. Re-test iperf; if still ~943 Mbps, the cap is -!# host-side (receiver RX bridge / IRQ / CPU). -!# -!# B) load-balance reports layer-2-3-4 BUT per-port counters show all traffic -!# on ONE tsys6-facing port: -!# -> Hash is configured but the LAG needs a bounce to pick it up. -!# Bounce the relevant port-channel: -!# config -!# interface port-channel N -!# shutdown -!# no shutdown -!# end -!# WARNING: drops all traffic on that LAG for ~2-5 s. Coordinate. -!# -!# C) load-balance still shows a layer-2-only policy: -!# -> The earlier change didn't commit to this context. Re-apply: -!# config -!# port-channel load-balance layer-2-3-4 -!# end -!# show port-channel load-balance -!# copy running-config startup-config -!# Then handle as case (B) if counters still aren't split. -!############################################################################### diff --git a/switches/pfv-r3-tor-stor.cmds b/switches/pfv-r3-tor-stor.cmds deleted file mode 100644 index 00b058f..0000000 --- a/switches/pfv-r3-tor-stor.cmds +++ /dev/null @@ -1,108 +0,0 @@ -!############################################################################### -!# pfv-r3-tor-stor -!# -!# Rack 3 storage TOR (VLAN1000 storage + VLAN1). -!# Carries the 4-port LACP to pfv-core-sw01 (Rack 5). -!# Most important switch for cross-rack storage throughput. -!# -!# Paste from enable mode after `terminal datadump` (DNOS 4/6) or -!# `terminal length 0` (DNOS 9 / OS9). Capture to returned-logs/pfv-r3-tor-stor.log -!############################################################################### - -enable -terminal datadump -terminal length 0 - -! --- Identify --------------------------------------------------------------- -show version -show system -show inventory -show running-config -show bootvar -show cpu utilization -show cpu -show memory -show processes -show logging - -! --- Interfaces (most important section) ------------------------------------ -show interface status -show interface description -show interface counters -show interface counters errors -show interface counters detail -show interfaces status -show interfaces description -show interfaces counters -show interfaces counters errors -show interfaces configuration - -! --- LACP / port-channel (the 4-port LAG to Rack 5) ------------------------ -show port-channel summary -show port-channel detail -show port-channel 1 -show port-channel 2 -show port-channel 3 -show port-channel 4 -show interfaces port-channel -show lacp -show lacp neighbor -show lacp 1 -show lag - -! --- VLANs (VLAN1000 is the storage VLAN) ---------------------------------- -show vlan -show vlan brief -show vlan id 1000 -show vlan detail -show interfaces vlan 1000 -show interfaces switchport - -! --- Spanning tree ---------------------------------------------------------- -show spanning-tree -show spanning-tree summary -show spanning-tree active -show spanning-tree detail -show spanning-tree mst - -! --- Multicast / IGMP (storage traffic) ------------------------------------- -show ip igmp snooping -show ip igmp snooping vlan 1000 -show ip igmp snooping vlan 1 - -! --- QoS / CoS / DCB / PFC (matters for storage) --------------------------- -show qos -show qos interface -show class-of-service -show dcb -show dcb priority -show priority-flow-control -show pfc - -! --- Jumbo / storm / MTU ---------------------------------------------------- -show ports jumbo-frame -show storm-control -show interfaces mtu - -! --- FDB / ARP / routing ---------------------------------------------------- -show mac address-table -show mac-address-table -show arp -show ip route - -! --- Environment / hardware / optics --------------------------------------- -show environment -show fan -show power supplies -show power -show hardware -show inventory -show transceiver -show interface transceiver -show media - -! --- Flow control (important for storage) ---------------------------------- -show flowcontrol -show interfaces flowcontrol - -exit diff --git a/switches/pfv-tor3-stor.cmds b/switches/pfv-tor3-stor.cmds new file mode 100644 index 0000000..732517f --- /dev/null +++ b/switches/pfv-tor3-stor.cmds @@ -0,0 +1,12 @@ +! pfv-tor3-stor — Neyland 24T (Radlan-based, rack 3 storage TOR) +! Radlan CLI uses different keywords than DNOS +enable +show system +show inventory +show interfaces configuration +show interfaces description +show port-channel +show lag +show vlan database +show vlan +show running-config