From ea3b0907aec2e2aa2171c8257e7ed44e9abdfa86 Mon Sep 17 00:00:00 2001 From: Charles N Wyble Date: Fri, 20 Feb 2026 09:24:59 -0500 Subject: [PATCH] feat(scripts): add comprehensive validation script for all MCP/LSP servers Created scripts/validate-all.sh to validate ALL servers with actual protocol handshake messages: Coverage: - 29 MCP servers (Design, Infrastructure, Content, Communication, Analytics, Productivity, Reverse Engineering) - 4 LSP servers (bash, docker, marksman, terraform-ls) Features: - Sends actual MCP initialize handshake to each container - Sends actual LSP initialize handshake to LSP containers - Categorizes results: PASSED, FAILED, SKIPPED, NEEDS_ENV, NEEDS_SERVICE, TRANSPORT_MISMATCH - Provides detailed summary with server names and versions - Color-coded output for easy scanning Usage: ./scripts/validate-all.sh # Test all servers ./scripts/validate-all.sh mcp # MCP servers only ./scripts/validate-all.sh lsp # LSP servers only ./scripts/validate-all.sh # Specific service This script implements the SDLC requirement for protocol validation. Every server must pass build + start + protocol handshake to be considered "working". --- scripts/validate-all.sh | 348 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100755 scripts/validate-all.sh diff --git a/scripts/validate-all.sh b/scripts/validate-all.sh new file mode 100755 index 0000000..75a8f44 --- /dev/null +++ b/scripts/validate-all.sh @@ -0,0 +1,348 @@ +#!/bin/bash +# Comprehensive MCP/LSP Server Validation Script +# Tests all servers with actual protocol messages +# +# Usage: +# ./scripts/validate-all.sh # Test all servers +# ./scripts/validate-all.sh mcp # Test only MCP servers +# ./scripts/validate-all.sh lsp # Test only LSP servers +# ./scripts/validate-all.sh # Test specific service + +set -e + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Protocol messages +MCP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{},"protocolVersion":"2024-11-05","clientInfo":{"name":"validate-all","version":"1.0.0"}},"id":1}' +LSP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"processId":null,"capabilities":{},"rootUri":null,"workspaceFolders":null},"id":1}' + +# Counters +PASSED=0 +FAILED=0 +SKIPPED=0 +NEEDS_ENV=0 +NEEDS_SERVICE=0 +TRANSPORT_MISMATCH=0 + +# Results arrays +declare -a RESULTS_PASS +declare -a RESULTS_FAIL +declare -a RESULTS_SKIP +declare -a RESULTS_ENV +declare -a RESULTS_SERVICE +declare -a RESULTS_TRANSPORT + +# MCP Servers configuration (container_name, timeout, env_vars...) +declare -A MCP_SERVERS +MCP_SERVERS=( + # Design & Engineering + ["kneldevstack-aimiddleware-blender-mcp"]="10" + ["kneldevstack-aimiddleware-freecad-mcp"]="10" + ["kneldevstack-aimiddleware-gimp-mcp"]="10" + ["kneldevstack-aimiddleware-kicad-mcp"]="10 KICAD_HOST=host.docker.internal KICAD_PORT=5555" + + # Hosting & Infrastructure + ["kneldevstack-aimiddleware-kubernetes-mcp"]="10" + ["kneldevstack-aimiddleware-docker-mcp"]="10" + ["kneldevstack-aimiddleware-proxmox-mcp"]="10 PROXMOX_HOST=https://proxmox.example.com PROXMOX_USER=root@pam PROXMOX_TOKEN=dummy PROXMOX_NODE=pve" + ["kneldevstack-aimiddleware-terraform-mcp"]="10" + ["kneldevstack-aimiddleware-cloudron-mcp"]="10 CLOUDRON_URL=https://cloudron.example.com CLOUDRON_TOKEN=dummy" + + # Development Tools + ["kneldevstack-aimiddleware-context7-mcp"]="10 UPSTASH_REDIS_REST_URL=https://dummy.upstash.io UPSTASH_REDIS_REST_TOKEN=dummy" + + # Content Management + ["kneldevstack-aimiddleware-nextcloud-mcp"]="10 NEXTCLOUD_HOST=https://nextcloud.example.com NEXTCLOUD_USERNAME=dummy NEXTCLOUD_PASSWORD=dummy" + ["kneldevstack-aimiddleware-ghost-mcp"]="10 GHOST_API_URL=https://ghost.example.com GHOST_ADMIN_API_KEY=012345678901234567890123:0123456789012345678901234567890123456789012345678901234567890123" + ["kneldevstack-aimiddleware-docspace-mcp"]="10 DOCSPACE_HOST=https://docspace.example.com DOCSPACE_TOKEN=dummy" + ["kneldevstack-aimiddleware-wordpress-mcp"]="10 WORDPRESS_URL=https://wordpress.example.com WORDPRESS_USERNAME=dummy WORDPRESS_APPLICATION_PASSWORD=dummy" + + # Communication & Collaboration + ["kneldevstack-aimiddleware-discourse-mcp"]="10 DISCOURSE_URL=https://discourse.example.com DISCOURSE_API_KEY=dummy DISCOURSE_API_USERNAME=dummy" + ["kneldevstack-aimiddleware-imap-mcp"]="10 IMAP_HOST=imap.example.com IMAP_PORT=993 IMAP_USERNAME=dummy IMAP_PASSWORD=dummy" + ["kneldevstack-aimiddleware-postizz-mcp"]="10 POSTIZ_API_KEY=dummy POSTIZ_URL=https://postiz.example.com" + + # Analytics & Security + ["kneldevstack-aimiddleware-matomo-mcp"]="10 MATOMO_URL=https://matomo.example.com MATOMO_TOKEN=dummy" + ["kneldevstack-aimiddleware-bitwarden-mcp"]="15 BITWARDEN_CLIENT_ID=dummy BITWARDEN_CLIENT_SECRET=dummy BITWARDEN_PASSWORD=dummy BITWARDEN_SERVER_URL=https://vault.bitwarden.com" + + # Productivity & Automation + ["kneldevstack-aimiddleware-audiobook-mcp"]="10" + ["kneldevstack-aimiddleware-snipeit-mcp"]="10 SNIPEIT_URL=https://snipeit.example.com SNIPEIT_TOKEN=dummy" + ["kneldevstack-aimiddleware-mcp-redmine"]="10 REDMINE_URL=https://redmine.example.com REDMINE_API_KEY=dummy" + ["kneldevstack-aimiddleware-mcp-ansible"]="10" + ["kneldevstack-aimiddleware-elasticsearch-mcp"]="10 ELASTICSEARCH_URL=http://localhost:9200 ELASTICSEARCH_USERNAME=dummy ELASTICSEARCH_PASSWORD=dummy" + ["kneldevstack-aimiddleware-drawio-mcp"]="10" + + # Additional Tools + ["kneldevstack-aimiddleware-penpot-mcp"]="10 PENPOT_URL=https://design.penpot.app PENPOT_TOKEN=dummy" + ["kneldevstack-aimiddleware-webserial-mcp"]="10 WEBSOCKET_URL=ws://host.docker.internal:3000" + + # Reverse Engineering + ["kneldevstack-aimiddleware-ghidra-mcp"]="15" + ["kneldevstack-aimiddleware-reverse-engineering-assistant"]="15" +) + +# LSP Servers configuration +declare -A LSP_SERVERS +LSP_SERVERS=( + ["kneldevstack-aimiddleware-bash-language-server"]="10" + ["kneldevstack-aimiddleware-docker-language-server"]="10" + ["kneldevstack-aimiddleware-marksman"]="10" + ["kneldevstack-aimiddleware-terraform-ls"]="10" +) + +# Test MCP server +test_mcp() { + local container=$1 + local timeout=$2 + shift 2 + local env_vars=("$@") + + # Check if image exists + if ! docker image inspect "$container" &>/dev/null; then + echo -e "${YELLOW}SKIP${NC} $container: Image not built" + SKIPPED=$((SKIPPED + 1)) + RESULTS_SKIP+=("$container: image not built") + return 2 + fi + + # Build env args + local env_args="" + for env_var in "${env_vars[@]}"; do + env_args="$env_args -e $env_var" + done + + # Run test + local result + result=$( (echo "$MCP_INIT" | timeout "$timeout" docker run --rm -i $env_args "$container" 2>&1) || true ) + + # Check for valid MCP response + if echo "$result" | grep -q '"result".*"serverInfo"'; then + local server_name version + server_name=$(echo "$result" | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4) + version=$(echo "$result" | grep -o '"version":"[^"]*"' | head -1 | cut -d'"' -f4) + echo -e "${GREEN}PASS${NC} $container: $server_name v${version:-unknown}" + PASSED=$((PASSED + 1)) + RESULTS_PASS+=("$container: $server_name v${version:-unknown}") + return 0 + fi + + # Check for specific failure modes + if echo "$result" | grep -qi "environment variable\|missing.*env\|required.*variable"; then + echo -e "${YELLOW}ENV${NC} $container: Needs environment variables" + NEEDS_ENV=$((NEEDS_ENV + 1)) + RESULTS_ENV+=("$container") + return 3 + fi + + if echo "$result" | grep -qi "connection refused\|not connected\|ECONNREFUSED\|ETIMEDOUT\|network unreachable"; then + echo -e "${YELLOW}SVC${NC} $container: Needs backend service" + NEEDS_SERVICE=$((NEEDS_SERVICE + 1)) + RESULTS_SERVICE+=("$container") + return 4 + fi + + if echo "$result" | grep -qi "http\|websocket\|ws://\|transport"; then + echo -e "${BLUE}TRN${NC} $container: Transport mismatch (not stdio)" + TRANSPORT_MISMATCH=$((TRANSPORT_MISMATCH + 1)) + RESULTS_TRANSPORT+=("$container") + return 5 + fi + + # Generic failure + echo -e "${RED}FAIL${NC} $container: No valid response" + FAILED=$((FAILED + 1)) + RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)") + return 1 +} + +# Test LSP server +test_lsp() { + local container=$1 + local timeout=$2 + + # Check if image exists + if ! docker image inspect "$container" &>/dev/null; then + echo -e "${YELLOW}SKIP${NC} $container: Image not built" + SKIPPED=$((SKIPPED + 1)) + RESULTS_SKIP+=("$container: image not built") + return 2 + fi + + # Run test + local result + result=$( (echo "$LSP_INIT" | timeout "$timeout" docker run --rm -i "$container" 2>&1) || true ) + + # Check for valid LSP response + if echo "$result" | grep -q '"result".*"capabilities"'; then + echo -e "${GREEN}PASS${NC} $container: LSP initialized" + PASSED=$((PASSED + 1)) + RESULTS_PASS+=("$container: LSP initialized") + return 0 + fi + + # Generic failure + echo -e "${RED}FAIL${NC} $container: No valid response" + FAILED=$((FAILED + 1)) + RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)") + return 1 +} + +# Print summary +print_summary() { + echo "" + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}VALIDATION SUMMARY${NC}" + echo -e "${BLUE}========================================${NC}" + echo "" + + echo -e "${GREEN}PASSED ($PASSED):${NC}" + if [ ${#RESULTS_PASS[@]} -gt 0 ]; then + for r in "${RESULTS_PASS[@]}"; do + echo " ✓ $r" + done + else + echo " (none)" + fi + echo "" + + if [ ${#RESULTS_ENV[@]} -gt 0 ]; then + echo -e "${YELLOW}NEEDS ENV VARS ($NEEDS_ENV):${NC}" + for r in "${RESULTS_ENV[@]}"; do + echo " ⚠ $r" + done + echo "" + fi + + if [ ${#RESULTS_SERVICE[@]} -gt 0 ]; then + echo -e "${YELLOW}NEEDS BACKEND SERVICE ($NEEDS_SERVICE):${NC}" + for r in "${RESULTS_SERVICE[@]}"; do + echo " ⚠ $r" + done + echo "" + fi + + if [ ${#RESULTS_TRANSPORT[@]} -gt 0 ]; then + echo -e "${BLUE}TRANSPORT MISMATCH ($TRANSPORT_MISMATCH):${NC}" + for r in "${RESULTS_TRANSPORT[@]}"; do + echo " ○ $r" + done + echo "" + fi + + if [ ${#RESULTS_SKIP[@]} -gt 0 ]; then + echo -e "${YELLOW}SKIPPED ($SKIPPED):${NC}" + for r in "${RESULTS_SKIP[@]}"; do + echo " - $r" + done + echo "" + fi + + if [ ${#RESULTS_FAIL[@]} -gt 0 ]; then + echo -e "${RED}FAILED ($FAILED):${NC}" + for r in "${RESULTS_FAIL[@]}"; do + echo " ✗ $r" + done + echo "" + fi + + echo -e "${BLUE}========================================${NC}" + echo -e "Total: ${GREEN}$PASSED passed${NC}, ${RED}$FAILED failed${NC}, ${YELLOW}$SKIPPED skipped${NC}" + echo -e " ${YELLOW}$NEEDS_ENV need env${NC}, ${YELLOW}$NEEDS_SERVICE need service${NC}, ${BLUE}$TRANSPORT_MISMATCH transport mismatch${NC}" + echo -e "${BLUE}========================================${NC}" + + # Return non-zero if any hard failures + [ $FAILED -eq 0 ] +} + +# Main +main() { + local mode="${1:-all}" + + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}KNEL-AIMiddleware Validation${NC}" + echo -e "${BLUE}========================================${NC}" + echo "" + + case "$mode" in + mcp) + echo -e "${BLUE}Testing MCP Servers...${NC}" + echo "" + for container in "${!MCP_SERVERS[@]}"; do + config="${MCP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + env_vars=("${parts[@]:1}") + test_mcp "$container" "$timeout" "${env_vars[@]}" || true + done + ;; + lsp) + echo -e "${BLUE}Testing LSP Servers...${NC}" + echo "" + for container in "${!LSP_SERVERS[@]}"; do + config="${LSP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + test_lsp "$container" "$timeout" || true + done + ;; + all) + echo -e "${BLUE}Testing MCP Servers...${NC}" + echo "" + for container in "${!MCP_SERVERS[@]}"; do + config="${MCP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + env_vars=("${parts[@]:1}") + test_mcp "$container" "$timeout" "${env_vars[@]}" || true + done + + echo "" + echo -e "${BLUE}Testing LSP Servers...${NC}" + echo "" + for container in "${!LSP_SERVERS[@]}"; do + config="${LSP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + test_lsp "$container" "$timeout" || true + done + ;; + *) + # Test specific service + if [[ -v MCP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then + container="kneldevstack-aimiddleware-$mode" + config="${MCP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + env_vars=("${parts[@]:1}") + test_mcp "$container" "$timeout" "${env_vars[@]}" + elif [[ -v LSP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then + container="kneldevstack-aimiddleware-$mode" + config="${LSP_SERVERS[$container]}" + read -ra parts <<< "$config" + timeout="${parts[0]}" + test_lsp "$container" "$timeout" + else + echo -e "${RED}Unknown service: $mode${NC}" + echo "Available MCP services: ${!MCP_SERVERS[@]#kneldevstack-aimiddleware-}" + echo "Available LSP services: ${!LSP_SERVERS[@]#kneldevstack-aimiddleware-}" + exit 1 + fi + ;; + esac + + if [ "$mode" != "all" ] && [ "$mode" != "mcp" ] && [ "$mode" != "lsp" ]; then + # Single service test, no summary + exit $? + fi + + print_summary +} + +main "$@"