KNEL-AIMiddleware/scripts/validate-all.sh

#!/bin/bash
# Comprehensive MCP/LSP Server Validation Script
# Tests all servers with actual protocol messages
#
# Usage:
#   ./scripts/validate-all.sh           # Test all servers
#   ./scripts/validate-all.sh mcp       # Test only MCP servers
#   ./scripts/validate-all.sh lsp       # Test only LSP servers
#   ./scripts/validate-all.sh <service> # Test specific service

set -e

# Colors
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Protocol messages
MCP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{},"protocolVersion":"2024-11-05","clientInfo":{"name":"validate-all","version":"1.0.0"}},"id":1}'
LSP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"processId":null,"capabilities":{},"rootUri":null,"workspaceFolders":null},"id":1}'

# Counters
PASSED=0
FAILED=0
SKIPPED=0
NEEDS_ENV=0
NEEDS_SERVICE=0
TRANSPORT_MISMATCH=0

# Results arrays
declare -a RESULTS_PASS
declare -a RESULTS_FAIL
declare -a RESULTS_SKIP
declare -a RESULTS_ENV
declare -a RESULTS_SERVICE
declare -a RESULTS_TRANSPORT

# MCP Servers configuration (container_name, timeout, env_vars...)
declare -A MCP_SERVERS
MCP_SERVERS=(
    # Design & Engineering
    ["kneldevstack-aimiddleware-blender-mcp"]="10"
    ["kneldevstack-aimiddleware-freecad-mcp"]="10"
    ["kneldevstack-aimiddleware-gimp-mcp"]="10"
    ["kneldevstack-aimiddleware-kicad-mcp"]="10 KICAD_HOST=host.docker.internal KICAD_PORT=5555"

    # Hosting & Infrastructure
    ["kneldevstack-aimiddleware-kubernetes-mcp"]="10"
    ["kneldevstack-aimiddleware-docker-mcp"]="10"
    ["kneldevstack-aimiddleware-proxmox-mcp"]="10 PROXMOX_HOST=https://proxmox.example.com PROXMOX_USER=root@pam PROXMOX_TOKEN=dummy PROXMOX_NODE=pve"
    ["kneldevstack-aimiddleware-terraform-mcp"]="10"
    ["kneldevstack-aimiddleware-cloudron-mcp"]="10 CLOUDRON_URL=https://cloudron.example.com CLOUDRON_TOKEN=dummy"

    # Development Tools
    ["kneldevstack-aimiddleware-context7-mcp"]="10 UPSTASH_REDIS_REST_URL=https://dummy.upstash.io UPSTASH_REDIS_REST_TOKEN=dummy"

    # Content Management
    ["kneldevstack-aimiddleware-nextcloud-mcp"]="10 NEXTCLOUD_HOST=https://nextcloud.example.com NEXTCLOUD_USERNAME=dummy NEXTCLOUD_PASSWORD=dummy"
    ["kneldevstack-aimiddleware-ghost-mcp"]="10 GHOST_API_URL=https://ghost.example.com GHOST_ADMIN_API_KEY=012345678901234567890123:0123456789012345678901234567890123456789012345678901234567890123"
    ["kneldevstack-aimiddleware-docspace-mcp"]="10 DOCSPACE_HOST=https://docspace.example.com DOCSPACE_TOKEN=dummy"
    ["kneldevstack-aimiddleware-wordpress-mcp"]="10 WORDPRESS_URL=https://wordpress.example.com WORDPRESS_USERNAME=dummy WORDPRESS_APPLICATION_PASSWORD=dummy"

    # Communication & Collaboration
    ["kneldevstack-aimiddleware-discourse-mcp"]="10 DISCOURSE_URL=https://discourse.example.com DISCOURSE_API_KEY=dummy DISCOURSE_API_USERNAME=dummy"
    ["kneldevstack-aimiddleware-imap-mcp"]="10 IMAP_HOST=imap.example.com IMAP_PORT=993 IMAP_USERNAME=dummy IMAP_PASSWORD=dummy"
    ["kneldevstack-aimiddleware-postizz-mcp"]="10 POSTIZ_API_KEY=dummy POSTIZ_URL=https://postiz.example.com"

    # Analytics & Security
    ["kneldevstack-aimiddleware-matomo-mcp"]="10 MATOMO_URL=https://matomo.example.com MATOMO_TOKEN=dummy"
    ["kneldevstack-aimiddleware-bitwarden-mcp"]="15 BITWARDEN_CLIENT_ID=dummy BITWARDEN_CLIENT_SECRET=dummy BITWARDEN_PASSWORD=dummy BITWARDEN_SERVER_URL=https://vault.bitwarden.com"

    # Productivity & Automation
    ["kneldevstack-aimiddleware-audiobook-mcp"]="10"
    ["kneldevstack-aimiddleware-snipeit-mcp"]="10 SNIPEIT_URL=https://snipeit.example.com SNIPEIT_TOKEN=dummy"
    ["kneldevstack-aimiddleware-mcp-redmine"]="10 REDMINE_URL=https://redmine.example.com REDMINE_API_KEY=dummy"
    ["kneldevstack-aimiddleware-mcp-ansible"]="10"
    ["kneldevstack-aimiddleware-elasticsearch-mcp"]="10 ELASTICSEARCH_URL=http://localhost:9200 ELASTICSEARCH_USERNAME=dummy ELASTICSEARCH_PASSWORD=dummy"
    ["kneldevstack-aimiddleware-drawio-mcp"]="10"

    # Additional Tools
    ["kneldevstack-aimiddleware-penpot-mcp"]="10 PENPOT_URL=https://design.penpot.app PENPOT_TOKEN=dummy"
    ["kneldevstack-aimiddleware-webserial-mcp"]="10 WEBSOCKET_URL=ws://host.docker.internal:3000"

    # Reverse Engineering
    ["kneldevstack-aimiddleware-ghidra-mcp"]="15"
    ["kneldevstack-aimiddleware-reverse-engineering-assistant"]="15"
)

# LSP Servers configuration
declare -A LSP_SERVERS
LSP_SERVERS=(
    ["kneldevstack-aimiddleware-bash-language-server"]="10"
    ["kneldevstack-aimiddleware-docker-language-server"]="10"
    ["kneldevstack-aimiddleware-marksman"]="10"
    ["kneldevstack-aimiddleware-terraform-ls"]="10"
)

# Test MCP server
test_mcp() {
    local container=$1
    local timeout=$2
    shift 2
    local env_vars=("$@")

    # Check if image exists
    if ! docker image inspect "$container" &>/dev/null; then
        echo -e "${YELLOW}SKIP${NC} $container: Image not built"
        SKIPPED=$((SKIPPED + 1))
        RESULTS_SKIP+=("$container: image not built")
        return 2
    fi

    # Build env args
    local env_args=""
    for env_var in "${env_vars[@]}"; do
        env_args="$env_args -e $env_var"
    done

    # Run test
    local result
    result=$( (echo "$MCP_INIT" | timeout "$timeout" docker run --rm -i $env_args "$container" 2>&1) || true )

    # Check for valid MCP response
    if echo "$result" | grep -q '"result".*"serverInfo"'; then
        local server_name version
        server_name=$(echo "$result" | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4)
        version=$(echo "$result" | grep -o '"version":"[^"]*"' | head -1 | cut -d'"' -f4)
        echo -e "${GREEN}PASS${NC} $container: $server_name v${version:-unknown}"
        PASSED=$((PASSED + 1))
        RESULTS_PASS+=("$container: $server_name v${version:-unknown}")
        return 0
    fi

    # Check for specific failure modes
    if echo "$result" | grep -qi "environment variable\|missing.*env\|required.*variable"; then
        echo -e "${YELLOW}ENV${NC} $container: Needs environment variables"
        NEEDS_ENV=$((NEEDS_ENV + 1))
        RESULTS_ENV+=("$container")
        return 3
    fi

    if echo "$result" | grep -qi "connection refused\|not connected\|ECONNREFUSED\|ETIMEDOUT\|network unreachable"; then
        echo -e "${YELLOW}SVC${NC} $container: Needs backend service"
        NEEDS_SERVICE=$((NEEDS_SERVICE + 1))
        RESULTS_SERVICE+=("$container")
        return 4
    fi

    if echo "$result" | grep -qi "http\|websocket\|ws://\|transport"; then
        echo -e "${BLUE}TRN${NC} $container: Transport mismatch (not stdio)"
        TRANSPORT_MISMATCH=$((TRANSPORT_MISMATCH + 1))
        RESULTS_TRANSPORT+=("$container")
        return 5
    fi

    # Generic failure
    echo -e "${RED}FAIL${NC} $container: No valid response"
    FAILED=$((FAILED + 1))
    RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)")
    return 1
}

# Test LSP server
test_lsp() {
    local container=$1
    local timeout=$2

    # Check if image exists
    if ! docker image inspect "$container" &>/dev/null; then
        echo -e "${YELLOW}SKIP${NC} $container: Image not built"
        SKIPPED=$((SKIPPED + 1))
        RESULTS_SKIP+=("$container: image not built")
        return 2
    fi

    # Run test
    local result
    result=$( (echo "$LSP_INIT" | timeout "$timeout" docker run --rm -i "$container" 2>&1) || true )

    # Check for valid LSP response
    if echo "$result" | grep -q '"result".*"capabilities"'; then
        echo -e "${GREEN}PASS${NC} $container: LSP initialized"
        PASSED=$((PASSED + 1))
        RESULTS_PASS+=("$container: LSP initialized")
        return 0
    fi

    # Generic failure
    echo -e "${RED}FAIL${NC} $container: No valid response"
    FAILED=$((FAILED + 1))
    RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)")
    return 1
}

# Print summary
print_summary() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}VALIDATION SUMMARY${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo ""

    echo -e "${GREEN}PASSED ($PASSED):${NC}"
    if [ ${#RESULTS_PASS[@]} -gt 0 ]; then
        for r in "${RESULTS_PASS[@]}"; do
            echo "  ✓ $r"
        done
    else
        echo "  (none)"
    fi
    echo ""

    if [ ${#RESULTS_ENV[@]} -gt 0 ]; then
        echo -e "${YELLOW}NEEDS ENV VARS ($NEEDS_ENV):${NC}"
        for r in "${RESULTS_ENV[@]}"; do
            echo "  ⚠ $r"
        done
        echo ""
    fi

    if [ ${#RESULTS_SERVICE[@]} -gt 0 ]; then
        echo -e "${YELLOW}NEEDS BACKEND SERVICE ($NEEDS_SERVICE):${NC}"
        for r in "${RESULTS_SERVICE[@]}"; do
            echo "  ⚠ $r"
        done
        echo ""
    fi

    if [ ${#RESULTS_TRANSPORT[@]} -gt 0 ]; then
        echo -e "${BLUE}TRANSPORT MISMATCH ($TRANSPORT_MISMATCH):${NC}"
        for r in "${RESULTS_TRANSPORT[@]}"; do
            echo "  ○ $r"
        done
        echo ""
    fi

    if [ ${#RESULTS_SKIP[@]} -gt 0 ]; then
        echo -e "${YELLOW}SKIPPED ($SKIPPED):${NC}"
        for r in "${RESULTS_SKIP[@]}"; do
            echo "  - $r"
        done
        echo ""
    fi

    if [ ${#RESULTS_FAIL[@]} -gt 0 ]; then
        echo -e "${RED}FAILED ($FAILED):${NC}"
        for r in "${RESULTS_FAIL[@]}"; do
            echo "  ✗ $r"
        done
        echo ""
    fi

    echo -e "${BLUE}========================================${NC}"
    echo -e "Total: ${GREEN}$PASSED passed${NC}, ${RED}$FAILED failed${NC}, ${YELLOW}$SKIPPED skipped${NC}"
    echo -e "         ${YELLOW}$NEEDS_ENV need env${NC}, ${YELLOW}$NEEDS_SERVICE need service${NC}, ${BLUE}$TRANSPORT_MISMATCH transport mismatch${NC}"
    echo -e "${BLUE}========================================${NC}"

    # Return non-zero if any hard failures
    [ $FAILED -eq 0 ]
}

# Main
main() {
    local mode="${1:-all}"

    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}KNEL-AIMiddleware Validation${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo ""

    case "$mode" in
        mcp)
            echo -e "${BLUE}Testing MCP Servers...${NC}"
            echo ""
            for container in "${!MCP_SERVERS[@]}"; do
                config="${MCP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                env_vars=("${parts[@]:1}")
                test_mcp "$container" "$timeout" "${env_vars[@]}" || true
            done
            ;;
        lsp)
            echo -e "${BLUE}Testing LSP Servers...${NC}"
            echo ""
            for container in "${!LSP_SERVERS[@]}"; do
                config="${LSP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                test_lsp "$container" "$timeout" || true
            done
            ;;
        all)
            echo -e "${BLUE}Testing MCP Servers...${NC}"
            echo ""
            for container in "${!MCP_SERVERS[@]}"; do
                config="${MCP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                env_vars=("${parts[@]:1}")
                test_mcp "$container" "$timeout" "${env_vars[@]}" || true
            done

            echo ""
            echo -e "${BLUE}Testing LSP Servers...${NC}"
            echo ""
            for container in "${!LSP_SERVERS[@]}"; do
                config="${LSP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                test_lsp "$container" "$timeout" || true
            done
            ;;
        *)
            # Test specific service
            if [[ -v MCP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then
                container="kneldevstack-aimiddleware-$mode"
                config="${MCP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                env_vars=("${parts[@]:1}")
                test_mcp "$container" "$timeout" "${env_vars[@]}"
            elif [[ -v LSP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then
                container="kneldevstack-aimiddleware-$mode"
                config="${LSP_SERVERS[$container]}"
                read -ra parts <<< "$config"
                timeout="${parts[0]}"
                test_lsp "$container" "$timeout"
            else
                echo -e "${RED}Unknown service: $mode${NC}"
                echo "Available MCP services: ${!MCP_SERVERS[@]#kneldevstack-aimiddleware-}"
                echo "Available LSP services: ${!LSP_SERVERS[@]#kneldevstack-aimiddleware-}"
                exit 1
            fi
            ;;
    esac

    if [ "$mode" != "all" ] && [ "$mode" != "mcp" ] && [ "$mode" != "lsp" ]; then
        # Single service test, no summary
        exit $?
    fi

    print_summary
}

main "$@"