feat(scripts): add comprehensive validation script for all MCP/LSP servers
Created scripts/validate-all.sh to validate ALL servers with actual protocol handshake messages: Coverage: - 29 MCP servers (Design, Infrastructure, Content, Communication, Analytics, Productivity, Reverse Engineering) - 4 LSP servers (bash, docker, marksman, terraform-ls) Features: - Sends actual MCP initialize handshake to each container - Sends actual LSP initialize handshake to LSP containers - Categorizes results: PASSED, FAILED, SKIPPED, NEEDS_ENV, NEEDS_SERVICE, TRANSPORT_MISMATCH - Provides detailed summary with server names and versions - Color-coded output for easy scanning Usage: ./scripts/validate-all.sh # Test all servers ./scripts/validate-all.sh mcp # MCP servers only ./scripts/validate-all.sh lsp # LSP servers only ./scripts/validate-all.sh <service> # Specific service This script implements the SDLC requirement for protocol validation. Every server must pass build + start + protocol handshake to be considered "working".
This commit is contained in:
Charles N Wyble
348
scripts/validate-all.sh
Executable file
348
scripts/validate-all.sh
Executable file
@@ -0,0 +1,348 @@
|
||||
#!/bin/bash
|
||||
# Comprehensive MCP/LSP Server Validation Script
|
||||
# Tests all servers with actual protocol messages
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/validate-all.sh # Test all servers
|
||||
# ./scripts/validate-all.sh mcp # Test only MCP servers
|
||||
# ./scripts/validate-all.sh lsp # Test only LSP servers
|
||||
# ./scripts/validate-all.sh <service> # Test specific service
|
||||
|
||||
set -e
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
# Protocol messages
|
||||
MCP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"capabilities":{},"protocolVersion":"2024-11-05","clientInfo":{"name":"validate-all","version":"1.0.0"}},"id":1}'
|
||||
LSP_INIT='{"jsonrpc":"2.0","method":"initialize","params":{"processId":null,"capabilities":{},"rootUri":null,"workspaceFolders":null},"id":1}'
|
||||
|
||||
# Counters
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
SKIPPED=0
|
||||
NEEDS_ENV=0
|
||||
NEEDS_SERVICE=0
|
||||
TRANSPORT_MISMATCH=0
|
||||
|
||||
# Results arrays
|
||||
declare -a RESULTS_PASS
|
||||
declare -a RESULTS_FAIL
|
||||
declare -a RESULTS_SKIP
|
||||
declare -a RESULTS_ENV
|
||||
declare -a RESULTS_SERVICE
|
||||
declare -a RESULTS_TRANSPORT
|
||||
|
||||
# MCP Servers configuration (container_name, timeout, env_vars...)
|
||||
declare -A MCP_SERVERS
|
||||
MCP_SERVERS=(
|
||||
# Design & Engineering
|
||||
["kneldevstack-aimiddleware-blender-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-freecad-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-gimp-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-kicad-mcp"]="10 KICAD_HOST=host.docker.internal KICAD_PORT=5555"
|
||||
|
||||
# Hosting & Infrastructure
|
||||
["kneldevstack-aimiddleware-kubernetes-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-docker-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-proxmox-mcp"]="10 PROXMOX_HOST=https://proxmox.example.com PROXMOX_USER=root@pam PROXMOX_TOKEN=dummy PROXMOX_NODE=pve"
|
||||
["kneldevstack-aimiddleware-terraform-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-cloudron-mcp"]="10 CLOUDRON_URL=https://cloudron.example.com CLOUDRON_TOKEN=dummy"
|
||||
|
||||
# Development Tools
|
||||
["kneldevstack-aimiddleware-context7-mcp"]="10 UPSTASH_REDIS_REST_URL=https://dummy.upstash.io UPSTASH_REDIS_REST_TOKEN=dummy"
|
||||
|
||||
# Content Management
|
||||
["kneldevstack-aimiddleware-nextcloud-mcp"]="10 NEXTCLOUD_HOST=https://nextcloud.example.com NEXTCLOUD_USERNAME=dummy NEXTCLOUD_PASSWORD=dummy"
|
||||
["kneldevstack-aimiddleware-ghost-mcp"]="10 GHOST_API_URL=https://ghost.example.com GHOST_ADMIN_API_KEY=012345678901234567890123:0123456789012345678901234567890123456789012345678901234567890123"
|
||||
["kneldevstack-aimiddleware-docspace-mcp"]="10 DOCSPACE_HOST=https://docspace.example.com DOCSPACE_TOKEN=dummy"
|
||||
["kneldevstack-aimiddleware-wordpress-mcp"]="10 WORDPRESS_URL=https://wordpress.example.com WORDPRESS_USERNAME=dummy WORDPRESS_APPLICATION_PASSWORD=dummy"
|
||||
|
||||
# Communication & Collaboration
|
||||
["kneldevstack-aimiddleware-discourse-mcp"]="10 DISCOURSE_URL=https://discourse.example.com DISCOURSE_API_KEY=dummy DISCOURSE_API_USERNAME=dummy"
|
||||
["kneldevstack-aimiddleware-imap-mcp"]="10 IMAP_HOST=imap.example.com IMAP_PORT=993 IMAP_USERNAME=dummy IMAP_PASSWORD=dummy"
|
||||
["kneldevstack-aimiddleware-postizz-mcp"]="10 POSTIZ_API_KEY=dummy POSTIZ_URL=https://postiz.example.com"
|
||||
|
||||
# Analytics & Security
|
||||
["kneldevstack-aimiddleware-matomo-mcp"]="10 MATOMO_URL=https://matomo.example.com MATOMO_TOKEN=dummy"
|
||||
["kneldevstack-aimiddleware-bitwarden-mcp"]="15 BITWARDEN_CLIENT_ID=dummy BITWARDEN_CLIENT_SECRET=dummy BITWARDEN_PASSWORD=dummy BITWARDEN_SERVER_URL=https://vault.bitwarden.com"
|
||||
|
||||
# Productivity & Automation
|
||||
["kneldevstack-aimiddleware-audiobook-mcp"]="10"
|
||||
["kneldevstack-aimiddleware-snipeit-mcp"]="10 SNIPEIT_URL=https://snipeit.example.com SNIPEIT_TOKEN=dummy"
|
||||
["kneldevstack-aimiddleware-mcp-redmine"]="10 REDMINE_URL=https://redmine.example.com REDMINE_API_KEY=dummy"
|
||||
["kneldevstack-aimiddleware-mcp-ansible"]="10"
|
||||
["kneldevstack-aimiddleware-elasticsearch-mcp"]="10 ELASTICSEARCH_URL=http://localhost:9200 ELASTICSEARCH_USERNAME=dummy ELASTICSEARCH_PASSWORD=dummy"
|
||||
["kneldevstack-aimiddleware-drawio-mcp"]="10"
|
||||
|
||||
# Additional Tools
|
||||
["kneldevstack-aimiddleware-penpot-mcp"]="10 PENPOT_URL=https://design.penpot.app PENPOT_TOKEN=dummy"
|
||||
["kneldevstack-aimiddleware-webserial-mcp"]="10 WEBSOCKET_URL=ws://host.docker.internal:3000"
|
||||
|
||||
# Reverse Engineering
|
||||
["kneldevstack-aimiddleware-ghidra-mcp"]="15"
|
||||
["kneldevstack-aimiddleware-reverse-engineering-assistant"]="15"
|
||||
)
|
||||
|
||||
# LSP Servers configuration
|
||||
declare -A LSP_SERVERS
|
||||
LSP_SERVERS=(
|
||||
["kneldevstack-aimiddleware-bash-language-server"]="10"
|
||||
["kneldevstack-aimiddleware-docker-language-server"]="10"
|
||||
["kneldevstack-aimiddleware-marksman"]="10"
|
||||
["kneldevstack-aimiddleware-terraform-ls"]="10"
|
||||
)
|
||||
|
||||
# Test MCP server
|
||||
test_mcp() {
|
||||
local container=$1
|
||||
local timeout=$2
|
||||
shift 2
|
||||
local env_vars=("$@")
|
||||
|
||||
# Check if image exists
|
||||
if ! docker image inspect "$container" &>/dev/null; then
|
||||
echo -e "${YELLOW}SKIP${NC} $container: Image not built"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
RESULTS_SKIP+=("$container: image not built")
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Build env args
|
||||
local env_args=""
|
||||
for env_var in "${env_vars[@]}"; do
|
||||
env_args="$env_args -e $env_var"
|
||||
done
|
||||
|
||||
# Run test
|
||||
local result
|
||||
result=$( (echo "$MCP_INIT" | timeout "$timeout" docker run --rm -i $env_args "$container" 2>&1) || true )
|
||||
|
||||
# Check for valid MCP response
|
||||
if echo "$result" | grep -q '"result".*"serverInfo"'; then
|
||||
local server_name version
|
||||
server_name=$(echo "$result" | grep -o '"name":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||||
version=$(echo "$result" | grep -o '"version":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||||
echo -e "${GREEN}PASS${NC} $container: $server_name v${version:-unknown}"
|
||||
PASSED=$((PASSED + 1))
|
||||
RESULTS_PASS+=("$container: $server_name v${version:-unknown}")
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check for specific failure modes
|
||||
if echo "$result" | grep -qi "environment variable\|missing.*env\|required.*variable"; then
|
||||
echo -e "${YELLOW}ENV${NC} $container: Needs environment variables"
|
||||
NEEDS_ENV=$((NEEDS_ENV + 1))
|
||||
RESULTS_ENV+=("$container")
|
||||
return 3
|
||||
fi
|
||||
|
||||
if echo "$result" | grep -qi "connection refused\|not connected\|ECONNREFUSED\|ETIMEDOUT\|network unreachable"; then
|
||||
echo -e "${YELLOW}SVC${NC} $container: Needs backend service"
|
||||
NEEDS_SERVICE=$((NEEDS_SERVICE + 1))
|
||||
RESULTS_SERVICE+=("$container")
|
||||
return 4
|
||||
fi
|
||||
|
||||
if echo "$result" | grep -qi "http\|websocket\|ws://\|transport"; then
|
||||
echo -e "${BLUE}TRN${NC} $container: Transport mismatch (not stdio)"
|
||||
TRANSPORT_MISMATCH=$((TRANSPORT_MISMATCH + 1))
|
||||
RESULTS_TRANSPORT+=("$container")
|
||||
return 5
|
||||
fi
|
||||
|
||||
# Generic failure
|
||||
echo -e "${RED}FAIL${NC} $container: No valid response"
|
||||
FAILED=$((FAILED + 1))
|
||||
RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)")
|
||||
return 1
|
||||
}
|
||||
|
||||
# Test LSP server
|
||||
test_lsp() {
|
||||
local container=$1
|
||||
local timeout=$2
|
||||
|
||||
# Check if image exists
|
||||
if ! docker image inspect "$container" &>/dev/null; then
|
||||
echo -e "${YELLOW}SKIP${NC} $container: Image not built"
|
||||
SKIPPED=$((SKIPPED + 1))
|
||||
RESULTS_SKIP+=("$container: image not built")
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Run test
|
||||
local result
|
||||
result=$( (echo "$LSP_INIT" | timeout "$timeout" docker run --rm -i "$container" 2>&1) || true )
|
||||
|
||||
# Check for valid LSP response
|
||||
if echo "$result" | grep -q '"result".*"capabilities"'; then
|
||||
echo -e "${GREEN}PASS${NC} $container: LSP initialized"
|
||||
PASSED=$((PASSED + 1))
|
||||
RESULTS_PASS+=("$container: LSP initialized")
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Generic failure
|
||||
echo -e "${RED}FAIL${NC} $container: No valid response"
|
||||
FAILED=$((FAILED + 1))
|
||||
RESULTS_FAIL+=("$container: $(echo "$result" | head -1 | cut -c1-60)")
|
||||
return 1
|
||||
}
|
||||
|
||||
# Print summary
|
||||
print_summary() {
|
||||
echo ""
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}VALIDATION SUMMARY${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}PASSED ($PASSED):${NC}"
|
||||
if [ ${#RESULTS_PASS[@]} -gt 0 ]; then
|
||||
for r in "${RESULTS_PASS[@]}"; do
|
||||
echo " ✓ $r"
|
||||
done
|
||||
else
|
||||
echo " (none)"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
if [ ${#RESULTS_ENV[@]} -gt 0 ]; then
|
||||
echo -e "${YELLOW}NEEDS ENV VARS ($NEEDS_ENV):${NC}"
|
||||
for r in "${RESULTS_ENV[@]}"; do
|
||||
echo " ⚠ $r"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ ${#RESULTS_SERVICE[@]} -gt 0 ]; then
|
||||
echo -e "${YELLOW}NEEDS BACKEND SERVICE ($NEEDS_SERVICE):${NC}"
|
||||
for r in "${RESULTS_SERVICE[@]}"; do
|
||||
echo " ⚠ $r"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ ${#RESULTS_TRANSPORT[@]} -gt 0 ]; then
|
||||
echo -e "${BLUE}TRANSPORT MISMATCH ($TRANSPORT_MISMATCH):${NC}"
|
||||
for r in "${RESULTS_TRANSPORT[@]}"; do
|
||||
echo " ○ $r"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ ${#RESULTS_SKIP[@]} -gt 0 ]; then
|
||||
echo -e "${YELLOW}SKIPPED ($SKIPPED):${NC}"
|
||||
for r in "${RESULTS_SKIP[@]}"; do
|
||||
echo " - $r"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
if [ ${#RESULTS_FAIL[@]} -gt 0 ]; then
|
||||
echo -e "${RED}FAILED ($FAILED):${NC}"
|
||||
for r in "${RESULTS_FAIL[@]}"; do
|
||||
echo " ✗ $r"
|
||||
done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "Total: ${GREEN}$PASSED passed${NC}, ${RED}$FAILED failed${NC}, ${YELLOW}$SKIPPED skipped${NC}"
|
||||
echo -e " ${YELLOW}$NEEDS_ENV need env${NC}, ${YELLOW}$NEEDS_SERVICE need service${NC}, ${BLUE}$TRANSPORT_MISMATCH transport mismatch${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
|
||||
# Return non-zero if any hard failures
|
||||
[ $FAILED -eq 0 ]
|
||||
}
|
||||
|
||||
# Main
|
||||
main() {
|
||||
local mode="${1:-all}"
|
||||
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo -e "${BLUE}KNEL-AIMiddleware Validation${NC}"
|
||||
echo -e "${BLUE}========================================${NC}"
|
||||
echo ""
|
||||
|
||||
case "$mode" in
|
||||
mcp)
|
||||
echo -e "${BLUE}Testing MCP Servers...${NC}"
|
||||
echo ""
|
||||
for container in "${!MCP_SERVERS[@]}"; do
|
||||
config="${MCP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
env_vars=("${parts[@]:1}")
|
||||
test_mcp "$container" "$timeout" "${env_vars[@]}" || true
|
||||
done
|
||||
;;
|
||||
lsp)
|
||||
echo -e "${BLUE}Testing LSP Servers...${NC}"
|
||||
echo ""
|
||||
for container in "${!LSP_SERVERS[@]}"; do
|
||||
config="${LSP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
test_lsp "$container" "$timeout" || true
|
||||
done
|
||||
;;
|
||||
all)
|
||||
echo -e "${BLUE}Testing MCP Servers...${NC}"
|
||||
echo ""
|
||||
for container in "${!MCP_SERVERS[@]}"; do
|
||||
config="${MCP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
env_vars=("${parts[@]:1}")
|
||||
test_mcp "$container" "$timeout" "${env_vars[@]}" || true
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}Testing LSP Servers...${NC}"
|
||||
echo ""
|
||||
for container in "${!LSP_SERVERS[@]}"; do
|
||||
config="${LSP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
test_lsp "$container" "$timeout" || true
|
||||
done
|
||||
;;
|
||||
*)
|
||||
# Test specific service
|
||||
if [[ -v MCP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then
|
||||
container="kneldevstack-aimiddleware-$mode"
|
||||
config="${MCP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
env_vars=("${parts[@]:1}")
|
||||
test_mcp "$container" "$timeout" "${env_vars[@]}"
|
||||
elif [[ -v LSP_SERVERS["kneldevstack-aimiddleware-$mode"] ]]; then
|
||||
container="kneldevstack-aimiddleware-$mode"
|
||||
config="${LSP_SERVERS[$container]}"
|
||||
read -ra parts <<< "$config"
|
||||
timeout="${parts[0]}"
|
||||
test_lsp "$container" "$timeout"
|
||||
else
|
||||
echo -e "${RED}Unknown service: $mode${NC}"
|
||||
echo "Available MCP services: ${!MCP_SERVERS[@]#kneldevstack-aimiddleware-}"
|
||||
echo "Available LSP services: ${!LSP_SERVERS[@]#kneldevstack-aimiddleware-}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$mode" != "all" ] && [ "$mode" != "mcp" ] && [ "$mode" != "lsp" ]; then
|
||||
# Single service test, no summary
|
||||
exit $?
|
||||
fi
|
||||
|
||||
print_summary
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user