diff --git a/13-microvm-setup.sh b/13-microvm-setup.sh new file mode 100755 index 0000000..25fe79d --- /dev/null +++ b/13-microvm-setup.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# 13-microvm-setup.sh — install Firecracker-class microVM tooling (#615) +# Idempotent; run as root: RUN=1 sudo -E ./13-microvm-setup.sh +# Engine note: qemu -M microvm + KVM is the default engine. Firecracker +# v1.16.1 is installed but its strict KVM capability check +# (USER_MEMORY/SET_TSS_ADDR/VAPIC) is refused by this host's nested KVM +# (Westmere under PVE) — retry MICROVM_ENGINE=firecracker after the +# #601 hardware upgrade. +set -euo pipefail +[ "${RUN:-0}" = 1 ] || { echo "gate: RUN=1 to execute"; exit 1; } +[ "$(id -u)" = 0 ] || { echo "must be root"; exit 1; } + +FC_VER=1.16.1 +FC_SHA=382a02a869e4d6d5cb14c40577f9545e8458021ea8b0b2d3fc10ec14d9c242e6 +FC_URL=https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VER} +IMG_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 +SUMS_URL=https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS + +BASE=/usr/local/lib/microvm/base +SRC=${SRC:-$(cd "$(dirname "$0")" && pwd)/microvm} + +echo "== packages" +DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + qemu-system-x86 qemu-utils dosfstools mtools curl + +echo "== kvm group" +for u in reachableceo reachableceo-offstage; do + id -nG "$u" | tr ' ' '\n' | grep -qx kvm || usermod -aG kvm "$u" +done + +echo "== firecracker v${FC_VER}" +if [ ! -x /usr/local/bin/firecracker ] || ! /usr/local/bin/firecracker --version 2>/dev/null | grep -q "$FC_VER"; then + t=$(mktemp -d) + curl -fsSL -o "$t/fc.tgz" "$FC_URL/firecracker-v${FC_VER}-x86_64.tgz" + echo "$FC_SHA $t/fc.tgz" | sha256sum -c - + tar xzf "$t/fc.tgz" -C "$t" release-v${FC_VER}-x86_64/firecracker-v${FC_VER}-x86_64 \ + release-v${FC_VER}-x86_64/jailer-v${FC_VER}-x86_64 + install -m0755 "$t/release-v${FC_VER}-x86_64/firecracker-v${FC_VER}-x86_64" /usr/local/bin/firecracker + install -m0755 "$t/release-v${FC_VER}-x86_64/jailer-v${FC_VER}-x86_64" /usr/local/bin/jailer + rm -rf "$t" +fi +/usr/local/bin/firecracker --version + +echo "== base assets" +mkdir -p "$BASE" +if [ ! -f "$BASE/debian-13-generic-amd64.qcow2" ]; then + curl -fsSL -o "$BASE/debian-13-generic-amd64.qcow2" "$IMG_URL" + curl -fsSL -o "$BASE/SHA512SUMS" "$SUMS_URL" + (cd "$BASE" && grep 'debian-13-generic-amd64.qcow2$' SHA512SUMS | sha512sum -c -) +fi +if [ ! -f "$BASE/vmlinuz" ] || [ ! -f "$BASE/initrd" ]; then + modprobe nbd max_part=8 + qemu-nbd -c /dev/nbd0 --read-only "$BASE/debian-13-generic-amd64.qcow2" + sleep 2 + mkdir -p /mnt/microvm-base + mount -o ro /dev/nbd0p1 /mnt/microvm-base + cp -L /mnt/microvm-base/boot/vmlinuz-* "$BASE/vmlinuz" + cp -L /mnt/microvm-base/boot/initrd.img-* "$BASE/initrd" + umount /mnt/microvm-base + qemu-nbd -d /dev/nbd0 +fi +if [ ! -f "$BASE/base-rootfs.raw" ]; then + qemu-img convert -O raw "$BASE/debian-13-generic-amd64.qcow2" "$BASE/base-rootfs.raw" + qemu-img resize "$BASE/base-rootfs.raw" 10G >/dev/null +fi +chmod 644 "$BASE"/vmlinuz "$BASE"/initrd "$BASE"/base-rootfs.raw "$BASE"/debian-13-generic-amd64.qcow2 +ls -lh "$BASE" + +echo "== tooling" +install -m0755 "$SRC/microvm" /usr/local/bin/microvm +install -m0755 "$SRC/microvm-net.sh" /usr/local/sbin/microvm-net.sh +install -m0644 "$SRC/microvm-net.service" /etc/systemd/system/microvm-net.service +systemctl daemon-reload +systemctl enable --now microvm-net.service +/usr/local/sbin/microvm-net.sh + +echo +echo "DONE. As a human account:" +echo " microvm create test1 && microvm start test1 && microvm ssh test1" +echo "(first ssh needs one logout/login if kvm group was just added)" diff --git a/TRACKING.md b/TRACKING.md index eab1c54..f599f67 100644 --- a/TRACKING.md +++ b/TRACKING.md @@ -17,6 +17,8 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 / | #610 | fleet | ✅ | Open Terminal 9-account fleet (tailscale-only :30000-30008) + OpenWebUI UAT PASS | | #611 | rdp | 🔄 | Jump/iPad + 5160x2160 monitor: EGFX frame-budget fix live (12-fix-rdp-bigres.sh); freerdp UAT all-clean; human UAT pending | | #612 | dotfiles | 🔄 | kali-addendum: Kali twoline prompt + completions (kali-comforts.sh) on top of omz, p10k = fallback (KALI_PROMPT=0); awaiting UAT | +| #615 | microvm | 🔄 | microvm tooling LIVE (qemu-microvm engine; firecracker blocked by nested-KVM caps until #601 hardware) + rebuild capture/docs next | +| #616 | macos | ⏳ | macOS VM on pfv-tsys5 (headless CI + iOS dev) — blocked by #601 | Inbox (mid-task interrupts): none. diff --git a/microvm/microvm b/microvm/microvm new file mode 100755 index 0000000..930c9a4 --- /dev/null +++ b/microvm/microvm @@ -0,0 +1,206 @@ +#!/bin/bash +# microvm — Firecracker-class microVMs on ultix-streaming (Redmine #615) +# Engine: qemu -M microvm + KVM (default) | Firecracker (MICROVM_ENGINE=firecracker; +# needs a KVM exposing USER_MEMORY/SET_TSS_ADDR/VAPIC caps — refused by this +# host's nested KVM until the #601 hardware upgrade; binary stays installed). +# Guests: Debian 13 cloud image, cloud-init NoCloud seed, static tap networking. +# Instances: ~/.microvm/. Host prep: microvm-net.service (taps mv-

). + +set -euo pipefail +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +BASE=/usr/local/lib/microvm/base +ROOT=$HOME/.microvm +DISK_GIB=${DISK_GIB:-10} +SSH_USER=debian + +case "$USER" in + reachableceo) PREFIX=r; SUBNET=172.30.0 ;; + reachableceo-offstage) PREFIX=o; SUBNET=172.30.1 ;; + *) echo "microvm: no tap pool for user '$USER' (add one in microvm-net.sh)" >&2; exit 1 ;; +esac + +die() { echo "microvm: $*" >&2; exit 1; } + +slot_ip() { # slot -> "host_ip guest_ip" + echo "$SUBNET.$((4*$1+1))" "$SUBNET.$((4*$1+2))" +} + +free_slot() { + local used slot + used=$(cat "$ROOT"/*/vm.conf 2>/dev/null | grep -E '^SLOT=' | cut -d= -f2 | sort -u || true) + for slot in 0 1 2 3; do + grep -qx "$slot" <<<"$used" || { echo "$slot"; return; } + done + die "all 4 tap slots busy; remove an instance first (microvm list / rm)" +} + +load_conf() { + [ -f "$ROOT/$1/vm.conf" ] || die "no such instance '$1' (microvm list)" + . "$ROOT/$1/vm.conf" +} + +cmd_create() { + local name=${1:-} mem=${2:-1024} vcpu=${3:-1} slot hostip guestip pubkey + [[ "$name" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || die "name: ^[a-z][a-z0-9-]{0,15}$" + [ -d "$ROOT/$name" ] && die "instance '$name' already exists" + slot=$(free_slot) + read -r hostip guestip <<<"$(slot_ip "$slot")" + mkdir -p "$ROOT/$name" + + pubkey=$(ls -1 "$HOME"/.ssh/id_*.pub 2>/dev/null | head -1) + [ -n "$pubkey" ] || die "no $HOME/.ssh/id_*.pub found (needed for guest login)" + + cat > "$ROOT/$name/user-data" < "$ROOT/$name/network-config" < "$ROOT/$name/meta-data" + + truncate -s 4M "$ROOT/$name/seed.img" + mkfs.vfat -n CIDATA "$ROOT/$name/seed.img" >/dev/null + mcopy -i "$ROOT/$name/seed.img" "$ROOT/$name/user-data" "$ROOT/$name/network-config" "$ROOT/$name/meta-data" :: + + echo "microvm: cloning base rootfs (${DISK_GIB}G sparse)..." + qemu-img convert -O raw "$BASE/base-rootfs.raw" "$ROOT/$name/rootfs.raw" + qemu-img resize "$ROOT/$name/rootfs.raw" "${DISK_GIB}G" >/dev/null + + if [ "$PREFIX" = r ]; then mnet=0; else mnet=1; fi + MAC=$(printf '52:54:00:30:%02x:%02x' $((mnet*16+slot)) $((slot+1))) + + cat > "$ROOT/$name/vm.conf" <"$ROOT/$NAME/qemu.log" 2>&1 & + echo $! > "$ROOT/$NAME/pid" +} + +start_fc() { + cat > "$ROOT/$NAME/config.json" <"$ROOT/$NAME/fc.log" 2>&1 & + echo $! > "$ROOT/$NAME/pid" +} + +running() { [ -f "$ROOT/$1/pid" ] && kill -0 "$(cat "$ROOT/$1/pid")" 2>/dev/null; } + +cmd_start() { + load_conf "$1" + running "$NAME" && die "'$NAME' already running (pid $(cat "$ROOT/$NAME/pid"))" + ip link show "$TAP" >/dev/null 2>&1 || die "tap $TAP missing — run: systemctl start microvm-net" + rm -f "$ROOT/$NAME/serial.log" + if [ "$ENGINE" != firecracker ] && ! id -nG | tr ' ' '\n' | grep -qx kvm; then + exec sg kvm -c "$0 start $NAME" + fi + if [ "$ENGINE" = firecracker ]; then start_fc; else start_qemu; fi + echo "microvm: '$NAME' starting — console: microvm console $NAME; ssh: microvm ssh $NAME (cloud-init ~10-20s)" +} + +cmd_ssh() { + load_conf "$1"; shift + exec ssh -o UserKnownHostsFile="$ROOT/$NAME/known_hosts" -o StrictHostKeyChecking=accept-new \ + -o ConnectTimeout=5 "$SSH_USER@$GUEST_IP" "$@" +} + +cmd_console() { + load_conf "$1" + [ -f "$ROOT/$NAME/serial.log" ] || die "no serial.log yet (start first)" + exec tail -f "$ROOT/$NAME/serial.log" +} + +cmd_stop() { + load_conf "$1" + running "$NAME" || { echo "microvm: '$NAME' not running"; return 0; } + ssh -o UserKnownHostsFile="$ROOT/$NAME/known_hosts" -o ConnectTimeout=3 -o BatchMode=yes \ + "$SSH_USER@$GUEST_IP" "sudo poweroff" 2>/dev/null || true + for _ in $(seq 1 20); do + running "$NAME" || { echo "microvm: '$NAME' stopped"; return 0; } + sleep 1 + done + kill "$(cat "$ROOT/$NAME/pid")" 2>/dev/null || true + echo "microvm: '$NAME' force-stopped" +} + +cmd_list() { + printf '%-16s %-8s %-15s %-9s %6s %5s %s\n' NAME STATE IP TAP MEM VCPUT ENGINE + for conf in "$ROOT"/*/vm.conf; do + [ -f "$conf" ] || continue + (. "$conf" + if running "$NAME"; then st=running; else st=stopped; fi + printf '%-16s %-8s %-15s %-9s %6s %5s %s\n' "$NAME" "$st" "$GUEST_IP" "$TAP" "$MEM" "$VCPU" "$ENGINE") + done +} + +cmd_rm() { + load_conf "$1" + running "$NAME" && die "'$NAME' is running — stop it first" + rm -rf "$ROOT/$NAME" + echo "microvm: '$NAME' removed" +} + +case "${1:-}" in + create) shift; [ $# -ge 1 ] || die "usage: microvm create NAME [MEM_MIB] [VCPU]"; cmd_create "$@" ;; + start) shift; [ $# -ge 1 ] || die "usage: microvm start NAME"; cmd_start "$@" ;; + ssh) shift; [ $# -ge 1 ] || die "usage: microvm ssh NAME [cmd...]"; cmd_ssh "$@" ;; + console) shift; [ $# -ge 1 ] || die "usage: microvm console NAME"; cmd_console "$@" ;; + stop) shift; [ $# -ge 1 ] || die "usage: microvm stop NAME"; cmd_stop "$@" ;; + list) cmd_list ;; + rm) shift; [ $# -ge 1 ] || die "usage: microvm rm NAME"; cmd_rm "$@" ;; + *) sed -n '2,8p' "$0"; echo; echo "usage: microvm {create|start|ssh|console|stop|list|rm} NAME [args]"; exit 1 ;; +esac diff --git a/microvm/microvm-net.service b/microvm/microvm-net.service new file mode 100644 index 0000000..cd19109 --- /dev/null +++ b/microvm/microvm-net.service @@ -0,0 +1,12 @@ +[Unit] +Description=microvm tap pool + NAT for user microVMs (#615) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/microvm-net.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/microvm/microvm-net.sh b/microvm/microvm-net.sh new file mode 100755 index 0000000..8e94e19 --- /dev/null +++ b/microvm/microvm-net.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# microvm-net.sh — host prep for microvm taps + NAT (idempotent; #615) +# Run by microvm-net.service at boot, and by 13-microvm-setup.sh. +set -euo pipefail + +# tap pool: mv-; per-tap /30 inside the user's /24 +POOLS=( + "reachableceo 172.30.0 r" + "reachableceo-offstage 172.30.1 o" +) + +sysctl -qw net.ipv4.ip_forward=1 + +for entry in "${POOLS[@]}"; do + read -r user subnet prefix <<<"$entry" + for slot in 0 1 2 3; do + tap="mv-$prefix$slot" + ip tuntap add mode tap user "$user" dev "$tap" 2>/dev/null || true + ip addr replace "$subnet.$((4*slot+1))/30" dev "$tap" + ip link set "$tap" up + done +done + +# NAT for both pools (iptables-nft; coexists with docker chains) +iptables -t nat -C POSTROUTING -s 172.30.0.0/23 -j MASQUERADE 2>/dev/null || \ + iptables -t nat -A POSTROUTING -s 172.30.0.0/23 -j MASQUERADE +iptables -C FORWARD -s 172.30.0.0/23 -j ACCEPT 2>/dev/null || \ + iptables -A FORWARD -s 172.30.0.0/23 -j ACCEPT +iptables -C FORWARD -d 172.30.0.0/23 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || \ + iptables -A FORWARD -d 172.30.0.0/23 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + +echo "microvm-net: taps + NAT ready" +ip -br addr | grep '^mv-'