microvm tooling: qemu-microvm engine, tap+NAT pools, cloud-init seeds (#615)
Firecracker-class microVMs for dev: 13-microvm-setup.sh (idempotent installer) + microvm CLI + microvm-net.service (per-user tap pools, /30s, NAT). Debian 13 cloud assets in /usr/local/lib/microvm/base. Firecracker v1.16.1 installed but its KVM cap check is refused by this nested Westmere KVM; qemu -M microvm engine default, FC retry after #601. Details + UAT: https://projects.knownelement.com/issues/615
This commit is contained in:
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
# 13-microvm-setup.sh — install Firecracker-class microVM tooling (#615)
|
||||
# Idempotent; run as root: RUN=1 sudo -E ./13-microvm-setup.sh
|
||||
# Engine note: qemu -M microvm + KVM is the default engine. Firecracker
|
||||
# v1.16.1 is installed but its strict KVM capability check
|
||||
# (USER_MEMORY/SET_TSS_ADDR/VAPIC) is refused by this host's nested KVM
|
||||
# (Westmere under PVE) — retry MICROVM_ENGINE=firecracker after the
|
||||
# #601 hardware upgrade.
|
||||
set -euo pipefail
|
||||
[ "${RUN:-0}" = 1 ] || { echo "gate: RUN=1 to execute"; exit 1; }
|
||||
[ "$(id -u)" = 0 ] || { echo "must be root"; exit 1; }
|
||||
|
||||
FC_VER=1.16.1
|
||||
FC_SHA=382a02a869e4d6d5cb14c40577f9545e8458021ea8b0b2d3fc10ec14d9c242e6
|
||||
FC_URL=https://github.com/firecracker-microvm/firecracker/releases/download/v${FC_VER}
|
||||
IMG_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
SUMS_URL=https://cloud.debian.org/images/cloud/trixie/latest/SHA512SUMS
|
||||
|
||||
BASE=/usr/local/lib/microvm/base
|
||||
SRC=${SRC:-$(cd "$(dirname "$0")" && pwd)/microvm}
|
||||
|
||||
echo "== packages"
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
||||
qemu-system-x86 qemu-utils dosfstools mtools curl
|
||||
|
||||
echo "== kvm group"
|
||||
for u in reachableceo reachableceo-offstage; do
|
||||
id -nG "$u" | tr ' ' '\n' | grep -qx kvm || usermod -aG kvm "$u"
|
||||
done
|
||||
|
||||
echo "== firecracker v${FC_VER}"
|
||||
if [ ! -x /usr/local/bin/firecracker ] || ! /usr/local/bin/firecracker --version 2>/dev/null | grep -q "$FC_VER"; then
|
||||
t=$(mktemp -d)
|
||||
curl -fsSL -o "$t/fc.tgz" "$FC_URL/firecracker-v${FC_VER}-x86_64.tgz"
|
||||
echo "$FC_SHA $t/fc.tgz" | sha256sum -c -
|
||||
tar xzf "$t/fc.tgz" -C "$t" release-v${FC_VER}-x86_64/firecracker-v${FC_VER}-x86_64 \
|
||||
release-v${FC_VER}-x86_64/jailer-v${FC_VER}-x86_64
|
||||
install -m0755 "$t/release-v${FC_VER}-x86_64/firecracker-v${FC_VER}-x86_64" /usr/local/bin/firecracker
|
||||
install -m0755 "$t/release-v${FC_VER}-x86_64/jailer-v${FC_VER}-x86_64" /usr/local/bin/jailer
|
||||
rm -rf "$t"
|
||||
fi
|
||||
/usr/local/bin/firecracker --version
|
||||
|
||||
echo "== base assets"
|
||||
mkdir -p "$BASE"
|
||||
if [ ! -f "$BASE/debian-13-generic-amd64.qcow2" ]; then
|
||||
curl -fsSL -o "$BASE/debian-13-generic-amd64.qcow2" "$IMG_URL"
|
||||
curl -fsSL -o "$BASE/SHA512SUMS" "$SUMS_URL"
|
||||
(cd "$BASE" && grep 'debian-13-generic-amd64.qcow2$' SHA512SUMS | sha512sum -c -)
|
||||
fi
|
||||
if [ ! -f "$BASE/vmlinuz" ] || [ ! -f "$BASE/initrd" ]; then
|
||||
modprobe nbd max_part=8
|
||||
qemu-nbd -c /dev/nbd0 --read-only "$BASE/debian-13-generic-amd64.qcow2"
|
||||
sleep 2
|
||||
mkdir -p /mnt/microvm-base
|
||||
mount -o ro /dev/nbd0p1 /mnt/microvm-base
|
||||
cp -L /mnt/microvm-base/boot/vmlinuz-* "$BASE/vmlinuz"
|
||||
cp -L /mnt/microvm-base/boot/initrd.img-* "$BASE/initrd"
|
||||
umount /mnt/microvm-base
|
||||
qemu-nbd -d /dev/nbd0
|
||||
fi
|
||||
if [ ! -f "$BASE/base-rootfs.raw" ]; then
|
||||
qemu-img convert -O raw "$BASE/debian-13-generic-amd64.qcow2" "$BASE/base-rootfs.raw"
|
||||
qemu-img resize "$BASE/base-rootfs.raw" 10G >/dev/null
|
||||
fi
|
||||
chmod 644 "$BASE"/vmlinuz "$BASE"/initrd "$BASE"/base-rootfs.raw "$BASE"/debian-13-generic-amd64.qcow2
|
||||
ls -lh "$BASE"
|
||||
|
||||
echo "== tooling"
|
||||
install -m0755 "$SRC/microvm" /usr/local/bin/microvm
|
||||
install -m0755 "$SRC/microvm-net.sh" /usr/local/sbin/microvm-net.sh
|
||||
install -m0644 "$SRC/microvm-net.service" /etc/systemd/system/microvm-net.service
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now microvm-net.service
|
||||
/usr/local/sbin/microvm-net.sh
|
||||
|
||||
echo
|
||||
echo "DONE. As a human account:"
|
||||
echo " microvm create test1 && microvm start test1 && microvm ssh test1"
|
||||
echo "(first ssh needs one logout/login if kvm group was just added)"
|
||||
@@ -17,6 +17,8 @@ One table, updated in place at each checkpoint. States: done ✅ / doing 🔄 /
|
||||
| #610 | fleet | ✅ | Open Terminal 9-account fleet (tailscale-only :30000-30008) + OpenWebUI UAT PASS |
|
||||
| #611 | rdp | 🔄 | Jump/iPad + 5160x2160 monitor: EGFX frame-budget fix live (12-fix-rdp-bigres.sh); freerdp UAT all-clean; human UAT pending |
|
||||
| #612 | dotfiles | 🔄 | kali-addendum: Kali twoline prompt + completions (kali-comforts.sh) on top of omz, p10k = fallback (KALI_PROMPT=0); awaiting UAT |
|
||||
| #615 | microvm | 🔄 | microvm tooling LIVE (qemu-microvm engine; firecracker blocked by nested-KVM caps until #601 hardware) + rebuild capture/docs next |
|
||||
| #616 | macos | ⏳ | macOS VM on pfv-tsys5 (headless CI + iOS dev) — blocked by #601 |
|
||||
|
||||
Inbox (mid-task interrupts): none.
|
||||
|
||||
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
# microvm — Firecracker-class microVMs on ultix-streaming (Redmine #615)
|
||||
# Engine: qemu -M microvm + KVM (default) | Firecracker (MICROVM_ENGINE=firecracker;
|
||||
# needs a KVM exposing USER_MEMORY/SET_TSS_ADDR/VAPIC caps — refused by this
|
||||
# host's nested KVM until the #601 hardware upgrade; binary stays installed).
|
||||
# Guests: Debian 13 cloud image, cloud-init NoCloud seed, static tap networking.
|
||||
# Instances: ~/.microvm/<name>. Host prep: microvm-net.service (taps mv-<p><i>).
|
||||
|
||||
set -euo pipefail
|
||||
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
BASE=/usr/local/lib/microvm/base
|
||||
ROOT=$HOME/.microvm
|
||||
DISK_GIB=${DISK_GIB:-10}
|
||||
SSH_USER=debian
|
||||
|
||||
case "$USER" in
|
||||
reachableceo) PREFIX=r; SUBNET=172.30.0 ;;
|
||||
reachableceo-offstage) PREFIX=o; SUBNET=172.30.1 ;;
|
||||
*) echo "microvm: no tap pool for user '$USER' (add one in microvm-net.sh)" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
die() { echo "microvm: $*" >&2; exit 1; }
|
||||
|
||||
slot_ip() { # slot -> "host_ip guest_ip"
|
||||
echo "$SUBNET.$((4*$1+1))" "$SUBNET.$((4*$1+2))"
|
||||
}
|
||||
|
||||
free_slot() {
|
||||
local used slot
|
||||
used=$(cat "$ROOT"/*/vm.conf 2>/dev/null | grep -E '^SLOT=' | cut -d= -f2 | sort -u || true)
|
||||
for slot in 0 1 2 3; do
|
||||
grep -qx "$slot" <<<"$used" || { echo "$slot"; return; }
|
||||
done
|
||||
die "all 4 tap slots busy; remove an instance first (microvm list / rm)"
|
||||
}
|
||||
|
||||
load_conf() {
|
||||
[ -f "$ROOT/$1/vm.conf" ] || die "no such instance '$1' (microvm list)"
|
||||
. "$ROOT/$1/vm.conf"
|
||||
}
|
||||
|
||||
cmd_create() {
|
||||
local name=${1:-} mem=${2:-1024} vcpu=${3:-1} slot hostip guestip pubkey
|
||||
[[ "$name" =~ ^[a-z][a-z0-9-]{0,15}$ ]] || die "name: ^[a-z][a-z0-9-]{0,15}$"
|
||||
[ -d "$ROOT/$name" ] && die "instance '$name' already exists"
|
||||
slot=$(free_slot)
|
||||
read -r hostip guestip <<<"$(slot_ip "$slot")"
|
||||
mkdir -p "$ROOT/$name"
|
||||
|
||||
pubkey=$(ls -1 "$HOME"/.ssh/id_*.pub 2>/dev/null | head -1)
|
||||
[ -n "$pubkey" ] || die "no $HOME/.ssh/id_*.pub found (needed for guest login)"
|
||||
|
||||
cat > "$ROOT/$name/user-data" <<EOF
|
||||
#cloud-config
|
||||
hostname: $name
|
||||
users:
|
||||
- name: $SSH_USER
|
||||
sudo: ALL=(ALL) NOPASSWD:ALL
|
||||
shell: /bin/bash
|
||||
lock_passwd: true
|
||||
ssh_authorized_keys:
|
||||
- $(cat "$pubkey")
|
||||
EOF
|
||||
cat > "$ROOT/$name/network-config" <<EOF
|
||||
version: 2
|
||||
ethernets:
|
||||
eth0:
|
||||
addresses: [$guestip/30]
|
||||
routes: [{via: $hostip, to: default}]
|
||||
nameservers: {addresses: [1.1.1.1, 9.9.9.9]}
|
||||
EOF
|
||||
printf 'instance-id: microvm-%s-%s\nlocal-hostname: %s\n' "$name" "$(date +%s)" "$name" > "$ROOT/$name/meta-data"
|
||||
|
||||
truncate -s 4M "$ROOT/$name/seed.img"
|
||||
mkfs.vfat -n CIDATA "$ROOT/$name/seed.img" >/dev/null
|
||||
mcopy -i "$ROOT/$name/seed.img" "$ROOT/$name/user-data" "$ROOT/$name/network-config" "$ROOT/$name/meta-data" ::
|
||||
|
||||
echo "microvm: cloning base rootfs (${DISK_GIB}G sparse)..."
|
||||
qemu-img convert -O raw "$BASE/base-rootfs.raw" "$ROOT/$name/rootfs.raw"
|
||||
qemu-img resize "$ROOT/$name/rootfs.raw" "${DISK_GIB}G" >/dev/null
|
||||
|
||||
if [ "$PREFIX" = r ]; then mnet=0; else mnet=1; fi
|
||||
MAC=$(printf '52:54:00:30:%02x:%02x' $((mnet*16+slot)) $((slot+1)))
|
||||
|
||||
cat > "$ROOT/$name/vm.conf" <<EOF
|
||||
NAME=$name
|
||||
SLOT=$slot
|
||||
MEM=$mem
|
||||
VCPU=$vcpu
|
||||
GUEST_IP=$guestip
|
||||
HOST_IP=$hostip
|
||||
TAP=mv-$PREFIX$slot
|
||||
MAC=$MAC
|
||||
ENGINE=\${MICROVM_ENGINE:-qemu}
|
||||
EOF
|
||||
echo "microvm: '$name' created — guest $guestip (gw $hostip, tap mv-$PREFIX$slot), $mem MiB / $vcpu vCPU"
|
||||
echo "microvm: start it: microvm start $name && microvm ssh $name"
|
||||
}
|
||||
|
||||
start_qemu() {
|
||||
nohup qemu-system-x86_64 \
|
||||
-accel kvm -cpu host -M microvm \
|
||||
-m "$MEM" -smp "$VCPU" \
|
||||
-kernel "$BASE/vmlinuz" -initrd "$BASE/initrd" \
|
||||
-append "console=ttyS0 reboot=k panic=-1 i8042.noaux random.trust_cpu=on root=/dev/vda1 rw" \
|
||||
-drive file="$ROOT/$NAME/rootfs.raw",if=none,id=r0,format=raw \
|
||||
-device virtio-blk-device,drive=r0 \
|
||||
-drive file="$ROOT/$NAME/seed.img",if=none,id=s0,format=raw,readonly=on \
|
||||
-device virtio-blk-device,drive=s0 \
|
||||
-netdev tap,id=n0,ifname="$TAP",script=no,downscript=no \
|
||||
-device virtio-net-device,netdev=n0,mac="$MAC" \
|
||||
-serial "file:$ROOT/$NAME/serial.log" \
|
||||
-display none -no-reboot \
|
||||
>"$ROOT/$NAME/qemu.log" 2>&1 &
|
||||
echo $! > "$ROOT/$NAME/pid"
|
||||
}
|
||||
|
||||
start_fc() {
|
||||
cat > "$ROOT/$NAME/config.json" <<EOF
|
||||
{
|
||||
"boot-source": {
|
||||
"kernel_image_path": "$BASE/vmlinuz",
|
||||
"initrd_path": "$BASE/initrd",
|
||||
"boot_args": "console=ttyS0 reboot=k panic=-1 i8042.noaux random.trust_cpu=on root=/dev/vda1 rw"
|
||||
},
|
||||
"drives": [
|
||||
{"drive_id": "rootfs", "path_on_host": "$ROOT/$NAME/rootfs.raw", "is_root_device": true, "is_read_only": false},
|
||||
{"drive_id": "seed", "path_on_host": "$ROOT/$NAME/seed.img", "is_root_device": false, "is_read_only": true}
|
||||
],
|
||||
"network-interfaces": [{"iface_id": "eth0", "guest_mac": "$MAC", "host_dev_name": "$TAP"}],
|
||||
"machine-config": {"vcpu_count": $VCPU, "mem_size_mib": $MEM, "smt": false},
|
||||
"serial": {"serial_out_path": "$ROOT/$NAME/serial.log"}
|
||||
}
|
||||
EOF
|
||||
nohup firecracker --api-sock "$ROOT/$NAME/fc.sock" --config-file "$ROOT/$NAME/config.json" \
|
||||
>"$ROOT/$NAME/fc.log" 2>&1 &
|
||||
echo $! > "$ROOT/$NAME/pid"
|
||||
}
|
||||
|
||||
running() { [ -f "$ROOT/$1/pid" ] && kill -0 "$(cat "$ROOT/$1/pid")" 2>/dev/null; }
|
||||
|
||||
cmd_start() {
|
||||
load_conf "$1"
|
||||
running "$NAME" && die "'$NAME' already running (pid $(cat "$ROOT/$NAME/pid"))"
|
||||
ip link show "$TAP" >/dev/null 2>&1 || die "tap $TAP missing — run: systemctl start microvm-net"
|
||||
rm -f "$ROOT/$NAME/serial.log"
|
||||
if [ "$ENGINE" != firecracker ] && ! id -nG | tr ' ' '\n' | grep -qx kvm; then
|
||||
exec sg kvm -c "$0 start $NAME"
|
||||
fi
|
||||
if [ "$ENGINE" = firecracker ]; then start_fc; else start_qemu; fi
|
||||
echo "microvm: '$NAME' starting — console: microvm console $NAME; ssh: microvm ssh $NAME (cloud-init ~10-20s)"
|
||||
}
|
||||
|
||||
cmd_ssh() {
|
||||
load_conf "$1"; shift
|
||||
exec ssh -o UserKnownHostsFile="$ROOT/$NAME/known_hosts" -o StrictHostKeyChecking=accept-new \
|
||||
-o ConnectTimeout=5 "$SSH_USER@$GUEST_IP" "$@"
|
||||
}
|
||||
|
||||
cmd_console() {
|
||||
load_conf "$1"
|
||||
[ -f "$ROOT/$NAME/serial.log" ] || die "no serial.log yet (start first)"
|
||||
exec tail -f "$ROOT/$NAME/serial.log"
|
||||
}
|
||||
|
||||
cmd_stop() {
|
||||
load_conf "$1"
|
||||
running "$NAME" || { echo "microvm: '$NAME' not running"; return 0; }
|
||||
ssh -o UserKnownHostsFile="$ROOT/$NAME/known_hosts" -o ConnectTimeout=3 -o BatchMode=yes \
|
||||
"$SSH_USER@$GUEST_IP" "sudo poweroff" 2>/dev/null || true
|
||||
for _ in $(seq 1 20); do
|
||||
running "$NAME" || { echo "microvm: '$NAME' stopped"; return 0; }
|
||||
sleep 1
|
||||
done
|
||||
kill "$(cat "$ROOT/$NAME/pid")" 2>/dev/null || true
|
||||
echo "microvm: '$NAME' force-stopped"
|
||||
}
|
||||
|
||||
cmd_list() {
|
||||
printf '%-16s %-8s %-15s %-9s %6s %5s %s\n' NAME STATE IP TAP MEM VCPUT ENGINE
|
||||
for conf in "$ROOT"/*/vm.conf; do
|
||||
[ -f "$conf" ] || continue
|
||||
(. "$conf"
|
||||
if running "$NAME"; then st=running; else st=stopped; fi
|
||||
printf '%-16s %-8s %-15s %-9s %6s %5s %s\n' "$NAME" "$st" "$GUEST_IP" "$TAP" "$MEM" "$VCPU" "$ENGINE")
|
||||
done
|
||||
}
|
||||
|
||||
cmd_rm() {
|
||||
load_conf "$1"
|
||||
running "$NAME" && die "'$NAME' is running — stop it first"
|
||||
rm -rf "$ROOT/$NAME"
|
||||
echo "microvm: '$NAME' removed"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
create) shift; [ $# -ge 1 ] || die "usage: microvm create NAME [MEM_MIB] [VCPU]"; cmd_create "$@" ;;
|
||||
start) shift; [ $# -ge 1 ] || die "usage: microvm start NAME"; cmd_start "$@" ;;
|
||||
ssh) shift; [ $# -ge 1 ] || die "usage: microvm ssh NAME [cmd...]"; cmd_ssh "$@" ;;
|
||||
console) shift; [ $# -ge 1 ] || die "usage: microvm console NAME"; cmd_console "$@" ;;
|
||||
stop) shift; [ $# -ge 1 ] || die "usage: microvm stop NAME"; cmd_stop "$@" ;;
|
||||
list) cmd_list ;;
|
||||
rm) shift; [ $# -ge 1 ] || die "usage: microvm rm NAME"; cmd_rm "$@" ;;
|
||||
*) sed -n '2,8p' "$0"; echo; echo "usage: microvm {create|start|ssh|console|stop|list|rm} NAME [args]"; exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=microvm tap pool + NAT for user microVMs (#615)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/microvm-net.sh
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# microvm-net.sh — host prep for microvm taps + NAT (idempotent; #615)
|
||||
# Run by microvm-net.service at boot, and by 13-microvm-setup.sh.
|
||||
set -euo pipefail
|
||||
|
||||
# tap pool: mv-<prefix><slot>; per-tap /30 inside the user's /24
|
||||
POOLS=(
|
||||
"reachableceo 172.30.0 r"
|
||||
"reachableceo-offstage 172.30.1 o"
|
||||
)
|
||||
|
||||
sysctl -qw net.ipv4.ip_forward=1
|
||||
|
||||
for entry in "${POOLS[@]}"; do
|
||||
read -r user subnet prefix <<<"$entry"
|
||||
for slot in 0 1 2 3; do
|
||||
tap="mv-$prefix$slot"
|
||||
ip tuntap add mode tap user "$user" dev "$tap" 2>/dev/null || true
|
||||
ip addr replace "$subnet.$((4*slot+1))/30" dev "$tap"
|
||||
ip link set "$tap" up
|
||||
done
|
||||
done
|
||||
|
||||
# NAT for both pools (iptables-nft; coexists with docker chains)
|
||||
iptables -t nat -C POSTROUTING -s 172.30.0.0/23 -j MASQUERADE 2>/dev/null || \
|
||||
iptables -t nat -A POSTROUTING -s 172.30.0.0/23 -j MASQUERADE
|
||||
iptables -C FORWARD -s 172.30.0.0/23 -j ACCEPT 2>/dev/null || \
|
||||
iptables -A FORWARD -s 172.30.0.0/23 -j ACCEPT
|
||||
iptables -C FORWARD -d 172.30.0.0/23 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT 2>/dev/null || \
|
||||
iptables -A FORWARD -d 172.30.0.0/23 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
|
||||
|
||||
echo "microvm-net: taps + NAT ready"
|
||||
ip -br addr | grep '^mv-'
|
||||
Reference in New Issue
Block a user