Forem: Sudhakar Daggubati

PCI GPU passthrough

Sudhakar Daggubati — Wed, 13 May 2026 09:33:37 +0000

Little snippet to prepare the host for GPU passthrough

#!/usr/bin/env bash
set -euo pipefail

echo  " This enables vfio passthroug on ubuntu 24+ "

# =========================
# CONFIG
# =========================
GPU_PCI="0000:01:00.0"
GPU_AUDIO_PCI="0000:01:00.1"
GPU_IDS="10de:2820,10de:22bd"


GRUB="/etc/default/grub"
VFIO_CONF="/etc/modprobe.d/vfio.conf"
BLACKLIST="/etc/modprobe.d/blacklist-nvidia.conf"
RECOVERY_FLAG="/boot/vfio_last_state"
SAFE_ENTRY_FILE="/etc/grub.d/40_vfio_safe_mode"

# ⚠️ IMPORTANT: replace once with `blkid`
ROOT_UUID="$(findmnt -no UUID /)"
ROOT_FSTYPE="$(findmnt -no FSTYPE /)"

# SAFETY CHECK: Ensure UUID was actually captured
if [[ -z "$ROOT_UUID" ]]; then
    echo "[!] ERROR: Could not determine ROOT_UUID."
    echo "    Check if findmnt is installed or if you have sufficient permissions."
    exit 1
fi


# 2. Check for Filesystem Compatibility
if [[ "$ROOT_FSTYPE" != ext* ]]; then
    echo "===================================================="
    echo " [!] WARNING: NON-EXT FILESYSTEM DETECTED ($ROOT_FSTYPE)"
    echo "===================================================="
    echo " Your Safe Mode GRUB entry is currently hardcoded for 'insmod ext2'."
    echo " To ensure recovery works, you should update the 'insmod' line"
    echo " in the ensure_safe_mode_entry function to 'insmod $ROOT_FSTYPE'."
    echo ""
    read -p "Continue anyway? (y/n): " confirm
    [[ "$confirm" != "y" ]] && exit 1
fi


# =========================
# BANNER
# =========================
show_recovery_banner() {
    echo ""
    echo "===================================================="
    echo " VFIO SAFE CONTROLLER"
    echo "===================================================="
    echo ""
    echo "RECOVERY OPTIONS:"
    echo "  → GRUB: 'Ubuntu (SAFE MODE - iGPU only)'"
    echo "  → OR 'Advanced options'"
    echo ""
    echo "SAFE MODE = NO VFIO, NO NVIDIA, iGPU ONLY"
    echo "===================================================="
    echo ""
}

# =========================
# VM SAFETY CHECK
# =========================
check_vms() {
    echo "[*] Checking running VMs..."

    RUNNING=$(virsh list --state-running 2>/dev/null | awk 'NR>2 {print $2}')

    if [[ -z "${RUNNING// }" ]]; then
        RUNNING=$(virsh list 2>/dev/null | awk 'NR>2 && $3=="running" {print $2}')
    fi

    if [[ -n "$RUNNING" ]]; then
        echo "[!] Running VMs detected:"
        echo "$RUNNING"
        return 1
    fi

    echo "[✓] No running VMs"
    return 0
}

# =========================
# PREFLIGHT
# =========================
preflight() {
    echo "[*] Preflight check..."

    local blocked=0
    local running_vms

    running_vms=$(virsh list --name 2>/dev/null | sed '/^$/d')

    if [[ -z "$running_vms" ]]; then
        echo "[✓] No running VMs"
        echo "[✓] Preflight OK"
        return 0
    fi

    for vm in $running_vms; do
        if virsh dumpxml "$vm" 2>/dev/null | grep -qE "$GPU_PCI|$GPU_AUDIO_PCI"; then
            echo "[!] Running VM '$vm' is using configured GPU passthrough devices"
            blocked=1
        else
            echo "[i] Running VM '$vm' does not use passthrough GPU"
        fi
    done

    if [[ $blocked -eq 1 ]]; then
        echo "[!] Stop affected VM(s) before changing VFIO state"
        return 1
    fi

    echo "[✓] No running VM currently uses the dGPU"
    echo "[✓] Preflight OK"
    return 0
}

# =========================
# STATUS
# =========================
status() {

    echo "===================="
    echo " SYSTEM STATUS"
    echo "===================="

    virsh list 2>/dev/null || true
    echo ""

    lspci -nnk -s ${GPU_PCI:5:7} || true
}

# =========================
# SAFE MODE GRUB ENTRY
# =========================
ensure_safe_mode_entry() {
    echo "[*] Ensuring SAFE MODE GRUB entry for $ROOT_FSTYPE..."

    local grub_mod="$ROOT_FSTYPE"
    [[ "$ROOT_FSTYPE" == ext* ]] && grub_mod="ext2"

    sudo tee "$SAFE_ENTRY_FILE" >/dev/null <<EOF
#!/bin/sh
exec tail -n +3 \$0

menuentry "Ubuntu (SAFE MODE - iGPU only, NO VFIO)" {
    insmod part_gpt
    insmod $grub_mod
    search --no-floppy --fs-uuid --set=root $ROOT_UUID
    linux /boot/vmlinuz root=UUID=$ROOT_UUID ro quiet splash intel_iommu=off modprobe.blacklist=vfio_pci,vfio,vfio_iommu_type1,nvidia,nvidia_drm,nvidia_modeset
    initrd /boot/initrd.img
}
EOF

    sudo chmod +x "$SAFE_ENTRY_FILE"
}


# =========================
# GRUB APPLY VFIO
# =========================
apply_vfio_grub() {
    sudo cp "$GRUB" "$GRUB.bak.$(date +%s)"

    sudo sed -i 's/vfio-pci.ids=[^ ]*//g' "$GRUB"
    sudo sed -i 's/intel_iommu=on//g' "$GRUB"
    sudo sed -i 's/iommu=pt//g' "$GRUB"

    #sudo sed -i "s/^GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"quiet splash intel_iommu=on iommu=pt vfio-pci.ids=$GPU_IDS /" "$GRUB"
    sudo sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on iommu=pt vfio-pci.ids='"$GPU_IDS"' rd.driver.pre=vfio-pci"/' "$GRUB"
}

verify_vfio_binding() {
    echo "[*] Checking GPU driver binding..."

    if lspci -nnk -s ${GPU_PCI:5:7} | grep -q "vfio-pci"; then
        echo "[✓] GPU correctly bound to VFIO"
        return 0
    else
        echo "[!] VFIO FAILED — GPU still not isolated"
        echo "    Do NOT launch VM"
        return 1
    fi
}


# =========================
# GRUB RESTORE (iGPU)
# =========================
restore_igpu_grub() {
    sudo cp "$GRUB" "$GRUB.bak.$(date +%s)"

    sudo sed -i 's/vfio-pci.ids=[^ ]*//g' "$GRUB"
    sudo sed -i 's/intel_iommu=on//g' "$GRUB"
    sudo sed -i 's/iommu=pt//g' "$GRUB"

    sudo sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"/' "$GRUB"
}

# =========================
# ENABLE VFIO
# =========================
enable_flow() {
    show_recovery_banner
    preflight || exit 1

    echo "[*] Enabling VFIO..."


    cat <<EOF | sudo tee "$BLACKLIST" >/dev/null
blacklist nvidia
blacklist nvidia_drm
blacklist nvidia_modeset
blacklist nvidia_uvm
blacklist nvidia_nouveau
EOF

    printf '%s\n' \
    "options vfio-pci ids=$GPU_IDS disable_vga=1" \
    "softdep nvidia pre: vfio-pci" \
    "softdep nvidia_drm pre: vfio-pci" \
    "softdep nvidia_modeset pre: vfio-pci" \
    "softdep nvidia_uvm pre: vfio-pci" \
    | sudo tee "$VFIO_CONF" >/dev/null

    # Block udev from loading nvidia modules
    sudo tee /etc/udev/rules.d/71-nvidia.rules >/dev/null << 'EOF'
# VFIO override — block nvidia module loading via udev
ACTION=="add", DEVPATH=="/bus/pci/drivers/nvidia", RUN+="/bin/false"
EOF

    sudo systemctl disable --now nvidia-persistenced.service 2>/dev/null || true
    sudo systemctl disable --now nvidia-powerd.service 2>/dev/null || true
    sudo systemctl disable --now nvidia-cdi-refresh.path 2>/dev/null || true



    apply_vfio_grub
    ensure_safe_mode_entry

    echo "$GPU_IDS" | sudo tee "$RECOVERY_FLAG" >/dev/null

    sudo update-initramfs -u
    sudo update-grub
    echo "[*] Verifying GRUB configuration..."
    grep -q "vfio-pci.ids=$GPU_IDS" "$GRUB" || {
        echo "[!] GRUB injection failed"
        exit 1
    }

    echo "[✓] VFIO enabled safely"
}

# =========================
# DISABLE VFIO
# =========================
disable_flow() {
    show_recovery_banner

    sudo rm -f "$VFIO_CONF"
    sudo rm -f "$BLACKLIST"
    sudo rm -f /etc/udev/rules.d/71-nvidia.rules
    sudo udevadm control --reload-rules

    restore_igpu_grub
    ensure_safe_mode_entry

    sudo rm -f "$RECOVERY_FLAG"

    # Re-enable nvidia services
    sudo systemctl enable --now nvidia-persistenced.service 2>/dev/null || true
    sudo systemctl enable --now nvidia-powerd.service 2>/dev/null || true
    sudo systemctl enable --now nvidia-cdi-refresh.path 2>/dev/null || true

    sudo update-initramfs -u
    sudo update-grub

    echo "[✓] VFIO disabled"
}

# =========================
# RECOVERY MODE
# =========================
recover_flow() {
    echo "[!] RECOVERY MODE ACTIVATED"

    restore_igpu_grub

    sudo rm -f "$VFIO_CONF"
    sudo rm -f "$BLACKLIST"
    sudo rm -f "$RECOVERY_FLAG"

    sudo rm -f /etc/udev/rules.d/71-nvidia.rules
    sudo udevadm control --reload-rules

    ensure_safe_mode_entry

    # Re-enable nvidia services
    sudo systemctl enable --now nvidia-persistenced.service 2>/dev/null || true
    sudo systemctl enable --now nvidia-powerd.service 2>/dev/null || true
    sudo systemctl enable --now nvidia-cdi-refresh.path 2>/dev/null || true

    sudo update-initramfs -u
    sudo update-grub

    echo "[✓] System restored to iGPU SAFE MODE"
}

# =========================
# MAIN CONTROLLER
# =========================
main() {
    show_recovery_banner

    case "${1:-}" in
        status)
            status
            ;;
        preflight)
            preflight
            ;;
        enable)
            enable_flow
            ;;
        disable)
            disable_flow
            ;;
        recover)
            recover_flow
            ;;
        verify)
            verify_vfio_binding || exit 1
            ;;

        *)
            echo "Usage: $0 {status|preflight|enable|disable|recover|verify}"
            ;;
    esac
}

main "$@"

open-source API gateway solutions and their managed offerings.

Sudhakar Daggubati — Fri, 27 Sep 2024 14:48:14 +0000

Kong and APISIX, two popular open-source #APIGateway solutions. #kong looks versatile Unified Gateway but how it fares against #APISIX backed by a similar enterprise API7 offering.

Key Features:

API Gateway and API Management
Plugin architecture
NGINX-based
Config management
Seamless Kubernetes integration
Security

Differentiation:

AI Gateway: Only Kong offers an AI gateway, leveraging AI for tasks like traffic management and anomaly detection.
Performance: API7 claims superior performance, but real-world benchmarks may vary.
Cost: Self-hosting Kong can be cost-effective, SaaS offering pricing are complex, its better to check the fine print and usage patterns Pricing Models
Service Mesh: Kong uses Envoy, while API7 uses Istio (which also uses Envoy).
Cloud Costs: Running AI and API gateways at scale in the cloud can be expensive, Network design, workload pattern and application architecture going to play a critical role in how costly its going to be.

Choosing the Right API Gateway:

APISIX(API7): If performance and native service mesh capabilities are your primary concerns, then API7 might be a good choice. While #APISIX is a top #CNCF project and cloud-native with its out-of-the-box plugins, API7's focus on performance and service mesh integration is a compelling factor.

Kong: if that #AIGateway is exiting, in-house team is capable and priority is for long standing community support and self management then considering kong is advisable; you could self mange or opt for enterprise option.

The best choice between Kong and APISIX depends on organizational inclinations and specific requirements, including the need for AI integration, performance demands, and budget constraints. Carefully evaluate your needs and consider the strengths and weaknesses of each platform to make an informed decision.

Considering both are open source in nature/base and a big community driving the adoption, over a period of time, feature coverage would eventually catch-up, so better to be cost & performance conscious when choosing the gateway. Watch out those pricing models; getting on is easy but not getting off :-)

PlatformEngineering

Automating DNS with Confidence: Terraform + DNScontrol

Sudhakar Daggubati — Mon, 23 Sep 2024 15:09:55 +0000

A split origin DNS setup with multi origin and multi subscription creating route via a gateway subnet is complex

Its even more challenging to keep it running smoothly and not to break things inadvertently; it's not a joke when people say its all DNS :-)

Terraform can automate most of this infra at scale but one aspect that it lacks is management of DNS in a complex setup in which one often need additional capabilities to test and validate before plan is applied.

There are multiple scenarios in which lack of this capability makes it hard to customize DNS and do not get into troubles.

Temporary zone and validate before apply

Use terraform to create temporary DNS zone
Use curl or https://httpie.io/ to validate the DNS entries
this setup ensures DNS changes are tested and impact is known
- Each provider has their own SDK, format that they support and API; for example terraform has no zone file import while azure does besides integration tests are non exist and complex to craft, mostly simple nslookup validation.

Combination of native DNS mgmt + Terraform

DNScontrol and Terraform are both powerful tools for managing DNS records, but they have different capabilities and use cases. Let's use them together to make a DNS management predictable and fault proof.

Designed specifically for DNS: DNScontrol is tailored for DNS management, offering features and integrations that are optimized for DNS-related tasks.
Flexibility: It provides a high level of flexibility, allowing you to define DNS records using various formats (e.g., YAML, JSON) and supports a wide range of DNS providers.

You could leave whole DNS management to DNSControl or use it for complex validations and DNS records mgmt and use AZ CLI to export and import in a CI/CD task with necessary approval flows +/- terraform.

kyverno-json; to extend kyverno policy rules framework beyond K8S resources

Sudhakar Daggubati — Wed, 14 Aug 2024 11:06:57 +0000

kyverno is a powerful policy engine for #Kubernetes artifacts governance, What if you need to extend same capabilities beyond Kubernetes resources?

Check kyverno-json, it is natural extension when one already having #kyverno as policy engine for #k8s governance.

Exploring as we got a config driven PaaS infra setup,by extending existing policy coverage to other configuration items and at plan stage itself and as a pipeline, we hope to minimize config induced errors.

Kyverno-json bridges the gap by allowing you to apply Kyverno policies to validate any JSON or YAML data. This opens doors to validating:

Terraform files: Ensure your infrastructure configurations adhere to best practices and security guidelines.
Dockerfiles: Validate image builds for compliance and prevent potential vulnerabilities.
Cloud configurations: Maintain consistency and avoid errors across your cloud infrastructure.
Authorization requests: Enforce granular access control at the request level.

Beyond Deployment-Time Validation:

With kyverno-json, validation extends beyond deployment time:

DevOps Pipelines: Integrate seamlessly into your DevOps pipelines for continuous validation.
Pre-commit hooks: Enforce validation before code commits, catching errors early in the development cycle.
Atlantis (Terraform PR Automation): Enhance your Terraform pull request automation with robust validation capabilities., atlantis also doubles up as self service tool for developers.
Makefiles: Utilize kyverno-json in your makefiles for streamlined validation as part of your build process.

Terraform plan can be validated taking its JSON output and passing to CLI when complex validation is required.

Terraform input validation is limited, kyverno-json covers a lot more with JMESPath ; a query language for JSON.