diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml new file mode 100644 index 00000000..1f7d9655 --- /dev/null +++ b/.github/workflows/debian-iso-e2e.yml @@ -0,0 +1,63 @@ +# Builds the unattended Debian ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Debian unattended USB ISO end-to-end + +on: + workflow_dispatch: + pull_request: + +permissions: + contents: read + +concurrency: + group: debian-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Install Debian unattended ISO via UEFI USB + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Verify Debian release alignment + run: bash iso/scripts/check_debian_release_alignment.sh + + - name: Build Debian unattended ISO + run: | + export BASE_OS=debian + export UNATTENDED=true + docker compose up --build + ls -lrt images/Dappnode-debian-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Enable ICMP for QEMU user networking + run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + + - name: Complete Debian unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi + run: | + debian_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-debian-*-amd64-netinst.iso' -print -quit) + test -n "${debian_iso}" + bash test/e2e_debian_iso.sh "${debian_iso}" + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: debian-unattended-usb-installation-logs + path: test-output/debian-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index deb5465a..45a09ec9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,37 +30,3 @@ jobs: - name: Uninstall DAppNode run: | sudo /bin/bash ./scripts/dappnode_uninstall.sh y - - iso: - name: test Debian and Ubuntu ISO - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v6 - - # Create Debian ISO - - name: create Debian ISO - run: | - export BASE_OS=debian - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Debian ISO creation - - name: verify Debian ISO - run: | - ls -lrt images/Dappnode-debian-*.iso - - # Create Ubuntu ISO - - name: create Ubuntu ISO - run: | - export BASE_OS=ubuntu - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Ubuntu ISO creation - - name: verify Ubuntu ISO - run: | - ls -lrt images/Dappnode-ubuntu-*.iso diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml new file mode 100644 index 00000000..49a105b7 --- /dev/null +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -0,0 +1,60 @@ +# Builds the unattended Ubuntu ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Ubuntu unattended USB ISO end-to-end + +on: + workflow_dispatch: + pull_request: + +permissions: + contents: read + +concurrency: + group: ubuntu-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Install Ubuntu unattended ISO via UEFI USB + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Build Ubuntu unattended ISO + run: | + export BASE_OS=ubuntu + export UNATTENDED=true + docker compose up --build + ls -lrt images/Dappnode-ubuntu-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Enable ICMP for QEMU user networking + run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + + - name: Complete Ubuntu unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi + run: | + ubuntu_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-ubuntu-*-live-server-amd64.iso' -print -quit) + test -n "${ubuntu_iso}" + bash test/e2e_ubuntu_iso.sh "${ubuntu_iso}" + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ubuntu-unattended-usb-installation-logs + path: test-output/ubuntu-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index ed3c1e18..15ec4cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ docs/_build -images \ No newline at end of file +images +test-output/ diff --git a/iso/debian_release.conf b/iso/debian_release.conf new file mode 100644 index 00000000..54e6b562 --- /dev/null +++ b/iso/debian_release.conf @@ -0,0 +1,3 @@ +# Debian release used by the ISO and its package repositories. +DEBIAN_SUITE="trixie" +DEBIAN_MAJOR_VERSION="13" diff --git a/iso/preseeds/preseed.cfg b/iso/preseeds/preseed.cfg index 1bd38005..fd55caea 100644 --- a/iso/preseeds/preseed.cfg +++ b/iso/preseeds/preseed.cfg @@ -25,7 +25,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/preseeds/preseed_unattended.cfg b/iso/preseeds/preseed_unattended.cfg index 8316fd7c..0521173e 100644 --- a/iso/preseeds/preseed_unattended.cfg +++ b/iso/preseeds/preseed_unattended.cfg @@ -62,7 +62,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/scripts/check_debian_release_alignment.sh b/iso/scripts/check_debian_release_alignment.sh new file mode 100755 index 00000000..2f237584 --- /dev/null +++ b/iso/scripts/check_debian_release_alignment.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -euo pipefail + +SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ISO_DIR=$(cd "${SCRIPTS_DIR}/.." && pwd) + +source "${ISO_DIR}/debian_release.conf" + +case "${DEBIAN_SUITE}" in + bullseye) suite_major_version="11" ;; + bookworm) suite_major_version="12" ;; + trixie) suite_major_version="13" ;; + forky) suite_major_version="14" ;; + *) + echo "[ERROR] Unknown Debian suite: ${DEBIAN_SUITE}" + exit 1 + ;; +esac + +if [ "${suite_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then + echo "[ERROR] Debian suite ${DEBIAN_SUITE} is Debian ${suite_major_version}, not Debian ${DEBIAN_MAJOR_VERSION}" + exit 1 +fi + +generator="${SCRIPTS_DIR}/generate_dappnode_iso_debian.sh" +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\([^"]*\)"$/\1/p' "${generator}") + +if [[ ! "${base_iso_name}" =~ ^debian-([0-9]+)\. ]]; then + echo "[ERROR] Could not determine the Debian major version from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi + +iso_major_version="${BASH_REMATCH[1]}" +if [ "${iso_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then + echo "[ERROR] Base ISO ${base_iso_name} is Debian ${iso_major_version}, but debian_release.conf expects Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE})" + exit 1 +fi + +expected_repository="d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware" +preseed_files=( + "${ISO_DIR}/preseeds/preseed.cfg" + "${ISO_DIR}/preseeds/preseed_unattended.cfg" +) + +for preseed_file in "${preseed_files[@]}"; do + repository_count=$(grep -Ec '^d-i apt-setup/local[0-9]+/repository string ' "${preseed_file}" || true) + if [ "${repository_count}" -ne 1 ]; then + echo "[ERROR] Expected exactly one additional Debian repository in ${preseed_file}, found ${repository_count}" + exit 1 + fi + + if ! grep -Fqx "${expected_repository}" "${preseed_file}"; then + echo "[ERROR] Debian repository in ${preseed_file} must use the @DEBIAN_SUITE@ placeholder" + exit 1 + fi +done + +echo "[INFO] Debian release alignment is valid: Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE}), ${base_iso_name}" diff --git a/iso/scripts/generate_dappnode_iso_debian.sh b/iso/scripts/generate_dappnode_iso_debian.sh index 4ec4a93d..349aca07 100755 --- a/iso/scripts/generate_dappnode_iso_debian.sh +++ b/iso/scripts/generate_dappnode_iso_debian.sh @@ -4,6 +4,7 @@ set -e SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") source ${SCRIPTS_DIR}/common_iso_generation.sh +source ${SCRIPTS_DIR}/../debian_release.conf BASE_ISO_NAME="debian-13.5.0-amd64-netinst.iso" BASE_ISO_VERSION="${BASE_ISO_NAME#debian-}" @@ -15,6 +16,9 @@ BASE_ISO_SHASUM="95838884f5ea6c82421dfe6baaa5a639dbbe6756c1e380f9fe7a7cb0c1949d2 DAPPNODE_ISO_NAME="${DAPPNODE_ISO_PREFIX}${BASE_ISO_NAME}" DAPPNODE_ISO_PATH="/images/${DAPPNODE_ISO_NAME}" +# Fail before downloading or building if the ISO and repositories target different releases. +bash "${SCRIPTS_DIR}/check_debian_release_alignment.sh" + customize_debian_preseed() { local iso_build_path=$1 local workdir=$2 @@ -50,6 +54,13 @@ customize_debian_preseed() { exit 1 } + # Keep package repositories aligned with the Debian release used by the ISO. + sed -i "s/@DEBIAN_SUITE@/${DEBIAN_SUITE}/g" "${tmp_initrd}/preseed.cfg" + if grep -q "@DEBIAN_SUITE@" "${tmp_initrd}/preseed.cfg"; then + echo "[ERROR] Could not render the Debian suite in the preseed file" + exit 1 + fi + # Recreate (and recompress) the initrd (cd "${tmp_initrd}" && find . -print0 | cpio -0 -ov -H newc | gzip >"${install_dir}/initrd.gz") || { echo "[ERROR] Could not create new initrd" diff --git a/scripts/check-disks.sh b/scripts/check-disks.sh index d7385c41..72070d31 100755 --- a/scripts/check-disks.sh +++ b/scripts/check-disks.sh @@ -12,4 +12,14 @@ for DISK in ${DEVICE}; do DISKS="${DISKS} ${DISK}"; done; DISKS=$(echo "${DISKS}" | sed "s/^ //g"); -debconf-set partman-auto/disk "$DISKS"; \ No newline at end of file +debconf-set partman-auto/disk "$DISKS"; + +# grub-installer/bootdev is preseeded to "default", which marks the question as +# already seen. grub-installer then resolves "default" to the first entry of +# grub-mkdevicemap, i.e. (hd0) -- the installer USB itself. Its safeguard against +# installing onto the installation media only runs for grub-pc, so on UEFI it +# happily targets the stick and then dies reading its partition table: +# Can't read partition table from /dev/sda +# Point it at the same non-USB disks partman is using. grub-installer accepts a +# space separated list and installs to each entry in turn. +debconf-set grub-installer/bootdev "$DISKS"; diff --git a/scripts/dappnode_install.sh b/scripts/dappnode_install.sh index 49ecf3f2..a25c8447 100755 --- a/scripts/dappnode_install.sh +++ b/scripts/dappnode_install.sh @@ -428,7 +428,16 @@ normalize_ipfs_version_ref() { local cid_path="$ref" local manifest_url="${IPFS_ENDPOINT%/}${cid_path}/dappnode_package.json" local manifest - manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" + local attempt + for attempt in 1 2 3 4 5; do + manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" + if [[ -n "$manifest" ]]; then + break + fi + if [[ "$attempt" -lt 5 ]]; then + sleep 2 + fi + done if [[ -z "$manifest" ]]; then error "Could not fetch IPFS manifest for ${comp} from: $manifest_url" error "Provide ${comp}_VERSION as /ipfs/: (example: /ipfs/Qm...:0.2.11)" diff --git a/scripts/dappnode_test_install.sh b/scripts/dappnode_test_install.sh index 4d5d704d..21f193d8 100755 --- a/scripts/dappnode_test_install.sh +++ b/scripts/dappnode_test_install.sh @@ -22,9 +22,33 @@ detect_installation_type() { fi } +# ICMP is filtered on many corporate, hotel and cloud networks, so a failed +# ping alone does not mean the machine is offline. Confirm over HTTPS before +# declaring the installation broken. +check_connectivity() { + ping -c 1 -q google.com >/dev/null 2>&1 && return 0 + + if command -v curl >/dev/null 2>&1; then + curl -fsS --max-time 10 -o /dev/null https://www.google.com/generate_204 && return 0 + fi + + if command -v wget >/dev/null 2>&1; then + wget -q --timeout=10 -O /dev/null https://www.google.com/generate_204 && return 0 + fi + + # Neither downloader is guaranteed to be present this early, so fall back to + # a plain TCP connect, which bash can do on its own. + if (exec 3<>/dev/tcp/www.google.com/443) 2>/dev/null; then + exec 3<&- 3>&- + return 0 + fi + + return 1 +} + components=(BIND IPFS VPN DAPPMANAGER WIFI) detect_installation_type -if ping -c 1 -q google.com >&/dev/null; then +if check_connectivity; then echo -e "\e[32m Connectivity OK\n \e[0m" else error_exit diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh new file mode 100755 index 00000000..9f9c2393 --- /dev/null +++ b/test/e2e_debian_iso.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Runs the generated unattended Debian ISO through the complete UEFI USB test. +# It adds Debian release and package-alignment checks to the shared harness. +set -Eeuo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) + +source "${REPO_ROOT}/iso/debian_release.conf" + +E2E_DISTRO="debian" +E2E_EXPECTED_ID="debian" +E2E_EXPECTED_VERSION="${DEBIAN_MAJOR_VERSION}" +E2E_EXPECTED_CODENAME="${DEBIAN_SUITE}" + +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@" diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh new file mode 100755 index 00000000..c0f80fea --- /dev/null +++ b/test/e2e_iso_install.sh @@ -0,0 +1,467 @@ +#!/bin/bash +# Boots an unattended DAppNode ISO as USB media and installs it onto a virtual disk. +# It completes the first-boot flow, reboots, and validates the running DAppNode system. +set -Eeuo pipefail + +run_e2e_iso_install() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + return 2 + fi + + local iso_path + iso_path=$(realpath "$1") + if [ ! -f "${iso_path}" ]; then + echo "[ERROR] ISO not found: ${iso_path}" + return 1 + fi + + local required_commands=(qemu-img qemu-system-x86_64 ssh sshpass socat) + local command_name + for command_name in "${required_commands[@]}"; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "[ERROR] Missing required command: ${command_name}" + return 1 + fi + done + + local output_dir=${E2E_OUTPUT_DIR:-"${PWD}/test-output/${E2E_DISTRO}-e2e"} + mkdir -p "${output_dir}" + output_dir=$(realpath "${output_dir}") + + temp_root=${RUNNER_TEMP:-/tmp} + vm_dir=$(mktemp -d "${temp_root%/}/dappnode-${E2E_DISTRO}-e2e.XXXXXX") + local disk_path="${vm_dir}/${E2E_DISTRO}.qcow2" + local installer_serial_log="${output_dir}/installer-serial.log" + local installer_process_log="${output_dir}/installer-qemu.log" + local installer_monitor_socket="${vm_dir}/installer-monitor.sock" + local installer_screenshot="${output_dir}/installer-screen.ppm" + local first_boot_monitor_socket="${vm_dir}/first-boot-monitor.sock" + local first_boot_screenshot="${output_dir}/first-boot-screen.ppm" + local first_boot_serial_log="${output_dir}/first-boot-serial.log" + local first_boot_process_log="${output_dir}/first-boot-qemu.log" + local system_serial_log="${output_dir}/final-system-serial.log" + local system_process_log="${output_dir}/final-system-qemu.log" + local installed_system_report="${output_dir}/installed-system-report.log" + + local vm_disk_size=${E2E_VM_DISK_SIZE:-32G} + local vm_memory_mb=${E2E_VM_MEMORY_MB:-4096} + local vm_cpus=${E2E_VM_CPUS:-2} + local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} + local boot_timeout_seconds=${E2E_BOOT_TIMEOUT_SECONDS:-600} + local first_boot_timeout_seconds=${E2E_FIRST_BOOT_TIMEOUT_SECONDS:-1800} + local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-1200} + local ssh_port=${E2E_SSH_PORT:-2222} + local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} + local boot_mode=${E2E_BOOT_MODE:-usb} + local firmware=${E2E_FIRMWARE:-uefi} + qemu_pid="" + + if [ "${boot_mode}" != "usb" ] || [ "${firmware}" != "uefi" ]; then + echo "[ERROR] This unattended ISO test requires E2E_BOOT_MODE=usb and E2E_FIRMWARE=uefi" + return 1 + fi + + cleanup_e2e_vm() { + local exit_code=$? + trap - EXIT INT TERM + + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + + if [ "${KEEP_E2E_VM:-false}" = "true" ]; then + echo "[INFO] Keeping VM files in ${vm_dir}" + else + case "${vm_dir}" in + "${temp_root%/}"/dappnode-"${E2E_DISTRO}"-e2e.*) rm -rf -- "${vm_dir}" ;; + *) echo "[WARN] Refusing to remove unexpected VM directory: ${vm_dir}" ;; + esac + fi + + exit "${exit_code}" + } + trap cleanup_e2e_vm EXIT INT TERM + + monitor_command() { + local socket=$1 + local command=$2 + [ -S "${socket}" ] || return 0 + printf '%s\n' "${command}" | \ + socat - "UNIX-CONNECT:${socket}" >/dev/null 2>&1 || true + } + + capture_vt_screens() { + # d-i draws its UI on VT1 and streams its syslog to VT4, so a failed + # installer step is only diagnosable if every console is captured. + local socket=$1 + local prefix=$2 + local vt + [ -S "${socket}" ] || return 0 + for vt in 1 2 3 4; do + monitor_command "${socket}" "sendkey alt-f${vt}" + sleep 1 + monitor_command "${socket}" "screendump ${prefix}-vt${vt}.ppm" + done + monitor_command "${socket}" "sendkey alt-f1" + } + + dump_guest_diagnostics() { + # SSH is already up whenever a first-boot wait times out, and the + # installer logs there explain far more than the serial console can. + declare -F ssh_guest >/dev/null 2>&1 || return 0 + echo "[INFO] Collecting guest diagnostics over SSH:" + ssh_guest " + echo '--- .firstboot ---'; ls -la /usr/src/dappnode/.firstboot 2>&1 + echo '--- dappnode processes ---'; ps aux | grep -E '[d]appnode_(test_)?install' 2>&1 + echo '--- docker ps ---'; docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 + echo '--- docker images ---'; docker images 2>&1 + echo '--- connectivity ---' + ping -c 1 -W 5 google.com 2>&1 | tail -3 + curl -sS -o /dev/null -w 'curl https: %{http_code}\n' --max-time 10 https://www.google.com/generate_204 2>&1 + echo '--- iso_install.log ---'; tail -n 60 /usr/src/dappnode/logs/iso_install.log 2>&1 + echo '--- dappnode_install.log ---'; tail -n 80 /usr/src/dappnode/logs/dappnode_install.log 2>&1 + " 2>&1 | sed 's/^/ /' || echo " [WARN] Could not collect guest diagnostics" + } + + show_failure_logs() { + capture_vt_screens "${installer_monitor_socket}" "${output_dir}/installer-screen" + capture_vt_screens "${first_boot_monitor_socket}" "${output_dir}/first-boot-screen" + monitor_command "${installer_monitor_socket}" "screendump ${installer_screenshot}" + monitor_command "${first_boot_monitor_socket}" "screendump ${first_boot_screenshot}" + dump_guest_diagnostics + echo "[INFO] Last installer serial output:" + tail -n 200 "${installer_serial_log}" 2>/dev/null || true + echo "[INFO] Last first-boot serial output:" + tail -n 200 "${first_boot_serial_log}" 2>/dev/null || true + echo "[INFO] Last final-system serial output:" + tail -n 200 "${system_serial_log}" 2>/dev/null || true + echo "[INFO] QEMU process output:" + tail -n 100 "${installer_process_log}" 2>/dev/null || true + tail -n 100 "${first_boot_process_log}" 2>/dev/null || true + tail -n 100 "${system_process_log}" 2>/dev/null || true + } + + wait_for_process_exit() { + local process_id=$1 + local timeout_seconds=$2 + local description=$3 + local deadline=$((SECONDS + timeout_seconds)) + + while kill -0 "${process_id}" 2>/dev/null; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + + wait "${process_id}" + } + + local qemu_acceleration=(-accel "tcg,thread=multi") + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + qemu_acceleration=(-accel kvm -cpu host) + echo "[INFO] Using KVM acceleration" + else + echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" + fi + + local ovmf_code="" + local ovmf_vars="" + local ovmf_candidate + for ovmf_candidate in /usr/share/OVMF/OVMF_CODE_4M.fd /usr/share/OVMF/OVMF_CODE.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_code=${ovmf_candidate} + break + fi + done + for ovmf_candidate in /usr/share/OVMF/OVMF_VARS_4M.fd /usr/share/OVMF/OVMF_VARS.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_vars=${ovmf_candidate} + break + fi + done + if [ -z "${ovmf_code}" ] || [ -z "${ovmf_vars}" ]; then + echo "[ERROR] UEFI firmware files were not found; install the ovmf package" + return 1 + fi + local writable_ovmf_vars="${vm_dir}/OVMF_VARS.fd" + cp "${ovmf_vars}" "${writable_ovmf_vars}" + local firmware_args=( + -drive "if=pflash,format=raw,readonly=on,file=${ovmf_code}" + -drive "if=pflash,format=raw,file=${writable_ovmf_vars}" + ) + + echo "[INFO] Creating ${vm_disk_size} virtual installation disk" + qemu-img create -q -f qcow2 "${disk_path}" "${vm_disk_size}" + + local target_disk_install_args=( + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" + -device "virtio-blk-pci,drive=target_disk,bootindex=2" + ) + local installer_media_args=( + -device "qemu-xhci,id=installer_xhci" + -drive "file=${iso_path},format=raw,if=none,readonly=on,id=installer_media" + -device "usb-storage,drive=installer_media,bootindex=1" + ) + + echo "[INFO] Booting the unattended ${E2E_DISTRO} ISO as ${firmware}/${boot_mode}" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + "${target_disk_install_args[@]}" \ + "${installer_media_args[@]}" \ + -boot menu=off \ + -nic user,model=virtio-net-pci \ + -display none \ + -monitor "unix:${installer_monitor_socket},server=on,wait=off" \ + -serial "file:${installer_serial_log}" \ + -no-reboot \ + >"${installer_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_process_exit "${qemu_pid}" "${install_timeout_seconds}" "the ${E2E_DISTRO} installer to complete"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Installer completed and virtual USB removed; starting the first disk boot" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ + -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + -boot menu=off \ + -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -display none \ + -monitor "unix:${first_boot_monitor_socket},server=on,wait=off" \ + -serial "file:${first_boot_serial_log}" \ + -no-reboot \ + >"${first_boot_process_log}" 2>&1 & + qemu_pid=$! + + local ssh_options=( + -p "${ssh_port}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o ConnectTimeout=5 + -o ConnectionAttempts=1 + -o LogLevel=ERROR + ) + + ssh_guest() { + SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" + } + + wait_for_ssh() { + local description=$1 + local timeout_seconds=$2 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for SSH on ${description}" + while ! ssh_guest true >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] ${description} VM exited before SSH became available" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for SSH on ${description}" + return 1 + fi + sleep 10 + done + } + + wait_for_guest_command() { + local timeout_seconds=$1 + local description=$2 + local guest_command=$3 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for ${description}" + while ! ssh_guest "${guest_command}" >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] VM exited while waiting for ${description}" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + } + + if ! wait_for_ssh "the first installed-system boot" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${first_boot_timeout_seconds}" \ + "DAppNode's first-boot installation test" \ + "test ! -e /usr/src/dappnode/.firstboot"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Acknowledging the completed first-boot test" + printf 'sendkey ret\n' | \ + socat - "UNIX-CONNECT:${first_boot_monitor_socket}" >/dev/null + + if ! wait_for_guest_command \ + 120 \ + "the first-boot installer process to exit" \ + "! pgrep -f '[d]appnode_test_install.sh|[/]usr/src/dappnode/scripts/dappnode_install.sh' >/dev/null"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Rebooting after the completed first-boot test" + ssh_guest "sudo -S -p '' systemctl reboot" <<<"${ssh_password}" >/dev/null 2>&1 || true + if ! wait_for_process_exit "${qemu_pid}" 120 "the first-boot VM to request its reboot"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Starting the final installed system without the virtual USB" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ + -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + -boot menu=off \ + -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -display none \ + -monitor none \ + -serial "file:${system_serial_log}" \ + >"${system_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_ssh "the final installed system" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${postinstall_timeout_seconds}" \ + "DAppNode core services to start" \ + "test ! -e /usr/src/dappnode/.firstboot && ! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local && docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth'"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Validating the completed unattended ${E2E_DISTRO} USB installation" + ssh_guest bash -s -- \ + "${E2E_DISTRO}" \ + "${E2E_EXPECTED_ID}" \ + "${E2E_EXPECTED_VERSION}" \ + "${E2E_EXPECTED_CODENAME}" <<'REMOTE_CHECKS' | tee "${installed_system_report}" +set -euo pipefail + +expected_distro=$1 +expected_id=$2 +expected_version=$3 +expected_codename=$4 + +source /etc/os-release +if [ "${expected_codename}" = "AUTO" ]; then + expected_codename=${VERSION_CODENAME} +fi +if [ "${ID}" != "${expected_id}" ] || [ "${VERSION_ID}" != "${expected_version}" ] || [ "${VERSION_CODENAME}" != "${expected_codename}" ]; then + echo "[ERROR] Expected ${expected_id} ${expected_version} (${expected_codename}), found ${PRETTY_NAME}" + exit 1 +fi + +apt_sources=$(grep -R -h -E --include='*.list' --include='*.sources' \ + '^(deb |Suites:)' /etc/apt 2>/dev/null || true) +if ! grep -Eq "(^|[[:space:]])${expected_codename}([[:space:]-]|$)" <<<"${apt_sources}"; then + echo "[ERROR] No ${expected_codename} APT source found on the installed system" + exit 1 +fi + +dpkg_audit=$(dpkg --audit) +if [ -n "${dpkg_audit}" ]; then + echo "[ERROR] dpkg reports incomplete or broken packages:" + echo "${dpkg_audit}" + exit 1 +fi + +package_version() { + dpkg-query -W -f='${Version}' "$1" +} + +case "${expected_distro}" in + debian) + for debian_suite in bullseye bookworm trixie forky; do + if [ "${debian_suite}" != "${expected_codename}" ] && \ + grep -R -q -E --include='*.list' --include='*.sources' \ + "(^|[[:space:]])${debian_suite}([[:space:]-]|$)" /etc/apt 2>/dev/null; then + echo "[ERROR] Found stale Debian suite ${debian_suite} in APT sources" + exit 1 + fi + done + + tasksel_version=$(package_version tasksel) + task_english_version=$(package_version task-english) + debconf_version=$(package_version debconf) + python_debconf_version=$(package_version python3-debconf) + + if [ "${tasksel_version}" != "${task_english_version}" ]; then + echo "[ERROR] tasksel (${tasksel_version}) and task-english (${task_english_version}) do not match" + exit 1 + fi + if [ "${debconf_version}" != "${python_debconf_version}" ]; then + echo "[ERROR] debconf (${debconf_version}) and python3-debconf (${python_debconf_version}) do not match" + exit 1 + fi + + echo "tasksel family: ${tasksel_version}" + echo "debconf family: ${debconf_version}" + ;; + ubuntu) + ubuntu_minimal_version=$(package_version ubuntu-minimal) + if ! grep -R -q -E --include='*.list' --include='*.sources' \ + 'https?://[^[:space:]]*ubuntu\.com/ubuntu' /etc/apt 2>/dev/null; then + echo "[ERROR] No official Ubuntu archive found in APT sources" + exit 1 + fi + if grep -R -H -E --include='*.list' --include='*.sources' \ + 'https?://deb\.debian\.org/debian' /etc/apt 2>/dev/null; then + echo "[ERROR] Found a Debian APT source on the installed Ubuntu system" + exit 1 + fi + echo "ubuntu-minimal: ${ubuntu_minimal_version}" + ;; + *) + echo "[ERROR] Unsupported installed-system check: ${expected_distro}" + exit 1 + ;; +esac + +test -x /usr/src/dappnode/scripts/dappnode_install.sh +test -s /usr/src/dappnode/logs/iso_install.log +test -s /usr/src/dappnode/logs/dappnode_install.log +test ! -e /usr/src/dappnode/.firstboot +! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local +docker --version +docker compose version +docker info >/dev/null +docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth' + +echo "Installed OS: ${PRETTY_NAME}" +echo "[INFO] End-to-end unattended ${expected_distro} UEFI USB installation checks passed" +REMOTE_CHECKS + + echo "[INFO] Complete unattended ${E2E_DISTRO} UEFI USB installation succeeded" +} diff --git a/test/e2e_ubuntu_iso.sh b/test/e2e_ubuntu_iso.sh new file mode 100755 index 00000000..2c00f16c --- /dev/null +++ b/test/e2e_ubuntu_iso.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Runs the generated unattended Ubuntu ISO through the complete UEFI USB test. +# It adds Ubuntu release and repository checks to the shared harness. +set -Eeuo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +GENERATOR="${REPO_ROOT}/iso/scripts/generate_dappnode_iso_ubuntu.sh" + +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\{0,1\}\([^"[:space:]]*\)"\{0,1\}$/\1/p' "${GENERATOR}") +if [[ ! "${base_iso_name}" =~ ^ubuntu-([0-9]+\.[0-9]+)(\.[0-9]+)?-live-server-amd64\.iso$ ]]; then + echo "[ERROR] Could not determine the Ubuntu release from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi + +E2E_DISTRO="ubuntu" +E2E_EXPECTED_ID="ubuntu" +E2E_EXPECTED_VERSION="${BASH_REMATCH[1]}" +E2E_EXPECTED_CODENAME="AUTO" + +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@"